commit 147188b143f67c27d8da396e759c19336f64c105 Author: nmzik Date: Thu Jul 16 10:21:41 2026 +0200 Initial commit diff --git a/.github/ISSUE_TEMPLATE/kytyps5-game-emulation.yaml b/.github/ISSUE_TEMPLATE/kytyps5-game-emulation.yaml new file mode 100644 index 000000000..02321f9be --- /dev/null +++ b/.github/ISSUE_TEMPLATE/kytyps5-game-emulation.yaml @@ -0,0 +1,156 @@ +name: Game Emulation Bug Report +description: Report KytyPS5 game boot, crash, freeze, rendering, audio, input, or gameplay behavior problems. +title: "[GAME BUG]: " +body: + - type: markdown + attributes: + value: | + ## Quick checks + + Before submitting, please: + - Search existing issues first. + - Re-test with a clean game dump. + - Attach the complete log file instead of edited snippets. + + - type: markdown + attributes: + value: | + ## Game and build + + - type: input + id: game-title + attributes: + label: Game title + description: Use the full title shown by the game or launcher. + placeholder: "Example: Astro Bot" + validations: + required: true + + - type: input + id: game-id + attributes: + label: Game ID / serial + description: Enter the ID or serial when available. If you cannot find it, write "Unknown". + placeholder: "Example: PPSA12345 or Unknown" + validations: + required: true + + - type: input + id: kyty-version + attributes: + label: KytyPS5 version + description: Include the exact build you tested so the report can be matched to the right code. + placeholder: "Example: v0.0.1" + validations: + required: true + + - type: input + id: regression-builds + attributes: + label: Last working build / first broken build + description: Optional. Fill this in if the same game worked in an older KytyPS5 build. + placeholder: "Example: last worked in v0.0.1, first broke in v0.0.1" + + - type: markdown + attributes: + value: | + ## Problem details + + - type: textarea + id: what-happened + attributes: + label: What happened? + description: Describe the failure or incorrect behavior as directly as possible. + placeholder: "Example: The game reaches the main menu, then freezes after selecting Continue." + validations: + required: true + + - type: textarea + id: steps-to-reproduce + attributes: + label: Steps to reproduce + description: List the shortest path that triggers the problem from a fresh KytyPS5 start. + placeholder: | + 1. Open KytyPS5. + 2. Boot the game. + 3. Select ... + validations: + required: true + + - type: textarea + id: expected-behavior + attributes: + label: Expected behavior + description: Explain what should happen at the same point in the game. + placeholder: "Example: The save should load and gameplay should continue normally." + validations: + required: true + + - type: markdown + attributes: + value: | + ## Host system + + - type: input + id: os + attributes: + label: OS + description: Include the operating system name, version, and edition when useful. + placeholder: "Example: Windows 11 Pro 25H2" + validations: + required: true + + - type: input + id: cpu + attributes: + label: CPU + description: Include the exact processor model. + placeholder: "Example: AMD Ryzen 9 9950X3D" + validations: + required: true + + - type: input + id: gpu + attributes: + label: GPU + description: Include the graphics card model and driver version when available. + placeholder: "Example: NVIDIA GeForce RTX 5080, driver 576.80" + validations: + required: true + + - type: input + id: ram-vram + attributes: + label: RAM / VRAM + description: Include both system memory and video memory. + placeholder: "Example: 64 GB DDR5 RAM / 16 GB VRAM" + validations: + required: true + + - type: markdown + attributes: + value: | + ## Evidence + + - type: textarea + id: log-file + attributes: + label: Log file upload + description: Drag the full KytyPS5 log (_kyty.txt) into this box, or link to the complete unmodified file. + placeholder: "Attach the full log file here." + validations: + required: true + + - type: textarea + id: screenshots-video + attributes: + label: Screenshots/video + description: Optional. Add images or footage when the problem is visible or audible. + placeholder: "Drag and drop media here, or paste links." + + - type: textarea + id: extra-notes + attributes: + label: Extra notes + description: Optional. Add settings, unusual hardware details, or anything else that may help. + placeholder: "Any extra context goes here." diff --git a/.gitignore b/.gitignore new file mode 100644 index 000000000..92f28ecb0 --- /dev/null +++ b/.gitignore @@ -0,0 +1,5 @@ +# Visual Studio 2015 cache/options directory +.vs/ +.idea/ +build/ +_Build/vscode-clang/ \ No newline at end of file diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 000000000..ea1f6aca4 --- /dev/null +++ b/.gitmodules @@ -0,0 +1,41 @@ +[submodule "3rdparty/SDL2"] + path = 3rdparty/SDL2 + url = https://github.com/libsdl-org/SDL.git + branch = SDL2 +[submodule "3rdparty/magic_enum"] + path = 3rdparty/magic_enum + url = https://github.com/Neargye/magic_enum.git +[submodule "3rdparty/xxHash"] + path = 3rdparty/xxHash + url = https://github.com/Cyan4973/xxHash.git +[submodule "3rdparty/Vulkan-Headers"] + path = 3rdparty/Vulkan-Headers + url = https://github.com/KhronosGroup/Vulkan-Headers.git +[submodule "3rdparty/SPIRV-Tools"] + path = 3rdparty/SPIRV-Tools + url = https://github.com/KhronosGroup/SPIRV-Tools.git +[submodule "3rdparty/SPIRV-Headers"] + path = 3rdparty/SPIRV-Headers + url = https://github.com/KhronosGroup/SPIRV-Headers.git +[submodule "3rdparty/VulkanMemoryAllocator"] + path = 3rdparty/VulkanMemoryAllocator + url = https://github.com/GPUOpen-LibrariesAndSDKs/VulkanMemoryAllocator.git +[submodule "3rdparty/ffmpeg-core"] + path = 3rdparty/ffmpeg-core + url = https://github.com/shadps4-emu/ext-ffmpeg-core.git + shallow = true +[submodule "3rdparty/fmt"] + path = 3rdparty/fmt + url = https://github.com/fmtlib/fmt.git +[submodule "3rdparty/tracy"] + path = 3rdparty/tracy + url = https://github.com/wolfpld/tracy.git +[submodule "3rdparty/spdlog"] + path = 3rdparty/spdlog + url = https://github.com/gabime/spdlog.git +[submodule "3rdparty/LibAtrac9"] + path = 3rdparty/LibAtrac9 + url = https://github.com/shadps4-emu/ext-LibAtrac9.git +[submodule "3rdparty/nlohmann_json"] + path = 3rdparty/nlohmann_json + url = https://github.com/nlohmann/json.git diff --git a/.vscode/launch.json b/.vscode/launch.json new file mode 100644 index 000000000..0c51cfa60 --- /dev/null +++ b/.vscode/launch.json @@ -0,0 +1,37 @@ +{ + "version": "0.2.0", + "configurations": [ + { + "name": "Debug launcher UI", + "type": "cppvsdbg", + "request": "launch", + "program": "${workspaceFolder}\\_Build\\vscode-clang\\launcher\\launcher.exe", + "args": [], + "stopAtEntry": false, + "cwd": "${workspaceFolder}\\_Build\\vscode-clang\\launcher", + "environment": [], + "externalConsole": false + }, + { + "name": "Debug kyty_emulator", + "type": "cppvsdbg", + "request": "launch", + "program": "${workspaceFolder}\\_Build\\vscode-clang\\kyty_emulator.exe", + "args": [ + "--game", "D:/gamesps5/PPSA07429", + "--printf-direction", "File", + "--graphics-debug-dump", "True" + ], + "stopAtEntry": false, + "cwd": "${workspaceFolder}\\_Build\\vscode-clang", + "environment": [], + "externalConsole": true + }, + { + "name": "Attach to running kyty_emulator", + "type": "cppvsdbg", + "request": "attach", + "processId": "${command:pickProcess}" + } + ] +} diff --git a/.vscode/settings.json b/.vscode/settings.json new file mode 100644 index 000000000..69f8e2895 --- /dev/null +++ b/.vscode/settings.json @@ -0,0 +1,16 @@ +{ + "cmake.sourceDirectory": "${workspaceFolder}/src", + "cmake.buildDirectory": "${workspaceFolder}/_Build/vscode-clang", + "cmake.generator": "Ninja", + "cmake.environment": { + "CC": "clang-cl", + "CXX": "clang-cl", + "PATH": "${env:ProgramFiles}/Microsoft Visual Studio/18/Community/VC/Tools/Llvm/x64/bin;${env:ProgramFiles}/Microsoft Visual Studio/18/Community/Common7/IDE/CommonExtensions/Microsoft/CMake/Ninja;${env:PATH}" + }, + "cmake.configureSettings": { + "CMAKE_C_COMPILER": "clang-cl", + "CMAKE_CXX_COMPILER": "clang-cl", + "CMAKE_PREFIX_PATH": "C:/Qt/6.11.0/msvc2022_64" + } +} + diff --git a/3rdparty/CMakeLists.txt b/3rdparty/CMakeLists.txt new file mode 100644 index 000000000..e61ef4508 --- /dev/null +++ b/3rdparty/CMakeLists.txt @@ -0,0 +1,103 @@ +set(BUILD_SHARED_LIBS OFF CACHE BOOL "" FORCE) + +add_subdirectory(cpuinfo) + +set(SDL_SHARED OFF CACHE BOOL "" FORCE) +add_subdirectory(SDL2) + +add_library(vulkan-1 UNKNOWN IMPORTED GLOBAL) + +if(MINGW) + find_library(Vulkan_Lib NAMES libvulkan-1.dll.a vulkan-1 + PATHS "$ENV{VULKAN_SDK}/Lib" "$ENV{VULKAN_SDK}/Lib32" NO_DEFAULT_PATH) + if(NOT Vulkan_Lib) + find_library(Vulkan_Lib NAMES libvulkan-1.dll.a vulkan-1) + endif() +elseif(WIN32) + find_library(Vulkan_Lib NAMES vulkan-1 + PATHS "$ENV{VULKAN_SDK}/Lib" "$ENV{VULKAN_SDK}/Lib32" NO_DEFAULT_PATH) + if(NOT Vulkan_Lib) + find_library(Vulkan_Lib NAMES vulkan-1) + endif() +elseif(LINUX) + find_library(Vulkan_Lib vulkan) +endif() + +if(NOT Vulkan_Lib OR Vulkan_Lib MATCHES "-NOTFOUND$") + message(FATAL_ERROR "Vulkan library was not found") +endif() + +set_target_properties(vulkan-1 PROPERTIES IMPORTED_LOCATION "${Vulkan_Lib}") + +set(SPIRV_SKIP_TESTS ON CACHE BOOL "" FORCE) +set(SPIRV_SKIP_EXECUTABLES ON CACHE BOOL "" FORCE) +set(SKIP_SPIRV_TOOLS_INSTALL ON CACHE BOOL "" FORCE) +set(SPIRV_WERROR OFF CACHE BOOL "" FORCE) +set(SPIRV-Headers_SOURCE_DIR "${CMAKE_CURRENT_SOURCE_DIR}/SPIRV-Headers" CACHE PATH "" FORCE) +add_subdirectory(SPIRV-Tools) + +add_library(spirv-tools ALIAS SPIRV-Tools-static) +add_library(spirv-tools-opt ALIAS SPIRV-Tools-opt) + +add_subdirectory(xxHash/build/cmake) + +set(FMT_DOC OFF CACHE BOOL "" FORCE) +set(FMT_INSTALL OFF CACHE BOOL "" FORCE) +set(FMT_MODULE OFF CACHE BOOL "" FORCE) +set(FMT_TEST OFF CACHE BOOL "" FORCE) +add_subdirectory(fmt EXCLUDE_FROM_ALL) + +set(JSON_BuildTests OFF CACHE BOOL "" FORCE) +set(JSON_Install OFF CACHE BOOL "" FORCE) +add_subdirectory(nlohmann_json EXCLUDE_FROM_ALL) + +set(SPDLOG_BUILD_ALL OFF CACHE BOOL "" FORCE) +set(SPDLOG_BUILD_EXAMPLE OFF CACHE BOOL "" FORCE) +set(SPDLOG_BUILD_TESTS OFF CACHE BOOL "" FORCE) +set(SPDLOG_BUILD_BENCH OFF CACHE BOOL "" FORCE) +set(SPDLOG_BUILD_WARNINGS OFF CACHE BOOL "" FORCE) +set(SPDLOG_INSTALL OFF CACHE BOOL "" FORCE) +set(SPDLOG_FMT_EXTERNAL ON CACHE BOOL "" FORCE) +set(SPDLOG_NO_EXCEPTIONS ON CACHE BOOL "" FORCE) +add_subdirectory(spdlog EXCLUDE_FROM_ALL) + +if (NOT TARGET FFmpeg::ffmpeg) + if(CMAKE_SYSTEM_PROCESSOR MATCHES "^(aarch64|arm64|ARM64)$") + set(ARCHITECTURE arm64) + else() + set(ARCHITECTURE x86_64) + endif() + add_subdirectory(ffmpeg-core) + add_library(FFmpeg::ffmpeg ALIAS ffmpeg) +endif() + +file(GLOB LIBATRAC9_SOURCES LibAtrac9/C/src/*.c) +add_library(LibAtrac9 STATIC ${LIBATRAC9_SOURCES}) +target_include_directories(LibAtrac9 PUBLIC LibAtrac9/C/src) +if(MSVC) + target_compile_options(LibAtrac9 PRIVATE /W0) +endif() + +set(TRACY_STATIC ON CACHE BOOL "" FORCE) +set(TRACY_ENABLE ON CACHE BOOL "" FORCE) +set(TRACY_ON_DEMAND ON CACHE BOOL "" FORCE) +set(TRACY_DELAYED_INIT ON CACHE BOOL "" FORCE) +set(TRACY_MANUAL_LIFETIME ON CACHE BOOL "" FORCE) +add_subdirectory(tracy EXCLUDE_FROM_ALL) +if (TARGET TracyClient) + if (MSVC) + target_compile_options(TracyClient PRIVATE /WX-) + endif() + if (CLANG) + target_compile_options(TracyClient PRIVATE + -Wno-error + -Wno-braced-scalar-init + -Wno-format + -Wno-unused-variable + ) + endif() +endif() + +if (CLANG AND MSVC) +add_subdirectory(winpthread) +endif() diff --git a/3rdparty/LibAtrac9 b/3rdparty/LibAtrac9 new file mode 160000 index 000000000..ec8899dad --- /dev/null +++ b/3rdparty/LibAtrac9 @@ -0,0 +1 @@ +Subproject commit ec8899dadf393f655f2871a94e0fe4b3d6220c9a diff --git a/3rdparty/SDL2 b/3rdparty/SDL2 new file mode 160000 index 000000000..4b69833bc --- /dev/null +++ b/3rdparty/SDL2 @@ -0,0 +1 @@ +Subproject commit 4b69833bc54abf3dd3288d4aa7afbba527775e5b diff --git a/3rdparty/SPIRV-Headers b/3rdparty/SPIRV-Headers new file mode 160000 index 000000000..01e057791 --- /dev/null +++ b/3rdparty/SPIRV-Headers @@ -0,0 +1 @@ +Subproject commit 01e0577914a75a2569c846778c2f93aa8e6feddd diff --git a/3rdparty/SPIRV-Tools b/3rdparty/SPIRV-Tools new file mode 160000 index 000000000..7f2d9ee92 --- /dev/null +++ b/3rdparty/SPIRV-Tools @@ -0,0 +1 @@ +Subproject commit 7f2d9ee926f98fc77a3ed1e1e0f113b8c9c49458 diff --git a/3rdparty/Vulkan-Headers b/3rdparty/Vulkan-Headers new file mode 160000 index 000000000..2fa203425 --- /dev/null +++ b/3rdparty/Vulkan-Headers @@ -0,0 +1 @@ +Subproject commit 2fa203425eb4af9dfc6b03f97ef72b0b5bcb8350 diff --git a/3rdparty/VulkanMemoryAllocator b/3rdparty/VulkanMemoryAllocator new file mode 160000 index 000000000..a1d434708 --- /dev/null +++ b/3rdparty/VulkanMemoryAllocator @@ -0,0 +1 @@ +Subproject commit a1d434708c217b2a6c7b365f1fe41fa03a562e59 diff --git a/3rdparty/cpuinfo/CMakeLists.txt b/3rdparty/cpuinfo/CMakeLists.txt new file mode 100644 index 000000000..92d218e79 --- /dev/null +++ b/3rdparty/cpuinfo/CMakeLists.txt @@ -0,0 +1,97 @@ +set(cpuinfo_src + src/init.c + src/api.c + src/cache.c + deps/clog/src/clog.c +) + +if(CMAKE_SYSTEM_PROCESSOR MATCHES "^(i[3-6]86|AMD64|x86(_64)?)$") + list(APPEND cpuinfo_src + src/x86/init.c + src/x86/info.c + src/x86/vendor.c + src/x86/uarch.c + src/x86/name.c + src/x86/topology.c + src/x86/isa.c + src/x86/cache/init.c + src/x86/cache/descriptor.c + src/x86/cache/deterministic.c) + if(LINUX OR ANDROID) + list(APPEND cpuinfo_src src/x86/linux/init.c src/x86/linux/cpuinfo.c) + else() + list(APPEND cpuinfo_src src/x86/windows/init.c) + endif() + +elseif(CMAKE_SYSTEM_PROCESSOR MATCHES "^(armv[5-8].*|aarch64|arm64)$") + list(APPEND cpuinfo_src src/arm/uarch.c src/arm/cache.c) + + if(LINUX OR ANDROID) + list(APPEND cpuinfo_src + src/arm/linux/init.c + src/arm/linux/cpuinfo.c + src/arm/linux/clusters.c + src/arm/linux/chipset.c + src/arm/linux/midr.c + src/arm/linux/hwcap.c) + + if(CMAKE_SYSTEM_PROCESSOR MATCHES "^armv[5-8]") + list(APPEND cpuinfo_src src/arm/linux/aarch32-isa.c) + + if(ANDROID AND ANDROID_ABI STREQUAL "armeabi") + set_source_files_properties(src/arm/linux/aarch32-isa.c PROPERTIES COMPILE_FLAGS -marm) + endif() + + elseif(CMAKE_SYSTEM_PROCESSOR MATCHES "^(aarch64|arm64)$") + list(APPEND cpuinfo_src src/arm/linux/aarch64-isa.c) + endif() + + endif() + + if(ANDROID) + list(APPEND cpuinfo_src src/arm/android/properties.c) + endif() + +endif() + +if(LINUX OR ANDROID) + list(APPEND cpuinfo_src + src/linux/smallfile.c + src/linux/multiline.c + src/linux/cpulist.c + src/linux/processors.c) +endif() + +if (LINUX) + set_source_files_properties(${cpuinfo_src} PROPERTIES COMPILE_FLAGS "-D_GNU_SOURCE=1") +endif() + +include_directories(include src deps/clog/include) + +add_library(cpuinfo STATIC ${cpuinfo_src}) + +if (MINGW) + if (CLANG) + target_compile_options(cpuinfo PRIVATE -Wno-unused-variable -Wno-implicit-function-declaration) + else() + target_compile_options(cpuinfo PRIVATE -Wno-maybe-uninitialized -Wno-unused-variable -Wno-implicit-function-declaration -Wno-format -Wno-format-extra-args) + endif() +elseif (LINUX) + if (CLANG) + else() + target_compile_options(cpuinfo PRIVATE -Wno-unused-result) + endif() +elseif (MSVC) + if (CLANG) + target_compile_options(cpuinfo PRIVATE -Wno-unused-variable -Wno-deprecated-declarations -Wno-implicit-function-declaration) + else() + endif() +elseif(ANDROID) + target_compile_options(cpuinfo PRIVATE -Wno-implicit-function-declaration) +endif() + + +target_include_directories(cpuinfo PUBLIC "${CMAKE_CURRENT_SOURCE_DIR}/include") + + + diff --git a/3rdparty/cpuinfo/LICENSE b/3rdparty/cpuinfo/LICENSE new file mode 100644 index 000000000..3f9a4f020 --- /dev/null +++ b/3rdparty/cpuinfo/LICENSE @@ -0,0 +1,27 @@ +Copyright (c) 2019 Google LLC +Copyright (c) 2017-2018 Facebook Inc. +Copyright (C) 2012-2017 Georgia Institute of Technology +Copyright (C) 2010-2012 Marat Dukhan + +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + +* Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. + +* Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/3rdparty/cpuinfo/deps/clog/LICENSE b/3rdparty/cpuinfo/deps/clog/LICENSE new file mode 100644 index 000000000..306de3d8f --- /dev/null +++ b/3rdparty/cpuinfo/deps/clog/LICENSE @@ -0,0 +1,26 @@ +Copyright (C) 2018 Marat Dukhan +Copyright (c) 2017-2018 Facebook Inc. +Copyright (c) 2017 Georgia Institute of Technology + +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + +* Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. + +* Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/3rdparty/cpuinfo/deps/clog/include/clog.h b/3rdparty/cpuinfo/deps/clog/include/clog.h new file mode 100644 index 000000000..414376116 --- /dev/null +++ b/3rdparty/cpuinfo/deps/clog/include/clog.h @@ -0,0 +1,100 @@ +#pragma once + +#include +#include +#include + +#define CLOG_NONE 0 +#define CLOG_FATAL 1 +#define CLOG_ERROR 2 +#define CLOG_WARNING 3 +#define CLOG_INFO 4 +#define CLOG_DEBUG 5 + +#ifndef CLOG_VISIBILITY + #if defined(__ELF__) + #define CLOG_VISIBILITY __attribute__((__visibility__("internal"))) + #elif defined(__MACH__) + #define CLOG_VISIBILITY __attribute__((__visibility__("hidden"))) + #else + #define CLOG_VISIBILITY + #endif +#endif + +#ifndef CLOG_ARGUMENTS_FORMAT + #if defined(__GNUC__) + #define CLOG_ARGUMENTS_FORMAT __attribute__((__format__(__printf__, 1, 2))) + #else + #define CLOG_ARGUMENTS_FORMAT + #endif +#endif + +#ifdef __cplusplus +extern "C" { +#endif + +CLOG_VISIBILITY void clog_vlog_debug(const char* module, const char* format, va_list args); +CLOG_VISIBILITY void clog_vlog_info(const char* module, const char* format, va_list args); +CLOG_VISIBILITY void clog_vlog_warning(const char* module, const char* format, va_list args); +CLOG_VISIBILITY void clog_vlog_error(const char* module, const char* format, va_list args); +CLOG_VISIBILITY void clog_vlog_fatal(const char* module, const char* format, va_list args); + +#define CLOG_DEFINE_LOG_DEBUG(log_debug_function_name, module, level) \ + CLOG_ARGUMENTS_FORMAT \ + inline static void log_debug_function_name(const char* format, ...) { \ + if (level >= CLOG_DEBUG) { \ + va_list args; \ + va_start(args, format); \ + clog_vlog_debug(module, format, args); \ + va_end(args); \ + } \ + } + +#define CLOG_DEFINE_LOG_INFO(log_info_function_name, module, level) \ + CLOG_ARGUMENTS_FORMAT \ + inline static void log_info_function_name(const char* format, ...) { \ + if (level >= CLOG_INFO) { \ + va_list args; \ + va_start(args, format); \ + clog_vlog_info(module, format, args); \ + va_end(args); \ + } \ + } + +#define CLOG_DEFINE_LOG_WARNING(log_warning_function_name, module, level) \ + CLOG_ARGUMENTS_FORMAT \ + inline static void log_warning_function_name(const char* format, ...) { \ + if (level >= CLOG_WARNING) { \ + va_list args; \ + va_start(args, format); \ + clog_vlog_warning(module, format, args); \ + va_end(args); \ + } \ + } + +#define CLOG_DEFINE_LOG_ERROR(log_error_function_name, module, level) \ + CLOG_ARGUMENTS_FORMAT \ + inline static void log_error_function_name(const char* format, ...) { \ + if (level >= CLOG_ERROR) { \ + va_list args; \ + va_start(args, format); \ + clog_vlog_error(module, format, args); \ + va_end(args); \ + } \ + } + +#define CLOG_DEFINE_LOG_FATAL(log_fatal_function_name, module, level) \ + CLOG_ARGUMENTS_FORMAT \ + inline static void log_fatal_function_name(const char* format, ...) { \ + if (level >= CLOG_FATAL) { \ + va_list args; \ + va_start(args, format); \ + clog_vlog_fatal(module, format, args); \ + va_end(args); \ + } \ + abort(); \ + } + +#ifdef __cplusplus +} /* extern "C" */ +#endif diff --git a/3rdparty/cpuinfo/deps/clog/src/clog.c b/3rdparty/cpuinfo/deps/clog/src/clog.c new file mode 100644 index 000000000..fe5d43e1f --- /dev/null +++ b/3rdparty/cpuinfo/deps/clog/src/clog.c @@ -0,0 +1,423 @@ +#include +#include +#include +#include +#ifdef _WIN32 + #include +#else + #include +#endif +#ifdef __ANDROID__ + #include +#endif + +#ifndef CLOG_LOG_TO_STDIO + #ifdef __ANDROID__ + #define CLOG_LOG_TO_STDIO 0 + #else + #define CLOG_LOG_TO_STDIO 1 + #endif +#endif + +#include + + +/* Messages up to this size are formatted entirely on-stack, and don't allocate heap memory */ +#define CLOG_STACK_BUFFER_SIZE 1024 + +#define CLOG_FATAL_PREFIX "Fatal error: " +#define CLOG_FATAL_PREFIX_LENGTH 13 +#define CLOG_FATAL_PREFIX_FORMAT "Fatal error in %s: " +#define CLOG_ERROR_PREFIX "Error: " +#define CLOG_ERROR_PREFIX_LENGTH 7 +#define CLOG_ERROR_PREFIX_FORMAT "Error in %s: " +#define CLOG_WARNING_PREFIX "Warning: " +#define CLOG_WARNING_PREFIX_LENGTH 9 +#define CLOG_WARNING_PREFIX_FORMAT "Warning in %s: " +#define CLOG_INFO_PREFIX "Note: " +#define CLOG_INFO_PREFIX_LENGTH 6 +#define CLOG_INFO_PREFIX_FORMAT "Note (%s): " +#define CLOG_DEBUG_PREFIX "Debug: " +#define CLOG_DEBUG_PREFIX_LENGTH 7 +#define CLOG_DEBUG_PREFIX_FORMAT "Debug (%s): " +#define CLOG_SUFFIX_LENGTH 1 + +void clog_vlog_fatal(const char* module, const char* format, va_list args) { + #if defined(__ANDROID__) && !CLOG_LOG_TO_STDIO + __android_log_vprint(ANDROID_LOG_FATAL, module, format, args); + #else + char stack_buffer[CLOG_STACK_BUFFER_SIZE]; + char* heap_buffer = NULL; + char* out_buffer = &stack_buffer[0]; + + /* The first call to vsnprintf will clobber args, thus need a copy in case a second vsnprintf call is needed */ + va_list args_copy; + va_copy(args_copy, args); + + int prefix_chars = CLOG_FATAL_PREFIX_LENGTH; + if (module == NULL) { + memcpy(stack_buffer, CLOG_FATAL_PREFIX, CLOG_FATAL_PREFIX_LENGTH); + } else { + prefix_chars = snprintf(stack_buffer, CLOG_STACK_BUFFER_SIZE, CLOG_FATAL_PREFIX_FORMAT, module); + if (prefix_chars < 0) { + /* Format error in prefix (possible if prefix is modified): skip prefix and continue as if nothing happened. */ + prefix_chars = 0; + } + } + + int format_chars; + if (prefix_chars + CLOG_SUFFIX_LENGTH >= CLOG_STACK_BUFFER_SIZE) { + /* + * Prefix + suffix alone would overflow the on-stack buffer, thus need to use on-heap buffer. + * Do not even try to format the string into on-stack buffer. + */ + format_chars = vsnprintf(NULL, 0, format, args); + } else { + format_chars = + vsnprintf( + &stack_buffer[prefix_chars], + CLOG_STACK_BUFFER_SIZE - prefix_chars - CLOG_SUFFIX_LENGTH, + format, + args); + } + if (format_chars < 0) { + /* Format error in the message: silently ignore this particular message. */ + goto cleanup; + } + if (prefix_chars + format_chars + CLOG_SUFFIX_LENGTH > CLOG_STACK_BUFFER_SIZE) { + /* Allocate a buffer on heap, and vsnprintf to this buffer */ + heap_buffer = malloc(prefix_chars + format_chars + CLOG_SUFFIX_LENGTH); + if (heap_buffer == NULL) { + goto cleanup; + } + + if (prefix_chars > CLOG_STACK_BUFFER_SIZE) { + /* Prefix didn't fit into on-stack buffer, re-format it again to on-heap buffer */ + snprintf(heap_buffer, prefix_chars + 1 /* for '\0'-terminator */, CLOG_FATAL_PREFIX_FORMAT, module); + } else { + /* Copy pre-formatted prefix from on-stack buffer to on-heap buffer */ + memcpy(heap_buffer, stack_buffer, prefix_chars); + } + vsnprintf(heap_buffer + prefix_chars, format_chars + CLOG_SUFFIX_LENGTH, format, args_copy); + out_buffer = heap_buffer; + } + out_buffer[prefix_chars + format_chars] = '\n'; + #ifdef _WIN32 + DWORD bytes_written; + WriteFile( + GetStdHandle(STD_ERROR_HANDLE), + out_buffer, prefix_chars + format_chars + CLOG_SUFFIX_LENGTH, + &bytes_written, NULL); + #else + write(STDERR_FILENO, out_buffer, prefix_chars + format_chars + CLOG_SUFFIX_LENGTH); + #endif + +cleanup: + free(heap_buffer); + va_end(args_copy); + #endif +} + +void clog_vlog_error(const char* module, const char* format, va_list args) { + #if defined(__ANDROID__) && !CLOG_LOG_TO_STDIO + __android_log_vprint(ANDROID_LOG_ERROR, module, format, args); + #else + char stack_buffer[CLOG_STACK_BUFFER_SIZE]; + char* heap_buffer = NULL; + char* out_buffer = &stack_buffer[0]; + + /* The first call to vsnprintf will clobber args, thus need a copy in case a second vsnprintf call is needed */ + va_list args_copy; + va_copy(args_copy, args); + + int prefix_chars = CLOG_ERROR_PREFIX_LENGTH; + if (module == NULL) { + memcpy(stack_buffer, CLOG_ERROR_PREFIX, CLOG_ERROR_PREFIX_LENGTH); + } else { + prefix_chars = snprintf(stack_buffer, CLOG_STACK_BUFFER_SIZE, CLOG_ERROR_PREFIX_FORMAT, module); + if (prefix_chars < 0) { + /* Format error in prefix (possible if prefix is modified): skip prefix and continue as if nothing happened. */ + prefix_chars = 0; + } + } + + int format_chars; + if (prefix_chars + CLOG_SUFFIX_LENGTH >= CLOG_STACK_BUFFER_SIZE) { + /* + * Prefix + suffix alone would overflow the on-stack buffer, thus need to use on-heap buffer. + * Do not even try to format the string into on-stack buffer. + */ + format_chars = vsnprintf(NULL, 0, format, args); + } else { + format_chars = + vsnprintf( + &stack_buffer[prefix_chars], + CLOG_STACK_BUFFER_SIZE - prefix_chars - CLOG_SUFFIX_LENGTH, + format, + args); + } + if (format_chars < 0) { + /* Format error in the message: silently ignore this particular message. */ + goto cleanup; + } + if (prefix_chars + format_chars + CLOG_SUFFIX_LENGTH > CLOG_STACK_BUFFER_SIZE) { + /* Allocate a buffer on heap, and vsnprintf to this buffer */ + heap_buffer = malloc(prefix_chars + format_chars + CLOG_SUFFIX_LENGTH); + if (heap_buffer == NULL) { + goto cleanup; + } + + if (prefix_chars > CLOG_STACK_BUFFER_SIZE) { + /* Prefix didn't fit into on-stack buffer, re-format it again to on-heap buffer */ + snprintf(heap_buffer, prefix_chars + 1 /* for '\0'-terminator */, CLOG_ERROR_PREFIX_FORMAT, module); + } else { + /* Copy pre-formatted prefix from on-stack buffer to on-heap buffer */ + memcpy(heap_buffer, stack_buffer, prefix_chars); + } + vsnprintf(heap_buffer + prefix_chars, format_chars + CLOG_SUFFIX_LENGTH, format, args_copy); + out_buffer = heap_buffer; + } + out_buffer[prefix_chars + format_chars] = '\n'; + #ifdef _WIN32 + DWORD bytes_written; + WriteFile( + GetStdHandle(STD_ERROR_HANDLE), + out_buffer, prefix_chars + format_chars + CLOG_SUFFIX_LENGTH, + &bytes_written, NULL); + #else + write(STDERR_FILENO, out_buffer, prefix_chars + format_chars + CLOG_SUFFIX_LENGTH); + #endif + +cleanup: + free(heap_buffer); + va_end(args_copy); + #endif +} + +void clog_vlog_warning(const char* module, const char* format, va_list args) { + #if defined(__ANDROID__) && !CLOG_LOG_TO_STDIO + __android_log_vprint(ANDROID_LOG_WARN, module, format, args); + #else + char stack_buffer[CLOG_STACK_BUFFER_SIZE]; + char* heap_buffer = NULL; + char* out_buffer = &stack_buffer[0]; + + /* The first call to vsnprintf will clobber args, thus need a copy in case a second vsnprintf call is needed */ + va_list args_copy; + va_copy(args_copy, args); + + int prefix_chars = CLOG_WARNING_PREFIX_LENGTH; + if (module == NULL) { + memcpy(stack_buffer, CLOG_WARNING_PREFIX, CLOG_WARNING_PREFIX_LENGTH); + } else { + prefix_chars = snprintf(stack_buffer, CLOG_STACK_BUFFER_SIZE, CLOG_WARNING_PREFIX_FORMAT, module); + if (prefix_chars < 0) { + /* Format error in prefix (possible if prefix is modified): skip prefix and continue as if nothing happened. */ + prefix_chars = 0; + } + } + + int format_chars; + if (prefix_chars + CLOG_SUFFIX_LENGTH >= CLOG_STACK_BUFFER_SIZE) { + /* + * Prefix + suffix alone would overflow the on-stack buffer, thus need to use on-heap buffer. + * Do not even try to format the string into on-stack buffer. + */ + format_chars = vsnprintf(NULL, 0, format, args); + } else { + format_chars = + vsnprintf( + &stack_buffer[prefix_chars], + CLOG_STACK_BUFFER_SIZE - prefix_chars - CLOG_SUFFIX_LENGTH, + format, + args); + } + if (format_chars < 0) { + /* Format error in the message: silently ignore this particular message. */ + goto cleanup; + } + if (prefix_chars + format_chars + CLOG_SUFFIX_LENGTH > CLOG_STACK_BUFFER_SIZE) { + /* Allocate a buffer on heap, and vsnprintf to this buffer */ + heap_buffer = malloc(prefix_chars + format_chars + CLOG_SUFFIX_LENGTH); + if (heap_buffer == NULL) { + goto cleanup; + } + + if (prefix_chars > CLOG_STACK_BUFFER_SIZE) { + /* Prefix didn't fit into on-stack buffer, re-format it again to on-heap buffer */ + snprintf(heap_buffer, prefix_chars + 1 /* for '\0'-terminator */, CLOG_WARNING_PREFIX_FORMAT, module); + } else { + /* Copy pre-formatted prefix from on-stack buffer to on-heap buffer */ + memcpy(heap_buffer, stack_buffer, prefix_chars); + } + vsnprintf(heap_buffer + prefix_chars, format_chars + CLOG_SUFFIX_LENGTH, format, args_copy); + out_buffer = heap_buffer; + } + out_buffer[prefix_chars + format_chars] = '\n'; + #ifdef _WIN32 + DWORD bytes_written; + WriteFile( + GetStdHandle(STD_ERROR_HANDLE), + out_buffer, prefix_chars + format_chars + CLOG_SUFFIX_LENGTH, + &bytes_written, NULL); + #else + write(STDERR_FILENO, out_buffer, prefix_chars + format_chars + CLOG_SUFFIX_LENGTH); + #endif + +cleanup: + free(heap_buffer); + va_end(args_copy); + #endif +} + +void clog_vlog_info(const char* module, const char* format, va_list args) { + #if defined(__ANDROID__) && !CLOG_LOG_TO_STDIO + __android_log_vprint(ANDROID_LOG_INFO, module, format, args); + #else + char stack_buffer[CLOG_STACK_BUFFER_SIZE]; + char* heap_buffer = NULL; + char* out_buffer = &stack_buffer[0]; + + /* The first call to vsnprintf will clobber args, thus need a copy in case a second vsnprintf call is needed */ + va_list args_copy; + va_copy(args_copy, args); + + int prefix_chars = CLOG_INFO_PREFIX_LENGTH; + if (module == NULL) { + memcpy(stack_buffer, CLOG_INFO_PREFIX, CLOG_INFO_PREFIX_LENGTH); + } else { + prefix_chars = snprintf(stack_buffer, CLOG_STACK_BUFFER_SIZE, CLOG_INFO_PREFIX_FORMAT, module); + if (prefix_chars < 0) { + /* Format error in prefix (possible if prefix is modified): skip prefix and continue as if nothing happened. */ + prefix_chars = 0; + } + } + + int format_chars; + if (prefix_chars + CLOG_SUFFIX_LENGTH >= CLOG_STACK_BUFFER_SIZE) { + /* + * Prefix + suffix alone would overflow the on-stack buffer, thus need to use on-heap buffer. + * Do not even try to format the string into on-stack buffer. + */ + format_chars = vsnprintf(NULL, 0, format, args); + } else { + format_chars = + vsnprintf( + &stack_buffer[prefix_chars], + CLOG_STACK_BUFFER_SIZE - prefix_chars - CLOG_SUFFIX_LENGTH, + format, + args); + } + if (format_chars < 0) { + /* Format error in the message: silently ignore this particular message. */ + goto cleanup; + } + if (prefix_chars + format_chars + CLOG_SUFFIX_LENGTH > CLOG_STACK_BUFFER_SIZE) { + /* Allocate a buffer on heap, and vsnprintf to this buffer */ + heap_buffer = malloc(prefix_chars + format_chars + CLOG_SUFFIX_LENGTH); + if (heap_buffer == NULL) { + goto cleanup; + } + + if (prefix_chars > CLOG_STACK_BUFFER_SIZE) { + /* Prefix didn't fit into on-stack buffer, re-format it again to on-heap buffer */ + snprintf(heap_buffer, prefix_chars + 1 /* for '\0'-terminator */, CLOG_INFO_PREFIX_FORMAT, module); + } else { + /* Copy pre-formatted prefix from on-stack buffer to on-heap buffer */ + memcpy(heap_buffer, stack_buffer, prefix_chars); + } + vsnprintf(heap_buffer + prefix_chars, format_chars + CLOG_SUFFIX_LENGTH, format, args_copy); + out_buffer = heap_buffer; + } + out_buffer[prefix_chars + format_chars] = '\n'; + #ifdef _WIN32 + DWORD bytes_written; + WriteFile( + GetStdHandle(STD_OUTPUT_HANDLE), + out_buffer, prefix_chars + format_chars + CLOG_SUFFIX_LENGTH, + &bytes_written, NULL); + #else + write(STDOUT_FILENO, out_buffer, prefix_chars + format_chars + CLOG_SUFFIX_LENGTH); + #endif + +cleanup: + free(heap_buffer); + va_end(args_copy); + #endif +} + +void clog_vlog_debug(const char* module, const char* format, va_list args) { + #if defined(__ANDROID__) && !CLOG_LOG_TO_STDIO + __android_log_vprint(ANDROID_LOG_DEBUG, module, format, args); + #else + char stack_buffer[CLOG_STACK_BUFFER_SIZE]; + char* heap_buffer = NULL; + char* out_buffer = &stack_buffer[0]; + + /* The first call to vsnprintf will clobber args, thus need a copy in case a second vsnprintf call is needed */ + va_list args_copy; + va_copy(args_copy, args); + + int prefix_chars = CLOG_DEBUG_PREFIX_LENGTH; + if (module == NULL) { + memcpy(stack_buffer, CLOG_DEBUG_PREFIX, CLOG_DEBUG_PREFIX_LENGTH); + } else { + prefix_chars = snprintf(stack_buffer, CLOG_STACK_BUFFER_SIZE, CLOG_DEBUG_PREFIX_FORMAT, module); + if (prefix_chars < 0) { + /* Format error in prefix (possible if prefix is modified): skip prefix and continue as if nothing happened. */ + prefix_chars = 0; + } + } + + int format_chars; + if (prefix_chars + CLOG_SUFFIX_LENGTH >= CLOG_STACK_BUFFER_SIZE) { + /* + * Prefix + suffix alone would overflow the on-stack buffer, thus need to use on-heap buffer. + * Do not even try to format the string into on-stack buffer. + */ + format_chars = vsnprintf(NULL, 0, format, args); + } else { + format_chars = + vsnprintf( + &stack_buffer[prefix_chars], + CLOG_STACK_BUFFER_SIZE - prefix_chars - CLOG_SUFFIX_LENGTH, + format, + args); + } + if (format_chars < 0) { + /* Format error in the message: silently ignore this particular message. */ + goto cleanup; + } + if (prefix_chars + format_chars + CLOG_SUFFIX_LENGTH > CLOG_STACK_BUFFER_SIZE) { + /* Allocate a buffer on heap, and vsnprintf to this buffer */ + heap_buffer = malloc(prefix_chars + format_chars + CLOG_SUFFIX_LENGTH); + if (heap_buffer == NULL) { + goto cleanup; + } + + if (prefix_chars > CLOG_STACK_BUFFER_SIZE) { + /* Prefix didn't fit into on-stack buffer, re-format it again to on-heap buffer */ + snprintf(heap_buffer, prefix_chars + 1 /* for '\0'-terminator */, CLOG_DEBUG_PREFIX_FORMAT, module); + } else { + /* Copy pre-formatted prefix from on-stack buffer to on-heap buffer */ + memcpy(heap_buffer, stack_buffer, prefix_chars); + } + vsnprintf(heap_buffer + prefix_chars, format_chars + CLOG_SUFFIX_LENGTH, format, args_copy); + out_buffer = heap_buffer; + } + out_buffer[prefix_chars + format_chars] = '\n'; + #ifdef _WIN32 + DWORD bytes_written; + WriteFile( + GetStdHandle(STD_OUTPUT_HANDLE), + out_buffer, prefix_chars + format_chars + CLOG_SUFFIX_LENGTH, + &bytes_written, NULL); + #else + write(STDOUT_FILENO, out_buffer, prefix_chars + format_chars + CLOG_SUFFIX_LENGTH); + #endif + +cleanup: + free(heap_buffer); + va_end(args_copy); + #endif +} diff --git a/3rdparty/cpuinfo/include/cpuinfo-mock.h b/3rdparty/cpuinfo/include/cpuinfo-mock.h new file mode 100644 index 000000000..3c1f637d9 --- /dev/null +++ b/3rdparty/cpuinfo/include/cpuinfo-mock.h @@ -0,0 +1,78 @@ +#pragma once +#ifndef CPUINFO_MOCK_H +#define CPUINFO_MOCK_H + +#include +#include + +#include +#if defined(__linux__) + #include +#endif + +#if !defined(CPUINFO_MOCK) || !(CPUINFO_MOCK) + #error This header is intended only for test use +#endif + + +#ifdef __cplusplus +extern "C" { +#endif + + +#if CPUINFO_ARCH_ARM + void CPUINFO_ABI cpuinfo_set_fpsid(uint32_t fpsid); + void CPUINFO_ABI cpuinfo_set_wcid(uint32_t wcid); +#endif /* CPUINFO_ARCH_ARM */ + +#if CPUINFO_ARCH_X86 || CPUINFO_ARCH_X86_64 + struct cpuinfo_mock_cpuid { + uint32_t input_eax; + uint32_t input_ecx; + uint32_t eax; + uint32_t ebx; + uint32_t ecx; + uint32_t edx; + }; + + void CPUINFO_ABI cpuinfo_mock_set_cpuid(struct cpuinfo_mock_cpuid* dump, size_t entries); + void CPUINFO_ABI cpuinfo_mock_get_cpuid(uint32_t eax, uint32_t regs[4]); + void CPUINFO_ABI cpuinfo_mock_get_cpuidex(uint32_t eax, uint32_t ecx, uint32_t regs[4]); +#endif /* CPUINFO_ARCH_X86 || CPUINFO_ARCH_X86_64 */ + +struct cpuinfo_mock_file { + const char* path; + size_t size; + const char* content; + size_t offset; +}; + +struct cpuinfo_mock_property { + const char* key; + const char* value; +}; + +#if defined(__linux__) + void CPUINFO_ABI cpuinfo_mock_filesystem(struct cpuinfo_mock_file* files); + int CPUINFO_ABI cpuinfo_mock_open(const char* path, int oflag); + int CPUINFO_ABI cpuinfo_mock_close(int fd); + ssize_t CPUINFO_ABI cpuinfo_mock_read(int fd, void* buffer, size_t capacity); + + #if CPUINFO_ARCH_ARM || CPUINFO_ARCH_ARM64 + void CPUINFO_ABI cpuinfo_set_hwcap(uint32_t hwcap); + #endif + #if CPUINFO_ARCH_ARM + void CPUINFO_ABI cpuinfo_set_hwcap2(uint32_t hwcap2); + #endif +#endif + +#if defined(__ANDROID__) + void CPUINFO_ABI cpuinfo_mock_android_properties(struct cpuinfo_mock_property* properties); + void CPUINFO_ABI cpuinfo_mock_gl_renderer(const char* renderer); +#endif + +#ifdef __cplusplus +} /* extern "C" */ +#endif + +#endif /* CPUINFO_MOCK_H */ diff --git a/3rdparty/cpuinfo/include/cpuinfo.h b/3rdparty/cpuinfo/include/cpuinfo.h new file mode 100644 index 000000000..258abd029 --- /dev/null +++ b/3rdparty/cpuinfo/include/cpuinfo.h @@ -0,0 +1,1894 @@ +#pragma once +#ifndef CPUINFO_H +#define CPUINFO_H + +#ifndef __cplusplus + #include +#endif + +#ifdef __APPLE__ + #include +#endif + +#include + +/* Identify architecture and define corresponding macro */ + +#if defined(__i386__) || defined(__i486__) || defined(__i586__) || defined(__i686__) || defined(_M_IX86) + #define CPUINFO_ARCH_X86 1 +#endif + +#if defined(__x86_64__) || defined(__x86_64) || defined(_M_X64) || defined(_M_AMD64) + #define CPUINFO_ARCH_X86_64 1 +#endif + +#if defined(__arm__) || defined(_M_ARM) + #define CPUINFO_ARCH_ARM 1 +#endif + +#if defined(__aarch64__) || defined(_M_ARM64) + #define CPUINFO_ARCH_ARM64 1 +#endif + +#if defined(__PPC64__) || defined(__powerpc64__) || defined(_ARCH_PPC64) + #define CPUINFO_ARCH_PPC64 1 +#endif + +#if defined(__asmjs__) + #define CPUINFO_ARCH_ASMJS 1 +#endif + +#if defined(__wasm__) + #if defined(__wasm_simd128__) + #define CPUINFO_ARCH_WASMSIMD 1 + #else + #define CPUINFO_ARCH_WASM 1 + #endif +#endif + +/* Define other architecture-specific macros as 0 */ + +#ifndef CPUINFO_ARCH_X86 + #define CPUINFO_ARCH_X86 0 +#endif + +#ifndef CPUINFO_ARCH_X86_64 + #define CPUINFO_ARCH_X86_64 0 +#endif + +#ifndef CPUINFO_ARCH_ARM + #define CPUINFO_ARCH_ARM 0 +#endif + +#ifndef CPUINFO_ARCH_ARM64 + #define CPUINFO_ARCH_ARM64 0 +#endif + +#ifndef CPUINFO_ARCH_PPC64 + #define CPUINFO_ARCH_PPC64 0 +#endif + +#ifndef CPUINFO_ARCH_ASMJS + #define CPUINFO_ARCH_ASMJS 0 +#endif + +#ifndef CPUINFO_ARCH_WASM + #define CPUINFO_ARCH_WASM 0 +#endif + +#ifndef CPUINFO_ARCH_WASMSIMD + #define CPUINFO_ARCH_WASMSIMD 0 +#endif + +#if CPUINFO_ARCH_X86 && defined(_MSC_VER) + #define CPUINFO_ABI __cdecl +#elif CPUINFO_ARCH_X86 && defined(__GNUC__) + #define CPUINFO_ABI __attribute__((__cdecl__)) +#else + #define CPUINFO_ABI +#endif + +#define CPUINFO_CACHE_UNIFIED 0x00000001 +#define CPUINFO_CACHE_INCLUSIVE 0x00000002 +#define CPUINFO_CACHE_COMPLEX_INDEXING 0x00000004 + +struct cpuinfo_cache { + /** Cache size in bytes */ + uint32_t size; + /** Number of ways of associativity */ + uint32_t associativity; + /** Number of sets */ + uint32_t sets; + /** Number of partitions */ + uint32_t partitions; + /** Line size in bytes */ + uint32_t line_size; + /** + * Binary characteristics of the cache (unified cache, inclusive cache, cache with complex indexing). + * + * @see CPUINFO_CACHE_UNIFIED, CPUINFO_CACHE_INCLUSIVE, CPUINFO_CACHE_COMPLEX_INDEXING + */ + uint32_t flags; + /** Index of the first logical processor that shares this cache */ + uint32_t processor_start; + /** Number of logical processors that share this cache */ + uint32_t processor_count; +}; + +struct cpuinfo_trace_cache { + uint32_t uops; + uint32_t associativity; +}; + +#define CPUINFO_PAGE_SIZE_4KB 0x1000 +#define CPUINFO_PAGE_SIZE_1MB 0x100000 +#define CPUINFO_PAGE_SIZE_2MB 0x200000 +#define CPUINFO_PAGE_SIZE_4MB 0x400000 +#define CPUINFO_PAGE_SIZE_16MB 0x1000000 +#define CPUINFO_PAGE_SIZE_1GB 0x40000000 + +struct cpuinfo_tlb { + uint32_t entries; + uint32_t associativity; + uint64_t pages; +}; + +/** Vendor of processor core design */ +enum cpuinfo_vendor { + /** Processor vendor is not known to the library, or the library failed to get vendor information from the OS. */ + cpuinfo_vendor_unknown = 0, + + /* Active vendors of modern CPUs */ + + /** + * Intel Corporation. Vendor of x86, x86-64, IA64, and ARM processor microarchitectures. + * + * Sold its ARM design subsidiary in 2006. The last ARM processor design was released in 2004. + */ + cpuinfo_vendor_intel = 1, + /** Advanced Micro Devices, Inc. Vendor of x86 and x86-64 processor microarchitectures. */ + cpuinfo_vendor_amd = 2, + /** ARM Holdings plc. Vendor of ARM and ARM64 processor microarchitectures. */ + cpuinfo_vendor_arm = 3, + /** Qualcomm Incorporated. Vendor of ARM and ARM64 processor microarchitectures. */ + cpuinfo_vendor_qualcomm = 4, + /** Apple Inc. Vendor of ARM and ARM64 processor microarchitectures. */ + cpuinfo_vendor_apple = 5, + /** Samsung Electronics Co., Ltd. Vendir if ARM64 processor microarchitectures. */ + cpuinfo_vendor_samsung = 6, + /** Nvidia Corporation. Vendor of ARM64-compatible processor microarchitectures. */ + cpuinfo_vendor_nvidia = 7, + /** MIPS Technologies, Inc. Vendor of MIPS processor microarchitectures. */ + cpuinfo_vendor_mips = 8, + /** International Business Machines Corporation. Vendor of PowerPC processor microarchitectures. */ + cpuinfo_vendor_ibm = 9, + /** Ingenic Semiconductor. Vendor of MIPS processor microarchitectures. */ + cpuinfo_vendor_ingenic = 10, + /** + * VIA Technologies, Inc. Vendor of x86 and x86-64 processor microarchitectures. + * + * Processors are designed by Centaur Technology, a subsidiary of VIA Technologies. + */ + cpuinfo_vendor_via = 11, + /** Cavium, Inc. Vendor of ARM64 processor microarchitectures. */ + cpuinfo_vendor_cavium = 12, + /** Broadcom, Inc. Vendor of ARM processor microarchitectures. */ + cpuinfo_vendor_broadcom = 13, + /** Applied Micro Circuits Corporation (APM). Vendor of ARM64 processor microarchitectures. */ + cpuinfo_vendor_apm = 14, + /** + * Huawei Technologies Co., Ltd. Vendor of ARM64 processor microarchitectures. + * + * Processors are designed by HiSilicon, a subsidiary of Huawei. + */ + cpuinfo_vendor_huawei = 15, + /** + * Hygon (Chengdu Haiguang Integrated Circuit Design Co., Ltd), Vendor of x86-64 processor microarchitectures. + * + * Processors are variants of AMD cores. + */ + cpuinfo_vendor_hygon = 16, + + /* Active vendors of embedded CPUs */ + + /** Texas Instruments Inc. Vendor of ARM processor microarchitectures. */ + cpuinfo_vendor_texas_instruments = 30, + /** Marvell Technology Group Ltd. Vendor of ARM processor microarchitectures. */ + cpuinfo_vendor_marvell = 31, + /** RDC Semiconductor Co., Ltd. Vendor of x86 processor microarchitectures. */ + cpuinfo_vendor_rdc = 32, + /** DM&P Electronics Inc. Vendor of x86 processor microarchitectures. */ + cpuinfo_vendor_dmp = 33, + /** Motorola, Inc. Vendor of PowerPC and ARM processor microarchitectures. */ + cpuinfo_vendor_motorola = 34, + + /* Defunct CPU vendors */ + + /** + * Transmeta Corporation. Vendor of x86 processor microarchitectures. + * + * Now defunct. The last processor design was released in 2004. + * Transmeta processors implemented VLIW ISA and used binary translation to execute x86 code. + */ + cpuinfo_vendor_transmeta = 50, + /** + * Cyrix Corporation. Vendor of x86 processor microarchitectures. + * + * Now defunct. The last processor design was released in 1996. + */ + cpuinfo_vendor_cyrix = 51, + /** + * Rise Technology. Vendor of x86 processor microarchitectures. + * + * Now defunct. The last processor design was released in 1999. + */ + cpuinfo_vendor_rise = 52, + /** + * National Semiconductor. Vendor of x86 processor microarchitectures. + * + * Sold its x86 design subsidiary in 1999. The last processor design was released in 1998. + */ + cpuinfo_vendor_nsc = 53, + /** + * Silicon Integrated Systems. Vendor of x86 processor microarchitectures. + * + * Sold its x86 design subsidiary in 2001. The last processor design was released in 2001. + */ + cpuinfo_vendor_sis = 54, + /** + * NexGen. Vendor of x86 processor microarchitectures. + * + * Now defunct. The last processor design was released in 1994. + * NexGen designed the first x86 microarchitecture which decomposed x86 instructions into simple microoperations. + */ + cpuinfo_vendor_nexgen = 55, + /** + * United Microelectronics Corporation. Vendor of x86 processor microarchitectures. + * + * Ceased x86 in the early 1990s. The last processor design was released in 1991. + * Designed U5C and U5D processors. Both are 486 level. + */ + cpuinfo_vendor_umc = 56, + /** + * Digital Equipment Corporation. Vendor of ARM processor microarchitecture. + * + * Sold its ARM designs in 1997. The last processor design was released in 1997. + */ + cpuinfo_vendor_dec = 57, +}; + +/** + * Processor microarchitecture + * + * Processors with different microarchitectures often have different instruction performance characteristics, + * and may have dramatically different pipeline organization. + */ +enum cpuinfo_uarch { + /** Microarchitecture is unknown, or the library failed to get information about the microarchitecture from OS */ + cpuinfo_uarch_unknown = 0, + + /** Pentium and Pentium MMX microarchitecture. */ + cpuinfo_uarch_p5 = 0x00100100, + /** Intel Quark microarchitecture. */ + cpuinfo_uarch_quark = 0x00100101, + + /** Pentium Pro, Pentium II, and Pentium III. */ + cpuinfo_uarch_p6 = 0x00100200, + /** Pentium M. */ + cpuinfo_uarch_dothan = 0x00100201, + /** Intel Core microarchitecture. */ + cpuinfo_uarch_yonah = 0x00100202, + /** Intel Core 2 microarchitecture on 65 nm process. */ + cpuinfo_uarch_conroe = 0x00100203, + /** Intel Core 2 microarchitecture on 45 nm process. */ + cpuinfo_uarch_penryn = 0x00100204, + /** Intel Nehalem and Westmere microarchitectures (Core i3/i5/i7 1st gen). */ + cpuinfo_uarch_nehalem = 0x00100205, + /** Intel Sandy Bridge microarchitecture (Core i3/i5/i7 2nd gen). */ + cpuinfo_uarch_sandy_bridge = 0x00100206, + /** Intel Ivy Bridge microarchitecture (Core i3/i5/i7 3rd gen). */ + cpuinfo_uarch_ivy_bridge = 0x00100207, + /** Intel Haswell microarchitecture (Core i3/i5/i7 4th gen). */ + cpuinfo_uarch_haswell = 0x00100208, + /** Intel Broadwell microarchitecture. */ + cpuinfo_uarch_broadwell = 0x00100209, + /** Intel Sky Lake microarchitecture (14 nm, including Kaby/Coffee/Whiskey/Amber/Comet/Cascade/Cooper Lake). */ + cpuinfo_uarch_sky_lake = 0x0010020A, + /** DEPRECATED (Intel Kaby Lake microarchitecture). */ + cpuinfo_uarch_kaby_lake = 0x0010020A, + /** Intel Palm Cove microarchitecture (10 nm, Cannon Lake). */ + cpuinfo_uarch_palm_cove = 0x0010020B, + /** Intel Sunny Cove microarchitecture (10 nm, Ice Lake). */ + cpuinfo_uarch_sunny_cove = 0x0010020C, + + /** Pentium 4 with Willamette, Northwood, or Foster cores. */ + cpuinfo_uarch_willamette = 0x00100300, + /** Pentium 4 with Prescott and later cores. */ + cpuinfo_uarch_prescott = 0x00100301, + + /** Intel Atom on 45 nm process. */ + cpuinfo_uarch_bonnell = 0x00100400, + /** Intel Atom on 32 nm process. */ + cpuinfo_uarch_saltwell = 0x00100401, + /** Intel Silvermont microarchitecture (22 nm out-of-order Atom). */ + cpuinfo_uarch_silvermont = 0x00100402, + /** Intel Airmont microarchitecture (14 nm out-of-order Atom). */ + cpuinfo_uarch_airmont = 0x00100403, + /** Intel Goldmont microarchitecture (Denverton, Apollo Lake). */ + cpuinfo_uarch_goldmont = 0x00100404, + /** Intel Goldmont Plus microarchitecture (Gemini Lake). */ + cpuinfo_uarch_goldmont_plus = 0x00100405, + + /** Intel Knights Ferry HPC boards. */ + cpuinfo_uarch_knights_ferry = 0x00100500, + /** Intel Knights Corner HPC boards (aka Xeon Phi). */ + cpuinfo_uarch_knights_corner = 0x00100501, + /** Intel Knights Landing microarchitecture (second-gen MIC). */ + cpuinfo_uarch_knights_landing = 0x00100502, + /** Intel Knights Hill microarchitecture (third-gen MIC). */ + cpuinfo_uarch_knights_hill = 0x00100503, + /** Intel Knights Mill Xeon Phi. */ + cpuinfo_uarch_knights_mill = 0x00100504, + + /** Intel/Marvell XScale series. */ + cpuinfo_uarch_xscale = 0x00100600, + + /** AMD K5. */ + cpuinfo_uarch_k5 = 0x00200100, + /** AMD K6 and alike. */ + cpuinfo_uarch_k6 = 0x00200101, + /** AMD Athlon and Duron. */ + cpuinfo_uarch_k7 = 0x00200102, + /** AMD Athlon 64, Opteron 64. */ + cpuinfo_uarch_k8 = 0x00200103, + /** AMD Family 10h (Barcelona, Istambul, Magny-Cours). */ + cpuinfo_uarch_k10 = 0x00200104, + /** + * AMD Bulldozer microarchitecture + * Zambezi FX-series CPUs, Zurich, Valencia and Interlagos Opteron CPUs. + */ + cpuinfo_uarch_bulldozer = 0x00200105, + /** + * AMD Piledriver microarchitecture + * Vishera FX-series CPUs, Trinity and Richland APUs, Delhi, Seoul, Abu Dhabi Opteron CPUs. + */ + cpuinfo_uarch_piledriver = 0x00200106, + /** AMD Steamroller microarchitecture (Kaveri APUs). */ + cpuinfo_uarch_steamroller = 0x00200107, + /** AMD Excavator microarchitecture (Carizzo APUs). */ + cpuinfo_uarch_excavator = 0x00200108, + /** AMD Zen microarchitecture (12/14 nm Ryzen and EPYC CPUs). */ + cpuinfo_uarch_zen = 0x00200109, + /** AMD Zen 2 microarchitecture (7 nm Ryzen and EPYC CPUs). */ + cpuinfo_uarch_zen2 = 0x0020010A, + /** AMD Zen 3 microarchitecture. */ + cpuinfo_uarch_zen3 = 0x0020010B, + + /** NSC Geode and AMD Geode GX and LX. */ + cpuinfo_uarch_geode = 0x00200200, + /** AMD Bobcat mobile microarchitecture. */ + cpuinfo_uarch_bobcat = 0x00200201, + /** AMD Jaguar mobile microarchitecture. */ + cpuinfo_uarch_jaguar = 0x00200202, + /** AMD Puma mobile microarchitecture. */ + cpuinfo_uarch_puma = 0x00200203, + + /** ARM7 series. */ + cpuinfo_uarch_arm7 = 0x00300100, + /** ARM9 series. */ + cpuinfo_uarch_arm9 = 0x00300101, + /** ARM 1136, ARM 1156, ARM 1176, or ARM 11MPCore. */ + cpuinfo_uarch_arm11 = 0x00300102, + + /** ARM Cortex-A5. */ + cpuinfo_uarch_cortex_a5 = 0x00300205, + /** ARM Cortex-A7. */ + cpuinfo_uarch_cortex_a7 = 0x00300207, + /** ARM Cortex-A8. */ + cpuinfo_uarch_cortex_a8 = 0x00300208, + /** ARM Cortex-A9. */ + cpuinfo_uarch_cortex_a9 = 0x00300209, + /** ARM Cortex-A12. */ + cpuinfo_uarch_cortex_a12 = 0x00300212, + /** ARM Cortex-A15. */ + cpuinfo_uarch_cortex_a15 = 0x00300215, + /** ARM Cortex-A17. */ + cpuinfo_uarch_cortex_a17 = 0x00300217, + + /** ARM Cortex-A32. */ + cpuinfo_uarch_cortex_a32 = 0x00300332, + /** ARM Cortex-A35. */ + cpuinfo_uarch_cortex_a35 = 0x00300335, + /** ARM Cortex-A53. */ + cpuinfo_uarch_cortex_a53 = 0x00300353, + /** ARM Cortex-A55 revision 0 (restricted dual-issue capabilities compared to revision 1+). */ + cpuinfo_uarch_cortex_a55r0 = 0x00300354, + /** ARM Cortex-A55. */ + cpuinfo_uarch_cortex_a55 = 0x00300355, + /** ARM Cortex-A57. */ + cpuinfo_uarch_cortex_a57 = 0x00300357, + /** ARM Cortex-A65. */ + cpuinfo_uarch_cortex_a65 = 0x00300365, + /** ARM Cortex-A72. */ + cpuinfo_uarch_cortex_a72 = 0x00300372, + /** ARM Cortex-A73. */ + cpuinfo_uarch_cortex_a73 = 0x00300373, + /** ARM Cortex-A75. */ + cpuinfo_uarch_cortex_a75 = 0x00300375, + /** ARM Cortex-A76. */ + cpuinfo_uarch_cortex_a76 = 0x00300376, + /** ARM Cortex-A77. */ + cpuinfo_uarch_cortex_a77 = 0x00300377, + /** ARM Cortex-A78. */ + cpuinfo_uarch_cortex_a78 = 0x00300378, + + /** ARM Neoverse N1. */ + cpuinfo_uarch_neoverse_n1 = 0x00300400, + /** ARM Neoverse E1. */ + cpuinfo_uarch_neoverse_e1 = 0x00300401, + /** ARM Neoverse V1. */ + cpuinfo_uarch_neoverse_v1 = 0x00300402, + /** ARM Neoverse N2. */ + cpuinfo_uarch_neoverse_n2 = 0x00300403, + + /** ARM Cortex-X1. */ + cpuinfo_uarch_cortex_x1 = 0x00300500, + + /** Qualcomm Scorpion. */ + cpuinfo_uarch_scorpion = 0x00400100, + /** Qualcomm Krait. */ + cpuinfo_uarch_krait = 0x00400101, + /** Qualcomm Kryo. */ + cpuinfo_uarch_kryo = 0x00400102, + /** Qualcomm Falkor. */ + cpuinfo_uarch_falkor = 0x00400103, + /** Qualcomm Saphira. */ + cpuinfo_uarch_saphira = 0x00400104, + + /** Nvidia Denver. */ + cpuinfo_uarch_denver = 0x00500100, + /** Nvidia Denver 2. */ + cpuinfo_uarch_denver2 = 0x00500101, + /** Nvidia Carmel. */ + cpuinfo_uarch_carmel = 0x00500102, + + /** Samsung Exynos M1 (Exynos 8890 big cores). */ + cpuinfo_uarch_exynos_m1 = 0x00600100, + /** Samsung Exynos M2 (Exynos 8895 big cores). */ + cpuinfo_uarch_exynos_m2 = 0x00600101, + /** Samsung Exynos M3 (Exynos 9810 big cores). */ + cpuinfo_uarch_exynos_m3 = 0x00600102, + /** Samsung Exynos M4 (Exynos 9820 big cores). */ + cpuinfo_uarch_exynos_m4 = 0x00600103, + /** Samsung Exynos M5 (Exynos 9830 big cores). */ + cpuinfo_uarch_exynos_m5 = 0x00600104, + + /* Deprecated synonym for Cortex-A76 */ + cpuinfo_uarch_cortex_a76ae = 0x00300376, + /* Deprecated names for Exynos. */ + cpuinfo_uarch_mongoose_m1 = 0x00600100, + cpuinfo_uarch_mongoose_m2 = 0x00600101, + cpuinfo_uarch_meerkat_m3 = 0x00600102, + cpuinfo_uarch_meerkat_m4 = 0x00600103, + + /** Apple A6 and A6X processors. */ + cpuinfo_uarch_swift = 0x00700100, + /** Apple A7 processor. */ + cpuinfo_uarch_cyclone = 0x00700101, + /** Apple A8 and A8X processor. */ + cpuinfo_uarch_typhoon = 0x00700102, + /** Apple A9 and A9X processor. */ + cpuinfo_uarch_twister = 0x00700103, + /** Apple A10 and A10X processor. */ + cpuinfo_uarch_hurricane = 0x00700104, + /** Apple A11 processor (big cores). */ + cpuinfo_uarch_monsoon = 0x00700105, + /** Apple A11 processor (little cores). */ + cpuinfo_uarch_mistral = 0x00700106, + /** Apple A12 processor (big cores). */ + cpuinfo_uarch_vortex = 0x00700107, + /** Apple A12 processor (little cores). */ + cpuinfo_uarch_tempest = 0x00700108, + /** Apple A13 processor (big cores). */ + cpuinfo_uarch_lightning = 0x00700109, + /** Apple A13 processor (little cores). */ + cpuinfo_uarch_thunder = 0x0070010A, + /** Apple M1 processor (big cores). */ + cpuinfo_uarch_firestorm = 0x0070010B, + /** Apple M1 processor (little cores). */ + cpuinfo_uarch_icestorm = 0x0070010C, + + /** Cavium ThunderX. */ + cpuinfo_uarch_thunderx = 0x00800100, + /** Cavium ThunderX2 (originally Broadcom Vulkan). */ + cpuinfo_uarch_thunderx2 = 0x00800200, + + /** Marvell PJ4. */ + cpuinfo_uarch_pj4 = 0x00900100, + + /** Broadcom Brahma B15. */ + cpuinfo_uarch_brahma_b15 = 0x00A00100, + /** Broadcom Brahma B53. */ + cpuinfo_uarch_brahma_b53 = 0x00A00101, + + /** Applied Micro X-Gene. */ + cpuinfo_uarch_xgene = 0x00B00100, + + /* Hygon Dhyana (a modification of AMD Zen for Chinese market). */ + cpuinfo_uarch_dhyana = 0x01000100, + + /** HiSilicon TaiShan v110 (Huawei Kunpeng 920 series processors). */ + cpuinfo_uarch_taishan_v110 = 0x00C00100, +}; + +struct cpuinfo_processor { + /** SMT (hyperthread) ID within a core */ + uint32_t smt_id; + /** Core containing this logical processor */ + const struct cpuinfo_core* core; + /** Cluster of cores containing this logical processor */ + const struct cpuinfo_cluster* cluster; + /** Physical package containing this logical processor */ + const struct cpuinfo_package* package; +#if defined(__linux__) + /** + * Linux-specific ID for the logical processor: + * - Linux kernel exposes information about this logical processor in /sys/devices/system/cpu/cpu/ + * - Bit in the cpu_set_t identifies this logical processor + */ + int linux_id; +#endif +#if defined(_WIN32) || defined(__CYGWIN__) + /** Windows-specific ID for the group containing the logical processor. */ + uint16_t windows_group_id; + /** + * Windows-specific ID of the logical processor within its group: + * - Bit in the KAFFINITY mask identifies this logical processor within its group. + */ + uint16_t windows_processor_id; +#endif +#if CPUINFO_ARCH_X86 || CPUINFO_ARCH_X86_64 + /** APIC ID (unique x86-specific ID of the logical processor) */ + uint32_t apic_id; +#endif + struct { + /** Level 1 instruction cache */ + const struct cpuinfo_cache* l1i; + /** Level 1 data cache */ + const struct cpuinfo_cache* l1d; + /** Level 2 unified or data cache */ + const struct cpuinfo_cache* l2; + /** Level 3 unified or data cache */ + const struct cpuinfo_cache* l3; + /** Level 4 unified or data cache */ + const struct cpuinfo_cache* l4; + } cache; +}; + +struct cpuinfo_core { + /** Index of the first logical processor on this core. */ + uint32_t processor_start; + /** Number of logical processors on this core */ + uint32_t processor_count; + /** Core ID within a package */ + uint32_t core_id; + /** Cluster containing this core */ + const struct cpuinfo_cluster* cluster; + /** Physical package containing this core. */ + const struct cpuinfo_package* package; + /** Vendor of the CPU microarchitecture for this core */ + enum cpuinfo_vendor vendor; + /** CPU microarchitecture for this core */ + enum cpuinfo_uarch uarch; +#if CPUINFO_ARCH_X86 || CPUINFO_ARCH_X86_64 + /** Value of CPUID leaf 1 EAX register for this core */ + uint32_t cpuid; +#elif CPUINFO_ARCH_ARM || CPUINFO_ARCH_ARM64 + /** Value of Main ID Register (MIDR) for this core */ + uint32_t midr; +#endif + /** Clock rate (non-Turbo) of the core, in Hz */ + uint64_t frequency; +}; + +struct cpuinfo_cluster { + /** Index of the first logical processor in the cluster */ + uint32_t processor_start; + /** Number of logical processors in the cluster */ + uint32_t processor_count; + /** Index of the first core in the cluster */ + uint32_t core_start; + /** Number of cores on the cluster */ + uint32_t core_count; + /** Cluster ID within a package */ + uint32_t cluster_id; + /** Physical package containing the cluster */ + const struct cpuinfo_package* package; + /** CPU microarchitecture vendor of the cores in the cluster */ + enum cpuinfo_vendor vendor; + /** CPU microarchitecture of the cores in the cluster */ + enum cpuinfo_uarch uarch; +#if CPUINFO_ARCH_X86 || CPUINFO_ARCH_X86_64 + /** Value of CPUID leaf 1 EAX register of the cores in the cluster */ + uint32_t cpuid; +#elif CPUINFO_ARCH_ARM || CPUINFO_ARCH_ARM64 + /** Value of Main ID Register (MIDR) of the cores in the cluster */ + uint32_t midr; +#endif + /** Clock rate (non-Turbo) of the cores in the cluster, in Hz */ + uint64_t frequency; +}; + +#define CPUINFO_PACKAGE_NAME_MAX 48 + +struct cpuinfo_package { + /** SoC or processor chip model name */ + char name[CPUINFO_PACKAGE_NAME_MAX]; + /** Index of the first logical processor on this physical package */ + uint32_t processor_start; + /** Number of logical processors on this physical package */ + uint32_t processor_count; + /** Index of the first core on this physical package */ + uint32_t core_start; + /** Number of cores on this physical package */ + uint32_t core_count; + /** Index of the first cluster of cores on this physical package */ + uint32_t cluster_start; + /** Number of clusters of cores on this physical package */ + uint32_t cluster_count; +}; + +struct cpuinfo_uarch_info { + /** Type of CPU microarchitecture */ + enum cpuinfo_uarch uarch; +#if CPUINFO_ARCH_X86 || CPUINFO_ARCH_X86_64 + /** Value of CPUID leaf 1 EAX register for the microarchitecture */ + uint32_t cpuid; +#elif CPUINFO_ARCH_ARM || CPUINFO_ARCH_ARM64 + /** Value of Main ID Register (MIDR) for the microarchitecture */ + uint32_t midr; +#endif + /** Number of logical processors with the microarchitecture */ + uint32_t processor_count; + /** Number of cores with the microarchitecture */ + uint32_t core_count; +}; + +#ifdef __cplusplus +extern "C" { +#endif + +bool CPUINFO_ABI cpuinfo_initialize(void); + +void CPUINFO_ABI cpuinfo_deinitialize(void); + +#if CPUINFO_ARCH_X86 || CPUINFO_ARCH_X86_64 + /* This structure is not a part of stable API. Use cpuinfo_has_x86_* functions instead. */ + struct cpuinfo_x86_isa { + #if CPUINFO_ARCH_X86 + bool rdtsc; + #endif + bool rdtscp; + bool rdpid; + bool sysenter; + #if CPUINFO_ARCH_X86 + bool syscall; + #endif + bool msr; + bool clzero; + bool clflush; + bool clflushopt; + bool mwait; + bool mwaitx; + #if CPUINFO_ARCH_X86 + bool emmx; + #endif + bool fxsave; + bool xsave; + #if CPUINFO_ARCH_X86 + bool fpu; + bool mmx; + bool mmx_plus; + #endif + bool three_d_now; + bool three_d_now_plus; + #if CPUINFO_ARCH_X86 + bool three_d_now_geode; + #endif + bool prefetch; + bool prefetchw; + bool prefetchwt1; + #if CPUINFO_ARCH_X86 + bool daz; + bool sse; + bool sse2; + #endif + bool sse3; + bool ssse3; + bool sse4_1; + bool sse4_2; + bool sse4a; + bool misaligned_sse; + bool avx; + bool fma3; + bool fma4; + bool xop; + bool f16c; + bool avx2; + bool avx512f; + bool avx512pf; + bool avx512er; + bool avx512cd; + bool avx512dq; + bool avx512bw; + bool avx512vl; + bool avx512ifma; + bool avx512vbmi; + bool avx512vbmi2; + bool avx512bitalg; + bool avx512vpopcntdq; + bool avx512vnni; + bool avx512bf16; + bool avx512vp2intersect; + bool avx512_4vnniw; + bool avx512_4fmaps; + bool hle; + bool rtm; + bool xtest; + bool mpx; + #if CPUINFO_ARCH_X86 + bool cmov; + bool cmpxchg8b; + #endif + bool cmpxchg16b; + bool clwb; + bool movbe; + #if CPUINFO_ARCH_X86_64 + bool lahf_sahf; + #endif + bool fs_gs_base; + bool lzcnt; + bool popcnt; + bool tbm; + bool bmi; + bool bmi2; + bool adx; + bool aes; + bool vaes; + bool pclmulqdq; + bool vpclmulqdq; + bool gfni; + bool rdrand; + bool rdseed; + bool sha; + bool rng; + bool ace; + bool ace2; + bool phe; + bool pmm; + bool lwp; + }; + + extern struct cpuinfo_x86_isa cpuinfo_isa; +#endif + +static inline bool cpuinfo_has_x86_rdtsc(void) { + #if CPUINFO_ARCH_X86_64 + return true; + #elif CPUINFO_ARCH_X86 + #if defined(__ANDROID__) + return true; + #else + return cpuinfo_isa.rdtsc; + #endif + #else + return false; + #endif +} + +static inline bool cpuinfo_has_x86_rdtscp(void) { + #if CPUINFO_ARCH_X86 || CPUINFO_ARCH_X86_64 + return cpuinfo_isa.rdtscp; + #else + return false; + #endif +} + +static inline bool cpuinfo_has_x86_rdpid(void) { + #if CPUINFO_ARCH_X86 || CPUINFO_ARCH_X86_64 + return cpuinfo_isa.rdpid; + #else + return false; + #endif +} + +static inline bool cpuinfo_has_x86_clzero(void) { + #if CPUINFO_ARCH_X86 || CPUINFO_ARCH_X86_64 + return cpuinfo_isa.clzero; + #else + return false; + #endif +} + +static inline bool cpuinfo_has_x86_mwait(void) { + #if CPUINFO_ARCH_X86 || CPUINFO_ARCH_X86_64 + return cpuinfo_isa.mwait; + #else + return false; + #endif +} + +static inline bool cpuinfo_has_x86_mwaitx(void) { + #if CPUINFO_ARCH_X86 || CPUINFO_ARCH_X86_64 + return cpuinfo_isa.mwaitx; + #else + return false; + #endif +} + +static inline bool cpuinfo_has_x86_fxsave(void) { + #if CPUINFO_ARCH_X86 || CPUINFO_ARCH_X86_64 + return cpuinfo_isa.fxsave; + #else + return false; + #endif +} + +static inline bool cpuinfo_has_x86_xsave(void) { + #if CPUINFO_ARCH_X86 || CPUINFO_ARCH_X86_64 + return cpuinfo_isa.xsave; + #else + return false; + #endif +} + +static inline bool cpuinfo_has_x86_fpu(void) { + #if CPUINFO_ARCH_X86_64 + return true; + #elif CPUINFO_ARCH_X86 + #if defined(__ANDROID__) + return true; + #else + return cpuinfo_isa.fpu; + #endif + #else + return false; + #endif +} + +static inline bool cpuinfo_has_x86_mmx(void) { + #if CPUINFO_ARCH_X86_64 + return true; + #elif CPUINFO_ARCH_X86 + #if defined(__ANDROID__) + return true; + #else + return cpuinfo_isa.mmx; + #endif + #else + return false; + #endif +} + +static inline bool cpuinfo_has_x86_mmx_plus(void) { + #if CPUINFO_ARCH_X86_64 + return true; + #elif CPUINFO_ARCH_X86 + #if defined(__ANDROID__) + return true; + #else + return cpuinfo_isa.mmx_plus; + #endif + #else + return false; + #endif +} + +static inline bool cpuinfo_has_x86_3dnow(void) { + #if CPUINFO_ARCH_X86 || CPUINFO_ARCH_X86_64 + return cpuinfo_isa.three_d_now; + #else + return false; + #endif +} + +static inline bool cpuinfo_has_x86_3dnow_plus(void) { + #if CPUINFO_ARCH_X86 || CPUINFO_ARCH_X86_64 + return cpuinfo_isa.three_d_now_plus; + #else + return false; + #endif +} + +static inline bool cpuinfo_has_x86_3dnow_geode(void) { + #if CPUINFO_ARCH_X86_64 + return false; + #elif CPUINFO_ARCH_X86 + #if defined(__ANDROID__) + return false; + #else + return cpuinfo_isa.three_d_now_geode; + #endif + #else + return false; + #endif +} + +static inline bool cpuinfo_has_x86_prefetch(void) { + #if CPUINFO_ARCH_X86 || CPUINFO_ARCH_X86_64 + return cpuinfo_isa.prefetch; + #else + return false; + #endif +} + +static inline bool cpuinfo_has_x86_prefetchw(void) { + #if CPUINFO_ARCH_X86 || CPUINFO_ARCH_X86_64 + return cpuinfo_isa.prefetchw; + #else + return false; + #endif +} + +static inline bool cpuinfo_has_x86_prefetchwt1(void) { + #if CPUINFO_ARCH_X86 || CPUINFO_ARCH_X86_64 + return cpuinfo_isa.prefetchwt1; + #else + return false; + #endif +} + +static inline bool cpuinfo_has_x86_daz(void) { + #if CPUINFO_ARCH_X86_64 + return true; + #elif CPUINFO_ARCH_X86 + #if defined(__ANDROID__) + return true; + #else + return cpuinfo_isa.daz; + #endif + #else + return false; + #endif +} + +static inline bool cpuinfo_has_x86_sse(void) { + #if CPUINFO_ARCH_X86_64 + return true; + #elif CPUINFO_ARCH_X86 + #if defined(__ANDROID__) + return true; + #else + return cpuinfo_isa.sse; + #endif + #else + return false; + #endif +} + +static inline bool cpuinfo_has_x86_sse2(void) { + #if CPUINFO_ARCH_X86_64 + return true; + #elif CPUINFO_ARCH_X86 + #if defined(__ANDROID__) + return true; + #else + return cpuinfo_isa.sse2; + #endif + #else + return false; + #endif +} + +static inline bool cpuinfo_has_x86_sse3(void) { + #if CPUINFO_ARCH_X86 || CPUINFO_ARCH_X86_64 + #if defined(__ANDROID__) + return true; + #else + return cpuinfo_isa.sse3; + #endif + #else + return false; + #endif +} + +static inline bool cpuinfo_has_x86_ssse3(void) { + #if CPUINFO_ARCH_X86 || CPUINFO_ARCH_X86_64 + #if defined(__ANDROID__) + return true; + #else + return cpuinfo_isa.ssse3; + #endif + #else + return false; + #endif +} + +static inline bool cpuinfo_has_x86_sse4_1(void) { + #if CPUINFO_ARCH_X86_64 + #if defined(__ANDROID__) + return true; + #else + return cpuinfo_isa.sse4_1; + #endif + #elif CPUINFO_ARCH_X86 + return cpuinfo_isa.sse4_1; + #else + return false; + #endif +} + +static inline bool cpuinfo_has_x86_sse4_2(void) { + #if CPUINFO_ARCH_X86_64 + #if defined(__ANDROID__) + return true; + #else + return cpuinfo_isa.sse4_2; + #endif + #elif CPUINFO_ARCH_X86 + return cpuinfo_isa.sse4_2; + #else + return false; + #endif +} + +static inline bool cpuinfo_has_x86_sse4a(void) { + #if CPUINFO_ARCH_X86 || CPUINFO_ARCH_X86_64 + return cpuinfo_isa.sse4a; + #else + return false; + #endif +} + +static inline bool cpuinfo_has_x86_misaligned_sse(void) { + #if CPUINFO_ARCH_X86 || CPUINFO_ARCH_X86_64 + return cpuinfo_isa.misaligned_sse; + #else + return false; + #endif +} + +static inline bool cpuinfo_has_x86_avx(void) { + #if CPUINFO_ARCH_X86 || CPUINFO_ARCH_X86_64 + return cpuinfo_isa.avx; + #else + return false; + #endif +} + +static inline bool cpuinfo_has_x86_fma3(void) { + #if CPUINFO_ARCH_X86 || CPUINFO_ARCH_X86_64 + return cpuinfo_isa.fma3; + #else + return false; + #endif +} + +static inline bool cpuinfo_has_x86_fma4(void) { + #if CPUINFO_ARCH_X86 || CPUINFO_ARCH_X86_64 + return cpuinfo_isa.fma4; + #else + return false; + #endif +} + +static inline bool cpuinfo_has_x86_xop(void) { + #if CPUINFO_ARCH_X86 || CPUINFO_ARCH_X86_64 + return cpuinfo_isa.xop; + #else + return false; + #endif +} + +static inline bool cpuinfo_has_x86_f16c(void) { + #if CPUINFO_ARCH_X86 || CPUINFO_ARCH_X86_64 + return cpuinfo_isa.f16c; + #else + return false; + #endif +} + +static inline bool cpuinfo_has_x86_avx2(void) { + #if CPUINFO_ARCH_X86 || CPUINFO_ARCH_X86_64 + return cpuinfo_isa.avx2; + #else + return false; + #endif +} + +static inline bool cpuinfo_has_x86_avx512f(void) { + #if CPUINFO_ARCH_X86 || CPUINFO_ARCH_X86_64 + return cpuinfo_isa.avx512f; + #else + return false; + #endif +} + +static inline bool cpuinfo_has_x86_avx512pf(void) { + #if CPUINFO_ARCH_X86 || CPUINFO_ARCH_X86_64 + return cpuinfo_isa.avx512pf; + #else + return false; + #endif +} + +static inline bool cpuinfo_has_x86_avx512er(void) { + #if CPUINFO_ARCH_X86 || CPUINFO_ARCH_X86_64 + return cpuinfo_isa.avx512er; + #else + return false; + #endif +} + +static inline bool cpuinfo_has_x86_avx512cd(void) { + #if CPUINFO_ARCH_X86 || CPUINFO_ARCH_X86_64 + return cpuinfo_isa.avx512cd; + #else + return false; + #endif +} + +static inline bool cpuinfo_has_x86_avx512dq(void) { + #if CPUINFO_ARCH_X86 || CPUINFO_ARCH_X86_64 + return cpuinfo_isa.avx512dq; + #else + return false; + #endif +} + +static inline bool cpuinfo_has_x86_avx512bw(void) { + #if CPUINFO_ARCH_X86 || CPUINFO_ARCH_X86_64 + return cpuinfo_isa.avx512bw; + #else + return false; + #endif +} + +static inline bool cpuinfo_has_x86_avx512vl(void) { + #if CPUINFO_ARCH_X86 || CPUINFO_ARCH_X86_64 + return cpuinfo_isa.avx512vl; + #else + return false; + #endif +} + +static inline bool cpuinfo_has_x86_avx512ifma(void) { + #if CPUINFO_ARCH_X86 || CPUINFO_ARCH_X86_64 + return cpuinfo_isa.avx512ifma; + #else + return false; + #endif +} + +static inline bool cpuinfo_has_x86_avx512vbmi(void) { + #if CPUINFO_ARCH_X86 || CPUINFO_ARCH_X86_64 + return cpuinfo_isa.avx512vbmi; + #else + return false; + #endif +} + +static inline bool cpuinfo_has_x86_avx512vbmi2(void) { + #if CPUINFO_ARCH_X86 || CPUINFO_ARCH_X86_64 + return cpuinfo_isa.avx512vbmi2; + #else + return false; + #endif +} + +static inline bool cpuinfo_has_x86_avx512bitalg(void) { + #if CPUINFO_ARCH_X86 || CPUINFO_ARCH_X86_64 + return cpuinfo_isa.avx512bitalg; + #else + return false; + #endif +} + +static inline bool cpuinfo_has_x86_avx512vpopcntdq(void) { + #if CPUINFO_ARCH_X86 || CPUINFO_ARCH_X86_64 + return cpuinfo_isa.avx512vpopcntdq; + #else + return false; + #endif +} + +static inline bool cpuinfo_has_x86_avx512vnni(void) { + #if CPUINFO_ARCH_X86 || CPUINFO_ARCH_X86_64 + return cpuinfo_isa.avx512vnni; + #else + return false; + #endif +} + +static inline bool cpuinfo_has_x86_avx512bf16(void) { + #if CPUINFO_ARCH_X86 || CPUINFO_ARCH_X86_64 + return cpuinfo_isa.avx512bf16; + #else + return false; + #endif +} + +static inline bool cpuinfo_has_x86_avx512vp2intersect(void) { + #if CPUINFO_ARCH_X86 || CPUINFO_ARCH_X86_64 + return cpuinfo_isa.avx512vp2intersect; + #else + return false; + #endif +} + +static inline bool cpuinfo_has_x86_avx512_4vnniw(void) { + #if CPUINFO_ARCH_X86 || CPUINFO_ARCH_X86_64 + return cpuinfo_isa.avx512_4vnniw; + #else + return false; + #endif +} + +static inline bool cpuinfo_has_x86_avx512_4fmaps(void) { + #if CPUINFO_ARCH_X86 || CPUINFO_ARCH_X86_64 + return cpuinfo_isa.avx512_4fmaps; + #else + return false; + #endif +} + +static inline bool cpuinfo_has_x86_hle(void) { + #if CPUINFO_ARCH_X86 || CPUINFO_ARCH_X86_64 + return cpuinfo_isa.hle; + #else + return false; + #endif +} + +static inline bool cpuinfo_has_x86_rtm(void) { + #if CPUINFO_ARCH_X86 || CPUINFO_ARCH_X86_64 + return cpuinfo_isa.rtm; + #else + return false; + #endif +} + +static inline bool cpuinfo_has_x86_xtest(void) { + #if CPUINFO_ARCH_X86 || CPUINFO_ARCH_X86_64 + return cpuinfo_isa.xtest; + #else + return false; + #endif +} + +static inline bool cpuinfo_has_x86_mpx(void) { + #if CPUINFO_ARCH_X86 || CPUINFO_ARCH_X86_64 + return cpuinfo_isa.mpx; + #else + return false; + #endif +} + +static inline bool cpuinfo_has_x86_cmov(void) { + #if CPUINFO_ARCH_X86_64 + return true; + #elif CPUINFO_ARCH_X86 + return cpuinfo_isa.cmov; + #else + return false; + #endif +} + +static inline bool cpuinfo_has_x86_cmpxchg8b(void) { + #if CPUINFO_ARCH_X86_64 + return true; + #elif CPUINFO_ARCH_X86 + return cpuinfo_isa.cmpxchg8b; + #else + return false; + #endif +} + +static inline bool cpuinfo_has_x86_cmpxchg16b(void) { + #if CPUINFO_ARCH_X86_64 + return cpuinfo_isa.cmpxchg16b; + #else + return false; + #endif +} + +static inline bool cpuinfo_has_x86_clwb(void) { + #if CPUINFO_ARCH_X86 || CPUINFO_ARCH_X86_64 + return cpuinfo_isa.clwb; + #else + return false; + #endif +} + +static inline bool cpuinfo_has_x86_movbe(void) { + #if CPUINFO_ARCH_X86 || CPUINFO_ARCH_X86_64 + return cpuinfo_isa.movbe; + #else + return false; + #endif +} + +static inline bool cpuinfo_has_x86_lahf_sahf(void) { + #if CPUINFO_ARCH_X86 + return true; + #elif CPUINFO_ARCH_X86_64 + return cpuinfo_isa.lahf_sahf; + #else + return false; + #endif +} + +static inline bool cpuinfo_has_x86_lzcnt(void) { + #if CPUINFO_ARCH_X86 || CPUINFO_ARCH_X86_64 + return cpuinfo_isa.lzcnt; + #else + return false; + #endif +} + +static inline bool cpuinfo_has_x86_popcnt(void) { + #if CPUINFO_ARCH_X86_64 + #if defined(__ANDROID__) + return true; + #else + return cpuinfo_isa.popcnt; + #endif + #elif CPUINFO_ARCH_X86 + return cpuinfo_isa.popcnt; + #else + return false; + #endif +} + +static inline bool cpuinfo_has_x86_tbm(void) { + #if CPUINFO_ARCH_X86 || CPUINFO_ARCH_X86_64 + return cpuinfo_isa.tbm; + #else + return false; + #endif +} + +static inline bool cpuinfo_has_x86_bmi(void) { + #if CPUINFO_ARCH_X86 || CPUINFO_ARCH_X86_64 + return cpuinfo_isa.bmi; + #else + return false; + #endif +} + +static inline bool cpuinfo_has_x86_bmi2(void) { + #if CPUINFO_ARCH_X86 || CPUINFO_ARCH_X86_64 + return cpuinfo_isa.bmi2; + #else + return false; + #endif +} + +static inline bool cpuinfo_has_x86_adx(void) { + #if CPUINFO_ARCH_X86 || CPUINFO_ARCH_X86_64 + return cpuinfo_isa.adx; + #else + return false; + #endif +} + +static inline bool cpuinfo_has_x86_aes(void) { + #if CPUINFO_ARCH_X86 || CPUINFO_ARCH_X86_64 + return cpuinfo_isa.aes; + #else + return false; + #endif +} + +static inline bool cpuinfo_has_x86_vaes(void) { + #if CPUINFO_ARCH_X86 || CPUINFO_ARCH_X86_64 + return cpuinfo_isa.vaes; + #else + return false; + #endif +} + +static inline bool cpuinfo_has_x86_pclmulqdq(void) { + #if CPUINFO_ARCH_X86 || CPUINFO_ARCH_X86_64 + return cpuinfo_isa.pclmulqdq; + #else + return false; + #endif +} + +static inline bool cpuinfo_has_x86_vpclmulqdq(void) { + #if CPUINFO_ARCH_X86 || CPUINFO_ARCH_X86_64 + return cpuinfo_isa.vpclmulqdq; + #else + return false; + #endif +} + +static inline bool cpuinfo_has_x86_gfni(void) { + #if CPUINFO_ARCH_X86 || CPUINFO_ARCH_X86_64 + return cpuinfo_isa.gfni; + #else + return false; + #endif +} + +static inline bool cpuinfo_has_x86_rdrand(void) { + #if CPUINFO_ARCH_X86 || CPUINFO_ARCH_X86_64 + return cpuinfo_isa.rdrand; + #else + return false; + #endif +} + +static inline bool cpuinfo_has_x86_rdseed(void) { + #if CPUINFO_ARCH_X86 || CPUINFO_ARCH_X86_64 + return cpuinfo_isa.rdseed; + #else + return false; + #endif +} + +static inline bool cpuinfo_has_x86_sha(void) { + #if CPUINFO_ARCH_X86 || CPUINFO_ARCH_X86_64 + return cpuinfo_isa.sha; + #else + return false; + #endif +} + +#if CPUINFO_ARCH_ARM || CPUINFO_ARCH_ARM64 + /* This structure is not a part of stable API. Use cpuinfo_has_arm_* functions instead. */ + struct cpuinfo_arm_isa { + #if CPUINFO_ARCH_ARM + bool thumb; + bool thumb2; + bool thumbee; + bool jazelle; + bool armv5e; + bool armv6; + bool armv6k; + bool armv7; + bool armv7mp; + bool armv8; + bool idiv; + + bool vfpv2; + bool vfpv3; + bool d32; + bool fp16; + bool fma; + + bool wmmx; + bool wmmx2; + bool neon; + #endif + #if CPUINFO_ARCH_ARM64 + bool atomics; + bool bf16; + bool sve; + bool svebf16; + bool sve2; + #endif + bool rdm; + bool fp16arith; + bool dot; + bool jscvt; + bool fcma; + + bool aes; + bool sha1; + bool sha2; + bool pmull; + bool crc32; + }; + + extern struct cpuinfo_arm_isa cpuinfo_isa; +#endif + +static inline bool cpuinfo_has_arm_thumb(void) { + #if CPUINFO_ARCH_ARM + return cpuinfo_isa.thumb; + #else + return false; + #endif +} + +static inline bool cpuinfo_has_arm_thumb2(void) { + #if CPUINFO_ARCH_ARM + return cpuinfo_isa.thumb2; + #else + return false; + #endif +} + +static inline bool cpuinfo_has_arm_v5e(void) { + #if CPUINFO_ARCH_ARM + return cpuinfo_isa.armv5e; + #else + return false; + #endif +} + +static inline bool cpuinfo_has_arm_v6(void) { + #if CPUINFO_ARCH_ARM + return cpuinfo_isa.armv6; + #else + return false; + #endif +} + +static inline bool cpuinfo_has_arm_v6k(void) { + #if CPUINFO_ARCH_ARM + return cpuinfo_isa.armv6k; + #else + return false; + #endif +} + +static inline bool cpuinfo_has_arm_v7(void) { + #if CPUINFO_ARCH_ARM + return cpuinfo_isa.armv7; + #else + return false; + #endif +} + +static inline bool cpuinfo_has_arm_v7mp(void) { + #if CPUINFO_ARCH_ARM + return cpuinfo_isa.armv7mp; + #else + return false; + #endif +} + +static inline bool cpuinfo_has_arm_v8(void) { + #if CPUINFO_ARCH_ARM64 + return true; + #elif CPUINFO_ARCH_ARM + return cpuinfo_isa.armv8; + #else + return false; + #endif +} + +static inline bool cpuinfo_has_arm_idiv(void) { + #if CPUINFO_ARCH_ARM64 + return true; + #elif CPUINFO_ARCH_ARM + return cpuinfo_isa.idiv; + #else + return false; + #endif +} + +static inline bool cpuinfo_has_arm_vfpv2(void) { + #if CPUINFO_ARCH_ARM + return cpuinfo_isa.vfpv2; + #else + return false; + #endif +} + +static inline bool cpuinfo_has_arm_vfpv3(void) { + #if CPUINFO_ARCH_ARM64 + return true; + #elif CPUINFO_ARCH_ARM + return cpuinfo_isa.vfpv3; + #else + return false; + #endif +} + +static inline bool cpuinfo_has_arm_vfpv3_d32(void) { + #if CPUINFO_ARCH_ARM64 + return true; + #elif CPUINFO_ARCH_ARM + return cpuinfo_isa.vfpv3 && cpuinfo_isa.d32; + #else + return false; + #endif +} + +static inline bool cpuinfo_has_arm_vfpv3_fp16(void) { + #if CPUINFO_ARCH_ARM64 + return true; + #elif CPUINFO_ARCH_ARM + return cpuinfo_isa.vfpv3 && cpuinfo_isa.fp16; + #else + return false; + #endif +} + +static inline bool cpuinfo_has_arm_vfpv3_fp16_d32(void) { + #if CPUINFO_ARCH_ARM64 + return true; + #elif CPUINFO_ARCH_ARM + return cpuinfo_isa.vfpv3 && cpuinfo_isa.fp16 && cpuinfo_isa.d32; + #else + return false; + #endif +} + +static inline bool cpuinfo_has_arm_vfpv4(void) { + #if CPUINFO_ARCH_ARM64 + return true; + #elif CPUINFO_ARCH_ARM + return cpuinfo_isa.vfpv3 && cpuinfo_isa.fma; + #else + return false; + #endif +} + +static inline bool cpuinfo_has_arm_vfpv4_d32(void) { + #if CPUINFO_ARCH_ARM64 + return true; + #elif CPUINFO_ARCH_ARM + return cpuinfo_isa.vfpv3 && cpuinfo_isa.fma && cpuinfo_isa.d32; + #else + return false; + #endif +} + +static inline bool cpuinfo_has_arm_wmmx(void) { + #if CPUINFO_ARCH_ARM + return cpuinfo_isa.wmmx; + #else + return false; + #endif +} + +static inline bool cpuinfo_has_arm_wmmx2(void) { + #if CPUINFO_ARCH_ARM + return cpuinfo_isa.wmmx2; + #else + return false; + #endif +} + +static inline bool cpuinfo_has_arm_neon(void) { + #if CPUINFO_ARCH_ARM64 + return true; + #elif CPUINFO_ARCH_ARM + return cpuinfo_isa.neon; + #else + return false; + #endif +} + +static inline bool cpuinfo_has_arm_neon_fp16(void) { + #if CPUINFO_ARCH_ARM64 + return true; + #elif CPUINFO_ARCH_ARM + return cpuinfo_isa.neon && cpuinfo_isa.fp16; + #else + return false; + #endif +} + +static inline bool cpuinfo_has_arm_neon_fma(void) { + #if CPUINFO_ARCH_ARM64 + return true; + #elif CPUINFO_ARCH_ARM + return cpuinfo_isa.neon && cpuinfo_isa.fma; + #else + return false; + #endif +} + +static inline bool cpuinfo_has_arm_neon_v8(void) { + #if CPUINFO_ARCH_ARM64 + return true; + #elif CPUINFO_ARCH_ARM + return cpuinfo_isa.neon && cpuinfo_isa.armv8; + #else + return false; + #endif +} + +static inline bool cpuinfo_has_arm_atomics(void) { + #if CPUINFO_ARCH_ARM64 + return cpuinfo_isa.atomics; + #else + return false; + #endif +} + +static inline bool cpuinfo_has_arm_neon_rdm(void) { + #if CPUINFO_ARCH_ARM || CPUINFO_ARCH_ARM64 + return cpuinfo_isa.rdm; + #else + return false; + #endif +} + +static inline bool cpuinfo_has_arm_neon_fp16_arith(void) { + #if CPUINFO_ARCH_ARM + return cpuinfo_isa.neon && cpuinfo_isa.fp16arith; + #elif CPUINFO_ARCH_ARM64 + return cpuinfo_isa.fp16arith; + #else + return false; + #endif +} + +static inline bool cpuinfo_has_arm_fp16_arith(void) { + #if CPUINFO_ARCH_ARM || CPUINFO_ARCH_ARM64 + return cpuinfo_isa.fp16arith; + #else + return false; + #endif +} + +static inline bool cpuinfo_has_arm_neon_dot(void) { + #if CPUINFO_ARCH_ARM || CPUINFO_ARCH_ARM64 + return cpuinfo_isa.dot; + #else + return false; + #endif +} + +static inline bool cpuinfo_has_arm_jscvt(void) { + #if CPUINFO_ARCH_ARM || CPUINFO_ARCH_ARM64 + return cpuinfo_isa.jscvt; + #else + return false; + #endif +} + +static inline bool cpuinfo_has_arm_fcma(void) { + #if CPUINFO_ARCH_ARM || CPUINFO_ARCH_ARM64 + return cpuinfo_isa.fcma; + #else + return false; + #endif +} + +static inline bool cpuinfo_has_arm_aes(void) { + #if CPUINFO_ARCH_ARM || CPUINFO_ARCH_ARM64 + return cpuinfo_isa.aes; + #else + return false; + #endif +} + +static inline bool cpuinfo_has_arm_sha1(void) { + #if CPUINFO_ARCH_ARM || CPUINFO_ARCH_ARM64 + return cpuinfo_isa.sha1; + #else + return false; + #endif +} + +static inline bool cpuinfo_has_arm_sha2(void) { + #if CPUINFO_ARCH_ARM || CPUINFO_ARCH_ARM64 + return cpuinfo_isa.sha2; + #else + return false; + #endif +} + +static inline bool cpuinfo_has_arm_pmull(void) { + #if CPUINFO_ARCH_ARM || CPUINFO_ARCH_ARM64 + return cpuinfo_isa.pmull; + #else + return false; + #endif +} + +static inline bool cpuinfo_has_arm_crc32(void) { + #if CPUINFO_ARCH_ARM || CPUINFO_ARCH_ARM64 + return cpuinfo_isa.crc32; + #else + return false; + #endif +} + +static inline bool cpuinfo_has_arm_sve(void) { + #if CPUINFO_ARCH_ARM64 + return cpuinfo_isa.sve; + #else + return false; + #endif +} + +static inline bool cpuinfo_has_arm_sve2(void) { + #if CPUINFO_ARCH_ARM64 + return cpuinfo_isa.sve2; + #else + return false; + #endif +} + +static inline bool cpuinfo_has_arm_bf16(void) { + #if CPUINFO_ARCH_ARM64 + return cpuinfo_isa.bf16; + #else + return false; + #endif +} + +static inline bool cpuinfo_has_arm_svebf16(void) { + #if CPUINFO_ARCH_ARM64 + return cpuinfo_isa.svebf16; + #else + return false; + #endif +} + +const struct cpuinfo_processor* CPUINFO_ABI cpuinfo_get_processors(void); +const struct cpuinfo_core* CPUINFO_ABI cpuinfo_get_cores(void); +const struct cpuinfo_cluster* CPUINFO_ABI cpuinfo_get_clusters(void); +const struct cpuinfo_package* CPUINFO_ABI cpuinfo_get_packages(void); +const struct cpuinfo_uarch_info* CPUINFO_ABI cpuinfo_get_uarchs(void); +const struct cpuinfo_cache* CPUINFO_ABI cpuinfo_get_l1i_caches(void); +const struct cpuinfo_cache* CPUINFO_ABI cpuinfo_get_l1d_caches(void); +const struct cpuinfo_cache* CPUINFO_ABI cpuinfo_get_l2_caches(void); +const struct cpuinfo_cache* CPUINFO_ABI cpuinfo_get_l3_caches(void); +const struct cpuinfo_cache* CPUINFO_ABI cpuinfo_get_l4_caches(void); + +const struct cpuinfo_processor* CPUINFO_ABI cpuinfo_get_processor(uint32_t index); +const struct cpuinfo_core* CPUINFO_ABI cpuinfo_get_core(uint32_t index); +const struct cpuinfo_cluster* CPUINFO_ABI cpuinfo_get_cluster(uint32_t index); +const struct cpuinfo_package* CPUINFO_ABI cpuinfo_get_package(uint32_t index); +const struct cpuinfo_uarch_info* CPUINFO_ABI cpuinfo_get_uarch(uint32_t index); +const struct cpuinfo_cache* CPUINFO_ABI cpuinfo_get_l1i_cache(uint32_t index); +const struct cpuinfo_cache* CPUINFO_ABI cpuinfo_get_l1d_cache(uint32_t index); +const struct cpuinfo_cache* CPUINFO_ABI cpuinfo_get_l2_cache(uint32_t index); +const struct cpuinfo_cache* CPUINFO_ABI cpuinfo_get_l3_cache(uint32_t index); +const struct cpuinfo_cache* CPUINFO_ABI cpuinfo_get_l4_cache(uint32_t index); + +uint32_t CPUINFO_ABI cpuinfo_get_processors_count(void); +uint32_t CPUINFO_ABI cpuinfo_get_cores_count(void); +uint32_t CPUINFO_ABI cpuinfo_get_clusters_count(void); +uint32_t CPUINFO_ABI cpuinfo_get_packages_count(void); +uint32_t CPUINFO_ABI cpuinfo_get_uarchs_count(void); +uint32_t CPUINFO_ABI cpuinfo_get_l1i_caches_count(void); +uint32_t CPUINFO_ABI cpuinfo_get_l1d_caches_count(void); +uint32_t CPUINFO_ABI cpuinfo_get_l2_caches_count(void); +uint32_t CPUINFO_ABI cpuinfo_get_l3_caches_count(void); +uint32_t CPUINFO_ABI cpuinfo_get_l4_caches_count(void); + +/** + * Returns upper bound on cache size. + */ +uint32_t CPUINFO_ABI cpuinfo_get_max_cache_size(void); + +/** + * Identify the logical processor that executes the current thread. + * + * There is no guarantee that the thread will stay on the same logical processor for any time. + * Callers should treat the result as only a hint, and be prepared to handle NULL return value. + */ +const struct cpuinfo_processor* CPUINFO_ABI cpuinfo_get_current_processor(void); + +/** + * Identify the core that executes the current thread. + * + * There is no guarantee that the thread will stay on the same core for any time. + * Callers should treat the result as only a hint, and be prepared to handle NULL return value. + */ +const struct cpuinfo_core* CPUINFO_ABI cpuinfo_get_current_core(void); + +/** + * Identify the microarchitecture index of the core that executes the current thread. + * If the system does not support such identification, the function returns 0. + * + * There is no guarantee that the thread will stay on the same type of core for any time. + * Callers should treat the result as only a hint. + */ +uint32_t CPUINFO_ABI cpuinfo_get_current_uarch_index(void); + +/** + * Identify the microarchitecture index of the core that executes the current thread. + * If the system does not support such identification, the function returns the user-specified default value. + * + * There is no guarantee that the thread will stay on the same type of core for any time. + * Callers should treat the result as only a hint. + */ +uint32_t CPUINFO_ABI cpuinfo_get_current_uarch_index_with_default(uint32_t default_uarch_index); + +#ifdef __cplusplus +} /* extern "C" */ +#endif + +#endif /* CPUINFO_H */ diff --git a/3rdparty/cpuinfo/src/api.c b/3rdparty/cpuinfo/src/api.c new file mode 100644 index 000000000..f91b421cc --- /dev/null +++ b/3rdparty/cpuinfo/src/api.c @@ -0,0 +1,410 @@ +#include +#include + +#include +#include +#include + +#ifdef __linux__ + #include + + #include + #include + #if !defined(__NR_getcpu) + #include + #endif +#endif + +bool cpuinfo_is_initialized = false; + +struct cpuinfo_processor* cpuinfo_processors = NULL; +struct cpuinfo_core* cpuinfo_cores = NULL; +struct cpuinfo_cluster* cpuinfo_clusters = NULL; +struct cpuinfo_package* cpuinfo_packages = NULL; +struct cpuinfo_cache* cpuinfo_cache[cpuinfo_cache_level_max] = { NULL }; + +uint32_t cpuinfo_processors_count = 0; +uint32_t cpuinfo_cores_count = 0; +uint32_t cpuinfo_clusters_count = 0; +uint32_t cpuinfo_packages_count = 0; +uint32_t cpuinfo_cache_count[cpuinfo_cache_level_max] = { 0 }; +uint32_t cpuinfo_max_cache_size = 0; + +#if CPUINFO_ARCH_ARM || CPUINFO_ARCH_ARM64 + struct cpuinfo_uarch_info* cpuinfo_uarchs = NULL; + uint32_t cpuinfo_uarchs_count = 0; +#else + struct cpuinfo_uarch_info cpuinfo_global_uarch = { cpuinfo_uarch_unknown }; +#endif + +#ifdef __linux__ + uint32_t cpuinfo_linux_cpu_max = 0; + const struct cpuinfo_processor** cpuinfo_linux_cpu_to_processor_map = NULL; + const struct cpuinfo_core** cpuinfo_linux_cpu_to_core_map = NULL; + #if CPUINFO_ARCH_ARM || CPUINFO_ARCH_ARM64 + const uint32_t* cpuinfo_linux_cpu_to_uarch_index_map = NULL; + #endif +#endif + + +const struct cpuinfo_processor* cpuinfo_get_processors(void) { + if CPUINFO_UNLIKELY(!cpuinfo_is_initialized) { + cpuinfo_log_fatal("cpuinfo_get_%s called before cpuinfo is initialized", "processors"); + } + return cpuinfo_processors; +} + +const struct cpuinfo_core* cpuinfo_get_cores(void) { + if CPUINFO_UNLIKELY(!cpuinfo_is_initialized) { + cpuinfo_log_fatal("cpuinfo_get_%s called before cpuinfo is initialized", "core"); + } + return cpuinfo_cores; +} + +const struct cpuinfo_cluster* cpuinfo_get_clusters(void) { + if CPUINFO_UNLIKELY(!cpuinfo_is_initialized) { + cpuinfo_log_fatal("cpuinfo_get_%s called before cpuinfo is initialized", "clusters"); + } + return cpuinfo_clusters; +} + +const struct cpuinfo_package* cpuinfo_get_packages(void) { + if CPUINFO_UNLIKELY(!cpuinfo_is_initialized) { + cpuinfo_log_fatal("cpuinfo_get_%s called before cpuinfo is initialized", "packages"); + } + return cpuinfo_packages; +} + +const struct cpuinfo_uarch_info* cpuinfo_get_uarchs() { + if (!cpuinfo_is_initialized) { + cpuinfo_log_fatal("cpuinfo_get_%s called before cpuinfo is initialized", "uarchs"); + } + #if CPUINFO_ARCH_ARM || CPUINFO_ARCH_ARM64 + return cpuinfo_uarchs; + #else + return &cpuinfo_global_uarch; + #endif +} + +const struct cpuinfo_processor* cpuinfo_get_processor(uint32_t index) { + if CPUINFO_UNLIKELY(!cpuinfo_is_initialized) { + cpuinfo_log_fatal("cpuinfo_get_%s called before cpuinfo is initialized", "processor"); + } + if CPUINFO_UNLIKELY(index >= cpuinfo_processors_count) { + return NULL; + } + return &cpuinfo_processors[index]; +} + +const struct cpuinfo_core* cpuinfo_get_core(uint32_t index) { + if CPUINFO_UNLIKELY(!cpuinfo_is_initialized) { + cpuinfo_log_fatal("cpuinfo_get_%s called before cpuinfo is initialized", "core"); + } + if CPUINFO_UNLIKELY(index >= cpuinfo_cores_count) { + return NULL; + } + return &cpuinfo_cores[index]; +} + +const struct cpuinfo_cluster* cpuinfo_get_cluster(uint32_t index) { + if CPUINFO_UNLIKELY(!cpuinfo_is_initialized) { + cpuinfo_log_fatal("cpuinfo_get_%s called before cpuinfo is initialized", "cluster"); + } + if CPUINFO_UNLIKELY(index >= cpuinfo_clusters_count) { + return NULL; + } + return &cpuinfo_clusters[index]; +} + +const struct cpuinfo_package* cpuinfo_get_package(uint32_t index) { + if CPUINFO_UNLIKELY(!cpuinfo_is_initialized) { + cpuinfo_log_fatal("cpuinfo_get_%s called before cpuinfo is initialized", "package"); + } + if CPUINFO_UNLIKELY(index >= cpuinfo_packages_count) { + return NULL; + } + return &cpuinfo_packages[index]; +} + +const struct cpuinfo_uarch_info* cpuinfo_get_uarch(uint32_t index) { + if (!cpuinfo_is_initialized) { + cpuinfo_log_fatal("cpuinfo_get_%s called before cpuinfo is initialized", "uarch"); + } + #if CPUINFO_ARCH_ARM || CPUINFO_ARCH_ARM64 + if CPUINFO_UNLIKELY(index >= cpuinfo_uarchs_count) { + return NULL; + } + return &cpuinfo_uarchs[index]; + #else + if CPUINFO_UNLIKELY(index != 0) { + return NULL; + } + return &cpuinfo_global_uarch; + #endif +} + +uint32_t cpuinfo_get_processors_count(void) { + if CPUINFO_UNLIKELY(!cpuinfo_is_initialized) { + cpuinfo_log_fatal("cpuinfo_get_%s called before cpuinfo is initialized", "processors_count"); + } + return cpuinfo_processors_count; +} + +uint32_t cpuinfo_get_cores_count(void) { + if CPUINFO_UNLIKELY(!cpuinfo_is_initialized) { + cpuinfo_log_fatal("cpuinfo_get_%s called before cpuinfo is initialized", "cores_count"); + } + return cpuinfo_cores_count; +} + +uint32_t cpuinfo_get_clusters_count(void) { + if CPUINFO_UNLIKELY(!cpuinfo_is_initialized) { + cpuinfo_log_fatal("cpuinfo_get_%s called before cpuinfo is initialized", "clusters_count"); + } + return cpuinfo_clusters_count; +} + +uint32_t cpuinfo_get_packages_count(void) { + if CPUINFO_UNLIKELY(!cpuinfo_is_initialized) { + cpuinfo_log_fatal("cpuinfo_get_%s called before cpuinfo is initialized", "packages_count"); + } + return cpuinfo_packages_count; +} + +uint32_t cpuinfo_get_uarchs_count(void) { + if (!cpuinfo_is_initialized) { + cpuinfo_log_fatal("cpuinfo_get_%s called before cpuinfo is initialized", "uarchs_count"); + } + #if CPUINFO_ARCH_ARM || CPUINFO_ARCH_ARM64 + return cpuinfo_uarchs_count; + #else + return 1; + #endif +} + +const struct cpuinfo_cache* CPUINFO_ABI cpuinfo_get_l1i_caches(void) { + if CPUINFO_UNLIKELY(!cpuinfo_is_initialized) { + cpuinfo_log_fatal("cpuinfo_get_%s called before cpuinfo is initialized", "l1i_caches"); + } + return cpuinfo_cache[cpuinfo_cache_level_1i]; +} + +const struct cpuinfo_cache* CPUINFO_ABI cpuinfo_get_l1d_caches(void) { + if CPUINFO_UNLIKELY(!cpuinfo_is_initialized) { + cpuinfo_log_fatal("cpuinfo_get_%s called before cpuinfo is initialized", "l1d_caches"); + } + return cpuinfo_cache[cpuinfo_cache_level_1d]; +} + +const struct cpuinfo_cache* CPUINFO_ABI cpuinfo_get_l2_caches(void) { + if CPUINFO_UNLIKELY(!cpuinfo_is_initialized) { + cpuinfo_log_fatal("cpuinfo_get_%s called before cpuinfo is initialized", "l2_caches"); + } + return cpuinfo_cache[cpuinfo_cache_level_2]; +} + +const struct cpuinfo_cache* CPUINFO_ABI cpuinfo_get_l3_caches(void) { + if CPUINFO_UNLIKELY(!cpuinfo_is_initialized) { + cpuinfo_log_fatal("cpuinfo_get_%s called before cpuinfo is initialized", "l3_caches"); + } + return cpuinfo_cache[cpuinfo_cache_level_3]; +} + +const struct cpuinfo_cache* CPUINFO_ABI cpuinfo_get_l4_caches(void) { + if CPUINFO_UNLIKELY(!cpuinfo_is_initialized) { + cpuinfo_log_fatal("cpuinfo_get_%s called before cpuinfo is initialized", "l4_caches"); + } + return cpuinfo_cache[cpuinfo_cache_level_4]; +} + +const struct cpuinfo_cache* CPUINFO_ABI cpuinfo_get_l1i_cache(uint32_t index) { + if CPUINFO_UNLIKELY(!cpuinfo_is_initialized) { + cpuinfo_log_fatal("cpuinfo_get_%s called before cpuinfo is initialized", "l1i_cache"); + } + if CPUINFO_UNLIKELY(index >= cpuinfo_cache_count[cpuinfo_cache_level_1i]) { + return NULL; + } + return &cpuinfo_cache[cpuinfo_cache_level_1i][index]; +} + +const struct cpuinfo_cache* CPUINFO_ABI cpuinfo_get_l1d_cache(uint32_t index) { + if CPUINFO_UNLIKELY(!cpuinfo_is_initialized) { + cpuinfo_log_fatal("cpuinfo_get_%s called before cpuinfo is initialized", "l1d_cache"); + } + if CPUINFO_UNLIKELY(index >= cpuinfo_cache_count[cpuinfo_cache_level_1d]) { + return NULL; + } + return &cpuinfo_cache[cpuinfo_cache_level_1d][index]; +} + +const struct cpuinfo_cache* CPUINFO_ABI cpuinfo_get_l2_cache(uint32_t index) { + if CPUINFO_UNLIKELY(!cpuinfo_is_initialized) { + cpuinfo_log_fatal("cpuinfo_get_%s called before cpuinfo is initialized", "l2_cache"); + } + if CPUINFO_UNLIKELY(index >= cpuinfo_cache_count[cpuinfo_cache_level_2]) { + return NULL; + } + return &cpuinfo_cache[cpuinfo_cache_level_2][index]; +} + +const struct cpuinfo_cache* CPUINFO_ABI cpuinfo_get_l3_cache(uint32_t index) { + if CPUINFO_UNLIKELY(!cpuinfo_is_initialized) { + cpuinfo_log_fatal("cpuinfo_get_%s called before cpuinfo is initialized", "l3_cache"); + } + if CPUINFO_UNLIKELY(index >= cpuinfo_cache_count[cpuinfo_cache_level_3]) { + return NULL; + } + return &cpuinfo_cache[cpuinfo_cache_level_3][index]; +} + +const struct cpuinfo_cache* CPUINFO_ABI cpuinfo_get_l4_cache(uint32_t index) { + if CPUINFO_UNLIKELY(!cpuinfo_is_initialized) { + cpuinfo_log_fatal("cpuinfo_get_%s called before cpuinfo is initialized", "l4_cache"); + } + if CPUINFO_UNLIKELY(index >= cpuinfo_cache_count[cpuinfo_cache_level_4]) { + return NULL; + } + return &cpuinfo_cache[cpuinfo_cache_level_4][index]; +} + +uint32_t CPUINFO_ABI cpuinfo_get_l1i_caches_count(void) { + if CPUINFO_UNLIKELY(!cpuinfo_is_initialized) { + cpuinfo_log_fatal("cpuinfo_get_%s called before cpuinfo is initialized", "l1i_caches_count"); + } + return cpuinfo_cache_count[cpuinfo_cache_level_1i]; +} + +uint32_t CPUINFO_ABI cpuinfo_get_l1d_caches_count(void) { + if CPUINFO_UNLIKELY(!cpuinfo_is_initialized) { + cpuinfo_log_fatal("cpuinfo_get_%s called before cpuinfo is initialized", "l1d_caches_count"); + } + return cpuinfo_cache_count[cpuinfo_cache_level_1d]; +} + +uint32_t CPUINFO_ABI cpuinfo_get_l2_caches_count(void) { + if CPUINFO_UNLIKELY(!cpuinfo_is_initialized) { + cpuinfo_log_fatal("cpuinfo_get_%s called before cpuinfo is initialized", "l2_caches_count"); + } + return cpuinfo_cache_count[cpuinfo_cache_level_2]; +} + +uint32_t CPUINFO_ABI cpuinfo_get_l3_caches_count(void) { + if CPUINFO_UNLIKELY(!cpuinfo_is_initialized) { + cpuinfo_log_fatal("cpuinfo_get_%s called before cpuinfo is initialized", "l3_caches_count"); + } + return cpuinfo_cache_count[cpuinfo_cache_level_3]; +} + +uint32_t CPUINFO_ABI cpuinfo_get_l4_caches_count(void) { + if CPUINFO_UNLIKELY(!cpuinfo_is_initialized) { + cpuinfo_log_fatal("cpuinfo_get_%s called before cpuinfo is initialized", "l4_caches_count"); + } + return cpuinfo_cache_count[cpuinfo_cache_level_4]; +} + +uint32_t CPUINFO_ABI cpuinfo_get_max_cache_size(void) { + if CPUINFO_UNLIKELY(!cpuinfo_is_initialized) { + cpuinfo_log_fatal("cpuinfo_get_%s called before cpuinfo is initialized", "max_cache_size"); + } + return cpuinfo_max_cache_size; +} + +const struct cpuinfo_processor* CPUINFO_ABI cpuinfo_get_current_processor(void) { + if CPUINFO_UNLIKELY(!cpuinfo_is_initialized) { + cpuinfo_log_fatal("cpuinfo_get_%s called before cpuinfo is initialized", "current_processor"); + } + #ifdef __linux__ + /* Initializing this variable silences a MemorySanitizer error. */ + unsigned cpu = 0; + if CPUINFO_UNLIKELY(syscall(__NR_getcpu, &cpu, NULL, NULL) != 0) { + return 0; + } + if CPUINFO_UNLIKELY((uint32_t) cpu >= cpuinfo_linux_cpu_max) { + return 0; + } + return cpuinfo_linux_cpu_to_processor_map[cpu]; + #else + return NULL; + #endif +} + +const struct cpuinfo_core* CPUINFO_ABI cpuinfo_get_current_core(void) { + if CPUINFO_UNLIKELY(!cpuinfo_is_initialized) { + cpuinfo_log_fatal("cpuinfo_get_%s called before cpuinfo is initialized", "current_core"); + } + #ifdef __linux__ + /* Initializing this variable silences a MemorySanitizer error. */ + unsigned cpu = 0; + if CPUINFO_UNLIKELY(syscall(__NR_getcpu, &cpu, NULL, NULL) != 0) { + return 0; + } + if CPUINFO_UNLIKELY((uint32_t) cpu >= cpuinfo_linux_cpu_max) { + return 0; + } + return cpuinfo_linux_cpu_to_core_map[cpu]; + #else + return NULL; + #endif +} + +uint32_t CPUINFO_ABI cpuinfo_get_current_uarch_index(void) { + if CPUINFO_UNLIKELY(!cpuinfo_is_initialized) { + cpuinfo_log_fatal("cpuinfo_get_%s called before cpuinfo is initialized", "current_uarch_index"); + } + #if CPUINFO_ARCH_ARM || CPUINFO_ARCH_ARM64 + #ifdef __linux__ + if (cpuinfo_linux_cpu_to_uarch_index_map == NULL) { + /* Special case: avoid syscall on systems with only a single type of cores */ + return 0; + } + + /* General case */ + /* Initializing this variable silences a MemorySanitizer error. */ + unsigned cpu = 0; + if CPUINFO_UNLIKELY(syscall(__NR_getcpu, &cpu, NULL, NULL) != 0) { + return 0; + } + if CPUINFO_UNLIKELY((uint32_t) cpu >= cpuinfo_linux_cpu_max) { + return 0; + } + return cpuinfo_linux_cpu_to_uarch_index_map[cpu]; + #else + /* Fallback: pretend to be on the big core. */ + return 0; + #endif + #else + /* Only ARM/ARM64 processors may include cores of different types in the same package. */ + return 0; + #endif +} + +uint32_t CPUINFO_ABI cpuinfo_get_current_uarch_index_with_default(uint32_t default_uarch_index) { + if CPUINFO_UNLIKELY(!cpuinfo_is_initialized) { + cpuinfo_log_fatal("cpuinfo_get_%s called before cpuinfo is initialized", "current_uarch_index_with_default"); + } + #if CPUINFO_ARCH_ARM || CPUINFO_ARCH_ARM64 + #ifdef __linux__ + if (cpuinfo_linux_cpu_to_uarch_index_map == NULL) { + /* Special case: avoid syscall on systems with only a single type of cores */ + return 0; + } + + /* General case */ + /* Initializing this variable silences a MemorySanitizer error. */ + unsigned cpu = 0; + if CPUINFO_UNLIKELY(syscall(__NR_getcpu, &cpu, NULL, NULL) != 0) { + return default_uarch_index; + } + if CPUINFO_UNLIKELY((uint32_t) cpu >= cpuinfo_linux_cpu_max) { + return default_uarch_index; + } + return cpuinfo_linux_cpu_to_uarch_index_map[cpu]; + #else + /* Fallback: no API to query current core, use default uarch index. */ + return default_uarch_index; + #endif + #else + /* Only ARM/ARM64 processors may include cores of different types in the same package. */ + return 0; + #endif +} diff --git a/3rdparty/cpuinfo/src/arm/android/api.h b/3rdparty/cpuinfo/src/arm/android/api.h new file mode 100644 index 000000000..228632ace --- /dev/null +++ b/3rdparty/cpuinfo/src/arm/android/api.h @@ -0,0 +1,20 @@ +#pragma once + +#include +#include +#include +#include + +enum cpuinfo_android_chipset_property { + cpuinfo_android_chipset_property_proc_cpuinfo_hardware = 0, + cpuinfo_android_chipset_property_ro_product_board, + cpuinfo_android_chipset_property_ro_board_platform, + cpuinfo_android_chipset_property_ro_mediatek_platform, + cpuinfo_android_chipset_property_ro_arch, + cpuinfo_android_chipset_property_ro_chipname, + cpuinfo_android_chipset_property_ro_hardware_chipname, + cpuinfo_android_chipset_property_max, +}; + +CPUINFO_INTERNAL void cpuinfo_arm_android_parse_properties( + struct cpuinfo_android_properties properties[restrict static 1]); diff --git a/3rdparty/cpuinfo/src/arm/android/properties.c b/3rdparty/cpuinfo/src/arm/android/properties.c new file mode 100644 index 000000000..5f93889d7 --- /dev/null +++ b/3rdparty/cpuinfo/src/arm/android/properties.c @@ -0,0 +1,67 @@ +#include +#include +#include +#include +#include + +#include + +#include +#include +#include +#include + +#if CPUINFO_MOCK + #include + + static struct cpuinfo_mock_property* cpuinfo_mock_properties = NULL; + + void CPUINFO_ABI cpuinfo_mock_android_properties(struct cpuinfo_mock_property* properties) { + cpuinfo_log_info("Android properties mocking enabled"); + cpuinfo_mock_properties = properties; + } + + static int cpuinfo_android_property_get(const char* key, char* value) { + if (cpuinfo_mock_properties != NULL) { + for (const struct cpuinfo_mock_property* prop = cpuinfo_mock_properties; prop->key != NULL; prop++) { + if (strncmp(key, prop->key, CPUINFO_BUILD_PROP_NAME_MAX) == 0) { + strncpy(value, prop->value, CPUINFO_BUILD_PROP_VALUE_MAX); + return (int) strnlen(prop->value, CPUINFO_BUILD_PROP_VALUE_MAX); + } + } + } + *value = '\0'; + return 0; + } +#else + static inline int cpuinfo_android_property_get(const char* key, char* value) { + return __system_property_get(key, value); + } +#endif + +void cpuinfo_arm_android_parse_properties(struct cpuinfo_android_properties properties[restrict static 1]) { + const int ro_product_board_length = + cpuinfo_android_property_get("ro.product.board", properties->ro_product_board); + cpuinfo_log_debug("read ro.product.board = \"%.*s\"", ro_product_board_length, properties->ro_product_board); + + const int ro_board_platform_length = + cpuinfo_android_property_get("ro.board.platform", properties->ro_board_platform); + cpuinfo_log_debug("read ro.board.platform = \"%.*s\"", ro_board_platform_length, properties->ro_board_platform); + + const int ro_mediatek_platform_length = + cpuinfo_android_property_get("ro.mediatek.platform", properties->ro_mediatek_platform); + cpuinfo_log_debug("read ro.mediatek.platform = \"%.*s\"", + ro_mediatek_platform_length, properties->ro_mediatek_platform); + + const int ro_arch_length = + cpuinfo_android_property_get("ro.arch", properties->ro_arch); + cpuinfo_log_debug("read ro.arch = \"%.*s\"", ro_arch_length, properties->ro_arch); + + const int ro_chipname_length = + cpuinfo_android_property_get("ro.chipname", properties->ro_chipname); + cpuinfo_log_debug("read ro.chipname = \"%.*s\"", ro_chipname_length, properties->ro_chipname); + + const int ro_hardware_chipname_length = + cpuinfo_android_property_get("ro.hardware.chipname", properties->ro_hardware_chipname); + cpuinfo_log_debug("read ro.hardware.chipname = \"%.*s\"", ro_hardware_chipname_length, properties->ro_hardware_chipname); +} diff --git a/3rdparty/cpuinfo/src/arm/api.h b/3rdparty/cpuinfo/src/arm/api.h new file mode 100644 index 000000000..48b99ddc9 --- /dev/null +++ b/3rdparty/cpuinfo/src/arm/api.h @@ -0,0 +1,122 @@ +#pragma once + +#include +#include + +#include +#include + +enum cpuinfo_arm_chipset_vendor { + cpuinfo_arm_chipset_vendor_unknown = 0, + cpuinfo_arm_chipset_vendor_qualcomm, + cpuinfo_arm_chipset_vendor_mediatek, + cpuinfo_arm_chipset_vendor_samsung, + cpuinfo_arm_chipset_vendor_hisilicon, + cpuinfo_arm_chipset_vendor_actions, + cpuinfo_arm_chipset_vendor_allwinner, + cpuinfo_arm_chipset_vendor_amlogic, + cpuinfo_arm_chipset_vendor_broadcom, + cpuinfo_arm_chipset_vendor_lg, + cpuinfo_arm_chipset_vendor_leadcore, + cpuinfo_arm_chipset_vendor_marvell, + cpuinfo_arm_chipset_vendor_mstar, + cpuinfo_arm_chipset_vendor_novathor, + cpuinfo_arm_chipset_vendor_nvidia, + cpuinfo_arm_chipset_vendor_pinecone, + cpuinfo_arm_chipset_vendor_renesas, + cpuinfo_arm_chipset_vendor_rockchip, + cpuinfo_arm_chipset_vendor_spreadtrum, + cpuinfo_arm_chipset_vendor_telechips, + cpuinfo_arm_chipset_vendor_texas_instruments, + cpuinfo_arm_chipset_vendor_wondermedia, + cpuinfo_arm_chipset_vendor_max, +}; + +enum cpuinfo_arm_chipset_series { + cpuinfo_arm_chipset_series_unknown = 0, + cpuinfo_arm_chipset_series_qualcomm_qsd, + cpuinfo_arm_chipset_series_qualcomm_msm, + cpuinfo_arm_chipset_series_qualcomm_apq, + cpuinfo_arm_chipset_series_qualcomm_snapdragon, + cpuinfo_arm_chipset_series_mediatek_mt, + cpuinfo_arm_chipset_series_samsung_exynos, + cpuinfo_arm_chipset_series_hisilicon_k3v, + cpuinfo_arm_chipset_series_hisilicon_hi, + cpuinfo_arm_chipset_series_hisilicon_kirin, + cpuinfo_arm_chipset_series_actions_atm, + cpuinfo_arm_chipset_series_allwinner_a, + cpuinfo_arm_chipset_series_amlogic_aml, + cpuinfo_arm_chipset_series_amlogic_s, + cpuinfo_arm_chipset_series_broadcom_bcm, + cpuinfo_arm_chipset_series_lg_nuclun, + cpuinfo_arm_chipset_series_leadcore_lc, + cpuinfo_arm_chipset_series_marvell_pxa, + cpuinfo_arm_chipset_series_mstar_6a, + cpuinfo_arm_chipset_series_novathor_u, + cpuinfo_arm_chipset_series_nvidia_tegra_t, + cpuinfo_arm_chipset_series_nvidia_tegra_ap, + cpuinfo_arm_chipset_series_nvidia_tegra_sl, + cpuinfo_arm_chipset_series_pinecone_surge_s, + cpuinfo_arm_chipset_series_renesas_mp, + cpuinfo_arm_chipset_series_rockchip_rk, + cpuinfo_arm_chipset_series_spreadtrum_sc, + cpuinfo_arm_chipset_series_telechips_tcc, + cpuinfo_arm_chipset_series_texas_instruments_omap, + cpuinfo_arm_chipset_series_wondermedia_wm, + cpuinfo_arm_chipset_series_max, +}; + +#define CPUINFO_ARM_CHIPSET_SUFFIX_MAX 8 + +struct cpuinfo_arm_chipset { + enum cpuinfo_arm_chipset_vendor vendor; + enum cpuinfo_arm_chipset_series series; + uint32_t model; + char suffix[CPUINFO_ARM_CHIPSET_SUFFIX_MAX]; +}; + +#define CPUINFO_ARM_CHIPSET_NAME_MAX CPUINFO_PACKAGE_NAME_MAX + +#ifndef __cplusplus + CPUINFO_INTERNAL void cpuinfo_arm_chipset_to_string( + const struct cpuinfo_arm_chipset chipset[restrict static 1], + char name[restrict static CPUINFO_ARM_CHIPSET_NAME_MAX]); + + CPUINFO_INTERNAL void cpuinfo_arm_fixup_chipset( + struct cpuinfo_arm_chipset chipset[restrict static 1], uint32_t cores, uint32_t max_cpu_freq_max); + + CPUINFO_INTERNAL void cpuinfo_arm_decode_vendor_uarch( + uint32_t midr, + #if CPUINFO_ARCH_ARM + bool has_vfpv4, + #endif + enum cpuinfo_vendor vendor[restrict static 1], + enum cpuinfo_uarch uarch[restrict static 1]); + + CPUINFO_INTERNAL void cpuinfo_arm_decode_cache( + enum cpuinfo_uarch uarch, + uint32_t cluster_cores, + uint32_t midr, + const struct cpuinfo_arm_chipset chipset[restrict static 1], + uint32_t cluster_id, + uint32_t arch_version, + struct cpuinfo_cache l1i[restrict static 1], + struct cpuinfo_cache l1d[restrict static 1], + struct cpuinfo_cache l2[restrict static 1], + struct cpuinfo_cache l3[restrict static 1]); + + CPUINFO_INTERNAL uint32_t cpuinfo_arm_compute_max_cache_size( + const struct cpuinfo_processor processor[restrict static 1]); +#else /* defined(__cplusplus) */ + CPUINFO_INTERNAL void cpuinfo_arm_decode_cache( + enum cpuinfo_uarch uarch, + uint32_t cluster_cores, + uint32_t midr, + const struct cpuinfo_arm_chipset chipset[1], + uint32_t cluster_id, + uint32_t arch_version, + struct cpuinfo_cache l1i[1], + struct cpuinfo_cache l1d[1], + struct cpuinfo_cache l2[1], + struct cpuinfo_cache l3[1]); +#endif diff --git a/3rdparty/cpuinfo/src/arm/cache.c b/3rdparty/cpuinfo/src/arm/cache.c new file mode 100644 index 000000000..1a6dd3817 --- /dev/null +++ b/3rdparty/cpuinfo/src/arm/cache.c @@ -0,0 +1,1737 @@ +#include + +#include +#include +#include +#include +#include + + +void cpuinfo_arm_decode_cache( + enum cpuinfo_uarch uarch, + uint32_t cluster_cores, + uint32_t midr, + const struct cpuinfo_arm_chipset chipset[restrict static 1], + uint32_t cluster_id, + uint32_t arch_version, + struct cpuinfo_cache l1i[restrict static 1], + struct cpuinfo_cache l1d[restrict static 1], + struct cpuinfo_cache l2[restrict static 1], + struct cpuinfo_cache l3[restrict static 1]) +{ + switch (uarch) { +#if CPUINFO_ARCH_ARM && !defined(__ARM_ARCH_7A__) && !defined(__ARM_ARCH_8A__) + case cpuinfo_uarch_xscale: + switch (midr_get_part(midr) >> 8) { + case 2: + /* + * PXA 210/25X/26X + * + * See "Computer Organization and Design, Revised Printing: The Hardware/Software Interface" + * by David A. Patterson, John L. Hennessy + */ + *l1i = (struct cpuinfo_cache) { + .size = 16 * 1024, + .associativity = 32, + .line_size = 32 + }; + *l1d = (struct cpuinfo_cache) { + .size = 16 * 1024, + .associativity = 4, + .line_size = 64 + }; + break; + case 4: + /* PXA 27X */ + *l1i = (struct cpuinfo_cache) { + .size = 32 * 1024, + .associativity = 32, + .line_size = 32 + }; + *l1d = (struct cpuinfo_cache) { + .size = 32 * 1024, + .associativity = 32, + .line_size = 32 + }; + break; + case 6: + /* + * PXA 3XX + * + * See http://download.intel.com/design/intelxscale/31628302.pdf + */ + *l1i = (struct cpuinfo_cache) { + .size = 32 * 1024, + .associativity = 4, + .line_size = 32 + }; + *l1d = (struct cpuinfo_cache) { + .size = 32 * 1024, + .associativity = 4, + .line_size = 32 + }; + *l2 = (struct cpuinfo_cache) { + .size = 256 * 1024, + .associativity = 8, + .line_size = 32 + }; + break; + } + break; + case cpuinfo_uarch_arm11: + *l1i = (struct cpuinfo_cache) { + .size = 16 * 1024, + .associativity = 4, + .line_size = 32 + }; + *l1d = (struct cpuinfo_cache) { + .size = 16 * 1024, + .associativity = 4, + .line_size = 32 + }; + break; +#endif /* CPUINFO_ARCH_ARM && !defined(__ARM_ARCH_7A__) && !defined(__ARM_ARCH_8A__) */ +#if CPUINFO_ARCH_ARM && !defined(__ARM_ARCH_8A__) + case cpuinfo_uarch_cortex_a5: + /* + * Cortex-A5 Technical Reference Manual: + * 7.1.1. Memory system + * The Cortex-A5 processor has separate instruction and data caches. + * The caches have the following features: + * - Data cache is 4-way set-associative. + * - Instruction cache is 2-way set-associative. + * - The cache line length is eight words. + * - You can configure the instruction and data caches independently during implementation + * to sizes of 4KB, 8KB, 16KB, 32KB, or 64KB. + * 1.1.3. System design components + * PrimeCell Level 2 Cache Controller (PL310) + * The addition of an on-chip secondary cache, also referred to as a Level 2 or L2 cache, is a + * recognized method of improving the performance of ARM-based systems when significant memory traffic + * is generated by the processor. The PrimeCell Level 2 Cache Controller reduces the number of external + * memory accesses and has been optimized for use with the Cortex-A5 processor. + * 8.1.7. Exclusive L2 cache + * The Cortex-A5 processor can be connected to an L2 cache that supports an exclusive cache mode. + * This mode must be activated both in the Cortex-A5 processor and in the L2 cache controller. + * + * +--------------------+-----------+-----------+----------+-----------+ + * | Processor model | L1D cache | L1I cache | L2 cache | Reference | + * +--------------------+-----------+-----------+----------+-----------+ + * | Qualcomm MSM7225A | | | | | + * | Qualcomm MSM7625A | | | | | + * | Qualcomm MSM7227A | | | | | + * | Qualcomm MSM7627A | 32K | 32K | 256K | Wiki [1] | + * | Qualcomm MSM7225AB | | | | | + * | Qualcomm MSM7225AB | | | | | + * | Qualcomm QSD8250 | | | | | + * | Qualcomm QSD8650 | | | | | + * +--------------------+-----------+-----------+----------+-----------+ + * | Spreadtrum SC6821 | 32K | 32K | ? | | + * | Spreadtrum SC6825 | 32K | 32K | 256K | Wiki [2] | + * | Spreadtrum SC8810 | ? | ? | ? | | + * | Spreadtrum SC8825 | 32K | 32K | ? | | + * +--------------------+-----------+-----------+----------+-----------+ + * + * [1] https://en.wikipedia.org/wiki/List_of_Qualcomm_Snapdragon_systems-on-chip#Snapdragon_S1 + * [2] https://en.wikipedia.org/wiki/Spreadtrum + */ + *l1i = (struct cpuinfo_cache) { + .size = 32 * 1024, + .associativity = 2, + .line_size = 32 + }; + *l1d = (struct cpuinfo_cache) { + .size = 32 * 1024, + .associativity = 4, + .line_size = 32 + }; + *l2 = (struct cpuinfo_cache) { + .size = 256 * 1024, + /* + * Follow NXP specification: "Eight-way set-associative 512 kB L2 cache with 32B line size" + * Reference: http://www.nxp.com/assets/documents/data/en/application-notes/AN4947.pdf + */ + .associativity = 8, + .line_size = 32 + }; + break; + case cpuinfo_uarch_cortex_a7: + /* + * Cortex-A7 MPCore Technical Reference Manual: + * 6.1. About the L1 memory system + * The L1 memory system consists of separate instruction and data caches. You can configure the + * instruction and data caches independently during implementation to sizes of 8KB, 16KB, 32KB, or 64KB. + * + * The L1 instruction memory system has the following features: + * - Instruction side cache line length of 32-bytes. + * - 2-way set-associative instruction cache. + * + * The L1 data memory system has the following features: + * - Data side cache line length of 64-bytes. + * - 4-way set-associative data cache. + * + * 7.1. About the L2 Memory system + * The L2 memory system consists of an: + * - Optional tightly-coupled L2 cache that includes: + * - Configurable L2 cache size of 128KB, 256KB, 512KB, and 1MB. + * - Fixed line length of 64 bytes + * - 8-way set-associative cache structure + * + * +--------------------+-------+-----------+-----------+-----------+-----------+ + * | Processor model | Cores | L1D cache | L1I cache | L2 cache | Reference | + * +--------------------+-------+-----------+-----------+-----------+-----------+ + * | Allwinner A20 | 2 | 32K | 32K | 256K | [1] | + * | Allwinner A23 | 2 | 32K | 32K | 256K | [2] | + * | Allwinner A31 | 4 | 32K | 32K | 1M | [3] | + * | Allwinner A31s | 4 | 32K | 32K | 1M | [4] | + * | Allwinner A33 | 4 | 32K | 32K | 512K | [5] | + * | Allwinner A80 Octa | 4(+4) | 32K | 32K | 512K(+2M) | [6] | + * | Allwinner A81T | 8 | 32K | 32K | 1M | [7] | + * +--------------------+-------+-----------+-----------+-----------+-----------+ + * | Broadcom BCM2836 | 4 | 32K | 32K | 512K | [8] | + * +--------------------+-------+-----------+-----------+-----------+-----------+ + * | Kirin 920 | 4(+4) | ? | ? | 512K | [9] | + * +--------------------+-------+-----------+-----------+-----------+-----------+ + * + * [1] https://linux-sunxi.org/A20 + * [2] https://linux-sunxi.org/A23 + * [3] http://dl.linux-sunxi.org/A31/A3x_release_document/A31/IC/A31%20datasheet%20V1.3%2020131106.pdf + * [4] https://github.com/allwinner-zh/documents/blob/master/A31s/A31s_Datasheet_v1.5_20150510.pdf + * [5] http://dl.linux-sunxi.org/A33/A33_Datasheet_release1.0.pdf + * [6] https://linux-sunxi.org/images/1/10/A80_Datasheet_Revision_1.0_0404.pdf + * [7] http://dl.linux-sunxi.org/A83T/A83T_datasheet_Revision_1.1.pdf + * [8] https://www.raspberrypi.org/forums/viewtopic.php?t=98428 + * [9] http://www.gizmochina.com/2014/10/07/hisilicon-kirin-920-tear-down/ + */ + *l1i = (struct cpuinfo_cache) { + .size = 32 * 1024, + .associativity = 2, + .line_size = 32 + }; + *l1d = (struct cpuinfo_cache) { + .size = 32 * 1024, + .associativity = 4, + .line_size = 64 + }; + *l2 = (struct cpuinfo_cache) { + .size = 128 * 1024 * cluster_cores, + .associativity = 8, + .line_size = 64 + }; + break; + case cpuinfo_uarch_cortex_a8: + /* + * Cortex-A8 Technical Reference Manual: + * 7.1. About the L1 memory system + * The L1 memory system consists of separate instruction and data caches in a Harvard arrangement. + * The L1 memory system provides the core with: + * - fixed line length of 64 bytes + * - support for 16KB or 32KB caches + * - 4-way set associative cache structure + * 8.1. About the L2 memory system + * The L2 memory system is tightly coupled to the L1 data cache and L1 instruction cache. + * The key features of the L2 memory system include: + * - configurable cache size of 0KB, 128KB, 256KB, 512KB, and 1MB + * - fixed line length of 64 bytes + * - 8-way set associative cache structure + * + * +----------------------+-----------+-----------+-----------+-----------+ + * | Processor model | L1D cache | L1I cache | L2 cache | Reference | + * +----------------------+-----------+-----------+-----------+-----------+ + * | Exynos 3 Single 3110 | 32K | 32K | 512K | [1] | + * +----------------------+-----------+-----------+-----------+-----------+ + * | TI DM 3730 | 32K | 32K | 256K | [2] | + * +----------------------+-----------+-----------+-----------+-----------+ + * + * [1] https://en.wikichip.org/w/images/0/04/Exynos_3110.pdf + * [2] https://www.ti.com/lit/ds/symlink/dm3725.pdf + */ + *l1i = (struct cpuinfo_cache) { + .size = 32 * 1024, + .associativity = 4, + .line_size = 64 + }; + *l1d = (struct cpuinfo_cache) { + .size = 32 * 1024, + .associativity = 4, + .line_size = 64 + }; + *l2 = (struct cpuinfo_cache) { + .associativity = 8, + .line_size = 64 + }; + switch (chipset->vendor) { + case cpuinfo_arm_chipset_vendor_samsung: + l2->size = 512 * 1024; + break; + default: + l2->size = 256 * 1024; + break; + } + + break; + case cpuinfo_uarch_cortex_a9: + /* + * ARM Cortex‑A9 Technical Reference Manual: + * 7.1.1 Memory system + * The Cortex‑A9 processor has separate instruction and data caches. + * The caches have the following features: + * - Both caches are 4-way set-associative. + * - The cache line length is eight words. + * - You can configure the instruction and data caches independently during implementation + * to sizes of 16KB, 32KB, or 64KB. + * 8.1.5 Exclusive L2 cache + * The Cortex‑A9 processor can be connected to an L2 cache that supports an exclusive cache mode. + * This mode must be activated both in the Cortex‑A9 processor and in the L2 cache controller. + * + * +--------------------+-------+-----------+-----------+-----------+-----------+ + * | Processor model | Cores | L1D cache | L1I cache | L2 cache | Reference | + * +--------------------+-------+-----------+-----------+-----------+-----------+ + * | Exynos 4 Dual 4210 | 2 | 32K | 32K | 1M | [1] | + * | Exynos 4 Dual 4212 | 2 | 32K | 32K | 1M | [2] | + * | Exynos 4 Quad 4412 | 4 | 32K | 32K | 1M | [3] | + * | Exynos 4 Quad 4415 | 4 | 32K | 32K | 1M | | + * | TI OMAP 4430 | 2 | 32K | 32K | 1M | [4] | + * | TI OMAP 4460 | 2 | 32K | 32K | 1M | [5] | + * +--------------------+-------+-----------+-----------+-----------+-----------+ + * + * [1] http://www.samsung.com/global/business/semiconductor/file/product/Exynos_4_Dual_45nm_User_Manaul_Public_REV1.00-0.pdf + * [2] http://www.samsung.com/global/business/semiconductor/file/product/Exynos_4_Dual_32nm_User_Manaul_Public_REV100-0.pdf + * [3] http://www.samsung.com/global/business/semiconductor/file/product/Exynos_4_Quad_User_Manaul_Public_REV1.00-0.pdf + * [4] https://www.hotchips.org/wp-content/uploads/hc_archives/hc21/2_mon/HC21.24.400.ClientProcessors-Epub/HC21.24.421.Witt-OMAP4430.pdf + * [5] http://www.anandtech.com/show/5310/samsung-galaxy-nexus-ice-cream-sandwich-review/9 + */ + + /* Use Exynos 4 specs */ + *l1i = (struct cpuinfo_cache) { + .size = 32 * 1024, + .associativity = 4, + .line_size = 32 + }; + *l1d = (struct cpuinfo_cache) { + .size = 32 * 1024, + .associativity = 4, + .line_size = 32 + }; + *l2 = (struct cpuinfo_cache) { + .size = 1024 * 1024, + /* OMAP4460 in Pandaboard ES has 16-way set-associative L2 cache */ + .associativity = 16, + .line_size = 32 + }; + break; + case cpuinfo_uarch_cortex_a15: + /* + * 6.1. About the L1 memory system + * The L1 memory system consists of separate instruction and data caches. + * The L1 instruction memory system has the following features: + * - 32KB 2-way set-associative instruction cache. + * - Fixed line length of 64 bytes. + * The L1 data memory system has the following features: + * - 32KB 2-way set-associative data cache. + * - Fixed line length of 64 bytes. + * 7.1. About the L2 memory system + * The features of the L2 memory system include: + * - Configurable L2 cache size of 512KB, 1MB, 2MB and 4MB. + * - Fixed line length of 64 bytes. + * - 16-way set-associative cache structure. + * + * +--------------------+-------+-----------+-----------+-----------+-----------+ + * | Processor model | Cores | L1D cache | L1I cache | L2 cache | Reference | + * +--------------------+-------+-----------+-----------+-----------+-----------+ + * | Exynos 5 Dual 5250 | 2 | 32K | 32K | 1M | [1] | + * | Exynos 5 Hexa 5260 | 2(+4) | 32K | 32K | 1M(+512K) | [2] | + * | Exynos 5 Octa 5410 | 4(+4) | 32K | 32K | 2M(+512K) | [3] | + * | Exynos 5 Octa 5420 | 4(+4) | 32K | 32K | 2M(+512K) | [3] | + * | Exynos 5 Octa 5422 | 4(+4) | 32K | 32K | 2M(+512K) | [3] | + * | Exynos 5 Octa 5430 | 4(+4) | 32K | 32K | 2M(+512K) | [3] | + * | Exynos 5 Octa 5800 | 4(+4) | 32K | 32K | 2M(+512K) | [3] | + * | Kirin 920 | 4(+4) | ? | ? | 2M(+512K) | [4] | + * +--------------------+-------+-----------+-----------+-----------+-----------+ + * + * [1] http://www.arndaleboard.org/wiki/downloads/supports/Exynos_5_Dual_User_Manaul_Public_REV1.00.pdf + * [2] http://www.yicsystem.com/wp-content/uploads/2014/08/Espresso5260P-Guide-Book.pdf + * [3] http://www.anandtech.com/show/6768/samsung-details-exynos-5-octa-architecture-power-at-isscc-13 + * [4] http://www.gizmochina.com/2014/10/07/hisilicon-kirin-920-tear-down/ + */ + *l1i = (struct cpuinfo_cache) { + .size = 32 * 1024, + .associativity = 2, + .line_size = 64 + }; + *l1d = (struct cpuinfo_cache) { + .size = 32 * 1024, + .associativity = 2, + .line_size = 64 + }; + *l2 = (struct cpuinfo_cache) { + .size = cluster_cores * 512 * 1024, + .associativity = 16, + .line_size = 64 + }; + break; + case cpuinfo_uarch_cortex_a17: + /* + * ARM Cortex-A17 MPCore Processor Technical Reference Manual: + * 6.1. About the L1 memory system + * The L1 memory system consists of separate instruction and data caches. + * The size of the instruction cache is implemented as either 32KB or 64KB. + * The size of the data cache is 32KB. + * + * The L1 instruction cache has the following features: + * - Instruction side cache line length of 64-bytes. + * - 4-way set-associative instruction cache. + * + * The L1 data cache has the following features: + * - Data side cache line length of 64-bytes. + * - 4-way set-associative data cache. + * + * 7.1. About the L2 Memory system + * An integrated L2 cache: + * - The cache size is implemented as either 256KB, 512KB, 1MB, 2MB, 4MB or 8MB. + * - A fixed line length of 64 bytes. + * - 16-way set-associative cache structure. + * + * +------------------+-------+-----------+-----------+-----------+-----------+ + * | Processor model | Cores | L1D cache | L1I cache | L2 cache | Reference | + * +------------------+-------+-----------+-----------+-----------+-----------+ + * | MediaTek MT6595 | 4(+4) | 32K | 32K | 2M(+512K) | [1] | + * +------------------+-------+-----------+-----------+-----------+-----------+ + * + * [1] https://blog.osakana.net/archives/5268 + */ + *l1i = (struct cpuinfo_cache) { + .size = 32 * 1024, + .associativity = 4, + .line_size = 64 + }; + *l1d = (struct cpuinfo_cache) { + .size = 32 * 1024, + .associativity = 4, + .line_size = 64 + }; + *l2 = (struct cpuinfo_cache) { + .size = cluster_cores * 512 * 1024, + .associativity = 16, + .line_size = 64 + }; + break; +#endif /* CPUINFO_ARCH_ARM && !defined(__ARM_ARCH_8A__) */ + case cpuinfo_uarch_cortex_a35: + /* + * ARM Cortex‑A35 Processor Technical Reference Manual: + * 6.1. About the L1 memory system + * The L1 memory system includes several power-saving and performance-enhancing features. + * These include separate instruction and data caches, which can be configured + * independently during implementation to sizes of 8KB, 16KB, 32KB, or 64KB. + * + * L1 instruction-side memory system + * A dedicated instruction cache that: + * - is virtually indexed and physically tagged. + * - is 2-way set associative. + * - is configurable to be 8KB, 16KB, 32KB, or 64KB. + * - uses a cache line length of 64 bytes. + * + * L1 data-side memory system + * A dedicated data cache that: + * - is physically indexed and physically tagged. + * - is 4-way set associative. + * - is configurable to be 8KB, 16KB, 32KB, or 64KB. + * - uses a cache line length of 64 bytes. + * + * 7.1. About the L2 memory system + * The L2 cache is 8-way set associative. + * Further features of the L2 cache are: + * - Configurable size of 128KB, 256KB, 512KB, and 1MB. + * - Fixed line length of 64 bytes. + * - Physically indexed and tagged. + * + * +-----------------+---------+-----------+-----------+-----------+-----------+ + * | Processor model | Cores | L1D cache | L1I cache | L2 cache | Reference | + * +-----------------+---------+-----------+-----------+-----------+-----------+ + * | MediaTek MT6599 | 4(+4+2) | ? | ? | ? | | + * +-----------------+---------+-----------+-----------+-----------+-----------+ + */ + *l1i = (struct cpuinfo_cache) { + .size = 16 * 1024, /* assumption based on low-end Cortex-A53 */ + .associativity = 2, + .line_size = 64 + }; + *l1d = (struct cpuinfo_cache) { + .size = 16 * 1024, /* assumption based on low-end Cortex-A53 */ + .associativity = 4, + .line_size = 64 + }; + *l2 = (struct cpuinfo_cache) { + .size = 256 * 1024, /* assumption based on low-end Cortex-A53 */ + .associativity = 8, + .line_size = 64 + }; + break; + case cpuinfo_uarch_cortex_a53: + /* + * ARM Cortex-A53 MPCore Processor Technical Reference Manual: + * 6.1. About the L1 memory system + * The L1 memory system consists of separate instruction and data caches. The implementer configures the + * instruction and data caches independently during implementation, to sizes of 8KB, 16KB, 32KB, or 64KB. + * + * The L1 Instruction memory system has the following key features: + * - Instruction side cache line length of 64 bytes. + * - 2-way set associative L1 Instruction cache. + * + * The L1 Data memory system has the following features: + * - Data side cache line length of 64 bytes. + * - 4-way set associative L1 Data cache. + * + * 7.1. About the L2 memory system + * The L2 memory system consists of an: + * - Optional tightly-coupled L2 cache that includes: + * - Configurable L2 cache size of 128KB, 256KB, 512KB, 1MB and 2MB. + * - Fixed line length of 64 bytes. + * - 16-way set-associative cache structure. + * + * +--------------------+-------+-----------+-----------+-----------+-----------+ + * | Processor model | Cores | L1D cache | L1I cache | L2 cache | Reference | + * +--------------------+-------+-----------+-----------+-----------+-----------+ + * | Broadcom BCM2837 | 4 | 16K | 16K | 512K | [1] | + * | Exynos 7420 | 4(+4) | 32K | 32K | 256K | [2, 3] | + * | Exynos 8890 | 4(+4) | 32K | 32K | 256K | [4] | + * | Rochchip RK3368 | 4+4 | 32K | 32K | 512K+256K | sysfs | + * | MediaTek MT8173C | 2(+2) | 32K | 32K | 512K(+1M) | sysfs | + * | Snapdragon 410 | 4 | 32K | 32K | 512K | [3] | + * | Snapdragon 630 | 4+4 | 32K | 32K | 1M+512K | sysfs | + * | Snapdragon 636 | 4(+4) | 32K+64K | 32K+64K | 1M+1M | sysfs | + * | Snapdragon 660 | 4(+4) | 32K+64K | 32K+64K | 1M+1M | sysfs | + * | Snapdragon 835 | 4(+4) | 32K+64K | 32K+64K | 1M(+2M) | sysfs | + * | Kirin 620 | 4+4 | 32K | 32K | 512K | [5] | + * +--------------------+-------+-----------+-----------+-----------+-----------+ + * + * [1] https://www.raspberrypi.org/forums/viewtopic.php?f=91&t=145766 + * [2] http://www.anandtech.com/show/9330/exynos-7420-deep-dive/2 + * [3] https://www.usenix.org/system/files/conference/usenixsecurity16/sec16_paper_lipp.pdf + * [4] http://www.boardset.com/products/products_v8890.php + * [5] http://mirror.lemaker.org/Hi6220V100_Multi-Mode_Application_Processor_Function_Description.pdf + */ + if (midr_is_qualcomm_cortex_a53_silver(midr)) { + /* Qualcomm-modified Cortex-A53 in Snapdragon 630/660/835 */ + + uint32_t l2_size = 512 * 1024; + switch (chipset->series) { + case cpuinfo_arm_chipset_series_qualcomm_msm: + if (chipset->model == 8998) { + /* Snapdragon 835 (MSM8998): 1 MB L2 (little cores only) */ + l2_size = 1024 * 1024; + } + break; + case cpuinfo_arm_chipset_series_qualcomm_snapdragon: + switch (chipset->model) { + case 630: + if (cluster_id == 0) { + /* Snapdragon 630: 1 MB L2 for the big cores */ + l2_size = 1024 * 1024; + } + break; + case 636: + /* Snapdragon 636: 1 MB L2 (little cores only) */ + l2_size = 1024 * 1024; + break; + case 660: + case 662: + /* Snapdragon 660: 1 MB L2 (little cores only) */ + l2_size = 1024 * 1024; + break; + } + break; + default: + break; + } + + *l1i = (struct cpuinfo_cache) { + .size = 32 * 1024, + .associativity = 2, + .line_size = 64 + }; + *l1d = (struct cpuinfo_cache) { + .size = 32 * 1024, + .associativity = 4, + .line_size = 64 + }; + *l2 = (struct cpuinfo_cache) { + .size = l2_size, + .associativity = 16, + .line_size = 64 + }; + } else { + /* Standard Cortex-A53 */ + + /* Use conservative values by default */ + uint32_t l1_size = 16 * 1024; + uint32_t l2_size = 256 * 1024; + switch (chipset->series) { + case cpuinfo_arm_chipset_series_qualcomm_msm: + l1_size = 32 * 1024; + l2_size = 512 * 1024; + switch (chipset->model) { + case 8937: /* Snapdragon 430 */ + case 8940: /* Snapdragon 435 */ + case 8953: /* Snapdragon 625 or 626 (8953PRO) */ + if (cluster_id == 0) { + /* 1M L2 for big cluster */ + l2_size = 1024 * 1024; + } + break; + case 8952: /* Snapdragon 617 */ + if (cluster_id != 0) { + /* 256K L2 for LITTLE cluster */ + l2_size = 256 * 1024; + } + break; + default: + /* Silence compiler warning about unhandled enum values */ + break; + } + break; + case cpuinfo_arm_chipset_series_qualcomm_apq: + l1_size = 32 * 1024; + l2_size = 512 * 1024; + break; + case cpuinfo_arm_chipset_series_qualcomm_snapdragon: + l1_size = 32 * 1024; + l2_size = 512 * 1024; + if (chipset->model == 450 && cluster_id == 0) { + /* Snapdragon 450: 1M L2 for big cluster */ + l2_size = 1024 * 1024; + } + break; + case cpuinfo_arm_chipset_series_hisilicon_hi: + l1_size = 32 * 1024; + l2_size = 512 * 1024; + break; + case cpuinfo_arm_chipset_series_hisilicon_kirin: + l1_size = 32 * 1024; + switch (chipset->model) { + case 970: /* Kirin 970 */ + l2_size = 1024 * 1024; + break; + default: + l2_size = 512 * 1024; + break; + } + break; + case cpuinfo_arm_chipset_series_mediatek_mt: + switch (chipset->model) { + case 8173: + l1_size = 32 * 1024; + l2_size = 512 * 1024; + break; + } + break; + case cpuinfo_arm_chipset_series_rockchip_rk: + l1_size = 32 * 1024; + switch (chipset->model) { + case 3368: + if (cluster_id == 0) { + /* RK3368: 512 KB L2 for the big cores */ + l2_size = 512 * 1024; + } + break; + } + break; + case cpuinfo_arm_chipset_series_broadcom_bcm: + switch (chipset->model) { + case 2837: /* BCM2837 */ + l2_size = 512 * 1024; + break; + } + break; + case cpuinfo_arm_chipset_series_samsung_exynos: + l1_size = 32 * 1024; + break; + default: + /* Silence compiler warning about unhandled enum values */ + break; + } + *l1i = (struct cpuinfo_cache) { + .size = l1_size, + .associativity = 2, + .line_size = 64 + }; + *l1d = (struct cpuinfo_cache) { + .size = l1_size, + .associativity = 4, + .line_size = 64 + }; + *l2 = (struct cpuinfo_cache) { + .size = l2_size, + .associativity = 16, + .line_size = 64 + }; + } + break; + case cpuinfo_uarch_cortex_a55r0: + case cpuinfo_uarch_cortex_a55: + /* + * ARM Cortex-A55 Core Technical Reference Manual + * A6.1. About the L1 memory system + * The Cortex®-A55 core's L1 memory system enhances core performance and power efficiency. + * It consists of separate instruction and data caches. You can configure instruction and data caches + * independently during implementation to sizes of 16KB, 32KB, or 64KB. + * + * L1 instruction-side memory system + * The L1 instruction-side memory system provides an instruction stream to the DPU. Its key features are: + * - 64-byte instruction side cache line length. + * - 4-way set associative L1 instruction cache. + * + * L1 data-side memory system + * - 64-byte data side cache line length. + * - 4-way set associative L1 data cache. + * + * A7.1 About the L2 memory system + * The Cortex-A55 L2 memory system is required to interface the Cortex-A55 cores to the L3 memory system. + * The L2 memory subsystem consists of: + * - An optional 4-way, set-associative L2 cache with a configurable size of 64KB, 128KB or 256KB. Cache + * lines have a fixed length of 64 bytes. + * + * The main features of the L2 memory system are: + * - Strictly exclusive with L1 data cache. + * - Pseudo-inclusive with L1 instruction cache. + * - Private per-core unified L2 cache. + * + * +--------------------+-------+-----------+-----------+-----------+----------+------------+ + * | Processor model | Cores | L1D cache | L1I cache | L2 cache | L3 cache | Reference | + * +--------------------+-------+-----------+-----------+-----------+----------+------------+ + * | Snapdragon 845 | 4(+4) | 32K | 32K | 128K | 2M | [1], sysfs | + * | Exynos 9810 | 4(+4) | ? | ? | None | 512K | [2] | + * | Kirin 980 | 4(+4) | 32K | 32K | 128K | 4M | [3] | + * +--------------------+-------+-----------+-----------+-----------+----------+------------+ + * + * [1] https://www.anandtech.com/show/12114/qualcomm-announces-snapdragon-845-soc + * [2] https://www.anandtech.com/show/12478/exynos-9810-handson-awkward-first-results + * [3] https://en.wikichip.org/wiki/hisilicon/kirin/980 + */ + if (midr_is_qualcomm_cortex_a55_silver(midr)) { + /* Qualcomm-modified Cortex-A55 in Snapdragon 670 / 710 / 845 */ + uint32_t l3_size = 1024 * 1024; + switch (chipset->series) { + case cpuinfo_arm_chipset_series_qualcomm_snapdragon: + /* Snapdragon 845: 2M L3 cache */ + if (chipset->model == 845) { + l3_size = 2 * 1024 * 1024; + } + break; + default: + break; + } + + *l1i = (struct cpuinfo_cache) { + .size = 32 * 1024, + .associativity = 4, + .line_size = 64, + }; + *l1d = (struct cpuinfo_cache) { + .size = 32 * 1024, + .associativity = 4, + .line_size = 64, + }; + *l2 = (struct cpuinfo_cache) { + .size = 128 * 1024, + .associativity = 4, + .line_size = 64, + }; + *l3 = (struct cpuinfo_cache) { + .size = l3_size, + .associativity = 16, + .line_size = 64, + }; + } else { + /* Standard Cortex-A55 */ + + *l1i = (struct cpuinfo_cache) { + .size = 32 * 1024, + .associativity = 4, + .line_size = 64, + }; + *l1d = (struct cpuinfo_cache) { + .size = 32 * 1024, + .associativity = 4, + .line_size = 64, + }; + if (chipset->series == cpuinfo_arm_chipset_series_samsung_exynos) { + *l2 = (struct cpuinfo_cache) { + .size = 512 * 1024, + /* DynamIQ */ + .associativity = 16, + .line_size = 64, + }; + } else { + uint32_t l3_size = 1024 * 1024; + switch (chipset->series) { + case cpuinfo_arm_chipset_series_hisilicon_kirin: + /* Kirin 980: 4M L3 cache */ + if (chipset->model == 980) { + l3_size = 4 * 1024 * 1024; + } + break; + default: + break; + } + *l2 = (struct cpuinfo_cache) { + .size = 128 * 1024, + .associativity = 4, + .line_size = 64, + }; + *l3 = (struct cpuinfo_cache) { + .size = l3_size, + /* DynamIQ */ + .associativity = 16, + .line_size = 64, + }; + } + } + break; + case cpuinfo_uarch_cortex_a57: + /* + * ARM Cortex-A57 MPCore Processor Technical Reference Manual: + * 6.1. About the L1 memory system + * The L1 memory system consists of separate instruction and data caches. + * + * The L1 instruction memory system has the following features: + * - 48KB 3-way set-associative instruction cache. + * - Fixed line length of 64 bytes. + * + * The L1 data memory system has the following features: + * - 32KB 2-way set-associative data cache. + * - Fixed line length of 64 bytes. + * + * 7.1 About the L2 memory system + * The features of the L2 memory system include: + * - Configurable L2 cache size of 512KB, 1MB, and 2MB. + * - Fixed line length of 64 bytes. + * - 16-way set-associative cache structure. + * - Inclusion property with L1 data caches. + * + * +--------------------+-------+-----------+-----------+-----------+-----------+ + * | Processor model | Cores | L1D cache | L1I cache | L2 cache | Reference | + * +--------------------+-------+-----------+-----------+-----------+-----------+ + * | Snapdragon 810 | 4(+4) | 32K | 48K | 2M | [1] | + * | Exynos 7420 | 4(+4) | 32K | 48K | 2M | [2] | + * | Jetson TX1 | 4 | 32K | 48K | 2M | [3] | + * +--------------------+-------+-----------+-----------+-----------+-----------+ + * + * [1] http://www.anandtech.com/show/9837/snapdragon-820-preview + * [2] http://www.anandtech.com/show/9330/exynos-7420-deep-dive/2 + * [3] https://devblogs.nvidia.com/parallelforall/jetson-tx2-delivers-twice-intelligence-edge/ + */ + *l1i = (struct cpuinfo_cache) { + .size = 48 * 1024, + .associativity = 3, + .line_size = 64 + }; + *l1d = (struct cpuinfo_cache) { + .size = 32 * 1024, + .associativity = 2, + .line_size = 64 + }; + *l2 = (struct cpuinfo_cache) { + .size = cluster_cores * 512 * 1024, + .associativity = 16, + .line_size = 64, + .flags = CPUINFO_CACHE_INCLUSIVE + }; + break; + case cpuinfo_uarch_cortex_a65: + { + /* + * ARM Cortex‑A65 Core Technical Reference Manual + * A6.1. About the L1 memory system + * The L1 memory system enhances the performance and power efficiency in the Cortex‑A65 core. + * It consists of separate instruction and data caches. You can configure instruction and data caches + * independently during implementation to sizes of 32KB or 64KB. + * + * L1 instruction-side memory system + * The L1 instruction-side memory system provides an instruction stream to the DPU. Its key features are: + * - 64-byte instruction side cache line length. + * - 4-way set associative L1 instruction cache. + * + * L1 data-side memory system + * - 64-byte data side cache line length. + * - 4-way set associative L1 data cache. + * + * A7.1 About the L2 memory system + * The Cortex‑A65 L2 memory system is required to interface the Cortex‑A65 cores to the L3 memory system. + * The L2 memory subsystem consists of: + * - An optional 4-way, set-associative L2 cache with a configurable size of 64KB, 128KB, or 256KB. + * Cache lines have a fixed length of 64 bytes. + * + * The main features of the L2 memory system are: + * - Strictly exclusive with L1 data cache. + * - Pseudo-inclusive with L1 instruction cache. + * - Private per-core unified L2 cache. + */ + const uint32_t l1_size = 32 * 1024; + const uint32_t l2_size = 128 * 1024; + const uint32_t l3_size = 512 * 1024; + *l1i = (struct cpuinfo_cache) { + .size = l1_size, + .associativity = 4, + .line_size = 64, + }; + *l1d = (struct cpuinfo_cache) { + .size = l1_size, + .associativity = 4, + .line_size = 64, + }; + *l2 = (struct cpuinfo_cache) { + .size = l2_size, + .associativity = 4, + .line_size = 64, + .flags = CPUINFO_CACHE_INCLUSIVE + }; + *l3 = (struct cpuinfo_cache) { + .size = l3_size, + /* DynamIQ */ + .associativity = 16, + .line_size = 64, + }; + break; + } + case cpuinfo_uarch_cortex_a72: + { + /* + * ARM Cortex-A72 MPCore Processor Technical Reference Manual + * 6.1. About the L1 memory system + * The L1 memory system consists of separate instruction and data caches. + * + * The L1 instruction memory system has the following features: + * - 48KB 3-way set-associative instruction cache. + * - Fixed line length of 64 bytes. + * + * The L1 data memory system has the following features: + * - 32KB 2-way set-associative data cache. + * - Fixed cache line length of 64 bytes. + * + * 7.1 About the L2 memory system + * The features of the L2 memory system include: + * - Configurable L2 cache size of 512KB, 1MB, 2MB and 4MB. + * - Fixed line length of 64 bytes. + * - Banked pipeline structures. + * - Inclusion property with L1 data caches. + * - 16-way set-associative cache structure. + * + * +---------------------+---------+-----------+-----------+------------+-----------+ + * | Processor model | Cores | L1D cache | L1I cache | L2 cache | Reference | + * +---------------------+---------+-----------+-----------+------------+-----------+ + * | Snapdragon 650 | 2(+4) | 32K(+32K) | 48K(+32K) | 1M(+512K) | [1] | + * | Snapdragon 652 | 4(+4) | 32K(+32K) | 48K(+32K) | 1M(+512K) | [2] | + * | Snapdragon 653 | 4(+4) | 32K(+32K) | 48K(+32K) | 1M(+512K) | [3] | + * | HiSilicon Kirin 950 | 4(+4) | 32K+32K | 48K+32K | ? | | + * | HiSilicon Kirin 955 | 4(+4) | 32K+32K | 48K+32K | ? | | + * | MediaTek MT8173C | 2(+2) | 32K(+32K) | 48K(+32K) | 1M(+512K) | sysfs | + * | MediaTek Helio X20 | 2(+4+4) | ? | ? | ? | | + * | MediaTek Helio X23 | 2(+4+4) | ? | ? | ? | | + * | MediaTek Helio X25 | 2(+4+4) | ? | ? | ? | | + * | MediaTek Helio X27 | 2(+4+4) | ? | ? | ? | | + * | Broadcom BCM2711 | 4 | 32K | 48K | 1M | [4] | + * +---------------------+---------+-----------+-----------+------------+-----------+ + * + * [1] http://pdadb.net/index.php?m=processor&id=578&c=qualcomm_snapdragon_618_msm8956__snapdragon_650 + * [2] http://pdadb.net/index.php?m=processor&id=667&c=qualcomm_snapdragon_620_apq8076__snapdragon_652 + * [3] http://pdadb.net/index.php?m=processor&id=692&c=qualcomm_snapdragon_653_msm8976sg__msm8976_pro + * [4] https://www.raspberrypi.org/documentation/hardware/raspberrypi/bcm2711/README.md + */ + uint32_t l2_size; + switch (chipset->series) { + case cpuinfo_arm_chipset_series_hisilicon_kirin: + l2_size = 2 * 1024 * 1024; + break; + default: + l2_size = 1024 * 1024; + break; + } + + *l1i = (struct cpuinfo_cache) { + .size = 48 * 1024, + .associativity = 3, + .line_size = 64 + }; + *l1d = (struct cpuinfo_cache) { + .size = 32 * 1024, + .associativity = 2, + .line_size = 64 + }; + *l2 = (struct cpuinfo_cache) { + .size = l2_size, + .associativity = 16, + .line_size = 64, + .flags = CPUINFO_CACHE_INCLUSIVE + }; + break; + } + case cpuinfo_uarch_cortex_a73: + { + /* + * ARM Cortex‑A73 MPCore Processor Technical Reference Manual + * 6.1. About the L1 memory system + * The L1 memory system consists of separate instruction and data caches. + * The size of the instruction cache is 64KB. + * The size of the data cache is configurable to either 32KB or 64KB. + * + * The L1 instruction memory system has the following key features: + * - Virtually Indexed, Physically Tagged (VIPT), four-way set-associative instruction cache. + * - Fixed cache line length of 64 bytes. + * + * The L1 data memory system has the following features: + * - ...the data cache behaves like an eight-way set associative PIPT cache (for 32KB configurations) + * and a 16-way set associative PIPT cache (for 64KB configurations). + * - Fixed cache line length of 64 bytes. + * + * 7.1 About the L2 memory system + * The L2 memory system consists of: + * - A tightly-integrated L2 cache with: + * - A configurable size of 256KB, 512KB, 1MB, 2MB, 4MB, or 8MB. + * - A 16-way, set-associative structure. + * - A fixed line length of 64 bytes. + * + * The ARM Cortex A73 - Artemis Unveiled [1] + * "ARM still envisions that most vendors will choose to use configurations of 1 to + * 2MB in consumer products. The L2 cache is inclusive of the L1 cache. " + * + * +---------------------+---------+-----------+-----------+-----------+-----------+ + * | Processor model | Cores | L1D cache | L1I cache | L2 cache | Reference | + * +---------------------+---------+-----------+-----------+-----------+-----------+ + * | HiSilicon Kirin 960 | 4(+4) | 64K+32K | 64K+32K | ? | [2] | + * | MediaTek Helio X30 | 2(+4+4) | ? | 64K+ ? | ? | | + * | Snapdragon 636 | 4(+4) | 64K(+32K) | 64K(+32K) | 1M(+1M) | sysfs | + * | Snapdragon 660 | 4(+4) | 64K+32K | 64K+32K | 1M(+1M) | [3] | + * | Snapdragon 835 | 4(+4) | 64K+32K | 64K+32K | 2M(+1M) | sysfs | + * +---------------------+---------+-----------+-----------+-----------+-----------+ + * + * [1] http://www.anandtech.com/show/10347/arm-cortex-a73-artemis-unveiled/2 + * [2] http://www.anandtech.com/show/11088/hisilicon-kirin-960-performance-and-power/3 + * [3] https://arstechnica.com/gadgets/2017/05/qualcomms-snapdragon-660-and-630-bring-more-high-end-features-to-midrange-chips/ + */ + uint32_t l1d_size = 32 * 1024; + uint32_t l2_size = 512 * 1024; + switch (chipset->series) { + case cpuinfo_arm_chipset_series_hisilicon_kirin: + l1d_size = 64 * 1024; + l2_size = 2 * 1024 * 1024; + break; + case cpuinfo_arm_chipset_series_mediatek_mt: + l1d_size = 64 * 1024; + l2_size = 1 * 1024 * 1024; /* TODO: verify assumption */ + break; + default: + switch (midr) { + case UINT32_C(0x51AF8001): /* Kryo 280 Gold */ + l1d_size = 64 * 1024; + l2_size = 2 * 1024 * 1024; + break; + case UINT32_C(0x51AF8002): /* Kryo 260 Gold */ + l1d_size = 64 * 1024; + l2_size = 1 * 1024 * 1024; + break; + } + } + + *l1i = (struct cpuinfo_cache) { + .size = 64 * 1024, + .associativity = 4, + .line_size = 64 + }; + *l1d = (struct cpuinfo_cache) { + .size = l1d_size, + .associativity = (l1d_size >> 12), + .line_size = 64 + }; + *l2 = (struct cpuinfo_cache) { + .size = l2_size, + .associativity = 16, + .line_size = 64, + .flags = CPUINFO_CACHE_INCLUSIVE + }; + break; + } + case cpuinfo_uarch_cortex_a75: + { + /* + * ARM Cortex-A75 Core Technical Reference Manual + * A6.1. About the L1 memory system + * The L1 memory system consists of separate instruction and data caches. Both have a fixed size of 64KB. + * + * A6.1.1 L1 instruction-side memory system + * The L1 instruction memory system has the following key features: + * - Virtually Indexed, Physically Tagged (VIPT), four-way set-associative instruction cache. + * - Fixed cache line length of 64 bytes. + * + * A6.1.2 L1 data-side memory system + * The L1 data memory system has the following features: + * - Physically Indexed, Physically Tagged (PIPT), 16-way set-associative L1 data cache. + * - Fixed cache line length of 64 bytes. + * - Pseudo-random cache replacement policy. + * + * A7.1 About the L2 memory system + * The L2 memory subsystem consist of: + * - An 8-way set associative L2 cache with a configurable size of 256KB or 512KB. + * Cache lines have a fixed length of 64 bytes. + * + * +--------------------+-------+-----------+-----------+-----------+----------+------------+ + * | Processor model | Cores | L1D cache | L1I cache | L2 cache | L3 cache | Reference | + * +--------------------+-------+-----------+-----------+-----------+----------+------------+ + * | Snapdragon 845 | 4(+4) | 64K | 64K | 256K | 2M | [1], sysfs | + * +--------------------+-------+-----------+-----------+-----------+----------+------------+ + * + * [1] https://www.anandtech.com/show/12114/qualcomm-announces-snapdragon-845-soc + */ + uint32_t l3_size = 1024 * 1024; + switch (chipset->series) { + case cpuinfo_arm_chipset_series_qualcomm_snapdragon: + /* Snapdragon 845: 2M L3 cache */ + if (chipset->model == 845) { + l3_size = 2 * 1024 * 1024; + } + break; + default: + break; + } + *l1i = (struct cpuinfo_cache) { + .size = 64 * 1024, + .associativity = 4, + .line_size = 64 + }; + *l1d = (struct cpuinfo_cache) { + .size = 64 * 1024, + .associativity = 16, + .line_size = 64 + }; + *l2 = (struct cpuinfo_cache) { + .size = 256 * 1024, + .associativity = 8, + .line_size = 64 + }; + *l3 = (struct cpuinfo_cache) { + .size = l3_size, + .associativity = 16, + .line_size = 64 + }; + break; + } + case cpuinfo_uarch_cortex_a76: + { + /* + * ARM Cortex-A76 Core Technical Reference Manual + * A6.1. About the L1 memory system + * The L1 memory system consists of separate instruction and data caches. Both have a fixed size of 64KB. + * + * A6.1.1 L1 instruction-side memory system + * The L1 instruction memory system has the following key features: + * - Virtually Indexed, Physically Tagged (VIPT), which behaves as a Physically Indexed, + * Physically Tagged (PIPT) 4-way set-associative L1 data cache. + * - Fixed cache line length of 64 bytes. + * + * A6.1.2 L1 data-side memory system + * The L1 data memory system has the following features: + * - Virtually Indexed, Physically Tagged (VIPT), which behaves as a Physically Indexed, + * Physically Tagged (PIPT) 4-way set-associative L1 data cache. + * - Fixed cache line length of 64 bytes. + * - Pseudo-LRU cache replacement policy. + * + * A7.1 About the L2 memory system + * The L2 memory subsystem consist of: + * - An 8-way set associative L2 cache with a configurable size of 128KB, 256KB or 512KB. + * Cache lines have a fixed length of 64 bytes. + * - Strictly inclusive with L1 data cache. Weakly inclusive with L1 instruction cache. + * - Dynamic biased replacement policy. + * - Modified Exclusive Shared Invalid (MESI) coherency. + * + * +--------------------+-------+-----------+-----------+-----------+----------+------------+ + * | Processor model | Cores | L1D cache | L1I cache | L2 cache | L3 cache | Reference | + * +--------------------+-------+-----------+-----------+-----------+----------+------------+ + * | Kirin 980 | 4(+4) | 64K | 64K | 512K | 4M | [1], [2] | + * +--------------------+-------+-----------+-----------+-----------+----------+------------+ + * + * [1] https://www.anandtech.com/show/13298/hisilicon-announces-the-kirin-980-first-a76-g76-on-7nm + * [2] https://en.wikichip.org/wiki/hisilicon/kirin/980 + */ + uint32_t l2_size = 256 * 1024; + uint32_t l3_size = 1024 * 1024; + switch (chipset->series) { + case cpuinfo_arm_chipset_series_hisilicon_kirin: + /* Kirin 980: 512K L2 cache + 4M L3 cache */ + if (chipset->model == 980) { + l2_size = 512 * 1024; + l3_size = 4 * 1024 * 1024; + } + break; + default: + break; + } + *l1i = (struct cpuinfo_cache) { + .size = 64 * 1024, + .associativity = 4, + .line_size = 64, + }; + *l1d = (struct cpuinfo_cache) { + .size = 64 * 1024, + .associativity = 4, + .line_size = 64, + }; + *l2 = (struct cpuinfo_cache) { + .size = l2_size, + .associativity = 8, + .line_size = 64, + .flags = CPUINFO_CACHE_INCLUSIVE, + }; + *l3 = (struct cpuinfo_cache) { + .size = l3_size, + .associativity = 16, + .line_size = 64, + }; + break; + } + case cpuinfo_uarch_cortex_a77: + { + /* + * ARM Cortex-A77 Core Technical Reference Manual + * A6.1. About the L1 memory system + * The L1 memory system consists of separate instruction and data caches. Both have a fixed size of 64KB. + * + * A6.1.1 L1 instruction-side memory system + * The L1 instruction memory system has the following key features: + * - Virtually Indexed, Physically Tagged (VIPT), which behaves as a Physically Indexed, + * Physically Tagged (PIPT) 4-way set-associative L1 data cache. + * - Fixed cache line length of 64 bytes. + * + * A6.1.2 L1 data-side memory system + * The L1 data memory system has the following features: + * - Virtually Indexed, Physically Tagged (VIPT), which behaves as a Physically Indexed, + * Physically Tagged (PIPT) 4-way set-associative L1 data cache. + * - Fixed cache line length of 64 bytes. + * - Pseudo-LRU cache replacement policy. + * + * A7.1 About the L2 memory system + * The L2 memory subsystem consist of: + * - An 8-way set associative L2 cache with a configurable size of 128KB, 256KB or 512KB. Cache lines + * have a fixed length of 64 bytes. + * - Strictly inclusive with L1 data cache. Weakly inclusive with L1 instruction cache. + */ + const uint32_t l2_size = 256 * 1024; + const uint32_t l3_size = 1024 * 1024; + *l1i = (struct cpuinfo_cache) { + .size = 64 * 1024, + .associativity = 4, + .line_size = 64, + }; + *l1d = (struct cpuinfo_cache) { + .size = 64 * 1024, + .associativity = 4, + .line_size = 64, + }; + *l2 = (struct cpuinfo_cache) { + .size = l2_size, + .associativity = 8, + .line_size = 64, + .flags = CPUINFO_CACHE_INCLUSIVE, + }; + *l3 = (struct cpuinfo_cache) { + .size = l3_size, + .associativity = 16, + .line_size = 64, + }; + break; + } + case cpuinfo_uarch_neoverse_n1: + case cpuinfo_uarch_neoverse_v1: + case cpuinfo_uarch_neoverse_n2: + { + /* + * ARM Neoverse-n1 Core Technical Reference Manual + * A6.1. About the L1 memory system + * The L1 memory system consists of separate instruction and data caches. Both have a fixed size of 64KB. + * + * A6.1.1 L1 instruction-side memory system + * The L1 instruction memory system has the following key features: + * - Virtually Indexed, Physically Tagged (VIPT), which behaves as a Physically Indexed, + * Physically Tagged (PIPT) 4-way set-associative L1 data cache. + * - Fixed cache line length of 64 bytes. + * + * A6.1.2 L1 data-side memory system + * The L1 data memory system has the following features: + * - Virtually Indexed, Physically Tagged (VIPT), which behaves as a Physically Indexed, + * Physically Tagged (PIPT) 4-way set-associative L1 data cache. + * - Fixed cache line length of 64 bytes. + * - Pseudo-LRU cache replacement policy. + * + * A7.1 About the L2 memory system + * The L2 memory subsystem consist of: + * - An 8-way set associative L2 cache with a configurable size of 256KB, 512KB, or 1024KB. Cache lines + * have a fixed length of 64 bytes. + * - Strictly inclusive with L1 data cache. + * - When configured with instruction cache hardware coherency, strictly inclusive with L1 instruction cache. + * - When configured without instruction cache hardware coherency, weakly inclusive with L1 instruction cache. + */ + + const uint32_t min_l2_size_KB= 256; + const uint32_t min_l3_size_KB = 0; + + *l1i = (struct cpuinfo_cache) { + .size = 64 * 1024, + .associativity = 4, + .line_size = 64, + }; + *l1d = (struct cpuinfo_cache) { + .size = 64 * 1024, + .associativity = 4, + .line_size = 64, + }; + *l2 = (struct cpuinfo_cache) { + .size = min_l2_size_KB * 1024, + .associativity = 8, + .line_size = 64, + .flags = CPUINFO_CACHE_INCLUSIVE, + }; + *l3 = (struct cpuinfo_cache) { + .size = min_l3_size_KB * 1024, + .associativity = 16, + .line_size = 64, + }; + break; + } +#if CPUINFO_ARCH_ARM && !defined(__ARM_ARCH_8A__) + case cpuinfo_uarch_scorpion: + /* + * - "The CPU includes 32KB instruction and data caches as + * well as a complete memory-management unit (MMU) suitable + * for high-level operating systems. The CPU also has + * 256KB of SRAM that can be allocated in 64KB increments + * to level-two (L2) cache or tightly coupled memory (TCM)." [1] + * We interpret it as L2 cache being 4-way set-associative on single-core Scorpion. + * - L1 Data Cache = 32 KB. 32 B/line. [2] + * - L2 Cache = 256 KB. 128 B/line. [2] + * - 256 KB (single-core) or 512 KB (dual-core) L2 cache [3] + * - Single or dual-core configuration [3] + * - For L1 cache assume the same associativity as Krait + * + * [1] https://www.qualcomm.com/media/documents/files/linley-report-on-dual-core-snapdragon.pdf + * [2] http://www.7-cpu.com/cpu/Snapdragon.html + * [3] https://en.wikipedia.org/wiki/Scorpion_(CPU) + */ + *l1i = (struct cpuinfo_cache) { + .size = 32 * 1024, + .associativity = 4, + .line_size = 32 + }; + *l1d = (struct cpuinfo_cache) { + .size = 32 * 1024, + .associativity = 4, + .line_size = 32 + }; + *l2 = (struct cpuinfo_cache) { + .size = cluster_cores * 256 * 1024, + .associativity = 4, + .line_size = 128 + }; + break; + case cpuinfo_uarch_krait: + /* + * - L0 Data cache = 4 KB. 64 B/line, direct mapped [1] + * - L0 Instruction cache = 4 KB. [1] + * - L1 Data cache = 16 KB. 64 B/line, 4-way [1] + * - L1 Instruction cache = 16 KB, 4-way [1] + * - L2 Cache = 1 MB, 128 B/line, 8-way. Each core has fast access only to 512 KB of L2 cache. [1] + * - L2 = 1MB (dual core) or 2MB (quad core), 8-way set associative [2] + * + * [1] http://www.7-cpu.com/cpu/Krait.html + * [2] http://www.anandtech.com/show/4940/qualcomm-new-snapdragon-s4-msm8960-krait-architecture/2 + */ + *l1i = (struct cpuinfo_cache) { + .size = 16 * 1024, + .associativity = 4, + .line_size = 64 /* assume same as L1D */ + }; + *l1d = (struct cpuinfo_cache) { + .size = 16 * 1024, + .associativity = 4, + .line_size = 64 + }; + *l2 = (struct cpuinfo_cache) { + .size = cluster_cores * 512 * 1024, + .associativity = 8, + .line_size = 128 + }; + break; +#endif /* CPUINFO_ARCH_ARM && !defined(__ARM_ARCH_8A__) */ + case cpuinfo_uarch_kryo: + /* + * +-----------------+-------+-----------+-----------+-----------+-----------+ + * | Processor model | Cores | L1D cache | L1I cache | L2 cache | Reference | + * +-----------------+-------+-----------+-----------+-----------+-----------+ + * | Snapdragon 820 | 2+2 | 24K | 32K | 1M+512K | [1, 2] | + * | Snapdragon 821 | 2+2 | ? | ? | 1M+512K | [1] | + * +-----------------+-------+-----------+-----------+-----------+-----------+ + * + * [1] http://www.anandtech.com/show/9837/snapdragon-820-preview/2 + * [2] https://www.inforcecomputing.com/public_docs/Inforce6601/Inforce_6601_Micro-SOM_FAQs_04-2016-1.pdf + */ + *l1i = (struct cpuinfo_cache) { + .size = 32 * 1024, + .associativity = 4, + .line_size = 64 + }; + *l1d = (struct cpuinfo_cache) { + .size = 24 * 1024, + .associativity = 3, + .line_size = 64 + }; + if (midr_is_kryo_silver(midr)) { + /* Kryo "Silver" */ + *l2 = (struct cpuinfo_cache) { + .size = 512 * 1024, + .associativity = 8, + .line_size = 128 + }; + } else { + /* Kryo "Gold" */ + *l2 = (struct cpuinfo_cache) { + .size = 1024 * 1024, + .associativity = 8, + .line_size = 128 + }; + } + break; + case cpuinfo_uarch_denver: + case cpuinfo_uarch_denver2: + /* + * The Denver chip includes a 128KB, 4-way level 1 instruction cache, a 64KB, 4-way level 2 data cache, + * and a 2MB, 16-way level 2 cache, all of which can service both cores. [1] + * + * All the caches have 64-byte lines. [2] + * + * [1] http://www.pcworld.com/article/2463900/nvidia-reveals-pc-like-performance-for-denver-tegra-k1.html + * [2] http://linleygroup.com/newsletters/newsletter_detail.php?num=5205&year=2014 + */ + *l1i = (struct cpuinfo_cache) { + .size = 128 * 1024, + .associativity = 4, + .line_size = 64 + }; + *l1d = (struct cpuinfo_cache) { + .size = 64 * 1024, + .associativity = 4, + .line_size = 64 + }; + *l2 = (struct cpuinfo_cache) { + .size = 2 * 1024 * 1024, + .associativity = 16, + .line_size = 64 + }; + break; + case cpuinfo_uarch_exynos_m1: + case cpuinfo_uarch_exynos_m2: + /* + * - "Moving past branch prediction we can see some elements of how the cache is set up for the L1 I$, + * namely 64 KB split into four sets with 128-byte line sizes for 128 cache lines per set" [1] + * - "For loads and stores, a 32 KB, 8-way set associative cache with 64 byte line size is used" [1] + * - "The L2 cache here is 2MB shared across all cores split into 16 sets. This memory is also split + * into 4 banks and has a 22 cycle latency" [1] + * + * +--------------------+-------+-----------+-----------+-----------+-----------+ + * | Processor model | Cores | L1D cache | L1I cache | L2 cache | Reference | + * +--------------------+-------+-----------+-----------+-----------+-----------+ + * | Exynos 8 Octa 8890 | 4(+4) | 64K | 32K | 2M | [1] | + * | Exynos 8 Octa 8895 | 4(+4) | 64K | 32K | 2M | [2] | + * +--------------------+-------+-----------+-----------+-----------+-----------+ + * + * [1] http://www.anandtech.com/show/10590/hot-chips-2016-exynos-m1-architecture-disclosed + * [2] https://www.extremetech.com/mobile/244949-samsungs-exynos-8895-features-custom-cpu-cores-first-10nm-chip-market + */ + *l1i = (struct cpuinfo_cache) { + .size = 64 * 1024, + .associativity = 4, + .line_size = 128 + }; + *l1d = (struct cpuinfo_cache) { + .size = 32 * 1024, + .associativity = 8, + .line_size = 64 + }; + *l2 = (struct cpuinfo_cache) { + .size = 2 * 1024 * 1024, + .associativity = 16, + .line_size = 64 + }; + break; + case cpuinfo_uarch_exynos_m3: + /* + * +--------------------+-------+-----------+-----------+-----------+----------+------------+ + * | Processor model | Cores | L1D cache | L1I cache | L2 cache | L3 cache | Reference | + * +--------------------+-------+-----------+-----------+-----------+----------+------------+ + * | Exynos 9810 | 4(+4) | 64K | ? | 512K | 4M | [1] | + * +--------------------+-------+-----------+-----------+-----------+----------+------------+ + * + * [1] https://www.anandtech.com/show/12478/exynos-9810-handson-awkward-first-results + */ + *l1i = (struct cpuinfo_cache) { + .size = 64 * 1024 /* assume same as in Exynos M1/M2 cores */, + .associativity = 4 /* assume same as in Exynos M1/M2 cores */, + .line_size = 128 /* assume same as in Exynos M1/M2 cores */ + }; + *l1d = (struct cpuinfo_cache) { + .size = 64 * 1024, + .associativity = 8 /* assume same as in Exynos M1/M2 cores */, + .line_size = 64 /* assume same as in Exynos M1/M2 cores */, + }; + *l2 = (struct cpuinfo_cache) { + .size = 512 * 1024, + .associativity = 16 /* assume same as in Exynos M1/M2 cores */, + .line_size = 64 /* assume same as in Exynos M1/M2 cores */, + }; + *l3 = (struct cpuinfo_cache) { + .size = 4 * 1024 * 1024, + .associativity = 16 /* assume DynamIQ cache */, + .line_size = 64 /* assume DynamIQ cache */, + }; + break; +#if CPUINFO_ARCH_ARM64 && !defined(__ANDROID__) + case cpuinfo_uarch_thunderx: + /* + * "78K-Icache and 32K-D cache per core, 16 MB shared L2 cache" [1] + * + * [1] https://www.cavium.com/pdfFiles/ThunderX_CP_PB_Rev1.pdf + */ + *l1i = (struct cpuinfo_cache) { + .size = 78 * 1024, + .associativity = 4 /* assumption */, + .line_size = 64 /* assumption */ + }; + *l1d = (struct cpuinfo_cache) { + .size = 32 * 1024, + .associativity = 4 /* assumption */, + .line_size = 64 /* assumption */ + }; + *l2 = (struct cpuinfo_cache) { + .size = 16 * 1024 * 1024, + .associativity = 8 /* assumption */, + .line_size = 64 /* assumption */ + }; + break; + case cpuinfo_uarch_taishan_v110: + /* + * It features private 64 KiB L1 instruction and data caches as well as 512 KiB of private L2. [1] + * + * +------------------+-------+-----------+-----------+-----------+----------+-----------+ + * | Processor model | Cores | L1D cache | L1I cache | L2 cache | L3 cache | Reference | + * +------------------+-------+-----------+-----------+-----------+----------+-----------+ + * | Kunpeng 920-3226 | 32 | 64K | 64K | 512K | 32M | [2] | + * +------------------+-------+-----------+-----------+-----------+----------+-----------+ + * | Kunpeng 920-4826 | 48 | 64K | 64K | 512K | 48M | [3] | + * +------------------+-------+-----------+-----------+-----------+----------+-----------+ + * | Kunpeng 920-6426 | 64 | 64K | 64K | 512K | 64M | [4] | + * +------------------+-------+-----------+-----------+-----------+----------+-----------+ + * + * [1] https://en.wikichip.org/wiki/hisilicon/microarchitectures/taishan_v110 + * [2] https://en.wikichip.org/wiki/hisilicon/kunpeng/920-3226 + * [3] https://en.wikichip.org/wiki/hisilicon/kunpeng/920-4826 + * [4] https://en.wikichip.org/wiki/hisilicon/kunpeng/920-6426 + */ + *l1i = (struct cpuinfo_cache) { + .size = 64 * 1024, + .associativity = 4 /* assumption */, + .line_size = 128 /* assumption */, + }; + *l1d = (struct cpuinfo_cache) { + .size = 64 * 1024, + .associativity = 4 /* assumption */, + .line_size = 128 /* assumption */, + }; + *l2 = (struct cpuinfo_cache) { + .size = 512 * 1024, + .associativity = 8 /* assumption */, + .line_size = 128 /* assumption */, + .flags = CPUINFO_CACHE_INCLUSIVE /* assumption */, + }; + *l3 = (struct cpuinfo_cache) { + .size = cluster_cores * 1024 * 1024, + .associativity = 16 /* assumption */, + .line_size = 128 /* assumption */, + }; + break; +#endif + case cpuinfo_uarch_cortex_a12: + case cpuinfo_uarch_cortex_a32: + default: + cpuinfo_log_warning("target uarch not recognized; using generic cache parameters"); + /* Follow OpenBLAS */ + if (arch_version >= 8) { + *l1i = (struct cpuinfo_cache) { + .size = 32 * 1024, + .associativity = 4, + .line_size = 64 + }; + *l1d = (struct cpuinfo_cache) { + .size = 32 * 1024, + .associativity = 4, + .line_size = 64 + }; + *l2 = (struct cpuinfo_cache) { + .size = cluster_cores * 256 * 1024, + .associativity = 8, + .line_size = 64 + }; + } else { + *l1i = (struct cpuinfo_cache) { + .size = 16 * 1024, + .associativity = 4, + .line_size = 32 + }; + *l1d = (struct cpuinfo_cache) { + .size = 16 * 1024, + .associativity = 4, + .line_size = 32 + }; + if (arch_version >= 7) { + *l2 = (struct cpuinfo_cache) { + .size = cluster_cores * 128 * 1024, + .associativity = 8, + .line_size = 32 + }; + } + } + break; + } + l1i->sets = l1i->size / (l1i->associativity * l1i->line_size); + l1i->partitions = 1; + l1d->sets = l1d->size / (l1d->associativity * l1d->line_size); + l1d->partitions = 1; + if (l2->size != 0) { + l2->sets = l2->size / (l2->associativity * l2->line_size); + l2->partitions = 1; + if (l3->size != 0) { + l3->sets = l3->size / (l3->associativity * l3->line_size); + l3->partitions = 1; + } + } +} + +uint32_t cpuinfo_arm_compute_max_cache_size(const struct cpuinfo_processor* processor) { + /* + * There is no precise way to detect cache size on ARM/ARM64, and cache size reported by cpuinfo + * may underestimate the actual cache size. Thus, we use microarchitecture-specific maximum. + */ + switch (processor->core->uarch) { + case cpuinfo_uarch_xscale: + case cpuinfo_uarch_arm11: + case cpuinfo_uarch_scorpion: + case cpuinfo_uarch_krait: + case cpuinfo_uarch_kryo: + case cpuinfo_uarch_exynos_m1: + case cpuinfo_uarch_exynos_m2: + case cpuinfo_uarch_exynos_m3: + /* cpuinfo-detected cache size always correct */ + return cpuinfo_compute_max_cache_size(processor); + case cpuinfo_uarch_cortex_a5: + /* Max observed (NXP Vybrid SoC) */ + return 512 * 1024; + case cpuinfo_uarch_cortex_a7: + /* + * Cortex-A7 MPCore Technical Reference Manual: + * 7.1. About the L2 Memory system + * The L2 memory system consists of an: + * - Optional tightly-coupled L2 cache that includes: + * - Configurable L2 cache size of 128KB, 256KB, 512KB, and 1MB. + */ + return 1024 * 1024; + case cpuinfo_uarch_cortex_a8: + /* + * Cortex-A8 Technical Reference Manual: + * 8.1. About the L2 memory system + * The key features of the L2 memory system include: + * - configurable cache size of 0KB, 128KB, 256KB, 512KB, and 1MB + */ + return 1024 * 1024; + case cpuinfo_uarch_cortex_a9: + /* Max observed (e.g. Exynos 4212) */ + return 1024 * 1024; + case cpuinfo_uarch_cortex_a12: + case cpuinfo_uarch_cortex_a17: + /* + * ARM Cortex-A17 MPCore Processor Technical Reference Manual: + * 7.1. About the L2 Memory system + * The key features of the L2 memory system include: + * - An integrated L2 cache: + * - The cache size is implemented as either 256KB, 512KB, 1MB, 2MB, 4MB or 8MB. + */ + return 8 * 1024 * 1024; + case cpuinfo_uarch_cortex_a15: + /* + * ARM Cortex-A15 MPCore Processor Technical Reference Manual: + * 7.1. About the L2 memory system + * The features of the L2 memory system include: + * - Configurable L2 cache size of 512KB, 1MB, 2MB and 4MB. + */ + return 4 * 1024 * 1024; + case cpuinfo_uarch_cortex_a35: + /* + * ARM Cortex‑A35 Processor Technical Reference Manual: + * 7.1 About the L2 memory system + * L2 cache + * - Further features of the L2 cache are: + * - Configurable size of 128KB, 256KB, 512KB, and 1MB. + */ + return 1024 * 1024; + case cpuinfo_uarch_cortex_a53: + /* + * ARM Cortex-A53 MPCore Processor Technical Reference Manual: + * 7.1. About the L2 memory system + * The L2 memory system consists of an: + * - Optional tightly-coupled L2 cache that includes: + * - Configurable L2 cache size of 128KB, 256KB, 512KB, 1MB and 2MB. + */ + return 2 * 1024 * 1024; + case cpuinfo_uarch_cortex_a57: + /* + * ARM Cortex-A57 MPCore Processor Technical Reference Manual: + * 7.1 About the L2 memory system + * The features of the L2 memory system include: + * - Configurable L2 cache size of 512KB, 1MB, and 2MB. + */ + return 2 * 1024 * 1024; + case cpuinfo_uarch_cortex_a72: + /* + * ARM Cortex-A72 MPCore Processor Technical Reference Manual: + * 7.1 About the L2 memory system + * The features of the L2 memory system include: + * - Configurable L2 cache size of 512KB, 1MB, 2MB and 4MB. + */ + return 4 * 1024 * 1024; + case cpuinfo_uarch_cortex_a73: + /* + * ARM Cortex‑A73 MPCore Processor Technical Reference Manual + * 7.1 About the L2 memory system + * The L2 memory system consists of: + * - A tightly-integrated L2 cache with: + * - A configurable size of 256KB, 512KB, 1MB, 2MB, 4MB, or 8MB. + */ + return 8 * 1024 * 1024; + case cpuinfo_uarch_cortex_a55: + case cpuinfo_uarch_neoverse_n1: + case cpuinfo_uarch_neoverse_v1: + case cpuinfo_uarch_neoverse_n2: + case cpuinfo_uarch_cortex_a75: + case cpuinfo_uarch_cortex_a76: + case cpuinfo_uarch_exynos_m4: + default: + /* + * ARM DynamIQ Shared Unit Technical Reference Manual + * 1.3 Implementation options + * L3_CACHE_SIZE + * - 256KB + * - 512KB + * - 1024KB + * - 1536KB + * - 2048KB + * - 3072KB + * - 4096KB + */ + return 4 * 1024 * 1024; + } +} diff --git a/3rdparty/cpuinfo/src/arm/linux/aarch32-isa.c b/3rdparty/cpuinfo/src/arm/linux/aarch32-isa.c new file mode 100644 index 000000000..d6f6a21d2 --- /dev/null +++ b/3rdparty/cpuinfo/src/arm/linux/aarch32-isa.c @@ -0,0 +1,271 @@ +#include + +#if CPUINFO_MOCK + #include +#endif +#include +#include +#include +#include + + +#if CPUINFO_MOCK + uint32_t cpuinfo_arm_fpsid = 0; + uint32_t cpuinfo_arm_mvfr0 = 0; + uint32_t cpuinfo_arm_wcid = 0; + + void cpuinfo_set_fpsid(uint32_t fpsid) { + cpuinfo_arm_fpsid = fpsid; + } + + void cpuinfo_set_wcid(uint32_t wcid) { + cpuinfo_arm_wcid = wcid; + } +#endif + + +void cpuinfo_arm_linux_decode_isa_from_proc_cpuinfo( + uint32_t features, + uint32_t features2, + uint32_t midr, + uint32_t architecture_version, + uint32_t architecture_flags, + const struct cpuinfo_arm_chipset chipset[restrict static 1], + struct cpuinfo_arm_isa isa[restrict static 1]) +{ + if (architecture_version >= 8) { + /* + * ARMv7 code running on ARMv8: IDIV, VFP, NEON are always supported, + * but may be not reported in /proc/cpuinfo features. + */ + isa->armv5e = true; + isa->armv6 = true; + isa->armv6k = true; + isa->armv7 = true; + isa->armv7mp = true; + isa->armv8 = true; + isa->thumb = true; + isa->thumb2 = true; + isa->idiv = true; + isa->vfpv3 = true; + isa->d32 = true; + isa->fp16 = true; + isa->fma = true; + isa->neon = true; + + /* + * NEON FP16 compute extension and VQRDMLAH/VQRDMLSH instructions are not indicated in /proc/cpuinfo. + * Use a MIDR-based heuristic to whitelist processors known to support it: + * - Processors with Cortex-A55 cores + * - Processors with Cortex-A65 cores + * - Processors with Cortex-A75 cores + * - Processors with Cortex-A76 cores + * - Processors with Cortex-A77 cores + * - Processors with Exynos M4 cores + * - Processors with Exynos M5 cores + * - Neoverse N1 cores + * - Neoverse V1 cores + * - Neoverse N2 cores + */ + if (chipset->series == cpuinfo_arm_chipset_series_samsung_exynos && chipset->model == 9810) { + /* Only little cores of Exynos 9810 support FP16 & RDM */ + cpuinfo_log_warning("FP16 arithmetics and RDM disabled: only little cores in Exynos 9810 support these extensions"); + } else { + switch (midr & (CPUINFO_ARM_MIDR_IMPLEMENTER_MASK | CPUINFO_ARM_MIDR_PART_MASK)) { + case UINT32_C(0x4100D050): /* Cortex-A55 */ + case UINT32_C(0x4100D060): /* Cortex-A65 */ + case UINT32_C(0x4100D0B0): /* Cortex-A76 */ + case UINT32_C(0x4100D0C0): /* Neoverse N1 */ + case UINT32_C(0x4100D0D0): /* Cortex-A77 */ + case UINT32_C(0x4100D0E0): /* Cortex-A76AE */ + case UINT32_C(0x4100D400): /* Neoverse V1 */ + case UINT32_C(0x4100D490): /* Neoverse N2 */ + case UINT32_C(0x4800D400): /* Cortex-A76 (HiSilicon) */ + case UINT32_C(0x51008020): /* Kryo 385 Gold (Cortex-A75) */ + case UINT32_C(0x51008030): /* Kryo 385 Silver (Cortex-A55) */ + case UINT32_C(0x51008040): /* Kryo 485 Gold (Cortex-A76) */ + case UINT32_C(0x51008050): /* Kryo 485 Silver (Cortex-A55) */ + case UINT32_C(0x53000030): /* Exynos M4 */ + case UINT32_C(0x53000040): /* Exynos M5 */ + isa->fp16arith = true; + isa->rdm = true; + break; + } + } + + /* + * NEON VDOT instructions are not indicated in /proc/cpuinfo. + * Use a MIDR-based heuristic to whitelist processors known to support it. + */ + switch (midr & (CPUINFO_ARM_MIDR_IMPLEMENTER_MASK | CPUINFO_ARM_MIDR_PART_MASK)) { + case UINT32_C(0x4100D0B0): /* Cortex-A76 */ + case UINT32_C(0x4100D0D0): /* Cortex-A77 */ + case UINT32_C(0x4100D0E0): /* Cortex-A76AE */ + case UINT32_C(0x4800D400): /* Cortex-A76 (HiSilicon) */ + case UINT32_C(0x51008040): /* Kryo 485 Gold (Cortex-A76) */ + case UINT32_C(0x51008050): /* Kryo 485 Silver (Cortex-A55) */ + case UINT32_C(0x53000030): /* Exynos-M4 */ + case UINT32_C(0x53000040): /* Exynos-M5 */ + isa->dot = true; + break; + case UINT32_C(0x4100D050): /* Cortex A55: revision 1 or later only */ + isa->dot = !!(midr_get_variant(midr) >= 1); + break; + case UINT32_C(0x4100D0A0): /* Cortex A75: revision 2 or later only */ + isa->dot = !!(midr_get_variant(midr) >= 2); + break; + } + } else { + /* ARMv7 or lower: use feature flags to detect optional features */ + + /* + * ARM11 (ARM 1136/1156/1176/11 MPCore) processors can report v7 architecture + * even though they support only ARMv6 instruction set. + */ + if (architecture_version == 7 && midr_is_arm11(midr)) { + cpuinfo_log_warning("kernel-reported architecture ARMv7 ignored due to mismatch with processor microarchitecture (ARM11)"); + architecture_version = 6; + } + + if (architecture_version < 7) { + const uint32_t armv7_features_mask = CPUINFO_ARM_LINUX_FEATURE_VFPV3 | CPUINFO_ARM_LINUX_FEATURE_VFPV3D16 | CPUINFO_ARM_LINUX_FEATURE_VFPD32 | + CPUINFO_ARM_LINUX_FEATURE_VFPV4 | CPUINFO_ARM_LINUX_FEATURE_NEON | CPUINFO_ARM_LINUX_FEATURE_IDIVT | CPUINFO_ARM_LINUX_FEATURE_IDIVA; + if (features & armv7_features_mask) { + architecture_version = 7; + } + } + if ((architecture_version >= 6) || (features & CPUINFO_ARM_LINUX_FEATURE_EDSP) || (architecture_flags & CPUINFO_ARM_LINUX_ARCH_E)) { + isa->armv5e = true; + } + if (architecture_version >= 6) { + isa->armv6 = true; + } + if (architecture_version >= 7) { + isa->armv6k = true; + isa->armv7 = true; + + /* + * ARMv7 MP extension (PLDW instruction) is not indicated in /proc/cpuinfo. + * Use heuristic list of supporting processors: + * - Processors supporting UDIV/SDIV instructions ("idiva" + "idivt" features in /proc/cpuinfo) + * - Cortex-A5 + * - Cortex-A9 + * - Dual-Core Scorpion + * - Krait (supports UDIV/SDIV, but kernels may not report it in /proc/cpuinfo) + * + * TODO: check single-core Qualcomm Scorpion. + */ + switch (midr & (CPUINFO_ARM_MIDR_IMPLEMENTER_MASK | CPUINFO_ARM_MIDR_PART_MASK)) { + case UINT32_C(0x4100C050): /* Cortex-A5 */ + case UINT32_C(0x4100C090): /* Cortex-A9 */ + case UINT32_C(0x510002D0): /* Scorpion (dual-core) */ + case UINT32_C(0x510004D0): /* Krait (dual-core) */ + case UINT32_C(0x510006F0): /* Krait (quad-core) */ + isa->armv7mp = true; + break; + default: + /* In practice IDIV instruction implies ARMv7+MP ISA */ + isa->armv7mp = (features & CPUINFO_ARM_LINUX_FEATURE_IDIV) == CPUINFO_ARM_LINUX_FEATURE_IDIV; + break; + } + } + + if (features & CPUINFO_ARM_LINUX_FEATURE_IWMMXT) { + const uint32_t wcid = read_wcid(); + cpuinfo_log_debug("WCID = 0x%08"PRIx32, wcid); + const uint32_t coprocessor_type = (wcid >> 8) & UINT32_C(0xFF); + if (coprocessor_type >= 0x10) { + isa->wmmx = true; + if (coprocessor_type >= 0x20) { + isa->wmmx2 = true; + } + } else { + cpuinfo_log_warning("WMMX ISA disabled: OS reported iwmmxt feature, " + "but WCID coprocessor type 0x%"PRIx32" indicates no WMMX support", + coprocessor_type); + } + } + + if ((features & CPUINFO_ARM_LINUX_FEATURE_THUMB) || (architecture_flags & CPUINFO_ARM_LINUX_ARCH_T)) { + isa->thumb = true; + + /* + * There is no separate feature flag for Thumb 2. + * All ARMv7 processors and ARM 1156 support Thumb 2. + */ + if (architecture_version >= 7 || midr_is_arm1156(midr)) { + isa->thumb2 = true; + } + } + if (features & CPUINFO_ARM_LINUX_FEATURE_THUMBEE) { + isa->thumbee = true; + } + if ((features & CPUINFO_ARM_LINUX_FEATURE_JAVA) || (architecture_flags & CPUINFO_ARM_LINUX_ARCH_J)) { + isa->jazelle = true; + } + + /* Qualcomm Krait may have buggy kernel configuration that doesn't report IDIV */ + if ((features & CPUINFO_ARM_LINUX_FEATURE_IDIV) == CPUINFO_ARM_LINUX_FEATURE_IDIV || midr_is_krait(midr)) { + isa->idiv = true; + } + + const uint32_t vfp_mask = \ + CPUINFO_ARM_LINUX_FEATURE_VFP | CPUINFO_ARM_LINUX_FEATURE_VFPV3 | CPUINFO_ARM_LINUX_FEATURE_VFPV3D16 | \ + CPUINFO_ARM_LINUX_FEATURE_VFPD32 | CPUINFO_ARM_LINUX_FEATURE_VFPV4 | CPUINFO_ARM_LINUX_FEATURE_NEON; + if (features & vfp_mask) { + const uint32_t vfpv3_mask = CPUINFO_ARM_LINUX_FEATURE_VFPV3 | CPUINFO_ARM_LINUX_FEATURE_VFPV3D16 | \ + CPUINFO_ARM_LINUX_FEATURE_VFPD32 | CPUINFO_ARM_LINUX_FEATURE_VFPV4 | CPUINFO_ARM_LINUX_FEATURE_NEON; + if ((architecture_version >= 7) || (features & vfpv3_mask)) { + isa->vfpv3 = true; + + const uint32_t d32_mask = CPUINFO_ARM_LINUX_FEATURE_VFPD32 | CPUINFO_ARM_LINUX_FEATURE_NEON; + if (features & d32_mask) { + isa->d32 = true; + } + } else { + #if defined(__ARM_ARCH_7A__) || defined(__ARM_ARCH_8A__) || defined(__ARM_ARCH) && (__ARM_ARCH >= 7) + isa->vfpv3 = true; + #else + const uint32_t fpsid = read_fpsid(); + cpuinfo_log_debug("FPSID = 0x%08"PRIx32, fpsid); + const uint32_t subarchitecture = (fpsid >> 16) & UINT32_C(0x7F); + if (subarchitecture >= 0x01) { + isa->vfpv2 = true; + } + #endif + } + } + if (features & CPUINFO_ARM_LINUX_FEATURE_NEON) { + isa->neon = true; + } + + /* + * There is no separate feature flag for FP16 support. + * VFPv4 implies VFPv3-FP16 support (and in practice, NEON-HP as well). + * Additionally, ARM Cortex-A9 and Qualcomm Scorpion support FP16. + */ + if ((features & CPUINFO_ARM_LINUX_FEATURE_VFPV4) || midr_is_cortex_a9(midr) || midr_is_scorpion(midr)) { + isa->fp16 = true; + } + + if (features & CPUINFO_ARM_LINUX_FEATURE_VFPV4) { + isa->fma = true; + } + } + + if (features2 & CPUINFO_ARM_LINUX_FEATURE2_AES) { + isa->aes = true; + } + if (features2 & CPUINFO_ARM_LINUX_FEATURE2_PMULL) { + isa->pmull = true; + } + if (features2 & CPUINFO_ARM_LINUX_FEATURE2_SHA1) { + isa->sha1 = true; + } + if (features2 & CPUINFO_ARM_LINUX_FEATURE2_SHA2) { + isa->sha2 = true; + } + if (features2 & CPUINFO_ARM_LINUX_FEATURE2_CRC32) { + isa->crc32 = true; + } +} diff --git a/3rdparty/cpuinfo/src/arm/linux/aarch64-isa.c b/3rdparty/cpuinfo/src/arm/linux/aarch64-isa.c new file mode 100644 index 000000000..7b18095c3 --- /dev/null +++ b/3rdparty/cpuinfo/src/arm/linux/aarch64-isa.c @@ -0,0 +1,139 @@ +#include + +#include +#include + + +void cpuinfo_arm64_linux_decode_isa_from_proc_cpuinfo( + uint32_t features, + uint32_t features2, + uint32_t midr, + const struct cpuinfo_arm_chipset chipset[restrict static 1], + struct cpuinfo_arm_isa isa[restrict static 1]) +{ + if (features & CPUINFO_ARM_LINUX_FEATURE_AES) { + isa->aes = true; + } + if (features & CPUINFO_ARM_LINUX_FEATURE_PMULL) { + isa->pmull = true; + } + if (features & CPUINFO_ARM_LINUX_FEATURE_SHA1) { + isa->sha1 = true; + } + if (features & CPUINFO_ARM_LINUX_FEATURE_SHA2) { + isa->sha2 = true; + } + if (features & CPUINFO_ARM_LINUX_FEATURE_CRC32) { + isa->crc32 = true; + } + if (features & CPUINFO_ARM_LINUX_FEATURE_ATOMICS) { + isa->atomics = true; + } + + /* + * Some phones ship with an old kernel configuration that doesn't report NEON FP16 compute extension and SQRDMLAH/SQRDMLSH/UQRDMLAH/UQRDMLSH instructions. + * Use a MIDR-based heuristic to whitelist processors known to support it: + * - Processors with Cortex-A55 cores + * - Processors with Cortex-A65 cores + * - Processors with Cortex-A75 cores + * - Processors with Cortex-A76 cores + * - Processors with Cortex-A77 cores + * - Processors with Exynos M4 cores + * - Processors with Exynos M5 cores + * - Neoverse N1 cores + * - Neoverse V1 cores + * - Neoverse N2 cores + */ + if (chipset->series == cpuinfo_arm_chipset_series_samsung_exynos && chipset->model == 9810) { + /* Exynos 9810 reports that it supports FP16 compute, but in fact only little cores do */ + cpuinfo_log_warning("FP16 arithmetics and RDM disabled: only little cores in Exynos 9810 support these extensions"); + } else { + const uint32_t fp16arith_mask = CPUINFO_ARM_LINUX_FEATURE_FPHP | CPUINFO_ARM_LINUX_FEATURE_ASIMDHP; + switch (midr & (CPUINFO_ARM_MIDR_IMPLEMENTER_MASK | CPUINFO_ARM_MIDR_PART_MASK)) { + case UINT32_C(0x4100D050): /* Cortex-A55 */ + case UINT32_C(0x4100D060): /* Cortex-A65 */ + case UINT32_C(0x4100D0B0): /* Cortex-A76 */ + case UINT32_C(0x4100D0C0): /* Neoverse N1 */ + case UINT32_C(0x4100D0D0): /* Cortex-A77 */ + case UINT32_C(0x4100D0E0): /* Cortex-A76AE */ + case UINT32_C(0x4100D400): /* Neoverse V1 */ + case UINT32_C(0x4100D490): /* Neoverse N2 */ + case UINT32_C(0x4800D400): /* Cortex-A76 (HiSilicon) */ + case UINT32_C(0x51008020): /* Kryo 385 Gold (Cortex-A75) */ + case UINT32_C(0x51008030): /* Kryo 385 Silver (Cortex-A55) */ + case UINT32_C(0x51008040): /* Kryo 485 Gold (Cortex-A76) */ + case UINT32_C(0x51008050): /* Kryo 485 Silver (Cortex-A55) */ + case UINT32_C(0x53000030): /* Exynos M4 */ + case UINT32_C(0x53000040): /* Exynos M5 */ + isa->fp16arith = true; + isa->rdm = true; + break; + default: + if ((features & fp16arith_mask) == fp16arith_mask) { + isa->fp16arith = true; + } else if (features & CPUINFO_ARM_LINUX_FEATURE_FPHP) { + cpuinfo_log_warning("FP16 arithmetics disabled: detected support only for scalar operations"); + } else if (features & CPUINFO_ARM_LINUX_FEATURE_ASIMDHP) { + cpuinfo_log_warning("FP16 arithmetics disabled: detected support only for SIMD operations"); + } + if (features & CPUINFO_ARM_LINUX_FEATURE_ASIMDRDM) { + isa->rdm = true; + } + break; + } + } + + /* + * Many phones ship with an old kernel configuration that doesn't report UDOT/SDOT instructions. + * Use a MIDR-based heuristic to whitelist processors known to support it. + */ + switch (midr & (CPUINFO_ARM_MIDR_IMPLEMENTER_MASK | CPUINFO_ARM_MIDR_PART_MASK)) { + case UINT32_C(0x4100D060): /* Cortex-A65 */ + case UINT32_C(0x4100D0B0): /* Cortex-A76 */ + case UINT32_C(0x4100D0C0): /* Neoverse N1 */ + case UINT32_C(0x4100D0D0): /* Cortex-A77 */ + case UINT32_C(0x4100D0E0): /* Cortex-A76AE */ + case UINT32_C(0x4100D400): /* Neoverse V1 */ + case UINT32_C(0x4100D490): /* Neoverse N2 */ + case UINT32_C(0x4100D4A0): /* Neoverse E1 */ + case UINT32_C(0x4800D400): /* Cortex-A76 (HiSilicon) */ + case UINT32_C(0x51008040): /* Kryo 485 Gold (Cortex-A76) */ + case UINT32_C(0x51008050): /* Kryo 485 Silver (Cortex-A55) */ + case UINT32_C(0x53000030): /* Exynos-M4 */ + case UINT32_C(0x53000040): /* Exynos-M5 */ + isa->dot = true; + break; + case UINT32_C(0x4100D050): /* Cortex A55: revision 1 or later only */ + isa->dot = !!(midr_get_variant(midr) >= 1); + break; + case UINT32_C(0x4100D0A0): /* Cortex A75: revision 2 or later only */ + isa->dot = !!(midr_get_variant(midr) >= 2); + break; + default: + if (features & CPUINFO_ARM_LINUX_FEATURE_ASIMDDP) { + isa->dot = true; + } + break; + } + if (features & CPUINFO_ARM_LINUX_FEATURE_JSCVT) { + isa->jscvt = true; + } + if (features & CPUINFO_ARM_LINUX_FEATURE_JSCVT) { + isa->jscvt = true; + } + if (features & CPUINFO_ARM_LINUX_FEATURE_FCMA) { + isa->fcma = true; + } + if (features & CPUINFO_ARM_LINUX_FEATURE_SVE) { + isa->sve = true; + } + if (features2 & CPUINFO_ARM_LINUX_FEATURE2_SVE2) { + isa->sve2 = true; + } + if (features2 & CPUINFO_ARM_LINUX_FEATURE2_BF16) { + isa->bf16 = true; + } + if (features2 & CPUINFO_ARM_LINUX_FEATURE2_SVEBF16) { + isa->svebf16 = true; + } +} diff --git a/3rdparty/cpuinfo/src/arm/linux/api.h b/3rdparty/cpuinfo/src/arm/linux/api.h new file mode 100644 index 000000000..1c09f827e --- /dev/null +++ b/3rdparty/cpuinfo/src/arm/linux/api.h @@ -0,0 +1,384 @@ +#pragma once + +#include +#include + +#include +#include +#include +#include +#include + +/* No hard limit in the kernel, maximum length observed on non-rogue kernels is 64 */ +#define CPUINFO_HARDWARE_VALUE_MAX 64 +/* No hard limit in the kernel, maximum length on Raspberry Pi is 8. Add 1 symbol to detect overly large revision strings */ +#define CPUINFO_REVISION_VALUE_MAX 9 + +#ifdef __ANDROID__ + /* As per include/sys/system_properties.h in Android NDK */ + #define CPUINFO_BUILD_PROP_NAME_MAX 32 + #define CPUINFO_BUILD_PROP_VALUE_MAX 92 + + struct cpuinfo_android_properties { + char proc_cpuinfo_hardware[CPUINFO_HARDWARE_VALUE_MAX]; + char ro_product_board[CPUINFO_BUILD_PROP_VALUE_MAX]; + char ro_board_platform[CPUINFO_BUILD_PROP_VALUE_MAX]; + char ro_mediatek_platform[CPUINFO_BUILD_PROP_VALUE_MAX]; + char ro_arch[CPUINFO_BUILD_PROP_VALUE_MAX]; + char ro_chipname[CPUINFO_BUILD_PROP_VALUE_MAX]; + char ro_hardware_chipname[CPUINFO_BUILD_PROP_VALUE_MAX]; + }; +#endif + +#define CPUINFO_ARM_LINUX_ARCH_T UINT32_C(0x00000001) +#define CPUINFO_ARM_LINUX_ARCH_E UINT32_C(0x00000002) +#define CPUINFO_ARM_LINUX_ARCH_J UINT32_C(0x00000004) + +#define CPUINFO_ARM_LINUX_ARCH_TE UINT32_C(0x00000003) +#define CPUINFO_ARM_LINUX_ARCH_TEJ UINT32_C(0x00000007) + +struct cpuinfo_arm_linux_proc_cpuinfo_cache { + uint32_t i_size; + uint32_t i_assoc; + uint32_t i_line_length; + uint32_t i_sets; + uint32_t d_size; + uint32_t d_assoc; + uint32_t d_line_length; + uint32_t d_sets; +}; + +#if CPUINFO_ARCH_ARM + /* arch/arm/include/uapi/asm/hwcap.h */ + + #define CPUINFO_ARM_LINUX_FEATURE_SWP UINT32_C(0x00000001) + #define CPUINFO_ARM_LINUX_FEATURE_HALF UINT32_C(0x00000002) + #define CPUINFO_ARM_LINUX_FEATURE_THUMB UINT32_C(0x00000004) + #define CPUINFO_ARM_LINUX_FEATURE_26BIT UINT32_C(0x00000008) + #define CPUINFO_ARM_LINUX_FEATURE_FASTMULT UINT32_C(0x00000010) + #define CPUINFO_ARM_LINUX_FEATURE_FPA UINT32_C(0x00000020) + #define CPUINFO_ARM_LINUX_FEATURE_VFP UINT32_C(0x00000040) + #define CPUINFO_ARM_LINUX_FEATURE_EDSP UINT32_C(0x00000080) + #define CPUINFO_ARM_LINUX_FEATURE_JAVA UINT32_C(0x00000100) + #define CPUINFO_ARM_LINUX_FEATURE_IWMMXT UINT32_C(0x00000200) + #define CPUINFO_ARM_LINUX_FEATURE_CRUNCH UINT32_C(0x00000400) + #define CPUINFO_ARM_LINUX_FEATURE_THUMBEE UINT32_C(0x00000800) + #define CPUINFO_ARM_LINUX_FEATURE_NEON UINT32_C(0x00001000) + #define CPUINFO_ARM_LINUX_FEATURE_VFPV3 UINT32_C(0x00002000) + #define CPUINFO_ARM_LINUX_FEATURE_VFPV3D16 UINT32_C(0x00004000) /* Also set for VFPv4 with 16 double-precision registers */ + #define CPUINFO_ARM_LINUX_FEATURE_TLS UINT32_C(0x00008000) + #define CPUINFO_ARM_LINUX_FEATURE_VFPV4 UINT32_C(0x00010000) + #define CPUINFO_ARM_LINUX_FEATURE_IDIVA UINT32_C(0x00020000) + #define CPUINFO_ARM_LINUX_FEATURE_IDIVT UINT32_C(0x00040000) + #define CPUINFO_ARM_LINUX_FEATURE_IDIV UINT32_C(0x00060000) + #define CPUINFO_ARM_LINUX_FEATURE_VFPD32 UINT32_C(0x00080000) + #define CPUINFO_ARM_LINUX_FEATURE_LPAE UINT32_C(0x00100000) + #define CPUINFO_ARM_LINUX_FEATURE_EVTSTRM UINT32_C(0x00200000) + + #define CPUINFO_ARM_LINUX_FEATURE2_AES UINT32_C(0x00000001) + #define CPUINFO_ARM_LINUX_FEATURE2_PMULL UINT32_C(0x00000002) + #define CPUINFO_ARM_LINUX_FEATURE2_SHA1 UINT32_C(0x00000004) + #define CPUINFO_ARM_LINUX_FEATURE2_SHA2 UINT32_C(0x00000008) + #define CPUINFO_ARM_LINUX_FEATURE2_CRC32 UINT32_C(0x00000010) +#elif CPUINFO_ARCH_ARM64 + /* arch/arm64/include/uapi/asm/hwcap.h */ + #define CPUINFO_ARM_LINUX_FEATURE_FP UINT32_C(0x00000001) + #define CPUINFO_ARM_LINUX_FEATURE_ASIMD UINT32_C(0x00000002) + #define CPUINFO_ARM_LINUX_FEATURE_EVTSTRM UINT32_C(0x00000004) + #define CPUINFO_ARM_LINUX_FEATURE_AES UINT32_C(0x00000008) + #define CPUINFO_ARM_LINUX_FEATURE_PMULL UINT32_C(0x00000010) + #define CPUINFO_ARM_LINUX_FEATURE_SHA1 UINT32_C(0x00000020) + #define CPUINFO_ARM_LINUX_FEATURE_SHA2 UINT32_C(0x00000040) + #define CPUINFO_ARM_LINUX_FEATURE_CRC32 UINT32_C(0x00000080) + #define CPUINFO_ARM_LINUX_FEATURE_ATOMICS UINT32_C(0x00000100) + #define CPUINFO_ARM_LINUX_FEATURE_FPHP UINT32_C(0x00000200) + #define CPUINFO_ARM_LINUX_FEATURE_ASIMDHP UINT32_C(0x00000400) + #define CPUINFO_ARM_LINUX_FEATURE_CPUID UINT32_C(0x00000800) + #define CPUINFO_ARM_LINUX_FEATURE_ASIMDRDM UINT32_C(0x00001000) + #define CPUINFO_ARM_LINUX_FEATURE_JSCVT UINT32_C(0x00002000) + #define CPUINFO_ARM_LINUX_FEATURE_FCMA UINT32_C(0x00004000) + #define CPUINFO_ARM_LINUX_FEATURE_LRCPC UINT32_C(0x00008000) + #define CPUINFO_ARM_LINUX_FEATURE_DCPOP UINT32_C(0x00010000) + #define CPUINFO_ARM_LINUX_FEATURE_SHA3 UINT32_C(0x00020000) + #define CPUINFO_ARM_LINUX_FEATURE_SM3 UINT32_C(0x00040000) + #define CPUINFO_ARM_LINUX_FEATURE_SM4 UINT32_C(0x00080000) + #define CPUINFO_ARM_LINUX_FEATURE_ASIMDDP UINT32_C(0x00100000) + #define CPUINFO_ARM_LINUX_FEATURE_SHA512 UINT32_C(0x00200000) + #define CPUINFO_ARM_LINUX_FEATURE_SVE UINT32_C(0x00400000) + #define CPUINFO_ARM_LINUX_FEATURE_ASIMDFHM UINT32_C(0x00800000) + #define CPUINFO_ARM_LINUX_FEATURE_DIT UINT32_C(0x01000000) + #define CPUINFO_ARM_LINUX_FEATURE_USCAT UINT32_C(0x02000000) + #define CPUINFO_ARM_LINUX_FEATURE_ILRCPC UINT32_C(0x04000000) + #define CPUINFO_ARM_LINUX_FEATURE_FLAGM UINT32_C(0x08000000) + #define CPUINFO_ARM_LINUX_FEATURE_SSBS UINT32_C(0x10000000) + #define CPUINFO_ARM_LINUX_FEATURE_SB UINT32_C(0x20000000) + #define CPUINFO_ARM_LINUX_FEATURE_PACA UINT32_C(0x40000000) + #define CPUINFO_ARM_LINUX_FEATURE_PACG UINT32_C(0x80000000) + + #define CPUINFO_ARM_LINUX_FEATURE2_DCPODP UINT32_C(0x00000001) + #define CPUINFO_ARM_LINUX_FEATURE2_SVE2 UINT32_C(0x00000002) + #define CPUINFO_ARM_LINUX_FEATURE2_SVEAES UINT32_C(0x00000004) + #define CPUINFO_ARM_LINUX_FEATURE2_SVEPMULL UINT32_C(0x00000008) + #define CPUINFO_ARM_LINUX_FEATURE2_SVEBITPERM UINT32_C(0x00000010) + #define CPUINFO_ARM_LINUX_FEATURE2_SVESHA3 UINT32_C(0x00000020) + #define CPUINFO_ARM_LINUX_FEATURE2_SVESM4 UINT32_C(0x00000040) + #define CPUINFO_ARM_LINUX_FEATURE2_FLAGM2 UINT32_C(0x00000080) + #define CPUINFO_ARM_LINUX_FEATURE2_FRINT UINT32_C(0x00000100) + #define CPUINFO_ARM_LINUX_FEATURE2_SVEI8MM UINT32_C(0x00000200) + #define CPUINFO_ARM_LINUX_FEATURE2_SVEF32MM UINT32_C(0x00000400) + #define CPUINFO_ARM_LINUX_FEATURE2_SVEF64MM UINT32_C(0x00000800) + #define CPUINFO_ARM_LINUX_FEATURE2_SVEBF16 UINT32_C(0x00001000) + #define CPUINFO_ARM_LINUX_FEATURE2_I8MM UINT32_C(0x00002000) + #define CPUINFO_ARM_LINUX_FEATURE2_BF16 UINT32_C(0x00004000) + #define CPUINFO_ARM_LINUX_FEATURE2_DGH UINT32_C(0x00008000) + #define CPUINFO_ARM_LINUX_FEATURE2_RNG UINT32_C(0x00010000) + #define CPUINFO_ARM_LINUX_FEATURE2_BTI UINT32_C(0x00020000) +#endif + +#define CPUINFO_ARM_LINUX_VALID_ARCHITECTURE UINT32_C(0x00010000) +#define CPUINFO_ARM_LINUX_VALID_IMPLEMENTER UINT32_C(0x00020000) +#define CPUINFO_ARM_LINUX_VALID_VARIANT UINT32_C(0x00040000) +#define CPUINFO_ARM_LINUX_VALID_PART UINT32_C(0x00080000) +#define CPUINFO_ARM_LINUX_VALID_REVISION UINT32_C(0x00100000) +#define CPUINFO_ARM_LINUX_VALID_PROCESSOR UINT32_C(0x00200000) +#define CPUINFO_ARM_LINUX_VALID_FEATURES UINT32_C(0x00400000) +#if CPUINFO_ARCH_ARM + #define CPUINFO_ARM_LINUX_VALID_ICACHE_SIZE UINT32_C(0x01000000) + #define CPUINFO_ARM_LINUX_VALID_ICACHE_SETS UINT32_C(0x02000000) + #define CPUINFO_ARM_LINUX_VALID_ICACHE_WAYS UINT32_C(0x04000000) + #define CPUINFO_ARM_LINUX_VALID_ICACHE_LINE UINT32_C(0x08000000) + #define CPUINFO_ARM_LINUX_VALID_DCACHE_SIZE UINT32_C(0x10000000) + #define CPUINFO_ARM_LINUX_VALID_DCACHE_SETS UINT32_C(0x20000000) + #define CPUINFO_ARM_LINUX_VALID_DCACHE_WAYS UINT32_C(0x40000000) + #define CPUINFO_ARM_LINUX_VALID_DCACHE_LINE UINT32_C(0x80000000) +#endif + +#define CPUINFO_ARM_LINUX_VALID_INFO UINT32_C(0x007F0000) +#define CPUINFO_ARM_LINUX_VALID_MIDR UINT32_C(0x003F0000) +#if CPUINFO_ARCH_ARM + #define CPUINFO_ARM_LINUX_VALID_ICACHE UINT32_C(0x0F000000) + #define CPUINFO_ARM_LINUX_VALID_DCACHE UINT32_C(0xF0000000) + #define CPUINFO_ARM_LINUX_VALID_CACHE_LINE UINT32_C(0x88000000) +#endif + +struct cpuinfo_arm_linux_processor { + uint32_t architecture_version; +#if CPUINFO_ARCH_ARM + uint32_t architecture_flags; + struct cpuinfo_arm_linux_proc_cpuinfo_cache proc_cpuinfo_cache; +#endif + uint32_t features; + uint32_t features2; + /** + * Main ID Register value. + */ + uint32_t midr; + enum cpuinfo_vendor vendor; + enum cpuinfo_uarch uarch; + uint32_t uarch_index; + /** + * ID of the physical package which includes this logical processor. + * The value is parsed from /sys/devices/system/cpu/cpu/topology/physical_package_id + */ + uint32_t package_id; + /** + * Minimum processor ID on the package which includes this logical processor. + * This value can serve as an ID for the cluster of logical processors: it is the + * same for all logical processors on the same package. + */ + uint32_t package_leader_id; + /** + * Number of logical processors in the package. + */ + uint32_t package_processor_count; + /** + * Maximum frequency, in kHZ. + * The value is parsed from /sys/devices/system/cpu/cpu/cpufreq/cpuinfo_max_freq + * If failed to read or parse the file, the value is 0. + */ + uint32_t max_frequency; + /** + * Minimum frequency, in kHZ. + * The value is parsed from /sys/devices/system/cpu/cpu/cpufreq/cpuinfo_min_freq + * If failed to read or parse the file, the value is 0. + */ + uint32_t min_frequency; + /** Linux processor ID */ + uint32_t system_processor_id; + uint32_t flags; +}; + +struct cpuinfo_arm_linux_cluster { + uint32_t processor_id_min; + uint32_t processor_id_max; +}; + +/* Returns true if the two processors do belong to the same cluster */ +static inline bool cpuinfo_arm_linux_processor_equals( + struct cpuinfo_arm_linux_processor processor_i[restrict static 1], + struct cpuinfo_arm_linux_processor processor_j[restrict static 1]) +{ + const uint32_t joint_flags = processor_i->flags & processor_j->flags; + + bool same_max_frequency = false; + if (joint_flags & CPUINFO_LINUX_FLAG_MAX_FREQUENCY) { + if (processor_i->max_frequency != processor_j->max_frequency) { + return false; + } else { + same_max_frequency = true; + } + } + + bool same_min_frequency = false; + if (joint_flags & CPUINFO_LINUX_FLAG_MIN_FREQUENCY) { + if (processor_i->min_frequency != processor_j->min_frequency) { + return false; + } else { + same_min_frequency = true; + } + } + + if ((joint_flags & CPUINFO_ARM_LINUX_VALID_MIDR) == CPUINFO_ARM_LINUX_VALID_MIDR) { + if (processor_i->midr == processor_j->midr) { + if (midr_is_cortex_a53(processor_i->midr)) { + return same_min_frequency & same_max_frequency; + } else { + return true; + } + } + } + + return same_max_frequency && same_min_frequency; +} + +/* Returns true if the two processors certainly don't belong to the same cluster */ +static inline bool cpuinfo_arm_linux_processor_not_equals( + struct cpuinfo_arm_linux_processor processor_i[restrict static 1], + struct cpuinfo_arm_linux_processor processor_j[restrict static 1]) +{ + const uint32_t joint_flags = processor_i->flags & processor_j->flags; + + if (joint_flags & CPUINFO_LINUX_FLAG_MAX_FREQUENCY) { + if (processor_i->max_frequency != processor_j->max_frequency) { + return true; + } + } + + if (joint_flags & CPUINFO_LINUX_FLAG_MIN_FREQUENCY) { + if (processor_i->min_frequency != processor_j->min_frequency) { + return true; + } + } + + if ((joint_flags & CPUINFO_ARM_LINUX_VALID_MIDR) == CPUINFO_ARM_LINUX_VALID_MIDR) { + if (processor_i->midr != processor_j->midr) { + return true; + } + } + + return false; +} + +CPUINFO_INTERNAL bool cpuinfo_arm_linux_parse_proc_cpuinfo( + char hardware[restrict static CPUINFO_HARDWARE_VALUE_MAX], + char revision[restrict static CPUINFO_REVISION_VALUE_MAX], + uint32_t max_processors_count, + struct cpuinfo_arm_linux_processor processors[restrict static max_processors_count]); + +#if CPUINFO_ARCH_ARM + CPUINFO_INTERNAL bool cpuinfo_arm_linux_hwcap_from_getauxval( + uint32_t hwcap[restrict static 1], + uint32_t hwcap2[restrict static 1]); + CPUINFO_INTERNAL bool cpuinfo_arm_linux_hwcap_from_procfs( + uint32_t hwcap[restrict static 1], + uint32_t hwcap2[restrict static 1]); + + CPUINFO_INTERNAL void cpuinfo_arm_linux_decode_isa_from_proc_cpuinfo( + uint32_t features, + uint32_t features2, + uint32_t midr, + uint32_t architecture_version, + uint32_t architecture_flags, + const struct cpuinfo_arm_chipset chipset[restrict static 1], + struct cpuinfo_arm_isa isa[restrict static 1]); +#elif CPUINFO_ARCH_ARM64 + CPUINFO_INTERNAL void cpuinfo_arm_linux_hwcap_from_getauxval( + uint32_t hwcap[restrict static 1], + uint32_t hwcap2[restrict static 1]); + + CPUINFO_INTERNAL void cpuinfo_arm64_linux_decode_isa_from_proc_cpuinfo( + uint32_t features, + uint32_t features2, + uint32_t midr, + const struct cpuinfo_arm_chipset chipset[restrict static 1], + struct cpuinfo_arm_isa isa[restrict static 1]); +#endif + +#ifdef __ANDROID__ + CPUINFO_INTERNAL struct cpuinfo_arm_chipset + cpuinfo_arm_android_decode_chipset( + const struct cpuinfo_android_properties properties[restrict static 1], + uint32_t cores, + uint32_t max_cpu_freq_max); +#else + CPUINFO_INTERNAL struct cpuinfo_arm_chipset + cpuinfo_arm_linux_decode_chipset( + const char hardware[restrict static CPUINFO_HARDWARE_VALUE_MAX], + const char revision[restrict static CPUINFO_REVISION_VALUE_MAX], + uint32_t cores, + uint32_t max_cpu_freq_max); +#endif + +CPUINFO_INTERNAL struct cpuinfo_arm_chipset + cpuinfo_arm_linux_decode_chipset_from_proc_cpuinfo_hardware( + const char proc_cpuinfo_hardware[restrict static CPUINFO_HARDWARE_VALUE_MAX], + uint32_t cores, uint32_t max_cpu_freq_max, bool is_tegra); + +#ifdef __ANDROID__ + CPUINFO_INTERNAL struct cpuinfo_arm_chipset + cpuinfo_arm_android_decode_chipset_from_ro_product_board( + const char ro_product_board[restrict static CPUINFO_BUILD_PROP_VALUE_MAX], + uint32_t cores, uint32_t max_cpu_freq_max); + CPUINFO_INTERNAL struct cpuinfo_arm_chipset + cpuinfo_arm_android_decode_chipset_from_ro_board_platform( + const char ro_board_platform[restrict static CPUINFO_BUILD_PROP_VALUE_MAX], + uint32_t cores, uint32_t max_cpu_freq_max); + CPUINFO_INTERNAL struct cpuinfo_arm_chipset + cpuinfo_arm_android_decode_chipset_from_ro_mediatek_platform( + const char ro_mediatek_platform[restrict static CPUINFO_BUILD_PROP_VALUE_MAX]); + CPUINFO_INTERNAL struct cpuinfo_arm_chipset + cpuinfo_arm_android_decode_chipset_from_ro_arch( + const char ro_arch[restrict static CPUINFO_BUILD_PROP_VALUE_MAX]); + CPUINFO_INTERNAL struct cpuinfo_arm_chipset + cpuinfo_arm_android_decode_chipset_from_ro_chipname( + const char ro_chipname[restrict static CPUINFO_BUILD_PROP_VALUE_MAX]); + CPUINFO_INTERNAL struct cpuinfo_arm_chipset + cpuinfo_arm_android_decode_chipset_from_ro_hardware_chipname( + const char ro_hardware_chipname[restrict static CPUINFO_BUILD_PROP_VALUE_MAX]); +#else + CPUINFO_INTERNAL struct cpuinfo_arm_chipset + cpuinfo_arm_linux_decode_chipset_from_proc_cpuinfo_revision( + const char proc_cpuinfo_revision[restrict static CPUINFO_REVISION_VALUE_MAX]); +#endif + +CPUINFO_INTERNAL bool cpuinfo_arm_linux_detect_core_clusters_by_heuristic( + uint32_t usable_processors, + uint32_t max_processors, + struct cpuinfo_arm_linux_processor processors[restrict static max_processors]); + +CPUINFO_INTERNAL void cpuinfo_arm_linux_detect_core_clusters_by_sequential_scan( + uint32_t max_processors, + struct cpuinfo_arm_linux_processor processors[restrict static max_processors]); + +CPUINFO_INTERNAL void cpuinfo_arm_linux_count_cluster_processors( + uint32_t max_processors, + struct cpuinfo_arm_linux_processor processors[restrict static max_processors]); + +CPUINFO_INTERNAL uint32_t cpuinfo_arm_linux_detect_cluster_midr( + const struct cpuinfo_arm_chipset chipset[restrict static 1], + uint32_t max_processors, + uint32_t usable_processors, + struct cpuinfo_arm_linux_processor processors[restrict static max_processors]); + +extern CPUINFO_INTERNAL const uint32_t* cpuinfo_linux_cpu_to_uarch_index_map; +extern CPUINFO_INTERNAL uint32_t cpuinfo_linux_cpu_to_uarch_index_map_entries; diff --git a/3rdparty/cpuinfo/src/arm/linux/chipset.c b/3rdparty/cpuinfo/src/arm/linux/chipset.c new file mode 100644 index 000000000..f369fba68 --- /dev/null +++ b/3rdparty/cpuinfo/src/arm/linux/chipset.c @@ -0,0 +1,3937 @@ +#include +#include +#include +#include + +#include +#ifdef __ANDROID__ + #include +#endif +#include +#include + + +static inline bool is_ascii_whitespace(char c) { + switch (c) { + case ' ': + case '\t': + case '\r': + case '\n': + return true; + default: + return false; + } +} + +static inline bool is_ascii_alphabetic(char c) { + const char lower_c = c | '\x20'; + return (uint8_t) (lower_c - 'a') <= (uint8_t) ('z' - 'a'); +} + +static inline bool is_ascii_alphabetic_uppercase(char c) { + return (uint8_t) (c - 'A') <= (uint8_t) ('Z' - 'A'); +} + +static inline bool is_ascii_numeric(char c) { + return (uint8_t) (c - '0') < 10; +} + +static inline uint16_t load_u16le(const void* ptr) { +#if defined(__ARM_ARCH_7A__) || defined(__aarch64__) + return *((const uint16_t*) ptr); +#else + const uint8_t* byte_ptr = (const uint8_t*) ptr; + return ((uint16_t) byte_ptr[1] << 8) | (uint16_t) byte_ptr[0]; +#endif +} + +static inline uint32_t load_u24le(const void* ptr) { +#if defined(__ARM_ARCH_7A__) || defined(__aarch64__) + return ((uint32_t) ((const uint8_t*) ptr)[2] << 16) | ((uint32_t) *((const uint16_t*) ptr)); +#else + const uint8_t* byte_ptr = (const uint8_t*) ptr; + return ((uint32_t) byte_ptr[2] << 16) | ((uint32_t) byte_ptr[1] << 8) | (uint32_t) byte_ptr[0]; +#endif +} + +static inline uint32_t load_u32le(const void* ptr) { +#if defined(__ARM_ARCH_7A__) || defined(__aarch64__) + return *((const uint32_t*) ptr); +#else + return ((uint32_t) ((const uint8_t*) ptr)[3] << 24) | load_u24le(ptr); +#endif +} + +/* + * Map from ARM chipset series ID to ARM chipset vendor ID. + * This map is used to avoid storing vendor IDs in tables. + */ +static enum cpuinfo_arm_chipset_vendor chipset_series_vendor[cpuinfo_arm_chipset_series_max] = { + [cpuinfo_arm_chipset_series_unknown] = cpuinfo_arm_chipset_vendor_unknown, + [cpuinfo_arm_chipset_series_qualcomm_qsd] = cpuinfo_arm_chipset_vendor_qualcomm, + [cpuinfo_arm_chipset_series_qualcomm_msm] = cpuinfo_arm_chipset_vendor_qualcomm, + [cpuinfo_arm_chipset_series_qualcomm_apq] = cpuinfo_arm_chipset_vendor_qualcomm, + [cpuinfo_arm_chipset_series_qualcomm_snapdragon] = cpuinfo_arm_chipset_vendor_qualcomm, + [cpuinfo_arm_chipset_series_mediatek_mt] = cpuinfo_arm_chipset_vendor_mediatek, + [cpuinfo_arm_chipset_series_samsung_exynos] = cpuinfo_arm_chipset_vendor_samsung, + [cpuinfo_arm_chipset_series_hisilicon_k3v] = cpuinfo_arm_chipset_vendor_hisilicon, + [cpuinfo_arm_chipset_series_hisilicon_hi] = cpuinfo_arm_chipset_vendor_hisilicon, + [cpuinfo_arm_chipset_series_hisilicon_kirin] = cpuinfo_arm_chipset_vendor_hisilicon, + [cpuinfo_arm_chipset_series_actions_atm] = cpuinfo_arm_chipset_vendor_actions, + [cpuinfo_arm_chipset_series_allwinner_a] = cpuinfo_arm_chipset_vendor_allwinner, + [cpuinfo_arm_chipset_series_amlogic_aml] = cpuinfo_arm_chipset_vendor_amlogic, + [cpuinfo_arm_chipset_series_amlogic_s] = cpuinfo_arm_chipset_vendor_amlogic, + [cpuinfo_arm_chipset_series_broadcom_bcm] = cpuinfo_arm_chipset_vendor_broadcom, + [cpuinfo_arm_chipset_series_lg_nuclun] = cpuinfo_arm_chipset_vendor_lg, + [cpuinfo_arm_chipset_series_leadcore_lc] = cpuinfo_arm_chipset_vendor_leadcore, + [cpuinfo_arm_chipset_series_marvell_pxa] = cpuinfo_arm_chipset_vendor_marvell, + [cpuinfo_arm_chipset_series_mstar_6a] = cpuinfo_arm_chipset_vendor_mstar, + [cpuinfo_arm_chipset_series_novathor_u] = cpuinfo_arm_chipset_vendor_novathor, + [cpuinfo_arm_chipset_series_nvidia_tegra_t] = cpuinfo_arm_chipset_vendor_nvidia, + [cpuinfo_arm_chipset_series_nvidia_tegra_ap] = cpuinfo_arm_chipset_vendor_nvidia, + [cpuinfo_arm_chipset_series_nvidia_tegra_sl] = cpuinfo_arm_chipset_vendor_nvidia, + [cpuinfo_arm_chipset_series_pinecone_surge_s] = cpuinfo_arm_chipset_vendor_pinecone, + [cpuinfo_arm_chipset_series_renesas_mp] = cpuinfo_arm_chipset_vendor_renesas, + [cpuinfo_arm_chipset_series_rockchip_rk] = cpuinfo_arm_chipset_vendor_rockchip, + [cpuinfo_arm_chipset_series_spreadtrum_sc] = cpuinfo_arm_chipset_vendor_spreadtrum, + [cpuinfo_arm_chipset_series_telechips_tcc] = cpuinfo_arm_chipset_vendor_telechips, + [cpuinfo_arm_chipset_series_texas_instruments_omap] = cpuinfo_arm_chipset_vendor_texas_instruments, + [cpuinfo_arm_chipset_series_wondermedia_wm] = cpuinfo_arm_chipset_vendor_wondermedia, +}; + +/** + * Tries to match /(MSM|APQ)\d{4}([A-Z\-]*)/ signature (case-insensitive) for Qualcomm MSM and APQ chipsets. + * If match successful, extracts model information into \p chipset argument. + * + * @param start - start of the platform identifier (/proc/cpuinfo Hardware string, ro.product.board, ro.board.platform + * or ro.chipname) to match. + * @param end - end of the platform identifier (/proc/cpuinfo Hardware string, ro.product.board, ro.board.platform or + * ro.chipname) to match. + * @param[out] chipset - location where chipset information will be stored upon a successful match. + * + * @returns true if signature matched, false otherwise. + */ +static bool match_msm_apq( + const char* start, const char* end, + struct cpuinfo_arm_chipset chipset[restrict static 1]) +{ + /* Expect at least 7 symbols: 3 symbols "MSM" or "APQ" + 4 digits */ + if (start + 7 > end) { + return false; + } + + /* Check that string starts with "MSM" or "APQ", case-insensitive. + * The first three characters are loaded as 24-bit little endian word, binary ORed with 0x20 to convert to lower + * case, and compared to "MSM" and "APQ" strings as integers. + */ + const uint32_t series_signature = UINT32_C(0x00202020) | load_u24le(start); + enum cpuinfo_arm_chipset_series series; + switch (series_signature) { + case UINT32_C(0x6D736D): /* "msm" = reverse("msm") */ + series = cpuinfo_arm_chipset_series_qualcomm_msm; + break; + case UINT32_C(0x717061): /* "qpa" = reverse("apq") */ + series = cpuinfo_arm_chipset_series_qualcomm_apq; + break; + default: + return false; + } + + /* Sometimes there is a space ' ' following the MSM/APQ series */ + const char* pos = start + 3; + if (*pos == ' ') { + pos++; + + /* Expect at least 4 more symbols (4-digit model number) */ + if (pos + 4 > end) { + return false; + } + } + + /* Validate and parse 4-digit model number */ + uint32_t model = 0; + for (uint32_t i = 0; i < 4; i++) { + const uint32_t digit = (uint32_t) (uint8_t) (*pos++) - '0'; + if (digit >= 10) { + /* Not really a digit */ + return false; + } + model = model * 10 + digit; + } + + /* Suffix is optional, so if we got to this point, parsing is successful. Commit parsed chipset. */ + *chipset = (struct cpuinfo_arm_chipset) { + .vendor = cpuinfo_arm_chipset_vendor_qualcomm, + .series = series, + .model = model, + }; + + /* Parse as many suffix characters as match the pattern [A-Za-z\-] */ + for (uint32_t i = 0; i < CPUINFO_ARM_CHIPSET_SUFFIX_MAX; i++) { + if (pos + i == end) { + break; + } + + const char c = pos[i]; + if (is_ascii_alphabetic(c)) { + /* Matched a letter [A-Za-z] */ + chipset->suffix[i] = c & '\xDF'; + } else if (c == '-') { + /* Matched a dash '-' */ + chipset->suffix[i] = c; + } else { + /* Neither of [A-Za-z\-] */ + break; + } + } + return true; +} + +/** + * Tries to match /SDM\d{3}$/ signature for Qualcomm Snapdragon chipsets. + * If match successful, extracts model information into \p chipset argument. + * + * @param start - start of the /proc/cpuinfo Hardware string to match. + * @param end - end of the /proc/cpuinfo Hardware string to match. + * @param[out] chipset - location where chipset information will be stored upon a successful match. + * + * @returns true if signature matched, false otherwise. + */ +static bool match_sdm( + const char* start, const char* end, + struct cpuinfo_arm_chipset chipset[restrict static 1]) +{ + /* Expect exactly 6 symbols: 3 symbols "SDM" + 3 digits */ + if (start + 6 != end) { + return false; + } + + /* Check that string starts with "SDM". + * The first three characters are loaded and compared as 24-bit little endian word. + */ + const uint32_t expected_sdm = load_u24le(start); + if (expected_sdm != UINT32_C(0x004D4453) /* "MDS" = reverse("SDM") */) { + return false; + } + + /* Validate and parse 3-digit model number */ + uint32_t model = 0; + for (uint32_t i = 3; i < 6; i++) { + const uint32_t digit = (uint32_t) (uint8_t) start[i] - '0'; + if (digit >= 10) { + /* Not really a digit */ + return false; + } + model = model * 10 + digit; + } + + /* Return parsed chipset. */ + *chipset = (struct cpuinfo_arm_chipset) { + .vendor = cpuinfo_arm_chipset_vendor_qualcomm, + .series = cpuinfo_arm_chipset_series_qualcomm_snapdragon, + .model = model, + }; + return true; +} + +/** + * Tries to match /SM\d{4}$/ signature for Qualcomm Snapdragon chipsets. + * If match successful, extracts model information into \p chipset argument. + * + * @param start - start of the /proc/cpuinfo Hardware string to match. + * @param end - end of the /proc/cpuinfo Hardware string to match. + * @param[out] chipset - location where chipset information will be stored upon a successful match. + * + * @returns true if signature matched, false otherwise. + */ +static bool match_sm( + const char* start, const char* end, + struct cpuinfo_arm_chipset chipset[restrict static 1]) +{ + /* Expect exactly 6 symbols: 2 symbols "SM" + 4 digits */ + if (start + 6 != end) { + return false; + } + + /* Check that string starts with "SM". + * The first three characters are loaded and compared as 16-bit little endian word. + */ + const uint32_t expected_sm = load_u16le(start); + if (expected_sm != UINT16_C(0x4D53) /* "MS" = reverse("SM") */) { + return false; + } + + /* Validate and parse 4-digit model number */ + uint32_t model = 0; + for (uint32_t i = 2; i < 6; i++) { + const uint32_t digit = (uint32_t) (uint8_t) start[i] - '0'; + if (digit >= 10) { + /* Not really a digit */ + return false; + } + model = model * 10 + digit; + } + + /* Return parsed chipset. */ + *chipset = (struct cpuinfo_arm_chipset) { + .vendor = cpuinfo_arm_chipset_vendor_qualcomm, + .series = cpuinfo_arm_chipset_series_qualcomm_snapdragon, + .model = model, + }; + return true; +} + + +struct special_map_entry { + const char* platform; + uint16_t model; + uint8_t series; + char suffix; +}; + +static const struct special_map_entry qualcomm_hardware_map_entries[] = { + { + /* "Kona" -> Qualcomm Kona */ + .platform = "Kona", + .series = cpuinfo_arm_chipset_series_qualcomm_snapdragon, + .model = 865, + }, + { + /* "Bengal" -> Qualcomm Bengal */ + .platform = "Bengal", + .series = cpuinfo_arm_chipset_series_qualcomm_snapdragon, + .model = 662, + }, + { + /* "Bengalp" -> Qualcomm Bengalp */ + .platform = "Bengalp", + .series = cpuinfo_arm_chipset_series_qualcomm_snapdragon, + .model = 662, + }, + { + /* "Lito" -> Qualcomm Lito */ + .platform = "Lito", + .series = cpuinfo_arm_chipset_series_qualcomm_snapdragon, + .model = 765, + .suffix = 'G' + }, + { + /* "Lagoon" -> Qualcomm Lagoon */ + .platform = "Lagoon", + .series = cpuinfo_arm_chipset_series_qualcomm_snapdragon, + .model = 0, + }, +}; + + +int strcicmp(char const *a, char const *b) +{ + for (;; a++, b++) { + int d = tolower((unsigned char)*a) - tolower((unsigned char)*b); + if (d != 0 || !*a) + return d; + } +} + +static bool match_qualcomm_special( + const char* start, const char* end, + struct cpuinfo_arm_chipset chipset[restrict static 1]) +{ + for (size_t i = 0; i < CPUINFO_COUNT_OF(qualcomm_hardware_map_entries); i++) { + int length = end - start; + if (strcicmp(qualcomm_hardware_map_entries[i].platform, start) == 0 && + qualcomm_hardware_map_entries[i].platform[length] == 0) + { + *chipset = (struct cpuinfo_arm_chipset) { + .vendor = chipset_series_vendor[qualcomm_hardware_map_entries[i].series], + .series = (enum cpuinfo_arm_chipset_series) qualcomm_hardware_map_entries[i].series, + .model = qualcomm_hardware_map_entries[i].model, + .suffix = { + [0] = qualcomm_hardware_map_entries[i].suffix, + }, + }; + return true; + } + } + return false; + +} + +/** + * Tries to match /Samsung Exynos\d{4}$/ signature (case-insensitive) for Samsung Exynos chipsets. + * If match successful, extracts model information into \p chipset argument. + * + * @param start - start of the /proc/cpuinfo Hardware string to match. + * @param end - end of the /proc/cpuinfo Hardware string to match. + * @param[out] chipset - location where chipset information will be stored upon a successful match. + * + * @returns true if signature matched, false otherwise. + */ +static bool match_samsung_exynos( + const char* start, const char* end, + struct cpuinfo_arm_chipset chipset[restrict static 1]) +{ + /* + * Expect at 18-19 symbols: + * - "Samsung" (7 symbols) + space + "Exynos" (6 symbols) + optional space 4-digit model number + */ + const size_t length = end - start; + switch (length) { + case 18: + case 19: + break; + default: + return false; + } + + /* + * Check that the string starts with "samsung exynos", case-insensitive. + * Blocks of 4 characters are loaded and compared as little-endian 32-bit word. + * Case-insensitive characters are binary ORed with 0x20 to convert them to lowercase. + */ + const uint32_t expected_sams = UINT32_C(0x20202000) | load_u32le(start); + if (expected_sams != UINT32_C(0x736D6153) /* "smaS" = reverse("Sams") */) { + return false; + } + const uint32_t expected_ung = UINT32_C(0x00202020) | load_u32le(start + 4); + if (expected_ung != UINT32_C(0x20676E75) /* " ung" = reverse("ung ") */) { + return false; + } + const uint32_t expected_exyn = UINT32_C(0x20202000) | load_u32le(start + 8); + if (expected_exyn != UINT32_C(0x6E797845) /* "nyxE" = reverse("Exyn") */) { + return false; + } + const uint16_t expected_os = UINT16_C(0x2020) | load_u16le(start + 12); + if (expected_os != UINT16_C(0x736F) /* "so" = reverse("os") */) { + return false; + } + + const char* pos = start + 14; + + /* There can be a space ' ' following the "Exynos" string */ + if (*pos == ' ') { + pos++; + + /* If optional space if present, we expect exactly 19 characters */ + if (length != 19) { + return false; + } + } + + /* Validate and parse 4-digit model number */ + uint32_t model = 0; + for (uint32_t i = 0; i < 4; i++) { + const uint32_t digit = (uint32_t) (uint8_t) (*pos++) - '0'; + if (digit >= 10) { + /* Not really a digit */ + return false; + } + model = model * 10 + digit; + } + + /* Return parsed chipset */ + *chipset = (struct cpuinfo_arm_chipset) { + .vendor = cpuinfo_arm_chipset_vendor_samsung, + .series = cpuinfo_arm_chipset_series_samsung_exynos, + .model = model, + }; + return true; +} + +/** + * Tries to match /exynos\d{4}$/ signature for Samsung Exynos chipsets. + * If match successful, extracts model information into \p chipset argument. + * + * @param start - start of the platform identifier (ro.board.platform or ro.chipname) to match. + * @param end - end of the platform identifier (ro.board.platform or ro.chipname) to match. + * @param[out] chipset - location where chipset information will be stored upon a successful match. + * + * @returns true if signature matched, false otherwise. + */ +static bool match_exynos( + const char* start, const char* end, + struct cpuinfo_arm_chipset chipset[restrict static 1]) +{ + /* Expect exactly 10 symbols: "exynos" (6 symbols) + 4-digit model number */ + if (start + 10 != end) { + return false; + } + + /* Load first 4 bytes as little endian 32-bit word */ + const uint32_t expected_exyn = load_u32le(start); + if (expected_exyn != UINT32_C(0x6E797865) /* "nyxe" = reverse("exyn") */ ) { + return false; + } + + /* Load next 2 bytes as little endian 16-bit word */ + const uint16_t expected_os = load_u16le(start + 4); + if (expected_os != UINT16_C(0x736F) /* "so" = reverse("os") */ ) { + return false; + } + + /* Check and parse 4-digit model number */ + uint32_t model = 0; + for (uint32_t i = 6; i < 10; i++) { + const uint32_t digit = (uint32_t) (uint8_t) start[i] - '0'; + if (digit >= 10) { + /* Not really a digit */ + return false; + } + model = model * 10 + digit; + } + + /* Return parsed chipset. */ + *chipset = (struct cpuinfo_arm_chipset) { + .vendor = cpuinfo_arm_chipset_vendor_samsung, + .series = cpuinfo_arm_chipset_series_samsung_exynos, + .model = model, + }; + return true; +} + +/** + * Tries to match /universal\d{4}$/ signature for Samsung Exynos chipsets. + * If match successful, extracts model information into \p chipset argument. + * + * @param start - start of the platform identifier (/proc/cpuinfo Hardware string, ro.product.board or ro.chipname) + * to match. + * @param end - end of the platform identifier (/proc/cpuinfo Hardware string, ro.product.board or ro.chipname) + * to match. + * @param[out] chipset - location where chipset information will be stored upon a successful match. + * + * @returns true if signature matched, false otherwise. + */ +static bool match_universal( + const char* start, const char* end, + struct cpuinfo_arm_chipset chipset[restrict static 1]) +{ + /* Expect exactly 13 symbols: "universal" (9 symbols) + 4-digit model number */ + if (start + 13 != end) { + return false; + } + + /* + * Check that the string starts with "universal". + * Blocks of 4 characters are loaded and compared as little-endian 32-bit word. + * Case-insensitive characters are binary ORed with 0x20 to convert them to lowercase. + */ + const uint8_t expected_u = UINT8_C(0x20) | (uint8_t) start[0]; + if (expected_u != UINT8_C(0x75) /* "u" */) { + return false; + } + const uint32_t expected_nive = UINT32_C(0x20202020) | load_u32le(start + 1); + if (expected_nive != UINT32_C(0x6576696E) /* "evin" = reverse("nive") */ ) { + return false; + } + const uint32_t expected_ersa = UINT32_C(0x20202020) | load_u32le(start + 5); + if (expected_ersa != UINT32_C(0x6C617372) /* "lasr" = reverse("rsal") */) { + return false; + } + + /* Validate and parse 4-digit model number */ + uint32_t model = 0; + for (uint32_t i = 9; i < 13; i++) { + const uint32_t digit = (uint32_t) (uint8_t) start[i] - '0'; + if (digit >= 10) { + /* Not really a digit */ + return false; + } + model = model * 10 + digit; + } + + /* Return parsed chipset. */ + *chipset = (struct cpuinfo_arm_chipset) { + .vendor = cpuinfo_arm_chipset_vendor_samsung, + .series = cpuinfo_arm_chipset_series_samsung_exynos, + .model = model, + }; + return true; +} + +/** + * Compares, case insensitively, a string to known values "SMDK4210" and "SMDK4x12" for Samsung Exynos chipsets. + * If platform identifier matches one of the SMDK* values, extracts model information into \p chipset argument. + * For "SMDK4x12" match, decodes the chipset name using number of cores. + * + * @param start - start of the platform identifier (/proc/cpuinfo Hardware string or ro.product.board) to match. + * @param end - end of the platform identifier (/proc/cpuinfo Hardware string or ro.product.board) to match. + * @param cores - number of cores in the chipset. + * @param[out] chipset - location where chipset information will be stored upon a successful match. + * + * @returns true if signature matched, false otherwise. + */ +static bool match_and_parse_smdk( + const char* start, const char* end, uint32_t cores, + struct cpuinfo_arm_chipset chipset[restrict static 1]) +{ + /* Expect exactly 8 symbols: "SMDK" (4 symbols) + 4-digit model number */ + if (start + 8 != end) { + return false; + } + + /* + * Check that string starts with "MT" (case-insensitive). + * The first four characters are loaded as a 32-bit little endian word and converted to lowercase. + */ + const uint32_t expected_smdk = UINT32_C(0x20202020) | load_u32le(start); + if (expected_smdk != UINT32_C(0x6B646D73) /* "kdms" = reverse("smdk") */) { + return false; + } + + /* + * Check that string ends with "4210" or "4x12". + * The last four characters are loaded and compared as a 32-bit little endian word. + */ + uint32_t model = 0; + const uint32_t expected_model = load_u32le(start + 4); + switch (expected_model) { + case UINT32_C(0x30313234): /* "0124" = reverse("4210") */ + model = 4210; + break; + case UINT32_C(0x32317834): /* "21x4" = reverse("4x12") */ + switch (cores) { + case 2: + model = 4212; + break; + case 4: + model = 4412; + break; + default: + cpuinfo_log_warning("system reported invalid %"PRIu32"-core Exynos 4x12 chipset", cores); + } + } + + if (model == 0) { + return false; + } + + *chipset = (struct cpuinfo_arm_chipset) { + .vendor = cpuinfo_arm_chipset_vendor_samsung, + .series = cpuinfo_arm_chipset_series_samsung_exynos, + .model = model, + }; + return true; +} + +/** + * Tries to match /MTK?\d{4}[A-Z/]*$/ signature for MediaTek MT chipsets. + * If match successful, extracts model information into \p chipset argument. + * + * @param start - start of the platform identifier (/proc/cpuinfo Hardware string, ro.product.board, ro.board.platform, + * ro.mediatek.platform, or ro.chipname) to match. + * @param end - end of the platform identifier (/proc/cpuinfo Hardware string, ro.product.board, ro.board.platform, + * ro.mediatek.platform, or ro.chipname) to match. + * @param match_end - indicates if the function should attempt to match through the end of the string and fail if there + * are unparsed characters in the end, or match only MTK signature, model number, and some of the + * suffix characters (the ones that pass validation). + * @param[out] chipset - location where chipset information will be stored upon a successful match. + * + * @returns true if signature matched, false otherwise. + */ +static bool match_mt( + const char* start, const char* end, bool match_end, + struct cpuinfo_arm_chipset chipset[restrict static 1]) +{ + /* Expect at least 6 symbols: "MT" (2 symbols) + 4-digit model number */ + if (start + 6 > end) { + return false; + } + + /* + * Check that string starts with "MT" (case-insensitive). + * The first two characters are loaded as 16-bit little endian word and converted to lowercase. + */ + const uint16_t mt = UINT16_C(0x2020) | load_u16le(start); + if (mt != UINT16_C(0x746D) /* "tm" */) { + return false; + } + + + /* Some images report "MTK" rather than "MT" */ + const char* pos = start + 2; + if (((uint8_t) *pos | UINT8_C(0x20)) == (uint8_t) 'k') { + pos++; + + /* Expect 4 more symbols after "MTK" (4-digit model number) */ + if (pos + 4 > end) { + return false; + } + } + + /* Validate and parse 4-digit model number */ + uint32_t model = 0; + for (uint32_t i = 0; i < 4; i++) { + const uint32_t digit = (uint32_t) (uint8_t) (*pos++) - '0'; + if (digit >= 10) { + /* Not really a digit */ + return false; + } + model = model * 10 + digit; + } + + /* Record parsed chipset. This implicitly zeroes-out suffix, which will be parsed later. */ + *chipset = (struct cpuinfo_arm_chipset) { + .vendor = cpuinfo_arm_chipset_vendor_mediatek, + .series = cpuinfo_arm_chipset_series_mediatek_mt, + .model = model, + }; + + if (match_end) { + /* Check that the potential suffix does not exceed maximum length */ + const size_t suffix_length = end - pos; + if (suffix_length > CPUINFO_ARM_CHIPSET_SUFFIX_MAX) { + return false; + } + + /* Validate suffix characters and copy them to chipset structure */ + for (size_t i = 0; i < suffix_length; i++) { + const char c = (*pos++); + if (is_ascii_alphabetic(c)) { + /* Matched a letter [A-Za-z], convert to uppercase */ + chipset->suffix[i] = c & '\xDF'; + } else if (c == '/') { + /* Matched a slash '/' */ + chipset->suffix[i] = c; + } else { + /* Invalid suffix character (neither of [A-Za-z/]) */ + return false; + } + } + } else { + /* Validate and parse as many suffix characters as we can */ + for (size_t i = 0; i < CPUINFO_ARM_CHIPSET_SUFFIX_MAX; i++) { + if (pos + i == end) { + break; + } + + const char c = pos[i]; + if (is_ascii_alphabetic(c)) { + /* Matched a letter [A-Za-z], convert to uppercase */ + chipset->suffix[i] = c & '\xDF'; + } else if (c == '/') { + /* Matched a slash '/' */ + chipset->suffix[i] = c; + } else { + /* Invalid suffix character (neither of [A-Za-z/]). This marks the end of the suffix. */ + break; + } + } + } + /* All suffix characters successfully validated and copied to chipset data */ + return true; +} + +/** + * Tries to match /[Kk]irin\s?\d{3}$/ signature for HiSilicon Kirin chipsets. + * If match successful, extracts model information into \p chipset argument. + * + * @param start - start of the /proc/cpuinfo Hardware string to match. + * @param end - end of the /proc/cpuinfo Hardware string to match. + * @param[out] chipset - location where chipset information will be stored upon a successful match. + * + * @returns true if signature matched, false otherwise. + */ +static bool match_kirin( + const char* start, const char* end, + struct cpuinfo_arm_chipset chipset[restrict static 1]) +{ + /* Expect 8-9 symbols: "Kirin" (5 symbols) + optional whitespace (1 symbol) + 3-digit model number */ + const size_t length = end - start; + switch (length) { + case 8: + case 9: + break; + default: + return false; + } + + /* Check that the string starts with "Kirin" or "kirin". */ + if (((uint8_t) start[0] | UINT8_C(0x20)) != (uint8_t) 'k') { + return false; + } + /* Symbols 1-5 are loaded and compared as little-endian 32-bit word. */ + const uint32_t irin = load_u32le(start + 1); + if (irin != UINT32_C(0x6E697269) /* "niri" = reverse("irin") */) { + return false; + } + + /* Check for optional whitespace after "Kirin" */ + if (is_ascii_whitespace(start[5])) { + /* When whitespace is present after "Kirin", expect 9 symbols total */ + if (length != 9) { + return false; + } + } + + /* Validate and parse 3-digit model number */ + uint32_t model = 0; + for (int32_t i = 0; i < 3; i++) { + const uint32_t digit = (uint32_t) (uint8_t) end[i - 3] - '0'; + if (digit >= 10) { + /* Not really a digit */ + return false; + } + model = model * 10 + digit; + } + + /* + * Thats it, return parsed chipset. + * Technically, Kirin 910T has a suffix, but it never appears in the form of "910T" string. + * Instead, Kirin 910T devices report "hi6620oem" string (handled outside of this function). + */ + *chipset = (struct cpuinfo_arm_chipset) { + .vendor = cpuinfo_arm_chipset_vendor_hisilicon, + .series = cpuinfo_arm_chipset_series_hisilicon_kirin, + .model = model, + }; + return true; +} + +/** + * Tries to match /rk\d{4}[a-z]?$/ signature for Rockchip RK chipsets. + * If match successful, extracts model information into \p chipset argument. + * + * @param start - start of the platform identifier (/proc/cpuinfo Hardware string or ro.board.platform) to match. + * @param end - end of the platform identifier (/proc/cpuinfo Hardware string or ro.board.platform) to match. + * @param[out] chipset - location where chipset information will be stored upon a successful match. + * + * @returns true if signature matched, false otherwise. + */ +static bool match_rk( + const char* start, const char* end, + struct cpuinfo_arm_chipset chipset[restrict static 1]) +{ + /* Expect 6-7 symbols: "RK" (2 symbols) + 4-digit model number + optional 1-letter suffix */ + const size_t length = end - start; + switch (length) { + case 6: + case 7: + break; + default: + return false; + } + + /* + * Check that string starts with "RK" (case-insensitive). + * The first two characters are loaded as 16-bit little endian word and converted to lowercase. + */ + const uint16_t expected_rk = UINT16_C(0x2020) | load_u16le(start); + if (expected_rk != UINT16_C(0x6B72) /* "kr" = reverse("rk") */) { + return false; + } + + /* Validate and parse 4-digit model number */ + uint32_t model = 0; + for (uint32_t i = 2; i < 6; i++) { + const uint32_t digit = (uint32_t) (uint8_t) start[i] - '0'; + if (digit >= 10) { + /* Not really a digit */ + return false; + } + model = model * 10 + digit; + } + + /* Parse optional suffix */ + char suffix = 0; + if (length == 7) { + /* Parse the suffix letter */ + const char c = start[6]; + if (is_ascii_alphabetic(c)) { + /* Convert to upper case */ + suffix = c & '\xDF'; + } else { + /* Invalid suffix character */ + return false; + } + } + + /* Return parsed chipset */ + *chipset = (struct cpuinfo_arm_chipset) { + .vendor = cpuinfo_arm_chipset_vendor_rockchip, + .series = cpuinfo_arm_chipset_series_rockchip_rk, + .model = model, + .suffix = { + [0] = suffix, + }, + }; + return true; +} + +/** + * Tries to match, case-insentitively, /s[cp]\d{4}[a-z]*|scx15$/ signature for Spreadtrum SC chipsets. + * If match successful, extracts model information into \p chipset argument. + * + * @param start - start of the platform identifier (/proc/cpuinfo Hardware string, ro.product.board, + * ro.board.platform, or ro.chipname) to match. + * @param end - end of the platform identifier (/proc/cpuinfo Hardware string, ro.product.board, + * ro.board.platform, or ro.chipname) to match. + * @param[out] chipset - location where chipset information will be stored upon a successful match. + * + * @returns true if signature matched, false otherwise. + */ +static bool match_sc( + const char* start, const char* end, + struct cpuinfo_arm_chipset chipset[restrict static 1]) +{ + /* Expect at least 5 symbols: "scx15" */ + if (start + 5 > end) { + return false; + } + + /* + * Check that string starts with "S[CP]" (case-insensitive). + * The first two characters are loaded as 16-bit little endian word and converted to lowercase. + */ + const uint16_t expected_sc_or_sp = UINT16_C(0x2020) | load_u16le(start); + switch (expected_sc_or_sp) { + case UINT16_C(0x6373): /* "cs" = reverse("sc") */ + case UINT16_C(0x7073): /* "ps" = reverse("sp") */ + break; + default: + return false; + } + + /* Special case: "scx" prefix (SC7715 reported as "scx15") */ + if ((start[2] | '\x20') == 'x') { + /* Expect exactly 5 characters: "scx15" */ + if (start + 5 != end) { + return false; + } + + /* Check that string ends with "15" */ + const uint16_t expected_15 = load_u16le(start + 3); + if (expected_15 != UINT16_C(0x3531) /* "51" = reverse("15") */ ) { + return false; + } + + *chipset = (struct cpuinfo_arm_chipset) { + .vendor = cpuinfo_arm_chipset_vendor_spreadtrum, + .series = cpuinfo_arm_chipset_series_spreadtrum_sc, + .model = 7715, + }; + return true; + } + + /* Expect at least 6 symbols: "S[CP]" (2 symbols) + 4-digit model number */ + if (start + 6 > end) { + return false; + } + + /* Validate and parse 4-digit model number */ + uint32_t model = 0; + for (uint32_t i = 2; i < 6; i++) { + const uint32_t digit = (uint32_t) (uint8_t) start[i] - '0'; + if (digit >= 10) { + /* Not really a digit */ + return false; + } + model = model * 10 + digit; + } + + /* Write parsed chipset */ + *chipset = (struct cpuinfo_arm_chipset) { + .vendor = cpuinfo_arm_chipset_vendor_spreadtrum, + .series = cpuinfo_arm_chipset_series_spreadtrum_sc, + .model = model, + }; + + /* Validate and copy suffix letters. If suffix is too long, truncate at CPUINFO_ARM_CHIPSET_SUFFIX_MAX letters. */ + const char* suffix = start + 6; + for (size_t i = 0; i < CPUINFO_ARM_CHIPSET_SUFFIX_MAX; i++) { + if (suffix + i == end) { + break; + } + + const char c = suffix[i]; + if (!is_ascii_alphabetic(c)) { + /* Invalid suffix character */ + return false; + } + /* Convert suffix letter to uppercase */ + chipset->suffix[i] = c & '\xDF'; + } + return true; +} + +/** + * Tries to match /lc\d{4}[a-z]?$/ signature for Leadcore LC chipsets. + * If match successful, extracts model information into \p chipset argument. + * + * @param start - start of the platform identifier (ro.product.board or ro.board.platform) to match. + * @param end - end of the platform identifier (ro.product.board or ro.board.platform) to match. + * @param[out] chipset - location where chipset information will be stored upon a successful match. + * + * @returns true if signature matched, false otherwise. + */ +static bool match_lc( + const char* start, const char* end, + struct cpuinfo_arm_chipset chipset[restrict static 1]) +{ + /* Expect at 6-7 symbols: "lc" (2 symbols) + 4-digit model number + optional 1-letter suffix */ + const size_t length = end - start; + switch (length) { + case 6: + case 7: + break; + default: + return false; + } + + /* Check that string starts with "lc". The first two characters are loaded as 16-bit little endian word */ + const uint16_t expected_lc = load_u16le(start); + if (expected_lc != UINT16_C(0x636C) /* "cl" = reverse("lc") */) { + return false; + } + + /* Validate and parse 4-digit model number */ + uint32_t model = 0; + for (uint32_t i = 2; i < 6; i++) { + const uint32_t digit = (uint32_t) (uint8_t) start[i] - '0'; + if (digit >= 10) { + /* Not really a digit */ + return false; + } + model = model * 10 + digit; + } + + /* Parse optional suffix letter */ + char suffix = 0; + if (length == 7) { + const char c = start[6]; + if (is_ascii_alphabetic(c)) { + /* Convert to uppercase */ + chipset->suffix[0] = c & '\xDF'; + } else { + /* Invalid suffix character */ + return false; + } + } + + /* Return parsed chipset */ + *chipset = (struct cpuinfo_arm_chipset) { + .vendor = cpuinfo_arm_chipset_vendor_leadcore, + .series = cpuinfo_arm_chipset_series_leadcore_lc, + .model = model, + .suffix = { + [0] = suffix, + }, + }; + return true; +} + +/** + * Tries to match /PXA(\d{3,4}|1L88)$/ signature for Marvell PXA chipsets. + * If match successful, extracts model information into \p chipset argument. + * + * @param start - start of the platform identifier (/proc/cpuinfo Hardware string, ro.product.board or ro.chipname) + * to match. + * @param end - end of the platform identifier (/proc/cpuinfo Hardaware string, ro.product.board or ro.chipname) to + * match. + * @param[out] chipset - location where chipset information will be stored upon a successful match. + * + * @returns true if signature matched, false otherwise. + */ +static bool match_pxa( + const char* start, const char* end, + struct cpuinfo_arm_chipset chipset[restrict static 1]) +{ + /* Expect 6-7 symbols: "PXA" (3 symbols) + 3-4 digit model number */ + const size_t length = end - start; + switch (length) { + case 6: + case 7: + break; + default: + return false; + } + + /* Check that the string starts with "PXA". Symbols 1-3 are loaded and compared as little-endian 16-bit word. */ + if (start[0] != 'P') { + return false; + } + const uint16_t expected_xa = load_u16le(start + 1); + if (expected_xa != UINT16_C(0x4158) /* "AX" = reverse("XA") */) { + return false; + } + + uint32_t model = 0; + + + /* Check for a very common typo: "PXA1L88" for "PXA1088" */ + if (length == 7) { + /* Load 4 model "number" symbols as a little endian 32-bit word and compare to "1L88" */ + const uint32_t expected_1L88 = load_u32le(start + 3); + if (expected_1L88 == UINT32_C(0x38384C31) /* "88L1" = reverse("1L88") */) { + model = 1088; + goto write_chipset; + } + } + + /* Check and parse 3-4 digit model number */ + for (uint32_t i = 3; i < length; i++) { + const uint32_t digit = (uint32_t) (uint8_t) start[i] - '0'; + if (digit >= 10) { + /* Not really a digit */ + return false; + } + model = model * 10 + digit; + } + + /* Return parsed chipset. */ +write_chipset: + *chipset = (struct cpuinfo_arm_chipset) { + .vendor = cpuinfo_arm_chipset_vendor_marvell, + .series = cpuinfo_arm_chipset_series_marvell_pxa, + .model = model, + }; + return true; +} + +/** + * Tries to match /BCM\d{4}$/ signature for Broadcom BCM chipsets. + * If match successful, extracts model information into \p chipset argument. + * + * @param start - start of the /proc/cpuinfo Hardware string to match. + * @param end - end of the /proc/cpuinfo Hardware string to match. + * @param[out] chipset - location where chipset information will be stored upon a successful match. + * + * @returns true if signature matched, false otherwise. + */ +static bool match_bcm( + const char* start, const char* end, + struct cpuinfo_arm_chipset chipset[restrict static 1]) +{ + /* Expect exactly 7 symbols: "BCM" (3 symbols) + 4-digit model number */ + if (start + 7 != end) { + return false; + } + + /* Check that the string starts with "BCM". + * The first three characters are loaded and compared as a 24-bit little endian word. + */ + const uint32_t expected_bcm = load_u24le(start); + if (expected_bcm != UINT32_C(0x004D4342) /* "MCB" = reverse("BCM") */) { + return false; + } + + /* Validate and parse 4-digit model number */ + uint32_t model = 0; + for (uint32_t i = 3; i < 7; i++) { + const uint32_t digit = (uint32_t) (uint8_t) start[i] - '0'; + if (digit >= 10) { + /* Not really a digit */ + return false; + } + model = model * 10 + digit; + } + + /* Return parsed chipset. */ + *chipset = (struct cpuinfo_arm_chipset) { + .vendor = cpuinfo_arm_chipset_vendor_broadcom, + .series = cpuinfo_arm_chipset_series_broadcom_bcm, + .model = model, + }; + return true; +} + +/** + * Tries to match /OMAP\d{4}$/ signature for Texas Instruments OMAP chipsets. + * If match successful, extracts model information into \p chipset argument. + * + * @param start - start of the /proc/cpuinfo Hardware string to match. + * @param end - end of the /proc/cpuinfo Hardware string to match. + * @param[out] chipset - location where chipset information will be stored upon a successful match. + * + * @returns true if signature matched, false otherwise. + */ +static bool match_omap( + const char* start, const char* end, + struct cpuinfo_arm_chipset chipset[restrict static 1]) +{ + /* Expect exactly 8 symbols: "OMAP" (4 symbols) + 4-digit model number */ + if (start + 8 != end) { + return false; + } + + /* Check that the string starts with "OMAP". Symbols 0-4 are loaded and compared as little-endian 32-bit word. */ + const uint32_t expected_omap = load_u32le(start); + if (expected_omap != UINT32_C(0x50414D4F) /* "PAMO" = reverse("OMAP") */) { + return false; + } + + /* Validate and parse 4-digit model number */ + uint32_t model = 0; + for (uint32_t i = 4; i < 8; i++) { + const uint32_t digit = (uint32_t) (uint8_t) start[i] - '0'; + if (digit >= 10) { + /* Not really a digit */ + return false; + } + model = model * 10 + digit; + } + + /* Return parsed chipset. */ + *chipset = (struct cpuinfo_arm_chipset) { + .vendor = cpuinfo_arm_chipset_vendor_texas_instruments, + .series = cpuinfo_arm_chipset_series_texas_instruments_omap, + .model = model, + }; + return true; +} + +/** + * Compares platform identifier string to known values for Broadcom chipsets. + * If the string matches one of the known values, the function decodes Broadcom chipset from frequency and number of + * cores into \p chipset argument. + * + * @param start - start of the platform identifier (ro.product.board or ro.board.platform) to match. + * @param end - end of the platform identifier (ro.product.board or ro.board.platform) to match. + * @param cores - number of cores in the chipset. + * @param max_cpu_freq_max - maximum of /sys/devices/system/cpu/cpu/cpofreq/cpu_freq_max values. + * @param[out] chipset - location where chipset information will be stored upon a successful match and decoding. + * + * @returns true if signature matched (even if exact model can't be decoded), false otherwise. + */ +static bool match_and_parse_broadcom( + const char* start, const char* end, uint32_t cores, uint32_t max_cpu_freq_max, + struct cpuinfo_arm_chipset chipset[restrict static 1]) +{ + /* Expect 4-6 symbols: "java" (4 symbols), "rhea" (4 symbols), "capri" (5 symbols), or "hawaii" (6 symbols) */ + const size_t length = end - start; + switch (length) { + case 4: + case 5: + case 6: + break; + default: + return false; + } + + /* + * Compare the platform identifier to known values for Broadcom chipsets: + * - "rhea" + * - "java" + * - "capri" + * - "hawaii" + * Upon a successful match, decode chipset name from frequency and number of cores. + */ + uint32_t model = 0; + char suffix = 0; + const uint32_t expected_platform = load_u32le(start); + switch (expected_platform) { + case UINT32_C(0x61656872): /* "aehr" = reverse("rhea") */ + if (length == 4) { + /* + * Detected "rhea" platform: + * - 1 core @ 849999 KHz -> BCM21654 + * - 1 core @ 999999 KHz -> BCM21654G + */ + if (cores == 1) { + model = 21654; + if (max_cpu_freq_max >= 999999) { + suffix = 'G'; + } + } + } + break; + case UINT32_C(0x6176616A): /* "avaj" = reverse("java") */ + if (length == 4) { + /* + * Detected "java" platform: + * - 4 cores -> BCM23550 + */ + if (cores == 4) { + model = 23550; + } + } + break; + case UINT32_C(0x61776168): /* "awah" = reverse("hawa") */ + if (length == 6) { + /* Check that string equals "hawaii" */ + const uint16_t expected_ii = load_u16le(start + 4); + if (expected_ii == UINT16_C(0x6969) /* "ii" */ ) { + /* + * Detected "hawaii" platform: + * - 1 core -> BCM21663 + * - 2 cores @ 999999 KHz -> BCM21664 + * - 2 cores @ 1200000 KHz -> BCM21664T + */ + switch (cores) { + case 1: + model = 21663; + break; + case 2: + model = 21664; + if (max_cpu_freq_max >= 1200000) { + suffix = 'T'; + } + break; + } + } + } + break; + case UINT32_C(0x72706163): /* "rpac" = reverse("capr") */ + if (length == 5) { + /* Check that string equals "capri" */ + if (start[4] == 'i') { + /* + * Detected "capri" platform: + * - 2 cores -> BCM28155 + */ + if (cores == 2) { + model = 28155; + } + } + } + break; + } + + if (model != 0) { + /* Chipset was successfully decoded */ + *chipset = (struct cpuinfo_arm_chipset) { + .vendor = cpuinfo_arm_chipset_vendor_broadcom, + .series = cpuinfo_arm_chipset_series_broadcom_bcm, + .model = model, + .suffix = { + [0] = suffix, + }, + }; + } + return model != 0; +} + +struct sunxi_map_entry { + uint8_t sunxi; + uint8_t cores; + uint8_t model; + char suffix; +}; + +static const struct sunxi_map_entry sunxi_map_entries[] = { +#if CPUINFO_ARCH_ARM + { + /* ("sun4i", 1) -> "A10" */ + .sunxi = 4, + .cores = 1, + .model = 10, + }, + { + /* ("sun5i", 1) -> "A13" */ + .sunxi = 5, + .cores = 1, + .model = 13, + }, + { + /* ("sun6i", 4) -> "A31" */ + .sunxi = 6, + .cores = 4, + .model = 31, + }, + { + /* ("sun7i", 2) -> "A20" */ + .sunxi = 7, + .cores = 2, + .model = 20, + + }, + { + /* ("sun8i", 2) -> "A23" */ + .sunxi = 8, + .cores = 2, + .model = 23, + }, + { + /* ("sun8i", 4) -> "A33" */ + .sunxi = 8, + .cores = 4, + .model = 33, + }, + { + /* ("sun8i", 8) -> "A83T" */ + .sunxi = 8, + .cores = 8, + .model = 83, + .suffix = 'T', + }, + { + /* ("sun9i", 8) -> "A80" */ + .sunxi = 9, + .cores = 8, + .model = 80, + }, +#endif /* CPUINFO_ARCH_ARM */ + { + /* ("sun50i", 4) -> "A64" */ + .sunxi = 50, + .cores = 4, + .model = 64, + }, +}; + +/** + * Tries to match /proc/cpuinfo Hardware string to Allwinner /sun\d+i/ signature. + * If the string matches signature, the function decodes Allwinner chipset from the number in the signature and the + * number of cores, and stores it in \p chipset argument. + * + * @param start - start of the /proc/cpuinfo Hardware string to match. + * @param end - end of the /proc/cpuinfo Hardware string to match. + * @param cores - number of cores in the chipset. + * @param[out] chipset - location where chipset information will be stored upon a successful match and decoding. + * + * @returns true if signature matched (even if exact model can't be decoded), false otherwise. + */ +static bool match_and_parse_sunxi( + const char* start, const char* end, uint32_t cores, + struct cpuinfo_arm_chipset chipset[restrict static 1]) +{ + /* Expect at least 5 symbols: "sun" (3 symbols) + platform id (1-2 digits) + "i" (1 symbol) */ + if (start + 5 > end) { + return false; + } + + /* Compare the first 3 characters to "sun" */ + if (start[0] != 's') { + return false; + } + const uint16_t expected_un = load_u16le(start + 1); + if (expected_un != UINT16_C(0x6E75) /* "nu" = reverse("un") */) { + return false; + } + + /* Check and parse the first (required) digit of the sunXi platform id */ + uint32_t sunxi_platform = 0; + { + const uint32_t digit = (uint32_t) (uint8_t) start[3] - '0'; + if (digit >= 10) { + /* Not really a digit */ + return false; + } + sunxi_platform = digit; + } + + /* Parse optional second digit of the sunXi platform id */ + const char* pos = start + 4; + { + const uint32_t digit = (uint32_t) (uint8_t) (*pos) - '0'; + if (digit < 10) { + sunxi_platform = sunxi_platform * 10 + digit; + if (++pos == end) { + /* Expected one more character, final 'i' letter */ + return false; + } + } + } + + /* Validate the final 'i' letter */ + if (*pos != 'i') { + return false; + } + + /* Compare sunXi platform id and number of cores to tabulated values to decode chipset name */ + uint32_t model = 0; + char suffix = 0; + for (size_t i = 0; i < CPUINFO_COUNT_OF(sunxi_map_entries); i++) { + if (sunxi_platform == sunxi_map_entries[i].sunxi && cores == sunxi_map_entries[i].cores) { + model = sunxi_map_entries[i].model; + suffix = sunxi_map_entries[i].suffix; + break; + } + } + + if (model == 0) { + cpuinfo_log_info("unrecognized %"PRIu32"-core Allwinner sun%"PRIu32" platform", cores, sunxi_platform); + } + /* Create chipset name from decoded data */ + *chipset = (struct cpuinfo_arm_chipset) { + .vendor = cpuinfo_arm_chipset_vendor_allwinner, + .series = cpuinfo_arm_chipset_series_allwinner_a, + .model = model, + .suffix = { + [0] = suffix, + }, + }; + return true; +} + +/** + * Compares /proc/cpuinfo Hardware string to "WMT" signature. + * If the string matches signature, the function decodes WonderMedia chipset from frequency and number of cores into + * \p chipset argument. + * + * @param start - start of the /proc/cpuinfo Hardware string to match. + * @param end - end of the /proc/cpuinfo Hardware string to match. + * @param cores - number of cores in the chipset. + * @param max_cpu_freq_max - maximum of /sys/devices/system/cpu/cpu/cpofreq/cpu_freq_max values. + * @param[out] chipset - location where chipset information will be stored upon a successful match and decoding. + * + * @returns true if signature matched (even if exact model can't be decoded), false otherwise. + */ +static bool match_and_parse_wmt( + const char* start, const char* end, uint32_t cores, uint32_t max_cpu_freq_max, + struct cpuinfo_arm_chipset chipset[restrict static 1]) +{ + /* Expected 3 symbols: "WMT" */ + if (start + 3 != end) { + return false; + } + + /* Compare string to "WMT" */ + if (start[0] != 'W') { + return false; + } + const uint16_t expected_mt = load_u16le(start + 1); + if (expected_mt != UINT16_C(0x544D) /* "TM" = reverse("MT") */) { + return false; + } + + /* Decode chipset name from frequency and number of cores */ + uint32_t model = 0; + switch (cores) { + case 1: + switch (max_cpu_freq_max) { + case 1008000: + /* 1 core @ 1008000 KHz -> WM8950 */ + model = 8950; + break; + case 1200000: + /* 1 core @ 1200000 KHz -> WM8850 */ + model = 8850; + break; + } + break; + case 2: + if (max_cpu_freq_max == 1500000) { + /* 2 cores @ 1500000 KHz -> WM8880 */ + model = 8880; + } + break; + } + + if (model == 0) { + cpuinfo_log_info("unrecognized WonderMedia platform with %"PRIu32" cores at %"PRIu32" KHz", + cores, max_cpu_freq_max); + } + *chipset = (struct cpuinfo_arm_chipset) { + .vendor = cpuinfo_arm_chipset_vendor_wondermedia, + .series = cpuinfo_arm_chipset_series_wondermedia_wm, + .model = model, + }; + return true; +} + +struct huawei_map_entry { + uint32_t platform; + uint32_t model; +}; + +static const struct huawei_map_entry huawei_platform_map[] = { + { + /* "ALP" -> Kirin 970 */ + .platform = UINT32_C(0x00504C41), /* "\0PLA" = reverse("ALP\0") */ + .model = 970, + }, + { + /* "BAC" -> Kirin 659 */ + .platform = UINT32_C(0x00434142), /* "\0CAB" = reverse("BAC\0") */ + .model = 659, + }, + { + /* "BLA" -> Kirin 970 */ + .platform = UINT32_C(0x00414C42), /* "\0ALB" = reverse("BLA\0") */ + .model = 970, + }, + { + /* "BKL" -> Kirin 970 */ + .platform = UINT32_C(0x004C4B42), /* "\0LKB" = reverse("BKL\0") */ + .model = 970, + }, + { + /* "CLT" -> Kirin 970 */ + .platform = UINT32_C(0x00544C43), /* "\0TLC" = reverse("CLT\0") */ + .model = 970, + }, + { + /* "COL" -> Kirin 970 */ + .platform = UINT32_C(0x004C4F43), /* "\0LOC" = reverse("COL\0") */ + .model = 970, + }, + { + /* "COR" -> Kirin 970 */ + .platform = UINT32_C(0x00524F43), /* "\0ROC" = reverse("COR\0") */ + .model = 970, + }, + { + /* "DUK" -> Kirin 960 */ + .platform = UINT32_C(0x004B5544), /* "\0KUD" = reverse("DUK\0") */ + .model = 960, + }, + { + /* "EML" -> Kirin 970 */ + .platform = UINT32_C(0x004C4D45), /* "\0LME" = reverse("EML\0") */ + .model = 970, + }, + { + /* "EVA" -> Kirin 955 */ + .platform = UINT32_C(0x00415645), /* "\0AVE" = reverse("EVA\0") */ + .model = 955, + }, + { + /* "FRD" -> Kirin 950 */ + .platform = UINT32_C(0x00445246), /* "\0DRF" = reverse("FRD\0") */ + .model = 950, + }, + { + /* "INE" -> Kirin 710 */ + .platform = UINT32_C(0x00454E49), /* "\0ENI" = reverse("INE\0") */ + .model = 710, + }, + { + /* "KNT" -> Kirin 950 */ + .platform = UINT32_C(0x00544E4B), /* "\0TNK" = reverse("KNT\0") */ + .model = 950, + }, + { + /* "LON" -> Kirin 960 */ + .platform = UINT32_C(0x004E4F4C), /* "\0NOL" = reverse("LON\0") */ + .model = 960, + }, + { + /* "LYA" -> Kirin 980 */ + .platform = UINT32_C(0x0041594C), /* "\0AYL" = reverse("LYA\0") */ + .model = 980, + }, + { + /* "MCN" -> Kirin 980 */ + .platform = UINT32_C(0x004E434D), /* "\0NCM" = reverse("MCN\0") */ + .model = 980, + }, + { + /* "MHA" -> Kirin 960 */ + .platform = UINT32_C(0x0041484D), /* "\0AHM" = reverse("MHA\0") */ + .model = 960, + }, + { + /* "NEO" -> Kirin 970 */ + .platform = UINT32_C(0x004F454E), /* "\0OEN" = reverse("NEO\0") */ + .model = 970, + }, + { + /* "NXT" -> Kirin 950 */ + .platform = UINT32_C(0x0054584E), /* "\0TXN" = reverse("NXT\0") */ + .model = 950, + }, + { + /* "PAN" -> Kirin 980 */ + .platform = UINT32_C(0x004E4150), /* "\0NAP" = reverse("PAN\0") */ + .model = 980, + }, + { + /* "PAR" -> Kirin 970 */ + .platform = UINT32_C(0x00524150), /* "\0RAP" = reverse("PAR\0") */ + .model = 970, + }, + { + /* "RVL" -> Kirin 970 */ + .platform = UINT32_C(0x004C5652), /* "\0LVR" = reverse("RVL\0") */ + .model = 970, + }, + { + /* "STF" -> Kirin 960 */ + .platform = UINT32_C(0x00465453), /* "\0FTS" = reverse("STF\0") */ + .model = 960, + }, + { + /* "SUE" -> Kirin 980 */ + .platform = UINT32_C(0x00455553), /* "\0EUS" = reverse("SUE\0") */ + .model = 980, + }, + { + /* "VIE" -> Kirin 955 */ + .platform = UINT32_C(0x00454956), /* "\0EIV" = reverse("VIE\0") */ + .model = 955, + }, + { + /* "VKY" -> Kirin 960 */ + .platform = UINT32_C(0x00594B56), /* "\0YKV" = reverse("VKY\0") */ + .model = 960, + }, + { + /* "VTR" -> Kirin 960 */ + .platform = UINT32_C(0x00525456), /* "\0RTV" = reverse("VTR\0") */ + .model = 960, + }, +}; + +/** + * Tries to match ro.product.board string to Huawei /([A-Z]{3})(\-[A-Z]?L\d{2})$/ signature where \1 is one of the + * known values for Huawei devices, which do not report chipset name elsewhere. + * If the string matches signature, the function decodes chipset (always HiSilicon Kirin for matched devices) from + * the Huawei platform ID in the signature and stores it in \p chipset argument. + * + * @param start - start of the ro.product.board string to match. + * @param end - end of the ro.product.board string to match. + * @param[out] chipset - location where chipset information will be stored upon a successful match and decoding. + * + * @returns true if signature matched, false otherwise. + */ +static bool match_and_parse_huawei( + const char* start, const char* end, + struct cpuinfo_arm_chipset chipset[restrict static 1]) +{ + /* + * Expect length of either 3, 7 or 8, exactly: + * - 3-letter platform identifier (see huawei_platform_map) + * - 3-letter platform identifier + '-' + 'L' + two digits + * - 3-letter platform identifier + '-' + capital letter + 'L' + two digits + */ + const size_t length = end - start; + switch (length) { + case 3: + case 7: + case 8: + break; + default: + return false; + } + + /* + * Try to find the first three-letter substring in among the tabulated entries for Huawei devices. + * The first three letters are loaded and compared as a little-endian 24-bit word. + */ + uint32_t model = 0; + const uint32_t target_platform_id = load_u24le(start); + for (uint32_t i = 0; i < CPUINFO_COUNT_OF(huawei_platform_map); i++) { + if (huawei_platform_map[i].platform == target_platform_id) { + model = huawei_platform_map[i].model; + break; + } + } + + if (model == 0) { + /* Platform does not match the tabulated Huawei entries */ + return false; + } + + if (length > 3) { + /* + * Check that: + * - The symbol after platform id is a dash + * - The symbol after it is an uppercase letter. For 7-symbol strings, the symbol is just 'L'. + */ + if (start[3] != '-' || !is_ascii_alphabetic_uppercase(start[4])) { + return false; + } + + /* Check that the last 3 entries are /L\d\d/ */ + if (end[-3] != 'L' || !is_ascii_numeric(end[-2]) || !is_ascii_numeric(end[-1])) { + return false; + } + } + + /* All checks succeeded, commit chipset name */ + *chipset = (struct cpuinfo_arm_chipset) { + .vendor = cpuinfo_arm_chipset_vendor_hisilicon, + .series = cpuinfo_arm_chipset_series_hisilicon_kirin, + .model = model, + }; + return true; +} + +/** + * Tries to match /tcc\d{3}x$/ signature for Telechips TCCXXXx chipsets. + * If match successful, extracts model information into \p chipset argument. + * + * @param start - start of the /proc/cpuinfo Hardware string to match. + * @param end - end of the /proc/cpuinfo Hardware string to match. + * @param[out] chipset - location where chipset information will be stored upon a successful match. + * + * @returns true if signature matched, false otherwise. + */ +static bool match_tcc( + const char* start, const char* end, + struct cpuinfo_arm_chipset chipset[restrict static 1]) +{ + /* Expect exactly 7 symbols: "tcc" (3 symbols) + 3-digit model number + fixed "x" suffix */ + if (start + 7 != end) { + return false; + } + + /* Quick check for the first character */ + if (start[0] != 't') { + return false; + } + + /* Load the next 2 bytes as little endian 16-bit word */ + const uint16_t expected_cc = load_u16le(start + 1); + if (expected_cc != UINT16_C(0x6363) /* "cc" */ ) { + return false; + } + + /* Check and parse 3-digit model number */ + uint32_t model = 0; + for (uint32_t i = 3; i < 6; i++) { + const uint32_t digit = (uint32_t) (uint8_t) start[i] - '0'; + if (digit >= 10) { + /* Not really a digit */ + return false; + } + model = model * 10 + digit; + } + + /* Check the fixed 'x' suffix in the end */ + if (start[6] != 'x') { + return false; + } + + /* Commit parsed chipset. */ + *chipset = (struct cpuinfo_arm_chipset) { + .vendor = cpuinfo_arm_chipset_vendor_telechips, + .series = cpuinfo_arm_chipset_series_telechips_tcc, + .model = model, + .suffix = { + [0] = 'X' + }, + }; + return true; +} + +/* + * Compares ro.board.platform string to Nvidia Tegra signatures ("tegra" and "tegra3") + * This check has effect on how /proc/cpuinfo Hardware string is interpreted. + * + * @param start - start of the ro.board.platform string to check. + * @param end - end of the ro.board.platform string to check. + * + * @returns true if the string matches an Nvidia Tegra signature, and false otherwise + */ +static bool is_tegra(const char* start, const char* end) { + /* Expect 5 ("tegra") or 6 ("tegra3") symbols */ + const size_t length = end - start; + switch (length) { + case 5: + case 6: + break; + default: + return false; + } + + /* Check that the first 5 characters match "tegra" */ + if (start[0] != 't') { + return false; + } + const uint32_t expected_egra = load_u32le(start + 1); + if (expected_egra != UINT32_C(0x61726765) /* "arge" = reverse("egra") */) { + return false; + } + + /* Check if the string is either "tegra" (length = 5) or "tegra3" (length != 5) and last character is '3' */ + return (length == 5 || start[5] == '3'); +} + +static const struct special_map_entry special_hardware_map_entries[] = { +#if CPUINFO_ARCH_ARM + { + /* "k3v2oem1" -> HiSilicon K3V2 */ + .platform = "k3v2oem1", + .series = cpuinfo_arm_chipset_series_hisilicon_k3v, + .model = 2, + }, + { + /* "hi6620oem" -> HiSilicon Kirin 910T */ + .platform = "hi6620oem", + .series = cpuinfo_arm_chipset_series_hisilicon_kirin, + .model = 910, + .suffix = 'T' + }, +#endif /* CPUINFO_ARCH_ARM */ + { + /* "hi6250" -> HiSilicon Kirin 650 */ + .platform = "hi6250", + .series = cpuinfo_arm_chipset_series_hisilicon_kirin, + .model = 650, + }, + { + /* "hi6210sft" -> HiSilicon Kirin 620 */ + .platform = "hi6210sft", + .series = cpuinfo_arm_chipset_series_hisilicon_kirin, + .model = 620, + }, + { + /* "hi3751" -> HiSilicon Hi3751 */ + .platform = "hi3751", + .series = cpuinfo_arm_chipset_series_hisilicon_hi, + .model = 3751, + }, +#if CPUINFO_ARCH_ARM + { + /* "hi3630" -> HiSilicon Kirin 920 */ + .platform = "hi3630", + .series = cpuinfo_arm_chipset_series_hisilicon_kirin, + .model = 920, + }, +#endif /* CPUINFO_ARCH_ARM */ + { + /* "hi3635" -> HiSilicon Kirin 930 */ + .platform = "hi3635", + .series = cpuinfo_arm_chipset_series_hisilicon_kirin, + .model = 930, + }, +#if CPUINFO_ARCH_ARM + { + /* "gs702a" -> Actions ATM7029 (Cortex-A5 + GC1000) */ + .platform = "gs702a", + .series = cpuinfo_arm_chipset_series_actions_atm, + .model = 7029, + }, + { + /* "gs702c" -> Actions ATM7029B (Cortex-A5 + SGX540) */ + .platform = "gs702c", + .series = cpuinfo_arm_chipset_series_actions_atm, + .model = 7029, + .suffix = 'B', + }, + { + /* "gs703d" -> Actions ATM7039S */ + .platform = "gs703d", + .series = cpuinfo_arm_chipset_series_actions_atm, + .model = 7039, + .suffix = 'S', + }, + { + /* "gs705a" -> Actions ATM7059A */ + .platform = "gs705a", + .series = cpuinfo_arm_chipset_series_actions_atm, + .model = 7059, + .suffix = 'A', + }, + { + /* "Amlogic Meson8" -> Amlogic S812 */ + .platform = "Amlogic Meson8", + .series = cpuinfo_arm_chipset_series_amlogic_s, + .model = 812, + }, + { + /* "Amlogic Meson8B" -> Amlogic S805 */ + .platform = "Amlogic Meson8B", + .series = cpuinfo_arm_chipset_series_amlogic_s, + .model = 805, + }, + { + /* "mapphone_CDMA" -> Texas Instruments OMAP4430 */ + .platform = "mapphone_CDMA", + .series = cpuinfo_arm_chipset_series_texas_instruments_omap, + .model = 4430, + }, + { + /* "Superior" -> Texas Instruments OMAP4470 */ + .platform = "Superior", + .series = cpuinfo_arm_chipset_series_texas_instruments_omap, + .model = 4470, + }, + { + /* "Tuna" (Samsung Galaxy Nexus) -> Texas Instruments OMAP4460 */ + .platform = "Tuna", + .series = cpuinfo_arm_chipset_series_texas_instruments_omap, + .model = 4460, + }, + { + /* "Manta" (Samsung Nexus 10) -> Samsung Exynos 5250 */ + .platform = "Manta", + .series = cpuinfo_arm_chipset_series_samsung_exynos, + .model = 5250, + }, + { + /* "Odin" -> LG Nuclun 7111 */ + .platform = "Odin", + .series = cpuinfo_arm_chipset_series_lg_nuclun, + .model = 7111, + }, + { + /* "Madison" -> MStar 6A338 */ + .platform = "Madison", + .series = cpuinfo_arm_chipset_series_mstar_6a, + .model = 338, + }, +#endif /* CPUINFO_ARCH_ARM */ +}; + +static const struct special_map_entry tegra_hardware_map_entries[] = { +#if CPUINFO_ARCH_ARM + { + /* "cardhu" (Nvidia Cardhu developer tablet) -> Tegra T30 */ + .platform = "cardhu", + .series = cpuinfo_arm_chipset_series_nvidia_tegra_t, + .model = 30, + }, + { + /* "kai" -> Tegra T30L */ + .platform = "kai", + .series = cpuinfo_arm_chipset_series_nvidia_tegra_t, + .model = 30, + .suffix = 'L', + }, + { + /* "p3" (Samsung Galaxy Tab 8.9) -> Tegra T20 */ + .platform = "p3", + .series = cpuinfo_arm_chipset_series_nvidia_tegra_t, + .model = 20, + }, + { + /* "n1" (Samsung Galaxy R / Samsung Captivate Glide) -> Tegra AP20H */ + .platform = "n1", + .series = cpuinfo_arm_chipset_series_nvidia_tegra_ap, + .model = 20, + .suffix = 'H', + }, + { + /* "SHW-M380S" (Samsung Galaxy Tab 10.1) -> Tegra T20 */ + .platform = "SHW-M380S", + .series = cpuinfo_arm_chipset_series_nvidia_tegra_t, + .model = 20, + }, + { + /* "m470" (Hisense Sero 7 Pro) -> Tegra T30L */ + .platform = "m470", + .series = cpuinfo_arm_chipset_series_nvidia_tegra_t, + .model = 30, + .suffix = 'L', + }, + { + /* "endeavoru" (HTC One X) -> Tegra AP33 */ + .platform = "endeavoru", + .series = cpuinfo_arm_chipset_series_nvidia_tegra_ap, + .model = 33, + }, + { + /* "evitareul" (HTC One X+) -> Tegra T33 */ + .platform = "evitareul", + .series = cpuinfo_arm_chipset_series_nvidia_tegra_t, + .model = 33, + }, + { + /* "enrc2b" (HTC One X+) -> Tegra T33 */ + .platform = "enrc2b", + .series = cpuinfo_arm_chipset_series_nvidia_tegra_t, + .model = 33, + }, + { + /* "mozart" (Asus Transformer Pad TF701T) -> Tegra T114 */ + .platform = "mozart", + .series = cpuinfo_arm_chipset_series_nvidia_tegra_t, + .model = 114, + }, + { + /* "tegratab" (Tegra Note 7) -> Tegra T114 */ + .platform = "tegratab", + .series = cpuinfo_arm_chipset_series_nvidia_tegra_t, + .model = 114, + }, + { + /* "tn8" (Nvidia Shield Tablet K1) -> Tegra T124 */ + .platform = "tn8", + .series = cpuinfo_arm_chipset_series_nvidia_tegra_t, + .model = 124, + }, + { + /* "roth" (Nvidia Shield Portable) -> Tegra T114 */ + .platform = "roth", + .series = cpuinfo_arm_chipset_series_nvidia_tegra_t, + .model = 114, + }, + { + /* "pisces" (Xiaomi Mi 3) -> Tegra T114 */ + .platform = "pisces", + .series = cpuinfo_arm_chipset_series_nvidia_tegra_t, + .model = 114, + }, + { + /* "mocha" (Xiaomi Mi Pad) -> Tegra T124 */ + .platform = "mocha", + .series = cpuinfo_arm_chipset_series_nvidia_tegra_t, + .model = 124, + }, + { + /* "stingray" (Motorola XOOM) -> Tegra AP20H */ + .platform = "stingray", + .series = cpuinfo_arm_chipset_series_nvidia_tegra_ap, + .model = 20, + .suffix = 'H', + }, + { + /* "Ceres" (Wiko Highway 4G) -> Tegra SL460N */ + .platform = "Ceres", + .series = cpuinfo_arm_chipset_series_nvidia_tegra_sl, + .model = 460, + .suffix = 'N', + }, + { + /* "MT799" (nabi 2 Tablet) -> Tegra T30 */ + .platform = "MT799", + .series = cpuinfo_arm_chipset_series_nvidia_tegra_t, + .model = 30, + }, + { + /* "t8400n" (nabi DreamTab HD8) -> Tegra T114 */ + .platform = "t8400n", + .series = cpuinfo_arm_chipset_series_nvidia_tegra_t, + .model = 114, + }, + { + /* "chagall" (Fujitsu Stylistic M532) -> Tegra T30 */ + .platform = "chagall", + .series = cpuinfo_arm_chipset_series_nvidia_tegra_t, + .model = 30, + }, + { + /* "ventana" (Asus Transformer TF101) -> Tegra T20 */ + .platform = "ventana", + .series = cpuinfo_arm_chipset_series_nvidia_tegra_t, + .model = 20, + }, + { + /* "bobsleigh" (Fujitsu Arrows Tab F-05E) -> Tegra T33 */ + .platform = "bobsleigh", + .series = cpuinfo_arm_chipset_series_nvidia_tegra_t, + .model = 33, + }, + { + /* "tegra_fjdev101" (Fujitsu Arrows X F-10D) -> Tegra AP33 */ + .platform = "tegra_fjdev101", + .series = cpuinfo_arm_chipset_series_nvidia_tegra_ap, + .model = 33, + }, + { + /* "tegra_fjdev103" (Fujitsu Arrows V F-04E) -> Tegra T33 */ + .platform = "tegra_fjdev103", + .series = cpuinfo_arm_chipset_series_nvidia_tegra_t, + .model = 33, + }, + { + /* "nbx03" (Sony Tablet S) -> Tegra T20 */ + .platform = "nbx03", + .series = cpuinfo_arm_chipset_series_nvidia_tegra_t, + .model = 20, + }, + { + /* "txs03" (Sony Xperia Tablet S) -> Tegra T30L */ + .platform = "txs03", + .series = cpuinfo_arm_chipset_series_nvidia_tegra_t, + .model = 30, + .suffix = 'L', + }, + { + /* "x3" (LG Optimus 4X HD P880) -> Tegra AP33 */ + .platform = "x3", + .series = cpuinfo_arm_chipset_series_nvidia_tegra_ap, + .model = 33, + }, + { + /* "vu10" (LG Optimus Vu P895) -> Tegra AP33 */ + .platform = "vu10", + .series = cpuinfo_arm_chipset_series_nvidia_tegra_ap, + .model = 33, + }, + { + /* "BIRCH" (HP Slate 7 Plus) -> Tegra T30L */ + .platform = "BIRCH", + .series = cpuinfo_arm_chipset_series_nvidia_tegra_t, + .model = 30, + .suffix = 'L', + }, + { + /* "macallan" (HP Slate 8 Pro) -> Tegra T114 */ + .platform = "macallan", + .series = cpuinfo_arm_chipset_series_nvidia_tegra_t, + .model = 114, + }, + { + /* "maya" (HP SlateBook 10 x2) -> Tegra T114 */ + .platform = "maya", + .series = cpuinfo_arm_chipset_series_nvidia_tegra_t, + .model = 114, + }, + { + /* "antares" (Toshiba AT100) -> Tegra T20 */ + .platform = "antares", + .series = cpuinfo_arm_chipset_series_nvidia_tegra_t, + .model = 20, + }, + { + /* "tostab12AL" (Toshiba AT300SE "Excite 10 SE") -> Tegra T30L */ + .platform = "tostab12AL", + .series = cpuinfo_arm_chipset_series_nvidia_tegra_t, + .model = 30, + .suffix = 'L', + }, + { + /* "tostab12BL" (Toshiba AT10-A "Excite Pure") -> Tegra T30L */ + .platform = "tostab12BL", + .series = cpuinfo_arm_chipset_series_nvidia_tegra_t, + .model = 30, + .suffix = 'L', + }, + { + /* "sphinx" (Toshiba AT270 "Excite 7.7") -> Tegra T30 */ + .platform = "sphinx", + .series = cpuinfo_arm_chipset_series_nvidia_tegra_t, + .model = 30, + }, + { + /* "tostab11BS" (Toshiba AT570 "Regza 7.7") -> Tegra T30 */ + .platform = "tostab11BS", + .series = cpuinfo_arm_chipset_series_nvidia_tegra_t, + .model = 30, + }, + { + /* "tostab12BA" (Toshiba AT10-LE-A "Excite Pro") -> Tegra T114 */ + .platform = "tostab12BA", + .series = cpuinfo_arm_chipset_series_nvidia_tegra_t, + .model = 114, + }, + { + /* "vangogh" (Acer Iconia Tab A100) -> Tegra T20 */ + .platform = "vangogh", + .series = cpuinfo_arm_chipset_series_nvidia_tegra_t, + .model = 20, + }, + { + /* "a110" (Acer Iconia Tab A110) -> Tegra T30L */ + .platform = "a110", + .series = cpuinfo_arm_chipset_series_nvidia_tegra_t, + .model = 30, + .suffix = 'L', + }, + { + /* "picasso_e" (Acer Iconia Tab A200) -> Tegra AP20H */ + .platform = "picasso_e", + .series = cpuinfo_arm_chipset_series_nvidia_tegra_ap, + .model = 20, + .suffix = 'H', + }, + { + /* "picasso_e2" (Acer Iconia Tab A210) -> Tegra T30L */ + .platform = "picasso_e2", + .series = cpuinfo_arm_chipset_series_nvidia_tegra_t, + .model = 30, + .suffix = 'L', + }, + { + /* "picasso" (Acer Iconia Tab A500) -> Tegra AP20H */ + .platform = "picasso", + .series = cpuinfo_arm_chipset_series_nvidia_tegra_ap, + .model = 20, + .suffix = 'H', + }, + { + /* "picasso_m" (Acer Iconia Tab A510) -> Tegra T30 */ + .platform = "picasso_m", + .series = cpuinfo_arm_chipset_series_nvidia_tegra_t, + .model = 30, + }, + { + /* "picasso_mf" (Acer Iconia Tab A700) -> Tegra T30 */ + .platform = "picasso_mf", + .series = cpuinfo_arm_chipset_series_nvidia_tegra_t, + .model = 30, + }, + { + /* "avalon" (Toshiba AT300 "Excite 10") -> Tegra T30L */ + .platform = "avalon", + .series = cpuinfo_arm_chipset_series_nvidia_tegra_t, + .model = 30, + .suffix = 'L', + }, + { + /* "NS_14T004" (iRiver NS-14T004) -> Tegra T30L */ + .platform = "NS_14T004", + .series = cpuinfo_arm_chipset_series_nvidia_tegra_t, + .model = 30, + .suffix = 'L', + }, + { + /* "WIKIPAD" (Wikipad) -> Tegra T30 */ + .platform = "WIKIPAD", + .series = cpuinfo_arm_chipset_series_nvidia_tegra_t, + .model = 30, + }, + { + /* "kb" (Pegatron Q00Q) -> Tegra T114 */ + .platform = "kb", + .series = cpuinfo_arm_chipset_series_nvidia_tegra_t, + .model = 114, + }, +#endif /* CPUINFO_ARCH_ARM */ + { + /* "foster_e" (Nvidia Shield TV, Flash) -> Tegra T210 */ + .platform = "foster_e", + .series = cpuinfo_arm_chipset_series_nvidia_tegra_t, + .model = 210, + }, + { + /* "foster_e_hdd" (Nvidia Shield TV, HDD) -> Tegra T210 */ + .platform = "foster_e_hdd", + .series = cpuinfo_arm_chipset_series_nvidia_tegra_t, + .model = 210, + }, + { + /* "darcy" (Nvidia Shield TV 2017) -> Tegra T210 */ + .platform = "darcy", + .series = cpuinfo_arm_chipset_series_nvidia_tegra_t, + .model = 210, + }, +}; + +/* + * Decodes chipset name from /proc/cpuinfo Hardware string. + * For some chipsets, the function relies frequency and on number of cores for chipset detection. + * + * @param[in] platform - /proc/cpuinfo Hardware string. + * @param cores - number of cores in the chipset. + * @param max_cpu_freq_max - maximum of /sys/devices/system/cpu/cpu/cpofreq/cpu_freq_max values. + * + * @returns Decoded chipset name. If chipset could not be decoded, the resulting structure would use `unknown` vendor + * and series identifiers. + */ +struct cpuinfo_arm_chipset cpuinfo_arm_linux_decode_chipset_from_proc_cpuinfo_hardware( + const char hardware[restrict static CPUINFO_HARDWARE_VALUE_MAX], + uint32_t cores, uint32_t max_cpu_freq_max, bool is_tegra) +{ + struct cpuinfo_arm_chipset chipset; + const size_t hardware_length = strnlen(hardware, CPUINFO_HARDWARE_VALUE_MAX); + const char* hardware_end = hardware + hardware_length; + + if (is_tegra) { + /* + * Nvidia Tegra-specific path: compare /proc/cpuinfo Hardware string to + * tabulated Hardware values for popular chipsets/devices with Tegra chipsets. + * This path is only used when ro.board.platform indicates a Tegra chipset + * (albeit does not indicate which exactly Tegra chipset). + */ + for (size_t i = 0; i < CPUINFO_COUNT_OF(tegra_hardware_map_entries); i++) { + if (strncmp(tegra_hardware_map_entries[i].platform, hardware, hardware_length) == 0 && + tegra_hardware_map_entries[i].platform[hardware_length] == 0) + { + cpuinfo_log_debug( + "found /proc/cpuinfo Hardware string \"%.*s\" in Nvidia Tegra chipset table", + (int) hardware_length, hardware); + /* Create chipset name from entry */ + return (struct cpuinfo_arm_chipset) { + .vendor = chipset_series_vendor[tegra_hardware_map_entries[i].series], + .series = (enum cpuinfo_arm_chipset_series) tegra_hardware_map_entries[i].series, + .model = tegra_hardware_map_entries[i].model, + .suffix = { + [0] = tegra_hardware_map_entries[i].suffix, + }, + }; + } + } + } else { + /* Generic path: consider all other vendors */ + + bool word_start = true; + for (const char* pos = hardware; pos != hardware_end; pos++) { + const char c = *pos; + switch (c) { + case ' ': + case '\t': + case ',': + word_start = true; + break; + default: + if (word_start && is_ascii_alphabetic(c)) { + /* Check Qualcomm MSM/APQ signature */ + if (match_msm_apq(pos, hardware_end, &chipset)) { + cpuinfo_log_debug( + "matched Qualcomm MSM/APQ signature in /proc/cpuinfo Hardware string \"%.*s\"", + (int) hardware_length, hardware); + return chipset; + } + + /* Check SDMxxx (Qualcomm Snapdragon) signature */ + if (match_sdm(pos, hardware_end, &chipset)) { + cpuinfo_log_debug( + "matched Qualcomm SDM signature in /proc/cpuinfo Hardware string \"%.*s\"", + (int) hardware_length, hardware); + return chipset; + } + + /* Check SMxxxx (Qualcomm Snapdragon) signature */ + if (match_sm(pos, hardware_end, &chipset)) { + cpuinfo_log_debug( + "matched Qualcomm SM signature in /proc/cpuinfo Hardware string \"%.*s\"", + (int) hardware_length, hardware); + return chipset; + } + + /* Check MediaTek MT signature */ + if (match_mt(pos, hardware_end, true, &chipset)) { + cpuinfo_log_debug( + "matched MediaTek MT signature in /proc/cpuinfo Hardware string \"%.*s\"", + (int) hardware_length, hardware); + return chipset; + } + + /* Check HiSilicon Kirin signature */ + if (match_kirin(pos, hardware_end, &chipset)) { + cpuinfo_log_debug( + "matched HiSilicon Kirin signature in /proc/cpuinfo Hardware string \"%.*s\"", + (int) hardware_length, hardware); + return chipset; + } + + /* Check Rockchip RK signature */ + if (match_rk(pos, hardware_end, &chipset)) { + cpuinfo_log_debug( + "matched Rockchip RK signature in /proc/cpuinfo Hardware string \"%.*s\"", + (int) hardware_length, hardware); + return chipset; + } + + if (match_qualcomm_special(pos, hardware_end, &chipset)) { + cpuinfo_log_debug( + "matched Qualcomm signature in /proc/cpuinfo Hardware string \"%.*s\"", + (int) hardware_length, hardware); + return chipset; + } + + } + word_start = false; + break; + } + } + + /* Check Samsung Exynos signature */ + if (match_samsung_exynos(hardware, hardware_end, &chipset)) { + cpuinfo_log_debug( + "matched Samsung Exynos signature in /proc/cpuinfo Hardware string \"%.*s\"", + (int) hardware_length, hardware); + return chipset; + } + + /* Check universalXXXX (Samsung Exynos) signature */ + if (match_universal(hardware, hardware_end, &chipset)) { + cpuinfo_log_debug( + "matched UNIVERSAL (Samsung Exynos) signature in /proc/cpuinfo Hardware string \"%.*s\"", + (int) hardware_length, hardware); + return chipset; + } + + #if CPUINFO_ARCH_ARM + /* Match /SMDK(4410|4x12)$/ */ + if (match_and_parse_smdk(hardware, hardware_end, cores, &chipset)) { + cpuinfo_log_debug( + "matched SMDK (Samsung Exynos) signature in /proc/cpuinfo Hardware string \"%.*s\"", + (int) hardware_length, hardware); + return chipset; + } + #endif + + /* Check Spreadtrum SC signature */ + if (match_sc(hardware, hardware_end, &chipset)) { + cpuinfo_log_debug( + "matched Spreadtrum SC signature in /proc/cpuinfo Hardware string \"%.*s\"", + (int) hardware_length, hardware); + return chipset; + } + + #if CPUINFO_ARCH_ARM + /* Check Marvell PXA signature */ + if (match_pxa(hardware, hardware_end, &chipset)) { + cpuinfo_log_debug( + "matched Marvell PXA signature in /proc/cpuinfo Hardware string \"%.*s\"", + (int) hardware_length, hardware); + return chipset; + } + #endif + + /* Match /sun\d+i/ signature and map to Allwinner chipset name */ + if (match_and_parse_sunxi(hardware, hardware_end, cores, &chipset)) { + cpuinfo_log_debug( + "matched sunxi (Allwinner Ax) signature in /proc/cpuinfo Hardware string \"%.*s\"", + (int) hardware_length, hardware); + return chipset; + } + + /* Check Broadcom BCM signature */ + if (match_bcm(hardware, hardware_end, &chipset)) { + cpuinfo_log_debug( + "matched Broadcom BCM signature in /proc/cpuinfo Hardware string \"%.*s\"", + (int) hardware_length, hardware); + return chipset; + } + + #if CPUINFO_ARCH_ARM + /* Check Texas Instruments OMAP signature */ + if (match_omap(hardware, hardware_end, &chipset)) { + cpuinfo_log_debug( + "matched Texas Instruments OMAP signature in /proc/cpuinfo Hardware string \"%.*s\"", + (int) hardware_length, hardware); + return chipset; + } + + /* Check WonderMedia WMT signature and decode chipset from frequency and number of cores */ + if (match_and_parse_wmt(hardware, hardware_end, cores, max_cpu_freq_max, &chipset)) { + cpuinfo_log_debug( + "matched WonderMedia WMT signature in /proc/cpuinfo Hardware string \"%.*s\"", + (int) hardware_length, hardware); + return chipset; + } + + #endif + + /* Check Telechips TCC signature */ + if (match_tcc(hardware, hardware_end, &chipset)) { + cpuinfo_log_debug( + "matched Telechips TCC signature in /proc/cpuinfo Hardware string \"%.*s\"", + (int) hardware_length, hardware); + return chipset; + } + + /* Compare to tabulated Hardware values for popular chipsets/devices which can't be otherwise detected */ + for (size_t i = 0; i < CPUINFO_COUNT_OF(special_hardware_map_entries); i++) { + if (strncmp(special_hardware_map_entries[i].platform, hardware, hardware_length) == 0 && + special_hardware_map_entries[i].platform[hardware_length] == 0) + { + cpuinfo_log_debug( + "found /proc/cpuinfo Hardware string \"%.*s\" in special chipset table", + (int) hardware_length, hardware); + /* Create chipset name from entry */ + return (struct cpuinfo_arm_chipset) { + .vendor = chipset_series_vendor[special_hardware_map_entries[i].series], + .series = (enum cpuinfo_arm_chipset_series) special_hardware_map_entries[i].series, + .model = special_hardware_map_entries[i].model, + .suffix = { + [0] = special_hardware_map_entries[i].suffix, + }, + }; + } + } + } + + return (struct cpuinfo_arm_chipset) { + .vendor = cpuinfo_arm_chipset_vendor_unknown, + .series = cpuinfo_arm_chipset_series_unknown, + }; +} + +#ifdef __ANDROID__ + static const struct special_map_entry special_board_map_entries[] = { + { + /* "hi6250" -> HiSilicon Kirin 650 */ + .platform = "hi6250", + .series = cpuinfo_arm_chipset_series_hisilicon_kirin, + .model = 650, + }, + { + /* "hi6210sft" -> HiSilicon Kirin 620 */ + .platform = "hi6210sft", + .series = cpuinfo_arm_chipset_series_hisilicon_kirin, + .model = 620, + }, +#if CPUINFO_ARCH_ARM + { + /* "hi3630" -> HiSilicon Kirin 920 */ + .platform = "hi3630", + .series = cpuinfo_arm_chipset_series_hisilicon_kirin, + .model = 920, + }, +#endif /* CPUINFO_ARCH_ARM */ + { + /* "hi3635" -> HiSilicon Kirin 930 */ + .platform = "hi3635", + .series = cpuinfo_arm_chipset_series_hisilicon_kirin, + .model = 930, + }, + { + /* "hi3650" -> HiSilicon Kirin 950 */ + .platform = "hi3650", + .series = cpuinfo_arm_chipset_series_hisilicon_kirin, + .model = 950, + }, + { + /* "hi3660" -> HiSilicon Kirin 960 */ + .platform = "hi3660", + .series = cpuinfo_arm_chipset_series_hisilicon_kirin, + .model = 960, + }, +#if CPUINFO_ARCH_ARM + { + /* "mp523x" -> Renesas MP5232 */ + .platform = "mp523x", + .series = cpuinfo_arm_chipset_series_renesas_mp, + .model = 5232, + }, +#endif /* CPUINFO_ARCH_ARM */ + { + /* "BEETHOVEN" (Huawei MadiaPad M3) -> HiSilicon Kirin 950 */ + .platform = "BEETHOVEN", + .series = cpuinfo_arm_chipset_series_hisilicon_kirin, + .model = 950, + }, +#if CPUINFO_ARCH_ARM + { + /* "hws7701u" (Huawei MediaPad 7 Youth) -> Rockchip RK3168 */ + .platform = "hws7701u", + .series = cpuinfo_arm_chipset_series_rockchip_rk, + .model = 3168, + }, + { + /* "g2mv" (LG G2 mini LTE) -> Nvidia Tegra SL460N */ + .platform = "g2mv", + .series = cpuinfo_arm_chipset_series_nvidia_tegra_sl, + .model = 460, + .suffix = 'N', + }, + { + /* "K00F" (Asus MeMO Pad 10) -> Rockchip RK3188 */ + .platform = "K00F", + .series = cpuinfo_arm_chipset_series_rockchip_rk, + .model = 3188, + }, + { + /* "T7H" (HP Slate 7) -> Rockchip RK3066 */ + .platform = "T7H", + .series = cpuinfo_arm_chipset_series_rockchip_rk, + .model = 3066, + }, + { + /* "tuna" (Samsung Galaxy Nexus) -> Texas Instruments OMAP4460 */ + .platform = "tuna", + .series = cpuinfo_arm_chipset_series_texas_instruments_omap, + .model = 4460, + }, + { + /* "grouper" (Asus Nexus 7 2012) -> Nvidia Tegra T30L */ + .platform = "grouper", + .series = cpuinfo_arm_chipset_series_nvidia_tegra_t, + .model = 30, + .suffix = 'L', + }, +#endif /* CPUINFO_ARCH_ARM */ + { + /* "flounder" (HTC Nexus 9) -> Nvidia Tegra T132 */ + .platform = "flounder", + .series = cpuinfo_arm_chipset_series_nvidia_tegra_t, + .model = 132, + }, + { + /* "dragon" (Google Pixel C) -> Nvidia Tegra T210 */ + .platform = "dragon", + .series = cpuinfo_arm_chipset_series_nvidia_tegra_t, + .model = 210, + }, + { + /* "sailfish" (Google Pixel) -> Qualcomm MSM8996PRO */ + .platform = "sailfish", + .series = cpuinfo_arm_chipset_series_qualcomm_msm, + .model = 8996, + .suffix = 'P', + }, + { + /* "marlin" (Google Pixel XL) -> Qualcomm MSM8996PRO */ + .platform = "marlin", + .series = cpuinfo_arm_chipset_series_qualcomm_msm, + .model = 8996, + .suffix = 'P', + }, + }; + + /* + * Decodes chipset name from ro.product.board Android system property. + * For some chipsets, the function relies frequency and on number of cores for chipset detection. + * + * @param[in] platform - ro.product.board value. + * @param cores - number of cores in the chipset. + * @param max_cpu_freq_max - maximum of /sys/devices/system/cpu/cpu/cpofreq/cpu_freq_max values. + * + * @returns Decoded chipset name. If chipset could not be decoded, the resulting structure would use `unknown` vendor + * and series identifiers. + */ + struct cpuinfo_arm_chipset cpuinfo_arm_android_decode_chipset_from_ro_product_board( + const char ro_product_board[restrict static CPUINFO_BUILD_PROP_VALUE_MAX], + uint32_t cores, uint32_t max_cpu_freq_max) + { + struct cpuinfo_arm_chipset chipset; + const char* board = ro_product_board; + const size_t board_length = strnlen(ro_product_board, CPUINFO_BUILD_PROP_VALUE_MAX); + const char* board_end = ro_product_board + board_length; + + /* Check Qualcomm MSM/APQ signature */ + if (match_msm_apq(board, board_end, &chipset)) { + cpuinfo_log_debug( + "matched Qualcomm MSM/APQ signature in ro.product.board string \"%.*s\"", (int) board_length, board); + return chipset; + } + + /* Check universaXXXX (Samsung Exynos) signature */ + if (match_universal(board, board_end, &chipset)) { + cpuinfo_log_debug( + "matched UNIVERSAL (Samsung Exynos) signature in ro.product.board string \"%.*s\"", + (int) board_length, board); + return chipset; + } + + #if CPUINFO_ARCH_ARM + /* Check SMDK (Samsung Exynos) signature */ + if (match_and_parse_smdk(board, board_end, cores, &chipset)) { + cpuinfo_log_debug( + "matched SMDK (Samsung Exynos) signature in ro.product.board string \"%.*s\"", + (int) board_length, board); + return chipset; + } + #endif + + /* Check MediaTek MT signature */ + if (match_mt(board, board_end, true, &chipset)) { + cpuinfo_log_debug( + "matched MediaTek MT signature in ro.product.board string \"%.*s\"", + (int) board_length, board); + return chipset; + } + + /* Check Spreadtrum SC signature */ + if (match_sc(board, board_end, &chipset)) { + cpuinfo_log_debug( + "matched Spreadtrum SC signature in ro.product.board string \"%.*s\"", + (int) board_length, board); + return chipset; + } + + #if CPUINFO_ARCH_ARM + /* Check Marvell PXA signature */ + if (match_pxa(board, board_end, &chipset)) { + cpuinfo_log_debug( + "matched Marvell PXA signature in ro.product.board string \"%.*s\"", + (int) board_length, board); + return chipset; + } + + /* Check Leadcore LCxxxx signature */ + if (match_lc(board, board_end, &chipset)) { + cpuinfo_log_debug( + "matched Leadcore LC signature in ro.product.board string \"%.*s\"", + (int) board_length, board); + return chipset; + } + + /* + * Compare to tabulated ro.product.board values for Broadcom chipsets and decode chipset from frequency and + * number of cores. + */ + if (match_and_parse_broadcom(board, board_end, cores, max_cpu_freq_max, &chipset)) { + cpuinfo_log_debug( + "found ro.product.board string \"%.*s\" in Broadcom chipset table", + (int) board_length, board); + return chipset; + } + #endif + + /* Compare to tabulated ro.product.board values for Huawei devices which don't report chipset elsewhere */ + if (match_and_parse_huawei(board, board_end, &chipset)) { + cpuinfo_log_debug( + "found ro.product.board string \"%.*s\" in Huawei chipset table", + (int) board_length, board); + return chipset; + } + + /* Compare to tabulated ro.product.board values for popular chipsets/devices which can't be otherwise detected */ + for (size_t i = 0; i < CPUINFO_COUNT_OF(special_board_map_entries); i++) { + if (strncmp(special_board_map_entries[i].platform, board, board_length) == 0 && + special_board_map_entries[i].platform[board_length] == 0) + { + cpuinfo_log_debug( + "found ro.product.board string \"%.*s\" in special chipset table", + (int) board_length, board); + /* Create chipset name from entry */ + return (struct cpuinfo_arm_chipset) { + .vendor = chipset_series_vendor[special_board_map_entries[i].series], + .series = (enum cpuinfo_arm_chipset_series) special_board_map_entries[i].series, + .model = special_board_map_entries[i].model, + .suffix = { + [0] = special_board_map_entries[i].suffix, + /* The suffix of MSM8996PRO is truncated at the first letter, reconstruct it here. */ + [1] = special_board_map_entries[i].suffix == 'P' ? 'R' : 0, + [2] = special_board_map_entries[i].suffix == 'P' ? 'O' : 0, + }, + }; + } + } + + return (struct cpuinfo_arm_chipset) { + .vendor = cpuinfo_arm_chipset_vendor_unknown, + .series = cpuinfo_arm_chipset_series_unknown, + }; + } + + struct amlogic_map_entry { + char ro_board_platform[6]; + uint16_t model; + uint8_t series; + char suffix[3]; + }; + + static const struct amlogic_map_entry amlogic_map_entries[] = { +#if CPUINFO_ARCH_ARM + { + /* "meson3" -> Amlogic AML8726-M */ + .ro_board_platform = "meson3", + .series = cpuinfo_arm_chipset_series_amlogic_aml, + .model = 8726, + .suffix = "-M", + }, + { + /* "meson6" -> Amlogic AML8726-MX */ + .ro_board_platform = "meson6", + .series = cpuinfo_arm_chipset_series_amlogic_aml, + .model = 8726, + .suffix = "-MX", + }, + { + /* "meson8" -> Amlogic S805 */ + .ro_board_platform = "meson8", + .series = cpuinfo_arm_chipset_series_amlogic_s, + .model = 805, + }, +#endif /* CPUINFO_ARCH_ARM */ + { + /* "gxbaby" -> Amlogic S905 */ + .ro_board_platform = "gxbaby", + .series = cpuinfo_arm_chipset_series_amlogic_s, + .model = 905, + }, + { + /* "gxl" -> Amlogic S905X */ + .ro_board_platform = "gxl", + .series = cpuinfo_arm_chipset_series_amlogic_s, + .model = 905, + .suffix = "X", + }, + { + /* "gxm" -> Amlogic S912 */ + .ro_board_platform = "gxm", + .series = cpuinfo_arm_chipset_series_amlogic_s, + .model = 912, + }, + }; + + static const struct special_map_entry special_platform_map_entries[] = { +#if CPUINFO_ARCH_ARM + { + /* "hi6620oem" -> HiSilicon Kirin 910T */ + .platform = "hi6620oem", + .series = cpuinfo_arm_chipset_series_hisilicon_kirin, + .model = 910, + .suffix = 'T', + }, +#endif /* CPUINFO_ARCH_ARM */ + { + /* "hi6250" -> HiSilicon Kirin 650 */ + .platform = "hi6250", + .series = cpuinfo_arm_chipset_series_hisilicon_kirin, + .model = 650, + }, + { + /* "hi6210sft" -> HiSilicon Kirin 620 */ + .platform = "hi6210sft", + .series = cpuinfo_arm_chipset_series_hisilicon_kirin, + .model = 620, + }, +#if CPUINFO_ARCH_ARM + { + /* "hi3630" -> HiSilicon Kirin 920 */ + .platform = "hi3630", + .series = cpuinfo_arm_chipset_series_hisilicon_kirin, + .model = 920, + }, +#endif /* CPUINFO_ARCH_ARM */ + { + /* "hi3635" -> HiSilicon Kirin 930 */ + .platform = "hi3635", + .series = cpuinfo_arm_chipset_series_hisilicon_kirin, + .model = 930, + }, + { + /* "hi3650" -> HiSilicon Kirin 950 */ + .platform = "hi3650", + .series = cpuinfo_arm_chipset_series_hisilicon_kirin, + .model = 950, + }, + { + /* "hi3660" -> HiSilicon Kirin 960 */ + .platform = "hi3660", + .series = cpuinfo_arm_chipset_series_hisilicon_kirin, + .model = 960, + }, +#if CPUINFO_ARCH_ARM + { + /* "k3v2oem1" -> HiSilicon K3V2 */ + .platform = "k3v2oem1", + .series = cpuinfo_arm_chipset_series_hisilicon_k3v, + .model = 2, + }, + { + /* "k3v200" -> HiSilicon K3V2 */ + .platform = "k3v200", + .series = cpuinfo_arm_chipset_series_hisilicon_k3v, + .model = 2, + }, + { + /* "montblanc" -> NovaThor U8500 */ + .platform = "montblanc", + .series = cpuinfo_arm_chipset_series_novathor_u, + .model = 8500, + }, +#endif /* CPUINFO_ARCH_ARM */ + { + /* "song" -> Pinecone Surge S1 */ + .platform = "song", + .series = cpuinfo_arm_chipset_series_pinecone_surge_s, + .model = 1, + }, +#if CPUINFO_ARCH_ARM + { + /* "rk322x" -> RockChip RK3229 */ + .platform = "rk322x", + .series = cpuinfo_arm_chipset_series_rockchip_rk, + .model = 3229, + }, +#endif /* CPUINFO_ARCH_ARM */ + { + /* "tegra132" -> Nvidia Tegra T132 */ + .platform = "tegra132", + .series = cpuinfo_arm_chipset_series_nvidia_tegra_t, + .model = 132, + }, + { + /* "tegra210_dragon" -> Nvidia Tegra T210 */ + .platform = "tegra210_dragon", + .series = cpuinfo_arm_chipset_series_nvidia_tegra_t, + .model = 210, + }, +#if CPUINFO_ARCH_ARM + { + /* "tegra4" -> Nvidia Tegra T114 */ + .platform = "tegra4", + .series = cpuinfo_arm_chipset_series_nvidia_tegra_t, + .model = 114, + }, + { + /* "s5pc110" -> Samsung Exynos 3110 */ + .platform = "s5pc110", + .series = cpuinfo_arm_chipset_series_samsung_exynos, + .model = 3110, + }, +#endif /* CPUINFO_ARCH_ARM */ + }; + + /* + * Decodes chipset name from ro.board.platform Android system property. + * For some chipsets, the function relies frequency and on number of cores for chipset detection. + * + * @param[in] platform - ro.board.platform value. + * @param cores - number of cores in the chipset. + * @param max_cpu_freq_max - maximum of /sys/devices/system/cpu/cpu/cpofreq/cpu_freq_max values. + * + * @returns Decoded chipset name. If chipset could not be decoded, the resulting structure would use `unknown` vendor + * and series identifiers. + */ + struct cpuinfo_arm_chipset cpuinfo_arm_android_decode_chipset_from_ro_board_platform( + const char platform[restrict static CPUINFO_BUILD_PROP_VALUE_MAX], + uint32_t cores, uint32_t max_cpu_freq_max) + { + struct cpuinfo_arm_chipset chipset; + const size_t platform_length = strnlen(platform, CPUINFO_BUILD_PROP_VALUE_MAX); + const char* platform_end = platform + platform_length; + + /* Check Qualcomm MSM/APQ signature */ + if (match_msm_apq(platform, platform_end, &chipset)) { + cpuinfo_log_debug( + "matched Qualcomm MSM/APQ signature in ro.board.platform string \"%.*s\"", + (int) platform_length, platform); + return chipset; + } + + /* Check exynosXXXX (Samsung Exynos) signature */ + if (match_exynos(platform, platform_end, &chipset)) { + cpuinfo_log_debug( + "matched exynosXXXX (Samsung Exynos) signature in ro.board.platform string \"%.*s\"", + (int) platform_length, platform); + return chipset; + } + + /* Check MediaTek MT signature */ + if (match_mt(platform, platform_end, true, &chipset)) { + cpuinfo_log_debug( + "matched MediaTek MT signature in ro.board.platform string \"%.*s\"", (int) platform_length, platform); + return chipset; + } + + /* Check HiSilicon Kirin signature */ + if (match_kirin(platform, platform_end, &chipset)) { + cpuinfo_log_debug( + "matched HiSilicon Kirin signature in ro.board.platform string \"%.*s\"", (int) platform_length, platform); + return chipset; + } + + /* Check Spreadtrum SC signature */ + if (match_sc(platform, platform_end, &chipset)) { + cpuinfo_log_debug( + "matched Spreadtrum SC signature in ro.board.platform string \"%.*s\"", (int) platform_length, platform); + return chipset; + } + + /* Check Rockchip RK signature */ + if (match_rk(platform, platform_end, &chipset)) { + cpuinfo_log_debug( + "matched Rockchip RK signature in ro.board.platform string \"%.*s\"", (int) platform_length, platform); + return chipset; + } + + #if CPUINFO_ARCH_ARM + /* Check Leadcore LCxxxx signature */ + if (match_lc(platform, platform_end, &chipset)) { + cpuinfo_log_debug( + "matched Leadcore LC signature in ro.board.platform string \"%.*s\"", (int) platform_length, platform); + return chipset; + } + #endif + + /* Compare to tabulated ro.board.platform values for Huawei devices which don't report chipset elsewhere */ + if (match_and_parse_huawei(platform, platform_end, &chipset)) { + cpuinfo_log_debug( + "found ro.board.platform string \"%.*s\" in Huawei chipset table", + (int) platform_length, platform); + return chipset; + } + + #if CPUINFO_ARCH_ARM + /* + * Compare to known ro.board.platform values for Broadcom devices and + * detect chipset from frequency and number of cores + */ + if (match_and_parse_broadcom(platform, platform_end, cores, max_cpu_freq_max, &chipset)) { + cpuinfo_log_debug( + "found ro.board.platform string \"%.*s\" in Broadcom chipset table", + (int) platform_length, platform); + return chipset; + } + + /* + * Compare to ro.board.platform value ("omap4") for OMAP4xxx chipsets. + * Upon successful match, detect OMAP4430 from frequency and number of cores. + */ + if (platform_length == 5 && cores == 2 && max_cpu_freq_max == 1008000 && memcmp(platform, "omap4", 5) == 0) { + cpuinfo_log_debug( + "matched Texas Instruments OMAP4 signature in ro.board.platform string \"%.*s\"", + (int) platform_length, platform); + + return (struct cpuinfo_arm_chipset) { + .vendor = cpuinfo_arm_chipset_vendor_texas_instruments, + .series = cpuinfo_arm_chipset_series_texas_instruments_omap, + .model = 4430, + }; + } + #endif + + /* + * Compare to tabulated ro.board.platform values for Amlogic chipsets/devices which can't be otherwise detected. + * The tabulated Amlogic ro.board.platform values have not more than 6 characters. + */ + if (platform_length <= 6) { + for (size_t i = 0; i < CPUINFO_COUNT_OF(amlogic_map_entries); i++) { + if (strncmp(amlogic_map_entries[i].ro_board_platform, platform, 6) == 0) { + cpuinfo_log_debug( + "found ro.board.platform string \"%.*s\" in Amlogic chipset table", + (int) platform_length, platform); + /* Create chipset name from entry */ + return (struct cpuinfo_arm_chipset) { + .vendor = cpuinfo_arm_chipset_vendor_amlogic, + .series = (enum cpuinfo_arm_chipset_series) amlogic_map_entries[i].series, + .model = amlogic_map_entries[i].model, + .suffix = { + [0] = amlogic_map_entries[i].suffix[0], + [1] = amlogic_map_entries[i].suffix[1], + [2] = amlogic_map_entries[i].suffix[2], + }, + }; + } + } + } + + /* Compare to tabulated ro.board.platform values for popular chipsets/devices which can't be otherwise detected */ + for (size_t i = 0; i < CPUINFO_COUNT_OF(special_platform_map_entries); i++) { + if (strncmp(special_platform_map_entries[i].platform, platform, platform_length) == 0 && + special_platform_map_entries[i].platform[platform_length] == 0) + { + /* Create chipset name from entry */ + cpuinfo_log_debug( + "found ro.board.platform string \"%.*s\" in special chipset table", (int) platform_length, platform); + return (struct cpuinfo_arm_chipset) { + .vendor = chipset_series_vendor[special_platform_map_entries[i].series], + .series = (enum cpuinfo_arm_chipset_series) special_platform_map_entries[i].series, + .model = special_platform_map_entries[i].model, + .suffix = { + [0] = special_platform_map_entries[i].suffix, + }, + }; + } + } + + /* None of the ro.board.platform signatures matched, indicate unknown chipset */ + return (struct cpuinfo_arm_chipset) { + .vendor = cpuinfo_arm_chipset_vendor_unknown, + .series = cpuinfo_arm_chipset_series_unknown, + }; + } + + /* + * Decodes chipset name from ro.mediatek.platform Android system property. + * + * @param[in] platform - ro.mediatek.platform value. + * + * @returns Decoded chipset name. If chipset could not be decoded, the resulting structure would use `unknown` + * vendor and series identifiers. + */ + struct cpuinfo_arm_chipset cpuinfo_arm_android_decode_chipset_from_ro_mediatek_platform( + const char platform[restrict static CPUINFO_BUILD_PROP_VALUE_MAX]) + { + struct cpuinfo_arm_chipset chipset; + const char* platform_end = platform + strnlen(platform, CPUINFO_BUILD_PROP_VALUE_MAX); + + /* Check MediaTek MT signature */ + if (match_mt(platform, platform_end, false, &chipset)) { + return chipset; + } + + return (struct cpuinfo_arm_chipset) { + .vendor = cpuinfo_arm_chipset_vendor_unknown, + .series = cpuinfo_arm_chipset_series_unknown, + }; + } + + + /* + * Decodes chipset name from ro.arch Android system property. + * + * The ro.arch property is matched only against Samsung Exynos signature. Systems with other chipset rarely + * configure ro.arch Android system property, and can be decoded through other properties, but some Exynos + * chipsets are identified only in ro.arch. + * + * @param[in] arch - ro.arch value. + * + * @returns Decoded chipset name. If chipset could not be decoded, the resulting structure would use `unknown` + * vendor and series identifiers. + */ + struct cpuinfo_arm_chipset cpuinfo_arm_android_decode_chipset_from_ro_arch( + const char arch[restrict static CPUINFO_BUILD_PROP_VALUE_MAX]) + { + struct cpuinfo_arm_chipset chipset; + const char* arch_end = arch + strnlen(arch, CPUINFO_BUILD_PROP_VALUE_MAX); + + /* Check Samsung exynosXXXX signature */ + if (match_exynos(arch, arch_end, &chipset)) { + return chipset; + } + + return (struct cpuinfo_arm_chipset) { + .vendor = cpuinfo_arm_chipset_vendor_unknown, + .series = cpuinfo_arm_chipset_series_unknown, + }; + } + + /* + * Decodes chipset name from ro.chipname or ro.hardware.chipname Android system property. + * + * @param[in] chipname - ro.chipname or ro.hardware.chipname value. + * + * @returns Decoded chipset name. If chipset could not be decoded, the resulting structure would use `unknown` vendor + * and series identifiers. + */ + + struct cpuinfo_arm_chipset cpuinfo_arm_android_decode_chipset_from_ro_chipname( + const char chipname[restrict static CPUINFO_BUILD_PROP_VALUE_MAX]) + { + struct cpuinfo_arm_chipset chipset; + const size_t chipname_length = strnlen(chipname, CPUINFO_BUILD_PROP_VALUE_MAX); + const char* chipname_end = chipname + chipname_length; + + /* Check Qualcomm MSM/APQ signatures */ + if (match_msm_apq(chipname, chipname_end, &chipset)) { + cpuinfo_log_debug( + "matched Qualcomm MSM/APQ signature in ro.chipname string \"%.*s\"", + (int) chipname_length, chipname); + return chipset; + } + + /* Check SMxxxx (Qualcomm Snapdragon) signature */ + if (match_sm(chipname, chipname_end, &chipset)) { + cpuinfo_log_debug( + "matched Qualcomm SM signature in /proc/cpuinfo Hardware string \"%.*s\"", + (int) chipname_length, chipname); + return chipset; + } + + /* Check exynosXXXX (Samsung Exynos) signature */ + if (match_exynos(chipname, chipname_end, &chipset)) { + cpuinfo_log_debug( + "matched exynosXXXX (Samsung Exynos) signature in ro.chipname string \"%.*s\"", + (int) chipname_length, chipname); + return chipset; + } + + /* Check universalXXXX (Samsung Exynos) signature */ + if (match_universal(chipname, chipname_end, &chipset)) { + cpuinfo_log_debug( + "matched UNIVERSAL (Samsung Exynos) signature in ro.chipname Hardware string \"%.*s\"", + (int) chipname_length, chipname); + return chipset; + } + + /* Check MediaTek MT signature */ + if (match_mt(chipname, chipname_end, true, &chipset)) { + cpuinfo_log_debug( + "matched MediaTek MT signature in ro.chipname string \"%.*s\"", + (int) chipname_length, chipname); + return chipset; + } + + /* Check Spreadtrum SC signature */ + if (match_sc(chipname, chipname_end, &chipset)) { + cpuinfo_log_debug( + "matched Spreadtrum SC signature in ro.chipname string \"%.*s\"", + (int) chipname_length, chipname); + return chipset; + } + + #if CPUINFO_ARCH_ARM + /* Check Marvell PXA signature */ + if (match_pxa(chipname, chipname_end, &chipset)) { + cpuinfo_log_debug( + "matched Marvell PXA signature in ro.chipname string \"%.*s\"", + (int) chipname_length, chipname); + return chipset; + } + + /* Compare to ro.chipname value ("mp523x") for Renesas MP5232 which can't be otherwise detected */ + if (chipname_length == 6 && memcmp(chipname, "mp523x", 6) == 0) { + cpuinfo_log_debug( + "matched Renesas MP5232 signature in ro.chipname string \"%.*s\"", + (int) chipname_length, chipname); + + return (struct cpuinfo_arm_chipset) { + .vendor = cpuinfo_arm_chipset_vendor_renesas, + .series = cpuinfo_arm_chipset_series_renesas_mp, + .model = 5232, + }; + } + #endif + + return (struct cpuinfo_arm_chipset) { + .vendor = cpuinfo_arm_chipset_vendor_unknown, + .series = cpuinfo_arm_chipset_series_unknown, + }; + } +#endif /* __ANDROID__ */ + +/* + * Fix common bugs, typos, and renames in chipset name. + * + * @param[in,out] chipset - chipset name to fix. + * @param cores - number of cores in the chipset. + * @param max_cpu_freq_max - maximum of /sys/devices/system/cpu/cpu/cpofreq/cpu_freq_max values. + */ +void cpuinfo_arm_fixup_chipset( + struct cpuinfo_arm_chipset chipset[restrict static 1], uint32_t cores, uint32_t max_cpu_freq_max) +{ + switch (chipset->series) { + case cpuinfo_arm_chipset_series_qualcomm_msm: + /* Check if there is suffix */ + if (chipset->suffix[0] == 0) { + /* No suffix, but the model may be misreported */ + switch (chipset->model) { + case 8216: + /* MSM8216 was renamed to MSM8916 */ + cpuinfo_log_info("reinterpreted MSM8216 chipset as MSM8916"); + chipset->model = 8916; + break; + case 8916: + /* Common bug: MSM8939 (Octa-core) reported as MSM8916 (Quad-core) */ + switch (cores) { + case 4: + break; + case 8: + cpuinfo_log_info("reinterpreted MSM8916 chipset with 8 cores as MSM8939"); + chipset->model = 8939; + break; + default: + cpuinfo_log_warning("system reported invalid %"PRIu32"-core MSM%"PRIu32" chipset", + cores, chipset->model); + chipset->model = 0; + } + break; + case 8937: + /* Common bug: MSM8917 (Quad-core) reported as MSM8937 (Octa-core) */ + switch (cores) { + case 4: + cpuinfo_log_info("reinterpreted MSM8937 chipset with 4 cores as MSM8917"); + chipset->model = 8917; + break; + case 8: + break; + default: + cpuinfo_log_warning("system reported invalid %"PRIu32"-core MSM%"PRIu32" chipset", + cores, chipset->model); + chipset->model = 0; + } + break; + case 8960: + /* Common bug: APQ8064 (Quad-core) reported as MSM8960 (Dual-core) */ + switch (cores) { + case 2: + break; + case 4: + cpuinfo_log_info("reinterpreted MSM8960 chipset with 4 cores as APQ8064"); + chipset->series = cpuinfo_arm_chipset_series_qualcomm_apq; + chipset->model = 8064; + break; + default: + cpuinfo_log_warning("system reported invalid %"PRIu32"-core MSM%"PRIu32" chipset", + cores, chipset->model); + chipset->model = 0; + } + break; + case 8996: + /* Common bug: MSM8994 (Octa-core) reported as MSM8996 (Quad-core) */ + switch (cores) { + case 4: + break; + case 8: + cpuinfo_log_info("reinterpreted MSM8996 chipset with 8 cores as MSM8994"); + chipset->model = 8994; + break; + default: + cpuinfo_log_warning("system reported invalid %"PRIu32"-core MSM%"PRIu32" chipset", + cores, chipset->model); + chipset->model = 0; + } + break; +#if CPUINFO_ARCH_ARM + case 8610: + /* Common bug: MSM8612 (Quad-core) reported as MSM8610 (Dual-core) */ + switch (cores) { + case 2: + break; + case 4: + cpuinfo_log_info("reinterpreted MSM8610 chipset with 4 cores as MSM8612"); + chipset->model = 8612; + break; + default: + cpuinfo_log_warning("system reported invalid %"PRIu32"-core MSM%"PRIu32" chipset", + cores, chipset->model); + chipset->model = 0; + } + break; +#endif /* CPUINFO_ARCH_ARM */ + } + } else { + /* Suffix may need correction */ + const uint32_t suffix_word = load_u32le(chipset->suffix); + if (suffix_word == UINT32_C(0x004D534D) /* "\0MSM" = reverse("MSM\0") */) { + /* + * Common bug: model name repeated twice, e.g. "MSM8916MSM8916" + * In this case, model matching code parses the second "MSM" as a suffix + */ + chipset->suffix[0] = 0; + chipset->suffix[1] = 0; + chipset->suffix[2] = 0; + } else { + switch (chipset->model) { + case 8976: + /* MSM8976SG -> MSM8976PRO */ + if (suffix_word == UINT32_C(0x00004753) /* "\0\0GS" = reverse("SG\0\0") */ ) { + chipset->suffix[0] = 'P'; + chipset->suffix[1] = 'R'; + chipset->suffix[2] = 'O'; + } + break; + case 8996: + /* MSM8996PRO -> MSM8996PRO-AB or MSM8996PRO-AC */ + if (suffix_word == UINT32_C(0x004F5250) /* "\0ORP" = reverse("PRO\0") */ ) { + chipset->suffix[3] = '-'; + chipset->suffix[4] = 'A'; + chipset->suffix[5] = 'B' + (char) (max_cpu_freq_max >= 2188800); + } + break; + } + } + } + break; + case cpuinfo_arm_chipset_series_qualcomm_apq: + { + /* Suffix may need correction */ + const uint32_t expected_apq = load_u32le(chipset->suffix); + if (expected_apq == UINT32_C(0x00515041) /* "\0QPA" = reverse("APQ\0") */) { + /* + * Common bug: model name repeated twice, e.g. "APQ8016APQ8016" + * In this case, model matching code parses the second "APQ" as a suffix + */ + chipset->suffix[0] = 0; + chipset->suffix[1] = 0; + chipset->suffix[2] = 0; + } + break; + } + case cpuinfo_arm_chipset_series_samsung_exynos: + switch (chipset->model) { +#if CPUINFO_ARCH_ARM + case 4410: + /* Exynos 4410 was renamed to Exynos 4412 */ + chipset->model = 4412; + break; + case 5420: + /* Common bug: Exynos 5260 (Hexa-core) reported as Exynos 5420 (Quad-core) */ + switch (cores) { + case 4: + break; + case 6: + cpuinfo_log_info("reinterpreted Exynos 5420 chipset with 6 cores as Exynos 5260"); + chipset->model = 5260; + break; + default: + cpuinfo_log_warning("system reported invalid %"PRIu32"-core Exynos 5420 chipset", cores); + chipset->model = 0; + } + break; +#endif /* CPUINFO_ARCH_ARM */ + case 7580: + /* Common bug: Exynos 7578 (Quad-core) reported as Exynos 7580 (Octa-core) */ + switch (cores) { + case 4: + cpuinfo_log_info("reinterpreted Exynos 7580 chipset with 4 cores as Exynos 7578"); + chipset->model = 7578; + break; + case 8: + break; + default: + cpuinfo_log_warning("system reported invalid %"PRIu32"-core Exynos 7580 chipset", cores); + chipset->model = 0; + } + break; + } + break; + case cpuinfo_arm_chipset_series_mediatek_mt: + if (chipset->model == 6752) { + /* Common bug: MT6732 (Quad-core) reported as MT6752 (Octa-core) */ + switch (cores) { + case 4: + cpuinfo_log_info("reinterpreted MT6752 chipset with 4 cores as MT6732"); + chipset->model = 6732; + break; + case 8: + break; + default: + cpuinfo_log_warning("system reported invalid %"PRIu32"-core MT6752 chipset", cores); + chipset->model = 0; + } + } + if (chipset->suffix[0] == 'T') { + /* Normalization: "TURBO" and "TRUBO" (apparently a typo) -> "T" */ + const uint32_t suffix_word = load_u32le(chipset->suffix + 1); + switch (suffix_word) { + case UINT32_C(0x4F425255): /* "OBRU" = reverse("URBO") */ + case UINT32_C(0x4F425552): /* "OBUR" = reverse("RUBO") */ + if (chipset->suffix[5] == 0) { + chipset->suffix[1] = 0; + chipset->suffix[2] = 0; + chipset->suffix[3] = 0; + chipset->suffix[4] = 0; + } + break; + } + } + break; + case cpuinfo_arm_chipset_series_rockchip_rk: + if (chipset->model == 3288) { + /* Common bug: Rockchip RK3399 (Hexa-core) always reported as RK3288 (Quad-core) */ + switch (cores) { + case 4: + break; + case 6: + cpuinfo_log_info("reinterpreted RK3288 chipset with 6 cores as RK3399"); + chipset->model = 3399; + break; + default: + cpuinfo_log_warning("system reported invalid %"PRIu32"-core RK3288 chipset", cores); + chipset->model = 0; + } + } + break; + default: + break; + } +} + +/* Map from ARM chipset vendor ID to its string representation */ +static const char* chipset_vendor_string[cpuinfo_arm_chipset_vendor_max] = { + [cpuinfo_arm_chipset_vendor_unknown] = "Unknown", + [cpuinfo_arm_chipset_vendor_qualcomm] = "Qualcomm", + [cpuinfo_arm_chipset_vendor_mediatek] = "MediaTek", + [cpuinfo_arm_chipset_vendor_samsung] = "Samsung", + [cpuinfo_arm_chipset_vendor_hisilicon] = "HiSilicon", + [cpuinfo_arm_chipset_vendor_actions] = "Actions", + [cpuinfo_arm_chipset_vendor_allwinner] = "Allwinner", + [cpuinfo_arm_chipset_vendor_amlogic] = "Amlogic", + [cpuinfo_arm_chipset_vendor_broadcom] = "Broadcom", + [cpuinfo_arm_chipset_vendor_lg] = "LG", + [cpuinfo_arm_chipset_vendor_leadcore] = "Leadcore", + [cpuinfo_arm_chipset_vendor_marvell] = "Marvell", + [cpuinfo_arm_chipset_vendor_mstar] = "MStar", + [cpuinfo_arm_chipset_vendor_novathor] = "NovaThor", + [cpuinfo_arm_chipset_vendor_nvidia] = "Nvidia", + [cpuinfo_arm_chipset_vendor_pinecone] = "Pinecone", + [cpuinfo_arm_chipset_vendor_renesas] = "Renesas", + [cpuinfo_arm_chipset_vendor_rockchip] = "Rockchip", + [cpuinfo_arm_chipset_vendor_spreadtrum] = "Spreadtrum", + [cpuinfo_arm_chipset_vendor_telechips] = "Telechips", + [cpuinfo_arm_chipset_vendor_texas_instruments] = "Texas Instruments", + [cpuinfo_arm_chipset_vendor_wondermedia] = "WonderMedia", +}; + +/* Map from ARM chipset series ID to its string representation */ +static const char* chipset_series_string[cpuinfo_arm_chipset_series_max] = { + [cpuinfo_arm_chipset_series_unknown] = NULL, + [cpuinfo_arm_chipset_series_qualcomm_qsd] = "QSD", + [cpuinfo_arm_chipset_series_qualcomm_msm] = "MSM", + [cpuinfo_arm_chipset_series_qualcomm_apq] = "APQ", + [cpuinfo_arm_chipset_series_qualcomm_snapdragon] = "Snapdragon ", + [cpuinfo_arm_chipset_series_mediatek_mt] = "MT", + [cpuinfo_arm_chipset_series_samsung_exynos] = "Exynos ", + [cpuinfo_arm_chipset_series_hisilicon_k3v] = "K3V", + [cpuinfo_arm_chipset_series_hisilicon_hi] = "Hi", + [cpuinfo_arm_chipset_series_hisilicon_kirin] = "Kirin ", + [cpuinfo_arm_chipset_series_actions_atm] = "ATM", + [cpuinfo_arm_chipset_series_allwinner_a] = "A", + [cpuinfo_arm_chipset_series_amlogic_aml] = "AML", + [cpuinfo_arm_chipset_series_amlogic_s] = "S", + [cpuinfo_arm_chipset_series_broadcom_bcm] = "BCM", + [cpuinfo_arm_chipset_series_lg_nuclun] = "Nuclun ", + [cpuinfo_arm_chipset_series_leadcore_lc] = "LC", + [cpuinfo_arm_chipset_series_marvell_pxa] = "PXA", + [cpuinfo_arm_chipset_series_mstar_6a] = "6A", + [cpuinfo_arm_chipset_series_novathor_u] = "U", + [cpuinfo_arm_chipset_series_nvidia_tegra_t] = "Tegra T", + [cpuinfo_arm_chipset_series_nvidia_tegra_ap] = "Tegra AP", + [cpuinfo_arm_chipset_series_nvidia_tegra_sl] = "Tegra SL", + [cpuinfo_arm_chipset_series_pinecone_surge_s] = "Surge S", + [cpuinfo_arm_chipset_series_renesas_mp] = "MP", + [cpuinfo_arm_chipset_series_rockchip_rk] = "RK", + [cpuinfo_arm_chipset_series_spreadtrum_sc] = "SC", + [cpuinfo_arm_chipset_series_telechips_tcc] = "TCC", + [cpuinfo_arm_chipset_series_texas_instruments_omap] = "OMAP", + [cpuinfo_arm_chipset_series_wondermedia_wm] = "WM", +}; + +/* Convert chipset name represented by cpuinfo_arm_chipset structure to a string representation */ +void cpuinfo_arm_chipset_to_string( + const struct cpuinfo_arm_chipset chipset[restrict static 1], + char name[restrict static CPUINFO_ARM_CHIPSET_NAME_MAX]) +{ + enum cpuinfo_arm_chipset_vendor vendor = chipset->vendor; + if (vendor >= cpuinfo_arm_chipset_vendor_max) { + vendor = cpuinfo_arm_chipset_vendor_unknown; + } + enum cpuinfo_arm_chipset_series series = chipset->series; + if (series >= cpuinfo_arm_chipset_series_max) { + series = cpuinfo_arm_chipset_series_unknown; + } + const char* vendor_string = chipset_vendor_string[vendor]; + const char* series_string = chipset_series_string[series]; + const uint32_t model = chipset->model; + if (model == 0) { + if (series == cpuinfo_arm_chipset_series_unknown) { + strncpy(name, vendor_string, CPUINFO_ARM_CHIPSET_NAME_MAX); + } else { + snprintf(name, CPUINFO_ARM_CHIPSET_NAME_MAX, + "%s %s", vendor_string, series_string); + } + } else { + const size_t suffix_length = strnlen(chipset->suffix, CPUINFO_ARM_CHIPSET_SUFFIX_MAX); + snprintf(name, CPUINFO_ARM_CHIPSET_NAME_MAX, + "%s %s%"PRIu32"%.*s", vendor_string, series_string, model, (int) suffix_length, chipset->suffix); + } +} + +#ifdef __ANDROID__ + static inline struct cpuinfo_arm_chipset disambiguate_qualcomm_chipset( + const struct cpuinfo_arm_chipset proc_cpuinfo_hardware_chipset[restrict static 1], + const struct cpuinfo_arm_chipset ro_product_board_chipset[restrict static 1], + const struct cpuinfo_arm_chipset ro_board_platform_chipset[restrict static 1], + const struct cpuinfo_arm_chipset ro_chipname_chipset[restrict static 1], + const struct cpuinfo_arm_chipset ro_hardware_chipname_chipset[restrict static 1]) + { + if (ro_hardware_chipname_chipset->series != cpuinfo_arm_chipset_series_unknown) { + return *ro_hardware_chipname_chipset; + } + if (ro_chipname_chipset->series != cpuinfo_arm_chipset_series_unknown) { + return *ro_chipname_chipset; + } + if (proc_cpuinfo_hardware_chipset->series != cpuinfo_arm_chipset_series_unknown) { + return *proc_cpuinfo_hardware_chipset; + } + if (ro_product_board_chipset->series != cpuinfo_arm_chipset_series_unknown) { + return *ro_product_board_chipset; + } + return *ro_board_platform_chipset; + } + + static inline struct cpuinfo_arm_chipset disambiguate_mediatek_chipset( + const struct cpuinfo_arm_chipset proc_cpuinfo_hardware_chipset[restrict static 1], + const struct cpuinfo_arm_chipset ro_product_board_chipset[restrict static 1], + const struct cpuinfo_arm_chipset ro_board_platform_chipset[restrict static 1], + const struct cpuinfo_arm_chipset ro_mediatek_platform_chipset[restrict static 1], + const struct cpuinfo_arm_chipset ro_chipname_chipset[restrict static 1]) + { + if (ro_chipname_chipset->series != cpuinfo_arm_chipset_series_unknown) { + return *ro_chipname_chipset; + } + if (proc_cpuinfo_hardware_chipset->series != cpuinfo_arm_chipset_series_unknown) { + return *proc_cpuinfo_hardware_chipset; + } + if (ro_product_board_chipset->series != cpuinfo_arm_chipset_series_unknown) { + return *ro_product_board_chipset; + } + if (ro_board_platform_chipset->series != cpuinfo_arm_chipset_series_unknown) { + return *ro_board_platform_chipset; + } + return *ro_mediatek_platform_chipset; + } + + static inline struct cpuinfo_arm_chipset disambiguate_hisilicon_chipset( + const struct cpuinfo_arm_chipset proc_cpuinfo_hardware_chipset[restrict static 1], + const struct cpuinfo_arm_chipset ro_product_board_chipset[restrict static 1], + const struct cpuinfo_arm_chipset ro_board_platform_chipset[restrict static 1]) + { + if (proc_cpuinfo_hardware_chipset->series != cpuinfo_arm_chipset_series_unknown) { + return *proc_cpuinfo_hardware_chipset; + } + if (ro_product_board_chipset->series != cpuinfo_arm_chipset_series_unknown) { + return *ro_product_board_chipset; + } + return *ro_board_platform_chipset; + } + + static inline struct cpuinfo_arm_chipset disambiguate_amlogic_chipset( + const struct cpuinfo_arm_chipset proc_cpuinfo_hardware_chipset[restrict static 1], + const struct cpuinfo_arm_chipset ro_board_platform_chipset[restrict static 1]) + { + if (proc_cpuinfo_hardware_chipset->series != cpuinfo_arm_chipset_series_unknown) { + return *proc_cpuinfo_hardware_chipset; + } + return *ro_board_platform_chipset; + } + + static inline struct cpuinfo_arm_chipset disambiguate_marvell_chipset( + const struct cpuinfo_arm_chipset proc_cpuinfo_hardware_chipset[restrict static 1], + const struct cpuinfo_arm_chipset ro_product_board_chipset[restrict static 1], + const struct cpuinfo_arm_chipset ro_chipname_chipset[restrict static 1]) + { + if (ro_chipname_chipset->series != cpuinfo_arm_chipset_series_unknown) { + return *ro_chipname_chipset; + } + if (ro_product_board_chipset->series != cpuinfo_arm_chipset_series_unknown) { + return *ro_product_board_chipset; + } + return *proc_cpuinfo_hardware_chipset; + } + + static inline struct cpuinfo_arm_chipset disambiguate_rockchip_chipset( + const struct cpuinfo_arm_chipset proc_cpuinfo_hardware_chipset[restrict static 1], + const struct cpuinfo_arm_chipset ro_product_board_chipset[restrict static 1], + const struct cpuinfo_arm_chipset ro_board_platform_chipset[restrict static 1]) + { + if (ro_product_board_chipset->series != cpuinfo_arm_chipset_series_unknown) { + return *ro_product_board_chipset; + } + if (proc_cpuinfo_hardware_chipset->series != cpuinfo_arm_chipset_series_unknown) { + return *proc_cpuinfo_hardware_chipset; + } + return *ro_board_platform_chipset; + } + + static inline struct cpuinfo_arm_chipset disambiguate_spreadtrum_chipset( + const struct cpuinfo_arm_chipset proc_cpuinfo_hardware_chipset[restrict static 1], + const struct cpuinfo_arm_chipset ro_product_board_chipset[restrict static 1], + const struct cpuinfo_arm_chipset ro_board_platform_chipset[restrict static 1], + const struct cpuinfo_arm_chipset ro_chipname_chipset[restrict static 1]) + { + if (ro_chipname_chipset->series != cpuinfo_arm_chipset_series_unknown) { + return *ro_chipname_chipset; + } + if (ro_product_board_chipset->series != cpuinfo_arm_chipset_series_unknown) { + return *ro_product_board_chipset; + } + if (proc_cpuinfo_hardware_chipset->series != cpuinfo_arm_chipset_series_unknown) { + return *proc_cpuinfo_hardware_chipset; + } + return *ro_board_platform_chipset; + } + + /* + * Decodes chipset name from Android system properties: + * - /proc/cpuinfo Hardware string + * - ro.product.board + * - ro.board.platform + * - ro.mediatek.platform + * - ro.chipname + * For some chipsets, the function relies frequency and on number of cores for chipset detection. + * + * @param[in] properties - structure with the Android system properties described above. + * @param cores - number of cores in the chipset. + * @param max_cpu_freq_max - maximum of /sys/devices/system/cpu/cpu/cpofreq/cpu_freq_max values. + * + * @returns Decoded chipset name. If chipset could not be decoded, the resulting structure would use `unknown` vendor + * and series identifiers. + */ + struct cpuinfo_arm_chipset cpuinfo_arm_android_decode_chipset( + const struct cpuinfo_android_properties properties[restrict static 1], + uint32_t cores, + uint32_t max_cpu_freq_max) + { + struct cpuinfo_arm_chipset chipset = { + .vendor = cpuinfo_arm_chipset_vendor_unknown, + .series = cpuinfo_arm_chipset_series_unknown, + }; + + const bool tegra_platform = is_tegra( + properties->ro_board_platform, + properties->ro_board_platform + strnlen(properties->ro_board_platform, CPUINFO_BUILD_PROP_VALUE_MAX)); + + struct cpuinfo_arm_chipset chipsets[cpuinfo_android_chipset_property_max] = { + [cpuinfo_android_chipset_property_proc_cpuinfo_hardware] = + cpuinfo_arm_linux_decode_chipset_from_proc_cpuinfo_hardware( + properties->proc_cpuinfo_hardware, cores, max_cpu_freq_max, tegra_platform), + [cpuinfo_android_chipset_property_ro_product_board] = + cpuinfo_arm_android_decode_chipset_from_ro_product_board( + properties->ro_product_board, cores, max_cpu_freq_max), + [cpuinfo_android_chipset_property_ro_board_platform] = + cpuinfo_arm_android_decode_chipset_from_ro_board_platform( + properties->ro_board_platform, cores, max_cpu_freq_max), + [cpuinfo_android_chipset_property_ro_mediatek_platform] = + cpuinfo_arm_android_decode_chipset_from_ro_mediatek_platform(properties->ro_mediatek_platform), + [cpuinfo_android_chipset_property_ro_arch] = + cpuinfo_arm_android_decode_chipset_from_ro_arch(properties->ro_arch), + [cpuinfo_android_chipset_property_ro_chipname] = + cpuinfo_arm_android_decode_chipset_from_ro_chipname(properties->ro_chipname), + [cpuinfo_android_chipset_property_ro_hardware_chipname] = + cpuinfo_arm_android_decode_chipset_from_ro_chipname(properties->ro_hardware_chipname), + }; + enum cpuinfo_arm_chipset_vendor vendor = cpuinfo_arm_chipset_vendor_unknown; + for (size_t i = 0; i < cpuinfo_android_chipset_property_max; i++) { + const enum cpuinfo_arm_chipset_vendor decoded_vendor = chipsets[i].vendor; + if (decoded_vendor != cpuinfo_arm_chipset_vendor_unknown) { + if (vendor == cpuinfo_arm_chipset_vendor_unknown) { + vendor = decoded_vendor; + } else if (vendor != decoded_vendor) { + /* Parsing different system properties produces different chipset vendors. This situation is rare. */ + cpuinfo_log_error( + "chipset detection failed: different chipset vendors reported in different system properties"); + goto finish; + } + } + } + if (vendor == cpuinfo_arm_chipset_vendor_unknown) { + cpuinfo_log_warning( + "chipset detection failed: none of the system properties matched known signatures"); + goto finish; + } + + /* Fix common bugs in reported chipsets */ + for (size_t i = 0; i < cpuinfo_android_chipset_property_max; i++) { + cpuinfo_arm_fixup_chipset(&chipsets[i], cores, max_cpu_freq_max); + } + + /* + * Propagate suffixes: consider all pairs of chipsets, if both chipsets in the pair are from the same series, + * and one's suffix is a prefix of another's chipset suffix, use the longest suffix. + */ + for (size_t i = 0; i < cpuinfo_android_chipset_property_max; i++) { + const size_t chipset_i_suffix_length = strnlen(chipsets[i].suffix, CPUINFO_ARM_CHIPSET_SUFFIX_MAX); + for (size_t j = 0; j < i; j++) { + if (chipsets[i].series == chipsets[j].series) { + const size_t chipset_j_suffix_length = strnlen(chipsets[j].suffix, CPUINFO_ARM_CHIPSET_SUFFIX_MAX); + if (chipset_i_suffix_length != chipset_j_suffix_length) { + const size_t common_prefix_length = (chipset_i_suffix_length < chipset_j_suffix_length) ? + chipset_i_suffix_length : chipset_j_suffix_length; + if (common_prefix_length == 0 || + memcmp(chipsets[i].suffix, chipsets[j].suffix, common_prefix_length) == 0) + { + if (chipset_i_suffix_length > chipset_j_suffix_length) { + memcpy(chipsets[j].suffix, chipsets[i].suffix, chipset_i_suffix_length); + } else { + memcpy(chipsets[i].suffix, chipsets[j].suffix, chipset_j_suffix_length); + } + } + } + } + } + } + + for (size_t i = 0; i < cpuinfo_android_chipset_property_max; i++) { + if (chipsets[i].series != cpuinfo_arm_chipset_series_unknown) { + if (chipset.series == cpuinfo_arm_chipset_series_unknown) { + chipset = chipsets[i]; + } else if (chipsets[i].series != chipset.series || chipsets[i].model != chipset.model || + strncmp(chipsets[i].suffix, chipset.suffix, CPUINFO_ARM_CHIPSET_SUFFIX_MAX) != 0) + { + cpuinfo_log_info( + "different chipsets reported in different system properties; " + "vendor-specific disambiguation heuristic would be used"); + switch (vendor) { + case cpuinfo_arm_chipset_vendor_qualcomm: + return disambiguate_qualcomm_chipset( + &chipsets[cpuinfo_android_chipset_property_proc_cpuinfo_hardware], + &chipsets[cpuinfo_android_chipset_property_ro_product_board], + &chipsets[cpuinfo_android_chipset_property_ro_board_platform], + &chipsets[cpuinfo_android_chipset_property_ro_chipname], + &chipsets[cpuinfo_android_chipset_property_ro_hardware_chipname]); + case cpuinfo_arm_chipset_vendor_mediatek: + return disambiguate_mediatek_chipset( + &chipsets[cpuinfo_android_chipset_property_proc_cpuinfo_hardware], + &chipsets[cpuinfo_android_chipset_property_ro_product_board], + &chipsets[cpuinfo_android_chipset_property_ro_board_platform], + &chipsets[cpuinfo_android_chipset_property_ro_mediatek_platform], + &chipsets[cpuinfo_android_chipset_property_ro_chipname]); + case cpuinfo_arm_chipset_vendor_hisilicon: + return disambiguate_hisilicon_chipset( + &chipsets[cpuinfo_android_chipset_property_proc_cpuinfo_hardware], + &chipsets[cpuinfo_android_chipset_property_ro_product_board], + &chipsets[cpuinfo_android_chipset_property_ro_board_platform]); + case cpuinfo_arm_chipset_vendor_amlogic: + return disambiguate_amlogic_chipset( + &chipsets[cpuinfo_android_chipset_property_proc_cpuinfo_hardware], + &chipsets[cpuinfo_android_chipset_property_ro_board_platform]); + case cpuinfo_arm_chipset_vendor_marvell: + return disambiguate_marvell_chipset( + &chipsets[cpuinfo_android_chipset_property_proc_cpuinfo_hardware], + &chipsets[cpuinfo_android_chipset_property_ro_product_board], + &chipsets[cpuinfo_android_chipset_property_ro_chipname]); + case cpuinfo_arm_chipset_vendor_rockchip: + return disambiguate_rockchip_chipset( + &chipsets[cpuinfo_android_chipset_property_proc_cpuinfo_hardware], + &chipsets[cpuinfo_android_chipset_property_ro_product_board], + &chipsets[cpuinfo_android_chipset_property_ro_board_platform]); + case cpuinfo_arm_chipset_vendor_spreadtrum: + return disambiguate_spreadtrum_chipset( + &chipsets[cpuinfo_android_chipset_property_proc_cpuinfo_hardware], + &chipsets[cpuinfo_android_chipset_property_ro_product_board], + &chipsets[cpuinfo_android_chipset_property_ro_board_platform], + &chipsets[cpuinfo_android_chipset_property_ro_chipname]); + default: + cpuinfo_log_error( + "chipset detection failed: " + "could not disambiguate different chipsets reported in different system properties"); + /* chipset variable contains valid, but inconsistent chipset information, overwrite it */ + chipset = (struct cpuinfo_arm_chipset) { + .vendor = cpuinfo_arm_chipset_vendor_unknown, + .series = cpuinfo_arm_chipset_series_unknown, + }; + goto finish; + } + } + } + } + + finish: + return chipset; + } +#else /* !defined(__ANDROID__) */ + /* + * Fix commonly misreported Broadcom BCM models on Raspberry Pi boards. + * + * @param[in,out] chipset - chipset name to fix. + * @param[in] revision - /proc/cpuinfo Revision string. + */ + void cpuinfo_arm_fixup_raspberry_pi_chipset( + struct cpuinfo_arm_chipset chipset[restrict static 1], + const char revision[restrict static CPUINFO_HARDWARE_VALUE_MAX]) + { + const size_t revision_length = strnlen(revision, CPUINFO_REVISION_VALUE_MAX); + + /* Parse revision codes according to https://www.raspberrypi.org/documentation/hardware/raspberrypi/revision-codes/README.md */ + #if CPUINFO_ARCH_ARM + if (revision_length == 4) { + /* + * Old-style revision codes. + * All Raspberry Pi models with old-style revision code use Broadcom BCM2835. + */ + + /* BCM2835 often misreported as BCM2708 */ + if (chipset->model == 2708) { + chipset->model = 2835; + } + return; + } + #endif + if ((size_t) (revision_length - 5) <= (size_t) (8 - 5) /* 5 <= length(revision) <= 8 */) { + /* New-style revision codes */ + + uint32_t model = 0; + switch (revision[revision_length - 4]) { + case '0': + /* BCM2835 */ + model = 2835; + break; + case '1': + /* BCM2836 */ + model = 2836; + break; + case '2': + /* BCM2837 */ + model = 2837; + break; + case '3': + /* BCM2711 */ + model = 2711; + break; + } + + if (model != 0) { + chipset->model = model; + chipset->suffix[0] = 0; + } + } + } + + /* + * Decodes chipset name from /proc/cpuinfo Hardware string. + * For some chipsets, the function relies frequency and on number of cores for chipset detection. + * + * @param[in] hardware - /proc/cpuinfo Hardware string. + * @param cores - number of cores in the chipset. + * @param max_cpu_freq_max - maximum of /sys/devices/system/cpu/cpu/cpofreq/cpu_freq_max values. + * + * @returns Decoded chipset name. If chipset could not be decoded, the resulting structure would use `unknown` vendor + * and series identifiers. + */ + struct cpuinfo_arm_chipset cpuinfo_arm_linux_decode_chipset( + const char hardware[restrict static CPUINFO_HARDWARE_VALUE_MAX], + const char revision[restrict static CPUINFO_REVISION_VALUE_MAX], + uint32_t cores, + uint32_t max_cpu_freq_max) + { + struct cpuinfo_arm_chipset chipset = + cpuinfo_arm_linux_decode_chipset_from_proc_cpuinfo_hardware( + hardware, cores, max_cpu_freq_max, false); + if (chipset.vendor == cpuinfo_arm_chipset_vendor_unknown) { + cpuinfo_log_warning( + "chipset detection failed: /proc/cpuinfo Hardware string did not match known signatures"); + } else if (chipset.vendor == cpuinfo_arm_chipset_vendor_broadcom) { + /* Raspberry Pi kernel reports bogus chipset models; detect chipset from RPi revision */ + cpuinfo_arm_fixup_raspberry_pi_chipset(&chipset, revision); + } else { + cpuinfo_arm_fixup_chipset(&chipset, cores, max_cpu_freq_max); + } + return chipset; + } + +#endif diff --git a/3rdparty/cpuinfo/src/arm/linux/clusters.c b/3rdparty/cpuinfo/src/arm/linux/clusters.c new file mode 100644 index 000000000..430773d1d --- /dev/null +++ b/3rdparty/cpuinfo/src/arm/linux/clusters.c @@ -0,0 +1,493 @@ +#include +#include +#include +#include + +#include +#include +#if defined(__ANDROID__) + #include +#endif +#include +#include +#include +#include +#include + +static inline bool bitmask_all(uint32_t bitfield, uint32_t mask) { + return (bitfield & mask) == mask; +} + +/* + * Assigns logical processors to clusters of cores using heuristic based on the typical configuration of clusters for + * 5, 6, 8, and 10 cores: + * - 5 cores (ARM32 Android only): 2 clusters of 4+1 cores + * - 6 cores: 2 clusters of 4+2 cores + * - 8 cores: 2 clusters of 4+4 cores + * - 10 cores: 3 clusters of 4+4+2 cores + * + * The function must be called after parsing OS-provided information on core clusters. + * Its purpose is to detect clusters of cores when OS-provided information is lacking or incomplete, i.e. + * - Linux kernel is not configured to report information in sysfs topology leaf. + * - Linux kernel reports topology information only for online cores, and only cores on one cluster are online, e.g.: + * - Exynos 8890 has 8 cores in 4+4 clusters, but only the first cluster of 4 cores is reported, and cluster + * configuration of logical processors 4-7 is not reported (all remaining processors 4-7 form cluster 1) + * - MT6797 has 10 cores in 4+4+2, but only the first cluster of 4 cores is reported, and cluster configuration + * of logical processors 4-9 is not reported (processors 4-7 form cluster 1, and processors 8-9 form cluster 2). + * + * Heuristic assignment of processors to the above pre-defined clusters fails if such assignment would contradict + * information provided by the operating system: + * - Any of the OS-reported processor clusters is different than the corresponding heuristic cluster. + * - Processors in a heuristic cluster have no OS-provided cluster siblings information, but have known and different + * minimum/maximum frequency. + * - Processors in a heuristic cluster have no OS-provided cluster siblings information, but have known and different + * MIDR components. + * + * If the heuristic assignment of processors to clusters of cores fails, all processors' clusters are unchanged. + * + * @param usable_processors - number of processors in the @p processors array with CPUINFO_LINUX_FLAG_VALID flags. + * @param max_processors - number of elements in the @p processors array. + * @param[in,out] processors - processor descriptors with pre-parsed POSSIBLE and PRESENT flags, minimum/maximum + * frequency, MIDR information, and core cluster (package siblings list) information. + * + * @retval true if the heuristic successfully assigned all processors into clusters of cores. + * @retval false if known details about processors contradict the heuristic configuration of core clusters. + */ +bool cpuinfo_arm_linux_detect_core_clusters_by_heuristic( + uint32_t usable_processors, + uint32_t max_processors, + struct cpuinfo_arm_linux_processor processors[restrict static max_processors]) +{ + uint32_t cluster_processors[3]; + switch (usable_processors) { + case 10: + cluster_processors[0] = 4; + cluster_processors[1] = 4; + cluster_processors[2] = 2; + break; + case 8: + cluster_processors[0] = 4; + cluster_processors[1] = 4; + break; + case 6: + cluster_processors[0] = 4; + cluster_processors[1] = 2; + break; +#if defined(__ANDROID__) && CPUINFO_ARCH_ARM + case 5: + /* + * The only processor with 5 cores is Leadcore L1860C (ARMv7, mobile), + * but this configuration is not too unreasonable for a virtualized ARM server. + */ + cluster_processors[0] = 4; + cluster_processors[1] = 1; + break; +#endif + default: + return false; + } + + /* + * Assignment of processors to core clusters is done in two passes: + * 1. Verify that the clusters proposed by heuristic are compatible with known details about processors. + * 2. If verification passed, update core clusters for the processors. + */ + + uint32_t cluster = 0; + uint32_t expected_cluster_processors = 0; + uint32_t cluster_start, cluster_flags, cluster_midr, cluster_max_frequency, cluster_min_frequency; + bool expected_cluster_exists; + for (uint32_t i = 0; i < max_processors; i++) { + if (bitmask_all(processors[i].flags, CPUINFO_LINUX_FLAG_VALID)) { + if (expected_cluster_processors == 0) { + /* Expect this processor to start a new cluster */ + + expected_cluster_exists = !!(processors[i].flags & CPUINFO_LINUX_FLAG_PACKAGE_CLUSTER); + if (expected_cluster_exists) { + if (processors[i].package_leader_id != i) { + cpuinfo_log_debug( + "heuristic detection of core clusters failed: " + "processor %"PRIu32" is expected to start a new cluster #%"PRIu32" with %"PRIu32" cores, " + "but system siblings lists reported it as a sibling of processor %"PRIu32, + i, cluster, cluster_processors[cluster], processors[i].package_leader_id); + return false; + } + } else { + cluster_flags = 0; + } + + cluster_start = i; + expected_cluster_processors = cluster_processors[cluster++]; + } else { + /* Expect this processor to belong to the same cluster as processor */ + + if (expected_cluster_exists) { + /* + * The cluster suggested by the heuristic was already parsed from system siblings lists. + * For all processors we expect in the cluster, check that: + * - They have pre-assigned cluster from siblings lists (CPUINFO_LINUX_FLAG_PACKAGE_CLUSTER flag). + * - They were assigned to the same cluster based on siblings lists + * (package_leader_id points to the first processor in the cluster). + */ + + if ((processors[i].flags & CPUINFO_LINUX_FLAG_PACKAGE_CLUSTER) == 0) { + cpuinfo_log_debug( + "heuristic detection of core clusters failed: " + "processor %"PRIu32" is expected to belong to the cluster of processor %"PRIu32", " + "but system siblings lists did not report it as a sibling of processor %"PRIu32, + i, cluster_start, cluster_start); + return false; + } + if (processors[i].package_leader_id != cluster_start) { + cpuinfo_log_debug( + "heuristic detection of core clusters failed: " + "processor %"PRIu32" is expected to belong to the cluster of processor %"PRIu32", " + "but system siblings lists reported it to belong to the cluster of processor %"PRIu32, + i, cluster_start, cluster_start); + return false; + } + } else { + /* + * The cluster suggest by the heuristic was not parsed from system siblings lists. + * For all processors we expect in the cluster, check that: + * - They have no pre-assigned cluster from siblings lists. + * - If their min/max CPU frequency is known, it is the same. + * - If any part of their MIDR (Implementer, Variant, Part, Revision) is known, it is the same. + */ + + if (processors[i].flags & CPUINFO_LINUX_FLAG_PACKAGE_CLUSTER) { + cpuinfo_log_debug( + "heuristic detection of core clusters failed: " + "processor %"PRIu32" is expected to be unassigned to any cluster, " + "but system siblings lists reported it to belong to the cluster of processor %"PRIu32, + i, processors[i].package_leader_id); + return false; + } + + if (processors[i].flags & CPUINFO_LINUX_FLAG_MIN_FREQUENCY) { + if (cluster_flags & CPUINFO_LINUX_FLAG_MIN_FREQUENCY) { + if (cluster_min_frequency != processors[i].min_frequency) { + cpuinfo_log_debug( + "heuristic detection of core clusters failed: " + "minimum frequency of processor %"PRIu32" (%"PRIu32" KHz) is different than of its expected cluster (%"PRIu32" KHz)", + i, processors[i].min_frequency, cluster_min_frequency); + return false; + } + } else { + cluster_min_frequency = processors[i].min_frequency; + cluster_flags |= CPUINFO_LINUX_FLAG_MIN_FREQUENCY; + } + } + + if (processors[i].flags & CPUINFO_LINUX_FLAG_MAX_FREQUENCY) { + if (cluster_flags & CPUINFO_LINUX_FLAG_MAX_FREQUENCY) { + if (cluster_max_frequency != processors[i].max_frequency) { + cpuinfo_log_debug( + "heuristic detection of core clusters failed: " + "maximum frequency of processor %"PRIu32" (%"PRIu32" KHz) is different than of its expected cluster (%"PRIu32" KHz)", + i, processors[i].max_frequency, cluster_max_frequency); + return false; + } + } else { + cluster_max_frequency = processors[i].max_frequency; + cluster_flags |= CPUINFO_LINUX_FLAG_MAX_FREQUENCY; + } + } + + if (processors[i].flags & CPUINFO_ARM_LINUX_VALID_IMPLEMENTER) { + if (cluster_flags & CPUINFO_ARM_LINUX_VALID_IMPLEMENTER) { + if ((cluster_midr & CPUINFO_ARM_MIDR_IMPLEMENTER_MASK) != (processors[i].midr & CPUINFO_ARM_MIDR_IMPLEMENTER_MASK)) { + cpuinfo_log_debug( + "heuristic detection of core clusters failed: " + "CPU Implementer of processor %"PRIu32" (0x%02"PRIx32") is different than of its expected cluster (0x%02"PRIx32")", + i, midr_get_implementer(processors[i].midr), midr_get_implementer(cluster_midr)); + return false; + } + } else { + cluster_midr = midr_copy_implementer(cluster_midr, processors[i].midr); + cluster_flags |= CPUINFO_ARM_LINUX_VALID_IMPLEMENTER; + } + } + + if (processors[i].flags & CPUINFO_ARM_LINUX_VALID_VARIANT) { + if (cluster_flags & CPUINFO_ARM_LINUX_VALID_VARIANT) { + if ((cluster_midr & CPUINFO_ARM_MIDR_VARIANT_MASK) != (processors[i].midr & CPUINFO_ARM_MIDR_VARIANT_MASK)) { + cpuinfo_log_debug( + "heuristic detection of core clusters failed: " + "CPU Variant of processor %"PRIu32" (0x%"PRIx32") is different than of its expected cluster (0x%"PRIx32")", + i, midr_get_variant(processors[i].midr), midr_get_variant(cluster_midr)); + return false; + } + } else { + cluster_midr = midr_copy_variant(cluster_midr, processors[i].midr); + cluster_flags |= CPUINFO_ARM_LINUX_VALID_VARIANT; + } + } + + if (processors[i].flags & CPUINFO_ARM_LINUX_VALID_PART) { + if (cluster_flags & CPUINFO_ARM_LINUX_VALID_PART) { + if ((cluster_midr & CPUINFO_ARM_MIDR_PART_MASK) != (processors[i].midr & CPUINFO_ARM_MIDR_PART_MASK)) { + cpuinfo_log_debug( + "heuristic detection of core clusters failed: " + "CPU Part of processor %"PRIu32" (0x%03"PRIx32") is different than of its expected cluster (0x%03"PRIx32")", + i, midr_get_part(processors[i].midr), midr_get_part(cluster_midr)); + return false; + } + } else { + cluster_midr = midr_copy_part(cluster_midr, processors[i].midr); + cluster_flags |= CPUINFO_ARM_LINUX_VALID_PART; + } + } + + if (processors[i].flags & CPUINFO_ARM_LINUX_VALID_REVISION) { + if (cluster_flags & CPUINFO_ARM_LINUX_VALID_REVISION) { + if ((cluster_midr & CPUINFO_ARM_MIDR_REVISION_MASK) != (processors[i].midr & CPUINFO_ARM_MIDR_REVISION_MASK)) { + cpuinfo_log_debug( + "heuristic detection of core clusters failed: " + "CPU Revision of processor %"PRIu32" (0x%"PRIx32") is different than of its expected cluster (0x%"PRIx32")", + i, midr_get_revision(cluster_midr), midr_get_revision(processors[i].midr)); + return false; + } + } else { + cluster_midr = midr_copy_revision(cluster_midr, processors[i].midr); + cluster_flags |= CPUINFO_ARM_LINUX_VALID_REVISION; + } + } + } + } + expected_cluster_processors--; + } + } + + /* Verification passed, assign all processors to new clusters */ + cluster = 0; + expected_cluster_processors = 0; + for (uint32_t i = 0; i < max_processors; i++) { + if (bitmask_all(processors[i].flags, CPUINFO_LINUX_FLAG_VALID)) { + if (expected_cluster_processors == 0) { + /* Expect this processor to start a new cluster */ + + cluster_start = i; + expected_cluster_processors = cluster_processors[cluster++]; + } else { + /* Expect this processor to belong to the same cluster as processor */ + + if (!(processors[i].flags & CPUINFO_LINUX_FLAG_PACKAGE_CLUSTER)) { + cpuinfo_log_debug("assigned processor %"PRIu32" to cluster of processor %"PRIu32" based on heuristic", + i, cluster_start); + } + + processors[i].package_leader_id = cluster_start; + processors[i].flags |= CPUINFO_LINUX_FLAG_PACKAGE_CLUSTER; + } + expected_cluster_processors--; + } + } + return true; +} + +/* + * Assigns logical processors to clusters of cores in sequential manner: + * - Clusters detected from OS-provided information are unchanged: + * - Processors assigned to these clusters stay assigned to the same clusters + * - No new processors are added to these clusters + * - Processors without pre-assigned cluster are clustered in one sequential scan: + * - If known details (min/max frequency, MIDR components) of a processor are compatible with a preceding + * processor, without pre-assigned cluster, the processor is assigned to the cluster of the preceding processor. + * - If known details (min/max frequency, MIDR components) of a processor are not compatible with a preceding + * processor, the processor is assigned to a newly created cluster. + * + * The function must be called after parsing OS-provided information on core clusters, and usually is called only + * if heuristic assignment of processors to clusters (cpuinfo_arm_linux_cluster_processors_by_heuristic) failed. + * + * Its purpose is to detect clusters of cores when OS-provided information is lacking or incomplete, i.e. + * - Linux kernel is not configured to report information in sysfs topology leaf. + * - Linux kernel reports topology information only for online cores, and all cores on some of the clusters are offline. + * + * Sequential assignment of processors to clusters always succeeds, and upon exit, all usable processors in the + * @p processors array have cluster information. + * + * @param max_processors - number of elements in the @p processors array. + * @param[in,out] processors - processor descriptors with pre-parsed POSSIBLE and PRESENT flags, minimum/maximum + * frequency, MIDR information, and core cluster (package siblings list) information. + * + * @retval true if the heuristic successfully assigned all processors into clusters of cores. + * @retval false if known details about processors contradict the heuristic configuration of core clusters. + */ +void cpuinfo_arm_linux_detect_core_clusters_by_sequential_scan( + uint32_t max_processors, + struct cpuinfo_arm_linux_processor processors[restrict static max_processors]) +{ + uint32_t cluster_flags = 0; + uint32_t cluster_processors = 0; + uint32_t cluster_start, cluster_midr, cluster_max_frequency, cluster_min_frequency; + for (uint32_t i = 0; i < max_processors; i++) { + if ((processors[i].flags & (CPUINFO_LINUX_FLAG_VALID | CPUINFO_LINUX_FLAG_PACKAGE_CLUSTER)) == CPUINFO_LINUX_FLAG_VALID) { + if (cluster_processors == 0) { + goto new_cluster; + } + + if (processors[i].flags & CPUINFO_LINUX_FLAG_MIN_FREQUENCY) { + if (cluster_flags & CPUINFO_LINUX_FLAG_MIN_FREQUENCY) { + if (cluster_min_frequency != processors[i].min_frequency) { + cpuinfo_log_info( + "minimum frequency of processor %"PRIu32" (%"PRIu32" KHz) is different than of preceding cluster (%"PRIu32" KHz); " + "processor %"PRIu32" starts to a new cluster", + i, processors[i].min_frequency, cluster_min_frequency, i); + goto new_cluster; + } + } else { + cluster_min_frequency = processors[i].min_frequency; + cluster_flags |= CPUINFO_LINUX_FLAG_MIN_FREQUENCY; + } + } + + if (processors[i].flags & CPUINFO_LINUX_FLAG_MAX_FREQUENCY) { + if (cluster_flags & CPUINFO_LINUX_FLAG_MAX_FREQUENCY) { + if (cluster_max_frequency != processors[i].max_frequency) { + cpuinfo_log_debug( + "maximum frequency of processor %"PRIu32" (%"PRIu32" KHz) is different than of preceding cluster (%"PRIu32" KHz); " + "processor %"PRIu32" starts a new cluster", + i, processors[i].max_frequency, cluster_max_frequency, i); + goto new_cluster; + } + } else { + cluster_max_frequency = processors[i].max_frequency; + cluster_flags |= CPUINFO_LINUX_FLAG_MAX_FREQUENCY; + } + } + + if (processors[i].flags & CPUINFO_ARM_LINUX_VALID_IMPLEMENTER) { + if (cluster_flags & CPUINFO_ARM_LINUX_VALID_IMPLEMENTER) { + if ((cluster_midr & CPUINFO_ARM_MIDR_IMPLEMENTER_MASK) != (processors[i].midr & CPUINFO_ARM_MIDR_IMPLEMENTER_MASK)) { + cpuinfo_log_debug( + "CPU Implementer of processor %"PRIu32" (0x%02"PRIx32") is different than of preceding cluster (0x%02"PRIx32"); " + "processor %"PRIu32" starts to a new cluster", + i, midr_get_implementer(processors[i].midr), midr_get_implementer(cluster_midr), i); + goto new_cluster; + } + } else { + cluster_midr = midr_copy_implementer(cluster_midr, processors[i].midr); + cluster_flags |= CPUINFO_ARM_LINUX_VALID_IMPLEMENTER; + } + } + + if (processors[i].flags & CPUINFO_ARM_LINUX_VALID_VARIANT) { + if (cluster_flags & CPUINFO_ARM_LINUX_VALID_VARIANT) { + if ((cluster_midr & CPUINFO_ARM_MIDR_VARIANT_MASK) != (processors[i].midr & CPUINFO_ARM_MIDR_VARIANT_MASK)) { + cpuinfo_log_debug( + "CPU Variant of processor %"PRIu32" (0x%"PRIx32") is different than of its expected cluster (0x%"PRIx32")" + "processor %"PRIu32" starts to a new cluster", + i, midr_get_variant(processors[i].midr), midr_get_variant(cluster_midr), i); + goto new_cluster; + } + } else { + cluster_midr = midr_copy_variant(cluster_midr, processors[i].midr); + cluster_flags |= CPUINFO_ARM_LINUX_VALID_VARIANT; + } + } + + if (processors[i].flags & CPUINFO_ARM_LINUX_VALID_PART) { + if (cluster_flags & CPUINFO_ARM_LINUX_VALID_PART) { + if ((cluster_midr & CPUINFO_ARM_MIDR_PART_MASK) != (processors[i].midr & CPUINFO_ARM_MIDR_PART_MASK)) { + cpuinfo_log_debug( + "CPU Part of processor %"PRIu32" (0x%03"PRIx32") is different than of its expected cluster (0x%03"PRIx32")" + "processor %"PRIu32" starts to a new cluster", + i, midr_get_part(processors[i].midr), midr_get_part(cluster_midr), i); + goto new_cluster; + } + } else { + cluster_midr = midr_copy_part(cluster_midr, processors[i].midr); + cluster_flags |= CPUINFO_ARM_LINUX_VALID_PART; + } + } + + if (processors[i].flags & CPUINFO_ARM_LINUX_VALID_REVISION) { + if (cluster_flags & CPUINFO_ARM_LINUX_VALID_REVISION) { + if ((cluster_midr & CPUINFO_ARM_MIDR_REVISION_MASK) != (processors[i].midr & CPUINFO_ARM_MIDR_REVISION_MASK)) { + cpuinfo_log_debug( + "CPU Revision of processor %"PRIu32" (0x%"PRIx32") is different than of its expected cluster (0x%"PRIx32")" + "processor %"PRIu32" starts to a new cluster", + i, midr_get_revision(cluster_midr), midr_get_revision(processors[i].midr), i); + goto new_cluster; + } + } else { + cluster_midr = midr_copy_revision(cluster_midr, processors[i].midr); + cluster_flags |= CPUINFO_ARM_LINUX_VALID_REVISION; + } + } + + /* All checks passed, attach processor to the preceding cluster */ + cluster_processors++; + processors[i].package_leader_id = cluster_start; + processors[i].flags |= CPUINFO_LINUX_FLAG_PACKAGE_CLUSTER; + cpuinfo_log_debug("assigned processor %"PRIu32" to preceding cluster of processor %"PRIu32, i, cluster_start); + continue; + +new_cluster: + /* Create a new cluster starting with processor i */ + cluster_start = i; + processors[i].package_leader_id = i; + processors[i].flags |= CPUINFO_LINUX_FLAG_PACKAGE_CLUSTER; + cluster_processors = 1; + + /* Copy known information from processor to cluster, and set the flags accordingly */ + cluster_flags = 0; + if (processors[i].flags & CPUINFO_LINUX_FLAG_MIN_FREQUENCY) { + cluster_min_frequency = processors[i].min_frequency; + cluster_flags |= CPUINFO_LINUX_FLAG_MIN_FREQUENCY; + } + if (processors[i].flags & CPUINFO_LINUX_FLAG_MAX_FREQUENCY) { + cluster_max_frequency = processors[i].max_frequency; + cluster_flags |= CPUINFO_LINUX_FLAG_MAX_FREQUENCY; + } + if (processors[i].flags & CPUINFO_ARM_LINUX_VALID_IMPLEMENTER) { + cluster_midr = midr_copy_implementer(cluster_midr, processors[i].midr); + cluster_flags |= CPUINFO_ARM_LINUX_VALID_IMPLEMENTER; + } + if (processors[i].flags & CPUINFO_ARM_LINUX_VALID_VARIANT) { + cluster_midr = midr_copy_variant(cluster_midr, processors[i].midr); + cluster_flags |= CPUINFO_ARM_LINUX_VALID_VARIANT; + } + if (processors[i].flags & CPUINFO_ARM_LINUX_VALID_PART) { + cluster_midr = midr_copy_part(cluster_midr, processors[i].midr); + cluster_flags |= CPUINFO_ARM_LINUX_VALID_PART; + } + if (processors[i].flags & CPUINFO_ARM_LINUX_VALID_REVISION) { + cluster_midr = midr_copy_revision(cluster_midr, processors[i].midr); + cluster_flags |= CPUINFO_ARM_LINUX_VALID_REVISION; + } + } + } +} + +/* + * Counts the number of logical processors in each core cluster. + * This function should be called after all processors are assigned to core clusters. + * + * @param max_processors - number of elements in the @p processors array. + * @param[in,out] processors - processor descriptors with pre-parsed POSSIBLE and PRESENT flags, + * and decoded core cluster (package_leader_id) information. + * The function expects the value of processors[i].package_processor_count to be zero. + * Upon return, processors[i].package_processor_count will contain the number of logical + * processors in the respective core cluster. + */ +void cpuinfo_arm_linux_count_cluster_processors( + uint32_t max_processors, + struct cpuinfo_arm_linux_processor processors[restrict static max_processors]) +{ + /* First pass: accumulate the number of processors at the group leader's package_processor_count */ + for (uint32_t i = 0; i < max_processors; i++) { + if (bitmask_all(processors[i].flags, CPUINFO_LINUX_FLAG_VALID)) { + const uint32_t package_leader_id = processors[i].package_leader_id; + processors[package_leader_id].package_processor_count += 1; + } + } + /* Second pass: copy the package_processor_count from the group leader processor */ + for (uint32_t i = 0; i < max_processors; i++) { + if (bitmask_all(processors[i].flags, CPUINFO_LINUX_FLAG_VALID)) { + const uint32_t package_leader_id = processors[i].package_leader_id; + processors[i].package_processor_count = processors[package_leader_id].package_processor_count; + } + } +} diff --git a/3rdparty/cpuinfo/src/arm/linux/cp.h b/3rdparty/cpuinfo/src/arm/linux/cp.h new file mode 100644 index 000000000..63940ec5d --- /dev/null +++ b/3rdparty/cpuinfo/src/arm/linux/cp.h @@ -0,0 +1,44 @@ +#include + + +#if CPUINFO_MOCK + extern uint32_t cpuinfo_arm_fpsid; + extern uint32_t cpuinfo_arm_mvfr0; + extern uint32_t cpuinfo_arm_wcid; + + static inline uint32_t read_fpsid(void) { + return cpuinfo_arm_fpsid; + } + + static inline uint32_t read_mvfr0(void) { + return cpuinfo_arm_mvfr0; + } + + static inline uint32_t read_wcid(void) { + return cpuinfo_arm_wcid; + } +#else + #if !defined(__ARM_ARCH_7A__) && !defined(__ARM_ARCH_8A__) && !(defined(__ARM_ARCH) && (__ARM_ARCH >= 7)) + /* + * CoProcessor 10 is inaccessible from user mode since ARMv7, + * and clang refuses to compile inline assembly when targeting ARMv7+ + */ + static inline uint32_t read_fpsid(void) { + uint32_t fpsid; + __asm__ __volatile__("MRC p10, 0x7, %[fpsid], cr0, cr0, 0" : [fpsid] "=r" (fpsid)); + return fpsid; + } + + static inline uint32_t read_mvfr0(void) { + uint32_t mvfr0; + __asm__ __volatile__("MRC p10, 0x7, %[mvfr0], cr7, cr0, 0" : [mvfr0] "=r" (mvfr0)); + return mvfr0; + } + #endif + + static inline uint32_t read_wcid(void) { + uint32_t wcid; + __asm__ __volatile__("MRC p1, 0, %[wcid], c0, c0" : [wcid] "=r" (wcid)); + return wcid; + } +#endif diff --git a/3rdparty/cpuinfo/src/arm/linux/cpuinfo.c b/3rdparty/cpuinfo/src/arm/linux/cpuinfo.c new file mode 100644 index 000000000..90e1631ee --- /dev/null +++ b/3rdparty/cpuinfo/src/arm/linux/cpuinfo.c @@ -0,0 +1,908 @@ +#include +#include +#include +#include +#include + +#include +#include +#include +#include + +/* + * Size, in chars, of the on-stack buffer used for parsing lines of /proc/cpuinfo. + * This is also the limit on the length of a single line. + */ +#define BUFFER_SIZE 1024 + + +static uint32_t parse_processor_number( + const char* processor_start, + const char* processor_end) +{ + const size_t processor_length = (size_t) (processor_end - processor_start); + + if (processor_length == 0) { + cpuinfo_log_warning("Processor number in /proc/cpuinfo is ignored: string is empty"); + return 0; + } + + uint32_t processor_number = 0; + for (const char* digit_ptr = processor_start; digit_ptr != processor_end; digit_ptr++) { + const uint32_t digit = (uint32_t) (*digit_ptr - '0'); + if (digit > 10) { + cpuinfo_log_warning("non-decimal suffix %.*s in /proc/cpuinfo processor number is ignored", + (int) (processor_end - digit_ptr), digit_ptr); + break; + } + + processor_number = processor_number * 10 + digit; + } + + return processor_number; +} + +/* + * Full list of ARM features reported in /proc/cpuinfo: + * + * * swp - support for SWP instruction (deprecated in ARMv7, can be removed in future) + * * half - support for half-word loads and stores. These instruction are part of ARMv4, + * so no need to check it on supported CPUs. + * * thumb - support for 16-bit Thumb instruction set. Note that BX instruction is detected + * by ARMv4T architecture, not by this flag. + * * 26bit - old CPUs merged 26-bit PC and program status register (flags) into 32-bit PC + * and had special instructions for working with packed PC. Now it is all deprecated. + * * fastmult - most old ARM CPUs could only compute 2 bits of multiplication result per clock + * cycle, but CPUs with M suffix (e.g. ARM7TDMI) could compute 4 bits per cycle. + * Of course, now it makes no sense. + * * fpa - floating point accelerator available. On original ARM ABI all floating-point operations + * generated FPA instructions. If FPA was not available, these instructions generated + * "illegal operation" interrupts, and the OS processed them by emulating the FPA instructions. + * Debian used this ABI before it switched to EABI. Now FPA is deprecated. + * * vfp - vector floating point instructions. Available on most modern CPUs (as part of VFPv3). + * Required by Android ARMv7A ABI and by Ubuntu on ARM. + * Note: there is no flag for VFPv2. + * * edsp - V5E instructions: saturating add/sub and 16-bit x 16-bit -> 32/64-bit multiplications. + * Required on Android, supported by all CPUs in production. + * * java - Jazelle extension. Supported on most CPUs. + * * iwmmxt - Intel/Marvell Wireless MMX instructions. 64-bit integer SIMD. + * Supported on XScale (Since PXA270) and Sheeva (PJ1, PJ4) architectures. + * Note that there is no flag for WMMX2 instructions. + * * crunch - Maverick Crunch instructions. Junk. + * * thumbee - ThumbEE instructions. Almost no documentation is available. + * * neon - NEON instructions (aka Advanced SIMD). MVFR1 register gives more + * fine-grained information on particular supported features, but + * the Linux kernel exports only a single flag for all of them. + * According to ARMv7A docs it also implies the availability of VFPv3 + * (with 32 double-precision registers d0-d31). + * * vfpv3 - VFPv3 instructions. Available on most modern CPUs. Augment VFPv2 by + * conversion to/from integers and load constant instructions. + * Required by Android ARMv7A ABI and by Ubuntu on ARM. + * * vfpv3d16 - VFPv3 instructions with only 16 double-precision registers (d0-d15). + * * tls - software thread ID registers. + * Used by kernel (and likely libc) for efficient implementation of TLS. + * * vfpv4 - fused multiply-add instructions. + * * idiva - DIV instructions available in ARM mode. + * * idivt - DIV instructions available in Thumb mode. + * * vfpd32 - VFP (of any version) with 32 double-precision registers d0-d31. + * * lpae - Large Physical Address Extension (physical address up to 40 bits). + * * evtstrm - generation of Event Stream by timer. + * * aes - AES instructions. + * * pmull - Polinomial Multiplication instructions. + * * sha1 - SHA1 instructions. + * * sha2 - SHA2 instructions. + * * crc32 - CRC32 instructions. + * + * /proc/cpuinfo on ARM is populated in file arch/arm/kernel/setup.c in Linux kernel + * Note that some devices may use patched Linux kernels with different feature names. + * However, the names above were checked on a large number of /proc/cpuinfo listings. + */ +static void parse_features( + const char* features_start, + const char* features_end, + struct cpuinfo_arm_linux_processor processor[restrict static 1]) +{ + const char* feature_start = features_start; + const char* feature_end; + + /* Mark the features as valid */ + processor->flags |= CPUINFO_ARM_LINUX_VALID_FEATURES | CPUINFO_ARM_LINUX_VALID_PROCESSOR; + + do { + feature_end = feature_start + 1; + for (; feature_end != features_end; feature_end++) { + if (*feature_end == ' ') { + break; + } + } + const size_t feature_length = (size_t) (feature_end - feature_start); + + switch (feature_length) { + case 2: + if (memcmp(feature_start, "fp", feature_length) == 0) { +#if CPUINFO_ARCH_ARM64 + processor->features |= CPUINFO_ARM_LINUX_FEATURE_FP; +#endif +#if CPUINFO_ARCH_ARM + } else if (memcmp(feature_start, "wp", feature_length) == 0) { + /* + * Some AArch64 kernels, including the one on Nexus 5X, + * erroneously report "swp" as "wp" to AArch32 programs + */ + processor->features |= CPUINFO_ARM_LINUX_FEATURE_SWP; +#endif + } else { + goto unexpected; + } + break; + case 3: + if (memcmp(feature_start, "aes", feature_length) == 0) { + #if CPUINFO_ARCH_ARM + processor->features2 |= CPUINFO_ARM_LINUX_FEATURE2_AES; + #elif CPUINFO_ARCH_ARM64 + processor->features |= CPUINFO_ARM_LINUX_FEATURE_AES; + #endif +#if CPUINFO_ARCH_ARM + } else if (memcmp(feature_start, "swp", feature_length) == 0) { + processor->features |= CPUINFO_ARM_LINUX_FEATURE_SWP; + } else if (memcmp(feature_start, "fpa", feature_length) == 0) { + processor->features |= CPUINFO_ARM_LINUX_FEATURE_FPA; + } else if (memcmp(feature_start, "vfp", feature_length) == 0) { + processor->features |= CPUINFO_ARM_LINUX_FEATURE_VFP; + } else if (memcmp(feature_start, "tls", feature_length) == 0) { + processor->features |= CPUINFO_ARM_LINUX_FEATURE_TLS; +#endif /* CPUINFO_ARCH_ARM */ + } else { + goto unexpected; + } + break; + case 4: + if (memcmp(feature_start, "sha1", feature_length) == 0) { + #if CPUINFO_ARCH_ARM + processor->features2 |= CPUINFO_ARM_LINUX_FEATURE2_SHA1; + #elif CPUINFO_ARCH_ARM64 + processor->features |= CPUINFO_ARM_LINUX_FEATURE_SHA1; + #endif + } else if (memcmp(feature_start, "sha2", feature_length) == 0) { + #if CPUINFO_ARCH_ARM + processor->features2 |= CPUINFO_ARM_LINUX_FEATURE2_SHA2; + #elif CPUINFO_ARCH_ARM64 + processor->features |= CPUINFO_ARM_LINUX_FEATURE_SHA2; + #endif + } else if (memcmp(feature_start, "fphp", feature_length) == 0) { + #if CPUINFO_ARCH_ARM64 + processor->features |= CPUINFO_ARM_LINUX_FEATURE_FPHP; + #endif + } else if (memcmp(feature_start, "fcma", feature_length) == 0) { + #if CPUINFO_ARCH_ARM64 + processor->features |= CPUINFO_ARM_LINUX_FEATURE_FCMA; + #endif +#if CPUINFO_ARCH_ARM + } else if (memcmp(feature_start, "half", feature_length) == 0) { + processor->features |= CPUINFO_ARM_LINUX_FEATURE_HALF; + } else if (memcmp(feature_start, "edsp", feature_length) == 0) { + processor->features |= CPUINFO_ARM_LINUX_FEATURE_EDSP; + } else if (memcmp(feature_start, "java", feature_length) == 0) { + processor->features |= CPUINFO_ARM_LINUX_FEATURE_JAVA; + } else if (memcmp(feature_start, "neon", feature_length) == 0) { + processor->features |= CPUINFO_ARM_LINUX_FEATURE_NEON; + } else if (memcmp(feature_start, "lpae", feature_length) == 0) { + processor->features |= CPUINFO_ARM_LINUX_FEATURE_LPAE; + } else if (memcmp(feature_start, "tlsi", feature_length) == 0) { + /* + * Some AArch64 kernels, including the one on Nexus 5X, + * erroneously report "tls" as "tlsi" to AArch32 programs + */ + processor->features |= CPUINFO_ARM_LINUX_FEATURE_TLS; +#endif /* CPUINFO_ARCH_ARM */ + } else { + goto unexpected; + } + break; + case 5: + if (memcmp(feature_start, "pmull", feature_length) == 0) { + #if CPUINFO_ARCH_ARM + processor->features2 |= CPUINFO_ARM_LINUX_FEATURE2_PMULL; + #elif CPUINFO_ARCH_ARM64 + processor->features |= CPUINFO_ARM_LINUX_FEATURE_PMULL; + #endif + } else if (memcmp(feature_start, "crc32", feature_length) == 0) { + #if CPUINFO_ARCH_ARM + processor->features2 |= CPUINFO_ARM_LINUX_FEATURE2_CRC32; + #elif CPUINFO_ARCH_ARM64 + processor->features |= CPUINFO_ARM_LINUX_FEATURE_CRC32; + #endif + } else if (memcmp(feature_start, "asimd", feature_length) == 0) { + #if CPUINFO_ARCH_ARM64 + processor->features |= CPUINFO_ARM_LINUX_FEATURE_ASIMD; + #endif + } else if (memcmp(feature_start, "cpuid", feature_length) == 0) { + #if CPUINFO_ARCH_ARM64 + processor->features |= CPUINFO_ARM_LINUX_FEATURE_CPUID; + #endif + } else if (memcmp(feature_start, "jscvt", feature_length) == 0) { + #if CPUINFO_ARCH_ARM64 + processor->features |= CPUINFO_ARM_LINUX_FEATURE_JSCVT; + #endif + } else if (memcmp(feature_start, "lrcpc", feature_length) == 0) { + #if CPUINFO_ARCH_ARM64 + processor->features |= CPUINFO_ARM_LINUX_FEATURE_LRCPC; + #endif +#if CPUINFO_ARCH_ARM + } else if (memcmp(feature_start, "thumb", feature_length) == 0) { + processor->features |= CPUINFO_ARM_LINUX_FEATURE_THUMB; + } else if (memcmp(feature_start, "26bit", feature_length) == 0) { + processor->features |= CPUINFO_ARM_LINUX_FEATURE_26BIT; + } else if (memcmp(feature_start, "vfpv3", feature_length) == 0) { + processor->features |= CPUINFO_ARM_LINUX_FEATURE_VFPV3; + } else if (memcmp(feature_start, "vfpv4", feature_length) == 0) { + processor->features |= CPUINFO_ARM_LINUX_FEATURE_VFPV4; + } else if (memcmp(feature_start, "idiva", feature_length) == 0) { + processor->features |= CPUINFO_ARM_LINUX_FEATURE_IDIVA; + } else if (memcmp(feature_start, "idivt", feature_length) == 0) { + processor->features |= CPUINFO_ARM_LINUX_FEATURE_IDIVT; +#endif /* CPUINFO_ARCH_ARM */ + } else { + goto unexpected; + } + break; +#if CPUINFO_ARCH_ARM + case 6: + if (memcmp(feature_start, "iwmmxt", feature_length) == 0) { + processor->features |= CPUINFO_ARM_LINUX_FEATURE_IWMMXT; + } else if (memcmp(feature_start, "crunch", feature_length) == 0) { + processor->features |= CPUINFO_ARM_LINUX_FEATURE_CRUNCH; + } else if (memcmp(feature_start, "vfpd32", feature_length) == 0) { + processor->features |= CPUINFO_ARM_LINUX_FEATURE_VFPD32; + } else { + goto unexpected; + } + break; +#endif /* CPUINFO_ARCH_ARM */ + case 7: + if (memcmp(feature_start, "evtstrm", feature_length) == 0) { + processor->features |= CPUINFO_ARM_LINUX_FEATURE_EVTSTRM; + } else if (memcmp(feature_start, "atomics", feature_length) == 0) { + #if CPUINFO_ARCH_ARM64 + processor->features |= CPUINFO_ARM_LINUX_FEATURE_ATOMICS; + #endif + } else if (memcmp(feature_start, "asimdhp", feature_length) == 0) { + #if CPUINFO_ARCH_ARM64 + processor->features |= CPUINFO_ARM_LINUX_FEATURE_ASIMDHP; + #endif +#if CPUINFO_ARCH_ARM + } else if (memcmp(feature_start, "thumbee", feature_length) == 0) { + processor->features |= CPUINFO_ARM_LINUX_FEATURE_THUMBEE; +#endif /* CPUINFO_ARCH_ARM */ + } else { + goto unexpected; + } + break; + case 8: + if (memcmp(feature_start, "asimdrdm", feature_length) == 0) { + #if CPUINFO_ARCH_ARM64 + processor->features |= CPUINFO_ARM_LINUX_FEATURE_ASIMDRDM; + #endif +#if CPUINFO_ARCH_ARM + } else if (memcmp(feature_start, "fastmult", feature_length) == 0) { + processor->features |= CPUINFO_ARM_LINUX_FEATURE_FASTMULT; + } else if (memcmp(feature_start, "vfpv3d16", feature_length) == 0) { + processor->features |= CPUINFO_ARM_LINUX_FEATURE_VFPV3D16; +#endif /* CPUINFO_ARCH_ARM */ + } else { + goto unexpected; + } + break; + default: + unexpected: + cpuinfo_log_warning("unexpected /proc/cpuinfo feature \"%.*s\" is ignored", + (int) feature_length, feature_start); + break; + } + feature_start = feature_end; + for (; feature_start != features_end; feature_start++) { + if (*feature_start != ' ') { + break; + } + } + } while (feature_start != feature_end); +} + +static void parse_cpu_architecture( + const char* cpu_architecture_start, + const char* cpu_architecture_end, + struct cpuinfo_arm_linux_processor processor[restrict static 1]) +{ + const size_t cpu_architecture_length = (size_t) (cpu_architecture_end - cpu_architecture_start); + /* Early AArch64 kernels report "CPU architecture: AArch64" instead of a numeric value 8 */ + if (cpu_architecture_length == 7) { + if (memcmp(cpu_architecture_start, "AArch64", cpu_architecture_length) == 0) { + processor->midr = midr_set_architecture(processor->midr, UINT32_C(0xF)); + processor->architecture_version = 8; + processor->flags |= CPUINFO_ARM_LINUX_VALID_ARCHITECTURE | CPUINFO_ARM_LINUX_VALID_PROCESSOR; + return; + } + } + + + uint32_t architecture = 0; + const char* cpu_architecture_ptr = cpu_architecture_start; + for (; cpu_architecture_ptr != cpu_architecture_end; cpu_architecture_ptr++) { + const uint32_t digit = (*cpu_architecture_ptr) - '0'; + + /* Verify that CPU architecture is a decimal number */ + if (digit >= 10) { + break; + } + + architecture = architecture * 10 + digit; + } + + if (cpu_architecture_ptr == cpu_architecture_start) { + cpuinfo_log_warning("CPU architecture %.*s in /proc/cpuinfo is ignored due to non-digit at the beginning of the string", + (int) cpu_architecture_length, cpu_architecture_start); + } else { + if (architecture != 0) { + processor->architecture_version = architecture; + processor->flags |= CPUINFO_ARM_LINUX_VALID_ARCHITECTURE | CPUINFO_ARM_LINUX_VALID_PROCESSOR; + + for (; cpu_architecture_ptr != cpu_architecture_end; cpu_architecture_ptr++) { + const char feature = *cpu_architecture_ptr; + switch (feature) { +#if CPUINFO_ARCH_ARM + case 'T': + processor->architecture_flags |= CPUINFO_ARM_LINUX_ARCH_T; + break; + case 'E': + processor->architecture_flags |= CPUINFO_ARM_LINUX_ARCH_E; + break; + case 'J': + processor->architecture_flags |= CPUINFO_ARM_LINUX_ARCH_J; + break; +#endif /* CPUINFO_ARCH_ARM */ + case ' ': + case '\t': + /* Ignore whitespace at the end */ + break; + default: + cpuinfo_log_warning("skipped unknown architectural feature '%c' for ARMv%"PRIu32, + feature, architecture); + break; + } + } + } else { + cpuinfo_log_warning("CPU architecture %.*s in /proc/cpuinfo is ignored due to invalid value (0)", + (int) cpu_architecture_length, cpu_architecture_start); + } + } + + uint32_t midr_architecture = UINT32_C(0xF); +#if CPUINFO_ARCH_ARM + switch (processor->architecture_version) { + case 6: + midr_architecture = UINT32_C(0x7); /* ARMv6 */ + break; + case 5: + if ((processor->architecture_flags & CPUINFO_ARM_LINUX_ARCH_TEJ) == CPUINFO_ARM_LINUX_ARCH_TEJ) { + midr_architecture = UINT32_C(0x6); /* ARMv5TEJ */ + } else if ((processor->architecture_flags & CPUINFO_ARM_LINUX_ARCH_TE) == CPUINFO_ARM_LINUX_ARCH_TE) { + midr_architecture = UINT32_C(0x5); /* ARMv5TE */ + } else { + midr_architecture = UINT32_C(0x4); /* ARMv5T */ + } + break; + } +#endif + processor->midr = midr_set_architecture(processor->midr, midr_architecture); +} + +static void parse_cpu_part( + const char* cpu_part_start, + const char* cpu_part_end, + struct cpuinfo_arm_linux_processor processor[restrict static 1]) +{ + const size_t cpu_part_length = (size_t) (cpu_part_end - cpu_part_start); + + /* + * CPU part should contain hex prefix (0x) and one to three hex digits. + * I have never seen less than three digits as a value of this field, + * but I don't think it is impossible to see such values in future. + * Value can not contain more than three hex digits since + * Main ID Register (MIDR) assigns only a 12-bit value for CPU part. + */ + if (cpu_part_length < 3 || cpu_part_length > 5) { + cpuinfo_log_warning("CPU part %.*s in /proc/cpuinfo is ignored due to unexpected length (%zu)", + (int) cpu_part_length, cpu_part_start, cpu_part_length); + return; + } + + /* Verify the presence of hex prefix */ + if (cpu_part_start[0] != '0' || cpu_part_start[1] != 'x') { + cpuinfo_log_warning("CPU part %.*s in /proc/cpuinfo is ignored due to lack of 0x prefix", + (int) cpu_part_length, cpu_part_start); + return; + } + + /* Verify that characters after hex prefix are hexadecimal digits and decode them */ + uint32_t cpu_part = 0; + for (const char* digit_ptr = cpu_part_start + 2; digit_ptr != cpu_part_end; digit_ptr++) { + const char digit_char = *digit_ptr; + uint32_t digit; + if (digit_char >= '0' && digit_char <= '9') { + digit = digit_char - '0'; + } else if ((uint32_t) (digit_char - 'A') < 6) { + digit = 10 + (digit_char - 'A'); + } else if ((uint32_t) (digit_char - 'a') < 6) { + digit = 10 + (digit_char - 'a'); + } else { + cpuinfo_log_warning("CPU part %.*s in /proc/cpuinfo is ignored due to unexpected non-hex character %c at offset %zu", + (int) cpu_part_length, cpu_part_start, digit_char, (size_t) (digit_ptr - cpu_part_start)); + return; + } + cpu_part = cpu_part * 16 + digit; + } + + processor->midr = midr_set_part(processor->midr, cpu_part); + processor->flags |= CPUINFO_ARM_LINUX_VALID_PART | CPUINFO_ARM_LINUX_VALID_PROCESSOR; +} + +static void parse_cpu_implementer( + const char* cpu_implementer_start, + const char* cpu_implementer_end, + struct cpuinfo_arm_linux_processor processor[restrict static 1]) +{ + const size_t cpu_implementer_length = cpu_implementer_end - cpu_implementer_start; + + /* + * Value should contain hex prefix (0x) and one or two hex digits. + * I have never seen single hex digit as a value of this field, + * but I don't think it is impossible in future. + * Value can not contain more than two hex digits since + * Main ID Register (MIDR) assigns only an 8-bit value for CPU implementer. + */ + switch (cpu_implementer_length) { + case 3: + case 4: + break; + default: + cpuinfo_log_warning("CPU implementer %.*s in /proc/cpuinfo is ignored due to unexpected length (%zu)", + (int) cpu_implementer_length, cpu_implementer_start, cpu_implementer_length); + return; + } + + /* Verify the presence of hex prefix */ + if (cpu_implementer_start[0] != '0' || cpu_implementer_start[1] != 'x') { + cpuinfo_log_warning("CPU implementer %.*s in /proc/cpuinfo is ignored due to lack of 0x prefix", + (int) cpu_implementer_length, cpu_implementer_start); + return; + } + + /* Verify that characters after hex prefix are hexadecimal digits and decode them */ + uint32_t cpu_implementer = 0; + for (const char* digit_ptr = cpu_implementer_start + 2; digit_ptr != cpu_implementer_end; digit_ptr++) { + const char digit_char = *digit_ptr; + uint32_t digit; + if (digit_char >= '0' && digit_char <= '9') { + digit = digit_char - '0'; + } else if ((uint32_t) (digit_char - 'A') < 6) { + digit = 10 + (digit_char - 'A'); + } else if ((uint32_t) (digit_char - 'a') < 6) { + digit = 10 + (digit_char - 'a'); + } else { + cpuinfo_log_warning("CPU implementer %.*s in /proc/cpuinfo is ignored due to unexpected non-hex character '%c' at offset %zu", + (int) cpu_implementer_length, cpu_implementer_start, digit_char, (size_t) (digit_ptr - cpu_implementer_start)); + return; + } + cpu_implementer = cpu_implementer * 16 + digit; + } + + processor->midr = midr_set_implementer(processor->midr, cpu_implementer); + processor->flags |= CPUINFO_ARM_LINUX_VALID_IMPLEMENTER | CPUINFO_ARM_LINUX_VALID_PROCESSOR; +} + +static void parse_cpu_variant( + const char* cpu_variant_start, + const char* cpu_variant_end, + struct cpuinfo_arm_linux_processor processor[restrict static 1]) +{ + const size_t cpu_variant_length = cpu_variant_end - cpu_variant_start; + + /* + * Value should contain hex prefix (0x) and one hex digit. + * Value can not contain more than one hex digits since + * Main ID Register (MIDR) assigns only a 4-bit value for CPU variant. + */ + if (cpu_variant_length != 3) { + cpuinfo_log_warning("CPU variant %.*s in /proc/cpuinfo is ignored due to unexpected length (%zu)", + (int) cpu_variant_length, cpu_variant_start, cpu_variant_length); + return; + } + + /* Skip if there is no hex prefix (0x) */ + if (cpu_variant_start[0] != '0' || cpu_variant_start[1] != 'x') { + cpuinfo_log_warning("CPU variant %.*s in /proc/cpuinfo is ignored due to lack of 0x prefix", + (int) cpu_variant_length, cpu_variant_start); + return; + } + + /* Check if the value after hex prefix is indeed a hex digit and decode it. */ + const char digit_char = cpu_variant_start[2]; + uint32_t cpu_variant; + if ((uint32_t) (digit_char - '0') < 10) { + cpu_variant = (uint32_t) (digit_char - '0'); + } else if ((uint32_t) (digit_char - 'A') < 6) { + cpu_variant = 10 + (uint32_t) (digit_char - 'A'); + } else if ((uint32_t) (digit_char - 'a') < 6) { + cpu_variant = 10 + (uint32_t) (digit_char - 'a'); + } else { + cpuinfo_log_warning("CPU variant %.*s in /proc/cpuinfo is ignored due to unexpected non-hex character '%c'", + (int) cpu_variant_length, cpu_variant_start, digit_char); + return; + } + + processor->midr = midr_set_variant(processor->midr, cpu_variant); + processor->flags |= CPUINFO_ARM_LINUX_VALID_VARIANT | CPUINFO_ARM_LINUX_VALID_PROCESSOR; +} + +static void parse_cpu_revision( + const char* cpu_revision_start, + const char* cpu_revision_end, + struct cpuinfo_arm_linux_processor processor[restrict static 1]) +{ + uint32_t cpu_revision = 0; + for (const char* digit_ptr = cpu_revision_start; digit_ptr != cpu_revision_end; digit_ptr++) { + const uint32_t digit = (uint32_t) (*digit_ptr - '0'); + + /* Verify that the character in CPU revision is a decimal digit */ + if (digit >= 10) { + cpuinfo_log_warning("CPU revision %.*s in /proc/cpuinfo is ignored due to unexpected non-digit character '%c' at offset %zu", + (int) (cpu_revision_end - cpu_revision_start), cpu_revision_start, + *digit_ptr, (size_t) (digit_ptr - cpu_revision_start)); + return; + } + + cpu_revision = cpu_revision * 10 + digit; + } + + processor->midr = midr_set_revision(processor->midr, cpu_revision); + processor->flags |= CPUINFO_ARM_LINUX_VALID_REVISION | CPUINFO_ARM_LINUX_VALID_PROCESSOR; +} + +#if CPUINFO_ARCH_ARM +/* + * Decode one of the cache-related numbers reported by Linux kernel + * for pre-ARMv7 architecture. + * An example cache-related information in /proc/cpuinfo: + * + * I size : 32768 + * I assoc : 4 + * I line length : 32 + * I sets : 256 + * D size : 16384 + * D assoc : 4 + * D line length : 32 + * D sets : 128 + * + */ +static void parse_cache_number( + const char* number_start, + const char* number_end, + const char* number_name, + uint32_t number_ptr[restrict static 1], + uint32_t flags[restrict static 1], + uint32_t number_mask) +{ + uint32_t number = 0; + for (const char* digit_ptr = number_start; digit_ptr != number_end; digit_ptr++) { + const uint32_t digit = *digit_ptr - '0'; + if (digit >= 10) { + cpuinfo_log_warning("%s %.*s in /proc/cpuinfo is ignored due to unexpected non-digit character '%c' at offset %zu", + number_name, (int) (number_end - number_start), number_start, + *digit_ptr, (size_t) (digit_ptr - number_start)); + return; + } + + number = number * 10 + digit; + } + + if (number == 0) { + cpuinfo_log_warning("%s %.*s in /proc/cpuinfo is ignored due to invalid value of zero reported by the kernel", + number_name, (int) (number_end - number_start), number_start); + } + + /* If the number specifies a cache line size, verify that is a reasonable power of 2 */ + if (number_mask & CPUINFO_ARM_LINUX_VALID_CACHE_LINE) { + switch (number) { + case 16: + case 32: + case 64: + case 128: + break; + default: + cpuinfo_log_warning("invalid %s %.*s is ignored: a value of 16, 32, 64, or 128 expected", + number_name, (int) (number_end - number_start), number_start); + } + } + + *number_ptr = number; + *flags |= number_mask | CPUINFO_ARM_LINUX_VALID_PROCESSOR; +} +#endif /* CPUINFO_ARCH_ARM */ + +struct proc_cpuinfo_parser_state { + char* hardware; + char* revision; + uint32_t processor_index; + uint32_t max_processors_count; + struct cpuinfo_arm_linux_processor* processors; + struct cpuinfo_arm_linux_processor dummy_processor; +}; + +/* + * Decode a single line of /proc/cpuinfo information. + * Lines have format [ ]*:[ ] + * An example of /proc/cpuinfo (from Pandaboard-ES): + * + * Processor : ARMv7 Processor rev 10 (v7l) + * processor : 0 + * BogoMIPS : 1392.74 + * + * processor : 1 + * BogoMIPS : 1363.33 + * + * Features : swp half thumb fastmult vfp edsp thumbee neon vfpv3 + * CPU implementer : 0x41 + * CPU architecture: 7 + * CPU variant : 0x2 + * CPU part : 0xc09 + * CPU revision : 10 + * + * Hardware : OMAP4 Panda board + * Revision : 0020 + * Serial : 0000000000000000 + */ +static bool parse_line( + const char* line_start, + const char* line_end, + struct proc_cpuinfo_parser_state state[restrict static 1], + uint64_t line_number) +{ + /* Empty line. Skip. */ + if (line_start == line_end) { + return true; + } + + /* Search for ':' on the line. */ + const char* separator = line_start; + for (; separator != line_end; separator++) { + if (*separator == ':') { + break; + } + } + /* Skip line if no ':' separator was found. */ + if (separator == line_end) { + cpuinfo_log_info("Line %.*s in /proc/cpuinfo is ignored: key/value separator ':' not found", + (int) (line_end - line_start), line_start); + return true; + } + + /* Skip trailing spaces in key part. */ + const char* key_end = separator; + for (; key_end != line_start; key_end--) { + if (key_end[-1] != ' ' && key_end[-1] != '\t') { + break; + } + } + /* Skip line if key contains nothing but spaces. */ + if (key_end == line_start) { + cpuinfo_log_info("Line %.*s in /proc/cpuinfo is ignored: key contains only spaces", + (int) (line_end - line_start), line_start); + return true; + } + + /* Skip leading spaces in value part. */ + const char* value_start = separator + 1; + for (; value_start != line_end; value_start++) { + if (*value_start != ' ') { + break; + } + } + /* Value part contains nothing but spaces. Skip line. */ + if (value_start == line_end) { + cpuinfo_log_info("Line %.*s in /proc/cpuinfo is ignored: value contains only spaces", + (int) (line_end - line_start), line_start); + return true; + } + + /* Skip trailing spaces in value part (if any) */ + const char* value_end = line_end; + for (; value_end != value_start; value_end--) { + if (value_end[-1] != ' ') { + break; + } + } + + const uint32_t processor_index = state->processor_index; + const uint32_t max_processors_count = state->max_processors_count; + struct cpuinfo_arm_linux_processor* processors = state->processors; + struct cpuinfo_arm_linux_processor* processor = &state->dummy_processor; + if (processor_index < max_processors_count) { + processor = &processors[processor_index]; + } + + const size_t key_length = key_end - line_start; + switch (key_length) { + case 6: + if (memcmp(line_start, "Serial", key_length) == 0) { + /* Usually contains just zeros, useless */ +#if CPUINFO_ARCH_ARM + } else if (memcmp(line_start, "I size", key_length) == 0) { + parse_cache_number(value_start, value_end, + "instruction cache size", &processor->proc_cpuinfo_cache.i_size, + &processor->flags, CPUINFO_ARM_LINUX_VALID_ICACHE_SIZE); + } else if (memcmp(line_start, "I sets", key_length) == 0) { + parse_cache_number(value_start, value_end, + "instruction cache sets", &processor->proc_cpuinfo_cache.i_sets, + &processor->flags, CPUINFO_ARM_LINUX_VALID_ICACHE_SETS); + } else if (memcmp(line_start, "D size", key_length) == 0) { + parse_cache_number(value_start, value_end, + "data cache size", &processor->proc_cpuinfo_cache.d_size, + &processor->flags, CPUINFO_ARM_LINUX_VALID_DCACHE_SIZE); + } else if (memcmp(line_start, "D sets", key_length) == 0) { + parse_cache_number(value_start, value_end, + "data cache sets", &processor->proc_cpuinfo_cache.d_sets, + &processor->flags, CPUINFO_ARM_LINUX_VALID_DCACHE_SETS); +#endif /* CPUINFO_ARCH_ARM */ + } else { + goto unknown; + } + break; +#if CPUINFO_ARCH_ARM + case 7: + if (memcmp(line_start, "I assoc", key_length) == 0) { + parse_cache_number(value_start, value_end, + "instruction cache associativity", &processor->proc_cpuinfo_cache.i_assoc, + &processor->flags, CPUINFO_ARM_LINUX_VALID_ICACHE_WAYS); + } else if (memcmp(line_start, "D assoc", key_length) == 0) { + parse_cache_number(value_start, value_end, + "data cache associativity", &processor->proc_cpuinfo_cache.d_assoc, + &processor->flags, CPUINFO_ARM_LINUX_VALID_DCACHE_WAYS); + } else { + goto unknown; + } + break; +#endif /* CPUINFO_ARCH_ARM */ + case 8: + if (memcmp(line_start, "CPU part", key_length) == 0) { + parse_cpu_part(value_start, value_end, processor); + } else if (memcmp(line_start, "Features", key_length) == 0) { + parse_features(value_start, value_end, processor); + } else if (memcmp(line_start, "BogoMIPS", key_length) == 0) { + /* BogoMIPS is useless, don't parse */ + } else if (memcmp(line_start, "Hardware", key_length) == 0) { + size_t value_length = value_end - value_start; + if (value_length > CPUINFO_HARDWARE_VALUE_MAX) { + cpuinfo_log_info( + "length of Hardware value \"%.*s\" in /proc/cpuinfo exceeds limit (%d): truncating to the limit", + (int) value_length, value_start, CPUINFO_HARDWARE_VALUE_MAX); + value_length = CPUINFO_HARDWARE_VALUE_MAX; + } else { + state->hardware[value_length] = '\0'; + } + memcpy(state->hardware, value_start, value_length); + cpuinfo_log_debug("parsed /proc/cpuinfo Hardware = \"%.*s\"", (int) value_length, value_start); + } else if (memcmp(line_start, "Revision", key_length) == 0) { + size_t value_length = value_end - value_start; + if (value_length > CPUINFO_REVISION_VALUE_MAX) { + cpuinfo_log_info( + "length of Revision value \"%.*s\" in /proc/cpuinfo exceeds limit (%d): truncating to the limit", + (int) value_length, value_start, CPUINFO_REVISION_VALUE_MAX); + value_length = CPUINFO_REVISION_VALUE_MAX; + } else { + state->revision[value_length] = '\0'; + } + memcpy(state->revision, value_start, value_length); + cpuinfo_log_debug("parsed /proc/cpuinfo Revision = \"%.*s\"", (int) value_length, value_start); + } else { + goto unknown; + } + break; + case 9: + if (memcmp(line_start, "processor", key_length) == 0) { + const uint32_t new_processor_index = parse_processor_number(value_start, value_end); + if (new_processor_index < processor_index) { + /* Strange: decreasing processor number */ + cpuinfo_log_warning( + "unexpectedly low processor number %"PRIu32" following processor %"PRIu32" in /proc/cpuinfo", + new_processor_index, processor_index); + } else if (new_processor_index > processor_index + 1) { + /* Strange, but common: skipped processor $(processor_index + 1) */ + cpuinfo_log_info( + "unexpectedly high processor number %"PRIu32" following processor %"PRIu32" in /proc/cpuinfo", + new_processor_index, processor_index); + } + if (new_processor_index < max_processors_count) { + /* Record that the processor was mentioned in /proc/cpuinfo */ + processors[new_processor_index].flags |= CPUINFO_ARM_LINUX_VALID_PROCESSOR; + } else { + /* Log and ignore processor */ + cpuinfo_log_warning("processor %"PRIu32" in /proc/cpuinfo is ignored: index exceeds system limit %"PRIu32, + new_processor_index, max_processors_count - 1); + } + state->processor_index = new_processor_index; + return true; + } else if (memcmp(line_start, "Processor", key_length) == 0) { + /* TODO: parse to fix misreported architecture, similar to Android's cpufeatures */ + } else { + goto unknown; + } + break; + case 11: + if (memcmp(line_start, "CPU variant", key_length) == 0) { + parse_cpu_variant(value_start, value_end, processor); + } else { + goto unknown; + } + break; + case 12: + if (memcmp(line_start, "CPU revision", key_length) == 0) { + parse_cpu_revision(value_start, value_end, processor); + } else { + goto unknown; + } + break; +#if CPUINFO_ARCH_ARM + case 13: + if (memcmp(line_start, "I line length", key_length) == 0) { + parse_cache_number(value_start, value_end, + "instruction cache line size", &processor->proc_cpuinfo_cache.i_line_length, + &processor->flags, CPUINFO_ARM_LINUX_VALID_ICACHE_LINE); + } else if (memcmp(line_start, "D line length", key_length) == 0) { + parse_cache_number(value_start, value_end, + "data cache line size", &processor->proc_cpuinfo_cache.d_line_length, + &processor->flags, CPUINFO_ARM_LINUX_VALID_DCACHE_LINE); + } else { + goto unknown; + } + break; +#endif /* CPUINFO_ARCH_ARM */ + case 15: + if (memcmp(line_start, "CPU implementer", key_length) == 0) { + parse_cpu_implementer(value_start, value_end, processor); + } else if (memcmp(line_start, "CPU implementor", key_length) == 0) { + parse_cpu_implementer(value_start, value_end, processor); + } else { + goto unknown; + } + break; + case 16: + if (memcmp(line_start, "CPU architecture", key_length) == 0) { + parse_cpu_architecture(value_start, value_end, processor); + } else { + goto unknown; + } + break; + default: + unknown: + cpuinfo_log_debug("unknown /proc/cpuinfo key: %.*s", (int) key_length, line_start); + + } + return true; +} + +bool cpuinfo_arm_linux_parse_proc_cpuinfo( + char hardware[restrict static CPUINFO_HARDWARE_VALUE_MAX], + char revision[restrict static CPUINFO_REVISION_VALUE_MAX], + uint32_t max_processors_count, + struct cpuinfo_arm_linux_processor processors[restrict static max_processors_count]) +{ + struct proc_cpuinfo_parser_state state = { + .hardware = hardware, + .revision = revision, + .processor_index = 0, + .max_processors_count = max_processors_count, + .processors = processors, + }; + return cpuinfo_linux_parse_multiline_file("/proc/cpuinfo", BUFFER_SIZE, + (cpuinfo_line_callback) parse_line, &state); +} diff --git a/3rdparty/cpuinfo/src/arm/linux/hwcap.c b/3rdparty/cpuinfo/src/arm/linux/hwcap.c new file mode 100644 index 000000000..35e9994f6 --- /dev/null +++ b/3rdparty/cpuinfo/src/arm/linux/hwcap.c @@ -0,0 +1,159 @@ +#include + +#include +#include +#include +#include +#include +#include +#include + +#if CPUINFO_MOCK + #include +#endif +#include +#include +#include + +#if CPUINFO_ARCH_ARM64 || CPUINFO_ARCH_ARM && !defined(__ANDROID__) + #include +#else + #define AT_HWCAP 16 + #define AT_HWCAP2 26 +#endif + + +#if CPUINFO_MOCK + static uint32_t mock_hwcap = 0; + void cpuinfo_set_hwcap(uint32_t hwcap) { + mock_hwcap = hwcap; + } + + static uint32_t mock_hwcap2 = 0; + void cpuinfo_set_hwcap2(uint32_t hwcap2) { + mock_hwcap2 = hwcap2; + } +#endif + + +#if CPUINFO_ARCH_ARM + typedef unsigned long (*getauxval_function_t)(unsigned long); + + bool cpuinfo_arm_linux_hwcap_from_getauxval( + uint32_t hwcap[restrict static 1], + uint32_t hwcap2[restrict static 1]) + { + #if CPUINFO_MOCK + *hwcap = mock_hwcap; + *hwcap2 = mock_hwcap2; + return true; + #elif defined(__ANDROID__) + /* Android: dynamically check if getauxval is supported */ + void* libc = NULL; + getauxval_function_t getauxval = NULL; + + dlerror(); + libc = dlopen("libc.so", RTLD_LAZY); + if (libc == NULL) { + cpuinfo_log_warning("failed to load libc.so: %s", dlerror()); + goto cleanup; + } + + getauxval = (getauxval_function_t) dlsym(libc, "getauxval"); + if (getauxval == NULL) { + cpuinfo_log_info("failed to locate getauxval in libc.so: %s", dlerror()); + goto cleanup; + } + + *hwcap = getauxval(AT_HWCAP); + *hwcap2 = getauxval(AT_HWCAP2); + + cleanup: + if (libc != NULL) { + dlclose(libc); + libc = NULL; + } + return getauxval != NULL; + #else + /* GNU/Linux: getauxval is always supported */ + *hwcap = getauxval(AT_HWCAP); + *hwcap2 = getauxval(AT_HWCAP2); + return true; + #endif + } + + #ifdef __ANDROID__ + bool cpuinfo_arm_linux_hwcap_from_procfs( + uint32_t hwcap[restrict static 1], + uint32_t hwcap2[restrict static 1]) + { + #if CPUINFO_MOCK + *hwcap = mock_hwcap; + *hwcap2 = mock_hwcap2; + return true; + #else + uint32_t hwcaps[2] = { 0, 0 }; + bool result = false; + int file = -1; + + file = open("/proc/self/auxv", O_RDONLY); + if (file == -1) { + cpuinfo_log_warning("failed to open /proc/self/auxv: %s", strerror(errno)); + goto cleanup; + } + + ssize_t bytes_read; + do { + Elf32_auxv_t elf_auxv; + bytes_read = read(file, &elf_auxv, sizeof(Elf32_auxv_t)); + if (bytes_read < 0) { + cpuinfo_log_warning("failed to read /proc/self/auxv: %s", strerror(errno)); + goto cleanup; + } else if (bytes_read > 0) { + if (bytes_read == sizeof(elf_auxv)) { + switch (elf_auxv.a_type) { + case AT_HWCAP: + hwcaps[0] = (uint32_t) elf_auxv.a_un.a_val; + break; + case AT_HWCAP2: + hwcaps[1] = (uint32_t) elf_auxv.a_un.a_val; + break; + } + } else { + cpuinfo_log_warning( + "failed to read %zu bytes from /proc/self/auxv: %zu bytes available", + sizeof(elf_auxv), (size_t) bytes_read); + goto cleanup; + } + } + } while (bytes_read == sizeof(Elf32_auxv_t)); + + /* Success, commit results */ + *hwcap = hwcaps[0]; + *hwcap2 = hwcaps[1]; + result = true; + + cleanup: + if (file != -1) { + close(file); + file = -1; + } + return result; + #endif + } + #endif /* __ANDROID__ */ +#elif CPUINFO_ARCH_ARM64 + void cpuinfo_arm_linux_hwcap_from_getauxval( + uint32_t hwcap[restrict static 1], + uint32_t hwcap2[restrict static 1]) + { + #if CPUINFO_MOCK + *hwcap = mock_hwcap; + *hwcap2 = mock_hwcap2; + #else + *hwcap = (uint32_t) getauxval(AT_HWCAP); + *hwcap2 = (uint32_t) getauxval(AT_HWCAP2); + return ; + #endif + } +#endif diff --git a/3rdparty/cpuinfo/src/arm/linux/init.c b/3rdparty/cpuinfo/src/arm/linux/init.c new file mode 100644 index 000000000..d3da5a9d2 --- /dev/null +++ b/3rdparty/cpuinfo/src/arm/linux/init.c @@ -0,0 +1,765 @@ +#include +#include +#include +#include + +#include +#include +#if defined(__ANDROID__) + #include +#endif +#include +#include +#include +#include +#include + + +struct cpuinfo_arm_isa cpuinfo_isa = { 0 }; + +static struct cpuinfo_package package = { { 0 } }; + +static inline bool bitmask_all(uint32_t bitfield, uint32_t mask) { + return (bitfield & mask) == mask; +} + +static inline uint32_t min(uint32_t a, uint32_t b) { + return a < b ? a : b; +} + +static inline int cmp(uint32_t a, uint32_t b) { + return (a > b) - (a < b); +} + +static bool cluster_siblings_parser( + uint32_t processor, uint32_t siblings_start, uint32_t siblings_end, + struct cpuinfo_arm_linux_processor* processors) +{ + processors[processor].flags |= CPUINFO_LINUX_FLAG_PACKAGE_CLUSTER; + uint32_t package_leader_id = processors[processor].package_leader_id; + + for (uint32_t sibling = siblings_start; sibling < siblings_end; sibling++) { + if (!bitmask_all(processors[sibling].flags, CPUINFO_LINUX_FLAG_VALID)) { + cpuinfo_log_info("invalid processor %"PRIu32" reported as a sibling for processor %"PRIu32, + sibling, processor); + continue; + } + + const uint32_t sibling_package_leader_id = processors[sibling].package_leader_id; + if (sibling_package_leader_id < package_leader_id) { + package_leader_id = sibling_package_leader_id; + } + + processors[sibling].package_leader_id = package_leader_id; + processors[sibling].flags |= CPUINFO_LINUX_FLAG_PACKAGE_CLUSTER; + } + + processors[processor].package_leader_id = package_leader_id; + + return true; +} + +static int cmp_arm_linux_processor(const void* ptr_a, const void* ptr_b) { + const struct cpuinfo_arm_linux_processor* processor_a = (const struct cpuinfo_arm_linux_processor*) ptr_a; + const struct cpuinfo_arm_linux_processor* processor_b = (const struct cpuinfo_arm_linux_processor*) ptr_b; + + /* Move usable processors towards the start of the array */ + const bool usable_a = bitmask_all(processor_a->flags, CPUINFO_LINUX_FLAG_VALID); + const bool usable_b = bitmask_all(processor_b->flags, CPUINFO_LINUX_FLAG_VALID); + if (usable_a != usable_b) { + return (int) usable_b - (int) usable_a; + } + + /* Compare based on core type (e.g. Cortex-A57 < Cortex-A53) */ + const uint32_t midr_a = processor_a->midr; + const uint32_t midr_b = processor_b->midr; + if (midr_a != midr_b) { + const uint32_t score_a = midr_score_core(midr_a); + const uint32_t score_b = midr_score_core(midr_b); + if (score_a != score_b) { + return score_a > score_b ? -1 : 1; + } + } + + /* Compare based on core frequency (e.g. 2.0 GHz < 1.2 GHz) */ + const uint32_t frequency_a = processor_a->max_frequency; + const uint32_t frequency_b = processor_b->max_frequency; + if (frequency_a != frequency_b) { + return frequency_a > frequency_b ? -1 : 1; + } + + /* Compare based on cluster leader id (i.e. cluster 1 < cluster 0) */ + const uint32_t cluster_a = processor_a->package_leader_id; + const uint32_t cluster_b = processor_b->package_leader_id; + if (cluster_a != cluster_b) { + return cluster_a > cluster_b ? -1 : 1; + } + + /* Compare based on system processor id (i.e. processor 0 < processor 1) */ + const uint32_t id_a = processor_a->system_processor_id; + const uint32_t id_b = processor_b->system_processor_id; + return cmp(id_a, id_b); +} + +void cpuinfo_arm_linux_init(void) { + struct cpuinfo_arm_linux_processor* arm_linux_processors = NULL; + struct cpuinfo_processor* processors = NULL; + struct cpuinfo_core* cores = NULL; + struct cpuinfo_cluster* clusters = NULL; + struct cpuinfo_uarch_info* uarchs = NULL; + struct cpuinfo_cache* l1i = NULL; + struct cpuinfo_cache* l1d = NULL; + struct cpuinfo_cache* l2 = NULL; + struct cpuinfo_cache* l3 = NULL; + const struct cpuinfo_processor** linux_cpu_to_processor_map = NULL; + const struct cpuinfo_core** linux_cpu_to_core_map = NULL; + uint32_t* linux_cpu_to_uarch_index_map = NULL; + + const uint32_t max_processors_count = cpuinfo_linux_get_max_processors_count(); + cpuinfo_log_debug("system maximum processors count: %"PRIu32, max_processors_count); + + const uint32_t max_possible_processors_count = 1 + + cpuinfo_linux_get_max_possible_processor(max_processors_count); + cpuinfo_log_debug("maximum possible processors count: %"PRIu32, max_possible_processors_count); + const uint32_t max_present_processors_count = 1 + + cpuinfo_linux_get_max_present_processor(max_processors_count); + cpuinfo_log_debug("maximum present processors count: %"PRIu32, max_present_processors_count); + + uint32_t valid_processor_mask = 0; + uint32_t arm_linux_processors_count = max_processors_count; + if (max_present_processors_count != 0) { + arm_linux_processors_count = min(arm_linux_processors_count, max_present_processors_count); + valid_processor_mask = CPUINFO_LINUX_FLAG_PRESENT; + } + if (max_possible_processors_count != 0) { + arm_linux_processors_count = min(arm_linux_processors_count, max_possible_processors_count); + valid_processor_mask |= CPUINFO_LINUX_FLAG_POSSIBLE; + } + if ((max_present_processors_count | max_possible_processors_count) == 0) { + cpuinfo_log_error("failed to parse both lists of possible and present processors"); + return; + } + + arm_linux_processors = calloc(arm_linux_processors_count, sizeof(struct cpuinfo_arm_linux_processor)); + if (arm_linux_processors == NULL) { + cpuinfo_log_error( + "failed to allocate %zu bytes for descriptions of %"PRIu32" ARM logical processors", + arm_linux_processors_count * sizeof(struct cpuinfo_arm_linux_processor), + arm_linux_processors_count); + return; + } + + if (max_possible_processors_count) { + cpuinfo_linux_detect_possible_processors( + arm_linux_processors_count, &arm_linux_processors->flags, + sizeof(struct cpuinfo_arm_linux_processor), + CPUINFO_LINUX_FLAG_POSSIBLE); + } + + if (max_present_processors_count) { + cpuinfo_linux_detect_present_processors( + arm_linux_processors_count, &arm_linux_processors->flags, + sizeof(struct cpuinfo_arm_linux_processor), + CPUINFO_LINUX_FLAG_PRESENT); + } + +#if defined(__ANDROID__) + struct cpuinfo_android_properties android_properties; + cpuinfo_arm_android_parse_properties(&android_properties); +#else + char proc_cpuinfo_hardware[CPUINFO_HARDWARE_VALUE_MAX]; +#endif + char proc_cpuinfo_revision[CPUINFO_REVISION_VALUE_MAX]; + + if (!cpuinfo_arm_linux_parse_proc_cpuinfo( +#if defined(__ANDROID__) + android_properties.proc_cpuinfo_hardware, +#else + proc_cpuinfo_hardware, +#endif + proc_cpuinfo_revision, + arm_linux_processors_count, + arm_linux_processors)) { + cpuinfo_log_error("failed to parse processor information from /proc/cpuinfo"); + return; + } + + for (uint32_t i = 0; i < arm_linux_processors_count; i++) { + if (bitmask_all(arm_linux_processors[i].flags, valid_processor_mask)) { + arm_linux_processors[i].flags |= CPUINFO_LINUX_FLAG_VALID; + cpuinfo_log_debug("parsed processor %"PRIu32" MIDR 0x%08"PRIx32, + i, arm_linux_processors[i].midr); + } + } + + uint32_t valid_processors = 0, last_midr = 0; + #if CPUINFO_ARCH_ARM + uint32_t last_architecture_version = 0, last_architecture_flags = 0; + #endif + for (uint32_t i = 0; i < arm_linux_processors_count; i++) { + arm_linux_processors[i].system_processor_id = i; + if (bitmask_all(arm_linux_processors[i].flags, CPUINFO_LINUX_FLAG_VALID)) { + valid_processors += 1; + + if (!(arm_linux_processors[i].flags & CPUINFO_ARM_LINUX_VALID_PROCESSOR)) { + /* + * Processor is in possible and present lists, but not reported in /proc/cpuinfo. + * This is fairly common: high-index processors can be not reported if they are offline. + */ + cpuinfo_log_info("processor %"PRIu32" is not listed in /proc/cpuinfo", i); + } + + if (bitmask_all(arm_linux_processors[i].flags, CPUINFO_ARM_LINUX_VALID_MIDR)) { + last_midr = arm_linux_processors[i].midr; + } + #if CPUINFO_ARCH_ARM + if (bitmask_all(arm_linux_processors[i].flags, CPUINFO_ARM_LINUX_VALID_ARCHITECTURE)) { + last_architecture_version = arm_linux_processors[i].architecture_version; + last_architecture_flags = arm_linux_processors[i].architecture_flags; + } + #endif + } else { + /* Processor reported in /proc/cpuinfo, but not in possible and/or present lists: log and ignore */ + if (!(arm_linux_processors[i].flags & CPUINFO_ARM_LINUX_VALID_PROCESSOR)) { + cpuinfo_log_warning("invalid processor %"PRIu32" reported in /proc/cpuinfo", i); + } + } + } + +#if defined(__ANDROID__) + const struct cpuinfo_arm_chipset chipset = + cpuinfo_arm_android_decode_chipset(&android_properties, valid_processors, 0); +#else + const struct cpuinfo_arm_chipset chipset = + cpuinfo_arm_linux_decode_chipset(proc_cpuinfo_hardware, proc_cpuinfo_revision, valid_processors, 0); +#endif + + #if CPUINFO_ARCH_ARM + uint32_t isa_features = 0, isa_features2 = 0; + #ifdef __ANDROID__ + /* + * On Android before API 20, libc.so does not provide getauxval function. + * Thus, we try to dynamically find it, or use two fallback mechanisms: + * 1. dlopen libc.so, and try to find getauxval + * 2. Parse /proc/self/auxv procfs file + * 3. Use features reported in /proc/cpuinfo + */ + if (!cpuinfo_arm_linux_hwcap_from_getauxval(&isa_features, &isa_features2)) { + /* getauxval can't be used, fall back to parsing /proc/self/auxv */ + if (!cpuinfo_arm_linux_hwcap_from_procfs(&isa_features, &isa_features2)) { + /* + * Reading /proc/self/auxv failed, probably due to file permissions. + * Use information from /proc/cpuinfo to detect ISA. + * + * If different processors report different ISA features, take the intersection. + */ + uint32_t processors_with_features = 0; + for (uint32_t i = 0; i < arm_linux_processors_count; i++) { + if (bitmask_all(arm_linux_processors[i].flags, CPUINFO_LINUX_FLAG_VALID | CPUINFO_ARM_LINUX_VALID_FEATURES)) { + if (processors_with_features == 0) { + isa_features = arm_linux_processors[i].features; + isa_features2 = arm_linux_processors[i].features2; + } else { + isa_features &= arm_linux_processors[i].features; + isa_features2 &= arm_linux_processors[i].features2; + } + processors_with_features += 1; + } + } + } + } + #else + /* On GNU/Linux getauxval is always available */ + cpuinfo_arm_linux_hwcap_from_getauxval(&isa_features, &isa_features2); + #endif + cpuinfo_arm_linux_decode_isa_from_proc_cpuinfo( + isa_features, isa_features2, + last_midr, last_architecture_version, last_architecture_flags, + &chipset, &cpuinfo_isa); + #elif CPUINFO_ARCH_ARM64 + uint32_t isa_features = 0, isa_features2 = 0; + /* getauxval is always available on ARM64 Android */ + cpuinfo_arm_linux_hwcap_from_getauxval(&isa_features, &isa_features2); + cpuinfo_arm64_linux_decode_isa_from_proc_cpuinfo( + isa_features, isa_features2, last_midr, &chipset, &cpuinfo_isa); + #endif + + /* Detect min/max frequency and package ID */ + for (uint32_t i = 0; i < arm_linux_processors_count; i++) { + if (bitmask_all(arm_linux_processors[i].flags, CPUINFO_LINUX_FLAG_VALID)) { + const uint32_t max_frequency = cpuinfo_linux_get_processor_max_frequency(i); + if (max_frequency != 0) { + arm_linux_processors[i].max_frequency = max_frequency; + arm_linux_processors[i].flags |= CPUINFO_LINUX_FLAG_MAX_FREQUENCY; + } + + const uint32_t min_frequency = cpuinfo_linux_get_processor_min_frequency(i); + if (min_frequency != 0) { + arm_linux_processors[i].min_frequency = min_frequency; + arm_linux_processors[i].flags |= CPUINFO_LINUX_FLAG_MIN_FREQUENCY; + } + + if (cpuinfo_linux_get_processor_package_id(i, &arm_linux_processors[i].package_id)) { + arm_linux_processors[i].flags |= CPUINFO_LINUX_FLAG_PACKAGE_ID; + } + } + } + + /* Initialize topology group IDs */ + for (uint32_t i = 0; i < arm_linux_processors_count; i++) { + arm_linux_processors[i].package_leader_id = i; + } + + /* Propagate topology group IDs among siblings */ + for (uint32_t i = 0; i < arm_linux_processors_count; i++) { + if (!bitmask_all(arm_linux_processors[i].flags, CPUINFO_LINUX_FLAG_VALID)) { + continue; + } + + if (arm_linux_processors[i].flags & CPUINFO_LINUX_FLAG_PACKAGE_ID) { + cpuinfo_linux_detect_core_siblings( + arm_linux_processors_count, i, + (cpuinfo_siblings_callback) cluster_siblings_parser, + arm_linux_processors); + } + } + + /* Propagate all cluster IDs */ + uint32_t clustered_processors = 0; + for (uint32_t i = 0; i < arm_linux_processors_count; i++) { + if (bitmask_all(arm_linux_processors[i].flags, CPUINFO_LINUX_FLAG_VALID | CPUINFO_LINUX_FLAG_PACKAGE_CLUSTER)) { + clustered_processors += 1; + + const uint32_t package_leader_id = arm_linux_processors[i].package_leader_id; + if (package_leader_id < i) { + arm_linux_processors[i].package_leader_id = arm_linux_processors[package_leader_id].package_leader_id; + } + + cpuinfo_log_debug("processor %"PRIu32" clustered with processor %"PRIu32" as inferred from system siblings lists", + i, arm_linux_processors[i].package_leader_id); + } + } + + if (clustered_processors != valid_processors) { + /* + * Topology information about some or all logical processors may be unavailable, for the following reasons: + * - Linux kernel is too old, or configured without support for topology information in sysfs. + * - Core is offline, and Linux kernel is configured to not report topology for offline cores. + * + * In this case, we assign processors to clusters using two methods: + * - Try heuristic cluster configurations (e.g. 6-core SoC usually has 4+2 big.LITTLE configuration). + * - If heuristic failed, assign processors to core clusters in a sequential scan. + */ + if (!cpuinfo_arm_linux_detect_core_clusters_by_heuristic(valid_processors, arm_linux_processors_count, arm_linux_processors)) { + cpuinfo_arm_linux_detect_core_clusters_by_sequential_scan(arm_linux_processors_count, arm_linux_processors); + } + } + + cpuinfo_arm_linux_count_cluster_processors(arm_linux_processors_count, arm_linux_processors); + + const uint32_t cluster_count = cpuinfo_arm_linux_detect_cluster_midr( + &chipset, + arm_linux_processors_count, valid_processors, arm_linux_processors); + + /* Initialize core vendor, uarch, MIDR, and frequency for every logical processor */ + for (uint32_t i = 0; i < arm_linux_processors_count; i++) { + if (bitmask_all(arm_linux_processors[i].flags, CPUINFO_LINUX_FLAG_VALID)) { + const uint32_t cluster_leader = arm_linux_processors[i].package_leader_id; + if (cluster_leader == i) { + /* Cluster leader: decode core vendor and uarch */ + cpuinfo_arm_decode_vendor_uarch( + arm_linux_processors[cluster_leader].midr, +#if CPUINFO_ARCH_ARM + !!(arm_linux_processors[cluster_leader].features & CPUINFO_ARM_LINUX_FEATURE_VFPV4), +#endif + &arm_linux_processors[cluster_leader].vendor, + &arm_linux_processors[cluster_leader].uarch); + } else { + /* Cluster non-leader: copy vendor, uarch, MIDR, and frequency from cluster leader */ + arm_linux_processors[i].flags |= arm_linux_processors[cluster_leader].flags & + (CPUINFO_ARM_LINUX_VALID_MIDR | CPUINFO_LINUX_FLAG_MAX_FREQUENCY); + arm_linux_processors[i].midr = arm_linux_processors[cluster_leader].midr; + arm_linux_processors[i].vendor = arm_linux_processors[cluster_leader].vendor; + arm_linux_processors[i].uarch = arm_linux_processors[cluster_leader].uarch; + arm_linux_processors[i].max_frequency = arm_linux_processors[cluster_leader].max_frequency; + } + } + } + + for (uint32_t i = 0; i < arm_linux_processors_count; i++) { + if (bitmask_all(arm_linux_processors[i].flags, CPUINFO_LINUX_FLAG_VALID)) { + cpuinfo_log_debug("post-analysis processor %"PRIu32": MIDR %08"PRIx32" frequency %"PRIu32, + i, arm_linux_processors[i].midr, arm_linux_processors[i].max_frequency); + } + } + + qsort(arm_linux_processors, arm_linux_processors_count, + sizeof(struct cpuinfo_arm_linux_processor), cmp_arm_linux_processor); + + for (uint32_t i = 0; i < arm_linux_processors_count; i++) { + if (bitmask_all(arm_linux_processors[i].flags, CPUINFO_LINUX_FLAG_VALID)) { + cpuinfo_log_debug("post-sort processor %"PRIu32": system id %"PRIu32" MIDR %08"PRIx32" frequency %"PRIu32, + i, arm_linux_processors[i].system_processor_id, arm_linux_processors[i].midr, arm_linux_processors[i].max_frequency); + } + } + + uint32_t uarchs_count = 0; + enum cpuinfo_uarch last_uarch; + for (uint32_t i = 0; i < arm_linux_processors_count; i++) { + if (bitmask_all(arm_linux_processors[i].flags, CPUINFO_LINUX_FLAG_VALID)) { + if (uarchs_count == 0 || arm_linux_processors[i].uarch != last_uarch) { + last_uarch = arm_linux_processors[i].uarch; + uarchs_count += 1; + } + arm_linux_processors[i].uarch_index = uarchs_count - 1; + } + } + + /* + * Assumptions: + * - No SMP (i.e. each core supports only one hardware thread). + * - Level 1 instruction and data caches are private to the core clusters. + * - Level 2 and level 3 cache is shared between cores in the same cluster. + */ + cpuinfo_arm_chipset_to_string(&chipset, package.name); + package.processor_count = valid_processors; + package.core_count = valid_processors; + package.cluster_count = cluster_count; + + processors = calloc(valid_processors, sizeof(struct cpuinfo_processor)); + if (processors == NULL) { + cpuinfo_log_error("failed to allocate %zu bytes for descriptions of %"PRIu32" logical processors", + valid_processors * sizeof(struct cpuinfo_processor), valid_processors); + goto cleanup; + } + + cores = calloc(valid_processors, sizeof(struct cpuinfo_core)); + if (cores == NULL) { + cpuinfo_log_error("failed to allocate %zu bytes for descriptions of %"PRIu32" cores", + valid_processors * sizeof(struct cpuinfo_core), valid_processors); + goto cleanup; + } + + clusters = calloc(cluster_count, sizeof(struct cpuinfo_cluster)); + if (clusters == NULL) { + cpuinfo_log_error("failed to allocate %zu bytes for descriptions of %"PRIu32" core clusters", + cluster_count * sizeof(struct cpuinfo_cluster), cluster_count); + goto cleanup; + } + + uarchs = calloc(uarchs_count, sizeof(struct cpuinfo_uarch_info)); + if (uarchs == NULL) { + cpuinfo_log_error("failed to allocate %zu bytes for descriptions of %"PRIu32" microarchitectures", + uarchs_count * sizeof(struct cpuinfo_uarch_info), uarchs_count); + goto cleanup; + } + + linux_cpu_to_processor_map = calloc(arm_linux_processors_count, sizeof(struct cpuinfo_processor*)); + if (linux_cpu_to_processor_map == NULL) { + cpuinfo_log_error("failed to allocate %zu bytes for %"PRIu32" logical processor mapping entries", + arm_linux_processors_count * sizeof(struct cpuinfo_processor*), arm_linux_processors_count); + goto cleanup; + } + + linux_cpu_to_core_map = calloc(arm_linux_processors_count, sizeof(struct cpuinfo_core*)); + if (linux_cpu_to_core_map == NULL) { + cpuinfo_log_error("failed to allocate %zu bytes for %"PRIu32" core mapping entries", + arm_linux_processors_count * sizeof(struct cpuinfo_core*), arm_linux_processors_count); + goto cleanup; + } + + if (uarchs_count > 1) { + linux_cpu_to_uarch_index_map = calloc(arm_linux_processors_count, sizeof(uint32_t)); + if (linux_cpu_to_uarch_index_map == NULL) { + cpuinfo_log_error("failed to allocate %zu bytes for %"PRIu32" uarch index mapping entries", + arm_linux_processors_count * sizeof(uint32_t), arm_linux_processors_count); + goto cleanup; + } + } + + l1i = calloc(valid_processors, sizeof(struct cpuinfo_cache)); + if (l1i == NULL) { + cpuinfo_log_error("failed to allocate %zu bytes for descriptions of %"PRIu32" L1I caches", + valid_processors * sizeof(struct cpuinfo_cache), valid_processors); + goto cleanup; + } + + l1d = calloc(valid_processors, sizeof(struct cpuinfo_cache)); + if (l1d == NULL) { + cpuinfo_log_error("failed to allocate %zu bytes for descriptions of %"PRIu32" L1D caches", + valid_processors * sizeof(struct cpuinfo_cache), valid_processors); + goto cleanup; + } + + uint32_t uarchs_index = 0; + for (uint32_t i = 0; i < arm_linux_processors_count; i++) { + if (bitmask_all(arm_linux_processors[i].flags, CPUINFO_LINUX_FLAG_VALID)) { + if (uarchs_index == 0 || arm_linux_processors[i].uarch != last_uarch) { + last_uarch = arm_linux_processors[i].uarch; + uarchs[uarchs_index] = (struct cpuinfo_uarch_info) { + .uarch = arm_linux_processors[i].uarch, + .midr = arm_linux_processors[i].midr, + }; + uarchs_index += 1; + } + uarchs[uarchs_index - 1].processor_count += 1; + uarchs[uarchs_index - 1].core_count += 1; + } + } + + uint32_t l2_count = 0, l3_count = 0, big_l3_size = 0, cluster_id = UINT32_MAX; + /* Indication whether L3 (if it exists) is shared between all cores */ + bool shared_l3 = true; + /* Populate cache information structures in l1i, l1d */ + for (uint32_t i = 0; i < valid_processors; i++) { + if (arm_linux_processors[i].package_leader_id == arm_linux_processors[i].system_processor_id) { + cluster_id += 1; + clusters[cluster_id] = (struct cpuinfo_cluster) { + .processor_start = i, + .processor_count = arm_linux_processors[i].package_processor_count, + .core_start = i, + .core_count = arm_linux_processors[i].package_processor_count, + .cluster_id = cluster_id, + .package = &package, + .vendor = arm_linux_processors[i].vendor, + .uarch = arm_linux_processors[i].uarch, + .midr = arm_linux_processors[i].midr, + }; + } + + processors[i].smt_id = 0; + processors[i].core = cores + i; + processors[i].cluster = clusters + cluster_id; + processors[i].package = &package; + processors[i].linux_id = (int) arm_linux_processors[i].system_processor_id; + processors[i].cache.l1i = l1i + i; + processors[i].cache.l1d = l1d + i; + linux_cpu_to_processor_map[arm_linux_processors[i].system_processor_id] = &processors[i]; + + cores[i].processor_start = i; + cores[i].processor_count = 1; + cores[i].core_id = i; + cores[i].cluster = clusters + cluster_id; + cores[i].package = &package; + cores[i].vendor = arm_linux_processors[i].vendor; + cores[i].uarch = arm_linux_processors[i].uarch; + cores[i].midr = arm_linux_processors[i].midr; + linux_cpu_to_core_map[arm_linux_processors[i].system_processor_id] = &cores[i]; + + if (linux_cpu_to_uarch_index_map != NULL) { + linux_cpu_to_uarch_index_map[arm_linux_processors[i].system_processor_id] = + arm_linux_processors[i].uarch_index; + } + + struct cpuinfo_cache temp_l2 = { 0 }, temp_l3 = { 0 }; + cpuinfo_arm_decode_cache( + arm_linux_processors[i].uarch, + arm_linux_processors[i].package_processor_count, + arm_linux_processors[i].midr, + &chipset, + cluster_id, + arm_linux_processors[i].architecture_version, + &l1i[i], &l1d[i], &temp_l2, &temp_l3); + l1i[i].processor_start = l1d[i].processor_start = i; + l1i[i].processor_count = l1d[i].processor_count = 1; + #if CPUINFO_ARCH_ARM + /* L1I reported in /proc/cpuinfo overrides defaults */ + if (bitmask_all(arm_linux_processors[i].flags, CPUINFO_ARM_LINUX_VALID_ICACHE)) { + l1i[i] = (struct cpuinfo_cache) { + .size = arm_linux_processors[i].proc_cpuinfo_cache.i_size, + .associativity = arm_linux_processors[i].proc_cpuinfo_cache.i_assoc, + .sets = arm_linux_processors[i].proc_cpuinfo_cache.i_sets, + .partitions = 1, + .line_size = arm_linux_processors[i].proc_cpuinfo_cache.i_line_length + }; + } + /* L1D reported in /proc/cpuinfo overrides defaults */ + if (bitmask_all(arm_linux_processors[i].flags, CPUINFO_ARM_LINUX_VALID_DCACHE)) { + l1d[i] = (struct cpuinfo_cache) { + .size = arm_linux_processors[i].proc_cpuinfo_cache.d_size, + .associativity = arm_linux_processors[i].proc_cpuinfo_cache.d_assoc, + .sets = arm_linux_processors[i].proc_cpuinfo_cache.d_sets, + .partitions = 1, + .line_size = arm_linux_processors[i].proc_cpuinfo_cache.d_line_length + }; + } + #endif + + if (temp_l3.size != 0) { + /* + * Assumptions: + * - L2 is private to each core + * - L3 is shared by cores in the same cluster + * - If cores in different clusters report the same L3, it is shared between all cores. + */ + l2_count += 1; + if (arm_linux_processors[i].package_leader_id == arm_linux_processors[i].system_processor_id) { + if (cluster_id == 0) { + big_l3_size = temp_l3.size; + l3_count = 1; + } else if (temp_l3.size != big_l3_size) { + /* If some cores have different L3 size, L3 is not shared between all cores */ + shared_l3 = false; + l3_count += 1; + } + } + } else { + /* If some cores don't have L3 cache, L3 is not shared between all cores */ + shared_l3 = false; + if (temp_l2.size != 0) { + /* Assume L2 is shared by cores in the same cluster */ + if (arm_linux_processors[i].package_leader_id == arm_linux_processors[i].system_processor_id) { + l2_count += 1; + } + } + } + } + + if (l2_count != 0) { + l2 = calloc(l2_count, sizeof(struct cpuinfo_cache)); + if (l2 == NULL) { + cpuinfo_log_error("failed to allocate %zu bytes for descriptions of %"PRIu32" L2 caches", + l2_count * sizeof(struct cpuinfo_cache), l2_count); + goto cleanup; + } + + if (l3_count != 0) { + l3 = calloc(l3_count, sizeof(struct cpuinfo_cache)); + if (l3 == NULL) { + cpuinfo_log_error("failed to allocate %zu bytes for descriptions of %"PRIu32" L3 caches", + l3_count * sizeof(struct cpuinfo_cache), l3_count); + goto cleanup; + } + } + } + + cluster_id = UINT32_MAX; + uint32_t l2_index = UINT32_MAX, l3_index = UINT32_MAX; + for (uint32_t i = 0; i < valid_processors; i++) { + if (arm_linux_processors[i].package_leader_id == arm_linux_processors[i].system_processor_id) { + cluster_id++; + } + + struct cpuinfo_cache dummy_l1i, dummy_l1d, temp_l2 = { 0 }, temp_l3 = { 0 }; + cpuinfo_arm_decode_cache( + arm_linux_processors[i].uarch, + arm_linux_processors[i].package_processor_count, + arm_linux_processors[i].midr, + &chipset, + cluster_id, + arm_linux_processors[i].architecture_version, + &dummy_l1i, &dummy_l1d, &temp_l2, &temp_l3); + + if (temp_l3.size != 0) { + /* + * Assumptions: + * - L2 is private to each core + * - L3 is shared by cores in the same cluster + * - If cores in different clusters report the same L3, it is shared between all cores. + */ + l2_index += 1; + l2[l2_index] = (struct cpuinfo_cache) { + .size = temp_l2.size, + .associativity = temp_l2.associativity, + .sets = temp_l2.sets, + .partitions = 1, + .line_size = temp_l2.line_size, + .flags = temp_l2.flags, + .processor_start = i, + .processor_count = 1, + }; + processors[i].cache.l2 = l2 + l2_index; + if (arm_linux_processors[i].package_leader_id == arm_linux_processors[i].system_processor_id) { + l3_index += 1; + if (l3_index < l3_count) { + l3[l3_index] = (struct cpuinfo_cache) { + .size = temp_l3.size, + .associativity = temp_l3.associativity, + .sets = temp_l3.sets, + .partitions = 1, + .line_size = temp_l3.line_size, + .flags = temp_l3.flags, + .processor_start = i, + .processor_count = + shared_l3 ? valid_processors : arm_linux_processors[i].package_processor_count, + }; + } + } + if (shared_l3) { + processors[i].cache.l3 = l3; + } else if (l3_index < l3_count) { + processors[i].cache.l3 = l3 + l3_index; + } + } else if (temp_l2.size != 0) { + /* Assume L2 is shared by cores in the same cluster */ + if (arm_linux_processors[i].package_leader_id == arm_linux_processors[i].system_processor_id) { + l2_index += 1; + l2[l2_index] = (struct cpuinfo_cache) { + .size = temp_l2.size, + .associativity = temp_l2.associativity, + .sets = temp_l2.sets, + .partitions = 1, + .line_size = temp_l2.line_size, + .flags = temp_l2.flags, + .processor_start = i, + .processor_count = arm_linux_processors[i].package_processor_count, + }; + } + processors[i].cache.l2 = l2 + l2_index; + } + } + + /* Commit */ + cpuinfo_processors = processors; + cpuinfo_cores = cores; + cpuinfo_clusters = clusters; + cpuinfo_packages = &package; + cpuinfo_uarchs = uarchs; + cpuinfo_cache[cpuinfo_cache_level_1i] = l1i; + cpuinfo_cache[cpuinfo_cache_level_1d] = l1d; + cpuinfo_cache[cpuinfo_cache_level_2] = l2; + cpuinfo_cache[cpuinfo_cache_level_3] = l3; + + cpuinfo_processors_count = valid_processors; + cpuinfo_cores_count = valid_processors; + cpuinfo_clusters_count = cluster_count; + cpuinfo_packages_count = 1; + cpuinfo_uarchs_count = uarchs_count; + cpuinfo_cache_count[cpuinfo_cache_level_1i] = valid_processors; + cpuinfo_cache_count[cpuinfo_cache_level_1d] = valid_processors; + cpuinfo_cache_count[cpuinfo_cache_level_2] = l2_count; + cpuinfo_cache_count[cpuinfo_cache_level_3] = l3_count; + cpuinfo_max_cache_size = cpuinfo_arm_compute_max_cache_size(&processors[0]); + + cpuinfo_linux_cpu_max = arm_linux_processors_count; + cpuinfo_linux_cpu_to_processor_map = linux_cpu_to_processor_map; + cpuinfo_linux_cpu_to_core_map = linux_cpu_to_core_map; + cpuinfo_linux_cpu_to_uarch_index_map = linux_cpu_to_uarch_index_map; + + __sync_synchronize(); + + cpuinfo_is_initialized = true; + + processors = NULL; + cores = NULL; + clusters = NULL; + uarchs = NULL; + l1i = l1d = l2 = l3 = NULL; + linux_cpu_to_processor_map = NULL; + linux_cpu_to_core_map = NULL; + linux_cpu_to_uarch_index_map = NULL; + +cleanup: + free(arm_linux_processors); + free(processors); + free(cores); + free(clusters); + free(uarchs); + free(l1i); + free(l1d); + free(l2); + free(l3); + free(linux_cpu_to_processor_map); + free(linux_cpu_to_core_map); + free(linux_cpu_to_uarch_index_map); +} diff --git a/3rdparty/cpuinfo/src/arm/linux/midr.c b/3rdparty/cpuinfo/src/arm/linux/midr.c new file mode 100644 index 000000000..0d8f03fa8 --- /dev/null +++ b/3rdparty/cpuinfo/src/arm/linux/midr.c @@ -0,0 +1,863 @@ +#include +#include +#include +#include + +#include +#include +#if defined(__ANDROID__) + #include +#endif +#include +#include +#include +#include +#include +#include + + +#define CLUSTERS_MAX 3 + +static inline bool bitmask_all(uint32_t bitfield, uint32_t mask) { + return (bitfield & mask) == mask; +} + +/* Description of core clusters configuration in a chipset (identified by series and model number) */ +struct cluster_config { + /* Number of cores (logical processors) */ + uint8_t cores; + /* ARM chipset series (see cpuinfo_arm_chipset_series enum) */ + uint8_t series; + /* Chipset model number (see cpuinfo_arm_chipset struct) */ + uint16_t model; + /* Number of heterogenous clusters in the CPU package */ + uint8_t clusters; + /* + * Number of cores in each cluster: + # - Symmetric configurations: [0] = # cores + * - big.LITTLE configurations: [0] = # LITTLE cores, [1] = # big cores + * - Max.Med.Min configurations: [0] = # Min cores, [1] = # Med cores, [2] = # Max cores + */ + uint8_t cluster_cores[CLUSTERS_MAX]; + /* + * MIDR of cores in each cluster: + * - Symmetric configurations: [0] = core MIDR + * - big.LITTLE configurations: [0] = LITTLE core MIDR, [1] = big core MIDR + * - Max.Med.Min configurations: [0] = Min core MIDR, [1] = Med core MIDR, [2] = Max core MIDR + */ + uint32_t cluster_midr[CLUSTERS_MAX]; +}; + +/* + * The list of chipsets where MIDR may not be unambigiously decoded at least on some devices. + * The typical reasons for impossibility to decoded MIDRs are buggy kernels, which either do not report all MIDR + * information (e.g. on ATM7029 kernel doesn't report CPU Part), or chipsets have more than one type of cores + * (i.e. 4x Cortex-A53 + 4x Cortex-A53 is out) and buggy kernels report MIDR information only about some cores + * in /proc/cpuinfo (either only online cores, or only the core that reads /proc/cpuinfo). On these kernels/chipsets, + * it is not possible to detect all core types by just parsing /proc/cpuinfo, so we use chipset name and this table to + * find their MIDR (and thus microarchitecture, cache, etc). + * + * Note: not all chipsets with heterogeneous multiprocessing need an entry in this table. The following HMP + * chipsets always list information about all cores in /proc/cpuinfo: + * + * - Snapdragon 660 + * - Snapdragon 820 (MSM8996) + * - Snapdragon 821 (MSM8996PRO) + * - Snapdragon 835 (MSM8998) + * - Exynos 8895 + * - Kirin 960 + * + * As these are all new processors, there is hope that this table won't uncontrollably grow over time. + */ +static const struct cluster_config cluster_configs[] = { +#if CPUINFO_ARCH_ARM + { + /* + * MSM8916 (Snapdragon 410): 4x Cortex-A53 + * Some AArch32 phones use non-standard /proc/cpuinfo format. + */ + .cores = 4, + .series = cpuinfo_arm_chipset_series_qualcomm_msm, + .model = UINT16_C(8916), + .clusters = 1, + .cluster_cores = { + [0] = 4, + }, + .cluster_midr = { + [0] = UINT32_C(0x410FD030), + }, + }, + { + /* + * MSM8939 (Snapdragon 615): 4x Cortex-A53 + 4x Cortex-A53 + * Some AArch32 phones use non-standard /proc/cpuinfo format. + */ + .cores = 8, + .series = cpuinfo_arm_chipset_series_qualcomm_msm, + .model = UINT16_C(8939), + .clusters = 2, + .cluster_cores = { + [0] = 4, + [1] = 4, + }, + .cluster_midr = { + [0] = UINT32_C(0x410FD034), + [1] = UINT32_C(0x410FD034), + }, + }, +#endif + { + /* MSM8956 (Snapdragon 650): 2x Cortex-A72 + 4x Cortex-A53 */ + .cores = 6, + .series = cpuinfo_arm_chipset_series_qualcomm_msm, + .model = UINT16_C(8956), + .clusters = 2, + .cluster_cores = { + [0] = 4, + [1] = 2, + }, + .cluster_midr = { + [0] = UINT32_C(0x410FD034), + [1] = UINT32_C(0x410FD080), + }, + }, + { + /* MSM8976/MSM8976PRO (Snapdragon 652/653): 4x Cortex-A72 + 4x Cortex-A53 */ + .cores = 8, + .series = cpuinfo_arm_chipset_series_qualcomm_msm, + .model = UINT16_C(8976), + .clusters = 2, + .cluster_cores = { + [0] = 4, + [1] = 4, + }, + .cluster_midr = { + [0] = UINT32_C(0x410FD034), + [1] = UINT32_C(0x410FD080), + }, + }, + { + /* MSM8992 (Snapdragon 808): 2x Cortex-A57 + 4x Cortex-A53 */ + .cores = 6, + .series = cpuinfo_arm_chipset_series_qualcomm_msm, + .model = UINT16_C(8992), + .clusters = 2, + .cluster_cores = { + [0] = 4, + [1] = 2, + }, + .cluster_midr = { + [0] = UINT32_C(0x410FD033), + [1] = UINT32_C(0x411FD072), + }, + }, + { + /* MSM8994/MSM8994V (Snapdragon 810): 4x Cortex-A57 + 4x Cortex-A53 */ + .cores = 8, + .series = cpuinfo_arm_chipset_series_qualcomm_msm, + .model = UINT16_C(8994), + .clusters = 2, + .cluster_cores = { + [0] = 4, + [1] = 4, + }, + .cluster_midr = { + [0] = UINT32_C(0x410FD032), + [1] = UINT32_C(0x411FD071), + }, + }, +#if CPUINFO_ARCH_ARM + { + /* Exynos 5422: 4x Cortex-A15 + 4x Cortex-A7 */ + .cores = 8, + .series = cpuinfo_arm_chipset_series_samsung_exynos, + .model = UINT16_C(5422), + .clusters = 2, + .cluster_cores = { + [0] = 4, + [1] = 4, + }, + .cluster_midr = { + [0] = UINT32_C(0x410FC073), + [1] = UINT32_C(0x412FC0F3), + }, + }, + { + /* Exynos 5430: 4x Cortex-A15 + 4x Cortex-A7 */ + .cores = 8, + .series = cpuinfo_arm_chipset_series_samsung_exynos, + .model = UINT16_C(5430), + .clusters = 2, + .cluster_cores = { + [0] = 4, + [1] = 4, + }, + .cluster_midr = { + [0] = UINT32_C(0x410FC074), + [1] = UINT32_C(0x413FC0F3), + }, + }, +#endif /* CPUINFO_ARCH_ARM */ + { + /* Exynos 5433: 4x Cortex-A57 + 4x Cortex-A53 */ + .cores = 8, + .series = cpuinfo_arm_chipset_series_samsung_exynos, + .model = UINT16_C(5433), + .clusters = 2, + .cluster_cores = { + [0] = 4, + [1] = 4, + }, + .cluster_midr = { + [0] = UINT32_C(0x410FD031), + [1] = UINT32_C(0x411FD070), + }, + }, + { + /* Exynos 7420: 4x Cortex-A57 + 4x Cortex-A53 */ + .cores = 8, + .series = cpuinfo_arm_chipset_series_samsung_exynos, + .model = UINT16_C(7420), + .clusters = 2, + .cluster_cores = { + [0] = 4, + [1] = 4, + }, + .cluster_midr = { + [0] = UINT32_C(0x410FD032), + [1] = UINT32_C(0x411FD070), + }, + }, + { + /* Exynos 8890: 4x Exynos M1 + 4x Cortex-A53 */ + .cores = 8, + .series = cpuinfo_arm_chipset_series_samsung_exynos, + .model = UINT16_C(8890), + .clusters = 2, + .cluster_cores = { + [0] = 4, + [1] = 4, + }, + .cluster_midr = { + [0] = UINT32_C(0x410FD034), + [1] = UINT32_C(0x531F0011), + }, + }, +#if CPUINFO_ARCH_ARM + { + /* Kirin 920: 4x Cortex-A15 + 4x Cortex-A7 */ + .cores = 8, + .series = cpuinfo_arm_chipset_series_hisilicon_kirin, + .model = UINT16_C(920), + .clusters = 2, + .cluster_cores = { + [0] = 4, + [1] = 4, + }, + .cluster_midr = { + [0] = UINT32_C(0x410FC075), + [1] = UINT32_C(0x413FC0F3), + }, + }, + { + /* Kirin 925: 4x Cortex-A15 + 4x Cortex-A7 */ + .cores = 8, + .series = cpuinfo_arm_chipset_series_hisilicon_kirin, + .model = UINT16_C(925), + .clusters = 2, + .cluster_cores = { + [0] = 4, + [1] = 4, + }, + .cluster_midr = { + [0] = UINT32_C(0x410FC075), + [1] = UINT32_C(0x413FC0F3), + }, + }, + { + /* Kirin 928: 4x Cortex-A15 + 4x Cortex-A7 */ + .cores = 8, + .series = cpuinfo_arm_chipset_series_hisilicon_kirin, + .model = UINT16_C(928), + .clusters = 2, + .cluster_cores = { + [0] = 4, + [1] = 4, + }, + .cluster_midr = { + [0] = UINT32_C(0x410FC075), + [1] = UINT32_C(0x413FC0F3), + }, + }, +#endif /* CPUINFO_ARCH_ARM */ + { + /* Kirin 950: 4x Cortex-A72 + 4x Cortex-A53 */ + .cores = 8, + .series = cpuinfo_arm_chipset_series_hisilicon_kirin, + .model = UINT16_C(950), + .clusters = 2, + .cluster_cores = { + [0] = 4, + [1] = 4, + }, + .cluster_midr = { + [0] = UINT32_C(0x410FD034), + [1] = UINT32_C(0x410FD080), + }, + }, + { + /* Kirin 955: 4x Cortex-A72 + 4x Cortex-A53 */ + .cores = 8, + .series = cpuinfo_arm_chipset_series_hisilicon_kirin, + .model = UINT16_C(955), + .clusters = 2, + .cluster_cores = { + [0] = 4, + [1] = 4, + }, + .cluster_midr = { + [0] = UINT32_C(0x410FD034), + [1] = UINT32_C(0x410FD080), + }, + }, +#if CPUINFO_ARCH_ARM + { + /* MediaTek MT8135: 2x Cortex-A7 + 2x Cortex-A15 */ + .cores = 4, + .series = cpuinfo_arm_chipset_series_mediatek_mt, + .model = UINT16_C(8135), + .clusters = 2, + .cluster_cores = { + [0] = 2, + [1] = 2, + }, + .cluster_midr = { + [0] = UINT32_C(0x410FC073), + [1] = UINT32_C(0x413FC0F2), + }, + }, +#endif + { + /* MediaTek MT8173: 2x Cortex-A72 + 2x Cortex-A53 */ + .cores = 4, + .series = cpuinfo_arm_chipset_series_mediatek_mt, + .model = UINT16_C(8173), + .clusters = 2, + .cluster_cores = { + [0] = 2, + [1] = 2, + }, + .cluster_midr = { + [0] = UINT32_C(0x410FD032), + [1] = UINT32_C(0x410FD080), + }, + }, + { + /* MediaTek MT8176: 2x Cortex-A72 + 4x Cortex-A53 */ + .cores = 6, + .series = cpuinfo_arm_chipset_series_mediatek_mt, + .model = UINT16_C(8176), + .clusters = 2, + .cluster_cores = { + [0] = 4, + [1] = 2, + }, + .cluster_midr = { + [0] = UINT32_C(0x410FD032), + [1] = UINT32_C(0x410FD080), + }, + }, +#if CPUINFO_ARCH_ARM64 + { + /* + * MediaTek MT8735: 4x Cortex-A53 + * Some AArch64 phones use non-standard /proc/cpuinfo format. + */ + .cores = 4, + .series = cpuinfo_arm_chipset_series_mediatek_mt, + .model = UINT16_C(8735), + .clusters = 1, + .cluster_cores = { + [0] = 4, + }, + .cluster_midr = { + [0] = UINT32_C(0x410FD034), + }, + }, +#endif +#if CPUINFO_ARCH_ARM + { + /* + * MediaTek MT6592: 4x Cortex-A7 + 4x Cortex-A7 + * Some phones use non-standard /proc/cpuinfo format. + */ + .cores = 4, + .series = cpuinfo_arm_chipset_series_mediatek_mt, + .model = UINT16_C(6592), + .clusters = 2, + .cluster_cores = { + [0] = 4, + [1] = 4, + }, + .cluster_midr = { + [0] = UINT32_C(0x410FC074), + [1] = UINT32_C(0x410FC074), + }, + }, + { + /* MediaTek MT6595: 4x Cortex-A17 + 4x Cortex-A7 */ + .cores = 8, + .series = cpuinfo_arm_chipset_series_mediatek_mt, + .model = UINT16_C(6595), + .clusters = 2, + .cluster_cores = { + [0] = 4, + [1] = 4, + }, + .cluster_midr = { + [0] = UINT32_C(0x410FC075), + [1] = UINT32_C(0x410FC0E0), + }, + }, +#endif + { + /* MediaTek MT6797: 2x Cortex-A72 + 4x Cortex-A53 + 4x Cortex-A53 */ + .cores = 10, + .series = cpuinfo_arm_chipset_series_mediatek_mt, + .model = UINT16_C(6797), + .clusters = 3, + .cluster_cores = { + [0] = 4, + [1] = 4, + [2] = 2, + }, + .cluster_midr = { + [0] = UINT32_C(0x410FD034), + [1] = UINT32_C(0x410FD034), + [2] = UINT32_C(0x410FD081), + }, + }, + { + /* MediaTek MT6799: 2x Cortex-A73 + 4x Cortex-A53 + 4x Cortex-A35 */ + .cores = 10, + .series = cpuinfo_arm_chipset_series_mediatek_mt, + .model = UINT16_C(6799), + .clusters = 3, + .cluster_cores = { + [0] = 4, + [1] = 4, + [2] = 2, + }, + .cluster_midr = { + [0] = UINT32_C(0x410FD041), + [1] = UINT32_C(0x410FD034), + [2] = UINT32_C(0x410FD092), + }, + }, + { + /* Rockchip RK3399: 2x Cortex-A72 + 4x Cortex-A53 */ + .cores = 6, + .series = cpuinfo_arm_chipset_series_rockchip_rk, + .model = UINT16_C(3399), + .clusters = 2, + .cluster_cores = { + [0] = 4, + [1] = 2, + }, + .cluster_midr = { + [0] = UINT32_C(0x410FD034), + [1] = UINT32_C(0x410FD082), + }, + }, +#if CPUINFO_ARCH_ARM + { + /* Actions ATM8029: 4x Cortex-A5 + * Most devices use non-standard /proc/cpuinfo format. + */ + .cores = 4, + .series = cpuinfo_arm_chipset_series_actions_atm, + .model = UINT16_C(7029), + .clusters = 1, + .cluster_cores = { + [0] = 4, + }, + .cluster_midr = { + [0] = UINT32_C(0x410FC051), + }, + }, +#endif +}; + +/* + * Searches chipset name in mapping of chipset name to cores' MIDR values. If match is successful, initializes MIDR + * for all clusters' leaders with tabulated values. + * + * @param[in] chipset - chipset (SoC) name information. + * @param clusters_count - number of CPU core clusters detected in the SoC. + * @param cluster_leaders - indices of core clusters' leaders in the @p processors array. + * @param processors_count - number of usable logical processors in the system. + * @param[in,out] processors - array of logical processor descriptions with pre-parsed MIDR, maximum frequency, + * and decoded core cluster (package_leader_id) information. + * Upon successful return, processors[i].midr for all clusters' leaders contains the + * tabulated MIDR values. + * @param verify_midr - indicated whether the function should check that the MIDR values to be assigned to leaders of + * core clusters are consistent with known parts of their parsed values. + * Set if to false if the only MIDR value parsed from /proc/cpuinfo is for the last processor + * reported in /proc/cpuinfo and thus can't be unambiguously attributed to that processor. + * + * @retval true if the chipset was found in the mapping and core clusters' leaders initialized with MIDR values. + * @retval false if the chipset was not found in the mapping, or any consistency check failed. + */ +static bool cpuinfo_arm_linux_detect_cluster_midr_by_chipset( + const struct cpuinfo_arm_chipset chipset[restrict static 1], + uint32_t clusters_count, + const uint32_t cluster_leaders[restrict static CLUSTERS_MAX], + uint32_t processors_count, + struct cpuinfo_arm_linux_processor processors[restrict static processors_count], + bool verify_midr) +{ + if (clusters_count <= CLUSTERS_MAX) { + for (uint32_t c = 0; c < CPUINFO_COUNT_OF(cluster_configs); c++) { + if (cluster_configs[c].model == chipset->model && cluster_configs[c].series == chipset->series) { + /* Verify that the total number of cores and clusters of cores matches expectation */ + if (cluster_configs[c].cores != processors_count || cluster_configs[c].clusters != clusters_count) { + return false; + } + + /* Verify that core cluster configuration matches expectation */ + for (uint32_t cluster = 0; cluster < clusters_count; cluster++) { + const uint32_t cluster_leader = cluster_leaders[cluster]; + if (cluster_configs[c].cluster_cores[cluster] != processors[cluster_leader].package_processor_count) { + return false; + } + } + + if (verify_midr) { + /* Verify known parts of MIDR */ + for (uint32_t cluster = 0; cluster < clusters_count; cluster++) { + const uint32_t cluster_leader = cluster_leaders[cluster]; + + /* Create a mask of known midr bits */ + uint32_t midr_mask = 0; + if (processors[cluster_leader].flags & CPUINFO_ARM_LINUX_VALID_IMPLEMENTER) { + midr_mask |= CPUINFO_ARM_MIDR_IMPLEMENTER_MASK; + } + if (processors[cluster_leader].flags & CPUINFO_ARM_LINUX_VALID_VARIANT) { + midr_mask |= CPUINFO_ARM_MIDR_VARIANT_MASK; + } + if (processors[cluster_leader].flags & CPUINFO_ARM_LINUX_VALID_PART) { + midr_mask |= CPUINFO_ARM_MIDR_PART_MASK; + } + if (processors[cluster_leader].flags & CPUINFO_ARM_LINUX_VALID_REVISION) { + midr_mask |= CPUINFO_ARM_MIDR_REVISION_MASK; + } + + /* Verify the bits under the mask */ + if ((processors[cluster_leader].midr ^ cluster_configs[c].cluster_midr[cluster]) & midr_mask) { + cpuinfo_log_debug("parsed MIDR of cluster %08"PRIu32" does not match tabulated value %08"PRIu32, + processors[cluster_leader].midr, cluster_configs[c].cluster_midr[cluster]); + return false; + } + } + } + + /* Assign MIDRs according to tabulated configurations */ + for (uint32_t cluster = 0; cluster < clusters_count; cluster++) { + const uint32_t cluster_leader = cluster_leaders[cluster]; + processors[cluster_leader].midr = cluster_configs[c].cluster_midr[cluster]; + processors[cluster_leader].flags |= CPUINFO_ARM_LINUX_VALID_MIDR; + cpuinfo_log_debug("cluster %"PRIu32" MIDR = 0x%08"PRIx32, cluster, cluster_configs[c].cluster_midr[cluster]); + } + return true; + } + } + } + return false; +} + +/* + * Initializes MIDR for leaders of core clusters using a heuristic for big.LITTLE systems: + * - If the only known MIDR is for the big core cluster, guess the matching MIDR for the LITTLE cluster. + * - Estimate which of the clusters is big using maximum frequency, if known, otherwise using system processor ID. + * - Initialize the MIDR for big and LITTLE core clusters using the guesstimates values. + * + * @param clusters_count - number of CPU core clusters detected in the SoC. + * @param cluster_with_midr_count - number of CPU core clusters in the SoC with known MIDR values. + * @param last_processor_with_midr - index of the last logical processor with known MIDR in the @p processors array. + * @param cluster_leaders - indices of core clusters' leaders in the @p processors array. + * @param[in,out] processors - array of logical processor descriptions with pre-parsed MIDR, maximum frequency, + * and decoded core cluster (package_leader_id) information. + * Upon successful return, processors[i].midr for all core clusters' leaders contains + * the heuristically detected MIDR value. + * @param verify_midr - indicated whether the function should check that the MIDR values to be assigned to leaders of + * core clusters are consistent with known parts of their parsed values. + * Set if to false if the only MIDR value parsed from /proc/cpuinfo is for the last processor + * reported in /proc/cpuinfo and thus can't be unambiguously attributed to that processor. + * + * @retval true if this is a big.LITTLE system with only one known MIDR and the CPU core clusters' leaders were + * initialized with MIDR values. + * @retval false if this is not a big.LITTLE system. + */ +static bool cpuinfo_arm_linux_detect_cluster_midr_by_big_little_heuristic( + uint32_t clusters_count, + uint32_t cluster_with_midr_count, + uint32_t last_processor_with_midr, + const uint32_t cluster_leaders[restrict static CLUSTERS_MAX], + struct cpuinfo_arm_linux_processor processors[restrict static last_processor_with_midr], + bool verify_midr) +{ + if (clusters_count != 2 || cluster_with_midr_count != 1) { + /* Not a big.LITTLE system, or MIDR is known for both/neither clusters */ + return false; + } + + const uint32_t midr_flags = + (processors[processors[last_processor_with_midr].package_leader_id].flags & CPUINFO_ARM_LINUX_VALID_MIDR); + const uint32_t big_midr = processors[processors[last_processor_with_midr].package_leader_id].midr; + const uint32_t little_midr = midr_little_core_for_big(big_midr); + + /* Default assumption: the first reported cluster is LITTLE cluster (this holds on most Linux kernels) */ + uint32_t little_cluster_leader = cluster_leaders[0]; + const uint32_t other_cluster_leader = cluster_leaders[1]; + /* If maximum frequency is known for both clusters, assume LITTLE cluster is the one with lower frequency */ + if (processors[little_cluster_leader].flags & processors[other_cluster_leader].flags & CPUINFO_LINUX_FLAG_MAX_FREQUENCY) { + if (processors[little_cluster_leader].max_frequency > processors[other_cluster_leader].max_frequency) { + little_cluster_leader = other_cluster_leader; + } + } + + if (verify_midr) { + /* Verify known parts of MIDR */ + for (uint32_t cluster = 0; cluster < clusters_count; cluster++) { + const uint32_t cluster_leader = cluster_leaders[cluster]; + + /* Create a mask of known midr bits */ + uint32_t midr_mask = 0; + if (processors[cluster_leader].flags & CPUINFO_ARM_LINUX_VALID_IMPLEMENTER) { + midr_mask |= CPUINFO_ARM_MIDR_IMPLEMENTER_MASK; + } + if (processors[cluster_leader].flags & CPUINFO_ARM_LINUX_VALID_VARIANT) { + midr_mask |= CPUINFO_ARM_MIDR_VARIANT_MASK; + } + if (processors[cluster_leader].flags & CPUINFO_ARM_LINUX_VALID_PART) { + midr_mask |= CPUINFO_ARM_MIDR_PART_MASK; + } + if (processors[cluster_leader].flags & CPUINFO_ARM_LINUX_VALID_REVISION) { + midr_mask |= CPUINFO_ARM_MIDR_REVISION_MASK; + } + + /* Verify the bits under the mask */ + const uint32_t midr = (cluster_leader == little_cluster_leader) ? little_midr : big_midr; + if ((processors[cluster_leader].midr ^ midr) & midr_mask) { + cpuinfo_log_debug( + "parsed MIDR %08"PRIu32" of cluster leader %"PRIu32" is inconsistent with expected value %08"PRIu32, + processors[cluster_leader].midr, cluster_leader, midr); + return false; + } + } + } + + for (uint32_t c = 0; c < clusters_count; c++) { + /* Skip cluster with already assigned MIDR */ + const uint32_t cluster_leader = cluster_leaders[c]; + if (bitmask_all(processors[cluster_leader].flags, CPUINFO_ARM_LINUX_VALID_MIDR)) { + continue; + } + + const uint32_t midr = (cluster_leader == little_cluster_leader) ? little_midr : big_midr; + cpuinfo_log_info("assume processor %"PRIu32" to have MIDR %08"PRIx32, cluster_leader, midr); + /* To be consistent, we copy the MIDR entirely, rather than by parts */ + processors[cluster_leader].midr = midr; + processors[cluster_leader].flags |= midr_flags; + } + return true; +} + +/* + * Initializes MIDR for leaders of core clusters in a single sequential scan: + * - Clusters preceding the first reported MIDR value are assumed to have default MIDR value. + * - Clusters following any reported MIDR value to have that MIDR value. + * + * @param default_midr - MIDR value that will be assigned to cluster leaders preceding any reported MIDR value. + * @param processors_count - number of logical processor descriptions in the @p processors array. + * @param[in,out] processors - array of logical processor descriptions with pre-parsed MIDR, maximum frequency, + * and decoded core cluster (package_leader_id) information. + * Upon successful return, processors[i].midr for all core clusters' leaders contains + * the assigned MIDR value. + */ +static void cpuinfo_arm_linux_detect_cluster_midr_by_sequential_scan( + uint32_t default_midr, + uint32_t processors_count, + struct cpuinfo_arm_linux_processor processors[restrict static processors_count]) +{ + uint32_t midr = default_midr; + for (uint32_t i = 0; i < processors_count; i++) { + if (bitmask_all(processors[i].flags, CPUINFO_LINUX_FLAG_VALID)) { + if (processors[i].package_leader_id == i) { + if (bitmask_all(processors[i].flags, CPUINFO_ARM_LINUX_VALID_MIDR)) { + midr = processors[i].midr; + } else { + cpuinfo_log_info("assume processor %"PRIu32" to have MIDR %08"PRIx32, i, midr); + /* To be consistent, we copy the MIDR entirely, rather than by parts */ + processors[i].midr = midr; + processors[i].flags |= CPUINFO_ARM_LINUX_VALID_MIDR; + } + } + } + } +} + +/* + * Detects MIDR of each CPU core clusters' leader. + * + * @param[in] chipset - chipset (SoC) name information. + * @param max_processors - number of processor descriptions in the @p processors array. + * @param usable_processors - number of processor descriptions in the @p processors array with both POSSIBLE and + * PRESENT flags. + * @param[in,out] processors - array of logical processor descriptions with pre-parsed MIDR, maximum frequency, + * and decoded core cluster (package_leader_id) information. + * Upon return, processors[i].midr for all clusters' leaders contains the MIDR value. + * + * @returns The number of core clusters + */ +uint32_t cpuinfo_arm_linux_detect_cluster_midr( + const struct cpuinfo_arm_chipset chipset[restrict static 1], + uint32_t max_processors, + uint32_t usable_processors, + struct cpuinfo_arm_linux_processor processors[restrict static max_processors]) +{ + uint32_t clusters_count = 0; + uint32_t cluster_leaders[CLUSTERS_MAX]; + uint32_t last_processor_in_cpuinfo = max_processors; + uint32_t last_processor_with_midr = max_processors; + uint32_t processors_with_midr_count = 0; + for (uint32_t i = 0; i < max_processors; i++) { + if (bitmask_all(processors[i].flags, CPUINFO_LINUX_FLAG_VALID)) { + if (processors[i].flags & CPUINFO_ARM_LINUX_VALID_PROCESSOR) { + last_processor_in_cpuinfo = i; + } + if (bitmask_all(processors[i].flags, CPUINFO_ARM_LINUX_VALID_IMPLEMENTER | CPUINFO_ARM_LINUX_VALID_PART)) { + last_processor_with_midr = i; + processors_with_midr_count += 1; + } + const uint32_t group_leader = processors[i].package_leader_id; + if (group_leader == i) { + if (clusters_count < CLUSTERS_MAX) { + cluster_leaders[clusters_count] = i; + } + clusters_count += 1; + } else { + /* Copy known bits of information to cluster leader */ + + if ((processors[i].flags & ~processors[group_leader].flags) & CPUINFO_LINUX_FLAG_MAX_FREQUENCY) { + processors[group_leader].max_frequency = processors[i].max_frequency; + processors[group_leader].flags |= CPUINFO_LINUX_FLAG_MAX_FREQUENCY; + } + if (!bitmask_all(processors[group_leader].flags, CPUINFO_ARM_LINUX_VALID_MIDR) && + bitmask_all(processors[i].flags, CPUINFO_ARM_LINUX_VALID_MIDR)) + { + processors[group_leader].midr = processors[i].midr; + processors[group_leader].flags |= CPUINFO_ARM_LINUX_VALID_MIDR; + } + } + } + } + cpuinfo_log_debug("detected %"PRIu32" core clusters", clusters_count); + + /* + * Two relations between reported /proc/cpuinfo information, and cores is possible: + * - /proc/cpuinfo reports information for all or some of the cores below the corresponding + * "processor : " lines. Information on offline cores may be missing. + * - /proc/cpuinfo reports information only once, after all "processor : " lines. + * The reported information may relate to processor #0 or to the processor which + * executed the system calls to read /proc/cpuinfo. It is also indistinguishable + * from /proc/cpuinfo reporting information only for the last core (e.g. if all other + * cores are offline). + * + * We detect the second case by checking if /proc/cpuinfo contains valid MIDR only for one, + * last reported, processor. Note, that the last reported core may be not the last + * present & possible processor, as /proc/cpuinfo may non-report high-index offline cores. + */ + if (processors_with_midr_count == 1 && last_processor_in_cpuinfo == last_processor_with_midr && clusters_count > 1) { + /* + * There are multiple core clusters, but /proc/cpuinfo reported MIDR only for one + * processor, and we don't even know which logical processor this information refers to. + * + * We make three attempts to detect MIDR for all clusters: + * 1. Search tabulated MIDR values for chipsets which have heterogeneous clusters and ship with Linux + * kernels which do not always report all cores in /proc/cpuinfo. If found, use the tabulated values. + * 2. For systems with 2 clusters and MIDR known for one cluster, assume big.LITTLE configuration, + * and estimate MIDR for the other cluster under assumption that MIDR for the big cluster is known. + * 3. Initialize MIDRs for all core clusters to the only parsed MIDR value. + */ + cpuinfo_log_debug("the only reported MIDR can not be attributed to a particular processor"); + + if (cpuinfo_arm_linux_detect_cluster_midr_by_chipset( + chipset, clusters_count, cluster_leaders, usable_processors, processors, false)) + { + return clusters_count; + } + + /* Try big.LITTLE heuristic */ + if (cpuinfo_arm_linux_detect_cluster_midr_by_big_little_heuristic( + clusters_count, 1, last_processor_with_midr, + cluster_leaders, processors, false)) + { + return clusters_count; + } + + /* Fall back to sequential initialization of MIDR values for core clusters */ + cpuinfo_arm_linux_detect_cluster_midr_by_sequential_scan( + processors[processors[last_processor_with_midr].package_leader_id].midr, + max_processors, processors); + } else if (processors_with_midr_count < usable_processors) { + /* + * /proc/cpuinfo reported MIDR only for some processors, and probably some core clusters do not have MIDR + * for any of the cores. Check if this is the case. + */ + uint32_t clusters_with_midr_count = 0; + for (uint32_t i = 0; i < max_processors; i++) { + if (bitmask_all(processors[i].flags, CPUINFO_LINUX_FLAG_VALID | CPUINFO_ARM_LINUX_VALID_MIDR)) { + if (processors[i].package_leader_id == i) { + clusters_with_midr_count += 1; + } + } + } + + if (clusters_with_midr_count < clusters_count) { + /* + * /proc/cpuinfo reported MIDR only for some clusters, need to reconstruct others. + * We make three attempts to detect MIDR for clusters without it: + * 1. Search tabulated MIDR values for chipsets which have heterogeneous clusters and ship with Linux + * kernels which do not always report all cores in /proc/cpuinfo. If found, use the tabulated values. + * 2. For systems with 2 clusters and MIDR known for one cluster, assume big.LITTLE configuration, + * and estimate MIDR for the other cluster under assumption that MIDR for the big cluster is known. + * 3. Initialize MIDRs for core clusters in a single sequential scan: + * - Clusters preceding the first reported MIDR value are assumed to have the last reported MIDR value. + * - Clusters following any reported MIDR value to have that MIDR value. + */ + + if (cpuinfo_arm_linux_detect_cluster_midr_by_chipset( + chipset, clusters_count, cluster_leaders, usable_processors, processors, true)) + { + return clusters_count; + } + + if (last_processor_with_midr != max_processors) { + /* Try big.LITTLE heuristic */ + if (cpuinfo_arm_linux_detect_cluster_midr_by_big_little_heuristic( + clusters_count, processors_with_midr_count, last_processor_with_midr, + cluster_leaders, processors, true)) + { + return clusters_count; + } + + /* Fall back to sequential initialization of MIDR values for core clusters */ + cpuinfo_arm_linux_detect_cluster_midr_by_sequential_scan( + processors[processors[last_processor_with_midr].package_leader_id].midr, + max_processors, processors); + } + } + } + return clusters_count; +} diff --git a/3rdparty/cpuinfo/src/arm/mach/init.c b/3rdparty/cpuinfo/src/arm/mach/init.c new file mode 100644 index 000000000..dbea578c4 --- /dev/null +++ b/3rdparty/cpuinfo/src/arm/mach/init.c @@ -0,0 +1,619 @@ +#include +#include +#include +#include +#include + +#include +#include +#include +#include + +#include +#include +#include +#include + +/* Polyfill recent CPUFAMILY_ARM_* values for older SDKs */ +#ifndef CPUFAMILY_ARM_MONSOON_MISTRAL + #define CPUFAMILY_ARM_MONSOON_MISTRAL 0xE81E7EF6 +#endif +#ifndef CPUFAMILY_ARM_VORTEX_TEMPEST + #define CPUFAMILY_ARM_VORTEX_TEMPEST 0x07D34B9F +#endif +#ifndef CPUFAMILY_ARM_LIGHTNING_THUNDER + #define CPUFAMILY_ARM_LIGHTNING_THUNDER 0x462504D2 +#endif +#ifndef CPUFAMILY_ARM_FIRESTORM_ICESTORM + #define CPUFAMILY_ARM_FIRESTORM_ICESTORM 0x1B588BB3 +#endif + +struct cpuinfo_arm_isa cpuinfo_isa = { +#if CPUINFO_ARCH_ARM + .thumb = true, + .thumb2 = true, + .thumbee = false, + .jazelle = false, + .armv5e = true, + .armv6 = true, + .armv6k = true, + .armv7 = true, + .vfpv2 = false, + .vfpv3 = true, + .d32 = true, + .wmmx = false, + .wmmx2 = false, + .neon = true, +#endif +#if CPUINFO_ARCH_ARM64 + .aes = true, + .sha1 = true, + .sha2 = true, + .pmull = true, + .crc32 = true, +#endif +}; + +static uint32_t get_sys_info(int type_specifier, const char* name) { + size_t size = 0; + uint32_t result = 0; + int mib[2] = { CTL_HW, type_specifier }; + if (sysctl(mib, 2, NULL, &size, NULL, 0) != 0) { + cpuinfo_log_info("sysctl(\"%s\") failed: %s", name, strerror(errno)); + } else if (size == sizeof(uint32_t)) { + sysctl(mib, 2, &result, &size, NULL, 0); + cpuinfo_log_debug("%s: %"PRIu32 ", size = %lu", name, result, size); + } else { + cpuinfo_log_info("sysctl does not support non-integer lookup for (\"%s\")", name); + } + return result; +} + +static uint32_t get_sys_info_by_name(const char* type_specifier) { + size_t size = 0; + uint32_t result = 0; + if (sysctlbyname(type_specifier, NULL, &size, NULL, 0) != 0) { + cpuinfo_log_info("sysctlbyname(\"%s\") failed: %s", type_specifier, strerror(errno)); + } else if (size == sizeof(uint32_t)) { + sysctlbyname(type_specifier, &result, &size, NULL, 0); + cpuinfo_log_debug("%s: %"PRIu32 ", size = %lu", type_specifier, result, size); + } else { + cpuinfo_log_info("sysctl does not support non-integer lookup for (\"%s\")", type_specifier); + } + return result; +} + +static enum cpuinfo_uarch decode_uarch(uint32_t cpu_family, uint32_t cpu_subtype, uint32_t core_index, uint32_t core_count) { + switch (cpu_family) { + case CPUFAMILY_ARM_SWIFT: + return cpuinfo_uarch_swift; + case CPUFAMILY_ARM_CYCLONE: + return cpuinfo_uarch_cyclone; + case CPUFAMILY_ARM_TYPHOON: + return cpuinfo_uarch_typhoon; + case CPUFAMILY_ARM_TWISTER: + return cpuinfo_uarch_twister; + case CPUFAMILY_ARM_HURRICANE: + return cpuinfo_uarch_hurricane; + case CPUFAMILY_ARM_MONSOON_MISTRAL: + /* 2x Monsoon + 4x Mistral cores */ + return core_index < 2 ? cpuinfo_uarch_monsoon : cpuinfo_uarch_mistral; + case CPUFAMILY_ARM_VORTEX_TEMPEST: + /* Hexa-core: 2x Vortex + 4x Tempest; Octa-core: 4x Cortex + 4x Tempest */ + return core_index + 4 < core_count ? cpuinfo_uarch_vortex : cpuinfo_uarch_tempest; + case CPUFAMILY_ARM_LIGHTNING_THUNDER: + /* Hexa-core: 2x Lightning + 4x Thunder; Octa-core (presumed): 4x Lightning + 4x Thunder */ + return core_index + 4 < core_count ? cpuinfo_uarch_lightning : cpuinfo_uarch_thunder; + case CPUFAMILY_ARM_FIRESTORM_ICESTORM: + /* Hexa-core: 2x Firestorm + 4x Icestorm; Octa-core: 4x Firestorm + 4x Icestorm */ + return core_index + 4 < core_count ? cpuinfo_uarch_firestorm : cpuinfo_uarch_icestorm; + default: + /* Use hw.cpusubtype for detection */ + break; + } + + #if CPUINFO_ARCH_ARM + switch (cpu_subtype) { + case CPU_SUBTYPE_ARM_V7: + return cpuinfo_uarch_cortex_a8; + case CPU_SUBTYPE_ARM_V7F: + return cpuinfo_uarch_cortex_a9; + case CPU_SUBTYPE_ARM_V7K: + return cpuinfo_uarch_cortex_a7; + default: + return cpuinfo_uarch_unknown; + } + #else + return cpuinfo_uarch_unknown; + #endif +} + +static void decode_package_name(char* package_name) { + size_t size; + if (sysctlbyname("hw.machine", NULL, &size, NULL, 0) != 0) { + cpuinfo_log_warning("sysctlbyname(\"hw.machine\") failed: %s", strerror(errno)); + return; + } + + char *machine_name = alloca(size); + if (sysctlbyname("hw.machine", machine_name, &size, NULL, 0) != 0) { + cpuinfo_log_warning("sysctlbyname(\"hw.machine\") failed: %s", strerror(errno)); + return; + } + cpuinfo_log_debug("hw.machine: %s", machine_name); + + char name[10]; + uint32_t major = 0, minor = 0; + if (sscanf(machine_name, "%9[^,0123456789]%"SCNu32",%"SCNu32, name, &major, &minor) != 3) { + cpuinfo_log_warning("parsing \"hw.machine\" failed: %s", strerror(errno)); + return; + } + + uint32_t chip_model = 0; + char suffix = '\0'; + if (strcmp(name, "iPhone") == 0) { + /* + * iPhone 4 and up are supported: + * - iPhone 4 [A4]: iPhone3,1, iPhone3,2, iPhone3,3 + * - iPhone 4S [A5]: iPhone4,1 + * - iPhone 5 [A6]: iPhone5,1, iPhone5,2 + * - iPhone 5c [A6]: iPhone5,3, iPhone5,4 + * - iPhone 5s [A7]: iPhone6,1, iPhone6,2 + * - iPhone 6 [A8]: iPhone7,2 + * - iPhone 6 Plus [A8]: iPhone7,1 + * - iPhone 6s [A9]: iPhone8,1 + * - iPhone 6s Plus [A9]: iPhone8,2 + * - iPhone SE [A9]: iPhone8,4 + * - iPhone 7 [A10]: iPhone9,1, iPhone9,3 + * - iPhone 7 Plus [A10]: iPhone9,2, iPhone9,4 + * - iPhone 8 [A11]: iPhone10,1, iPhone10,4 + * - iPhone 8 Plus [A11]: iPhone10,2, iPhone10,5 + * - iPhone X [A11]: iPhone10,3, iPhone10,6 + * - iPhone XS [A12]: iPhone11,2, + * - iPhone XS Max [A12]: iPhone11,4, iPhone11,6 + * - iPhone XR [A12]: iPhone11,8 + */ + chip_model = major + 1; + } else if (strcmp(name, "iPad") == 0) { + switch (major) { + /* iPad 2 and up are supported */ + case 2: + /* + * iPad 2 [A5]: iPad2,1, iPad2,2, iPad2,3, iPad2,4 + * iPad mini [A5]: iPad2,5, iPad2,6, iPad2,7 + */ + chip_model = major + 3; + break; + case 3: + /* + * iPad 3rd Gen [A5X]: iPad3,1, iPad3,2, iPad3,3 + * iPad 4th Gen [A6X]: iPad3,4, iPad3,5, iPad3,6 + */ + chip_model = (minor <= 3) ? 5 : 6; + suffix = 'X'; + break; + case 4: + /* + * iPad Air [A7]: iPad4,1, iPad4,2, iPad4,3 + * iPad mini Retina [A7]: iPad4,4, iPad4,5, iPad4,6 + * iPad mini 3 [A7]: iPad4,7, iPad4,8, iPad4,9 + */ + chip_model = major + 3; + break; + case 5: + /* + * iPad mini 4 [A8]: iPad5,1, iPad5,2 + * iPad Air 2 [A8X]: iPad5,3, iPad5,4 + */ + chip_model = major + 3; + suffix = (minor <= 2) ? '\0' : 'X'; + break; + case 6: + /* + * iPad Pro 9.7" [A9X]: iPad6,3, iPad6,4 + * iPad Pro [A9X]: iPad6,7, iPad6,8 + * iPad 5th Gen [A9]: iPad6,11, iPad6,12 + */ + chip_model = major + 3; + suffix = minor <= 8 ? 'X' : '\0'; + break; + case 7: + /* + * iPad Pro 12.9" [A10X]: iPad7,1, iPad7,2 + * iPad Pro 10.5" [A10X]: iPad7,3, iPad7,4 + * iPad 6th Gen [A10]: iPad7,5, iPad7,6 + */ + chip_model = major + 3; + suffix = minor <= 4 ? 'X' : '\0'; + break; + default: + cpuinfo_log_info("unknown iPad: %s", machine_name); + break; + } + } else if (strcmp(name, "iPod") == 0) { + switch (major) { + case 5: + chip_model = 5; + break; + /* iPod touch (5th Gen) [A5]: iPod5,1 */ + case 7: + /* iPod touch (6th Gen, 2015) [A8]: iPod7,1 */ + chip_model = 8; + break; + default: + cpuinfo_log_info("unknown iPod: %s", machine_name); + break; + } + } else { + cpuinfo_log_info("unknown device: %s", machine_name); + } + if (chip_model != 0) { + snprintf(package_name, CPUINFO_PACKAGE_NAME_MAX, "Apple A%"PRIu32"%c", chip_model, suffix); + } +} + +void cpuinfo_arm_mach_init(void) { + struct cpuinfo_processor* processors = NULL; + struct cpuinfo_core* cores = NULL; + struct cpuinfo_cluster* clusters = NULL; + struct cpuinfo_package* packages = NULL; + struct cpuinfo_uarch_info* uarchs = NULL; + struct cpuinfo_cache* l1i = NULL; + struct cpuinfo_cache* l1d = NULL; + struct cpuinfo_cache* l2 = NULL; + struct cpuinfo_cache* l3 = NULL; + + struct cpuinfo_mach_topology mach_topology = cpuinfo_mach_detect_topology(); + processors = calloc(mach_topology.threads, sizeof(struct cpuinfo_processor)); + if (processors == NULL) { + cpuinfo_log_error("failed to allocate %zu bytes for descriptions of %"PRIu32" logical processors", + mach_topology.threads * sizeof(struct cpuinfo_processor), mach_topology.threads); + goto cleanup; + } + cores = calloc(mach_topology.cores, sizeof(struct cpuinfo_core)); + if (cores == NULL) { + cpuinfo_log_error("failed to allocate %zu bytes for descriptions of %"PRIu32" cores", + mach_topology.cores * sizeof(struct cpuinfo_core), mach_topology.cores); + goto cleanup; + } + packages = calloc(mach_topology.packages, sizeof(struct cpuinfo_package)); + if (packages == NULL) { + cpuinfo_log_error("failed to allocate %zu bytes for descriptions of %"PRIu32" packages", + mach_topology.packages * sizeof(struct cpuinfo_package), mach_topology.packages); + goto cleanup; + } + + const uint32_t threads_per_core = mach_topology.threads / mach_topology.cores; + const uint32_t threads_per_package = mach_topology.threads / mach_topology.packages; + const uint32_t cores_per_package = mach_topology.cores / mach_topology.packages; + + for (uint32_t i = 0; i < mach_topology.packages; i++) { + packages[i] = (struct cpuinfo_package) { + .processor_start = i * threads_per_package, + .processor_count = threads_per_package, + .core_start = i * cores_per_package, + .core_count = cores_per_package, + }; + decode_package_name(packages[i].name); + } + + + const uint32_t cpu_family = get_sys_info_by_name("hw.cpufamily"); + const uint32_t cpu_type = get_sys_info_by_name("hw.cputype"); + const uint32_t cpu_subtype = get_sys_info_by_name("hw.cpusubtype"); + switch (cpu_type) { + case CPU_TYPE_ARM64: + cpuinfo_isa.aes = true; + cpuinfo_isa.sha1 = true; + cpuinfo_isa.sha2 = true; + cpuinfo_isa.pmull = true; + cpuinfo_isa.crc32 = true; + break; +#if CPUINFO_ARCH_ARM + case CPU_TYPE_ARM: + switch (cpu_subtype) { + case CPU_SUBTYPE_ARM_V8: + cpuinfo_isa.armv8 = true; + cpuinfo_isa.aes = true; + cpuinfo_isa.sha1 = true; + cpuinfo_isa.sha2 = true; + cpuinfo_isa.pmull = true; + cpuinfo_isa.crc32 = true; + /* Fall-through to add ARMv7S features */ + case CPU_SUBTYPE_ARM_V7S: + case CPU_SUBTYPE_ARM_V7K: + cpuinfo_isa.fma = true; + /* Fall-through to add ARMv7F features */ + case CPU_SUBTYPE_ARM_V7F: + cpuinfo_isa.armv7mp = true; + cpuinfo_isa.fp16 = true; + /* Fall-through to add ARMv7 features */ + case CPU_SUBTYPE_ARM_V7: + break; + default: + break; + } + break; +#endif + } + /* + * Support for ARMv8.1 Atomics & FP16 arithmetic instructions is supposed to be detected via + * sysctlbyname calls with "hw.optional.armv8_1_atomics" and "hw.optional.neon_fp16" arguments + * (see https://devstreaming-cdn.apple.com/videos/wwdc/2018/409t8zw7rumablsh/409/409_whats_new_in_llvm.pdf), + * but on new iOS versions these calls just fail with EPERM. + * + * Thus, we whitelist CPUs known to support these instructions. + */ + switch (cpu_family) { + case CPUFAMILY_ARM_MONSOON_MISTRAL: + case CPUFAMILY_ARM_VORTEX_TEMPEST: + case CPUFAMILY_ARM_LIGHTNING_THUNDER: + case CPUFAMILY_ARM_FIRESTORM_ICESTORM: + #if CPUINFO_ARCH_ARM64 + cpuinfo_isa.atomics = true; + #endif + cpuinfo_isa.fp16arith = true; + } + + /* + * There does not yet seem to exist an OS mechanism to detect support for + * ARMv8.2 optional dot-product instructions, so we currently whitelist CPUs + * known to support these instruction. + */ + switch (cpu_family) { + case CPUFAMILY_ARM_LIGHTNING_THUNDER: + case CPUFAMILY_ARM_FIRESTORM_ICESTORM: + cpuinfo_isa.dot = true; + } + + uint32_t num_clusters = 1; + for (uint32_t i = 0; i < mach_topology.cores; i++) { + cores[i] = (struct cpuinfo_core) { + .processor_start = i * threads_per_core, + .processor_count = threads_per_core, + .core_id = i % cores_per_package, + .package = packages + i / cores_per_package, + .vendor = cpuinfo_vendor_apple, + .uarch = decode_uarch(cpu_family, cpu_subtype, i, mach_topology.cores), + }; + if (i != 0 && cores[i].uarch != cores[i - 1].uarch) { + num_clusters++; + } + } + for (uint32_t i = 0; i < mach_topology.threads; i++) { + const uint32_t smt_id = i % threads_per_core; + const uint32_t core_id = i / threads_per_core; + const uint32_t package_id = i / threads_per_package; + + processors[i].smt_id = smt_id; + processors[i].core = &cores[core_id]; + processors[i].package = &packages[package_id]; + } + + clusters = calloc(num_clusters, sizeof(struct cpuinfo_cluster)); + if (clusters == NULL) { + cpuinfo_log_error( + "failed to allocate %zu bytes for descriptions of %"PRIu32" clusters", + num_clusters * sizeof(struct cpuinfo_cluster), num_clusters); + goto cleanup; + } + uarchs = calloc(num_clusters, sizeof(struct cpuinfo_uarch_info)); + if (uarchs == NULL) { + cpuinfo_log_error( + "failed to allocate %zu bytes for descriptions of %"PRIu32" uarchs", + num_clusters * sizeof(enum cpuinfo_uarch), num_clusters); + goto cleanup; + } + uint32_t cluster_idx = UINT32_MAX; + for (uint32_t i = 0; i < mach_topology.cores; i++) { + if (i == 0 || cores[i].uarch != cores[i - 1].uarch) { + cluster_idx++; + uarchs[cluster_idx] = (struct cpuinfo_uarch_info) { + .uarch = cores[i].uarch, + .processor_count = 1, + .core_count = 1, + }; + clusters[cluster_idx] = (struct cpuinfo_cluster) { + .processor_start = i * threads_per_core, + .processor_count = 1, + .core_start = i, + .core_count = 1, + .cluster_id = cluster_idx, + .package = cores[i].package, + .vendor = cores[i].vendor, + .uarch = cores[i].uarch, + }; + } else { + uarchs[cluster_idx].processor_count++; + uarchs[cluster_idx].core_count++; + clusters[cluster_idx].processor_count++; + clusters[cluster_idx].core_count++; + } + cores[i].cluster = &clusters[cluster_idx]; + } + + for (uint32_t i = 0; i < mach_topology.threads; i++) { + const uint32_t core_id = i / threads_per_core; + processors[i].cluster = cores[core_id].cluster; + } + + for (uint32_t i = 0; i < mach_topology.packages; i++) { + packages[i].cluster_start = 0; + packages[i].cluster_count = num_clusters; + } + + const uint32_t cacheline_size = get_sys_info(HW_CACHELINE, "HW_CACHELINE"); + const uint32_t l1d_cache_size = get_sys_info(HW_L1DCACHESIZE, "HW_L1DCACHESIZE"); + const uint32_t l1i_cache_size = get_sys_info(HW_L1ICACHESIZE, "HW_L1ICACHESIZE"); + const uint32_t l2_cache_size = get_sys_info(HW_L2CACHESIZE, "HW_L2CACHESIZE"); + const uint32_t l3_cache_size = get_sys_info(HW_L3CACHESIZE, "HW_L3CACHESIZE"); + const uint32_t l1_cache_associativity = 4; + const uint32_t l2_cache_associativity = 8; + const uint32_t l3_cache_associativity = 16; + const uint32_t cache_partitions = 1; + const uint32_t cache_flags = 0; + + uint32_t threads_per_l1 = 0, l1_count = 0; + if (l1i_cache_size != 0 || l1d_cache_size != 0) { + /* Assume L1 caches are private to each core */ + threads_per_l1 = 1; + l1_count = mach_topology.threads / threads_per_l1; + cpuinfo_log_debug("detected %"PRIu32" L1 caches", l1_count); + } + + uint32_t threads_per_l2 = 0, l2_count = 0; + if (l2_cache_size != 0) { + /* Assume L2 cache is shared between all cores */ + threads_per_l2 = mach_topology.cores; + l2_count = 1; + cpuinfo_log_debug("detected %"PRIu32" L2 caches", l2_count); + } + + uint32_t threads_per_l3 = 0, l3_count = 0; + if (l3_cache_size != 0) { + /* Assume L3 cache is shared between all cores */ + threads_per_l3 = mach_topology.cores; + l3_count = 1; + cpuinfo_log_debug("detected %"PRIu32" L3 caches", l3_count); + } + + if (l1i_cache_size != 0) { + l1i = calloc(l1_count, sizeof(struct cpuinfo_cache)); + if (l1i == NULL) { + cpuinfo_log_error("failed to allocate %zu bytes for descriptions of %"PRIu32" L1I caches", + l1_count * sizeof(struct cpuinfo_cache), l1_count); + goto cleanup; + } + for (uint32_t c = 0; c < l1_count; c++) { + l1i[c] = (struct cpuinfo_cache) { + .size = l1i_cache_size, + .associativity = l1_cache_associativity, + .sets = l1i_cache_size / (l1_cache_associativity * cacheline_size), + .partitions = cache_partitions, + .line_size = cacheline_size, + .flags = cache_flags, + .processor_start = c * threads_per_l1, + .processor_count = threads_per_l1, + }; + } + for (uint32_t t = 0; t < mach_topology.threads; t++) { + processors[t].cache.l1i = &l1i[t / threads_per_l1]; + } + } + + if (l1d_cache_size != 0) { + l1d = calloc(l1_count, sizeof(struct cpuinfo_cache)); + if (l1d == NULL) { + cpuinfo_log_error("failed to allocate %zu bytes for descriptions of %"PRIu32" L1D caches", + l1_count * sizeof(struct cpuinfo_cache), l1_count); + goto cleanup; + } + for (uint32_t c = 0; c < l1_count; c++) { + l1d[c] = (struct cpuinfo_cache) { + .size = l1d_cache_size, + .associativity = l1_cache_associativity, + .sets = l1d_cache_size / (l1_cache_associativity * cacheline_size), + .partitions = cache_partitions, + .line_size = cacheline_size, + .flags = cache_flags, + .processor_start = c * threads_per_l1, + .processor_count = threads_per_l1, + }; + } + for (uint32_t t = 0; t < mach_topology.threads; t++) { + processors[t].cache.l1d = &l1d[t / threads_per_l1]; + } + } + + if (l2_count != 0) { + l2 = calloc(l2_count, sizeof(struct cpuinfo_cache)); + if (l2 == NULL) { + cpuinfo_log_error("failed to allocate %zu bytes for descriptions of %"PRIu32" L2 caches", + l2_count * sizeof(struct cpuinfo_cache), l2_count); + goto cleanup; + } + for (uint32_t c = 0; c < l2_count; c++) { + l2[c] = (struct cpuinfo_cache) { + .size = l2_cache_size, + .associativity = l2_cache_associativity, + .sets = l2_cache_size / (l2_cache_associativity * cacheline_size), + .partitions = cache_partitions, + .line_size = cacheline_size, + .flags = cache_flags, + .processor_start = c * threads_per_l2, + .processor_count = threads_per_l2, + }; + } + for (uint32_t t = 0; t < mach_topology.threads; t++) { + processors[t].cache.l2 = &l2[0]; + } + } + + if (l3_count != 0) { + l3 = calloc(l3_count, sizeof(struct cpuinfo_cache)); + if (l3 == NULL) { + cpuinfo_log_error("failed to allocate %zu bytes for descriptions of %"PRIu32" L3 caches", + l3_count * sizeof(struct cpuinfo_cache), l3_count); + goto cleanup; + } + for (uint32_t c = 0; c < l3_count; c++) { + l3[c] = (struct cpuinfo_cache) { + .size = l3_cache_size, + .associativity = l3_cache_associativity, + .sets = l3_cache_size / (l3_cache_associativity * cacheline_size), + .partitions = cache_partitions, + .line_size = cacheline_size, + .flags = cache_flags, + .processor_start = c * threads_per_l3, + .processor_count = threads_per_l3, + }; + } + for (uint32_t t = 0; t < mach_topology.threads; t++) { + processors[t].cache.l3 = &l3[0]; + } + } + + /* Commit changes */ + cpuinfo_processors = processors; + cpuinfo_cores = cores; + cpuinfo_clusters = clusters; + cpuinfo_packages = packages; + cpuinfo_uarchs = uarchs; + cpuinfo_cache[cpuinfo_cache_level_1i] = l1i; + cpuinfo_cache[cpuinfo_cache_level_1d] = l1d; + cpuinfo_cache[cpuinfo_cache_level_2] = l2; + cpuinfo_cache[cpuinfo_cache_level_3] = l3; + + cpuinfo_processors_count = mach_topology.threads; + cpuinfo_cores_count = mach_topology.cores; + cpuinfo_clusters_count = num_clusters; + cpuinfo_packages_count = mach_topology.packages; + cpuinfo_uarchs_count = num_clusters; + cpuinfo_cache_count[cpuinfo_cache_level_1i] = l1_count; + cpuinfo_cache_count[cpuinfo_cache_level_1d] = l1_count; + cpuinfo_cache_count[cpuinfo_cache_level_2] = l2_count; + cpuinfo_cache_count[cpuinfo_cache_level_3] = l3_count; + cpuinfo_max_cache_size = cpuinfo_compute_max_cache_size(&processors[0]); + + __sync_synchronize(); + + cpuinfo_is_initialized = true; + + processors = NULL; + cores = NULL; + clusters = NULL; + packages = NULL; + uarchs = NULL; + l1i = l1d = l2 = l3 = NULL; + +cleanup: + free(processors); + free(cores); + free(clusters); + free(packages); + free(uarchs); + free(l1i); + free(l1d); + free(l2); + free(l3); +} diff --git a/3rdparty/cpuinfo/src/arm/midr.h b/3rdparty/cpuinfo/src/arm/midr.h new file mode 100644 index 000000000..6329783ad --- /dev/null +++ b/3rdparty/cpuinfo/src/arm/midr.h @@ -0,0 +1,260 @@ +#pragma once +#include + + +#define CPUINFO_ARM_MIDR_IMPLEMENTER_MASK UINT32_C(0xFF000000) +#define CPUINFO_ARM_MIDR_VARIANT_MASK UINT32_C(0x00F00000) +#define CPUINFO_ARM_MIDR_ARCHITECTURE_MASK UINT32_C(0x000F0000) +#define CPUINFO_ARM_MIDR_PART_MASK UINT32_C(0x0000FFF0) +#define CPUINFO_ARM_MIDR_REVISION_MASK UINT32_C(0x0000000F) + +#define CPUINFO_ARM_MIDR_IMPLEMENTER_OFFSET 24 +#define CPUINFO_ARM_MIDR_VARIANT_OFFSET 20 +#define CPUINFO_ARM_MIDR_ARCHITECTURE_OFFSET 16 +#define CPUINFO_ARM_MIDR_PART_OFFSET 4 +#define CPUINFO_ARM_MIDR_REVISION_OFFSET 0 + +#define CPUINFO_ARM_MIDR_ARM1156 UINT32_C(0x410FB560) +#define CPUINFO_ARM_MIDR_CORTEX_A7 UINT32_C(0x410FC070) +#define CPUINFO_ARM_MIDR_CORTEX_A9 UINT32_C(0x410FC090) +#define CPUINFO_ARM_MIDR_CORTEX_A15 UINT32_C(0x410FC0F0) +#define CPUINFO_ARM_MIDR_CORTEX_A17 UINT32_C(0x410FC0E0) +#define CPUINFO_ARM_MIDR_CORTEX_A35 UINT32_C(0x410FD040) +#define CPUINFO_ARM_MIDR_CORTEX_A53 UINT32_C(0x410FD030) +#define CPUINFO_ARM_MIDR_CORTEX_A55 UINT32_C(0x410FD050) +#define CPUINFO_ARM_MIDR_CORTEX_A57 UINT32_C(0x410FD070) +#define CPUINFO_ARM_MIDR_CORTEX_A72 UINT32_C(0x410FD080) +#define CPUINFO_ARM_MIDR_CORTEX_A73 UINT32_C(0x410FD090) +#define CPUINFO_ARM_MIDR_CORTEX_A75 UINT32_C(0x410FD0A0) +#define CPUINFO_ARM_MIDR_KRYO280_GOLD UINT32_C(0x51AF8001) +#define CPUINFO_ARM_MIDR_KRYO280_SILVER UINT32_C(0x51AF8014) +#define CPUINFO_ARM_MIDR_KRYO385_GOLD UINT32_C(0x518F802D) +#define CPUINFO_ARM_MIDR_KRYO385_SILVER UINT32_C(0x518F803C) +#define CPUINFO_ARM_MIDR_KRYO_SILVER_821 UINT32_C(0x510F2010) +#define CPUINFO_ARM_MIDR_KRYO_GOLD UINT32_C(0x510F2050) +#define CPUINFO_ARM_MIDR_KRYO_SILVER_820 UINT32_C(0x510F2110) +#define CPUINFO_ARM_MIDR_EXYNOS_M1_M2 UINT32_C(0x530F0010) +#define CPUINFO_ARM_MIDR_DENVER2 UINT32_C(0x4E0F0030) + +inline static uint32_t midr_set_implementer(uint32_t midr, uint32_t implementer) { + return (midr & ~CPUINFO_ARM_MIDR_IMPLEMENTER_MASK) | + ((implementer << CPUINFO_ARM_MIDR_IMPLEMENTER_OFFSET) & CPUINFO_ARM_MIDR_IMPLEMENTER_MASK); +} + +inline static uint32_t midr_set_variant(uint32_t midr, uint32_t variant) { + return (midr & ~CPUINFO_ARM_MIDR_VARIANT_MASK) | + ((variant << CPUINFO_ARM_MIDR_VARIANT_OFFSET) & CPUINFO_ARM_MIDR_VARIANT_MASK); +} + +inline static uint32_t midr_set_architecture(uint32_t midr, uint32_t architecture) { + return (midr & ~CPUINFO_ARM_MIDR_ARCHITECTURE_MASK) | + ((architecture << CPUINFO_ARM_MIDR_ARCHITECTURE_OFFSET) & CPUINFO_ARM_MIDR_ARCHITECTURE_MASK); +} + +inline static uint32_t midr_set_part(uint32_t midr, uint32_t part) { + return (midr & ~CPUINFO_ARM_MIDR_PART_MASK) | + ((part << CPUINFO_ARM_MIDR_PART_OFFSET) & CPUINFO_ARM_MIDR_PART_MASK); +} + +inline static uint32_t midr_set_revision(uint32_t midr, uint32_t revision) { + return (midr & ~CPUINFO_ARM_MIDR_REVISION_MASK) | + ((revision << CPUINFO_ARM_MIDR_REVISION_OFFSET) & CPUINFO_ARM_MIDR_REVISION_MASK); +} + +inline static uint32_t midr_get_variant(uint32_t midr) { + return (midr & CPUINFO_ARM_MIDR_VARIANT_MASK) >> CPUINFO_ARM_MIDR_VARIANT_OFFSET; +} + +inline static uint32_t midr_get_implementer(uint32_t midr) { + return (midr & CPUINFO_ARM_MIDR_IMPLEMENTER_MASK) >> CPUINFO_ARM_MIDR_IMPLEMENTER_OFFSET; +} + +inline static uint32_t midr_get_part(uint32_t midr) { + return (midr & CPUINFO_ARM_MIDR_PART_MASK) >> CPUINFO_ARM_MIDR_PART_OFFSET; +} + +inline static uint32_t midr_get_revision(uint32_t midr) { + return (midr & CPUINFO_ARM_MIDR_REVISION_MASK) >> CPUINFO_ARM_MIDR_REVISION_OFFSET; +} + +inline static uint32_t midr_copy_implementer(uint32_t midr, uint32_t other_midr) { + return (midr & ~CPUINFO_ARM_MIDR_IMPLEMENTER_MASK) | (other_midr & CPUINFO_ARM_MIDR_IMPLEMENTER_MASK); +} + +inline static uint32_t midr_copy_variant(uint32_t midr, uint32_t other_midr) { + return (midr & ~CPUINFO_ARM_MIDR_VARIANT_MASK) | (other_midr & CPUINFO_ARM_MIDR_VARIANT_MASK); +} + +inline static uint32_t midr_copy_architecture(uint32_t midr, uint32_t other_midr) { + return (midr & ~CPUINFO_ARM_MIDR_ARCHITECTURE_MASK) | (other_midr & CPUINFO_ARM_MIDR_ARCHITECTURE_MASK); +} + +inline static uint32_t midr_copy_part(uint32_t midr, uint32_t other_midr) { + return (midr & ~CPUINFO_ARM_MIDR_PART_MASK) | (other_midr & CPUINFO_ARM_MIDR_PART_MASK); +} + +inline static uint32_t midr_copy_revision(uint32_t midr, uint32_t other_midr) { + return (midr & ~CPUINFO_ARM_MIDR_REVISION_MASK) | (other_midr & CPUINFO_ARM_MIDR_REVISION_MASK); +} + +inline static bool midr_is_arm1156(uint32_t midr) { + const uint32_t uarch_mask = CPUINFO_ARM_MIDR_IMPLEMENTER_MASK | CPUINFO_ARM_MIDR_PART_MASK; + return (midr & uarch_mask) == (CPUINFO_ARM_MIDR_ARM1156 & uarch_mask); +} + +inline static bool midr_is_arm11(uint32_t midr) { + return (midr & (CPUINFO_ARM_MIDR_IMPLEMENTER_MASK | 0x0000F000)) == UINT32_C(0x4100B000); +} + +inline static bool midr_is_cortex_a9(uint32_t midr) { + const uint32_t uarch_mask = CPUINFO_ARM_MIDR_IMPLEMENTER_MASK | CPUINFO_ARM_MIDR_PART_MASK; + return (midr & uarch_mask) == (CPUINFO_ARM_MIDR_CORTEX_A9 & uarch_mask); +} + +inline static bool midr_is_scorpion(uint32_t midr) { + switch (midr & (CPUINFO_ARM_MIDR_IMPLEMENTER_MASK | CPUINFO_ARM_MIDR_PART_MASK)) { + case UINT32_C(0x510000F0): + case UINT32_C(0x510002D0): + return true; + default: + return false; + } +} + +inline static bool midr_is_krait(uint32_t midr) { + switch (midr & (CPUINFO_ARM_MIDR_IMPLEMENTER_MASK | CPUINFO_ARM_MIDR_PART_MASK)) { + case UINT32_C(0x510004D0): + case UINT32_C(0x510006F0): + return true; + default: + return false; + } +} + +inline static bool midr_is_cortex_a53(uint32_t midr) { + const uint32_t uarch_mask = CPUINFO_ARM_MIDR_IMPLEMENTER_MASK | CPUINFO_ARM_MIDR_PART_MASK; + return (midr & uarch_mask) == (CPUINFO_ARM_MIDR_CORTEX_A53 & uarch_mask); +} + +inline static bool midr_is_qualcomm_cortex_a53_silver(uint32_t midr) { + const uint32_t uarch_mask = CPUINFO_ARM_MIDR_IMPLEMENTER_MASK | CPUINFO_ARM_MIDR_PART_MASK; + return (midr & uarch_mask) == (CPUINFO_ARM_MIDR_KRYO280_SILVER & uarch_mask); +} + +inline static bool midr_is_qualcomm_cortex_a55_silver(uint32_t midr) { + const uint32_t uarch_mask = CPUINFO_ARM_MIDR_IMPLEMENTER_MASK | CPUINFO_ARM_MIDR_PART_MASK; + return (midr & uarch_mask) == (CPUINFO_ARM_MIDR_KRYO385_SILVER & uarch_mask); +} + +inline static bool midr_is_kryo280_gold(uint32_t midr) { + const uint32_t uarch_mask = CPUINFO_ARM_MIDR_IMPLEMENTER_MASK | CPUINFO_ARM_MIDR_PART_MASK; + return (midr & uarch_mask) == (CPUINFO_ARM_MIDR_KRYO280_GOLD & uarch_mask); +} + +inline static bool midr_is_kryo_silver(uint32_t midr) { + const uint32_t uarch_mask = + CPUINFO_ARM_MIDR_IMPLEMENTER_MASK | CPUINFO_ARM_MIDR_ARCHITECTURE_MASK | CPUINFO_ARM_MIDR_PART_MASK; + switch (midr & uarch_mask) { + case CPUINFO_ARM_MIDR_KRYO_SILVER_820: + case CPUINFO_ARM_MIDR_KRYO_SILVER_821: + return true; + default: + return false; + } +} + +inline static bool midr_is_kryo_gold(uint32_t midr) { + const uint32_t uarch_mask = CPUINFO_ARM_MIDR_IMPLEMENTER_MASK | CPUINFO_ARM_MIDR_PART_MASK; + return (midr & uarch_mask) == (CPUINFO_ARM_MIDR_KRYO_GOLD & uarch_mask); +} + +inline static uint32_t midr_score_core(uint32_t midr) { + const uint32_t core_mask = CPUINFO_ARM_MIDR_IMPLEMENTER_MASK | CPUINFO_ARM_MIDR_PART_MASK; + switch (midr & core_mask) { + case UINT32_C(0x53000030): /* Exynos M4 */ + case UINT32_C(0x53000040): /* Exynos M5 */ + case UINT32_C(0x4100D440): /* Cortex-X1 */ + /* These cores are in big role w.r.t Cortex-A75/-A76/-A77/-A78 */ + return 6; + case UINT32_C(0x4E000030): /* Denver 2 */ + case UINT32_C(0x53000010): /* Exynos M1 and Exynos M2 */ + case UINT32_C(0x53000020): /* Exynos M3 */ + case UINT32_C(0x51008040): /* Kryo 485 Gold / Gold Prime */ + case UINT32_C(0x51008020): /* Kryo 385 Gold */ + case UINT32_C(0x51008000): /* Kryo 260 / 280 Gold */ + case UINT32_C(0x51002050): /* Kryo Gold */ + case UINT32_C(0x4800D400): /* Cortex-A76 (HiSilicon) */ + case UINT32_C(0x4100D490): /* Neoverse N2 */ + case UINT32_C(0x4100D410): /* Cortex-A78 */ + case UINT32_C(0x4100D400): /* Neoverse V1 */ + case UINT32_C(0x4100D0D0): /* Cortex-A77 */ + case UINT32_C(0x4100D0E0): /* Cortex-A76AE */ + case UINT32_C(0x4100D0C0): /* Neoverse-N1 */ + case UINT32_C(0x4100D0B0): /* Cortex-A76 */ + case UINT32_C(0x4100D0A0): /* Cortex-A75 */ + case UINT32_C(0x4100D090): /* Cortex-A73 */ + case UINT32_C(0x4100D080): /* Cortex-A72 */ +#if CPUINFO_ARCH_ARM + case UINT32_C(0x4100C0F0): /* Cortex-A15 */ + case UINT32_C(0x4100C0E0): /* Cortex-A17 */ + case UINT32_C(0x4100C0D0): /* Rockchip RK3288 cores */ + case UINT32_C(0x4100C0C0): /* Cortex-A12 */ +#endif /* CPUINFO_ARCH_ARM */ + /* These cores are always in big role */ + return 5; + case UINT32_C(0x4100D070): /* Cortex-A57 */ + /* Cortex-A57 can be in LITTLE role w.r.t. Denver 2, or in big role w.r.t. Cortex-A53 */ + return 4; +#if CPUINFO_ARCH_ARM64 + case UINT32_C(0x4100D060): /* Cortex-A65 */ +#endif /* CPUINFO_ARCH_ARM64 */ + case UINT32_C(0x4100D050): /* Cortex-A55 */ + case UINT32_C(0x4100D030): /* Cortex-A53 */ + /* Cortex-A53 is usually in LITTLE role, but can be in big role w.r.t. Cortex-A35 */ + return 2; + case UINT32_C(0x4100D040): /* Cortex-A35 */ +#if CPUINFO_ARCH_ARM + case UINT32_C(0x4100C070): /* Cortex-A7 */ +#endif /* CPUINFO_ARCH_ARM */ + case UINT32_C(0x51008050): /* Kryo 485 Silver */ + case UINT32_C(0x51008030): /* Kryo 385 Silver */ + case UINT32_C(0x51008010): /* Kryo 260 / 280 Silver */ + case UINT32_C(0x51002110): /* Kryo Silver (Snapdragon 820) */ + case UINT32_C(0x51002010): /* Kryo Silver (Snapdragon 821) */ + /* These cores are always in LITTLE core */ + return 1; + default: + /* + * Unknown cores, or cores which do not have big/LITTLE roles. + * To be future-proof w.r.t. cores not yet recognized in cpuinfo, assume position between + * Cortex-A57/A72/A73/A75 and Cortex-A53/A55. Then at least future cores paired with + * one of these known cores will be properly scored. + */ + return 3; + } +} + +inline static uint32_t midr_little_core_for_big(uint32_t midr) { + const uint32_t core_mask = + CPUINFO_ARM_MIDR_IMPLEMENTER_MASK | CPUINFO_ARM_MIDR_ARCHITECTURE_MASK | CPUINFO_ARM_MIDR_PART_MASK; + switch (midr & core_mask) { + case CPUINFO_ARM_MIDR_CORTEX_A75: + return CPUINFO_ARM_MIDR_CORTEX_A55; + case CPUINFO_ARM_MIDR_CORTEX_A73: + case CPUINFO_ARM_MIDR_CORTEX_A72: + case CPUINFO_ARM_MIDR_CORTEX_A57: + case CPUINFO_ARM_MIDR_EXYNOS_M1_M2: + return CPUINFO_ARM_MIDR_CORTEX_A53; + case CPUINFO_ARM_MIDR_CORTEX_A17: + case CPUINFO_ARM_MIDR_CORTEX_A15: + return CPUINFO_ARM_MIDR_CORTEX_A7; + case CPUINFO_ARM_MIDR_KRYO280_GOLD: + return CPUINFO_ARM_MIDR_KRYO280_SILVER; + case CPUINFO_ARM_MIDR_KRYO_GOLD: + return CPUINFO_ARM_MIDR_KRYO_SILVER_820; + case CPUINFO_ARM_MIDR_DENVER2: + return CPUINFO_ARM_MIDR_CORTEX_A57; + default: + return midr; + } +} diff --git a/3rdparty/cpuinfo/src/arm/tlb.c b/3rdparty/cpuinfo/src/arm/tlb.c new file mode 100644 index 000000000..9beb8327e --- /dev/null +++ b/3rdparty/cpuinfo/src/arm/tlb.c @@ -0,0 +1,133 @@ + + +switch (uarch) { + case cpuinfo_uarch_cortex_a5: + /* + * Cortex-A5 Technical Reference Manual: + * 6.3.1. Micro TLB + * The first level of caching for the page table information is a micro TLB of + * 10 entries that is implemented on each of the instruction and data sides. + * 6.3.2. Main TLB + * Misses from the instruction and data micro TLBs are handled by a unified main TLB. + * The main TLB is 128-entry two-way set-associative. + */ + break; + case cpuinfo_uarch_cortex_a7: + /* + * Cortex-A7 MPCore Technical Reference Manual: + * 5.3.1. Micro TLB + * The first level of caching for the page table information is a micro TLB of + * 10 entries that is implemented on each of the instruction and data sides. + * 5.3.2. Main TLB + * Misses from the micro TLBs are handled by a unified main TLB. This is a 256-entry 2-way + * set-associative structure. The main TLB supports all the VMSAv7 page sizes of + * 4KB, 64KB, 1MB and 16MB in addition to the LPAE page sizes of 2MB and 1G. + */ + break; + case cpuinfo_uarch_cortex_a8: + /* + * Cortex-A8 Technical Reference Manual: + * 6.1. About the MMU + * The MMU features include the following: + * - separate, fully-associative, 32-entry data and instruction TLBs + * - TLB entries that support 4KB, 64KB, 1MB, and 16MB pages + */ + break; + case cpuinfo_uarch_cortex_a9: + /* + * ARM Cortex‑A9 Technical Reference Manual: + * 6.2.1 Micro TLB + * The first level of caching for the page table information is a micro TLB of 32 entries on the data side, + * and configurable 32 or 64 entries on the instruction side. + * 6.2.2 Main TLB + * The main TLB is implemented as a combination of: + * - A fully-associative, lockable array of four elements. + * - A 2-way associative structure of 2x32, 2x64, 2x128 or 2x256 entries. + */ + break; + case cpuinfo_uarch_cortex_a15: + /* + * ARM Cortex-A15 MPCore Processor Technical Reference Manual: + * 5.2.1. L1 instruction TLB + * The L1 instruction TLB is a 32-entry fully-associative structure. This TLB caches entries at the 4KB + * granularity of Virtual Address (VA) to Physical Address (PA) mapping only. If the page tables map the + * memory region to a larger granularity than 4K, it only allocates one mapping for the particular 4K region + * to which the current access corresponds. + * 5.2.2. L1 data TLB + * There are two separate 32-entry fully-associative TLBs that are used for data loads and stores, + * respectively. Similar to the L1 instruction TLB, both of these cache entries at the 4KB granularity of + * VA to PA mappings only. At implementation time, the Cortex-A15 MPCore processor can be configured with + * the -l1tlb_1m option, to have the L1 data TLB cache entries at both the 4KB and 1MB granularity. + * With this configuration, any translation that results in a 1MB or larger page is cached in the L1 data + * TLB as a 1MB entry. Any translation that results in a page smaller than 1MB is cached in the L1 data TLB + * as a 4KB entry. By default, all translations are cached in the L1 data TLB as a 4KB entry. + * 5.2.3. L2 TLB + * Misses from the L1 instruction and data TLBs are handled by a unified L2 TLB. This is a 512-entry 4-way + * set-associative structure. The L2 TLB supports all the VMSAv7 page sizes of 4K, 64K, 1MB and 16MB in + * addition to the LPAE page sizes of 2MB and 1GB. + */ + break; + case cpuinfo_uarch_cortex_a17: + /* + * ARM Cortex-A17 MPCore Processor Technical Reference Manual: + * 5.2.1. Instruction micro TLB + * The instruction micro TLB is implemented as a 32, 48 or 64 entry, fully-associative structure. This TLB + * caches entries at the 4KB and 1MB granularity of Virtual Address (VA) to Physical Address (PA) mapping + * only. If the translation tables map the memory region to a larger granularity than 4KB or 1MB, it only + * allocates one mapping for the particular 4KB region to which the current access corresponds. + * 5.2.2. Data micro TLB + * The data micro TLB is a 32 entry fully-associative TLB that is used for data loads and stores. The cache + * entries have a 4KB and 1MB granularity of VA to PA mappings only. + * 5.2.3. Unified main TLB + * Misses from the instruction and data micro TLBs are handled by a unified main TLB. This is a 1024 entry + * 4-way set-associative structure. The main TLB supports all the VMSAv7 page sizes of 4K, 64K, 1MB and 16MB + * in addition to the LPAE page sizes of 2MB and 1GB. + */ + break; + case cpuinfo_uarch_cortex_a35: + /* + * ARM Cortex‑A35 Processor Technical Reference Manual: + * A6.2 TLB Organization + * Micro TLB + * The first level of caching for the translation table information is a micro TLB of ten entries that + * is implemented on each of the instruction and data sides. + * Main TLB + * A unified main TLB handles misses from the micro TLBs. It has a 512-entry, 2-way, set-associative + * structure and supports all VMSAv8 block sizes, except 1GB. If it fetches a 1GB block, the TLB splits + * it into 512MB blocks and stores the appropriate block for the lookup. + */ + break; + case cpuinfo_uarch_cortex_a53: + /* + * ARM Cortex-A53 MPCore Processor Technical Reference Manual: + * 5.2.1. Micro TLB + * The first level of caching for the translation table information is a micro TLB of ten entries that is + * implemented on each of the instruction and data sides. + * 5.2.2. Main TLB + * A unified main TLB handles misses from the micro TLBs. This is a 512-entry, 4-way, set-associative + * structure. The main TLB supports all VMSAv8 block sizes, except 1GB. If a 1GB block is fetched, it is + * split into 512MB blocks and the appropriate block for the lookup stored. + */ + break; + case cpuinfo_uarch_cortex_a57: + /* + * ARM® Cortex-A57 MPCore Processor Technical Reference Manual: + * 5.2.1 L1 instruction TLB + * The L1 instruction TLB is a 48-entry fully-associative structure. This TLB caches entries of three + * different page sizes, natively 4KB, 64KB, and 1MB, of VA to PA mappings. If the page tables map the memory + * region to a larger granularity than 1MB, it only allocates one mapping for the particular 1MB region to + * which the current access corresponds. + * 5.2.2 L1 data TLB + * The L1 data TLB is a 32-entry fully-associative TLB that is used for data loads and stores. This TLB + * caches entries of three different page sizes, natively 4KB, 64KB, and 1MB, of VA to PA mappings. + * 5.2.3 L2 TLB + * Misses from the L1 instruction and data TLBs are handled by a unified L2 TLB. This is a 1024-entry 4-way + * set-associative structure. The L2 TLB supports the page sizes of 4K, 64K, 1MB and 16MB. It also supports + * page sizes of 2MB and 1GB for the long descriptor format translation in AArch32 state and in AArch64 state + * when using the 4KB translation granule. In addition, the L2 TLB supports the 512MB page map size defined + * for the AArch64 translations that use a 64KB translation granule. + */ + break; +} + + diff --git a/3rdparty/cpuinfo/src/arm/uarch.c b/3rdparty/cpuinfo/src/arm/uarch.c new file mode 100644 index 000000000..346e1c188 --- /dev/null +++ b/3rdparty/cpuinfo/src/arm/uarch.c @@ -0,0 +1,370 @@ +#include + +#include +#include +#include + + +void cpuinfo_arm_decode_vendor_uarch( + uint32_t midr, +#if CPUINFO_ARCH_ARM + bool has_vfpv4, +#endif /* CPUINFO_ARCH_ARM */ + enum cpuinfo_vendor vendor[restrict static 1], + enum cpuinfo_uarch uarch[restrict static 1]) +{ + switch (midr_get_implementer(midr)) { + case 'A': + *vendor = cpuinfo_vendor_arm; + switch (midr_get_part(midr)) { +#if CPUINFO_ARCH_ARM + case 0xC05: + *uarch = cpuinfo_uarch_cortex_a5; + break; + case 0xC07: + *uarch = cpuinfo_uarch_cortex_a7; + break; + case 0xC08: + *uarch = cpuinfo_uarch_cortex_a8; + break; + case 0xC09: + *uarch = cpuinfo_uarch_cortex_a9; + break; + case 0xC0C: + *uarch = cpuinfo_uarch_cortex_a12; + break; + case 0xC0E: + *uarch = cpuinfo_uarch_cortex_a17; + break; + case 0xC0D: + /* + * Rockchip RK3288 only. + * Core information is ambiguous: some sources specify Cortex-A12, others - Cortex-A17. + * Assume it is Cortex-A12. + */ + *uarch = cpuinfo_uarch_cortex_a12; + break; + case 0xC0F: + *uarch = cpuinfo_uarch_cortex_a15; + break; +#endif /* CPUINFO_ARCH_ARM */ + case 0xD01: + *uarch = cpuinfo_uarch_cortex_a32; + break; + case 0xD03: + *uarch = cpuinfo_uarch_cortex_a53; + break; + case 0xD04: + *uarch = cpuinfo_uarch_cortex_a35; + break; + case 0xD05: + // Note: use Variant, not Revision, field + *uarch = (midr & CPUINFO_ARM_MIDR_VARIANT_MASK) == 0 ? + cpuinfo_uarch_cortex_a55r0 : cpuinfo_uarch_cortex_a55; + break; + case 0xD06: + *uarch = cpuinfo_uarch_cortex_a65; + break; + case 0xD07: + *uarch = cpuinfo_uarch_cortex_a57; + break; + case 0xD08: + *uarch = cpuinfo_uarch_cortex_a72; + break; + case 0xD09: + *uarch = cpuinfo_uarch_cortex_a73; + break; + case 0xD0A: + *uarch = cpuinfo_uarch_cortex_a75; + break; + case 0xD0B: + *uarch = cpuinfo_uarch_cortex_a76; + break; +#if CPUINFO_ARCH_ARM64 && !defined(__ANDROID__) + case 0xD0C: + *uarch = cpuinfo_uarch_neoverse_n1; + break; +#endif /* CPUINFO_ARCH_ARM64 && !defined(__ANDROID__) */ + case 0xD0D: + *uarch = cpuinfo_uarch_cortex_a77; + break; + case 0xD0E: /* Cortex-A76AE */ + *uarch = cpuinfo_uarch_cortex_a76; + break; +#if CPUINFO_ARCH_ARM64 && !defined(__ANDROID__) + case 0xD40: + *uarch = cpuinfo_uarch_neoverse_v1; + break; +#endif /* CPUINFO_ARCH_ARM64 && !defined(__ANDROID__) */ + case 0xD41: /* Cortex-A78 */ + *uarch = cpuinfo_uarch_cortex_a78; + break; + case 0xD44: /* Cortex-X1 */ + *uarch = cpuinfo_uarch_cortex_x1; + break; +#if CPUINFO_ARCH_ARM64 && !defined(__ANDROID__) + case 0xD49: + *uarch = cpuinfo_uarch_neoverse_n2; + break; + case 0xD4A: + *uarch = cpuinfo_uarch_neoverse_e1; + break; +#endif /* CPUINFO_ARCH_ARM64 && !defined(__ANDROID__) */ + default: + switch (midr_get_part(midr) >> 8) { +#if CPUINFO_ARCH_ARM + case 7: + *uarch = cpuinfo_uarch_arm7; + break; + case 9: + *uarch = cpuinfo_uarch_arm9; + break; + case 11: + *uarch = cpuinfo_uarch_arm11; + break; +#endif /* CPUINFO_ARCH_ARM */ + default: + cpuinfo_log_warning("unknown ARM CPU part 0x%03"PRIx32" ignored", midr_get_part(midr)); + } + } + break; + case 'B': + *vendor = cpuinfo_vendor_broadcom; + switch (midr_get_part(midr)) { + case 0x00F: + *uarch = cpuinfo_uarch_brahma_b15; + break; + case 0x100: + *uarch = cpuinfo_uarch_brahma_b53; + break; +#if CPUINFO_ARCH_ARM64 && !defined(__ANDROID__) + case 0x516: + /* Broadcom Vulkan was sold to Cavium before it reached the market, so we identify it as Cavium ThunderX2 */ + *vendor = cpuinfo_vendor_cavium; + *uarch = cpuinfo_uarch_thunderx2; + break; +#endif + default: + cpuinfo_log_warning("unknown Broadcom CPU part 0x%03"PRIx32" ignored", midr_get_part(midr)); + } + break; +#if CPUINFO_ARCH_ARM64 && !defined(__ANDROID__) + case 'C': + *vendor = cpuinfo_vendor_cavium; + switch (midr_get_part(midr)) { + case 0x0A0: /* ThunderX */ + case 0x0A1: /* ThunderX 88XX */ + case 0x0A2: /* ThunderX 81XX */ + case 0x0A3: /* ThunderX 83XX */ + *uarch = cpuinfo_uarch_thunderx; + break; + case 0x0AF: /* ThunderX2 99XX */ + *uarch = cpuinfo_uarch_thunderx2; + break; + default: + cpuinfo_log_warning("unknown Cavium CPU part 0x%03"PRIx32" ignored", midr_get_part(midr)); + } + break; +#endif + case 'H': + *vendor = cpuinfo_vendor_huawei; + switch (midr_get_part(midr)) { +#if CPUINFO_ARCH_ARM64 && !defined(__ANDROID__) + case 0xD01: /* Kunpeng 920 series */ + *uarch = cpuinfo_uarch_taishan_v110; + break; +#endif + case 0xD40: /* Kirin 980 Big/Medium cores -> Cortex-A76 */ + *vendor = cpuinfo_vendor_arm; + *uarch = cpuinfo_uarch_cortex_a76; + break; + default: + cpuinfo_log_warning("unknown Huawei CPU part 0x%03"PRIx32" ignored", midr_get_part(midr)); + } + break; +#if CPUINFO_ARCH_ARM + case 'i': + *vendor = cpuinfo_vendor_intel; + switch (midr_get_part(midr) >> 8) { + case 2: /* PXA 210/25X/26X */ + case 4: /* PXA 27X */ + case 6: /* PXA 3XX */ + *uarch = cpuinfo_uarch_xscale; + break; + default: + cpuinfo_log_warning("unknown Intel CPU part 0x%03"PRIx32" ignored", midr_get_part(midr)); + } + break; +#endif /* CPUINFO_ARCH_ARM */ + case 'N': + *vendor = cpuinfo_vendor_nvidia; + switch (midr_get_part(midr)) { + case 0x000: + *uarch = cpuinfo_uarch_denver; + break; + case 0x003: + *uarch = cpuinfo_uarch_denver2; + break; + case 0x004: + *uarch = cpuinfo_uarch_carmel; + break; + default: + cpuinfo_log_warning("unknown Nvidia CPU part 0x%03"PRIx32" ignored", midr_get_part(midr)); + } + break; +#if !defined(__ANDROID__) + case 'P': + *vendor = cpuinfo_vendor_apm; + switch (midr_get_part(midr)) { + case 0x000: + *uarch = cpuinfo_uarch_xgene; + break; + default: + cpuinfo_log_warning("unknown Applied Micro CPU part 0x%03"PRIx32" ignored", midr_get_part(midr)); + } + break; +#endif + case 'Q': + *vendor = cpuinfo_vendor_qualcomm; + switch (midr_get_part(midr)) { +#if CPUINFO_ARCH_ARM + case 0x00F: + /* Mostly Scorpions, but some Cortex A5 may report this value as well */ + if (has_vfpv4) { + /* Unlike Scorpion, Cortex-A5 comes with VFPv4 */ + *vendor = cpuinfo_vendor_arm; + *uarch = cpuinfo_uarch_cortex_a5; + } else { + *uarch = cpuinfo_uarch_scorpion; + } + break; + case 0x02D: /* Dual-core Scorpions */ + *uarch = cpuinfo_uarch_scorpion; + break; + case 0x04D: + /* + * Dual-core Krait: + * - r1p0 -> Krait 200 + * - r1p4 -> Krait 200 + * - r2p0 -> Krait 300 + */ + case 0x06F: + /* + * Quad-core Krait: + * - r0p1 -> Krait 200 + * - r0p2 -> Krait 200 + * - r1p0 -> Krait 300 + * - r2p0 -> Krait 400 (Snapdragon 800 MSMxxxx) + * - r2p1 -> Krait 400 (Snapdragon 801 MSMxxxxPRO) + * - r3p1 -> Krait 450 + */ + *uarch = cpuinfo_uarch_krait; + break; +#endif /* CPUINFO_ARCH_ARM */ + case 0x201: /* Qualcomm Snapdragon 821: Low-power Kryo "Silver" */ + case 0x205: /* Qualcomm Snapdragon 820 & 821: High-performance Kryo "Gold" */ + case 0x211: /* Qualcomm Snapdragon 820: Low-power Kryo "Silver" */ + *uarch = cpuinfo_uarch_kryo; + break; + case 0x800: /* High-performance Kryo 260 (r10p2) / Kryo 280 (r10p1) "Gold" -> Cortex-A73 */ + *vendor = cpuinfo_vendor_arm; + *uarch = cpuinfo_uarch_cortex_a73; + break; + case 0x801: /* Low-power Kryo 260 / 280 "Silver" -> Cortex-A53 */ + *vendor = cpuinfo_vendor_arm; + *uarch = cpuinfo_uarch_cortex_a53; + break; + case 0x802: /* High-performance Kryo 385 "Gold" -> Cortex-A75 */ + *vendor = cpuinfo_vendor_arm; + *uarch = cpuinfo_uarch_cortex_a75; + break; + case 0x803: /* Low-power Kryo 385 "Silver" -> Cortex-A55r0 */ + *vendor = cpuinfo_vendor_arm; + *uarch = cpuinfo_uarch_cortex_a55r0; + break; + case 0x804: /* High-performance Kryo 485 "Gold" / "Gold Prime" -> Cortex-A76 */ + *vendor = cpuinfo_vendor_arm; + *uarch = cpuinfo_uarch_cortex_a76; + break; + case 0x805: /* Low-performance Kryo 485 "Silver" -> Cortex-A55 */ + *vendor = cpuinfo_vendor_arm; + *uarch = cpuinfo_uarch_cortex_a55; + break; +#if CPUINFO_ARCH_ARM64 && !defined(__ANDROID__) + case 0xC00: + *uarch = cpuinfo_uarch_falkor; + break; + case 0xC01: + *uarch = cpuinfo_uarch_saphira; + break; +#endif /* CPUINFO_ARCH_ARM64 && !defined(__ANDROID__) */ + default: + cpuinfo_log_warning("unknown Qualcomm CPU part 0x%03"PRIx32" ignored", midr_get_part(midr)); + } + break; + case 'S': + *vendor = cpuinfo_vendor_samsung; + switch (midr & (CPUINFO_ARM_MIDR_VARIANT_MASK | CPUINFO_ARM_MIDR_PART_MASK)) { + case 0x00100010: + /* + * Exynos 8890 MIDR = 0x531F0011, assume Exynos M1 has: + * - CPU variant 0x1 + * - CPU part 0x001 + */ + *uarch = cpuinfo_uarch_exynos_m1; + break; + case 0x00400010: + /* + * Exynos 8895 MIDR = 0x534F0010, assume Exynos M2 has: + * - CPU variant 0x4 + * - CPU part 0x001 + */ + *uarch = cpuinfo_uarch_exynos_m2; + break; + case 0x00100020: + /* + * Exynos 9810 MIDR = 0x531F0020, assume Exynos M3 has: + * - CPU variant 0x1 + * - CPU part 0x002 + */ + *uarch = cpuinfo_uarch_exynos_m3; + break; + case 0x00100030: + /* + * Exynos 9820 MIDR = 0x531F0030, assume Exynos M4 has: + * - CPU variant 0x1 + * - CPU part 0x003 + */ + *uarch = cpuinfo_uarch_exynos_m4; + break; + case 0x00100040: + /* + * Exynos 9820 MIDR = 0x531F0040, assume Exynos M5 has: + * - CPU variant 0x1 + * - CPU part 0x004 + */ + *uarch = cpuinfo_uarch_exynos_m5; + break; + default: + cpuinfo_log_warning("unknown Samsung CPU variant 0x%01"PRIx32" part 0x%03"PRIx32" ignored", + midr_get_variant(midr), midr_get_part(midr)); + } + break; +#if CPUINFO_ARCH_ARM + case 'V': + *vendor = cpuinfo_vendor_marvell; + switch (midr_get_part(midr)) { + case 0x581: /* PJ4 / PJ4B */ + case 0x584: /* PJ4B-MP / PJ4C */ + *uarch = cpuinfo_uarch_pj4; + break; + default: + cpuinfo_log_warning("unknown Marvell CPU part 0x%03"PRIx32" ignored", midr_get_part(midr)); + } + break; +#endif /* CPUINFO_ARCH_ARM */ + default: + cpuinfo_log_warning("unknown CPU implementer '%c' (0x%02"PRIx32") with CPU part 0x%03"PRIx32" ignored", + (char) midr_get_implementer(midr), midr_get_implementer(midr), midr_get_part(midr)); + } +} diff --git a/3rdparty/cpuinfo/src/cache.c b/3rdparty/cpuinfo/src/cache.c new file mode 100644 index 000000000..b976b8796 --- /dev/null +++ b/3rdparty/cpuinfo/src/cache.c @@ -0,0 +1,18 @@ +#include + +#include +#include + + +uint32_t cpuinfo_compute_max_cache_size(const struct cpuinfo_processor* processor) { + if (processor->cache.l4 != NULL) { + return processor->cache.l4->size; + } else if (processor->cache.l3 != NULL) { + return processor->cache.l3->size; + } else if (processor->cache.l2 != NULL) { + return processor->cache.l2->size; + } else if (processor->cache.l1d != NULL) { + return processor->cache.l1d->size; + } + return 0; +} diff --git a/3rdparty/cpuinfo/src/cpuinfo/common.h b/3rdparty/cpuinfo/src/cpuinfo/common.h new file mode 100644 index 000000000..b2b404d74 --- /dev/null +++ b/3rdparty/cpuinfo/src/cpuinfo/common.h @@ -0,0 +1,40 @@ +/* + * Copyright (c) Facebook, Inc. and its affiliates. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. + */ + +#pragma once + + +#define CPUINFO_COUNT_OF(array) (sizeof(array) / sizeof(0[array])) + +#if defined(__GNUC__) + #define CPUINFO_LIKELY(condition) (__builtin_expect(!!(condition), 1)) + #define CPUINFO_UNLIKELY(condition) (__builtin_expect(!!(condition), 0)) +#else + #define CPUINFO_LIKELY(condition) (!!(condition)) + #define CPUINFO_UNLIKELY(condition) (!!(condition)) +#endif + +#ifndef CPUINFO_INTERNAL + #if defined(__ELF__) + #define CPUINFO_INTERNAL __attribute__((__visibility__("internal"))) + #elif defined(__MACH__) + #define CPUINFO_INTERNAL __attribute__((__visibility__("hidden"))) + #else + #define CPUINFO_INTERNAL + #endif +#endif + +#ifndef CPUINFO_PRIVATE + #if defined(__ELF__) + #define CPUINFO_PRIVATE __attribute__((__visibility__("hidden"))) + #elif defined(__MACH__) + #define CPUINFO_PRIVATE __attribute__((__visibility__("hidden"))) + #else + #define CPUINFO_PRIVATE + #endif +#endif diff --git a/3rdparty/cpuinfo/src/cpuinfo/internal-api.h b/3rdparty/cpuinfo/src/cpuinfo/internal-api.h new file mode 100644 index 000000000..9c23d7c8b --- /dev/null +++ b/3rdparty/cpuinfo/src/cpuinfo/internal-api.h @@ -0,0 +1,62 @@ +#pragma once + +#include +#include + +#if defined(_WIN32) || defined(__CYGWIN__) + #include +#endif + +#include +#include + + +enum cpuinfo_cache_level { + cpuinfo_cache_level_1i = 0, + cpuinfo_cache_level_1d = 1, + cpuinfo_cache_level_2 = 2, + cpuinfo_cache_level_3 = 3, + cpuinfo_cache_level_4 = 4, + cpuinfo_cache_level_max = 5, +}; + +extern CPUINFO_INTERNAL bool cpuinfo_is_initialized; + +extern CPUINFO_INTERNAL struct cpuinfo_processor* cpuinfo_processors; +extern CPUINFO_INTERNAL struct cpuinfo_core* cpuinfo_cores; +extern CPUINFO_INTERNAL struct cpuinfo_cluster* cpuinfo_clusters; +extern CPUINFO_INTERNAL struct cpuinfo_package* cpuinfo_packages; +extern CPUINFO_INTERNAL struct cpuinfo_cache* cpuinfo_cache[cpuinfo_cache_level_max]; + +extern CPUINFO_INTERNAL uint32_t cpuinfo_processors_count; +extern CPUINFO_INTERNAL uint32_t cpuinfo_cores_count; +extern CPUINFO_INTERNAL uint32_t cpuinfo_clusters_count; +extern CPUINFO_INTERNAL uint32_t cpuinfo_packages_count; +extern CPUINFO_INTERNAL uint32_t cpuinfo_cache_count[cpuinfo_cache_level_max]; +extern CPUINFO_INTERNAL uint32_t cpuinfo_max_cache_size; + +#if CPUINFO_ARCH_ARM || CPUINFO_ARCH_ARM64 + extern CPUINFO_INTERNAL struct cpuinfo_uarch_info* cpuinfo_uarchs; + extern CPUINFO_INTERNAL uint32_t cpuinfo_uarchs_count; +#else + extern CPUINFO_INTERNAL struct cpuinfo_uarch_info cpuinfo_global_uarch; +#endif + +#ifdef __linux__ + extern CPUINFO_INTERNAL uint32_t cpuinfo_linux_cpu_max; + extern CPUINFO_INTERNAL const struct cpuinfo_processor** cpuinfo_linux_cpu_to_processor_map; + extern CPUINFO_INTERNAL const struct cpuinfo_core** cpuinfo_linux_cpu_to_core_map; +#endif + +CPUINFO_PRIVATE void cpuinfo_x86_mach_init(void); +CPUINFO_PRIVATE void cpuinfo_x86_linux_init(void); +#if defined(_WIN32) || defined(__CYGWIN__) + CPUINFO_PRIVATE BOOL CALLBACK cpuinfo_x86_windows_init(PINIT_ONCE init_once, PVOID parameter, PVOID* context); +#endif +CPUINFO_PRIVATE void cpuinfo_arm_mach_init(void); +CPUINFO_PRIVATE void cpuinfo_arm_linux_init(void); +CPUINFO_PRIVATE void cpuinfo_emscripten_init(void); + +CPUINFO_PRIVATE uint32_t cpuinfo_compute_max_cache_size(const struct cpuinfo_processor* processor); + +typedef void (*cpuinfo_processor_callback)(uint32_t); diff --git a/3rdparty/cpuinfo/src/cpuinfo/log.h b/3rdparty/cpuinfo/src/cpuinfo/log.h new file mode 100644 index 000000000..dac8cdb66 --- /dev/null +++ b/3rdparty/cpuinfo/src/cpuinfo/log.h @@ -0,0 +1,17 @@ +#pragma once + +#include + +#include + +#define CPUINFO_LOG_DEBUG_PARSERS 0 + +#ifndef CPUINFO_LOG_LEVEL + #define CPUINFO_LOG_LEVEL CLOG_ERROR +#endif + +CLOG_DEFINE_LOG_DEBUG(cpuinfo_log_debug, "cpuinfo", CPUINFO_LOG_LEVEL); +CLOG_DEFINE_LOG_INFO(cpuinfo_log_info, "cpuinfo", CPUINFO_LOG_LEVEL); +CLOG_DEFINE_LOG_WARNING(cpuinfo_log_warning, "cpuinfo", CPUINFO_LOG_LEVEL); +CLOG_DEFINE_LOG_ERROR(cpuinfo_log_error, "cpuinfo", CPUINFO_LOG_LEVEL); +CLOG_DEFINE_LOG_FATAL(cpuinfo_log_fatal, "cpuinfo", CPUINFO_LOG_LEVEL); diff --git a/3rdparty/cpuinfo/src/cpuinfo/utils.h b/3rdparty/cpuinfo/src/cpuinfo/utils.h new file mode 100644 index 000000000..157baad9a --- /dev/null +++ b/3rdparty/cpuinfo/src/cpuinfo/utils.h @@ -0,0 +1,19 @@ +#pragma once + +#include + + +inline static uint32_t bit_length(uint32_t n) { + const uint32_t n_minus_1 = n - 1; + if (n_minus_1 == 0) { + return 0; + } else { + #ifdef _MSC_VER + unsigned long bsr; + _BitScanReverse(&bsr, n_minus_1); + return bsr + 1; + #else + return 32 - __builtin_clz(n_minus_1); + #endif + } +} diff --git a/3rdparty/cpuinfo/src/emscripten/init.c b/3rdparty/cpuinfo/src/emscripten/init.c new file mode 100644 index 000000000..ce4bdea2e --- /dev/null +++ b/3rdparty/cpuinfo/src/emscripten/init.c @@ -0,0 +1,277 @@ +#include +#include +#include +#include +#include + +#include + +#include +#include +#include + + +static const volatile float infinity = INFINITY; + +static struct cpuinfo_package static_package = { }; + +static struct cpuinfo_cache static_x86_l3 = { + .size = 2 * 1024 * 1024, + .associativity = 16, + .sets = 2048, + .partitions = 1, + .line_size = 64, +}; + +void cpuinfo_emscripten_init(void) { + struct cpuinfo_processor* processors = NULL; + struct cpuinfo_core* cores = NULL; + struct cpuinfo_cluster* clusters = NULL; + struct cpuinfo_cache* l1i = NULL; + struct cpuinfo_cache* l1d = NULL; + struct cpuinfo_cache* l2 = NULL; + + const bool is_x86 = signbit(infinity - infinity); + + int logical_cores_count = emscripten_num_logical_cores(); + if (logical_cores_count <= 0) { + logical_cores_count = 1; + } + uint32_t processor_count = (uint32_t) logical_cores_count; + uint32_t core_count = processor_count; + uint32_t cluster_count = 1; + uint32_t big_cluster_core_count = core_count; + uint32_t processors_per_core = 1; + if (is_x86) { + if (processor_count % 2 == 0) { + processors_per_core = 2; + core_count = processor_count / 2; + big_cluster_core_count = core_count; + } + } else { + /* Assume ARM/ARM64 */ + if (processor_count > 4) { + /* Assume big.LITTLE architecture */ + cluster_count = 2; + big_cluster_core_count = processor_count >= 8 ? 4 : 2; + } + } + uint32_t l2_count = is_x86 ? core_count : cluster_count; + + processors = calloc(processor_count, sizeof(struct cpuinfo_processor)); + if (processors == NULL) { + cpuinfo_log_error("failed to allocate %zu bytes for descriptions of %"PRIu32" logical processors", + processor_count * sizeof(struct cpuinfo_processor), processor_count); + goto cleanup; + } + cores = calloc(processor_count, sizeof(struct cpuinfo_core)); + if (cores == NULL) { + cpuinfo_log_error("failed to allocate %zu bytes for descriptions of %"PRIu32" cores", + processor_count * sizeof(struct cpuinfo_core), processor_count); + goto cleanup; + } + clusters = calloc(cluster_count, sizeof(struct cpuinfo_cluster)); + if (clusters == NULL) { + cpuinfo_log_error("failed to allocate %zu bytes for descriptions of %"PRIu32" clusters", + cluster_count * sizeof(struct cpuinfo_cluster), cluster_count); + goto cleanup; + } + + l1i = calloc(core_count, sizeof(struct cpuinfo_cache)); + if (l1i == NULL) { + cpuinfo_log_error("failed to allocate %zu bytes for descriptions of %"PRIu32" L1I caches", + core_count * sizeof(struct cpuinfo_cache), core_count); + goto cleanup; + } + + l1d = calloc(core_count, sizeof(struct cpuinfo_cache)); + if (l1d == NULL) { + cpuinfo_log_error("failed to allocate %zu bytes for descriptions of %"PRIu32" L1D caches", + core_count * sizeof(struct cpuinfo_cache), core_count); + goto cleanup; + } + + l2 = calloc(l2_count, sizeof(struct cpuinfo_cache)); + if (l2 == NULL) { + cpuinfo_log_error("failed to allocate %zu bytes for descriptions of %"PRIu32" L2 caches", + l2_count * sizeof(struct cpuinfo_cache), l2_count); + goto cleanup; + } + + static_package.processor_count = processor_count; + static_package.core_count = core_count; + static_package.cluster_count = cluster_count; + if (is_x86) { + strncpy(static_package.name, "x86 vCPU", CPUINFO_PACKAGE_NAME_MAX); + } else { + strncpy(static_package.name, "ARM vCPU", CPUINFO_PACKAGE_NAME_MAX); + } + + for (uint32_t i = 0; i < core_count; i++) { + for (uint32_t j = 0; j < processors_per_core; j++) { + processors[i * processors_per_core + j] = (struct cpuinfo_processor) { + .smt_id = j, + .core = cores + i, + .cluster = clusters + (uint32_t) (i >= big_cluster_core_count), + .package = &static_package, + .cache.l1i = l1i + i, + .cache.l1d = l1d + i, + .cache.l2 = is_x86 ? l2 + i : l2 + (uint32_t) (i >= big_cluster_core_count), + .cache.l3 = is_x86 ? &static_x86_l3 : NULL, + }; + } + + cores[i] = (struct cpuinfo_core) { + .processor_start = i * processors_per_core, + .processor_count = processors_per_core, + .core_id = i, + .cluster = clusters + (uint32_t) (i >= big_cluster_core_count), + .package = &static_package, + .vendor = cpuinfo_vendor_unknown, + .uarch = cpuinfo_uarch_unknown, + .frequency = 0, + }; + + l1i[i] = (struct cpuinfo_cache) { + .size = 32 * 1024, + .associativity = 4, + .sets = 128, + .partitions = 1, + .line_size = 64, + .processor_start = i * processors_per_core, + .processor_count = processors_per_core, + }; + + l1d[i] = (struct cpuinfo_cache) { + .size = 32 * 1024, + .associativity = 4, + .sets = 128, + .partitions = 1, + .line_size = 64, + .processor_start = i * processors_per_core, + .processor_count = processors_per_core, + }; + + if (is_x86) { + l2[i] = (struct cpuinfo_cache) { + .size = 256 * 1024, + .associativity = 8, + .sets = 512, + .partitions = 1, + .line_size = 64, + .processor_start = i * processors_per_core, + .processor_count = processors_per_core, + }; + } + } + + if (is_x86) { + clusters[0] = (struct cpuinfo_cluster) { + .processor_start = 0, + .processor_count = processor_count, + .core_start = 0, + .core_count = core_count, + .cluster_id = 0, + .package = &static_package, + .vendor = cpuinfo_vendor_unknown, + .uarch = cpuinfo_uarch_unknown, + .frequency = 0, + }; + + static_x86_l3.processor_count = processor_count; + } else { + clusters[0] = (struct cpuinfo_cluster) { + .processor_start = 0, + .processor_count = big_cluster_core_count, + .core_start = 0, + .core_count = big_cluster_core_count, + .cluster_id = 0, + .package = &static_package, + .vendor = cpuinfo_vendor_unknown, + .uarch = cpuinfo_uarch_unknown, + .frequency = 0, + }; + + l2[0] = (struct cpuinfo_cache) { + .size = 1024 * 1024, + .associativity = 8, + .sets = 2048, + .partitions = 1, + .line_size = 64, + .processor_start = 0, + .processor_count = big_cluster_core_count, + }; + + if (cluster_count > 1) { + l2[1] = (struct cpuinfo_cache) { + .size = 256 * 1024, + .associativity = 8, + .sets = 512, + .partitions = 1, + .line_size = 64, + .processor_start = big_cluster_core_count, + .processor_count = processor_count - big_cluster_core_count, + }; + + clusters[1] = (struct cpuinfo_cluster) { + .processor_start = big_cluster_core_count, + .processor_count = processor_count - big_cluster_core_count, + .core_start = big_cluster_core_count, + .core_count = processor_count - big_cluster_core_count, + .cluster_id = 1, + .package = &static_package, + .vendor = cpuinfo_vendor_unknown, + .uarch = cpuinfo_uarch_unknown, + .frequency = 0, + }; + } + } + + /* Commit changes */ + cpuinfo_cache[cpuinfo_cache_level_1i] = l1i; + cpuinfo_cache[cpuinfo_cache_level_1d] = l1d; + cpuinfo_cache[cpuinfo_cache_level_2] = l2; + if (is_x86) { + cpuinfo_cache[cpuinfo_cache_level_3] = &static_x86_l3; + } + + cpuinfo_processors = processors; + cpuinfo_cores = cores; + cpuinfo_clusters = clusters; + cpuinfo_packages = &static_package; + + cpuinfo_cache_count[cpuinfo_cache_level_1i] = processor_count; + cpuinfo_cache_count[cpuinfo_cache_level_1d] = processor_count; + cpuinfo_cache_count[cpuinfo_cache_level_2] = l2_count; + if (is_x86) { + cpuinfo_cache_count[cpuinfo_cache_level_3] = 1; + } + + cpuinfo_global_uarch = (struct cpuinfo_uarch_info) { + .uarch = cpuinfo_uarch_unknown, + .processor_count = processor_count, + .core_count = core_count, + }; + + cpuinfo_processors_count = processor_count; + cpuinfo_cores_count = processor_count; + cpuinfo_clusters_count = cluster_count; + cpuinfo_packages_count = 1; + + cpuinfo_max_cache_size = is_x86 ? 128 * 1024 * 1024 : 8 * 1024 * 1024; + + cpuinfo_is_initialized = true; + + processors = NULL; + cores = NULL; + clusters = NULL; + l1i = l1d = l2 = NULL; + +cleanup: + free(processors); + free(cores); + free(clusters); + free(l1i); + free(l1d); + free(l2); +} diff --git a/3rdparty/cpuinfo/src/init.c b/3rdparty/cpuinfo/src/init.c new file mode 100644 index 000000000..d61e7be6e --- /dev/null +++ b/3rdparty/cpuinfo/src/init.c @@ -0,0 +1,59 @@ +#if defined(_WIN32) || defined(__CYGWIN__) + #include +#elif !defined(__EMSCRIPTEN__) || defined(__EMSCRIPTEN_PTHREADS__) + #include +#endif + +#include +#include +#include + +#ifdef __APPLE__ + #include "TargetConditionals.h" +#endif + + +#if defined(_WIN32) || defined(__CYGWIN__) + static INIT_ONCE init_guard = INIT_ONCE_STATIC_INIT; +#elif !defined(__EMSCRIPTEN__) || defined(__EMSCRIPTEN_PTHREADS__) + static pthread_once_t init_guard = PTHREAD_ONCE_INIT; +#else + static bool init_guard = false; +#endif + +bool CPUINFO_ABI cpuinfo_initialize(void) { +#if CPUINFO_ARCH_X86 || CPUINFO_ARCH_X86_64 + #if defined(__MACH__) && defined(__APPLE__) + pthread_once(&init_guard, &cpuinfo_x86_mach_init); + #elif defined(__linux__) + pthread_once(&init_guard, &cpuinfo_x86_linux_init); + #elif defined(_WIN32) || defined(__CYGWIN__) + InitOnceExecuteOnce(&init_guard, &cpuinfo_x86_windows_init, NULL, NULL); + #else + cpuinfo_log_error("operating system is not supported in cpuinfo"); + #endif +#elif CPUINFO_ARCH_ARM || CPUINFO_ARCH_ARM64 + #if defined(__linux__) + pthread_once(&init_guard, &cpuinfo_arm_linux_init); + #elif defined(__MACH__) && defined(__APPLE__) + pthread_once(&init_guard, &cpuinfo_arm_mach_init); + #else + cpuinfo_log_error("operating system is not supported in cpuinfo"); + #endif +#elif CPUINFO_ARCH_ASMJS || CPUINFO_ARCH_WASM || CPUINFO_ARCH_WASMSIMD + #if defined(__EMSCRIPTEN_PTHREADS__) + pthread_once(&init_guard, &cpuinfo_emscripten_init); + #else + if (!init_guard) { + cpuinfo_emscripten_init(); + } + init_guard = true; + #endif +#else + cpuinfo_log_error("processor architecture is not supported in cpuinfo"); +#endif + return cpuinfo_is_initialized; +} + +void CPUINFO_ABI cpuinfo_deinitialize(void) { +} diff --git a/3rdparty/cpuinfo/src/linux/api.h b/3rdparty/cpuinfo/src/linux/api.h new file mode 100644 index 000000000..f55b8ac73 --- /dev/null +++ b/3rdparty/cpuinfo/src/linux/api.h @@ -0,0 +1,59 @@ +#pragma once + +#include +#include +#include + +#include +#include + + +#define CPUINFO_LINUX_FLAG_PRESENT UINT32_C(0x00000001) +#define CPUINFO_LINUX_FLAG_POSSIBLE UINT32_C(0x00000002) +#define CPUINFO_LINUX_FLAG_MAX_FREQUENCY UINT32_C(0x00000004) +#define CPUINFO_LINUX_FLAG_MIN_FREQUENCY UINT32_C(0x00000008) +#define CPUINFO_LINUX_FLAG_SMT_ID UINT32_C(0x00000010) +#define CPUINFO_LINUX_FLAG_CORE_ID UINT32_C(0x00000020) +#define CPUINFO_LINUX_FLAG_PACKAGE_ID UINT32_C(0x00000040) +#define CPUINFO_LINUX_FLAG_APIC_ID UINT32_C(0x00000080) +#define CPUINFO_LINUX_FLAG_SMT_CLUSTER UINT32_C(0x00000100) +#define CPUINFO_LINUX_FLAG_CORE_CLUSTER UINT32_C(0x00000200) +#define CPUINFO_LINUX_FLAG_PACKAGE_CLUSTER UINT32_C(0x00000400) +#define CPUINFO_LINUX_FLAG_PROC_CPUINFO UINT32_C(0x00000800) +#define CPUINFO_LINUX_FLAG_VALID UINT32_C(0x00001000) + + +typedef bool (*cpuinfo_cpulist_callback)(uint32_t, uint32_t, void*); +CPUINFO_INTERNAL bool cpuinfo_linux_parse_cpulist(const char* filename, cpuinfo_cpulist_callback callback, void* context); +typedef bool (*cpuinfo_smallfile_callback)(const char*, const char*, void*); +CPUINFO_INTERNAL bool cpuinfo_linux_parse_small_file(const char* filename, size_t buffer_size, cpuinfo_smallfile_callback, void* context); +typedef bool (*cpuinfo_line_callback)(const char*, const char*, void*, uint64_t); +CPUINFO_INTERNAL bool cpuinfo_linux_parse_multiline_file(const char* filename, size_t buffer_size, cpuinfo_line_callback, void* context); + +CPUINFO_INTERNAL uint32_t cpuinfo_linux_get_max_processors_count(void); +CPUINFO_INTERNAL uint32_t cpuinfo_linux_get_max_possible_processor(uint32_t max_processors_count); +CPUINFO_INTERNAL uint32_t cpuinfo_linux_get_max_present_processor(uint32_t max_processors_count); +CPUINFO_INTERNAL uint32_t cpuinfo_linux_get_processor_min_frequency(uint32_t processor); +CPUINFO_INTERNAL uint32_t cpuinfo_linux_get_processor_max_frequency(uint32_t processor); +CPUINFO_INTERNAL bool cpuinfo_linux_get_processor_package_id(uint32_t processor, uint32_t package_id[restrict static 1]); +CPUINFO_INTERNAL bool cpuinfo_linux_get_processor_core_id(uint32_t processor, uint32_t core_id[restrict static 1]); + +CPUINFO_INTERNAL bool cpuinfo_linux_detect_possible_processors(uint32_t max_processors_count, + uint32_t* processor0_flags, uint32_t processor_struct_size, uint32_t possible_flag); +CPUINFO_INTERNAL bool cpuinfo_linux_detect_present_processors(uint32_t max_processors_count, + uint32_t* processor0_flags, uint32_t processor_struct_size, uint32_t present_flag); + +typedef bool (*cpuinfo_siblings_callback)(uint32_t, uint32_t, uint32_t, void*); +CPUINFO_INTERNAL bool cpuinfo_linux_detect_core_siblings( + uint32_t max_processors_count, + uint32_t processor, + cpuinfo_siblings_callback callback, + void* context); +CPUINFO_INTERNAL bool cpuinfo_linux_detect_thread_siblings( + uint32_t max_processors_count, + uint32_t processor, + cpuinfo_siblings_callback callback, + void* context); + +extern CPUINFO_INTERNAL const struct cpuinfo_processor** cpuinfo_linux_cpu_to_processor_map; +extern CPUINFO_INTERNAL const struct cpuinfo_core** cpuinfo_linux_cpu_to_core_map; diff --git a/3rdparty/cpuinfo/src/linux/cpulist.c b/3rdparty/cpuinfo/src/linux/cpulist.c new file mode 100644 index 000000000..287198638 --- /dev/null +++ b/3rdparty/cpuinfo/src/linux/cpulist.c @@ -0,0 +1,214 @@ +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include + +#if CPUINFO_MOCK + #include +#endif +#include +#include + + +/* + * Size, in chars, of the on-stack buffer used for parsing cpu lists. + * This is also the limit on the length of a single entry + * ( or -) + * in the cpu list. + */ +#define BUFFER_SIZE 256 + + +/* Locale-independent */ +inline static bool is_whitespace(char c) { + switch (c) { + case ' ': + case '\t': + case '\n': + case '\r': + return true; + default: + return false; + } +} + +inline static const char* parse_number(const char* string, const char* end, uint32_t number_ptr[restrict static 1]) { + uint32_t number = 0; + while (string != end) { + const uint32_t digit = (uint32_t) (*string) - (uint32_t) '0'; + if (digit >= 10) { + break; + } + number = number * UINT32_C(10) + digit; + string += 1; + } + *number_ptr = number; + return string; +} + +inline static bool parse_entry(const char* entry_start, const char* entry_end, cpuinfo_cpulist_callback callback, void* context) { + /* Skip whitespace at the beginning of an entry */ + for (; entry_start != entry_end; entry_start++) { + if (!is_whitespace(*entry_start)) { + break; + } + } + /* Skip whitespace at the end of an entry */ + for (; entry_end != entry_start; entry_end--) { + if (!is_whitespace(entry_end[-1])) { + break; + } + } + + const size_t entry_length = (size_t) (entry_end - entry_start); + if (entry_length == 0) { + cpuinfo_log_warning("unexpected zero-length cpu list entry ignored"); + return false; + } + + #if CPUINFO_LOG_DEBUG_PARSERS + cpuinfo_log_debug("parse cpu list entry \"%.*s\" (%zu chars)", (int) entry_length, entry_start, entry_length); + #endif + uint32_t first_cpu, last_cpu; + + const char* number_end = parse_number(entry_start, entry_end, &first_cpu); + if (number_end == entry_start) { + /* Failed to parse the number; ignore the entry */ + cpuinfo_log_warning("invalid character '%c' in the cpu list entry \"%.*s\": entry is ignored", + entry_start[0], (int) entry_length, entry_start); + return false; + } else if (number_end == entry_end) { + /* Completely parsed the entry */ + #if CPUINFO_LOG_DEBUG_PARSERS + cpuinfo_log_debug("cpulist: call callback with list_start = %"PRIu32", list_end = %"PRIu32, + first_cpu, first_cpu + 1); + #endif + return callback(first_cpu, first_cpu + 1, context); + } + + /* Parse the second part of the entry */ + if (*number_end != '-') { + cpuinfo_log_warning("invalid character '%c' in the cpu list entry \"%.*s\": entry is ignored", + *number_end, (int) entry_length, entry_start); + return false; + } + + const char* number_start = number_end + 1; + number_end = parse_number(number_start, entry_end, &last_cpu); + if (number_end == number_start) { + /* Failed to parse the second number; ignore the entry */ + cpuinfo_log_warning("invalid character '%c' in the cpu list entry \"%.*s\": entry is ignored", + *number_start, (int) entry_length, entry_start); + return false; + } + + if (number_end != entry_end) { + /* Partially parsed the entry; ignore unparsed characters and continue with the parsed part */ + cpuinfo_log_warning("ignored invalid characters \"%.*s\" at the end of cpu list entry \"%.*s\"", + (int) (entry_end - number_end), number_start, (int) entry_length, entry_start); + } + + if (last_cpu < first_cpu) { + cpuinfo_log_warning("ignored cpu list entry \"%.*s\": invalid range %"PRIu32"-%"PRIu32, + (int) entry_length, entry_start, first_cpu, last_cpu); + return false; + } + + /* Parsed both parts of the entry; update CPU set */ + #if CPUINFO_LOG_DEBUG_PARSERS + cpuinfo_log_debug("cpulist: call callback with list_start = %"PRIu32", list_end = %"PRIu32, + first_cpu, last_cpu + 1); + #endif + return callback(first_cpu, last_cpu + 1, context); +} + +bool cpuinfo_linux_parse_cpulist(const char* filename, cpuinfo_cpulist_callback callback, void* context) { + bool status = true; + int file = -1; + char buffer[BUFFER_SIZE]; + #if CPUINFO_LOG_DEBUG_PARSERS + cpuinfo_log_debug("parsing cpu list from file %s", filename); + #endif + +#if CPUINFO_MOCK + file = cpuinfo_mock_open(filename, O_RDONLY); +#else + file = open(filename, O_RDONLY); +#endif + if (file == -1) { + cpuinfo_log_info("failed to open %s: %s", filename, strerror(errno)); + status = false; + goto cleanup; + } + + size_t position = 0; + const char* buffer_end = &buffer[BUFFER_SIZE]; + char* data_start = buffer; + ssize_t bytes_read; + do { +#if CPUINFO_MOCK + bytes_read = cpuinfo_mock_read(file, data_start, (size_t) (buffer_end - data_start)); +#else + bytes_read = read(file, data_start, (size_t) (buffer_end - data_start)); +#endif + if (bytes_read < 0) { + cpuinfo_log_info("failed to read file %s at position %zu: %s", filename, position, strerror(errno)); + status = false; + goto cleanup; + } + + position += (size_t) bytes_read; + const char* data_end = data_start + (size_t) bytes_read; + const char* entry_start = buffer; + + if (bytes_read == 0) { + /* No more data in the file: process the remaining text in the buffer as a single entry */ + const char* entry_end = data_end; + const bool entry_status = parse_entry(entry_start, entry_end, callback, context); + status &= entry_status; + } else { + const char* entry_end; + do { + /* Find the end of the entry, as indicated by a comma (',') */ + for (entry_end = entry_start; entry_end != data_end; entry_end++) { + if (*entry_end == ',') { + break; + } + } + + /* + * If we located separator at the end of the entry, parse it. + * Otherwise, there may be more data at the end; read the file once again. + */ + if (entry_end != data_end) { + const bool entry_status = parse_entry(entry_start, entry_end, callback, context); + status &= entry_status; + entry_start = entry_end + 1; + } + } while (entry_end != data_end); + + /* Move remaining partial entry data at the end to the beginning of the buffer */ + const size_t entry_length = (size_t) (entry_end - entry_start); + memmove(buffer, entry_start, entry_length); + data_start = &buffer[entry_length]; + } + } while (bytes_read != 0); + +cleanup: + if (file != -1) { +#if CPUINFO_MOCK + cpuinfo_mock_close(file); +#else + close(file); +#endif + file = -1; + } + return status; +} diff --git a/3rdparty/cpuinfo/src/linux/mockfile.c b/3rdparty/cpuinfo/src/linux/mockfile.c new file mode 100644 index 000000000..138acfeb9 --- /dev/null +++ b/3rdparty/cpuinfo/src/linux/mockfile.c @@ -0,0 +1,105 @@ +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include + +#if !CPUINFO_MOCK + #error This file should be built only in mock mode +#endif + +#include +#include +#include +#include + + +static struct cpuinfo_mock_file* cpuinfo_mock_files = NULL; +static uint32_t cpuinfo_mock_file_count = 0; + + +void CPUINFO_ABI cpuinfo_mock_filesystem(struct cpuinfo_mock_file* files) { + cpuinfo_log_info("filesystem mocking enabled"); + uint32_t file_count = 0; + while (files[file_count].path != NULL) { + /* Indicate that file is not opened */ + files[file_count].offset = SIZE_MAX; + file_count += 1; + } + cpuinfo_mock_files = files; + cpuinfo_mock_file_count = file_count; +} + +int CPUINFO_ABI cpuinfo_mock_open(const char* path, int oflag) { + if (cpuinfo_mock_files == NULL) { + cpuinfo_log_warning("cpuinfo_mock_open called without mock filesystem; redictering to open"); + return open(path, oflag); + } + + for (uint32_t i = 0; i < cpuinfo_mock_file_count; i++) { + if (strcmp(cpuinfo_mock_files[i].path, path) == 0) { + if (oflag != O_RDONLY) { + errno = EACCES; + return -1; + } + if (cpuinfo_mock_files[i].offset != SIZE_MAX) { + errno = ENFILE; + return -1; + } + cpuinfo_mock_files[i].offset = 0; + return (int) i; + } + } + errno = ENOENT; + return -1; +} + +int CPUINFO_ABI cpuinfo_mock_close(int fd) { + if (cpuinfo_mock_files == NULL) { + cpuinfo_log_warning("cpuinfo_mock_close called without mock filesystem; redictering to close"); + return close(fd); + } + + if ((unsigned int) fd >= cpuinfo_mock_file_count) { + errno = EBADF; + return -1; + } + if (cpuinfo_mock_files[fd].offset == SIZE_MAX) { + errno = EBADF; + return -1; + } + cpuinfo_mock_files[fd].offset = SIZE_MAX; + return 0; +} + +ssize_t CPUINFO_ABI cpuinfo_mock_read(int fd, void* buffer, size_t capacity) { + if (cpuinfo_mock_files == NULL) { + cpuinfo_log_warning("cpuinfo_mock_read called without mock filesystem; redictering to read"); + return read(fd, buffer, capacity); + } + + if ((unsigned int) fd >= cpuinfo_mock_file_count) { + errno = EBADF; + return -1; + } + if (cpuinfo_mock_files[fd].offset == SIZE_MAX) { + errno = EBADF; + return -1; + } + + const size_t offset = cpuinfo_mock_files[fd].offset; + size_t count = cpuinfo_mock_files[fd].size - offset; + if (count > capacity) { + count = capacity; + } + memcpy(buffer, (void*) cpuinfo_mock_files[fd].content + offset, count); + cpuinfo_mock_files[fd].offset += count; + return (ssize_t) count; +} diff --git a/3rdparty/cpuinfo/src/linux/multiline.c b/3rdparty/cpuinfo/src/linux/multiline.c new file mode 100644 index 000000000..1feeb9b1c --- /dev/null +++ b/3rdparty/cpuinfo/src/linux/multiline.c @@ -0,0 +1,106 @@ +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include + +#if CPUINFO_MOCK + #include +#endif +#include +#include + + +bool cpuinfo_linux_parse_multiline_file(const char* filename, size_t buffer_size, cpuinfo_line_callback callback, void* context) +{ + int file = -1; + bool status = false; + char* buffer = (char*) alloca(buffer_size); + +#if CPUINFO_MOCK + file = cpuinfo_mock_open(filename, O_RDONLY); +#else + file = open(filename, O_RDONLY); +#endif + if (file == -1) { + cpuinfo_log_info("failed to open %s: %s", filename, strerror(errno)); + goto cleanup; + } + + /* Only used for error reporting */ + size_t position = 0; + uint64_t line_number = 1; + const char* buffer_end = &buffer[buffer_size]; + char* data_start = buffer; + ssize_t bytes_read; + do { +#if CPUINFO_MOCK + bytes_read = cpuinfo_mock_read(file, data_start, (size_t) (buffer_end - data_start)); +#else + bytes_read = read(file, data_start, (size_t) (buffer_end - data_start)); +#endif + if (bytes_read < 0) { + cpuinfo_log_info("failed to read file %s at position %zu: %s", + filename, position, strerror(errno)); + goto cleanup; + } + + position += (size_t) bytes_read; + const char* data_end = data_start + (size_t) bytes_read; + const char* line_start = buffer; + + if (bytes_read == 0) { + /* No more data in the file: process the remaining text in the buffer as a single entry */ + const char* line_end = data_end; + if (!callback(line_start, line_end, context, line_number)) { + goto cleanup; + } + } else { + const char* line_end; + do { + /* Find the end of the entry, as indicated by newline character ('\n') */ + for (line_end = line_start; line_end != data_end; line_end++) { + if (*line_end == '\n') { + break; + } + } + + /* + * If we located separator at the end of the entry, parse it. + * Otherwise, there may be more data at the end; read the file once again. + */ + if (line_end != data_end) { + if (!callback(line_start, line_end, context, line_number++)) { + goto cleanup; + } + line_start = line_end + 1; + } + } while (line_end != data_end); + + /* Move remaining partial line data at the end to the beginning of the buffer */ + const size_t line_length = (size_t) (line_end - line_start); + memmove(buffer, line_start, line_length); + data_start = &buffer[line_length]; + } + } while (bytes_read != 0); + + /* Commit */ + status = true; + +cleanup: + if (file != -1) { +#if CPUINFO_MOCK + cpuinfo_mock_close(file); +#else + close(file); +#endif + file = -1; + } + return status; +} diff --git a/3rdparty/cpuinfo/src/linux/processors.c b/3rdparty/cpuinfo/src/linux/processors.c new file mode 100644 index 000000000..aedba7438 --- /dev/null +++ b/3rdparty/cpuinfo/src/linux/processors.c @@ -0,0 +1,406 @@ +#include +#include +#include +#include +#include + +#if !defined(__ANDROID__) + /* + * sched.h is only used for CPU_SETSIZE constant. + * Android NDK headers before platform 21 do have this constant in sched.h + */ + #include +#endif + +#include +#include + + +#define STRINGIFY(token) #token + +#define KERNEL_MAX_FILENAME "/sys/devices/system/cpu/kernel_max" +#define KERNEL_MAX_FILESIZE 32 +#define FREQUENCY_FILENAME_SIZE (sizeof("/sys/devices/system/cpu/cpu" STRINGIFY(UINT32_MAX) "/cpufreq/cpuinfo_max_freq")) +#define MAX_FREQUENCY_FILENAME_FORMAT "/sys/devices/system/cpu/cpu%" PRIu32 "/cpufreq/cpuinfo_max_freq" +#define MIN_FREQUENCY_FILENAME_FORMAT "/sys/devices/system/cpu/cpu%" PRIu32 "/cpufreq/cpuinfo_min_freq" +#define FREQUENCY_FILESIZE 32 +#define PACKAGE_ID_FILENAME_SIZE (sizeof("/sys/devices/system/cpu/cpu" STRINGIFY(UINT32_MAX) "/topology/physical_package_id")) +#define PACKAGE_ID_FILENAME_FORMAT "/sys/devices/system/cpu/cpu%" PRIu32 "/topology/physical_package_id" +#define PACKAGE_ID_FILESIZE 32 +#define CORE_ID_FILENAME_SIZE (sizeof("/sys/devices/system/cpu/cpu" STRINGIFY(UINT32_MAX) "/topology/core_id")) +#define CORE_ID_FILENAME_FORMAT "/sys/devices/system/cpu/cpu%" PRIu32 "/topology/core_id" +#define CORE_ID_FILESIZE 32 + +#define CORE_SIBLINGS_FILENAME_SIZE (sizeof("/sys/devices/system/cpu/cpu" STRINGIFY(UINT32_MAX) "/topology/core_siblings_list")) +#define CORE_SIBLINGS_FILENAME_FORMAT "/sys/devices/system/cpu/cpu%" PRIu32 "/topology/core_siblings_list" +#define THREAD_SIBLINGS_FILENAME_SIZE (sizeof("/sys/devices/system/cpu/cpu" STRINGIFY(UINT32_MAX) "/topology/thread_siblings_list")) +#define THREAD_SIBLINGS_FILENAME_FORMAT "/sys/devices/system/cpu/cpu%" PRIu32 "/topology/thread_siblings_list" + +#define POSSIBLE_CPULIST_FILENAME "/sys/devices/system/cpu/possible" +#define PRESENT_CPULIST_FILENAME "/sys/devices/system/cpu/present" + + +inline static const char* parse_number(const char* start, const char* end, uint32_t number_ptr[restrict static 1]) { + uint32_t number = 0; + const char* parsed = start; + for (; parsed != end; parsed++) { + const uint32_t digit = (uint32_t) (uint8_t) (*parsed) - (uint32_t) '0'; + if (digit >= 10) { + break; + } + number = number * UINT32_C(10) + digit; + } + *number_ptr = number; + return parsed; +} + +/* Locale-independent */ +inline static bool is_whitespace(char c) { + switch (c) { + case ' ': + case '\t': + case '\n': + case '\r': + return true; + default: + return false; + } +} + +#if defined(__ANDROID__) && !defined(CPU_SETSIZE) + /* + * Android NDK headers before platform 21 do not define CPU_SETSIZE, + * so we hard-code its value, as defined in platform 21 headers + */ + #if defined(__LP64__) + static const uint32_t default_max_processors_count = 1024; + #else + static const uint32_t default_max_processors_count = 32; + #endif +#else + static const uint32_t default_max_processors_count = CPU_SETSIZE; +#endif + +static bool uint32_parser(const char* text_start, const char* text_end, void* context) { + if (text_start == text_end) { + cpuinfo_log_error("failed to parse file %s: file is empty", KERNEL_MAX_FILENAME); + return false; + } + + uint32_t kernel_max = 0; + const char* parsed_end = parse_number(text_start, text_end, &kernel_max); + if (parsed_end == text_start) { + cpuinfo_log_error("failed to parse file %s: \"%.*s\" is not an unsigned number", + KERNEL_MAX_FILENAME, (int) (text_end - text_start), text_start); + return false; + } else { + for (const char* char_ptr = parsed_end; char_ptr != text_end; char_ptr++) { + if (!is_whitespace(*char_ptr)) { + cpuinfo_log_warning("non-whitespace characters \"%.*s\" following number in file %s are ignored", + (int) (text_end - char_ptr), char_ptr, KERNEL_MAX_FILENAME); + break; + } + } + } + + uint32_t* kernel_max_ptr = (uint32_t*) context; + *kernel_max_ptr = kernel_max; + return true; +} + +uint32_t cpuinfo_linux_get_max_processors_count(void) { + uint32_t kernel_max; + if (cpuinfo_linux_parse_small_file(KERNEL_MAX_FILENAME, KERNEL_MAX_FILESIZE, uint32_parser, &kernel_max)) { + cpuinfo_log_debug("parsed kernel_max value of %"PRIu32" from %s", kernel_max, KERNEL_MAX_FILENAME); + + if (kernel_max >= default_max_processors_count) { + cpuinfo_log_warning("kernel_max value of %"PRIu32" parsed from %s exceeds platform-default limit %"PRIu32, + kernel_max, KERNEL_MAX_FILENAME, default_max_processors_count - 1); + } + + return kernel_max + 1; + } else { + cpuinfo_log_warning("using platform-default max processors count = %"PRIu32, default_max_processors_count); + return default_max_processors_count; + } +} + +uint32_t cpuinfo_linux_get_processor_max_frequency(uint32_t processor) { + char max_frequency_filename[FREQUENCY_FILENAME_SIZE]; + const int chars_formatted = snprintf( + max_frequency_filename, FREQUENCY_FILENAME_SIZE, MAX_FREQUENCY_FILENAME_FORMAT, processor); + if ((unsigned int) chars_formatted >= FREQUENCY_FILENAME_SIZE) { + cpuinfo_log_warning("failed to format filename for max frequency of processor %"PRIu32, processor); + return 0; + } + + uint32_t max_frequency; + if (cpuinfo_linux_parse_small_file(max_frequency_filename, FREQUENCY_FILESIZE, uint32_parser, &max_frequency)) { + cpuinfo_log_debug("parsed max frequency value of %"PRIu32" KHz for logical processor %"PRIu32" from %s", + max_frequency, processor, max_frequency_filename); + return max_frequency; + } else { + cpuinfo_log_warning("failed to parse max frequency for processor %"PRIu32" from %s", + processor, max_frequency_filename); + return 0; + } +} + +uint32_t cpuinfo_linux_get_processor_min_frequency(uint32_t processor) { + char min_frequency_filename[FREQUENCY_FILENAME_SIZE]; + const int chars_formatted = snprintf( + min_frequency_filename, FREQUENCY_FILENAME_SIZE, MIN_FREQUENCY_FILENAME_FORMAT, processor); + if ((unsigned int) chars_formatted >= FREQUENCY_FILENAME_SIZE) { + cpuinfo_log_warning("failed to format filename for min frequency of processor %"PRIu32, processor); + return 0; + } + + uint32_t min_frequency; + if (cpuinfo_linux_parse_small_file(min_frequency_filename, FREQUENCY_FILESIZE, uint32_parser, &min_frequency)) { + cpuinfo_log_debug("parsed min frequency value of %"PRIu32" KHz for logical processor %"PRIu32" from %s", + min_frequency, processor, min_frequency_filename); + return min_frequency; + } else { + /* + * This error is less severe than parsing max frequency, because min frequency is only useful for clustering, + * while max frequency is also needed for peak FLOPS calculation. + */ + cpuinfo_log_info("failed to parse min frequency for processor %"PRIu32" from %s", + processor, min_frequency_filename); + return 0; + } +} + +bool cpuinfo_linux_get_processor_core_id(uint32_t processor, uint32_t core_id_ptr[restrict static 1]) { + char core_id_filename[PACKAGE_ID_FILENAME_SIZE]; + const int chars_formatted = snprintf( + core_id_filename, CORE_ID_FILENAME_SIZE, CORE_ID_FILENAME_FORMAT, processor); + if ((unsigned int) chars_formatted >= CORE_ID_FILENAME_SIZE) { + cpuinfo_log_warning("failed to format filename for core id of processor %"PRIu32, processor); + return 0; + } + + uint32_t core_id; + if (cpuinfo_linux_parse_small_file(core_id_filename, CORE_ID_FILESIZE, uint32_parser, &core_id)) { + cpuinfo_log_debug("parsed core id value of %"PRIu32" for logical processor %"PRIu32" from %s", + core_id, processor, core_id_filename); + *core_id_ptr = core_id; + return true; + } else { + cpuinfo_log_info("failed to parse core id for processor %"PRIu32" from %s", + processor, core_id_filename); + return false; + } +} + +bool cpuinfo_linux_get_processor_package_id(uint32_t processor, uint32_t package_id_ptr[restrict static 1]) { + char package_id_filename[PACKAGE_ID_FILENAME_SIZE]; + const int chars_formatted = snprintf( + package_id_filename, PACKAGE_ID_FILENAME_SIZE, PACKAGE_ID_FILENAME_FORMAT, processor); + if ((unsigned int) chars_formatted >= PACKAGE_ID_FILENAME_SIZE) { + cpuinfo_log_warning("failed to format filename for package id of processor %"PRIu32, processor); + return 0; + } + + uint32_t package_id; + if (cpuinfo_linux_parse_small_file(package_id_filename, PACKAGE_ID_FILESIZE, uint32_parser, &package_id)) { + cpuinfo_log_debug("parsed package id value of %"PRIu32" for logical processor %"PRIu32" from %s", + package_id, processor, package_id_filename); + *package_id_ptr = package_id; + return true; + } else { + cpuinfo_log_info("failed to parse package id for processor %"PRIu32" from %s", + processor, package_id_filename); + return false; + } +} + +static bool max_processor_number_parser(uint32_t processor_list_start, uint32_t processor_list_end, void* context) { + uint32_t* processor_number_ptr = (uint32_t*) context; + const uint32_t processor_list_last = processor_list_end - 1; + if (*processor_number_ptr < processor_list_last) { + *processor_number_ptr = processor_list_last; + } + return true; +} + +uint32_t cpuinfo_linux_get_max_possible_processor(uint32_t max_processors_count) { + uint32_t max_possible_processor = 0; + if (!cpuinfo_linux_parse_cpulist(POSSIBLE_CPULIST_FILENAME, max_processor_number_parser, &max_possible_processor)) { + #if CPUINFO_ARCH_ARM || CPUINFO_ARCH_ARM64 + cpuinfo_log_error("failed to parse the list of possible processors in %s", POSSIBLE_CPULIST_FILENAME); + #else + cpuinfo_log_warning("failed to parse the list of possible processors in %s", POSSIBLE_CPULIST_FILENAME); + #endif + return UINT32_MAX; + } + if (max_possible_processor >= max_processors_count) { + cpuinfo_log_warning( + "maximum possible processor number %"PRIu32" exceeds system limit %"PRIu32": truncating to the latter", + max_possible_processor, max_processors_count - 1); + max_possible_processor = max_processors_count - 1; + } + return max_possible_processor; +} + +uint32_t cpuinfo_linux_get_max_present_processor(uint32_t max_processors_count) { + uint32_t max_present_processor = 0; + if (!cpuinfo_linux_parse_cpulist(PRESENT_CPULIST_FILENAME, max_processor_number_parser, &max_present_processor)) { + #if CPUINFO_ARCH_ARM || CPUINFO_ARCH_ARM64 + cpuinfo_log_error("failed to parse the list of present processors in %s", PRESENT_CPULIST_FILENAME); + #else + cpuinfo_log_warning("failed to parse the list of present processors in %s", PRESENT_CPULIST_FILENAME); + #endif + return UINT32_MAX; + } + if (max_present_processor >= max_processors_count) { + cpuinfo_log_warning( + "maximum present processor number %"PRIu32" exceeds system limit %"PRIu32": truncating to the latter", + max_present_processor, max_processors_count - 1); + max_present_processor = max_processors_count - 1; + } + return max_present_processor; +} + +struct detect_processors_context { + uint32_t max_processors_count; + uint32_t* processor0_flags; + uint32_t processor_struct_size; + uint32_t detected_flag; +}; + +static bool detect_processor_parser(uint32_t processor_list_start, uint32_t processor_list_end, void* context) { + const uint32_t max_processors_count = ((struct detect_processors_context*) context)->max_processors_count; + const uint32_t* processor0_flags = ((struct detect_processors_context*) context)->processor0_flags; + const uint32_t processor_struct_size = ((struct detect_processors_context*) context)->processor_struct_size; + const uint32_t detected_flag = ((struct detect_processors_context*) context)->detected_flag; + + for (uint32_t processor = processor_list_start; processor < processor_list_end; processor++) { + if (processor >= max_processors_count) { + break; + } + *((uint32_t*) ((uintptr_t) processor0_flags + processor_struct_size * processor)) |= detected_flag; + } + return true; +} + +bool cpuinfo_linux_detect_possible_processors(uint32_t max_processors_count, + uint32_t* processor0_flags, uint32_t processor_struct_size, uint32_t possible_flag) +{ + struct detect_processors_context context = { + .max_processors_count = max_processors_count, + .processor0_flags = processor0_flags, + .processor_struct_size = processor_struct_size, + .detected_flag = possible_flag, + }; + if (cpuinfo_linux_parse_cpulist(POSSIBLE_CPULIST_FILENAME, detect_processor_parser, &context)) { + return true; + } else { + cpuinfo_log_warning("failed to parse the list of possible processors in %s", POSSIBLE_CPULIST_FILENAME); + return false; + } +} + +bool cpuinfo_linux_detect_present_processors(uint32_t max_processors_count, + uint32_t* processor0_flags, uint32_t processor_struct_size, uint32_t present_flag) +{ + struct detect_processors_context context = { + .max_processors_count = max_processors_count, + .processor0_flags = processor0_flags, + .processor_struct_size = processor_struct_size, + .detected_flag = present_flag, + }; + if (cpuinfo_linux_parse_cpulist(PRESENT_CPULIST_FILENAME, detect_processor_parser, &context)) { + return true; + } else { + cpuinfo_log_warning("failed to parse the list of present processors in %s", PRESENT_CPULIST_FILENAME); + return false; + } +} + +struct siblings_context { + const char* group_name; + uint32_t max_processors_count; + uint32_t processor; + cpuinfo_siblings_callback callback; + void* callback_context; +}; + +static bool siblings_parser(uint32_t sibling_list_start, uint32_t sibling_list_end, struct siblings_context* context) { + const char* group_name = context->group_name; + const uint32_t max_processors_count = context->max_processors_count; + const uint32_t processor = context->processor; + + if (sibling_list_end > max_processors_count) { + cpuinfo_log_warning("ignore %s siblings %"PRIu32"-%"PRIu32" of processor %"PRIu32, + group_name, max_processors_count, sibling_list_end - 1, processor); + sibling_list_end = max_processors_count; + } + + return context->callback(processor, sibling_list_start, sibling_list_end, context->callback_context); +} + +bool cpuinfo_linux_detect_core_siblings( + uint32_t max_processors_count, + uint32_t processor, + cpuinfo_siblings_callback callback, + void* context) +{ + char core_siblings_filename[CORE_SIBLINGS_FILENAME_SIZE]; + const int chars_formatted = snprintf( + core_siblings_filename, CORE_SIBLINGS_FILENAME_SIZE, CORE_SIBLINGS_FILENAME_FORMAT, processor); + if ((unsigned int) chars_formatted >= CORE_SIBLINGS_FILENAME_SIZE) { + cpuinfo_log_warning("failed to format filename for core siblings of processor %"PRIu32, processor); + return false; + } + + struct siblings_context siblings_context = { + .group_name = "package", + .max_processors_count = max_processors_count, + .processor = processor, + .callback = callback, + .callback_context = context, + }; + if (cpuinfo_linux_parse_cpulist(core_siblings_filename, + (cpuinfo_cpulist_callback) siblings_parser, &siblings_context)) + { + return true; + } else { + cpuinfo_log_info("failed to parse the list of core siblings for processor %"PRIu32" from %s", + processor, core_siblings_filename); + return false; + } +} + +bool cpuinfo_linux_detect_thread_siblings( + uint32_t max_processors_count, + uint32_t processor, + cpuinfo_siblings_callback callback, + void* context) +{ + char thread_siblings_filename[THREAD_SIBLINGS_FILENAME_SIZE]; + const int chars_formatted = snprintf( + thread_siblings_filename, THREAD_SIBLINGS_FILENAME_SIZE, THREAD_SIBLINGS_FILENAME_FORMAT, processor); + if ((unsigned int) chars_formatted >= THREAD_SIBLINGS_FILENAME_SIZE) { + cpuinfo_log_warning("failed to format filename for thread siblings of processor %"PRIu32, processor); + return false; + } + + struct siblings_context siblings_context = { + .group_name = "core", + .max_processors_count = max_processors_count, + .processor = processor, + .callback = callback, + .callback_context = context, + }; + if (cpuinfo_linux_parse_cpulist(thread_siblings_filename, + (cpuinfo_cpulist_callback) siblings_parser, &siblings_context)) + { + return true; + } else { + cpuinfo_log_info("failed to parse the list of thread siblings for processor %"PRIu32" from %s", + processor, thread_siblings_filename); + return false; + } +} + diff --git a/3rdparty/cpuinfo/src/linux/smallfile.c b/3rdparty/cpuinfo/src/linux/smallfile.c new file mode 100644 index 000000000..98cde00e5 --- /dev/null +++ b/3rdparty/cpuinfo/src/linux/smallfile.c @@ -0,0 +1,70 @@ +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include + +#if CPUINFO_MOCK + #include +#endif +#include +#include + + +bool cpuinfo_linux_parse_small_file(const char* filename, size_t buffer_size, cpuinfo_smallfile_callback callback, void* context) { + int file = -1; + bool status = false; + char* buffer = (char*) alloca(buffer_size); + + #if CPUINFO_LOG_DEBUG_PARSERS + cpuinfo_log_debug("parsing small file %s", filename); + #endif + +#if CPUINFO_MOCK + file = cpuinfo_mock_open(filename, O_RDONLY); +#else + file = open(filename, O_RDONLY); +#endif + if (file == -1) { + cpuinfo_log_info("failed to open %s: %s", filename, strerror(errno)); + goto cleanup; + } + + size_t buffer_position = 0; + ssize_t bytes_read; + do { +#if CPUINFO_MOCK + bytes_read = cpuinfo_mock_read(file, &buffer[buffer_position], buffer_size - buffer_position); +#else + bytes_read = read(file, &buffer[buffer_position], buffer_size - buffer_position); +#endif + if (bytes_read < 0) { + cpuinfo_log_info("failed to read file %s at position %zu: %s", filename, buffer_position, strerror(errno)); + goto cleanup; + } + buffer_position += (size_t) bytes_read; + if (buffer_position >= buffer_size) { + cpuinfo_log_error("failed to read file %s: insufficient buffer of size %zu", filename, buffer_size); + goto cleanup; + } + } while (bytes_read != 0); + + status = callback(buffer, &buffer[buffer_position], context); + +cleanup: + if (file != -1) { +#if CPUINFO_MOCK + cpuinfo_mock_close(file); +#else + close(file); +#endif + file = -1; + } + return status; +} diff --git a/3rdparty/cpuinfo/src/mach/api.h b/3rdparty/cpuinfo/src/mach/api.h new file mode 100644 index 000000000..fdef5bdff --- /dev/null +++ b/3rdparty/cpuinfo/src/mach/api.h @@ -0,0 +1,16 @@ +#pragma once + +#include + +#define CPUINFO_MACH_MAX_CACHE_LEVELS 8 + + +struct cpuinfo_mach_topology { + uint32_t packages; + uint32_t cores; + uint32_t threads; + uint32_t threads_per_cache[CPUINFO_MACH_MAX_CACHE_LEVELS]; +}; + + +struct cpuinfo_mach_topology cpuinfo_mach_detect_topology(void); diff --git a/3rdparty/cpuinfo/src/mach/topology.c b/3rdparty/cpuinfo/src/mach/topology.c new file mode 100644 index 000000000..b56343bb2 --- /dev/null +++ b/3rdparty/cpuinfo/src/mach/topology.c @@ -0,0 +1,73 @@ +#include +#include +#include + +#include +#include + +#include +#include + +#include + + +struct cpuinfo_mach_topology cpuinfo_mach_detect_topology(void) { + int cores = 1; + size_t sizeof_cores = sizeof(cores); + if (sysctlbyname("hw.physicalcpu_max", &cores, &sizeof_cores, NULL, 0) != 0) { + cpuinfo_log_error("sysctlbyname(\"hw.physicalcpu_max\") failed: %s", strerror(errno)); + } else if (cores <= 0) { + cpuinfo_log_error("sysctlbyname(\"hw.physicalcpu_max\") returned invalid value %d", cores); + cores = 1; + } + + int threads = 1; + size_t sizeof_threads = sizeof(threads); + if (sysctlbyname("hw.logicalcpu_max", &threads, &sizeof_threads, NULL, 0) != 0) { + cpuinfo_log_error("sysctlbyname(\"hw.logicalcpu_max\") failed: %s", strerror(errno)); + } else if (threads <= 0) { + cpuinfo_log_error("sysctlbyname(\"hw.logicalcpu_max\") returned invalid value %d", threads); + threads = cores; + } + + int packages = 1; +#if !TARGET_OS_IPHONE + size_t sizeof_packages = sizeof(packages); + if (sysctlbyname("hw.packages", &packages, &sizeof_packages, NULL, 0) != 0) { + cpuinfo_log_error("sysctlbyname(\"hw.packages\") failed: %s", strerror(errno)); + } else if (packages <= 0) { + cpuinfo_log_error("sysctlbyname(\"hw.packages\") returned invalid value %d", packages); + packages = 1; + } +#endif + + cpuinfo_log_debug("mach topology: packages = %d, cores = %d, threads = %d", packages, (int) cores, (int) threads); + struct cpuinfo_mach_topology topology = { + .packages = (uint32_t) packages, + .cores = (uint32_t) cores, + .threads = (uint32_t) threads + }; + +#if !TARGET_OS_IPHONE + size_t cacheconfig_size = 0; + if (sysctlbyname("hw.cacheconfig", NULL, &cacheconfig_size, NULL, 0) != 0) { + cpuinfo_log_error("sysctlbyname(\"hw.cacheconfig\") failed: %s", strerror(errno)); + } else { + uint64_t* cacheconfig = alloca(cacheconfig_size); + if (sysctlbyname("hw.cacheconfig", cacheconfig, &cacheconfig_size, NULL, 0) != 0) { + cpuinfo_log_error("sysctlbyname(\"hw.cacheconfig\") failed: %s", strerror(errno)); + } else { + size_t cache_configs = cacheconfig_size / sizeof(uint64_t); + cpuinfo_log_debug("mach hw.cacheconfig count: %zu", cache_configs); + if (cache_configs > CPUINFO_MACH_MAX_CACHE_LEVELS) { + cache_configs = CPUINFO_MACH_MAX_CACHE_LEVELS; + } + for (size_t i = 0; i < cache_configs; i++) { + cpuinfo_log_debug("mach hw.cacheconfig[%zu]: %"PRIu64, i, cacheconfig[i]); + topology.threads_per_cache[i] = cacheconfig[i]; + } + } + } +#endif + return topology; +} diff --git a/3rdparty/cpuinfo/src/x86/api.h b/3rdparty/cpuinfo/src/x86/api.h new file mode 100644 index 000000000..213c2d881 --- /dev/null +++ b/3rdparty/cpuinfo/src/x86/api.h @@ -0,0 +1,159 @@ +#pragma once + +#include +#include + +#include +#include + + +struct cpuid_regs { + uint32_t eax; + uint32_t ebx; + uint32_t ecx; + uint32_t edx; +}; + +struct cpuinfo_x86_cache { + uint32_t size; + uint32_t associativity; + uint32_t sets; + uint32_t partitions; + uint32_t line_size; + uint32_t flags; + uint32_t apic_bits; +}; + +struct cpuinfo_x86_caches { + struct cpuinfo_trace_cache trace; + struct cpuinfo_x86_cache l1i; + struct cpuinfo_x86_cache l1d; + struct cpuinfo_x86_cache l2; + struct cpuinfo_x86_cache l3; + struct cpuinfo_x86_cache l4; + uint32_t prefetch_size; +}; + +struct cpuinfo_x86_model_info { + uint32_t model; + uint32_t family; + + uint32_t base_model; + uint32_t base_family; + uint32_t stepping; + uint32_t extended_model; + uint32_t extended_family; + uint32_t processor_type; +}; + +struct cpuinfo_x86_topology { + uint32_t apic_id; + uint32_t thread_bits_offset; + uint32_t thread_bits_length; + uint32_t core_bits_offset; + uint32_t core_bits_length; +}; + +struct cpuinfo_x86_processor { + uint32_t cpuid; + enum cpuinfo_vendor vendor; + enum cpuinfo_uarch uarch; +#ifdef __linux__ + int linux_id; +#endif + struct cpuinfo_x86_caches cache; + struct { + struct cpuinfo_tlb itlb_4KB; + struct cpuinfo_tlb itlb_2MB; + struct cpuinfo_tlb itlb_4MB; + struct cpuinfo_tlb dtlb0_4KB; + struct cpuinfo_tlb dtlb0_2MB; + struct cpuinfo_tlb dtlb0_4MB; + struct cpuinfo_tlb dtlb_4KB; + struct cpuinfo_tlb dtlb_2MB; + struct cpuinfo_tlb dtlb_4MB; + struct cpuinfo_tlb dtlb_1GB; + struct cpuinfo_tlb stlb2_4KB; + struct cpuinfo_tlb stlb2_2MB; + struct cpuinfo_tlb stlb2_1GB; + } tlb; + struct cpuinfo_x86_topology topology; + char brand_string[CPUINFO_PACKAGE_NAME_MAX]; +}; + +CPUINFO_INTERNAL void cpuinfo_x86_init_processor(struct cpuinfo_x86_processor* processor); + +CPUINFO_INTERNAL enum cpuinfo_vendor cpuinfo_x86_decode_vendor(uint32_t ebx, uint32_t ecx, uint32_t edx); +CPUINFO_INTERNAL struct cpuinfo_x86_model_info cpuinfo_x86_decode_model_info(uint32_t eax); +CPUINFO_INTERNAL enum cpuinfo_uarch cpuinfo_x86_decode_uarch( + enum cpuinfo_vendor vendor, + const struct cpuinfo_x86_model_info* model_info); + +CPUINFO_INTERNAL struct cpuinfo_x86_isa cpuinfo_x86_detect_isa( + const struct cpuid_regs basic_info, const struct cpuid_regs extended_info, + uint32_t max_base_index, uint32_t max_extended_index, + enum cpuinfo_vendor vendor, enum cpuinfo_uarch uarch); + +CPUINFO_INTERNAL void cpuinfo_x86_detect_topology( + uint32_t max_base_index, + uint32_t max_extended_index, + struct cpuid_regs leaf1, + struct cpuinfo_x86_topology* topology); + +CPUINFO_INTERNAL void cpuinfo_x86_detect_cache( + uint32_t max_base_index, uint32_t max_extended_index, + bool amd_topology_extensions, + enum cpuinfo_vendor vendor, + const struct cpuinfo_x86_model_info* model_info, + struct cpuinfo_x86_caches* cache, + struct cpuinfo_tlb* itlb_4KB, + struct cpuinfo_tlb* itlb_2MB, + struct cpuinfo_tlb* itlb_4MB, + struct cpuinfo_tlb* dtlb0_4KB, + struct cpuinfo_tlb* dtlb0_2MB, + struct cpuinfo_tlb* dtlb0_4MB, + struct cpuinfo_tlb* dtlb_4KB, + struct cpuinfo_tlb* dtlb_2MB, + struct cpuinfo_tlb* dtlb_4MB, + struct cpuinfo_tlb* dtlb_1GB, + struct cpuinfo_tlb* stlb2_4KB, + struct cpuinfo_tlb* stlb2_2MB, + struct cpuinfo_tlb* stlb2_1GB, + uint32_t* log2_package_cores_max); + +CPUINFO_INTERNAL void cpuinfo_x86_decode_cache_descriptor( + uint8_t descriptor, enum cpuinfo_vendor vendor, + const struct cpuinfo_x86_model_info* model_info, + struct cpuinfo_x86_caches* cache, + struct cpuinfo_tlb* itlb_4KB, + struct cpuinfo_tlb* itlb_2MB, + struct cpuinfo_tlb* itlb_4MB, + struct cpuinfo_tlb* dtlb0_4KB, + struct cpuinfo_tlb* dtlb0_2MB, + struct cpuinfo_tlb* dtlb0_4MB, + struct cpuinfo_tlb* dtlb_4KB, + struct cpuinfo_tlb* dtlb_2MB, + struct cpuinfo_tlb* dtlb_4MB, + struct cpuinfo_tlb* dtlb_1GB, + struct cpuinfo_tlb* stlb2_4KB, + struct cpuinfo_tlb* stlb2_2MB, + struct cpuinfo_tlb* stlb2_1GB, + uint32_t* prefetch_size); + +CPUINFO_INTERNAL bool cpuinfo_x86_decode_deterministic_cache_parameters( + struct cpuid_regs regs, + struct cpuinfo_x86_caches* cache, + uint32_t* package_cores_max); + +CPUINFO_INTERNAL bool cpuinfo_x86_decode_cache_properties( + struct cpuid_regs regs, + struct cpuinfo_x86_caches* cache); + +CPUINFO_INTERNAL uint32_t cpuinfo_x86_normalize_brand_string( + const char raw_name[48], + char normalized_name[48]); + +CPUINFO_INTERNAL uint32_t cpuinfo_x86_format_package_name( + enum cpuinfo_vendor vendor, + const char normalized_brand_string[48], + char package_name[CPUINFO_PACKAGE_NAME_MAX]); diff --git a/3rdparty/cpuinfo/src/x86/cache/descriptor.c b/3rdparty/cpuinfo/src/x86/cache/descriptor.c new file mode 100644 index 000000000..69d38cc39 --- /dev/null +++ b/3rdparty/cpuinfo/src/x86/cache/descriptor.c @@ -0,0 +1,1726 @@ +#include + +#include +#include + + +void cpuinfo_x86_decode_cache_descriptor( + uint8_t descriptor, enum cpuinfo_vendor vendor, + const struct cpuinfo_x86_model_info* model_info, + struct cpuinfo_x86_caches* cache, + struct cpuinfo_tlb* itlb_4KB, + struct cpuinfo_tlb* itlb_2MB, + struct cpuinfo_tlb* itlb_4MB, + struct cpuinfo_tlb* dtlb0_4KB, + struct cpuinfo_tlb* dtlb0_2MB, + struct cpuinfo_tlb* dtlb0_4MB, + struct cpuinfo_tlb* dtlb_4KB, + struct cpuinfo_tlb* dtlb_2MB, + struct cpuinfo_tlb* dtlb_4MB, + struct cpuinfo_tlb* dtlb_1GB, + struct cpuinfo_tlb* stlb2_4KB, + struct cpuinfo_tlb* stlb2_2MB, + struct cpuinfo_tlb* stlb2_1GB, + uint32_t* prefetch_size) +{ + /* + * Descriptors are parsed according to: + * - Application Note 485: Intel Processor Indentification and CPUID Instruction, May 2012, Order Number 241618-039 + * - Intel 64 and IA-32 Architectures Software Developer’s Manual, Volume 2 (2A, 2B, 2C & 2D): Instruction Set + * Reference, A-Z, December 2016. Order Number: 325383-061US + * - Cyrix CPU Detection Guide, Preliminary Revision 1.01 + * - Geode(TM) GX1 Processor Series: Low Power Integrated x86 Solution + */ + switch (descriptor) { + case 0x01: + /* + * Intel ISA Reference: + * "Instruction TLB: 4 KByte pages, 4-way set associative, 32 entries" + * Application Note 485: + * "Instruction TLB: 4-KB Pages, 4-way set associative, 32 entries" + */ + *itlb_4KB = (struct cpuinfo_tlb) { + .entries = 32, + .associativity = 4, + .pages = CPUINFO_PAGE_SIZE_4KB, + }; + break; + case 0x02: + /* + * Intel ISA Reference: + * "Instruction TLB: 4 MByte pages, fully associative, 2 entries" + * Application Note 485: + * "Instruction TLB: 4-MB Pages, fully associative, 2 entries" + */ + *itlb_4MB = (struct cpuinfo_tlb) { + .entries = 2, + .associativity = 2, + .pages = CPUINFO_PAGE_SIZE_4MB, + }; + break; + case 0x03: + /* + * Intel ISA Reference: + * "Data TLB: 4 KByte pages, 4-way set associative, 64 entries" + * Application Note 485: + * "Data TLB: 4-KB Pages, 4-way set associative, 64 entries" + */ + *dtlb_4KB = (struct cpuinfo_tlb) { + .entries = 64, + .associativity = 4, + .pages = CPUINFO_PAGE_SIZE_4KB, + }; + break; + case 0x04: + /* + * Intel ISA Reference: + * "Data TLB: 4 MByte pages, 4-way set associative, 8 entries" + * Application Note 485: + * "Data TLB: 4-MB Pages, 4-way set associative, 8 entries" + */ + *dtlb_4MB = (struct cpuinfo_tlb) { + .entries = 8, + .associativity = 4, + .pages = CPUINFO_PAGE_SIZE_4MB, + }; + break; + case 0x05: + /* + * Intel ISA Reference: + * "Data TLB1: 4 MByte pages, 4-way set associative, 32 entries" + * Application Note 485: + * "Data TLB: 4-MB Pages, 4-way set associative, 32 entries" + */ + *dtlb_4MB = (struct cpuinfo_tlb) { + .entries = 32, + .associativity = 4, + .pages = CPUINFO_PAGE_SIZE_4MB, + }; + break; + case 0x06: + /* + * Intel ISA Reference: + * "1st-level instruction cache: 8 KBytes, 4-way set associative, 32 byte line size" + * Application Note 485: + * "1st-level instruction cache: 8-KB, 4-way set associative, 32-byte line size" + */ + cache->l1i = (struct cpuinfo_x86_cache) { + .size = 8 * 1024, + .associativity = 4, + .sets = 64, + .partitions = 1, + .line_size = 32, + }; + break; + case 0x08: + /* + * Intel ISA Reference: + * "1st-level instruction cache: 16 KBytes, 4-way set associative, 32 byte line size" + * Application Note 485: + * "1st-level instruction cache: 16-KB, 4-way set associative, 32-byte line size" + */ + cache->l1i = (struct cpuinfo_x86_cache) { + .size = 16 * 1024, + .associativity = 4, + .sets = 128, + .partitions = 1, + .line_size = 32, + }; + break; + case 0x09: + /* + * Intel ISA Reference: + * "1st-level instruction cache: 32KBytes, 4-way set associative, 64 byte line size" + * Application Note 485: + * "1st-level Instruction Cache: 32-KB, 4-way set associative, 64-byte line size" + */ + cache->l1i = (struct cpuinfo_x86_cache) { + .size = 32 * 1024, + .associativity = 4, + .sets = 128, + .partitions = 1, + .line_size = 64, + }; + break; + case 0x0A: + /* + * Intel ISA Reference: + * "1st-level data cache: 8 KBytes, 2-way set associative, 32 byte line size" + * Application Note 485: + * "1st-level data cache: 8-KB, 2-way set associative, 32-byte line size" + */ + cache->l1d = (struct cpuinfo_x86_cache) { + .size = 8 * 1024, + .associativity = 2, + .sets = 128, + .partitions = 1, + .line_size = 32, + }; + break; + case 0x0B: + /* + * Intel ISA Reference: + * "Instruction TLB: 4 MByte pages, 4-way set associative, 4 entries" + * Application Note 485: + * "Instruction TLB: 4-MB pages, 4-way set associative, 4 entries" + */ + *itlb_4MB = (struct cpuinfo_tlb) { + .entries = 4, + .associativity = 4, + .pages = CPUINFO_PAGE_SIZE_4MB, + }; + break; + case 0x0C: + /* + * Intel ISA Reference: + * "1st-level data cache: 16 KBytes, 4-way set associative, 32 byte line size" + * Application Note 485: + * "1st-level data cache: 16-KB, 4-way set associative, 32-byte line size" + */ + cache->l1d = (struct cpuinfo_x86_cache) { + .size = 16 * 1024, + .associativity = 4, + .sets = 128, + .partitions = 1, + .line_size = 32, + }; + break; + case 0x0D: + /* + * Intel ISA Reference: + * "1st-level data cache: 16 KBytes, 4-way set associative, 64 byte line size" + * Application Note 485: + * "1st-level Data Cache: 16-KB, 4-way set associative, 64-byte line size" + */ + cache->l1d = (struct cpuinfo_x86_cache) { + .size = 16 * 1024, + .associativity = 4, + .sets = 64, + .partitions = 1, + .line_size = 64, + }; + break; + case 0x0E: + /* + * Intel ISA Reference: + * "1st-level data cache: 24 KBytes, 6-way set associative, 64 byte line size" + * Application Note 485: + * "1st-level Data Cache: 24-KB, 6-way set associative, 64-byte line size" + */ + cache->l1d = (struct cpuinfo_x86_cache) { + .size = 24 * 1024, + .associativity = 6, + .sets = 64, + .partitions = 1, + .line_size = 64, + }; + break; + case 0x1D: + /* + * Intel ISA Reference: + * "2nd-level cache: 128 KBytes, 2-way set associative, 64 byte line size" + */ + cache->l2 = (struct cpuinfo_x86_cache) { + .size = 128 * 1024, + .associativity = 2, + .sets = 1024, + .partitions = 1, + .line_size = 64, + .flags = CPUINFO_CACHE_INCLUSIVE, + }; + case 0x21: + /* + * Intel ISA Reference: + * "2nd-level cache: 256 KBytes, 8-way set associative, 64 byte line size" + * Application Note 485: + * "2nd-level cache: 256-KB, 8-way set associative, 64-byte line size" + */ + cache->l2 = (struct cpuinfo_x86_cache) { + .size = 256 * 1024, + .associativity = 8, + .sets = 512, + .partitions = 1, + .line_size = 64, + .flags = CPUINFO_CACHE_INCLUSIVE, + }; + break; + case 0x22: + /* + * Intel ISA Reference: + * "3rd-level cache: 512 KBytes, 4-way set associative, 64 byte line size, 2 lines per sector" + * Application Note 485: + * "3rd-level cache: 512-KB, 4-way set associative, sectored cache, 64-byte line size" + */ + cache->l3 = (struct cpuinfo_x86_cache) { + .size = 512 * 1024, + .associativity = 4, + .sets = 2048, + .partitions = 1, + .line_size = 64, + .flags = CPUINFO_CACHE_INCLUSIVE, + }; + break; + case 0x23: + /* + * Intel ISA Reference: + * "3rd-level cache: 1 MBytes, 8-way set associative, 64 byte line size, 2 lines per sector" + * Application Note 485: + * "3rd-level cache: 1-MB, 8-way set associative, sectored cache, 64-byte line size" + */ + cache->l3 = (struct cpuinfo_x86_cache) { + .size = 1024 * 1024, + .associativity = 8, + .sets = 2048, + .partitions = 1, + .line_size = 64, + .flags = CPUINFO_CACHE_INCLUSIVE, + }; + break; + case 0x24: + /* + * Intel ISA Reference: + * "2nd-level cache: 1 MBytes, 16-way set associative, 64 byte line size" + */ + cache->l2 = (struct cpuinfo_x86_cache) { + .size = 1024 * 1024, + .associativity = 16, + .sets = 1024, + .partitions = 1, + .line_size = 64, + .flags = CPUINFO_CACHE_INCLUSIVE, + }; + break; + case 0x25: + /* + * Intel ISA Reference: + * "3rd-level cache: 2 MBytes, 8-way set associative, 64 byte line size, 2 lines per sector" + * Application Note 485: + * "3rd-level cache: 2-MB, 8-way set associative, sectored cache, 64-byte line size" + */ + cache->l3 = (struct cpuinfo_x86_cache) { + .size = 2 * 1024 * 1024, + .associativity = 8, + .sets = 4096, + .partitions = 1, + .line_size = 64, + .flags = CPUINFO_CACHE_INCLUSIVE, + }; + break; + case 0x29: + /* + * Intel ISA Reference: + * "3rd-level cache: 4 MBytes, 8-way set associative, 64 byte line size, 2 lines per sector" + * Application Note 485: + * "3rd-level cache: 4-MB, 8-way set associative, sectored cache, 64-byte line size" + */ + cache->l3 = (struct cpuinfo_x86_cache) { + .size = 4 * 1024 * 1024, + .associativity = 8, + .sets = 8192, + .partitions = 1, + .line_size = 64, + .flags = CPUINFO_CACHE_INCLUSIVE, + }; + break; + case 0x2C: + /* + * Intel ISA Reference: + * "1st-level data cache: 32 KBytes, 8-way set associative, 64 byte line size" + * Application Note 485: + * "1st-level data cache: 32-KB, 8-way set associative, 64-byte line size" + */ + cache->l1d = (struct cpuinfo_x86_cache) { + .size = 32 * 1024, + .associativity = 8, + .sets = 64, + .partitions = 1, + .line_size = 64, + }; + break; + case 0x30: + /* + * Intel ISA Reference: + * "1st-level instruction cache: 32 KBytes, 8-way set associative, 64 byte line size" + * Application Note 485: + * "1st-level instruction cache: 32-KB, 8-way set associative, 64-byte line size" + */ + cache->l1i = (struct cpuinfo_x86_cache) { + .size = 32 * 1024, + .associativity = 8, + .sets = 64, + .partitions = 1, + .line_size = 64, + }; + break; + case 0x39: + /* Where does this come from? */ + cache->l2 = (struct cpuinfo_x86_cache) { + .size = 128 * 1024, + .associativity = 4, + .sets = 512, + .partitions = 1, + .line_size = 64, + .flags = CPUINFO_CACHE_INCLUSIVE, + }; + break; + case 0x3A: + /* Where does this come from? */ + cache->l2 = (struct cpuinfo_x86_cache) { + .size = 192 * 1024, + .associativity = 6, + .sets = 512, + .partitions = 1, + .line_size = 64, + .flags = CPUINFO_CACHE_INCLUSIVE, + }; + break; + case 0x3B: + /* Where does this come from? */ + cache->l2 = (struct cpuinfo_x86_cache) { + .size = 128 * 1024, + .associativity = 2, + .sets = 1024, + .partitions = 1, + .line_size = 64, + .flags = CPUINFO_CACHE_INCLUSIVE, + }; + break; + case 0x3C: + /* Where does this come from? */ + cache->l2 = (struct cpuinfo_x86_cache) { + .size = 256 * 1024, + .associativity = 4, + .sets = 1024, + .partitions = 1, + .line_size = 64, + .flags = CPUINFO_CACHE_INCLUSIVE, + }; + break; + case 0x3D: + /* Where does this come from? */ + cache->l2 = (struct cpuinfo_x86_cache) { + .size = 384 * 1024, + .associativity = 6, + .sets = 1024, + .partitions = 1, + .line_size = 64, + .flags = CPUINFO_CACHE_INCLUSIVE, + }; + break; + case 0x3E: + /* Where does this come from? */ + cache->l2 = (struct cpuinfo_x86_cache) { + .size = 512 * 1024, + .associativity = 4, + .sets = 2048, + .partitions = 1, + .line_size = 64, + .flags = CPUINFO_CACHE_INCLUSIVE, + }; + break; + case 0x40: + /* + * Intel ISA Reference: + * "No 2nd-level cache or, if processor contains a valid 2nd-level cache, no 3rd-level cache" + * Application Note 485: + * "No 2nd-level cache or, if processor contains a valid 2nd-level cache, no 3rd-level cache" + */ + break; + case 0x41: + /* + * Intel ISA Reference: + * "2nd-level cache: 128 KBytes, 4-way set associative, 32 byte line size" + * Application Note 485: + * "2nd-level cache: 128-KB, 4-way set associative, 32-byte line size" + */ + cache->l2 = (struct cpuinfo_x86_cache) { + .size = 128 * 1024, + .associativity = 4, + .sets = 1024, + .partitions = 1, + .line_size = 32, + .flags = CPUINFO_CACHE_INCLUSIVE, + }; + break; + case 0x42: + /* + * Intel ISA Reference: + * "2nd-level cache: 256 KBytes, 4-way set associative, 32 byte line size" + * Application Note 485: + * "2nd-level cache: 256-KB, 4-way set associative, 32-byte line size" + */ + cache->l2 = (struct cpuinfo_x86_cache) { + .size = 256 * 1024, + .associativity = 4, + .sets = 2048, + .partitions = 1, + .line_size = 32, + .flags = CPUINFO_CACHE_INCLUSIVE, + }; + break; + case 0x43: + /* + * Intel ISA Reference: + * "2nd-level cache: 512 KBytes, 4-way set associative, 32 byte line size" + * Application Note 485: + * "2nd-level cache: 512-KB, 4-way set associative, 32-byte line size" + */ + cache->l2 = (struct cpuinfo_x86_cache) { + .size = 512 * 1024, + .associativity = 4, + .sets = 4096, + .partitions = 1, + .line_size = 32, + .flags = CPUINFO_CACHE_INCLUSIVE, + }; + break; + case 0x44: + /* + * Intel ISA Reference: + * "2nd-level cache: 1 MByte, 4-way set associative, 32 byte line size" + * Application Note 485: + * "2nd-level cache: 1-MB, 4-way set associative, 32-byte line size" + */ + cache->l2 = (struct cpuinfo_x86_cache) { + .size = 1024 * 1024, + .associativity = 4, + .sets = 8192, + .partitions = 1, + .line_size = 32, + .flags = CPUINFO_CACHE_INCLUSIVE, + }; + break; + case 0x45: + /* + * Intel ISA Reference: + * "2nd-level cache: 2 MByte, 4-way set associative, 32 byte line size" + * Application Note 485: + * "2nd-level cache: 2-MB, 4-way set associative, 32-byte line size" + */ + cache->l2 = (struct cpuinfo_x86_cache) { + .size = 2 * 1024 * 1024, + .associativity = 4, + .sets = 16384, + .partitions = 1, + .line_size = 32, + .flags = CPUINFO_CACHE_INCLUSIVE, + }; + break; + case 0x46: + /* + * Intel ISA Reference: + * "3rd-level cache: 4 MByte, 4-way set associative, 64 byte line size" + * Application Note 485: + * "3rd-level cache: 4-MB, 4-way set associative, 64-byte line size" + */ + cache->l3 = (struct cpuinfo_x86_cache) { + .size = 4 * 1024 * 1024, + .associativity = 4, + .sets = 16384, + .partitions = 1, + .line_size = 64, + .flags = CPUINFO_CACHE_INCLUSIVE, + }; + break; + case 0x47: + /* + * Intel ISA Reference: + * "3rd-level cache: 8 MByte, 8-way set associative, 64 byte line size" + * Application Note 485: + * "3rd-level cache: 8-MB, 8-way set associative, 64-byte line size" + */ + cache->l3 = (struct cpuinfo_x86_cache) { + .size = 8 * 1024 * 1024, + .associativity = 8, + .sets = 16384, + .partitions = 1, + .line_size = 64, + .flags = CPUINFO_CACHE_INCLUSIVE, + }; + break; + case 0x48: + /* + * Intel ISA Reference: + * "2nd-level cache: 3MByte, 12-way set associative, 64 byte line size" + * Application Note 485: + * "2nd-level cache: 3-MB, 12-way set associative, 64-byte line size, unified on-die" + */ + cache->l2 = (struct cpuinfo_x86_cache) { + .size = 3 * 1024 * 1024, + .associativity = 12, + .sets = 4096, + .partitions = 1, + .line_size = 64, + .flags = CPUINFO_CACHE_INCLUSIVE, + }; + break; + case 0x49: + /* + * Intel ISA Reference: + * "3rd-level cache: 4MB, 16-way set associative, 64-byte line size (Intel Xeon processor MP, + * Family 0FH, Model 06H); 2nd-level cache: 4 MByte, 16-way set associative, 64 byte line size" + * Application Note 485: + * "3rd-level cache: 4-MB, 16-way set associative, 64-byte line size (Intel Xeon processor MP, + * Family 0Fh, Model 06h) + * 2nd-level cache: 4-MB, 16-way set associative, 64-byte line size" + */ + if ((vendor == cpuinfo_vendor_intel) && (model_info->model == 0x06) && (model_info->family == 0x0F)) { + cache->l3 = (struct cpuinfo_x86_cache) { + .size = 4 * 1024 * 1024, + .associativity = 16, + .sets = 4096, + .partitions = 1, + .line_size = 64, + .flags = CPUINFO_CACHE_INCLUSIVE, + }; + } else { + cache->l2 = (struct cpuinfo_x86_cache) { + .size = 4 * 1024 * 1024, + .associativity = 16, + .sets = 4096, + .partitions = 1, + .line_size = 64, + .flags = CPUINFO_CACHE_INCLUSIVE, + }; + } + break; + case 0x4A: + /* + * Intel ISA Reference: + * "3rd-level cache: 6MByte, 12-way set associative, 64 byte line size" + * Application Note 485: + * "3rd-level cache: 6-MB, 12-way set associative, 64-byte line size" + */ + cache->l3 = (struct cpuinfo_x86_cache) { + .size = 6 * 1024 * 1024, + .associativity = 12, + .sets = 8192, + .partitions = 1, + .line_size = 64, + .flags = CPUINFO_CACHE_INCLUSIVE, + }; + break; + case 0x4B: + /* + * Intel ISA Reference: + * "3rd-level cache: 8MByte, 16-way set associative, 64 byte line size" + * Application Note 485: + * "3rd-level cache: 8-MB, 16-way set associative, 64-byte line size" + */ + cache->l3 = (struct cpuinfo_x86_cache) { + .size = 8 * 1024 * 1024, + .associativity = 16, + .sets = 8192, + .partitions = 1, + .line_size = 64, + .flags = CPUINFO_CACHE_INCLUSIVE, + }; + break; + case 0x4C: + /* + * Intel ISA Reference: + * "3rd-level cache: 12MByte, 12-way set associative, 64 byte line size" + * Application Note 485: + * "3rd-level cache: 12-MB, 12-way set associative, 64-byte line size" + */ + cache->l3 = (struct cpuinfo_x86_cache) { + .size = 12 * 1024 * 1024, + .associativity = 12, + .sets = 16384, + .partitions = 1, + .line_size = 64, + .flags = CPUINFO_CACHE_INCLUSIVE, + }; + break; + case 0x4D: + /* + * Intel ISA Reference: + * "3rd-level cache: 16MByte, 16-way set associative, 64 byte line size" + * Application Note 485: + * "3rd-level cache: 16-MB, 16-way set associative, 64-byte line size" + */ + cache->l3 = (struct cpuinfo_x86_cache) { + .size = 16 * 1024 * 1024, + .associativity = 16, + .sets = 16384, + .partitions = 1, + .line_size = 64, + .flags = CPUINFO_CACHE_INCLUSIVE, + }; + break; + case 0x4E: + /* + * Intel ISA Reference: + * "2nd-level cache: 6MByte, 24-way set associative, 64 byte line size" + * Application Note 485: + * "2nd-level cache: 6-MB, 24-way set associative, 64-byte line size" + */ + cache->l2 = (struct cpuinfo_x86_cache) { + .size = 6 * 1024 * 1024, + .associativity = 24, + .sets = 4096, + .partitions = 1, + .line_size = 64, + .flags = CPUINFO_CACHE_INCLUSIVE, + }; + break; + case 0x4F: + /* + * Intel ISA Reference: + * "Instruction TLB: 4 KByte pages, 32 entries" + * Application Note 485: + * "Instruction TLB: 4-KB pages, 32 entries" + */ + *itlb_4KB = (struct cpuinfo_tlb) { + .entries = 32, + /* Assume full associativity from nearby entries: manual lacks detail */ + .associativity = 32, + .pages = CPUINFO_PAGE_SIZE_4KB, + }; + break; + case 0x50: + /* + * Intel ISA Reference: + * "Instruction TLB: 4 KByte and 2-MByte or 4-MByte pages, 64 entries" + * Application Note 485: + * "Instruction TLB: 4-KB, 2-MB or 4-MB pages, fully associative, 64 entries" + */ + *itlb_4KB = *itlb_2MB = *itlb_4MB = (struct cpuinfo_tlb) { + .entries = 64, + .associativity = 64, + .pages = CPUINFO_PAGE_SIZE_4KB | CPUINFO_PAGE_SIZE_2MB | CPUINFO_PAGE_SIZE_4MB, + }; + break; + case 0x51: + /* + * Intel ISA Reference: + * "Instruction TLB: 4 KByte and 2-MByte or 4-MByte pages, 128 entries" + * Application Note 485: + * "Instruction TLB: 4-KB, 2-MB or 4-MB pages, fully associative, 128 entries" + */ + *itlb_4KB = *itlb_2MB = *itlb_4MB = (struct cpuinfo_tlb) { + .entries = 128, + .associativity = 128, + .pages = CPUINFO_PAGE_SIZE_4KB | CPUINFO_PAGE_SIZE_2MB | CPUINFO_PAGE_SIZE_4MB, + }; + break; + case 0x52: + /* + * Intel ISA Reference: + * "Instruction TLB: 4 KByte and 2-MByte or 4-MByte pages, 256 entries" + * Application Note 485: + * "Instruction TLB: 4-KB, 2-MB or 4-MB pages, fully associative, 256 entries" + */ + *itlb_4KB = *itlb_2MB = *itlb_4MB = (struct cpuinfo_tlb) { + .entries = 256, + .associativity = 256, + .pages = CPUINFO_PAGE_SIZE_4KB | CPUINFO_PAGE_SIZE_2MB | CPUINFO_PAGE_SIZE_4MB, + }; + break; + case 0x55: + /* + * Intel ISA Reference: + * "Instruction TLB: 2-MByte or 4-MByte pages, fully associative, 7 entries" + * Application Note 485: + * "Instruction TLB: 2-MB or 4-MB pages, fully associative, 7 entries" + */ + *itlb_2MB = *itlb_4MB = (struct cpuinfo_tlb) { + .entries = 7, + .associativity = 7, + .pages = CPUINFO_PAGE_SIZE_2MB | CPUINFO_PAGE_SIZE_4MB, + }; + break; + case 0x56: + /* + * Intel ISA Reference: + * "Data TLB0: 4 MByte pages, 4-way set associative, 16 entries" + * Application Note 485: + * "L1 Data TLB: 4-MB pages, 4-way set associative, 16 entries" + */ + *dtlb0_4MB = (struct cpuinfo_tlb) { + .entries = 16, + .associativity = 4, + .pages = CPUINFO_PAGE_SIZE_4MB, + }; + break; + case 0x57: + /* + * Intel ISA Reference: + * "Data TLB0: 4 KByte pages, 4-way associative, 16 entries" + * Application Note 485: + * "L1 Data TLB: 4-KB pages, 4-way set associative, 16 entries" + */ + *dtlb0_4KB = (struct cpuinfo_tlb) { + .entries = 16, + .associativity = 4, + .pages = CPUINFO_PAGE_SIZE_4KB, + }; + break; + case 0x59: + /* + * Intel ISA Reference: + * "Data TLB0: 4 KByte pages, fully associative, 16 entries" + * Application Note 485: + * "Data TLB0: 4-KB pages, fully associative, 16 entries" + */ + *dtlb0_4KB = (struct cpuinfo_tlb) { + .entries = 16, + .associativity = 16, + .pages = CPUINFO_PAGE_SIZE_4KB, + }; + break; + case 0x5A: + /* + * Intel ISA Reference: + * "Data TLB0: 2 MByte or 4 MByte pages, 4-way set associative, 32 entries" + * Application Note 485: + * "Data TLB0: 2-MB or 4-MB pages, 4-way associative, 32 entries" + */ + *dtlb0_2MB = *dtlb0_4MB = (struct cpuinfo_tlb) { + .entries = 32, + .associativity = 4, + .pages = CPUINFO_PAGE_SIZE_2MB | CPUINFO_PAGE_SIZE_4MB, + }; + break; + case 0x5B: + /* + * Intel ISA Reference: + * "Data TLB: 4 KByte and 4 MByte pages, 64 entries" + * Application Note 485: + * "Data TLB: 4-KB or 4-MB pages, fully associative, 64 entries" + */ + *dtlb_4KB = *dtlb_4MB = (struct cpuinfo_tlb) { + .entries = 64, + .associativity = 64, + .pages = CPUINFO_PAGE_SIZE_4KB | CPUINFO_PAGE_SIZE_4MB, + }; + break; + case 0x5C: + /* + * Intel ISA Reference: + * "Data TLB: 4 KByte and 4 MByte pages, 128 entries" + * Application Note 485: + * "Data TLB: 4-KB or 4-MB pages, fully associative, 128 entries" + */ + *dtlb_4KB = *dtlb_4MB = (struct cpuinfo_tlb) { + .entries = 128, + .associativity = 128, + .pages = CPUINFO_PAGE_SIZE_4KB | CPUINFO_PAGE_SIZE_4MB, + }; + break; + case 0x5D: + /* + * Intel ISA Reference: + * "Data TLB: 4 KByte and 4 MByte pages, 256 entries" + * Application Note 485: + * "Data TLB: 4-KB or 4-MB pages, fully associative, 256 entries" + */ + *dtlb_4KB = *dtlb_4MB = (struct cpuinfo_tlb) { + .entries = 256, + .associativity = 256, + .pages = CPUINFO_PAGE_SIZE_4KB | CPUINFO_PAGE_SIZE_4MB, + }; + break; + case 0x60: + /* + * Application Note 485: + * "1st-level data cache: 16-KB, 8-way set associative, sectored cache, 64-byte line size" + */ + cache->l1d = (struct cpuinfo_x86_cache) { + .size = 16 * 1024, + .associativity = 8, + .sets = 32, + .partitions = 1, + .line_size = 64, + }; + break; + case 0x61: + /* + * Intel ISA Reference: + * "Instruction TLB: 4 KByte pages, fully associative, 48 entries" + */ + *itlb_4KB = (struct cpuinfo_tlb) { + .entries = 48, + .associativity = 48, + .pages = CPUINFO_PAGE_SIZE_4KB, + }; + break; + case 0x63: + /* + * Intel ISA Reference: + * "Data TLB: 2 MByte or 4 MByte pages, 4-way set associative, 32 entries and + * a separate array with 1 GByte pages, 4-way set associative, 4 entries" + */ + *dtlb_2MB = *dtlb_4MB = (struct cpuinfo_tlb) { + .entries = 32, + .associativity = 4, + .pages = CPUINFO_PAGE_SIZE_2MB | CPUINFO_PAGE_SIZE_4MB, + }; + *dtlb_1GB = (struct cpuinfo_tlb) { + .entries = 4, + .associativity = 4, + .pages = CPUINFO_PAGE_SIZE_1GB, + }; + break; + case 0x64: + /* + * Intel ISA Reference: + * "Data TLB: 4 KByte pages, 4-way set associative, 512 entries" + * + */ + *dtlb_4KB = (struct cpuinfo_tlb) { + .entries = 512, + .associativity = 4, + .pages = CPUINFO_PAGE_SIZE_4KB, + }; + break; + case 0x66: + /* + * Application Note 485: + * "1st-level data cache: 8-KB, 4-way set associative, sectored cache, 64-byte line size" + */ + cache->l1d = (struct cpuinfo_x86_cache) { + .size = 8 * 1024, + .associativity = 4, + .sets = 32, + .partitions = 1, + .line_size = 64, + }; + break; + case 0x67: + /* + * Application Note 485: + * "1st-level data cache: 16-KB, 4-way set associative, sectored cache, 64-byte line size" + */ + cache->l1d = (struct cpuinfo_x86_cache) { + .size = 16 * 1024, + .associativity = 4, + .sets = 64, + .partitions = 1, + .line_size = 64, + }; + break; + case 0x68: + /* + * Application Note 485: + * "1st-level data cache: 32-KB, 4 way set associative, sectored cache, 64-byte line size" + */ + cache->l1d = (struct cpuinfo_x86_cache) { + .size = 32 * 1024, + .associativity = 4, + .sets = 128, + .partitions = 1, + .line_size = 64, + }; + break; + case 0x6A: + /* + * Intel ISA Reference: + * "uTLB: 4 KByte pages, 8-way set associative, 64 entries" + */ + + /* uTLB is, an fact, a normal 1-level DTLB on Silvermont & Knoghts Landing */ + *dtlb_4KB = (struct cpuinfo_tlb) { + .entries = 64, + .associativity = 8, + .pages = CPUINFO_PAGE_SIZE_4KB, + }; + break; + case 0x6B: + /* + * Intel ISA Reference: + * "DTLB: 4 KByte pages, 8-way set associative, 256 entries" + */ + *dtlb_4KB = (struct cpuinfo_tlb) { + .entries = 256, + .associativity = 8, + .pages = CPUINFO_PAGE_SIZE_4KB, + }; + break; + case 0x6C: + /* + * Intel ISA Reference: + * "DTLB: 2M/4M pages, 8-way set associative, 128 entries" + */ + *dtlb_2MB = *dtlb_4MB = (struct cpuinfo_tlb) { + .entries = 128, + .associativity = 8, + .pages = CPUINFO_PAGE_SIZE_2MB | CPUINFO_PAGE_SIZE_4MB, + }; + break; + case 0x6D: + /* + * Intel ISA Reference: + * "DTLB: 1 GByte pages, fully associative, 16 entries" + */ + *dtlb_1GB = (struct cpuinfo_tlb) { + .entries = 16, + .associativity = 16, + .pages = CPUINFO_PAGE_SIZE_1GB, + }; + break; + case 0x70: + /* + * Intel ISA Reference: + * "Trace cache: 12 K-uop, 8-way set associative" + * Application Note 485: + * "Trace cache: 12K-uops, 8-way set associative" + * Cyrix CPU Detection Guide and Geode GX1 Processor Series: + * "TLB, 32 entries, 4-way set associative, 4K-Byte Pages" + */ + switch (vendor) { +#if CPUINFO_ARCH_X86 + case cpuinfo_vendor_cyrix: + case cpuinfo_vendor_nsc: + *dtlb_4KB = *itlb_4KB = (struct cpuinfo_tlb) { + .entries = 32, + .associativity = 4, + .pages = CPUINFO_PAGE_SIZE_4KB, + }; + break; +#endif /* CPUINFO_ARCH_X86 */ + default: + cache->trace = (struct cpuinfo_trace_cache) { + .uops = 12 * 1024, + .associativity = 8, + }; + } + break; + case 0x71: + /* + * Intel ISA Reference: + * "Trace cache: 16 K-uop, 8-way set associative" + * Application Note 485: + * "Trace cache: 16K-uops, 8-way set associative" + */ + cache->trace = (struct cpuinfo_trace_cache) { + .uops = 16 * 1024, + .associativity = 8, + }; + break; + case 0x72: + /* + * Intel ISA Reference: + * "Trace cache: 32 K-μop, 8-way set associative" + * Application Note 485: + * "Trace cache: 32K-uops, 8-way set associative" + */ + cache->trace = (struct cpuinfo_trace_cache) { + .uops = 32 * 1024, + .associativity = 8, + }; + break; + case 0x73: + /* Where does this come from? */ + cache->trace = (struct cpuinfo_trace_cache) { + .uops = 64 * 1024, + .associativity = 8, + }; + break; + case 0x76: + /* + * Intel ISA Reference: + * "Instruction TLB: 2M/4M pages, fully associative, 8 entries" + * Application Note 485: + * "Instruction TLB: 2M/4M pages, fully associative, 8 entries" + */ + *itlb_2MB = *itlb_4MB = (struct cpuinfo_tlb) { + .entries = 8, + .associativity = 8, + .pages = CPUINFO_PAGE_SIZE_2MB | CPUINFO_PAGE_SIZE_4MB, + }; + break; + case 0x78: + /* + * Intel ISA Reference: + * "2nd-level cache: 1 MByte, 4-way set associative, 64byte line size" + * Application Note 485: + * "2nd-level cache: 1-MB, 4-way set associative, 64-byte line size" + */ + cache->l2 = (struct cpuinfo_x86_cache) { + .size = 1024 * 1024, + .associativity = 4, + .sets = 4096, + .partitions = 1, + .line_size = 64, + .flags = CPUINFO_CACHE_INCLUSIVE, + }; + break; + case 0x79: + /* + * Intel ISA Reference: + * "2nd-level cache: 128 KByte, 8-way set associative, 64 byte line size, 2 lines per sector" + * Application Note 485: + * "2nd-level cache: 128-KB, 8-way set associative, sectored cache, 64-byte line size" + */ + cache->l2 = (struct cpuinfo_x86_cache) { + .size = 128 * 1024, + .associativity = 8, + .sets = 256, + .partitions = 1, + .line_size = 64, + .flags = CPUINFO_CACHE_INCLUSIVE, + }; + break; + case 0x7A: + /* + * Intel ISA Reference: + * "2nd-level cache: 256 KByte, 8-way set associative, 64 byte line size, 2 lines per sector" + * Application Note 485: + * "2nd-level cache: 256-KB, 8-way set associative, sectored cache, 64-byte line size" + */ + cache->l2 = (struct cpuinfo_x86_cache) { + .size = 256 * 1024, + .associativity = 8, + .sets = 512, + .partitions = 1, + .line_size = 64, + .flags = CPUINFO_CACHE_INCLUSIVE, + }; + break; + case 0x7B: + /* + * Intel ISA Reference: + * "2nd-level cache: 512 KByte, 8-way set associative, 64 byte line size, 2 lines per sector" + * Application Note 485: + * "2nd-level cache: 512-KB, 8-way set associative, sectored cache, 64-byte line size" + */ + cache->l2 = (struct cpuinfo_x86_cache) { + .size = 512 * 1024, + .associativity = 8, + .sets = 1024, + .partitions = 1, + .line_size = 64, + .flags = CPUINFO_CACHE_INCLUSIVE, + }; + break; + case 0x7C: + /* + * Intel ISA Reference: + * "2nd-level cache: 1 MByte, 8-way set associative, 64 byte line size, 2 lines per sector" + * Application Note 485: + * "2nd-level cache: 1-MB, 8-way set associative, sectored cache, 64-byte line size" + */ + cache->l2 = (struct cpuinfo_x86_cache) { + .size = 1024 * 1024, + .associativity = 8, + .sets = 2048, + .partitions = 1, + .line_size = 64, + .flags = CPUINFO_CACHE_INCLUSIVE, + }; + break; + case 0x7D: + /* + * Intel ISA Reference: + * "2nd-level cache: 2 MByte, 8-way set associative, 64byte line size" + * Application Note 485: + * "2nd-level cache: 2-MB, 8-way set associative, 64-byte line size" + */ + cache->l2 = (struct cpuinfo_x86_cache) { + .size = 2 * 1024 * 1024, + .associativity = 8, + .sets = 4096, + .partitions = 1, + .line_size = 64, + .flags = CPUINFO_CACHE_INCLUSIVE, + }; + break; + case 0x7F: + /* + * Intel ISA Reference: + * "2nd-level cache: 512 KByte, 2-way set associative, 64-byte line size" + * Application Note 485: + * "2nd-level cache: 512-KB, 2-way set associative, 64-byte line size" + */ + cache->l2 = (struct cpuinfo_x86_cache) { + .size = 512 * 1024, + .associativity = 2, + .sets = 4096, + .partitions = 1, + .line_size = 64, + .flags = CPUINFO_CACHE_INCLUSIVE, + }; + break; + case 0x80: + /* + * Intel ISA Reference: + * "2nd-level cache: 512 KByte, 8-way set associative, 64-byte line size" + * Application Note 485: + * "2nd-level cache: 512-KB, 8-way set associative, 64-byte line size" + * Cyrix CPU Detection Guide and Geode GX1 Processor Series: + * "Level 1 Cache, 16K, 4-way set associative, 16 Bytes/Line" + */ + switch (vendor) { +#if CPUINFO_ARCH_X86 && !defined(__ANDROID__) + case cpuinfo_vendor_cyrix: + case cpuinfo_vendor_nsc: + cache->l1i = cache->l1d = (struct cpuinfo_x86_cache) { + .size = 16 * 1024, + .associativity = 4, + .sets = 256, + .partitions = 1, + .line_size = 16, + .flags = CPUINFO_CACHE_UNIFIED, + }; + break; +#endif /* CPUINFO_ARCH_X86 */ + default: + cache->l2 = (struct cpuinfo_x86_cache) { + .size = 512 * 1024, + .associativity = 8, + .sets = 1024, + .partitions = 1, + .line_size = 64, + .flags = CPUINFO_CACHE_INCLUSIVE, + }; + } + break; + case 0x82: + /* + * Intel ISA Reference: + * "2nd-level cache: 256 KByte, 8-way set associative, 32 byte line size" + * Application Note 485: + * "2nd-level cache: 256-KB, 8-way set associative, 32-byte line size" + */ + cache->l2 = (struct cpuinfo_x86_cache) { + .size = 256 * 1024, + .associativity = 4, + .sets = 2048, + .partitions = 1, + .line_size = 32, + .flags = CPUINFO_CACHE_INCLUSIVE, + }; + break; + case 0x83: + /* + * Intel ISA Reference: + * "2nd-level cache: 512 KByte, 8-way set associative, 32 byte line size" + * Application Note 485: + * "2nd-level cache: 512-KB, 8-way set associative, 32-byte line size" + */ + cache->l2 = (struct cpuinfo_x86_cache) { + .size = 512 * 1024, + .associativity = 8, + .sets = 2048, + .partitions = 1, + .line_size = 32, + .flags = CPUINFO_CACHE_INCLUSIVE, + }; + break; + case 0x84: + /* + * Intel ISA Reference: + * "2nd-level cache: 1 MByte, 8-way set associative, 32 byte line size" + * Application Note 485: + * "2nd-level cache: 1-MB, 8-way set associative, 32-byte line size" + */ + cache->l2 = (struct cpuinfo_x86_cache) { + .size = 1024 * 1024, + .associativity = 8, + .sets = 4096, + .partitions = 1, + .line_size = 32, + .flags = CPUINFO_CACHE_INCLUSIVE, + }; + break; + case 0x85: + /* + * Intel ISA Reference: + * "2nd-level cache: 2 MByte, 8-way set associative, 32 byte line size" + * Application Note 485: + * "2nd-level cache: 2-MB, 8-way set associative, 32-byte line size" + */ + cache->l2 = (struct cpuinfo_x86_cache) { + .size = 2 * 1024 * 1024, + .associativity = 8, + .sets = 8192, + .partitions = 1, + .line_size = 32, + .flags = CPUINFO_CACHE_INCLUSIVE, + }; + break; + case 0x86: + /* + * Intel ISA Reference: + * "2nd-level cache: 512 KByte, 4-way set associative, 64 byte line size" + * Application Note 485: + * "2nd-level cache: 512-KB, 4-way set associative, 64-byte line size" + */ + cache->l2 = (struct cpuinfo_x86_cache) { + .size = 512 * 1024, + .associativity = 4, + .sets = 2048, + .partitions = 1, + .line_size = 64, + .flags = CPUINFO_CACHE_INCLUSIVE, + }; + break; + case 0x87: + /* + * Intel ISA Reference: + * "2nd-level cache: 1 MByte, 8-way set associative, 64 byte line size" + * Application Note 485: + * "2nd-level cache: 1-MB, 8-way set associative, 64-byte line size" + */ + cache->l2 = (struct cpuinfo_x86_cache) { + .size = 1024 * 1024, + .associativity = 8, + .sets = 2048, + .partitions = 1, + .line_size = 64, + .flags = CPUINFO_CACHE_INCLUSIVE, + }; + break; + case 0xA0: + /* + * Intel ISA Reference: + * "DTLB: 4k pages, fully associative, 32 entries" + */ + *dtlb_4KB = (struct cpuinfo_tlb) { + .entries = 32, + .associativity = 32, + .pages = CPUINFO_PAGE_SIZE_4KB, + }; + break; + case 0xB0: + /* + * Intel ISA Reference: + * "Instruction TLB: 4 KByte pages, 4-way set associative, 128 entries" + * Application Note 485: + * "Instruction TLB: 4-KB Pages, 4-way set associative, 128 entries" + */ + *itlb_4KB = (struct cpuinfo_tlb) { + .entries = 128, + .associativity = 4, + .pages = CPUINFO_PAGE_SIZE_4KB, + }; + break; + case 0xB1: + /* + * Intel ISA Reference: + * "Instruction TLB: 2M pages, 4-way, 8 entries or 4M pages, 4-way, 4 entries" + * Application Note 485: + * "Instruction TLB: 2-MB pages, 4-way, 8 entries or 4M pages, 4-way, 4 entries" + */ + *itlb_2MB = (struct cpuinfo_tlb) { + .entries = 8, + .associativity = 4, + .pages = CPUINFO_PAGE_SIZE_2MB | CPUINFO_PAGE_SIZE_4MB, + }; + *itlb_4MB = (struct cpuinfo_tlb) { + .entries = 4, + .associativity = 4, + .pages = CPUINFO_PAGE_SIZE_2MB | CPUINFO_PAGE_SIZE_4MB, + }; + break; + case 0xB2: + /* + * Intel ISA Reference: + * "Instruction TLB: 4KByte pages, 4-way set associative, 64 entries" + * Application Note 485: + * "Instruction TLB: 4-KB pages, 4-way set associative, 64 entries" + */ + *itlb_4KB = (struct cpuinfo_tlb) { + .entries = 64, + .associativity = 4, + .pages = CPUINFO_PAGE_SIZE_4KB, + }; + break; + case 0xB3: + /* + * Intel ISA Reference: + * "Data TLB: 4 KByte pages, 4-way set associative, 128 entries" + * Application Note 485: + * "Data TLB: 4-KB Pages, 4-way set associative, 128 entries" + */ + *dtlb_4KB = (struct cpuinfo_tlb) { + .entries = 128, + .associativity = 4, + .pages = CPUINFO_PAGE_SIZE_4KB, + }; + break; + case 0xB4: + /* + * Intel ISA Reference: + * "Data TLB1: 4 KByte pages, 4-way associative, 256 entries" + * Application Note 485: + * "Data TLB: 4-KB Pages, 4-way set associative, 256 entries" + */ + *dtlb_4KB = (struct cpuinfo_tlb) { + .entries = 256, + .associativity = 4, + .pages = CPUINFO_PAGE_SIZE_4KB, + }; + break; + case 0xB5: + /* + * Intel ISA Reference: + * "Instruction TLB: 4KByte pages, 8-way set associative, 64 entries" + */ + *itlb_4KB = (struct cpuinfo_tlb) { + .entries = 64, + .associativity = 8, + .pages = CPUINFO_PAGE_SIZE_4KB, + }; + break; + case 0xB6: + /* + * Intel ISA Reference: + * "Instruction TLB: 4KByte pages, 8-way set associative, 128 entries" + */ + *itlb_4KB = (struct cpuinfo_tlb) { + .entries = 128, + .associativity = 8, + .pages = CPUINFO_PAGE_SIZE_4KB, + }; + break; + case 0xBA: + /* + * Intel ISA Reference: + * "Data TLB1: 4 KByte pages, 4-way associative, 64 entries" + * Application Note 485: + * "Data TLB: 4-KB Pages, 4-way set associative, 64 entries" + */ + *itlb_4KB = (struct cpuinfo_tlb) { + .entries = 64, + .associativity = 4, + .pages = CPUINFO_PAGE_SIZE_4KB, + }; + break; + case 0xC0: + /* + * Intel ISA Reference: + * "Data TLB: 4 KByte and 4 MByte pages, 4-way associative, 8 entries" + * Application Note 485: + * "Data TLB: 4-KB or 4-MB Pages, 4-way set associative, 8 entries" + */ + *itlb_4KB = *itlb_4MB = (struct cpuinfo_tlb) { + .entries = 8, + .associativity = 4, + .pages = CPUINFO_PAGE_SIZE_4KB | CPUINFO_PAGE_SIZE_4MB, + }; + break; + case 0xC1: + /* + * Intel ISA Reference: + * "Shared 2nd-Level TLB: 4 KByte/2MByte pages, 8-way associative, 1024 entries" + */ + *stlb2_4KB = *stlb2_2MB = (struct cpuinfo_tlb) { + .entries = 1024, + .associativity = 8, + .pages = CPUINFO_PAGE_SIZE_4KB | CPUINFO_PAGE_SIZE_2MB, + }; + break; + case 0xC2: + /* + * Intel ISA Reference: + * "DTLB: 4 KByte/2 MByte pages, 4-way associative, 16 entries" + */ + *dtlb_4KB = *dtlb_2MB = (struct cpuinfo_tlb) { + .entries = 16, + .associativity = 4, + .pages = CPUINFO_PAGE_SIZE_4KB | CPUINFO_PAGE_SIZE_2MB, + }; + break; + case 0xC3: + /* + * Intel ISA Reference: + * "Shared 2nd-Level TLB: 4 KByte/2 MByte pages, 6-way associative, 1536 entries. + * Also 1GBbyte pages, 4-way, 16 entries." + */ + *stlb2_4KB = *stlb2_2MB = (struct cpuinfo_tlb) { + .entries = 1536, + .associativity = 6, + .pages = CPUINFO_PAGE_SIZE_4KB | CPUINFO_PAGE_SIZE_2MB, + }; + *stlb2_1GB = (struct cpuinfo_tlb) { + .entries = 16, + .associativity = 4, + .pages = CPUINFO_PAGE_SIZE_1GB, + }; + break; + case 0xC4: + /* + * Intel ISA Reference: + * "DTLB: 2M/4M Byte pages, 4-way associative, 32 entries" + */ + *dtlb_2MB = *dtlb_4MB = (struct cpuinfo_tlb) { + .entries = 32, + .associativity = 4, + .pages = CPUINFO_PAGE_SIZE_2MB | CPUINFO_PAGE_SIZE_4MB, + }; + break; + case 0xCA: + /* + * Intel ISA Reference: + * "Shared 2nd-Level TLB: 4 KByte pages, 4-way associative, 512 entries" + * Application Note 485: + * "Shared 2nd-level TLB: 4 KB pages, 4-way set associative, 512 entries" + */ + *stlb2_4KB = (struct cpuinfo_tlb) { + .entries = 512, + .associativity = 4, + .pages = CPUINFO_PAGE_SIZE_4KB, + }; + break; + case 0xD0: + /* + * Intel ISA Reference: + * "3rd-level cache: 512 KByte, 4-way set associative, 64 byte line size" + * Application Note 485: + * "3rd-level cache: 512-kB, 4-way set associative, 64-byte line size" + */ + cache->l3 = (struct cpuinfo_x86_cache) { + .size = 512 * 1024, + .associativity = 4, + .sets = 2048, + .partitions = 1, + .line_size = 64, + .flags = CPUINFO_CACHE_INCLUSIVE, + }; + break; + case 0xD1: + /* + * Intel ISA Reference: + * "3rd-level cache: 1 MByte, 4-way set associative, 64 byte line size" + * Application Note 485: + * "3rd-level cache: 1-MB, 4-way set associative, 64-byte line size" + */ + cache->l3 = (struct cpuinfo_x86_cache) { + .size = 1024 * 1024, + .associativity = 4, + .sets = 4096, + .partitions = 1, + .line_size = 64, + .flags = CPUINFO_CACHE_INCLUSIVE, + }; + break; + case 0xD2: + /* + * Intel ISA Reference: + * "3rd-level cache: 2 MByte, 4-way set associative, 64 byte line size" + * Application Note 485: + * "3rd-level cache: 2-MB, 4-way set associative, 64-byte line size" + */ + cache->l3 = (struct cpuinfo_x86_cache) { + .size = 2 * 1024 * 2014, + .associativity = 4, + .sets = 8192, + .partitions = 1, + .line_size = 64, + .flags = CPUINFO_CACHE_INCLUSIVE, + }; + break; + case 0xD6: + /* + * Intel ISA Reference: + * "3rd-level cache: 1 MByte, 8-way set associative, 64 byte line size" + * Application Note 485: + * "3rd-level cache: 1-MB, 8-way set associative, 64-byte line size" + */ + cache->l3 = (struct cpuinfo_x86_cache) { + .size = 1024 * 1024, + .associativity = 8, + .sets = 2048, + .partitions = 1, + .line_size = 64, + .flags = CPUINFO_CACHE_INCLUSIVE, + }; + break; + case 0xD7: + /* + * Intel ISA Reference: + * "3rd-level cache: 2 MByte, 8-way set associative, 64 byte line size" + * Application Note 485: + * "3rd-level cache: 2-MB, 8-way set associative, 64-byte line size" + */ + cache->l3 = (struct cpuinfo_x86_cache) { + .size = 2 * 1024 * 1024, + .associativity = 8, + .sets = 4096, + .partitions = 1, + .line_size = 64, + .flags = CPUINFO_CACHE_INCLUSIVE, + }; + break; + case 0xD8: + /* + * Intel ISA Reference: + * "3rd-level cache: 4 MByte, 8-way set associative, 64 byte line size" + * Application Note 485: + * "3rd-level cache: 4-MB, 8-way set associative, 64-byte line size" + */ + cache->l3 = (struct cpuinfo_x86_cache) { + .size = 4 * 1024 * 1024, + .associativity = 8, + .sets = 8192, + .partitions = 1, + .line_size = 64, + .flags = CPUINFO_CACHE_INCLUSIVE, + }; + break; + case 0xDC: + /* + * Intel ISA Reference: + * "3rd-level cache: 1.5 MByte, 12-way set associative, 64 byte line size" + * Application Note 485: + * "3rd-level cache: 1.5-MB, 12-way set associative, 64-byte line size" + */ + cache->l3 = (struct cpuinfo_x86_cache) { + .size = 3 * 512 * 1024, + .associativity = 12, + .sets = 2048, + .partitions = 1, + .line_size = 64, + .flags = CPUINFO_CACHE_INCLUSIVE, + }; + break; + case 0xDD: + /* + * Intel ISA Reference: + * "3rd-level cache: 3 MByte, 12-way set associative, 64 byte line size" + * Application Note 485: + * "3rd-level cache: 3-MB, 12-way set associative, 64-byte line size" + */ + cache->l3 = (struct cpuinfo_x86_cache) { + .size = 3 * 1024 * 1024, + .associativity = 12, + .sets = 4096, + .partitions = 1, + .line_size = 64, + .flags = CPUINFO_CACHE_INCLUSIVE, + }; + break; + case 0xDE: + /* + * Intel ISA Reference: + * "3rd-level cache: 6 MByte, 12-way set associative, 64 byte line size" + * Application Note 485: + * "3rd-level cache: 6-MB, 12-way set associative, 64-byte line size" + */ + cache->l3 = (struct cpuinfo_x86_cache) { + .size = 6 * 1024 * 1024, + .associativity = 12, + .sets = 8192, + .partitions = 1, + .line_size = 64, + .flags = CPUINFO_CACHE_INCLUSIVE, + }; + break; + case 0xE2: + /* + * Intel ISA Reference: + * "3rd-level cache: 2 MByte, 16-way set associative, 64 byte line size" + * Application Note 485: + * "3rd-level cache: 2-MB, 16-way set associative, 64-byte line size" + */ + cache->l3 = (struct cpuinfo_x86_cache) { + .size = 2 * 1024 * 1024, + .associativity = 16, + .sets = 2048, + .partitions = 1, + .line_size = 64, + .flags = CPUINFO_CACHE_INCLUSIVE, + }; + break; + case 0xE3: + /* + * Intel ISA Reference: + * "3rd-level cache: 4 MByte, 16-way set associative, 64 byte line size" + * Application Note 485: + * "3rd-level cache: 4-MB, 16-way set associative, 64-byte line size" + */ + cache->l3 = (struct cpuinfo_x86_cache) { + .size = 4 * 1024 * 1024, + .associativity = 16, + .sets = 4096, + .partitions = 1, + .line_size = 64, + .flags = CPUINFO_CACHE_INCLUSIVE, + }; + break; + case 0xE4: + /* + * Intel ISA Reference: + * "3rd-level cache: 8 MByte, 16-way set associative, 64 byte line size" + * Application Note 485: + * "3rd-level cache: 8-MB, 16-way set associative, 64-byte line size" + */ + cache->l3 = (struct cpuinfo_x86_cache) { + .size = 8 * 1024 * 1024, + .associativity = 16, + .sets = 8192, + .partitions = 1, + .line_size = 64, + .flags = CPUINFO_CACHE_INCLUSIVE, + }; + break; + case 0xEA: + /* + * Intel ISA Reference: + * "3rd-level cache: 12MByte, 24-way set associative, 64 byte line size" + * Application Note 485: + * "3rd-level cache: 12-MB, 24-way set associative, 64-byte line size" + */ + cache->l3 = (struct cpuinfo_x86_cache) { + .size = 12 * 1024 * 1024, + .associativity = 24, + .sets = 8192, + .partitions = 1, + .line_size = 64, + .flags = CPUINFO_CACHE_INCLUSIVE, + }; + break; + case 0xEB: + /* + * Intel ISA Reference: + * "3rd-level cache: 18MByte, 24-way set associative, 64 byte line size" + * Application Note 485: + * "3rd-level cache: 18-MB, 24-way set associative, 64-byte line size" + */ + cache->l3 = (struct cpuinfo_x86_cache) { + .size = 18 * 1024 * 1024, + .associativity = 24, + .sets = 12288, + .partitions = 1, + .line_size = 64, + .flags = CPUINFO_CACHE_INCLUSIVE, + }; + break; + case 0xEC: + /* + * Intel ISA Reference: + * "3rd-level cache: 24MByte, 24-way set associative, 64 byte line size" + * Application Note 485: + * "3rd-level cache: 24-MB, 24-way set associative, 64-byte line size" + */ + cache->l3 = (struct cpuinfo_x86_cache) { + .size = 24 * 1024 * 1024, + .associativity = 24, + .sets = 16384, + .partitions = 1, + .line_size = 64, + .flags = CPUINFO_CACHE_INCLUSIVE, + }; + break; + case 0xF0: + /* + * Intel ISA Reference: + * "64-Byte prefetching" + * Application Note 485: + * "64-byte Prefetching" + */ + cache->prefetch_size = 64; + break; + case 0xF1: + /* + * Intel ISA Reference: + * "128-Byte prefetching" + * Application Note 485: + * "128-byte Prefetching" + */ + cache->prefetch_size = 128; + break; + } +} diff --git a/3rdparty/cpuinfo/src/x86/cache/deterministic.c b/3rdparty/cpuinfo/src/x86/cache/deterministic.c new file mode 100644 index 000000000..befd5029a --- /dev/null +++ b/3rdparty/cpuinfo/src/x86/cache/deterministic.c @@ -0,0 +1,257 @@ +#include + +#include +#include +#include +#include + + +enum cache_type { + cache_type_none = 0, + cache_type_data = 1, + cache_type_instruction = 2, + cache_type_unified = 3, +}; + +bool cpuinfo_x86_decode_deterministic_cache_parameters( + struct cpuid_regs regs, + struct cpuinfo_x86_caches* cache, + uint32_t* package_cores_max) +{ + const uint32_t type = regs.eax & UINT32_C(0x1F); + if (type == cache_type_none) { + return false; + } + + /* Level starts at 1 */ + const uint32_t level = (regs.eax >> 5) & UINT32_C(0x7); + + const uint32_t sets = 1 + regs.ecx; + const uint32_t line_size = 1 + (regs.ebx & UINT32_C(0x00000FFF)); + const uint32_t partitions = 1 + ((regs.ebx >> 12) & UINT32_C(0x000003FF)); + const uint32_t associativity = 1 + (regs.ebx >> 22); + + *package_cores_max = 1 + (regs.eax >> 26); + const uint32_t processors = 1 + ((regs.eax >> 14) & UINT32_C(0x00000FFF)); + const uint32_t apic_bits = bit_length(processors); + + uint32_t flags = 0; + if (regs.edx & UINT32_C(0x00000002)) { + flags |= CPUINFO_CACHE_INCLUSIVE; + } + if (regs.edx & UINT32_C(0x00000004)) { + flags |= CPUINFO_CACHE_COMPLEX_INDEXING; + } + switch (level) { + case 1: + switch (type) { + case cache_type_unified: + cache->l1d = cache->l1i = (struct cpuinfo_x86_cache) { + .size = associativity * partitions * line_size * sets, + .associativity = associativity, + .sets = sets, + .partitions = partitions, + .line_size = line_size, + .flags = flags | CPUINFO_CACHE_UNIFIED, + .apic_bits = apic_bits + }; + break; + case cache_type_data: + cache->l1d = (struct cpuinfo_x86_cache) { + .size = associativity * partitions * line_size * sets, + .associativity = associativity, + .sets = sets, + .partitions = partitions, + .line_size = line_size, + .flags = flags, + .apic_bits = apic_bits + }; + break; + case cache_type_instruction: + cache->l1i = (struct cpuinfo_x86_cache) { + .size = associativity * partitions * line_size * sets, + .associativity = associativity, + .sets = sets, + .partitions = partitions, + .line_size = line_size, + .flags = flags, + .apic_bits = apic_bits + }; + break; + } + break; + case 2: + switch (type) { + case cache_type_instruction: + cpuinfo_log_warning("unexpected L2 instruction cache reported in leaf 0x00000004 is ignored"); + break; + case cache_type_unified: + flags |= CPUINFO_CACHE_UNIFIED; + case cache_type_data: + cache->l2 = (struct cpuinfo_x86_cache) { + .size = associativity * partitions * line_size * sets, + .associativity = associativity, + .sets = sets, + .partitions = partitions, + .line_size = line_size, + .flags = flags, + .apic_bits = apic_bits + }; + break; + } + break; + case 3: + switch (type) { + case cache_type_instruction: + cpuinfo_log_warning("unexpected L3 instruction cache reported in leaf 0x00000004 is ignored"); + break; + case cache_type_unified: + flags |= CPUINFO_CACHE_UNIFIED; + case cache_type_data: + cache->l3 = (struct cpuinfo_x86_cache) { + .size = associativity * partitions * line_size * sets, + .associativity = associativity, + .sets = sets, + .partitions = partitions, + .line_size = line_size, + .flags = flags, + .apic_bits = apic_bits + }; + break; + } + break; + case 4: + switch (type) { + case cache_type_instruction: + cpuinfo_log_warning("unexpected L4 instruction cache reported in leaf 0x00000004 is ignored"); + break; + case cache_type_unified: + flags |= CPUINFO_CACHE_UNIFIED; + case cache_type_data: + cache->l4 = (struct cpuinfo_x86_cache) { + .size = associativity * partitions * line_size * sets, + .associativity = associativity, + .sets = sets, + .partitions = partitions, + .line_size = line_size, + .flags = flags, + .apic_bits = apic_bits + }; + break; + } + break; + default: + cpuinfo_log_warning("unexpected L%"PRIu32" cache reported in leaf 0x00000004 is ignored", level); + break; + } + return true; +} + + +bool cpuinfo_x86_decode_cache_properties( + struct cpuid_regs regs, + struct cpuinfo_x86_caches* cache) +{ + const uint32_t type = regs.eax & UINT32_C(0x1F); + if (type == cache_type_none) { + return false; + } + + const uint32_t level = (regs.eax >> 5) & UINT32_C(0x7); + const uint32_t cores = 1 + ((regs.eax >> 14) & UINT32_C(0x00000FFF)); + const uint32_t apic_bits = bit_length(cores); + + const uint32_t sets = 1 + regs.ecx; + const uint32_t line_size = 1 + (regs.ebx & UINT32_C(0x00000FFF)); + const uint32_t partitions = 1 + ((regs.ebx >> 12) & UINT32_C(0x000003FF)); + const uint32_t associativity = 1 + (regs.ebx >> 22); + + uint32_t flags = 0; + if (regs.edx & UINT32_C(0x00000002)) { + flags |= CPUINFO_CACHE_INCLUSIVE; + } + + switch (level) { + case 1: + switch (type) { + case cache_type_unified: + cache->l1d = cache->l1i = (struct cpuinfo_x86_cache) { + .size = associativity * partitions * line_size * sets, + .associativity = associativity, + .sets = sets, + .partitions = partitions, + .line_size = line_size, + .flags = flags | CPUINFO_CACHE_UNIFIED, + .apic_bits = apic_bits + }; + break; + case cache_type_data: + cache->l1d = (struct cpuinfo_x86_cache) { + .size = associativity * partitions * line_size * sets, + .associativity = associativity, + .sets = sets, + .partitions = partitions, + .line_size = line_size, + .flags = flags, + .apic_bits = apic_bits + }; + break; + case cache_type_instruction: + cache->l1i = (struct cpuinfo_x86_cache) { + .size = associativity * partitions * line_size * sets, + .associativity = associativity, + .sets = sets, + .partitions = partitions, + .line_size = line_size, + .flags = flags, + .apic_bits = apic_bits + }; + break; + } + break; + case 2: + switch (type) { + case cache_type_instruction: + cpuinfo_log_warning("unexpected L2 instruction cache reported in leaf 0x8000001D is ignored"); + break; + case cache_type_unified: + flags |= CPUINFO_CACHE_UNIFIED; + case cache_type_data: + cache->l2 = (struct cpuinfo_x86_cache) { + .size = associativity * partitions * line_size * sets, + .associativity = associativity, + .sets = sets, + .partitions = partitions, + .line_size = line_size, + .flags = flags, + .apic_bits = apic_bits + }; + break; + } + break; + case 3: + switch (type) { + case cache_type_instruction: + cpuinfo_log_warning("unexpected L3 instruction cache reported in leaf 0x8000001D is ignored"); + break; + case cache_type_unified: + flags |= CPUINFO_CACHE_UNIFIED; + case cache_type_data: + cache->l3 = (struct cpuinfo_x86_cache) { + .size = associativity * partitions * line_size * sets, + .associativity = associativity, + .sets = sets, + .partitions = partitions, + .line_size = line_size, + .flags = flags, + .apic_bits = apic_bits + }; + break; + } + break; + default: + cpuinfo_log_warning("unexpected L%"PRIu32" cache reported in leaf 0x8000001D is ignored", level); + break; + } + return true; +} diff --git a/3rdparty/cpuinfo/src/x86/cache/init.c b/3rdparty/cpuinfo/src/x86/cache/init.c new file mode 100644 index 000000000..dd1f1ea37 --- /dev/null +++ b/3rdparty/cpuinfo/src/x86/cache/init.c @@ -0,0 +1,88 @@ +#include + +#include +#include +#include +#include +#include + + +union cpuinfo_x86_cache_descriptors { + struct cpuid_regs regs; + uint8_t as_bytes[16]; +}; + +enum cache_type { + cache_type_none = 0, + cache_type_data = 1, + cache_type_instruction = 2, + cache_type_unified = 3, +}; + +void cpuinfo_x86_detect_cache( + uint32_t max_base_index, uint32_t max_extended_index, + bool amd_topology_extensions, + enum cpuinfo_vendor vendor, + const struct cpuinfo_x86_model_info* model_info, + struct cpuinfo_x86_caches* cache, + struct cpuinfo_tlb* itlb_4KB, + struct cpuinfo_tlb* itlb_2MB, + struct cpuinfo_tlb* itlb_4MB, + struct cpuinfo_tlb* dtlb0_4KB, + struct cpuinfo_tlb* dtlb0_2MB, + struct cpuinfo_tlb* dtlb0_4MB, + struct cpuinfo_tlb* dtlb_4KB, + struct cpuinfo_tlb* dtlb_2MB, + struct cpuinfo_tlb* dtlb_4MB, + struct cpuinfo_tlb* dtlb_1GB, + struct cpuinfo_tlb* stlb2_4KB, + struct cpuinfo_tlb* stlb2_2MB, + struct cpuinfo_tlb* stlb2_1GB, + uint32_t* log2_package_cores_max) +{ + if (max_base_index >= 2) { + union cpuinfo_x86_cache_descriptors descriptors; + descriptors.regs = cpuid(2); + uint32_t iterations = (uint8_t) descriptors.as_bytes[0]; + if (iterations != 0) { +iterate_descriptors: + for (uint32_t i = 1 /* note: not 0 */; i < 16; i++) { + const uint8_t descriptor = descriptors.as_bytes[i]; + if (descriptor != 0) { + cpuinfo_x86_decode_cache_descriptor( + descriptor, vendor, model_info, + cache, + itlb_4KB, itlb_2MB, itlb_4MB, + dtlb0_4KB, dtlb0_2MB, dtlb0_4MB, + dtlb_4KB, dtlb_2MB, dtlb_4MB, dtlb_1GB, + stlb2_4KB, stlb2_2MB, stlb2_1GB, + &cache->prefetch_size); + } + } + if (--iterations != 0) { + descriptors.regs = cpuid(2); + goto iterate_descriptors; + } + } + + if (vendor != cpuinfo_vendor_amd && vendor != cpuinfo_vendor_hygon && max_base_index >= 4) { + struct cpuid_regs leaf4; + uint32_t input_ecx = 0; + uint32_t package_cores_max = 0; + do { + leaf4 = cpuidex(4, input_ecx++); + } while (cpuinfo_x86_decode_deterministic_cache_parameters( + leaf4, cache, &package_cores_max)); + if (package_cores_max != 0) { + *log2_package_cores_max = bit_length(package_cores_max); + } + } + } + if (amd_topology_extensions && max_extended_index >= UINT32_C(0x8000001D)) { + struct cpuid_regs leaf0x8000001D; + uint32_t input_ecx = 0; + do { + leaf0x8000001D = cpuidex(UINT32_C(0x8000001D), input_ecx++); + } while (cpuinfo_x86_decode_cache_properties(leaf0x8000001D, cache)); + } +} diff --git a/3rdparty/cpuinfo/src/x86/cpuid.h b/3rdparty/cpuinfo/src/x86/cpuid.h new file mode 100644 index 000000000..9e9e01318 --- /dev/null +++ b/3rdparty/cpuinfo/src/x86/cpuid.h @@ -0,0 +1,79 @@ +#pragma once +#include + +#if defined(__GNUC__) + #include +#elif defined(_MSC_VER) + #include +#endif + +#if CPUINFO_MOCK + #include +#endif +#include + + +#if defined(__GNUC__) || defined(_MSC_VER) + static inline struct cpuid_regs cpuid(uint32_t eax) { + #if CPUINFO_MOCK + uint32_t regs_array[4]; + cpuinfo_mock_get_cpuid(eax, regs_array); + return (struct cpuid_regs) { + .eax = regs_array[0], + .ebx = regs_array[1], + .ecx = regs_array[2], + .edx = regs_array[3], + }; + #else + struct cpuid_regs regs; + #if defined(__GNUC__) + __cpuid(eax, regs.eax, regs.ebx, regs.ecx, regs.edx); + #else + int regs_array[4]; + __cpuid(regs_array, (int) eax); + regs.eax = regs_array[0]; + regs.ebx = regs_array[1]; + regs.ecx = regs_array[2]; + regs.edx = regs_array[3]; + #endif + return regs; + #endif + } + + static inline struct cpuid_regs cpuidex(uint32_t eax, uint32_t ecx) { + #if CPUINFO_MOCK + uint32_t regs_array[4]; + cpuinfo_mock_get_cpuidex(eax, ecx, regs_array); + return (struct cpuid_regs) { + .eax = regs_array[0], + .ebx = regs_array[1], + .ecx = regs_array[2], + .edx = regs_array[3], + }; + #else + struct cpuid_regs regs; + #if defined(__GNUC__) + __cpuid_count(eax, ecx, regs.eax, regs.ebx, regs.ecx, regs.edx); + #else + int regs_array[4]; + __cpuidex(regs_array, (int) eax, (int) ecx); + regs.eax = regs_array[0]; + regs.ebx = regs_array[1]; + regs.ecx = regs_array[2]; + regs.edx = regs_array[3]; + #endif + return regs; + #endif + } +#endif + +static inline uint64_t xgetbv(uint32_t ext_ctrl_reg) { + #ifdef _MSC_VER + return (uint64_t)_xgetbv((unsigned int)ext_ctrl_reg); + #else + uint32_t lo, hi; + __asm__(".byte 0x0F, 0x01, 0xD0" : "=a" (lo), "=d" (hi) : "c" (ext_ctrl_reg)); + return ((uint64_t) hi << 32) | (uint64_t) lo; + #endif +} + diff --git a/3rdparty/cpuinfo/src/x86/info.c b/3rdparty/cpuinfo/src/x86/info.c new file mode 100644 index 000000000..ceb6b8457 --- /dev/null +++ b/3rdparty/cpuinfo/src/x86/info.c @@ -0,0 +1,19 @@ +#include + +#include +#include + + +struct cpuinfo_x86_model_info cpuinfo_x86_decode_model_info(uint32_t eax) { + struct cpuinfo_x86_model_info model_info; + model_info.stepping = eax & 0xF; + model_info.base_model = (eax >> 4) & 0xF; + model_info.base_family = (eax >> 8) & 0xF; + model_info.processor_type = (eax >> 12) & 0x3; + model_info.extended_model = (eax >> 16) & 0xF; + model_info.extended_family = (eax >> 20) & 0xFF; + + model_info.family = model_info.base_family + model_info.extended_family; + model_info.model = model_info.base_model + (model_info.extended_model << 4); + return model_info; +} diff --git a/3rdparty/cpuinfo/src/x86/init.c b/3rdparty/cpuinfo/src/x86/init.c new file mode 100644 index 000000000..244359cdd --- /dev/null +++ b/3rdparty/cpuinfo/src/x86/init.c @@ -0,0 +1,75 @@ +#include +#include + +#include +#include +#include +#include +#include +#include + + +struct cpuinfo_x86_isa cpuinfo_isa = { 0 }; +CPUINFO_INTERNAL uint32_t cpuinfo_x86_clflush_size = 0; + +void cpuinfo_x86_init_processor(struct cpuinfo_x86_processor* processor) { + const struct cpuid_regs leaf0 = cpuid(0); + const uint32_t max_base_index = leaf0.eax; + const enum cpuinfo_vendor vendor = processor->vendor = + cpuinfo_x86_decode_vendor(leaf0.ebx, leaf0.ecx, leaf0.edx); + + const struct cpuid_regs leaf0x80000000 = cpuid(UINT32_C(0x80000000)); + const uint32_t max_extended_index = + leaf0x80000000.eax >= UINT32_C(0x80000000) ? leaf0x80000000.eax : 0; + + const struct cpuid_regs leaf0x80000001 = max_extended_index >= UINT32_C(0x80000001) ? + cpuid(UINT32_C(0x80000001)) : (struct cpuid_regs) { 0, 0, 0, 0 }; + + if (max_base_index >= 1) { + const struct cpuid_regs leaf1 = cpuid(1); + processor->cpuid = leaf1.eax; + + const struct cpuinfo_x86_model_info model_info = cpuinfo_x86_decode_model_info(leaf1.eax); + const enum cpuinfo_uarch uarch = processor->uarch = + cpuinfo_x86_decode_uarch(vendor, &model_info); + + cpuinfo_x86_clflush_size = ((leaf1.ebx >> 8) & UINT32_C(0x000000FF)) * 8; + + /* + * Topology extensions support: + * - AMD: ecx[bit 22] in extended info (reserved bit on Intel CPUs). + */ + const bool amd_topology_extensions = !!(leaf0x80000001.ecx & UINT32_C(0x00400000)); + + cpuinfo_x86_detect_cache( + max_base_index, max_extended_index, amd_topology_extensions, vendor, &model_info, + &processor->cache, + &processor->tlb.itlb_4KB, + &processor->tlb.itlb_2MB, + &processor->tlb.itlb_4MB, + &processor->tlb.dtlb0_4KB, + &processor->tlb.dtlb0_2MB, + &processor->tlb.dtlb0_4MB, + &processor->tlb.dtlb_4KB, + &processor->tlb.dtlb_2MB, + &processor->tlb.dtlb_4MB, + &processor->tlb.dtlb_1GB, + &processor->tlb.stlb2_4KB, + &processor->tlb.stlb2_2MB, + &processor->tlb.stlb2_1GB, + &processor->topology.core_bits_length); + + cpuinfo_x86_detect_topology(max_base_index, max_extended_index, leaf1, &processor->topology); + + cpuinfo_isa = cpuinfo_x86_detect_isa(leaf1, leaf0x80000001, + max_base_index, max_extended_index, vendor, uarch); + } + if (max_extended_index >= UINT32_C(0x80000004)) { + struct cpuid_regs brand_string[3]; + for (uint32_t i = 0; i < 3; i++) { + brand_string[i] = cpuid(UINT32_C(0x80000002) + i); + } + memcpy(processor->brand_string, brand_string, sizeof(processor->brand_string)); + cpuinfo_log_debug("raw CPUID brand string: \"%48s\"", processor->brand_string); + } +} diff --git a/3rdparty/cpuinfo/src/x86/isa.c b/3rdparty/cpuinfo/src/x86/isa.c new file mode 100644 index 000000000..f2e5a281b --- /dev/null +++ b/3rdparty/cpuinfo/src/x86/isa.c @@ -0,0 +1,724 @@ +#include +#include +#include + +#include +#include + + +#if CPUINFO_ARCH_X86 + #ifdef _MSC_VER + #pragma pack(push, 2) + #endif + struct fxsave_region { + uint16_t fpu_control_word; + uint16_t fpu_status_word; + uint16_t fpu_tag_word; + uint16_t fpu_opcode; + uint32_t fpu_instruction_pointer_offset; + uint32_t fpu_instruction_pointer_selector; + uint32_t fpu_operand_pointer_offset; + uint32_t fpu_operand_pointer_selector; + uint32_t mxcsr_state; + uint32_t mxcsr_mask; + uint64_t fpu_registers[8 * 2]; + uint64_t xmm_registers[8 * 2]; + uint64_t padding[28]; + } + #ifndef _MSC_VER + __attribute__((__aligned__(16), __packed__)) + #endif + ; /* end of fxsave_region structure */ + #ifdef _MSC_VER + #pragma pack(pop, 2) + #endif +#endif + + +struct cpuinfo_x86_isa cpuinfo_x86_detect_isa( + const struct cpuid_regs basic_info, const struct cpuid_regs extended_info, + uint32_t max_base_index, uint32_t max_extended_index, + enum cpuinfo_vendor vendor, enum cpuinfo_uarch uarch) +{ + struct cpuinfo_x86_isa isa = { 0 }; + + const struct cpuid_regs structured_feature_info0 = + (max_base_index >= 7) ? cpuidex(7, 0) : (struct cpuid_regs) { 0, 0, 0, 0}; + const struct cpuid_regs structured_feature_info1 = + (max_base_index >= 7) ? cpuidex(7, 1) : (struct cpuid_regs) { 0, 0, 0, 0}; + + const uint32_t processor_capacity_info_index = UINT32_C(0x80000008); + const struct cpuid_regs processor_capacity_info = + (max_extended_index >= processor_capacity_info_index) ? + cpuid(processor_capacity_info_index) : (struct cpuid_regs) { 0, 0, 0, 0 }; + + bool avx_regs = false, avx512_regs = false, mpx_regs = false; + /* + * OSXSAVE: Operating system enabled XSAVE instructions for application use: + * - Intel, AMD: ecx[bit 26] in basic info = XSAVE/XRSTOR instructions supported by a chip. + * - Intel, AMD: ecx[bit 27] in basic info = XSAVE/XRSTOR instructions enabled by OS. + */ + const uint32_t osxsave_mask = UINT32_C(0x0C000000); + if ((basic_info.ecx & osxsave_mask) == osxsave_mask) { + uint64_t xcr0_valid_bits = 0; + if (max_base_index >= 0xD) { + const struct cpuid_regs regs = cpuidex(0xD, 0); + xcr0_valid_bits = ((uint64_t) regs.edx << 32) | regs.eax; + } + + const uint64_t xfeature_enabled_mask = xgetbv(0); + + /* + * AVX registers: + * - Intel, AMD: XFEATURE_ENABLED_MASK[bit 1] for low 128 bits of ymm registers + * - Intel, AMD: XFEATURE_ENABLED_MASK[bit 2] for high 128 bits of ymm registers + */ + const uint64_t avx_regs_mask = UINT64_C(0x0000000000000006); + if ((xcr0_valid_bits & avx_regs_mask) == avx_regs_mask) { + avx_regs = (xfeature_enabled_mask & avx_regs_mask) == avx_regs_mask; + } + + /* + * AVX512 registers: + * - Intel, AMD: XFEATURE_ENABLED_MASK[bit 1] for low 128 bits of zmm registers + * - Intel, AMD: XFEATURE_ENABLED_MASK[bit 2] for bits 128-255 of zmm registers + * - Intel: XFEATURE_ENABLED_MASK[bit 5] for 8 64-bit OpMask registers (k0-k7) + * - Intel: XFEATURE_ENABLED_MASK[bit 6] for the high 256 bits of the zmm registers zmm0-zmm15 + * - Intel: XFEATURE_ENABLED_MASK[bit 7] for the 512-bit zmm registers zmm16-zmm31 + */ + const uint64_t avx512_regs_mask = UINT64_C(0x00000000000000E6); + if ((xcr0_valid_bits & avx512_regs_mask) == avx512_regs_mask) { + avx512_regs = (xfeature_enabled_mask & avx512_regs_mask) == avx512_regs_mask; + } + + /* + * MPX registers: + * - Intel: XFEATURE_ENABLED_MASK[bit 3] for BNDREGS + * - Intel: XFEATURE_ENABLED_MASK[bit 4] for BNDCSR + */ + const uint64_t mpx_regs_mask = UINT64_C(0x0000000000000018); + if ((xcr0_valid_bits & mpx_regs_mask) == mpx_regs_mask) { + mpx_regs = (xfeature_enabled_mask & mpx_regs_mask) == mpx_regs_mask; + } + } + +#if CPUINFO_ARCH_X86 + /* + * RDTSC instruction: + * - Intel, AMD: edx[bit 4] in basic info. + * - AMD: edx[bit 4] in extended info (reserved bit on Intel CPUs). + */ + isa.rdtsc = !!((basic_info.edx | extended_info.edx) & UINT32_C(0x00000010)); +#endif + + /* + * SYSENTER/SYSEXIT instructions: + * - Intel, AMD: edx[bit 11] in basic info. + */ + isa.sysenter = !!(basic_info.edx & UINT32_C(0x00000800)); + +#if CPUINFO_ARCH_X86 + /* + * SYSCALL/SYSRET instructions: + * - Intel, AMD: edx[bit 11] in extended info. + */ + isa.syscall = !!(extended_info.edx & UINT32_C(0x00000800)); +#endif + + /* + * RDMSR/WRMSR instructions: + * - Intel, AMD: edx[bit 5] in basic info. + * - AMD: edx[bit 5] in extended info (reserved bit on Intel CPUs). + */ + isa.msr = !!((basic_info.edx | extended_info.edx) & UINT32_C(0x00000020)); + + /* + * CLZERO instruction: + * - AMD: ebx[bit 0] in processor capacity info (reserved bit on Intel CPUs). + */ + isa.clzero = !!(processor_capacity_info.ebx & UINT32_C(0x00000001)); + + /* + * CLFLUSH instruction: + * - Intel, AMD: edx[bit 19] in basic info. + */ + isa.clflush = !!(basic_info.edx & UINT32_C(0x00080000)); + + /* + * CLFLUSHOPT instruction: + * - Intel: ebx[bit 23] in structured feature info (ecx = 0). + */ + isa.clflushopt = !!(structured_feature_info0.ebx & UINT32_C(0x00800000)); + + /* + * MWAIT/MONITOR instructions: + * - Intel, AMD: ecx[bit 3] in basic info. + */ + isa.mwait = !!(basic_info.ecx & UINT32_C(0x00000008)); + + /* + * MWAITX/MONITORX instructions: + * - AMD: ecx[bit 29] in extended info. + */ + isa.mwaitx = !!(extended_info.ecx & UINT32_C(0x20000000)); + + /* + * FXSAVE/FXRSTOR instructions: + * - Intel, AMD: edx[bit 24] in basic info. + * - AMD: edx[bit 24] in extended info (zero bit on Intel CPUs, EMMX bit on Cyrix CPUs). + */ + switch (vendor) { +#if CPUINFO_ARCH_X86 + case cpuinfo_vendor_cyrix: + case cpuinfo_vendor_nsc: + isa.emmx = !!(extended_info.edx & UINT32_C(0x01000000)); + break; +#endif + default: + isa.fxsave = !!((basic_info.edx | extended_info.edx) & UINT32_C(0x01000000)); + break; + } + + /* + * XSAVE/XRSTOR instructions: + * - Intel, AMD: ecx[bit 26] in basic info. + */ + isa.xsave = !!(basic_info.ecx & UINT32_C(0x04000000)); + +#if CPUINFO_ARCH_X86 + /* + * x87 FPU instructions: + * - Intel, AMD: edx[bit 0] in basic info. + * - AMD: edx[bit 0] in extended info (reserved bit on Intel CPUs). + */ + isa.fpu = !!((basic_info.edx | extended_info.edx) & UINT32_C(0x00000001)); + + /* + * MMX instructions: + * - Intel, AMD: edx[bit 23] in basic info. + * - AMD: edx[bit 23] in extended info (zero bit on Intel CPUs). + */ + isa.mmx = !!((basic_info.edx | extended_info.edx) & UINT32_C(0x00800000)); + + /* + * MMX+/Integer SSE instructions: + * - Intel, AMD: edx[bit 25] in basic info (SSE feature flag). + * - Pre-SSE AMD: edx[bit 22] in extended info (zero bit on Intel CPUs). + */ + isa.mmx_plus = !!((basic_info.edx & UINT32_C(0x02000000)) | (extended_info.edx & UINT32_C(0x00400000))); +#endif + + /* + * 3dnow! instructions: + * - AMD: edx[bit 31] of extended info (zero bit on Intel CPUs). + */ + isa.three_d_now = !!(extended_info.edx & UINT32_C(0x80000000)); + + /* + * 3dnow!+ instructions: + * - AMD: edx[bit 30] of extended info (zero bit on Intel CPUs). + */ + isa.three_d_now_plus = !!(extended_info.edx & UINT32_C(0x40000000)); + +#if CPUINFO_ARCH_X86 + /* + * 3dnow! Geode instructions: + * - No CPUID bit, detect as Geode microarchitecture + 3dnow!+ support + */ + isa.three_d_now_geode = isa.three_d_now_plus && (uarch == cpuinfo_uarch_geode); +#endif + + /* + * PREFETCH instruction: + * - AMD: ecx[bit 8] of extended info (one of 3dnow! prefetch instructions). + * On Intel this bit indicates PREFETCHW, but not PREFETCH support. + * - AMD: edx[bit 31] of extended info (implied by 3dnow! support). Reserved bit on Intel CPUs. + * - AMD: edx[bit 30] of extended info (implied by 3dnow!+ support). Reserved bit on Intel CPUs. + * - AMD: edx[bit 29] of extended info (x86-64 support). Does not imply PREFETCH support on non-AMD CPUs!!! + */ + switch (vendor) { + case cpuinfo_vendor_intel: + /* + * Instruction is not documented in the manual, + * and the 3dnow! prefetch CPUID bit indicates PREFETCHW instruction. + */ + break; + case cpuinfo_vendor_amd: + case cpuinfo_vendor_hygon: + isa.prefetch = !!((extended_info.ecx & UINT32_C(0x00000100)) | (extended_info.edx & UINT32_C(0xE0000000))); + break; + default: + /* + * Conservatively assume, that 3dnow!/3dnow!+ support implies PREFETCH support, but + * 3dnow! prefetch CPUID bit follows Intel spec (PREFETCHW, but not PREFETCH). + */ + isa.prefetch = !!(extended_info.edx & UINT32_C(0xC0000000)); + break; + } + + /* + * PREFETCHW instruction: + * - AMD: ecx[bit 8] of extended info (one of 3dnow! prefetch instructions). + * - Intel: ecx[bit 8] of extended info (PREFETCHW instruction only). + * - AMD: edx[bit 31] of extended info (implied by 3dnow! support). Reserved bit on Intel CPUs. + * - AMD: edx[bit 30] of extended info (implied by 3dnow!+ support). Reserved bit on Intel CPUs. + * - AMD: edx[bit 29] of extended info (x86-64 support). Does not imply PREFETCHW support on non-AMD CPUs!!! + */ + switch (vendor) { + case cpuinfo_vendor_amd: + case cpuinfo_vendor_hygon: + isa.prefetchw = !!((extended_info.ecx & UINT32_C(0x00000100)) | (extended_info.edx & UINT32_C(0xE0000000))); + break; + default: + /* Assume, that 3dnow!/3dnow!+ support implies PREFETCHW support, not implications from x86-64 support */ + isa.prefetchw = !!((extended_info.ecx & UINT32_C(0x00000100)) | (extended_info.edx & UINT32_C(0xC0000000))); + break; + } + + /* + * PREFETCHWT1 instruction: + * - Intel: ecx[bit 0] of structured feature info (ecx = 0). Reserved bit on AMD. + */ + isa.prefetchwt1 = !!(structured_feature_info0.ecx & UINT32_C(0x00000001)); + +#if CPUINFO_ARCH_X86 + /* + * SSE instructions: + * - Intel, AMD: edx[bit 25] in basic info. + */ + isa.sse = !!(basic_info.edx & UINT32_C(0x02000000)); + + /* + * SSE2 instructions: + * - Intel, AMD: edx[bit 26] in basic info. + */ + isa.sse2 = !!(basic_info.edx & UINT32_C(0x04000000)); +#endif + + /* + * SSE3 instructions: + * - Intel, AMD: ecx[bit 0] in basic info. + */ + isa.sse3 = !!(basic_info.ecx & UINT32_C(0x00000001)); + +#if CPUINFO_ARCH_X86 + /* + * CPUs with x86-64 or SSE3 always support DAZ (denormals-as-zero) mode. + * Only early Pentium 4 models may not support it. + */ + if (isa.sse3) { + isa.daz = true; + } else { + /* Detect DAZ support from masked MXCSR bits */ + if (isa.sse && isa.fxsave) { + struct fxsave_region region = { 0 }; + #ifdef _MSC_VER + _fxsave(®ion); + #else + __asm__ __volatile__ ("fxsave %[region];" : [region] "+m" (region)); + #endif + + /* + * Denormals-as-zero (DAZ) flag: + * - Intel, AMD: MXCSR[bit 6] + */ + isa.daz = !!(region.mxcsr_mask & UINT32_C(0x00000040)); + } + } +#endif + + /* + * SSSE3 instructions: + * - Intel, AMD: ecx[bit 9] in basic info. + */ + isa.ssse3 = !!(basic_info.ecx & UINT32_C(0x0000200)); + + + /* + * SSE4.1 instructions: + * - Intel, AMD: ecx[bit 19] in basic info. + */ + isa.sse4_1 = !!(basic_info.ecx & UINT32_C(0x00080000)); + + /* + * SSE4.2 instructions: + * - Intel: ecx[bit 20] in basic info (reserved bit on AMD CPUs). + */ + isa.sse4_2 = !!(basic_info.ecx & UINT32_C(0x00100000)); + + /* + * SSE4A instructions: + * - AMD: ecx[bit 6] in extended info (reserved bit on Intel CPUs). + */ + isa.sse4a = !!(extended_info.ecx & UINT32_C(0x00000040)); + + /* + * Misaligned memory operands in SSE instructions: + * - AMD: ecx[bit 7] in extended info (reserved bit on Intel CPUs). + */ + isa.misaligned_sse = !!(extended_info.ecx & UINT32_C(0x00000080)); + + /* + * AVX instructions: + * - Intel, AMD: ecx[bit 28] in basic info. + */ + isa.avx = avx_regs && !!(basic_info.ecx & UINT32_C(0x10000000)); + + /* + * FMA3 instructions: + * - Intel: ecx[bit 12] in basic info (reserved bit on AMD CPUs). + */ + isa.fma3 = avx_regs && !!(basic_info.ecx & UINT32_C(0x00001000)); + + /* + * FMA4 instructions: + * - AMD: ecx[bit 16] in extended info (reserved bit on Intel CPUs). + */ + isa.fma4 = avx_regs && !!(extended_info.ecx & UINT32_C(0x00010000)); + + /* + * XOP instructions: + * - AMD: ecx[bit 11] in extended info (reserved bit on Intel CPUs). + */ + isa.xop = avx_regs && !!(extended_info.ecx & UINT32_C(0x00000800)); + + /* + * F16C instructions: + * - Intel, AMD: ecx[bit 29] in basic info. + */ + isa.f16c = avx_regs && !!(basic_info.ecx & UINT32_C(0x20000000)); + + /* + * AVX2 instructions: + * - Intel: ebx[bit 5] in structured feature info (ecx = 0). + */ + isa.avx2 = avx_regs && !!(structured_feature_info0.ebx & UINT32_C(0x00000020)); + + /* + * AVX512F instructions: + * - Intel: ebx[bit 16] in structured feature info (ecx = 0). + */ + isa.avx512f = avx512_regs && !!(structured_feature_info0.ebx & UINT32_C(0x00010000)); + + /* + * AVX512PF instructions: + * - Intel: ebx[bit 26] in structured feature info (ecx = 0). + */ + isa.avx512pf = avx512_regs && !!(structured_feature_info0.ebx & UINT32_C(0x04000000)); + + /* + * AVX512ER instructions: + * - Intel: ebx[bit 27] in structured feature info (ecx = 0). + */ + isa.avx512er = avx512_regs && !!(structured_feature_info0.ebx & UINT32_C(0x08000000)); + + /* + * AVX512CD instructions: + * - Intel: ebx[bit 28] in structured feature info (ecx = 0). + */ + isa.avx512cd = avx512_regs && !!(structured_feature_info0.ebx & UINT32_C(0x10000000)); + + /* + * AVX512DQ instructions: + * - Intel: ebx[bit 17] in structured feature info (ecx = 0). + */ + isa.avx512dq = avx512_regs && !!(structured_feature_info0.ebx & UINT32_C(0x00020000)); + + /* + * AVX512BW instructions: + * - Intel: ebx[bit 30] in structured feature info (ecx = 0). + */ + isa.avx512bw = avx512_regs && !!(structured_feature_info0.ebx & UINT32_C(0x40000000)); + + /* + * AVX512VL instructions: + * - Intel: ebx[bit 31] in structured feature info (ecx = 0). + */ + isa.avx512vl = avx512_regs && !!(structured_feature_info0.ebx & UINT32_C(0x80000000)); + + /* + * AVX512IFMA instructions: + * - Intel: ebx[bit 21] in structured feature info (ecx = 0). + */ + isa.avx512ifma = avx512_regs && !!(structured_feature_info0.ebx & UINT32_C(0x00200000)); + + /* + * AVX512VBMI instructions: + * - Intel: ecx[bit 1] in structured feature info (ecx = 0). + */ + isa.avx512vbmi = avx512_regs && !!(structured_feature_info0.ecx & UINT32_C(0x00000002)); + + /* + * AVX512VBMI2 instructions: + * - Intel: ecx[bit 6] in structured feature info (ecx = 0). + */ + isa.avx512vbmi2 = avx512_regs && !!(structured_feature_info0.ecx & UINT32_C(0x00000040)); + + /* + * AVX512BITALG instructions: + * - Intel: ecx[bit 12] in structured feature info (ecx = 0). + */ + isa.avx512bitalg = avx512_regs && !!(structured_feature_info0.ecx & UINT32_C(0x00001000)); + + /* + * AVX512VPOPCNTDQ instructions: + * - Intel: ecx[bit 14] in structured feature info (ecx = 0). + */ + isa.avx512vpopcntdq = avx512_regs && !!(structured_feature_info0.ecx & UINT32_C(0x00004000)); + + /* + * AVX512VNNI instructions: + * - Intel: ecx[bit 11] in structured feature info (ecx = 0). + */ + isa.avx512vnni = avx512_regs && !!(structured_feature_info0.ecx & UINT32_C(0x00000800)); + + /* + * AVX512_4VNNIW instructions: + * - Intel: edx[bit 2] in structured feature info (ecx = 0). + */ + isa.avx512_4vnniw = avx512_regs && !!(structured_feature_info0.edx & UINT32_C(0x00000004)); + + /* + * AVX512_4FMAPS instructions: + * - Intel: edx[bit 3] in structured feature info (ecx = 0). + */ + isa.avx512_4fmaps = avx512_regs && !!(structured_feature_info0.edx & UINT32_C(0x00000008)); + + /* + * AVX512_VP2INTERSECT instructions: + * - Intel: edx[bit 8] in structured feature info (ecx = 0). + */ + isa.avx512vp2intersect = avx512_regs && !!(structured_feature_info0.edx & UINT32_C(0x00000100)); + + /* + * AVX512_BF16 instructions: + * - Intel: eax[bit 5] in structured feature info (ecx = 1). + */ + isa.avx512bf16 = avx512_regs && !!(structured_feature_info1.eax & UINT32_C(0x00000020)); + + /* + * HLE instructions: + * - Intel: ebx[bit 4] in structured feature info (ecx = 0). + */ + isa.hle = !!(structured_feature_info0.ebx & UINT32_C(0x00000010)); + + /* + * RTM instructions: + * - Intel: ebx[bit 11] in structured feature info (ecx = 0). + */ + isa.rtm = !!(structured_feature_info0.ebx & UINT32_C(0x00000800)); + + /* + * XTEST instruction: + * - Intel: either HLE or RTM is supported + */ + isa.xtest = isa.hle || isa.rtm; + + /* + * MPX registers and instructions: + * - Intel: ebx[bit 14] in structured feature info (ecx = 0). + */ + isa.mpx = mpx_regs && !!(structured_feature_info0.ebx & UINT32_C(0x00004000)); + +#if CPUINFO_ARCH_X86 + /* + * CMOV instructions: + * - Intel, AMD: edx[bit 15] in basic info. + * - AMD: edx[bit 15] in extended info (zero bit on Intel CPUs). + */ + isa.cmov = !!((basic_info.edx | extended_info.edx) & UINT32_C(0x00008000)); + + /* + * CMPXCHG8B instruction: + * - Intel, AMD: edx[bit 8] in basic info. + * - AMD: edx[bit 8] in extended info (reserved bit on Intel CPUs). + */ + isa.cmpxchg8b = !!((basic_info.edx | extended_info.edx) & UINT32_C(0x00000100)); +#endif + + /* + * CMPXCHG16B instruction: + * - Intel, AMD: ecx[bit 13] in basic info. + */ + isa.cmpxchg16b = !!(basic_info.ecx & UINT32_C(0x00002000)); + + /* + * CLWB instruction: + * - Intel: ebx[bit 24] in structured feature info (ecx = 0). + */ + isa.clwb = !!(structured_feature_info0.ebx & UINT32_C(0x01000000)); + + /* + * MOVBE instruction: + * - Intel: ecx[bit 22] in basic info. + */ + isa.movbe = !!(basic_info.ecx & UINT32_C(0x00400000)); + +#if CPUINFO_ARCH_X86_64 + /* + * Some early x86-64 CPUs lack LAHF & SAHF instructions. + * A special CPU feature bit must be checked to ensure their availability: + * - Intel, AMD: ecx[bit 0] in extended info. + */ + isa.lahf_sahf = !!(extended_info.ecx & UINT32_C(0x00000001)); +#endif + + /* + * RDFSBASE/RDGSBASE/WRFSBASE/WRGSBASE instructions. + * - Intel: ebx[bit 0] in structured feature info (ecx = 0). + */ + isa.fs_gs_base = !!(structured_feature_info0.ebx & UINT32_C(0x00000001)); + + /* + * LZCNT instruction: + * - Intel, AMD: ecx[bit 5] in extended info. + */ + isa.lzcnt = !!(extended_info.ecx & UINT32_C(0x00000020)); + + /* + * POPCNT instruction: + * - Intel, AMD: ecx[bit 23] in basic info. + */ + isa.popcnt = !!(basic_info.ecx & UINT32_C(0x00800000)); + + /* + * TBM instructions: + * - AMD: ecx[bit 21] in extended info (reserved bit on Intel CPUs). + */ + isa.tbm = !!(extended_info.ecx & UINT32_C(0x00200000)); + + /* + * BMI instructions: + * - Intel, AMD: ebx[bit 3] in structured feature info (ecx = 0). + */ + isa.bmi = !!(structured_feature_info0.ebx & UINT32_C(0x00000008)); + + /* + * BMI2 instructions: + * - Intel: ebx[bit 8] in structured feature info (ecx = 0). + */ + isa.bmi2 = !!(structured_feature_info0.ebx & UINT32_C(0x00000100)); + + /* + * ADCX/ADOX instructions: + * - Intel: ebx[bit 19] in structured feature info (ecx = 0). + */ + isa.adx = !!(structured_feature_info0.ebx & UINT32_C(0x00080000)); + + /* + * AES instructions: + * - Intel: ecx[bit 25] in basic info (reserved bit on AMD CPUs). + */ + isa.aes = !!(basic_info.ecx & UINT32_C(0x02000000)); + + /* + * VAES instructions: + * - Intel: ecx[bit 9] in structured feature info (ecx = 0). + */ + isa.vaes = !!(structured_feature_info0.ecx & UINT32_C(0x00000200)); + + /* + * PCLMULQDQ instruction: + * - Intel: ecx[bit 1] in basic info (reserved bit on AMD CPUs). + */ + isa.pclmulqdq = !!(basic_info.ecx & UINT32_C(0x00000002)); + + /* + * VPCLMULQDQ instruction: + * - Intel: ecx[bit 10] in structured feature info (ecx = 0). + */ + isa.vpclmulqdq = !!(structured_feature_info0.ecx & UINT32_C(0x00000400)); + + /* + * GFNI instructions: + * - Intel: ecx[bit 8] in structured feature info (ecx = 0). + */ + isa.gfni = !!(structured_feature_info0.ecx & UINT32_C(0x00000100)); + + /* + * RDRAND instruction: + * - Intel: ecx[bit 30] in basic info (reserved bit on AMD CPUs). + */ + isa.rdrand = !!(basic_info.ecx & UINT32_C(0x40000000)); + + /* + * RDSEED instruction: + * - Intel: ebx[bit 18] in structured feature info (ecx = 0). + */ + isa.rdseed = !!(structured_feature_info0.ebx & UINT32_C(0x00040000)); + + /* + * SHA instructions: + * - Intel: ebx[bit 29] in structured feature info (ecx = 0). + */ + isa.sha = !!(structured_feature_info0.ebx & UINT32_C(0x20000000)); + + if (vendor == cpuinfo_vendor_via) { + const struct cpuid_regs padlock_meta_info = cpuid(UINT32_C(0xC0000000)); + const uint32_t max_padlock_index = padlock_meta_info.eax; + const uint32_t padlock_info_index = UINT32_C(0xC0000001); + if (max_padlock_index >= padlock_info_index) { + const struct cpuid_regs padlock_info = cpuid(padlock_info_index); + + /* + * Padlock RNG extension: + * - VIA: edx[bit 2] in padlock info = RNG exists on chip flag. + * - VIA: edx[bit 3] in padlock info = RNG enabled by OS. + */ + const uint32_t padlock_rng_mask = UINT32_C(0x0000000C); + isa.rng = (padlock_info.edx & padlock_rng_mask) == padlock_rng_mask; + + /* + * Padlock ACE extension: + * - VIA: edx[bit 6] in padlock info = ACE exists on chip flag. + * - VIA: edx[bit 7] in padlock info = ACE enabled by OS. + */ + const uint32_t padlock_ace_mask = UINT32_C(0x000000C0); + isa.ace = (padlock_info.edx & padlock_ace_mask) == padlock_ace_mask; + + /* + * Padlock ACE 2 extension: + * - VIA: edx[bit 8] in padlock info = ACE2 exists on chip flag. + * - VIA: edx[bit 9] in padlock info = ACE 2 enabled by OS. + */ + const uint32_t padlock_ace2_mask = UINT32_C(0x00000300); + isa.ace2 = (padlock_info.edx & padlock_ace2_mask) == padlock_ace2_mask; + + /* + * Padlock PHE extension: + * - VIA: edx[bit 10] in padlock info = PHE exists on chip flag. + * - VIA: edx[bit 11] in padlock info = PHE enabled by OS. + */ + const uint32_t padlock_phe_mask = UINT32_C(0x00000C00); + isa.phe = (padlock_info.edx & padlock_phe_mask) == padlock_phe_mask; + + /* + * Padlock PMM extension: + * - VIA: edx[bit 12] in padlock info = PMM exists on chip flag. + * - VIA: edx[bit 13] in padlock info = PMM enabled by OS. + */ + const uint32_t padlock_pmm_mask = UINT32_C(0x00003000); + isa.pmm = (padlock_info.edx & padlock_pmm_mask) == padlock_pmm_mask; + } + } + + /* + * LWP instructions: + * - AMD: ecx[bit 15] in extended info (reserved bit on Intel CPUs). + */ + isa.lwp = !!(extended_info.ecx & UINT32_C(0x00008000)); + + /* + * RDTSCP instruction: + * - Intel, AMD: edx[bit 27] in extended info. + */ + isa.rdtscp = !!(extended_info.edx & UINT32_C(0x08000000)); + + /* + * RDPID instruction: + * - Intel: ecx[bit 22] in structured feature info (ecx = 0). + */ + isa.rdpid = !!(structured_feature_info0.ecx & UINT32_C(0x00400000)); + + return isa; +} diff --git a/3rdparty/cpuinfo/src/x86/linux/api.h b/3rdparty/cpuinfo/src/x86/linux/api.h new file mode 100644 index 000000000..1c9485b19 --- /dev/null +++ b/3rdparty/cpuinfo/src/x86/linux/api.h @@ -0,0 +1,20 @@ +#pragma once + +#include +#include + +#include +#include +#include +#include + + +struct cpuinfo_x86_linux_processor { + uint32_t apic_id; + uint32_t linux_id; + uint32_t flags; +}; + +CPUINFO_INTERNAL bool cpuinfo_x86_linux_parse_proc_cpuinfo( + uint32_t max_processors_count, + struct cpuinfo_x86_linux_processor processors[restrict static max_processors_count]); diff --git a/3rdparty/cpuinfo/src/x86/linux/cpuinfo.c b/3rdparty/cpuinfo/src/x86/linux/cpuinfo.c new file mode 100644 index 000000000..90ff81435 --- /dev/null +++ b/3rdparty/cpuinfo/src/x86/linux/cpuinfo.c @@ -0,0 +1,207 @@ +#include +#include +#include +#include +#include + +#include +#include +#include + +/* + * Size, in chars, of the on-stack buffer used for parsing lines of /proc/cpuinfo. + * This is also the limit on the length of a single line. + */ +#define BUFFER_SIZE 2048 + + +static uint32_t parse_processor_number( + const char* processor_start, + const char* processor_end) +{ + const size_t processor_length = (size_t) (processor_end - processor_start); + + if (processor_length == 0) { + cpuinfo_log_warning("Processor number in /proc/cpuinfo is ignored: string is empty"); + return 0; + } + + uint32_t processor_number = 0; + for (const char* digit_ptr = processor_start; digit_ptr != processor_end; digit_ptr++) { + const uint32_t digit = (uint32_t) (*digit_ptr - '0'); + if (digit > 10) { + cpuinfo_log_warning("non-decimal suffix %.*s in /proc/cpuinfo processor number is ignored", + (int) (processor_end - digit_ptr), digit_ptr); + break; + } + + processor_number = processor_number * 10 + digit; + } + + return processor_number; +} + +/* + * Decode APIC ID reported by Linux kernel for x86/x86-64 architecture. + * Example of APIC ID reported in /proc/cpuinfo: + * + * apicid : 2 + */ +static void parse_apic_id( + const char* apic_start, + const char* apic_end, + struct cpuinfo_x86_linux_processor processor[restrict static 1]) +{ + uint32_t apic_id = 0; + for (const char* digit_ptr = apic_start; digit_ptr != apic_end; digit_ptr++) { + const uint32_t digit = *digit_ptr - '0'; + if (digit >= 10) { + cpuinfo_log_warning("APIC ID %.*s in /proc/cpuinfo is ignored due to unexpected non-digit character '%c' at offset %zu", + (int) (apic_end - apic_start), apic_start, + *digit_ptr, (size_t) (digit_ptr - apic_start)); + return; + } + + apic_id = apic_id * 10 + digit; + } + + processor->apic_id = apic_id; + processor->flags |= CPUINFO_LINUX_FLAG_APIC_ID; +} + +struct proc_cpuinfo_parser_state { + uint32_t processor_index; + uint32_t max_processors_count; + struct cpuinfo_x86_linux_processor* processors; + struct cpuinfo_x86_linux_processor dummy_processor; +}; + +/* + * Decode a single line of /proc/cpuinfo information. + * Lines have format [ ]*:[ ] + */ +static bool parse_line( + const char* line_start, + const char* line_end, + struct proc_cpuinfo_parser_state state[restrict static 1], + uint64_t line_number) +{ + /* Empty line. Skip. */ + if (line_start == line_end) { + return true; + } + + /* Search for ':' on the line. */ + const char* separator = line_start; + for (; separator != line_end; separator++) { + if (*separator == ':') { + break; + } + } + /* Skip line if no ':' separator was found. */ + if (separator == line_end) { + cpuinfo_log_info("Line %.*s in /proc/cpuinfo is ignored: key/value separator ':' not found", + (int) (line_end - line_start), line_start); + return true; + } + + /* Skip trailing spaces in key part. */ + const char* key_end = separator; + for (; key_end != line_start; key_end--) { + if (key_end[-1] != ' ' && key_end[-1] != '\t') { + break; + } + } + /* Skip line if key contains nothing but spaces. */ + if (key_end == line_start) { + cpuinfo_log_info("Line %.*s in /proc/cpuinfo is ignored: key contains only spaces", + (int) (line_end - line_start), line_start); + return true; + } + + /* Skip leading spaces in value part. */ + const char* value_start = separator + 1; + for (; value_start != line_end; value_start++) { + if (*value_start != ' ') { + break; + } + } + /* Value part contains nothing but spaces. Skip line. */ + if (value_start == line_end) { + cpuinfo_log_info("Line %.*s in /proc/cpuinfo is ignored: value contains only spaces", + (int) (line_end - line_start), line_start); + return true; + } + + /* Skip trailing spaces in value part (if any) */ + const char* value_end = line_end; + for (; value_end != value_start; value_end--) { + if (value_end[-1] != ' ') { + break; + } + } + + const uint32_t processor_index = state->processor_index; + const uint32_t max_processors_count = state->max_processors_count; + struct cpuinfo_x86_linux_processor* processors = state->processors; + struct cpuinfo_x86_linux_processor* processor = &state->dummy_processor; + if (processor_index < max_processors_count) { + processor = &processors[processor_index]; + } + + const size_t key_length = key_end - line_start; + switch (key_length) { + case 6: + if (memcmp(line_start, "apicid", key_length) == 0) { + parse_apic_id(value_start, value_end, processor); + } else { + goto unknown; + } + break; + case 9: + if (memcmp(line_start, "processor", key_length) == 0) { + const uint32_t new_processor_index = parse_processor_number(value_start, value_end); + if (new_processor_index < processor_index) { + /* Strange: decreasing processor number */ + cpuinfo_log_warning( + "unexpectedly low processor number %"PRIu32" following processor %"PRIu32" in /proc/cpuinfo", + new_processor_index, processor_index); + } else if (new_processor_index > processor_index + 1) { + /* Strange, but common: skipped processor $(processor_index + 1) */ + cpuinfo_log_info( + "unexpectedly high processor number %"PRIu32" following processor %"PRIu32" in /proc/cpuinfo", + new_processor_index, processor_index); + } + if (new_processor_index >= max_processors_count) { + /* Log and ignore processor */ + cpuinfo_log_warning("processor %"PRIu32" in /proc/cpuinfo is ignored: index exceeds system limit %"PRIu32, + new_processor_index, max_processors_count - 1); + } else { + processors[new_processor_index].flags |= CPUINFO_LINUX_FLAG_PROC_CPUINFO; + } + state->processor_index = new_processor_index; + return true; + } else { + goto unknown; + } + break; + default: + unknown: + cpuinfo_log_debug("unknown /proc/cpuinfo key: %.*s", (int) key_length, line_start); + + } + return true; +} + +bool cpuinfo_x86_linux_parse_proc_cpuinfo( + uint32_t max_processors_count, + struct cpuinfo_x86_linux_processor processors[restrict static max_processors_count]) +{ + struct proc_cpuinfo_parser_state state = { + .processor_index = 0, + .max_processors_count = max_processors_count, + .processors = processors, + }; + return cpuinfo_linux_parse_multiline_file("/proc/cpuinfo", BUFFER_SIZE, + (cpuinfo_line_callback) parse_line, &state); +} diff --git a/3rdparty/cpuinfo/src/x86/linux/init.c b/3rdparty/cpuinfo/src/x86/linux/init.c new file mode 100644 index 000000000..f5657890e --- /dev/null +++ b/3rdparty/cpuinfo/src/x86/linux/init.c @@ -0,0 +1,629 @@ +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include + + +static inline uint32_t bit_mask(uint32_t bits) { + return (UINT32_C(1) << bits) - UINT32_C(1); +} + +static inline bool bitmask_all(uint32_t bitfield, uint32_t mask) { + return (bitfield & mask) == mask; +} + +static inline uint32_t min(uint32_t a, uint32_t b) { + return a < b ? a : b; +} + +static inline int cmp(uint32_t a, uint32_t b) { + return (a > b) - (a < b); +} + +static int cmp_x86_linux_processor(const void* ptr_a, const void* ptr_b) { + const struct cpuinfo_x86_linux_processor* processor_a = (const struct cpuinfo_x86_linux_processor*) ptr_a; + const struct cpuinfo_x86_linux_processor* processor_b = (const struct cpuinfo_x86_linux_processor*) ptr_b; + + /* Move usable processors towards the start of the array */ + const bool usable_a = bitmask_all(processor_a->flags, CPUINFO_LINUX_FLAG_VALID); + const bool usable_b = bitmask_all(processor_b->flags, CPUINFO_LINUX_FLAG_VALID); + if (usable_a != usable_b) { + return (int) usable_b - (int) usable_a; + } + + /* Compare based on APIC ID (i.e. processor 0 < processor 1) */ + const uint32_t id_a = processor_a->apic_id; + const uint32_t id_b = processor_b->apic_id; + return cmp(id_a, id_b); +} + +static void cpuinfo_x86_count_objects( + uint32_t linux_processors_count, + const struct cpuinfo_x86_linux_processor linux_processors[restrict static linux_processors_count], + const struct cpuinfo_x86_processor processor[restrict static 1], + uint32_t valid_processor_mask, + uint32_t llc_apic_bits, + uint32_t cores_count_ptr[restrict static 1], + uint32_t clusters_count_ptr[restrict static 1], + uint32_t packages_count_ptr[restrict static 1], + uint32_t l1i_count_ptr[restrict static 1], + uint32_t l1d_count_ptr[restrict static 1], + uint32_t l2_count_ptr[restrict static 1], + uint32_t l3_count_ptr[restrict static 1], + uint32_t l4_count_ptr[restrict static 1]) +{ + const uint32_t core_apic_mask = + ~(bit_mask(processor->topology.thread_bits_length) << processor->topology.thread_bits_offset); + const uint32_t package_apic_mask = + core_apic_mask & ~(bit_mask(processor->topology.core_bits_length) << processor->topology.core_bits_offset); + const uint32_t llc_apic_mask = ~bit_mask(llc_apic_bits); + const uint32_t cluster_apic_mask = package_apic_mask | llc_apic_mask; + + uint32_t cores_count = 0, clusters_count = 0, packages_count = 0; + uint32_t l1i_count = 0, l1d_count = 0, l2_count = 0, l3_count = 0, l4_count = 0; + uint32_t last_core_id = UINT32_MAX, last_cluster_id = UINT32_MAX, last_package_id = UINT32_MAX; + uint32_t last_l1i_id = UINT32_MAX, last_l1d_id = UINT32_MAX; + uint32_t last_l2_id = UINT32_MAX, last_l3_id = UINT32_MAX, last_l4_id = UINT32_MAX; + for (uint32_t i = 0; i < linux_processors_count; i++) { + if (bitmask_all(linux_processors[i].flags, valid_processor_mask)) { + const uint32_t apic_id = linux_processors[i].apic_id; + cpuinfo_log_debug("APID ID %"PRIu32": system processor %"PRIu32, apic_id, linux_processors[i].linux_id); + + /* All bits of APIC ID except thread ID mask */ + const uint32_t core_id = apic_id & core_apic_mask; + if (core_id != last_core_id) { + last_core_id = core_id; + cores_count++; + } + /* All bits of APIC ID except thread ID and core ID masks */ + const uint32_t package_id = apic_id & package_apic_mask; + if (package_id != last_package_id) { + last_package_id = package_id; + packages_count++; + } + /* Bits of APIC ID which are part of either LLC or package ID mask */ + const uint32_t cluster_id = apic_id & cluster_apic_mask; + if (cluster_id != last_cluster_id) { + last_cluster_id = cluster_id; + clusters_count++; + } + if (processor->cache.l1i.size != 0) { + const uint32_t l1i_id = apic_id & ~bit_mask(processor->cache.l1i.apic_bits); + if (l1i_id != last_l1i_id) { + last_l1i_id = l1i_id; + l1i_count++; + } + } + if (processor->cache.l1d.size != 0) { + const uint32_t l1d_id = apic_id & ~bit_mask(processor->cache.l1d.apic_bits); + if (l1d_id != last_l1d_id) { + last_l1d_id = l1d_id; + l1d_count++; + } + } + if (processor->cache.l2.size != 0) { + const uint32_t l2_id = apic_id & ~bit_mask(processor->cache.l2.apic_bits); + if (l2_id != last_l2_id) { + last_l2_id = l2_id; + l2_count++; + } + } + if (processor->cache.l3.size != 0) { + const uint32_t l3_id = apic_id & ~bit_mask(processor->cache.l3.apic_bits); + if (l3_id != last_l3_id) { + last_l3_id = l3_id; + l3_count++; + } + } + if (processor->cache.l4.size != 0) { + const uint32_t l4_id = apic_id & ~bit_mask(processor->cache.l4.apic_bits); + if (l4_id != last_l4_id) { + last_l4_id = l4_id; + l4_count++; + } + } + } + } + *cores_count_ptr = cores_count; + *clusters_count_ptr = clusters_count; + *packages_count_ptr = packages_count; + *l1i_count_ptr = l1i_count; + *l1d_count_ptr = l1d_count; + *l2_count_ptr = l2_count; + *l3_count_ptr = l3_count; + *l4_count_ptr = l4_count; +} + +void cpuinfo_x86_linux_init(void) { + struct cpuinfo_x86_linux_processor* x86_linux_processors = NULL; + struct cpuinfo_processor* processors = NULL; + struct cpuinfo_core* cores = NULL; + struct cpuinfo_cluster* clusters = NULL; + struct cpuinfo_package* packages = NULL; + const struct cpuinfo_processor** linux_cpu_to_processor_map = NULL; + const struct cpuinfo_core** linux_cpu_to_core_map = NULL; + struct cpuinfo_cache* l1i = NULL; + struct cpuinfo_cache* l1d = NULL; + struct cpuinfo_cache* l2 = NULL; + struct cpuinfo_cache* l3 = NULL; + struct cpuinfo_cache* l4 = NULL; + + const uint32_t max_processors_count = cpuinfo_linux_get_max_processors_count(); + cpuinfo_log_debug("system maximum processors count: %"PRIu32, max_processors_count); + + const uint32_t max_possible_processors_count = 1 + + cpuinfo_linux_get_max_possible_processor(max_processors_count); + cpuinfo_log_debug("maximum possible processors count: %"PRIu32, max_possible_processors_count); + const uint32_t max_present_processors_count = 1 + + cpuinfo_linux_get_max_present_processor(max_processors_count); + cpuinfo_log_debug("maximum present processors count: %"PRIu32, max_present_processors_count); + + uint32_t valid_processor_mask = 0; + uint32_t x86_linux_processors_count = max_processors_count; + if (max_present_processors_count != 0) { + x86_linux_processors_count = min(x86_linux_processors_count, max_present_processors_count); + valid_processor_mask = CPUINFO_LINUX_FLAG_PRESENT; + } else { + valid_processor_mask = CPUINFO_LINUX_FLAG_PROC_CPUINFO; + } + if (max_possible_processors_count != 0) { + x86_linux_processors_count = min(x86_linux_processors_count, max_possible_processors_count); + valid_processor_mask |= CPUINFO_LINUX_FLAG_POSSIBLE; + } + + x86_linux_processors = calloc(x86_linux_processors_count, sizeof(struct cpuinfo_x86_linux_processor)); + if (x86_linux_processors == NULL) { + cpuinfo_log_error( + "failed to allocate %zu bytes for descriptions of %"PRIu32" x86 logical processors", + x86_linux_processors_count * sizeof(struct cpuinfo_x86_linux_processor), + x86_linux_processors_count); + return; + } + + if (max_possible_processors_count != 0) { + cpuinfo_linux_detect_possible_processors( + x86_linux_processors_count, &x86_linux_processors->flags, + sizeof(struct cpuinfo_x86_linux_processor), + CPUINFO_LINUX_FLAG_POSSIBLE); + } + + if (max_present_processors_count != 0) { + cpuinfo_linux_detect_present_processors( + x86_linux_processors_count, &x86_linux_processors->flags, + sizeof(struct cpuinfo_x86_linux_processor), + CPUINFO_LINUX_FLAG_PRESENT); + } + + if (!cpuinfo_x86_linux_parse_proc_cpuinfo(x86_linux_processors_count, x86_linux_processors)) { + cpuinfo_log_error("failed to parse processor information from /proc/cpuinfo"); + return; + } + + for (uint32_t i = 0; i < x86_linux_processors_count; i++) { + if (bitmask_all(x86_linux_processors[i].flags, valid_processor_mask)) { + x86_linux_processors[i].flags |= CPUINFO_LINUX_FLAG_VALID; + } + } + + struct cpuinfo_x86_processor x86_processor; + memset(&x86_processor, 0, sizeof(x86_processor)); + cpuinfo_x86_init_processor(&x86_processor); + char brand_string[48]; + cpuinfo_x86_normalize_brand_string(x86_processor.brand_string, brand_string); + + uint32_t processors_count = 0; + for (uint32_t i = 0; i < x86_linux_processors_count; i++) { + if (bitmask_all(x86_linux_processors[i].flags, CPUINFO_LINUX_FLAG_VALID)) { + x86_linux_processors[i].linux_id = i; + processors_count++; + } + } + + qsort(x86_linux_processors, x86_linux_processors_count, sizeof(struct cpuinfo_x86_linux_processor), + cmp_x86_linux_processor); + + processors = calloc(processors_count, sizeof(struct cpuinfo_processor)); + if (processors == NULL) { + cpuinfo_log_error("failed to allocate %zu bytes for descriptions of %"PRIu32" logical processors", + processors_count * sizeof(struct cpuinfo_processor), processors_count); + goto cleanup; + } + + uint32_t llc_apic_bits = 0; + if (x86_processor.cache.l4.size != 0) { + llc_apic_bits = x86_processor.cache.l4.apic_bits; + } else if (x86_processor.cache.l3.size != 0) { + llc_apic_bits = x86_processor.cache.l3.apic_bits; + } else if (x86_processor.cache.l2.size != 0) { + llc_apic_bits = x86_processor.cache.l2.apic_bits; + } else if (x86_processor.cache.l1d.size != 0) { + llc_apic_bits = x86_processor.cache.l1d.apic_bits; + } + uint32_t packages_count = 0, clusters_count = 0, cores_count = 0; + uint32_t l1i_count = 0, l1d_count = 0, l2_count = 0, l3_count = 0, l4_count = 0; + cpuinfo_x86_count_objects( + x86_linux_processors_count, x86_linux_processors, &x86_processor, valid_processor_mask, llc_apic_bits, + &cores_count, &clusters_count, &packages_count, &l1i_count, &l1d_count, &l2_count, &l3_count, &l4_count); + + cpuinfo_log_debug("detected %"PRIu32" cores", cores_count); + cpuinfo_log_debug("detected %"PRIu32" clusters", clusters_count); + cpuinfo_log_debug("detected %"PRIu32" packages", packages_count); + cpuinfo_log_debug("detected %"PRIu32" L1I caches", l1i_count); + cpuinfo_log_debug("detected %"PRIu32" L1D caches", l1d_count); + cpuinfo_log_debug("detected %"PRIu32" L2 caches", l2_count); + cpuinfo_log_debug("detected %"PRIu32" L3 caches", l3_count); + cpuinfo_log_debug("detected %"PRIu32" L4 caches", l4_count); + + linux_cpu_to_processor_map = calloc(x86_linux_processors_count, sizeof(struct cpuinfo_processor*)); + if (linux_cpu_to_processor_map == NULL) { + cpuinfo_log_error("failed to allocate %zu bytes for mapping entries of %"PRIu32" logical processors", + x86_linux_processors_count * sizeof(struct cpuinfo_processor*), + x86_linux_processors_count); + goto cleanup; + } + + linux_cpu_to_core_map = calloc(x86_linux_processors_count, sizeof(struct cpuinfo_core*)); + if (linux_cpu_to_core_map == NULL) { + cpuinfo_log_error("failed to allocate %zu bytes for mapping entries of %"PRIu32" cores", + x86_linux_processors_count * sizeof(struct cpuinfo_core*), + x86_linux_processors_count); + goto cleanup; + } + + cores = calloc(cores_count, sizeof(struct cpuinfo_core)); + if (cores == NULL) { + cpuinfo_log_error("failed to allocate %zu bytes for descriptions of %"PRIu32" cores", + cores_count * sizeof(struct cpuinfo_core), cores_count); + goto cleanup; + } + + clusters = calloc(clusters_count, sizeof(struct cpuinfo_cluster)); + if (clusters == NULL) { + cpuinfo_log_error("failed to allocate %zu bytes for descriptions of %"PRIu32" core clusters", + clusters_count * sizeof(struct cpuinfo_cluster), clusters_count); + goto cleanup; + } + + packages = calloc(packages_count, sizeof(struct cpuinfo_package)); + if (packages == NULL) { + cpuinfo_log_error("failed to allocate %zu bytes for descriptions of %"PRIu32" physical packages", + packages_count * sizeof(struct cpuinfo_package), packages_count); + goto cleanup; + } + + if (l1i_count != 0) { + l1i = calloc(l1i_count, sizeof(struct cpuinfo_cache)); + if (l1i == NULL) { + cpuinfo_log_error("failed to allocate %zu bytes for descriptions of %"PRIu32" L1I caches", + l1i_count * sizeof(struct cpuinfo_cache), l1i_count); + goto cleanup; + } + } + if (l1d_count != 0) { + l1d = calloc(l1d_count, sizeof(struct cpuinfo_cache)); + if (l1d == NULL) { + cpuinfo_log_error("failed to allocate %zu bytes for descriptions of %"PRIu32" L1D caches", + l1d_count * sizeof(struct cpuinfo_cache), l1d_count); + goto cleanup; + } + } + if (l2_count != 0) { + l2 = calloc(l2_count, sizeof(struct cpuinfo_cache)); + if (l2 == NULL) { + cpuinfo_log_error("failed to allocate %zu bytes for descriptions of %"PRIu32" L2 caches", + l2_count * sizeof(struct cpuinfo_cache), l2_count); + goto cleanup; + } + } + if (l3_count != 0) { + l3 = calloc(l3_count, sizeof(struct cpuinfo_cache)); + if (l3 == NULL) { + cpuinfo_log_error("failed to allocate %zu bytes for descriptions of %"PRIu32" L3 caches", + l3_count * sizeof(struct cpuinfo_cache), l3_count); + goto cleanup; + } + } + if (l4_count != 0) { + l4 = calloc(l4_count, sizeof(struct cpuinfo_cache)); + if (l4 == NULL) { + cpuinfo_log_error("failed to allocate %zu bytes for descriptions of %"PRIu32" L4 caches", + l4_count * sizeof(struct cpuinfo_cache), l4_count); + goto cleanup; + } + } + + const uint32_t core_apic_mask = + ~(bit_mask(x86_processor.topology.thread_bits_length) << x86_processor.topology.thread_bits_offset); + const uint32_t package_apic_mask = + core_apic_mask & ~(bit_mask(x86_processor.topology.core_bits_length) << x86_processor.topology.core_bits_offset); + const uint32_t llc_apic_mask = ~bit_mask(llc_apic_bits); + const uint32_t cluster_apic_mask = package_apic_mask | llc_apic_mask; + + uint32_t processor_index = UINT32_MAX, core_index = UINT32_MAX, cluster_index = UINT32_MAX, package_index = UINT32_MAX; + uint32_t l1i_index = UINT32_MAX, l1d_index = UINT32_MAX, l2_index = UINT32_MAX, l3_index = UINT32_MAX, l4_index = UINT32_MAX; + uint32_t cluster_id = 0, core_id = 0, smt_id = 0; + uint32_t last_apic_core_id = UINT32_MAX, last_apic_cluster_id = UINT32_MAX, last_apic_package_id = UINT32_MAX; + uint32_t last_l1i_id = UINT32_MAX, last_l1d_id = UINT32_MAX; + uint32_t last_l2_id = UINT32_MAX, last_l3_id = UINT32_MAX, last_l4_id = UINT32_MAX; + for (uint32_t i = 0; i < x86_linux_processors_count; i++) { + if (bitmask_all(x86_linux_processors[i].flags, CPUINFO_LINUX_FLAG_VALID)) { + const uint32_t apic_id = x86_linux_processors[i].apic_id; + processor_index++; + smt_id++; + + /* All bits of APIC ID except thread ID mask */ + const uint32_t apid_core_id = apic_id & core_apic_mask; + if (apid_core_id != last_apic_core_id) { + core_index++; + core_id++; + smt_id = 0; + } + /* Bits of APIC ID which are part of either LLC or package ID mask */ + const uint32_t apic_cluster_id = apic_id & cluster_apic_mask; + if (apic_cluster_id != last_apic_cluster_id) { + cluster_index++; + cluster_id++; + } + /* All bits of APIC ID except thread ID and core ID masks */ + const uint32_t apic_package_id = apic_id & package_apic_mask; + if (apic_package_id != last_apic_package_id) { + package_index++; + core_id = 0; + cluster_id = 0; + } + + /* Initialize logical processor object */ + processors[processor_index].smt_id = smt_id; + processors[processor_index].core = cores + core_index; + processors[processor_index].cluster = clusters + cluster_index; + processors[processor_index].package = packages + package_index; + processors[processor_index].linux_id = x86_linux_processors[i].linux_id; + processors[processor_index].apic_id = x86_linux_processors[i].apic_id; + + if (apid_core_id != last_apic_core_id) { + /* new core */ + cores[core_index] = (struct cpuinfo_core) { + .processor_start = processor_index, + .processor_count = 1, + .core_id = core_id, + .cluster = clusters + cluster_index, + .package = packages + package_index, + .vendor = x86_processor.vendor, + .uarch = x86_processor.uarch, + .cpuid = x86_processor.cpuid, + }; + clusters[cluster_index].core_count += 1; + packages[package_index].core_count += 1; + last_apic_core_id = apid_core_id; + } else { + /* another logical processor on the same core */ + cores[core_index].processor_count++; + } + + if (apic_cluster_id != last_apic_cluster_id) { + /* new cluster */ + clusters[cluster_index].processor_start = processor_index; + clusters[cluster_index].processor_count = 1; + clusters[cluster_index].core_start = core_index; + clusters[cluster_index].cluster_id = cluster_id; + clusters[cluster_index].package = packages + package_index; + clusters[cluster_index].vendor = x86_processor.vendor; + clusters[cluster_index].uarch = x86_processor.uarch; + clusters[cluster_index].cpuid = x86_processor.cpuid; + packages[package_index].cluster_count += 1; + last_apic_cluster_id = apic_cluster_id; + } else { + /* another logical processor on the same cluster */ + clusters[cluster_index].processor_count++; + } + + if (apic_package_id != last_apic_package_id) { + /* new package */ + packages[package_index].processor_start = processor_index; + packages[package_index].processor_count = 1; + packages[package_index].core_start = core_index; + packages[package_index].cluster_start = cluster_index; + cpuinfo_x86_format_package_name(x86_processor.vendor, brand_string, packages[package_index].name); + last_apic_package_id = apic_package_id; + } else { + /* another logical processor on the same package */ + packages[package_index].processor_count++; + } + + linux_cpu_to_processor_map[x86_linux_processors[i].linux_id] = processors + processor_index; + linux_cpu_to_core_map[x86_linux_processors[i].linux_id] = cores + core_index; + + if (x86_processor.cache.l1i.size != 0) { + const uint32_t l1i_id = apic_id & ~bit_mask(x86_processor.cache.l1i.apic_bits); + processors[i].cache.l1i = &l1i[l1i_index]; + if (l1i_id != last_l1i_id) { + /* new cache */ + last_l1i_id = l1i_id; + l1i[++l1i_index] = (struct cpuinfo_cache) { + .size = x86_processor.cache.l1i.size, + .associativity = x86_processor.cache.l1i.associativity, + .sets = x86_processor.cache.l1i.sets, + .partitions = x86_processor.cache.l1i.partitions, + .line_size = x86_processor.cache.l1i.line_size, + .flags = x86_processor.cache.l1i.flags, + .processor_start = processor_index, + .processor_count = 1, + }; + } else { + /* another processor sharing the same cache */ + l1i[l1i_index].processor_count += 1; + } + processors[i].cache.l1i = &l1i[l1i_index]; + } else { + /* reset cache id */ + last_l1i_id = UINT32_MAX; + } + if (x86_processor.cache.l1d.size != 0) { + const uint32_t l1d_id = apic_id & ~bit_mask(x86_processor.cache.l1d.apic_bits); + processors[i].cache.l1d = &l1d[l1d_index]; + if (l1d_id != last_l1d_id) { + /* new cache */ + last_l1d_id = l1d_id; + l1d[++l1d_index] = (struct cpuinfo_cache) { + .size = x86_processor.cache.l1d.size, + .associativity = x86_processor.cache.l1d.associativity, + .sets = x86_processor.cache.l1d.sets, + .partitions = x86_processor.cache.l1d.partitions, + .line_size = x86_processor.cache.l1d.line_size, + .flags = x86_processor.cache.l1d.flags, + .processor_start = processor_index, + .processor_count = 1, + }; + } else { + /* another processor sharing the same cache */ + l1d[l1d_index].processor_count += 1; + } + processors[i].cache.l1d = &l1d[l1d_index]; + } else { + /* reset cache id */ + last_l1d_id = UINT32_MAX; + } + if (x86_processor.cache.l2.size != 0) { + const uint32_t l2_id = apic_id & ~bit_mask(x86_processor.cache.l2.apic_bits); + processors[i].cache.l2 = &l2[l2_index]; + if (l2_id != last_l2_id) { + /* new cache */ + last_l2_id = l2_id; + l2[++l2_index] = (struct cpuinfo_cache) { + .size = x86_processor.cache.l2.size, + .associativity = x86_processor.cache.l2.associativity, + .sets = x86_processor.cache.l2.sets, + .partitions = x86_processor.cache.l2.partitions, + .line_size = x86_processor.cache.l2.line_size, + .flags = x86_processor.cache.l2.flags, + .processor_start = processor_index, + .processor_count = 1, + }; + } else { + /* another processor sharing the same cache */ + l2[l2_index].processor_count += 1; + } + processors[i].cache.l2 = &l2[l2_index]; + } else { + /* reset cache id */ + last_l2_id = UINT32_MAX; + } + if (x86_processor.cache.l3.size != 0) { + const uint32_t l3_id = apic_id & ~bit_mask(x86_processor.cache.l3.apic_bits); + processors[i].cache.l3 = &l3[l3_index]; + if (l3_id != last_l3_id) { + /* new cache */ + last_l3_id = l3_id; + l3[++l3_index] = (struct cpuinfo_cache) { + .size = x86_processor.cache.l3.size, + .associativity = x86_processor.cache.l3.associativity, + .sets = x86_processor.cache.l3.sets, + .partitions = x86_processor.cache.l3.partitions, + .line_size = x86_processor.cache.l3.line_size, + .flags = x86_processor.cache.l3.flags, + .processor_start = processor_index, + .processor_count = 1, + }; + } else { + /* another processor sharing the same cache */ + l3[l3_index].processor_count += 1; + } + processors[i].cache.l3 = &l3[l3_index]; + } else { + /* reset cache id */ + last_l3_id = UINT32_MAX; + } + if (x86_processor.cache.l4.size != 0) { + const uint32_t l4_id = apic_id & ~bit_mask(x86_processor.cache.l4.apic_bits); + processors[i].cache.l4 = &l4[l4_index]; + if (l4_id != last_l4_id) { + /* new cache */ + last_l4_id = l4_id; + l4[++l4_index] = (struct cpuinfo_cache) { + .size = x86_processor.cache.l4.size, + .associativity = x86_processor.cache.l4.associativity, + .sets = x86_processor.cache.l4.sets, + .partitions = x86_processor.cache.l4.partitions, + .line_size = x86_processor.cache.l4.line_size, + .flags = x86_processor.cache.l4.flags, + .processor_start = processor_index, + .processor_count = 1, + }; + } else { + /* another processor sharing the same cache */ + l4[l4_index].processor_count += 1; + } + processors[i].cache.l4 = &l4[l4_index]; + } else { + /* reset cache id */ + last_l4_id = UINT32_MAX; + } + } + } + + /* Commit changes */ + cpuinfo_processors = processors; + cpuinfo_cores = cores; + cpuinfo_clusters = clusters; + cpuinfo_packages = packages; + cpuinfo_cache[cpuinfo_cache_level_1i] = l1i; + cpuinfo_cache[cpuinfo_cache_level_1d] = l1d; + cpuinfo_cache[cpuinfo_cache_level_2] = l2; + cpuinfo_cache[cpuinfo_cache_level_3] = l3; + cpuinfo_cache[cpuinfo_cache_level_4] = l4; + + cpuinfo_processors_count = processors_count; + cpuinfo_cores_count = cores_count; + cpuinfo_clusters_count = clusters_count; + cpuinfo_packages_count = packages_count; + cpuinfo_cache_count[cpuinfo_cache_level_1i] = l1i_count; + cpuinfo_cache_count[cpuinfo_cache_level_1d] = l1d_count; + cpuinfo_cache_count[cpuinfo_cache_level_2] = l2_count; + cpuinfo_cache_count[cpuinfo_cache_level_3] = l3_count; + cpuinfo_cache_count[cpuinfo_cache_level_4] = l4_count; + cpuinfo_max_cache_size = cpuinfo_compute_max_cache_size(&processors[0]); + + cpuinfo_global_uarch = (struct cpuinfo_uarch_info) { + .uarch = x86_processor.uarch, + .cpuid = x86_processor.cpuid, + .processor_count = processors_count, + .core_count = cores_count, + }; + + cpuinfo_linux_cpu_max = x86_linux_processors_count; + cpuinfo_linux_cpu_to_processor_map = linux_cpu_to_processor_map; + cpuinfo_linux_cpu_to_core_map = linux_cpu_to_core_map; + + __sync_synchronize(); + + cpuinfo_is_initialized = true; + + processors = NULL; + cores = NULL; + clusters = NULL; + packages = NULL; + l1i = l1d = l2 = l3 = l4 = NULL; + linux_cpu_to_processor_map = NULL; + linux_cpu_to_core_map = NULL; + +cleanup: + free(x86_linux_processors); + free(processors); + free(cores); + free(clusters); + free(packages); + free(l1i); + free(l1d); + free(l2); + free(l3); + free(l4); + free(linux_cpu_to_processor_map); + free(linux_cpu_to_core_map); +} diff --git a/3rdparty/cpuinfo/src/x86/mach/init.c b/3rdparty/cpuinfo/src/x86/mach/init.c new file mode 100644 index 000000000..b44d3adf5 --- /dev/null +++ b/3rdparty/cpuinfo/src/x86/mach/init.c @@ -0,0 +1,356 @@ +#include +#include +#include + +#include +#include +#include +#include +#include + + +static inline uint32_t max(uint32_t a, uint32_t b) { + return a > b ? a : b; +} + +static inline uint32_t bit_mask(uint32_t bits) { + return (UINT32_C(1) << bits) - UINT32_C(1); +} + +void cpuinfo_x86_mach_init(void) { + struct cpuinfo_processor* processors = NULL; + struct cpuinfo_core* cores = NULL; + struct cpuinfo_cluster* clusters = NULL; + struct cpuinfo_package* packages = NULL; + struct cpuinfo_cache* l1i = NULL; + struct cpuinfo_cache* l1d = NULL; + struct cpuinfo_cache* l2 = NULL; + struct cpuinfo_cache* l3 = NULL; + struct cpuinfo_cache* l4 = NULL; + + struct cpuinfo_mach_topology mach_topology = cpuinfo_mach_detect_topology(); + processors = calloc(mach_topology.threads, sizeof(struct cpuinfo_processor)); + if (processors == NULL) { + cpuinfo_log_error("failed to allocate %zu bytes for descriptions of %"PRIu32" logical processors", + mach_topology.threads * sizeof(struct cpuinfo_processor), mach_topology.threads); + goto cleanup; + } + cores = calloc(mach_topology.cores, sizeof(struct cpuinfo_core)); + if (cores == NULL) { + cpuinfo_log_error("failed to allocate %zu bytes for descriptions of %"PRIu32" cores", + mach_topology.cores * sizeof(struct cpuinfo_core), mach_topology.cores); + goto cleanup; + } + /* On x86 cluster of cores is a physical package */ + clusters = calloc(mach_topology.packages, sizeof(struct cpuinfo_cluster)); + if (clusters == NULL) { + cpuinfo_log_error("failed to allocate %zu bytes for descriptions of %"PRIu32" core clusters", + mach_topology.packages * sizeof(struct cpuinfo_cluster), mach_topology.packages); + goto cleanup; + } + packages = calloc(mach_topology.packages, sizeof(struct cpuinfo_package)); + if (packages == NULL) { + cpuinfo_log_error("failed to allocate %zu bytes for descriptions of %"PRIu32" physical packages", + mach_topology.packages * sizeof(struct cpuinfo_package), mach_topology.packages); + goto cleanup; + } + + struct cpuinfo_x86_processor x86_processor; + memset(&x86_processor, 0, sizeof(x86_processor)); + cpuinfo_x86_init_processor(&x86_processor); + char brand_string[48]; + cpuinfo_x86_normalize_brand_string(x86_processor.brand_string, brand_string); + + const uint32_t threads_per_core = mach_topology.threads / mach_topology.cores; + const uint32_t threads_per_package = mach_topology.threads / mach_topology.packages; + const uint32_t cores_per_package = mach_topology.cores / mach_topology.packages; + for (uint32_t i = 0; i < mach_topology.packages; i++) { + clusters[i] = (struct cpuinfo_cluster) { + .processor_start = i * threads_per_package, + .processor_count = threads_per_package, + .core_start = i * cores_per_package, + .core_count = cores_per_package, + .cluster_id = 0, + .package = packages + i, + .vendor = x86_processor.vendor, + .uarch = x86_processor.uarch, + .cpuid = x86_processor.cpuid, + }; + packages[i].processor_start = i * threads_per_package; + packages[i].processor_count = threads_per_package; + packages[i].core_start = i * cores_per_package; + packages[i].core_count = cores_per_package; + packages[i].cluster_start = i; + packages[i].cluster_count = 1; + cpuinfo_x86_format_package_name(x86_processor.vendor, brand_string, packages[i].name); + } + for (uint32_t i = 0; i < mach_topology.cores; i++) { + cores[i] = (struct cpuinfo_core) { + .processor_start = i * threads_per_core, + .processor_count = threads_per_core, + .core_id = i % cores_per_package, + .cluster = clusters + i / cores_per_package, + .package = packages + i / cores_per_package, + .vendor = x86_processor.vendor, + .uarch = x86_processor.uarch, + .cpuid = x86_processor.cpuid, + }; + } + for (uint32_t i = 0; i < mach_topology.threads; i++) { + const uint32_t smt_id = i % threads_per_core; + const uint32_t core_id = i / threads_per_core; + const uint32_t package_id = i / threads_per_package; + + /* Reconstruct APIC IDs from topology components */ + const uint32_t thread_bits_mask = bit_mask(x86_processor.topology.thread_bits_length); + const uint32_t core_bits_mask = bit_mask(x86_processor.topology.core_bits_length); + const uint32_t package_bits_offset = max( + x86_processor.topology.thread_bits_offset + x86_processor.topology.thread_bits_length, + x86_processor.topology.core_bits_offset + x86_processor.topology.core_bits_length); + const uint32_t apic_id = + ((smt_id & thread_bits_mask) << x86_processor.topology.thread_bits_offset) | + ((core_id & core_bits_mask) << x86_processor.topology.core_bits_offset) | + (package_id << package_bits_offset); + cpuinfo_log_debug("reconstructed APIC ID 0x%08"PRIx32" for thread %"PRIu32, apic_id, i); + + processors[i].smt_id = smt_id; + processors[i].core = cores + i / threads_per_core; + processors[i].cluster = clusters + i / threads_per_package; + processors[i].package = packages + i / threads_per_package; + processors[i].apic_id = apic_id; + } + + uint32_t threads_per_l1 = 0, l1_count = 0; + if (x86_processor.cache.l1i.size != 0 || x86_processor.cache.l1d.size != 0) { + threads_per_l1 = mach_topology.threads_per_cache[1]; + if (threads_per_l1 == 0) { + /* Assume that threads on the same core share L1 */ + threads_per_l1 = mach_topology.threads / mach_topology.cores; + cpuinfo_log_warning("Mach kernel did not report number of threads sharing L1 cache; assume %"PRIu32, + threads_per_l1); + } + l1_count = mach_topology.threads / threads_per_l1; + cpuinfo_log_debug("detected %"PRIu32" L1 caches", l1_count); + } + + uint32_t threads_per_l2 = 0, l2_count = 0; + if (x86_processor.cache.l2.size != 0) { + threads_per_l2 = mach_topology.threads_per_cache[2]; + if (threads_per_l2 == 0) { + if (x86_processor.cache.l3.size != 0) { + /* This is not a last-level cache; assume that threads on the same core share L2 */ + threads_per_l2 = mach_topology.threads / mach_topology.cores; + } else { + /* This is a last-level cache; assume that threads on the same package share L2 */ + threads_per_l2 = mach_topology.threads / mach_topology.packages; + } + cpuinfo_log_warning("Mach kernel did not report number of threads sharing L2 cache; assume %"PRIu32, + threads_per_l2); + } + l2_count = mach_topology.threads / threads_per_l2; + cpuinfo_log_debug("detected %"PRIu32" L2 caches", l2_count); + } + + uint32_t threads_per_l3 = 0, l3_count = 0; + if (x86_processor.cache.l3.size != 0) { + threads_per_l3 = mach_topology.threads_per_cache[3]; + if (threads_per_l3 == 0) { + /* + * Assume that threads on the same package share L3. + * However, is it not necessarily the last-level cache (there may be L4 cache as well) + */ + threads_per_l3 = mach_topology.threads / mach_topology.packages; + cpuinfo_log_warning("Mach kernel did not report number of threads sharing L3 cache; assume %"PRIu32, + threads_per_l3); + } + l3_count = mach_topology.threads / threads_per_l3; + cpuinfo_log_debug("detected %"PRIu32" L3 caches", l3_count); + } + + uint32_t threads_per_l4 = 0, l4_count = 0; + if (x86_processor.cache.l4.size != 0) { + threads_per_l4 = mach_topology.threads_per_cache[4]; + if (threads_per_l4 == 0) { + /* + * Assume that all threads share this L4. + * As of now, L4 cache exists only on notebook x86 CPUs, which are single-package, + * but multi-socket systems could have shared L4 (like on IBM POWER8). + */ + threads_per_l4 = mach_topology.threads; + cpuinfo_log_warning("Mach kernel did not report number of threads sharing L4 cache; assume %"PRIu32, + threads_per_l4); + } + l4_count = mach_topology.threads / threads_per_l4; + cpuinfo_log_debug("detected %"PRIu32" L4 caches", l4_count); + } + + if (x86_processor.cache.l1i.size != 0) { + l1i = calloc(l1_count, sizeof(struct cpuinfo_cache)); + if (l1i == NULL) { + cpuinfo_log_error("failed to allocate %zu bytes for descriptions of %"PRIu32" L1I caches", + l1_count * sizeof(struct cpuinfo_cache), l1_count); + return; + } + for (uint32_t c = 0; c < l1_count; c++) { + l1i[c] = (struct cpuinfo_cache) { + .size = x86_processor.cache.l1i.size, + .associativity = x86_processor.cache.l1i.associativity, + .sets = x86_processor.cache.l1i.sets, + .partitions = x86_processor.cache.l1i.partitions, + .line_size = x86_processor.cache.l1i.line_size, + .flags = x86_processor.cache.l1i.flags, + .processor_start = c * threads_per_l1, + .processor_count = threads_per_l1, + }; + } + for (uint32_t t = 0; t < mach_topology.threads; t++) { + processors[t].cache.l1i = &l1i[t / threads_per_l1]; + } + } + + if (x86_processor.cache.l1d.size != 0) { + l1d = calloc(l1_count, sizeof(struct cpuinfo_cache)); + if (l1d == NULL) { + cpuinfo_log_error("failed to allocate %zu bytes for descriptions of %"PRIu32" L1D caches", + l1_count * sizeof(struct cpuinfo_cache), l1_count); + return; + } + for (uint32_t c = 0; c < l1_count; c++) { + l1d[c] = (struct cpuinfo_cache) { + .size = x86_processor.cache.l1d.size, + .associativity = x86_processor.cache.l1d.associativity, + .sets = x86_processor.cache.l1d.sets, + .partitions = x86_processor.cache.l1d.partitions, + .line_size = x86_processor.cache.l1d.line_size, + .flags = x86_processor.cache.l1d.flags, + .processor_start = c * threads_per_l1, + .processor_count = threads_per_l1, + }; + } + for (uint32_t t = 0; t < mach_topology.threads; t++) { + processors[t].cache.l1d = &l1d[t / threads_per_l1]; + } + } + + if (l2_count != 0) { + l2 = calloc(l2_count, sizeof(struct cpuinfo_cache)); + if (l2 == NULL) { + cpuinfo_log_error("failed to allocate %zu bytes for descriptions of %"PRIu32" L2 caches", + l2_count * sizeof(struct cpuinfo_cache), l2_count); + return; + } + for (uint32_t c = 0; c < l2_count; c++) { + l2[c] = (struct cpuinfo_cache) { + .size = x86_processor.cache.l2.size, + .associativity = x86_processor.cache.l2.associativity, + .sets = x86_processor.cache.l2.sets, + .partitions = x86_processor.cache.l2.partitions, + .line_size = x86_processor.cache.l2.line_size, + .flags = x86_processor.cache.l2.flags, + .processor_start = c * threads_per_l2, + .processor_count = threads_per_l2, + }; + } + for (uint32_t t = 0; t < mach_topology.threads; t++) { + processors[t].cache.l2 = &l2[t / threads_per_l2]; + } + } + + if (l3_count != 0) { + l3 = calloc(l3_count, sizeof(struct cpuinfo_cache)); + if (l3 == NULL) { + cpuinfo_log_error("failed to allocate %zu bytes for descriptions of %"PRIu32" L3 caches", + l3_count * sizeof(struct cpuinfo_cache), l3_count); + return; + } + for (uint32_t c = 0; c < l3_count; c++) { + l3[c] = (struct cpuinfo_cache) { + .size = x86_processor.cache.l3.size, + .associativity = x86_processor.cache.l3.associativity, + .sets = x86_processor.cache.l3.sets, + .partitions = x86_processor.cache.l3.partitions, + .line_size = x86_processor.cache.l3.line_size, + .flags = x86_processor.cache.l3.flags, + .processor_start = c * threads_per_l3, + .processor_count = threads_per_l3, + }; + } + for (uint32_t t = 0; t < mach_topology.threads; t++) { + processors[t].cache.l3 = &l3[t / threads_per_l3]; + } + } + + if (l4_count != 0) { + l4 = calloc(l4_count, sizeof(struct cpuinfo_cache)); + if (l4 == NULL) { + cpuinfo_log_error("failed to allocate %zu bytes for descriptions of %"PRIu32" L4 caches", + l4_count * sizeof(struct cpuinfo_cache), l4_count); + return; + } + for (uint32_t c = 0; c < l4_count; c++) { + l4[c] = (struct cpuinfo_cache) { + .size = x86_processor.cache.l4.size, + .associativity = x86_processor.cache.l4.associativity, + .sets = x86_processor.cache.l4.sets, + .partitions = x86_processor.cache.l4.partitions, + .line_size = x86_processor.cache.l4.line_size, + .flags = x86_processor.cache.l4.flags, + .processor_start = c * threads_per_l4, + .processor_count = threads_per_l4, + }; + } + for (uint32_t t = 0; t < mach_topology.threads; t++) { + processors[t].cache.l4 = &l4[t / threads_per_l4]; + } + } + + /* Commit changes */ + cpuinfo_processors = processors; + cpuinfo_cores = cores; + cpuinfo_clusters = clusters; + cpuinfo_packages = packages; + cpuinfo_cache[cpuinfo_cache_level_1i] = l1i; + cpuinfo_cache[cpuinfo_cache_level_1d] = l1d; + cpuinfo_cache[cpuinfo_cache_level_2] = l2; + cpuinfo_cache[cpuinfo_cache_level_3] = l3; + cpuinfo_cache[cpuinfo_cache_level_4] = l4; + + cpuinfo_processors_count = mach_topology.threads; + cpuinfo_cores_count = mach_topology.cores; + cpuinfo_clusters_count = mach_topology.packages; + cpuinfo_packages_count = mach_topology.packages; + cpuinfo_cache_count[cpuinfo_cache_level_1i] = l1_count; + cpuinfo_cache_count[cpuinfo_cache_level_1d] = l1_count; + cpuinfo_cache_count[cpuinfo_cache_level_2] = l2_count; + cpuinfo_cache_count[cpuinfo_cache_level_3] = l3_count; + cpuinfo_cache_count[cpuinfo_cache_level_4] = l4_count; + cpuinfo_max_cache_size = cpuinfo_compute_max_cache_size(&processors[0]); + + cpuinfo_global_uarch = (struct cpuinfo_uarch_info) { + .uarch = x86_processor.uarch, + .cpuid = x86_processor.cpuid, + .processor_count = mach_topology.threads, + .core_count = mach_topology.cores, + }; + + __sync_synchronize(); + + cpuinfo_is_initialized = true; + + processors = NULL; + cores = NULL; + clusters = NULL; + packages = NULL; + l1i = l1d = l2 = l3 = l4 = NULL; + +cleanup: + free(processors); + free(cores); + free(clusters); + free(packages); + free(l1i); + free(l1d); + free(l2); + free(l3); + free(l4); +} diff --git a/3rdparty/cpuinfo/src/x86/mockcpuid.c b/3rdparty/cpuinfo/src/x86/mockcpuid.c new file mode 100644 index 000000000..2631f09ba --- /dev/null +++ b/3rdparty/cpuinfo/src/x86/mockcpuid.c @@ -0,0 +1,70 @@ +#include +#include + +#if !CPUINFO_MOCK + #error This file should be built only in mock mode +#endif + +#include + + +static struct cpuinfo_mock_cpuid* cpuinfo_mock_cpuid_data = NULL; +static uint32_t cpuinfo_mock_cpuid_entries = 0; +static uint32_t cpuinfo_mock_cpuid_leaf4_iteration = 0; + +void CPUINFO_ABI cpuinfo_mock_set_cpuid(struct cpuinfo_mock_cpuid* dump, size_t entries) { + cpuinfo_mock_cpuid_data = dump; + cpuinfo_mock_cpuid_entries = entries; +}; + +void CPUINFO_ABI cpuinfo_mock_get_cpuid(uint32_t eax, uint32_t regs[4]) { + if (eax != 4) { + cpuinfo_mock_cpuid_leaf4_iteration = 0; + } + if (cpuinfo_mock_cpuid_data != NULL && cpuinfo_mock_cpuid_entries != 0) { + if (eax == 4) { + uint32_t skip_entries = cpuinfo_mock_cpuid_leaf4_iteration; + for (uint32_t i = 0; i < cpuinfo_mock_cpuid_entries; i++) { + if (eax == cpuinfo_mock_cpuid_data[i].input_eax) { + if (skip_entries-- == 0) { + regs[0] = cpuinfo_mock_cpuid_data[i].eax; + regs[1] = cpuinfo_mock_cpuid_data[i].ebx; + regs[2] = cpuinfo_mock_cpuid_data[i].ecx; + regs[3] = cpuinfo_mock_cpuid_data[i].edx; + cpuinfo_mock_cpuid_leaf4_iteration++; + return; + } + } + } + } else { + for (uint32_t i = 0; i < cpuinfo_mock_cpuid_entries; i++) { + if (eax == cpuinfo_mock_cpuid_data[i].input_eax) { + regs[0] = cpuinfo_mock_cpuid_data[i].eax; + regs[1] = cpuinfo_mock_cpuid_data[i].ebx; + regs[2] = cpuinfo_mock_cpuid_data[i].ecx; + regs[3] = cpuinfo_mock_cpuid_data[i].edx; + return; + } + } + } + } + regs[0] = regs[1] = regs[2] = regs[3] = 0; +} + +void CPUINFO_ABI cpuinfo_mock_get_cpuidex(uint32_t eax, uint32_t ecx, uint32_t regs[4]) { + cpuinfo_mock_cpuid_leaf4_iteration = 0; + if (cpuinfo_mock_cpuid_data != NULL && cpuinfo_mock_cpuid_entries != 0) { + for (uint32_t i = 0; i < cpuinfo_mock_cpuid_entries; i++) { + if (eax == cpuinfo_mock_cpuid_data[i].input_eax && + ecx == cpuinfo_mock_cpuid_data[i].input_ecx) + { + regs[0] = cpuinfo_mock_cpuid_data[i].eax; + regs[1] = cpuinfo_mock_cpuid_data[i].ebx; + regs[2] = cpuinfo_mock_cpuid_data[i].ecx; + regs[3] = cpuinfo_mock_cpuid_data[i].edx; + return; + } + } + } + regs[0] = regs[1] = regs[2] = regs[3] = 0; +} diff --git a/3rdparty/cpuinfo/src/x86/name.c b/3rdparty/cpuinfo/src/x86/name.c new file mode 100644 index 000000000..957a0d88b --- /dev/null +++ b/3rdparty/cpuinfo/src/x86/name.c @@ -0,0 +1,708 @@ +#include +#include +#include +#include +#include + +#include +#include +#include + + +/* The state of the parser to be preserved between parsing different tokens. */ +struct parser_state { + /* + * Pointer to the start of the previous token if it is "model". + * NULL if previous token is not "model". + */ + char* context_model; + /* + * Pointer to the start of the previous token if it is a single-uppercase-letter token. + * NULL if previous token is anything different. + */ + char* context_upper_letter; + /* + * Pointer to the start of the previous token if it is "Dual". + * NULL if previous token is not "Dual". + */ + char* context_dual; + /* + * Pointer to the start of the previous token if it is "Core", "Dual-Core", "QuadCore", etc. + * NULL if previous token is anything different. + */ + char* context_core; + /* + * Pointer to the start of the previous token if it is "Eng" or "Engineering", etc. + * NULL if previous token is anything different. + */ + char* context_engineering; + /* + * Pointer to the '@' symbol in the brand string (separates frequency specification). + * NULL if there is no '@' symbol. + */ + char* frequency_separator; + /* Indicates whether the brand string (after transformations) contains frequency. */ + bool frequency_token; + /* Indicates whether the processor is of Xeon family (contains "Xeon" substring). */ + bool xeon; + /* Indicates whether the processor model number was already parsed. */ + bool parsed_model_number; + /* Indicates whether the processor is an engineering sample (contains "Engineering Sample" or "Eng Sample" substrings). */ + bool engineering_sample; +}; + +/** @brief Resets information about the previous token. Keeps all other state information. */ +static void reset_context(struct parser_state* state) { + state->context_model = NULL; + state->context_upper_letter = NULL; + state->context_dual = NULL; + state->context_core = NULL; +} + +/** + * @brief Overwrites the supplied string with space characters if it exactly matches the given string. + * @param string The string to be compared against other string, and erased in case of matching. + * @param length The length of the two string to be compared against each other. + * @param target The string to compare against. + * @retval true If the two strings match and the first supplied string was erased (overwritten with space characters). + * @retval false If the two strings are different and the first supplied string remained unchanged. + */ +static inline bool erase_matching(char* string, size_t length, const char* target) { + const bool match = memcmp(string, target, length) == 0; + if (match) { + memset(string, ' ', length); + } + return match; +} + +/** + * @brief Checks if the supplied ASCII character is an uppercase latin letter. + * @param character The character to analyse. + * @retval true If the supplied character is an uppercase latin letter ('A' to 'Z'). + * @retval false If the supplied character is anything different. + */ +static inline bool is_upper_letter(char character) { + return (uint32_t) (character - 'A') <= (uint32_t)('Z' - 'A'); +} + +/** + * @brief Checks if the supplied ASCII character is a digit. + * @param character The character to analyse. + * @retval true If the supplied character is a digit ('0' to '9'). + * @retval false If the supplied character is anything different. + */ +static inline bool is_digit(char character) { + return (uint32_t) (character - '0') < UINT32_C(10); +} + +static inline bool is_zero_number(const char* token_start, const char* token_end) { + for (const char* char_ptr = token_start; char_ptr != token_end; char_ptr++) { + if (*char_ptr != '0') { + return false; + } + } + return true; +} + +static inline bool is_space(const char* token_start, const char* token_end) { + for (const char* char_ptr = token_start; char_ptr != token_end; char_ptr++) { + if (*char_ptr != ' ') { + return false; + } + } + return true; +} + +static inline bool is_number(const char* token_start, const char* token_end) { + for (const char* char_ptr = token_start; char_ptr != token_end; char_ptr++) { + if (!is_digit(*char_ptr)) { + return false; + } + } + return true; +} + +static inline bool is_model_number(const char* token_start, const char* token_end) { + for (const char* char_ptr = token_start + 1; char_ptr < token_end; char_ptr++) { + if (is_digit(char_ptr[-1]) && is_digit(char_ptr[0])) { + return true; + } + } + return false; +} + +static inline bool is_frequency(const char* token_start, const char* token_end) { + const size_t token_length = (size_t) (token_end - token_start); + if (token_length > 3 && token_end[-2] == 'H' && token_end[-1] == 'z') { + switch (token_end[-3]) { + case 'K': + case 'M': + case 'G': + return true; + } + } + return false; +} + +/** + * @warning Input and output tokens can overlap + */ +static inline char* move_token(const char* token_start, const char* token_end, char* output_ptr) { + const size_t token_length = (size_t) (token_end - token_start); + memmove(output_ptr, token_start, token_length); + return output_ptr + token_length; +} + +static bool transform_token(char* token_start, char* token_end, struct parser_state* state) { + const struct parser_state previousState = *state; + reset_context(state); + + size_t token_length = (size_t) (token_end - token_start); + + if (state->frequency_separator != NULL) { + if (token_start > state->frequency_separator) { + if (state->parsed_model_number) { + memset(token_start, ' ', token_length); + } + } + } + + + /* Early AMD and Cyrix processors have "tm" suffix for trademark, e.g. + * "AMD-K6tm w/ multimedia extensions" + * "Cyrix MediaGXtm MMXtm Enhanced" + */ + if (token_length > 2) { + const char context_char = token_end[-3]; + if (is_digit(context_char) || is_upper_letter(context_char)) { + if (erase_matching(token_end - 2, 2, "tm")) { + token_end -= 2; + token_length -= 2; + } + } + } + if (token_length > 4) { + /* Some early AMD CPUs have "AMD-" at the beginning, e.g. + * "AMD-K5(tm) Processor" + * "AMD-K6tm w/ multimedia extensions" + * "AMD-K6(tm) 3D+ Processor" + * "AMD-K6(tm)-III Processor" + */ + if (erase_matching(token_start, 4, "AMD-")) { + token_start += 4; + token_length -= 4; + } + } + switch (token_length) { + case 1: + /* + * On some Intel processors there is a space between the first letter of + * the name and the number after it, e.g. + * "Intel(R) Core(TM) i7 CPU X 990 @ 3.47GHz" + * "Intel(R) Core(TM) CPU Q 820 @ 1.73GHz" + * We want to merge these parts together, in reverse order, i.e. "X 990" -> "990X", "820" -> "820Q" + */ + if (is_upper_letter(token_start[0])) { + state->context_upper_letter = token_start; + return true; + } + break; + case 2: + /* Erase everything after "w/" in "AMD-K6tm w/ multimedia extensions" */ + if (erase_matching(token_start, token_length, "w/")) { + return false; + } + /* + * Intel Xeon processors since Ivy Bridge use versions, e.g. + * "Intel Xeon E3-1230 v2" + * Some processor branch strings report them as "V", others report as "v". + * Normalize the former (upper-case) to the latter (lower-case) version + */ + if (token_start[0] == 'V' && is_digit(token_start[1])) { + token_start[0] = 'v'; + return true; + } + break; + case 3: + /* + * Erase "CPU" in brand string on Intel processors, e.g. + * "Intel(R) Core(TM) i5 CPU 650 @ 3.20GHz" + * "Intel(R) Xeon(R) CPU X3210 @ 2.13GHz" + * "Intel(R) Atom(TM) CPU Z2760 @ 1.80GHz" + */ + if (erase_matching(token_start, token_length, "CPU")) { + return true; + } + /* + * Erase everything after "SOC" on AMD System-on-Chips, e.g. + * "AMD GX-212JC SOC with Radeon(TM) R2E Graphics \0" + */ + if (erase_matching(token_start, token_length, "SOC")) { + return false; + } + /* + * Erase "AMD" in brand string on AMD processors, e.g. + * "AMD Athlon(tm) Processor" + * "AMD Engineering Sample" + * "Quad-Core AMD Opteron(tm) Processor 2344 HE" + */ + if (erase_matching(token_start, token_length, "AMD")) { + return true; + } + /* + * Erase "VIA" in brand string on VIA processors, e.g. + * "VIA C3 Ezra" + * "VIA C7-M Processor 1200MHz" + * "VIA Nano L3050@1800MHz" + */ + if (erase_matching(token_start, token_length, "VIA")) { + return true; + } + /* Erase "IDT" in brand string on early Centaur processors, e.g. "IDT WinChip 2-3D" */ + if (erase_matching(token_start, token_length, "IDT")) { + return true; + } + /* + * Erase everything starting with "MMX" in + * "Cyrix MediaGXtm MMXtm Enhanced" ("tm" suffix is removed by this point) + */ + if (erase_matching(token_start, token_length, "MMX")) { + return false; + } + /* + * Erase everything starting with "APU" on AMD processors, e.g. + * "AMD A10-4600M APU with Radeon(tm) HD Graphics" + * "AMD A10-7850K APU with Radeon(TM) R7 Graphics" + * "AMD A6-6310 APU with AMD Radeon R4 Graphics" + */ + if (erase_matching(token_start, token_length, "APU")) { + return false; + } + /* + * Remember to discard string if it contains "Eng Sample", + * e.g. "Eng Sample, ZD302046W4K43_36/30/20_2/8_A" + */ + if (memcmp(token_start, "Eng", token_length) == 0) { + state->context_engineering = token_start; + } + break; + case 4: + /* Remember to erase "Dual Core" in "AMD Athlon(tm) 64 X2 Dual Core Processor 3800+" */ + if (memcmp(token_start, "Dual", token_length) == 0) { + state->context_dual = token_start; + } + /* Remember if the processor is on Xeon family */ + if (memcmp(token_start, "Xeon", token_length) == 0) { + state->xeon = true; + } + /* Erase "Dual Core" in "AMD Athlon(tm) 64 X2 Dual Core Processor 3800+" */ + if (previousState.context_dual != NULL) { + if (memcmp(token_start, "Core", token_length) == 0) { + memset(previousState.context_dual, ' ', (size_t) (token_end - previousState.context_dual)); + state->context_core = token_end; + return true; + } + } + break; + case 5: + /* + * Erase "Intel" in brand string on Intel processors, e.g. + * "Intel(R) Xeon(R) CPU X3210 @ 2.13GHz" + * "Intel(R) Atom(TM) CPU D2700 @ 2.13GHz" + * "Genuine Intel(R) processor 800MHz" + */ + if (erase_matching(token_start, token_length, "Intel")) { + return true; + } + /* + * Erase "Cyrix" in brand string on Cyrix processors, e.g. + * "Cyrix MediaGXtm MMXtm Enhanced" + */ + if (erase_matching(token_start, token_length, "Cyrix")) { + return true; + } + /* + * Erase everything following "Geode" (but not "Geode" token itself) on Geode processors, e.g. + * "Geode(TM) Integrated Processor by AMD PCS" + * "Geode(TM) Integrated Processor by National Semi" + */ + if (memcmp(token_start, "Geode", token_length) == 0) { + return false; + } + /* Remember to erase "model unknown" in "AMD Processor model unknown" */ + if (memcmp(token_start, "model", token_length) == 0) { + state->context_model = token_start; + return true; + } + break; + case 6: + /* + * Erase everything starting with "Radeon" or "RADEON" on AMD APUs, e.g. + * "A8-7670K Radeon R7, 10 Compute Cores 4C+6G" + * "FX-8800P Radeon R7, 12 Compute Cores 4C+8G" + * "A12-9800 RADEON R7, 12 COMPUTE CORES 4C+8G" + * "A9-9410 RADEON R5, 5 COMPUTE CORES 2C+3G" + */ + if (erase_matching(token_start, token_length, "Radeon") || erase_matching(token_start, token_length, "RADEON")) { + return false; + } + /* + * Erase "Mobile" when it is not part of the processor name, + * e.g. in "AMD Turion(tm) X2 Ultra Dual-Core Mobile ZM-82" + */ + if (previousState.context_core != NULL) { + if (erase_matching(token_start, token_length, "Mobile")) { + return true; + } + } + /* Erase "family" in "Intel(R) Pentium(R) III CPU family 1266MHz" */ + if (erase_matching(token_start, token_length, "family")) { + return true; + } + /* Discard the string if it contains "Engineering Sample" */ + if (previousState.context_engineering != NULL) { + if (memcmp(token_start, "Sample", token_length) == 0) { + state->engineering_sample = true; + return false; + } + } + break; + case 7: + /* + * Erase "Geniune" in brand string on Intel engineering samples, e.g. + * "Genuine Intel(R) processor 800MHz" + * "Genuine Intel(R) CPU @ 2.13GHz" + * "Genuine Intel(R) CPU 0000 @ 1.73GHz" + */ + if (erase_matching(token_start, token_length, "Genuine")) { + return true; + } + /* + * Erase "12-core" in brand string on AMD Threadripper, e.g. + * "AMD Ryzen Threadripper 1920X 12-Core Processor" + */ + if (erase_matching(token_start, token_length, "12-Core")) { + return true; + } + /* + * Erase "16-core" in brand string on AMD Threadripper, e.g. + * "AMD Ryzen Threadripper 1950X 16-Core Processor" + */ + if (erase_matching(token_start, token_length, "16-Core")) { + return true; + } + /* Erase "model unknown" in "AMD Processor model unknown" */ + if (previousState.context_model != NULL) { + if (memcmp(token_start, "unknown", token_length) == 0) { + memset(previousState.context_model, ' ', token_end - previousState.context_model); + return true; + } + } + /* + * Discard the string if it contains "Eng Sample:" or "Eng Sample," e.g. + * "AMD Eng Sample, ZD302046W4K43_36/30/20_2/8_A" + * "AMD Eng Sample: 2D3151A2M88E4_35/31_N" + */ + if (previousState.context_engineering != NULL) { + if (memcmp(token_start, "Sample,", token_length) == 0 || memcmp(token_start, "Sample:", token_length) == 0) { + state->engineering_sample = true; + return false; + } + } + break; + case 8: + /* Erase "QuadCore" in "VIA QuadCore L4700 @ 1.2+ GHz" */ + if (erase_matching(token_start, token_length, "QuadCore")) { + state->context_core = token_end; + return true; + } + /* Erase "Six-Core" in "AMD FX(tm)-6100 Six-Core Processor" */ + if (erase_matching(token_start, token_length, "Six-Core")) { + state->context_core = token_end; + return true; + } + break; + case 9: + if (erase_matching(token_start, token_length, "Processor")) { + return true; + } + if (erase_matching(token_start, token_length, "processor")) { + return true; + } + /* Erase "Dual-Core" in "Pentium(R) Dual-Core CPU T4200 @ 2.00GHz" */ + if (erase_matching(token_start, token_length, "Dual-Core")) { + state->context_core = token_end; + return true; + } + /* Erase "Quad-Core" in AMD processors, e.g. + * "Quad-Core AMD Opteron(tm) Processor 2347 HE" + * "AMD FX(tm)-4170 Quad-Core Processor" + */ + if (erase_matching(token_start, token_length, "Quad-Core")) { + state->context_core = token_end; + return true; + } + /* Erase "Transmeta" in brand string on Transmeta processors, e.g. + * "Transmeta(tm) Crusoe(tm) Processor TM5800" + * "Transmeta Efficeon(tm) Processor TM8000" + */ + if (erase_matching(token_start, token_length, "Transmeta")) { + return true; + } + break; + case 10: + /* + * Erase "Eight-Core" in AMD processors, e.g. + * "AMD FX(tm)-8150 Eight-Core Processor" + */ + if (erase_matching(token_start, token_length, "Eight-Core")) { + state->context_core = token_end; + return true; + } + break; + case 11: + /* + * Erase "Triple-Core" in AMD processors, e.g. + * "AMD Phenom(tm) II N830 Triple-Core Processor" + * "AMD Phenom(tm) 8650 Triple-Core Processor" + */ + if (erase_matching(token_start, token_length, "Triple-Core")) { + state->context_core = token_end; + return true; + } + /* + * Remember to discard string if it contains "Engineering Sample", + * e.g. "AMD Engineering Sample" + */ + if (memcmp(token_start, "Engineering", token_length) == 0) { + state->context_engineering = token_start; + return true; + } + break; + } + if (is_zero_number(token_start, token_end)) { + memset(token_start, ' ', token_length); + return true; + } + /* On some Intel processors the last letter of the name is put before the number, + * and an additional space it added, e.g. + * "Intel(R) Core(TM) i7 CPU X 990 @ 3.47GHz" + * "Intel(R) Core(TM) CPU Q 820 @ 1.73GHz" + * "Intel(R) Core(TM) i5 CPU M 480 @ 2.67GHz" + * We fix this issue, i.e. "X 990" -> "990X", "Q 820" -> "820Q" + */ + if (previousState.context_upper_letter != 0) { + /* A single letter token followed by 2-to-5 digit letter is merged together */ + switch (token_length) { + case 2: + case 3: + case 4: + case 5: + if (is_number(token_start, token_end)) { + /* Load the previous single-letter token */ + const char letter = *previousState.context_upper_letter; + /* Erase the previous single-letter token */ + *previousState.context_upper_letter = ' '; + /* Move the current token one position to the left */ + move_token(token_start, token_end, token_start - 1); + token_start -= 1; + /* + * Add the letter on the end + * Note: accessing token_start[-1] is safe because this is not the first token + */ + token_end[-1] = letter; + } + } + } + if (state->frequency_separator != NULL) { + if (is_model_number(token_start, token_end)) { + state->parsed_model_number = true; + } + } + if (is_frequency(token_start, token_end)) { + state->frequency_token = true; + } + return true; +} + +uint32_t cpuinfo_x86_normalize_brand_string( + const char raw_name[48], + char normalized_name[48]) +{ + normalized_name[0] = '\0'; + char name[48]; + memcpy(name, raw_name, sizeof(name)); + + /* + * First find the end of the string + * Start search from the end because some brand strings contain zeroes in the middle + */ + char* name_end = &name[48]; + while (name_end[-1] == '\0') { + /* + * Adject name_end by 1 position and check that we didn't reach the start of the brand string. + * This is possible if all characters are zero. + */ + if (--name_end == name) { + /* All characters are zeros */ + return 0; + } + } + + struct parser_state parser_state = { 0 }; + + /* Now unify all whitespace characters: replace tabs and '\0' with spaces */ + { + bool inside_parentheses = false; + for (char* char_ptr = name; char_ptr != name_end; char_ptr++) { + switch (*char_ptr) { + case '(': + inside_parentheses = true; + *char_ptr = ' '; + break; + case ')': + inside_parentheses = false; + *char_ptr = ' '; + break; + case '@': + parser_state.frequency_separator = char_ptr; + case '\0': + case '\t': + *char_ptr = ' '; + break; + default: + if (inside_parentheses) { + *char_ptr = ' '; + } + } + } + } + + /* Iterate through all tokens and erase redundant parts */ + { + bool is_token = false; + char* token_start; + for (char* char_ptr = name; char_ptr != name_end; char_ptr++) { + if (*char_ptr == ' ') { + if (is_token) { + is_token = false; + if (!transform_token(token_start, char_ptr, &parser_state)) { + name_end = char_ptr; + break; + } + } + } else { + if (!is_token) { + is_token = true; + token_start = char_ptr; + } + } + } + if (is_token) { + transform_token(token_start, name_end, &parser_state); + } + } + + /* If this is an engineering sample, return empty string */ + if (parser_state.engineering_sample) { + return 0; + } + + /* Check if there is some string before the frequency separator. */ + if (parser_state.frequency_separator != NULL) { + if (is_space(name, parser_state.frequency_separator)) { + /* If only frequency is available, return empty string */ + return 0; + } + } + + /* Compact tokens: collapse multiple spacing into one */ + { + char* output_ptr = normalized_name; + char* token_start; + bool is_token = false; + bool previous_token_ends_with_dash = true; + bool current_token_starts_with_dash = false; + uint32_t token_count = 1; + for (char* char_ptr = name; char_ptr != name_end; char_ptr++) { + const char character = *char_ptr; + if (character == ' ') { + if (is_token) { + is_token = false; + if (!current_token_starts_with_dash && !previous_token_ends_with_dash) { + token_count += 1; + *output_ptr++ = ' '; + } + output_ptr = move_token(token_start, char_ptr, output_ptr); + /* Note: char_ptr[-1] exists because there is a token before this space */ + previous_token_ends_with_dash = (char_ptr[-1] == '-'); + } + } else { + if (!is_token) { + is_token = true; + token_start = char_ptr; + current_token_starts_with_dash = (character == '-'); + } + } + } + if (is_token) { + if (!current_token_starts_with_dash && !previous_token_ends_with_dash) { + token_count += 1; + *output_ptr++ = ' '; + } + output_ptr = move_token(token_start, name_end, output_ptr); + } + if (parser_state.frequency_token && token_count <= 1) { + /* The only remaining part is frequency */ + normalized_name[0] = '\0'; + return 0; + } + if (output_ptr < &normalized_name[48]) { + *output_ptr = '\0'; + } else { + normalized_name[47] = '\0'; + } + return (uint32_t) (output_ptr - normalized_name); + } +} + +static const char* vendor_string_map[] = { + [cpuinfo_vendor_intel] = "Intel", + [cpuinfo_vendor_amd] = "AMD", + [cpuinfo_vendor_via] = "VIA", + [cpuinfo_vendor_hygon] = "Hygon", + [cpuinfo_vendor_rdc] = "RDC", + [cpuinfo_vendor_dmp] = "DM&P", + [cpuinfo_vendor_transmeta] = "Transmeta", + [cpuinfo_vendor_cyrix] = "Cyrix", + [cpuinfo_vendor_rise] = "Rise", + [cpuinfo_vendor_nsc] = "NSC", + [cpuinfo_vendor_sis] = "SiS", + [cpuinfo_vendor_nexgen] = "NexGen", + [cpuinfo_vendor_umc] = "UMC", +}; + +uint32_t cpuinfo_x86_format_package_name( + enum cpuinfo_vendor vendor, + const char normalized_brand_string[48], + char package_name[CPUINFO_PACKAGE_NAME_MAX]) +{ + if (normalized_brand_string[0] == '\0') { + package_name[0] = '\0'; + return 0; + } + + const char* vendor_string = NULL; + if ((uint32_t) vendor < (uint32_t) CPUINFO_COUNT_OF(vendor_string_map)) { + vendor_string = vendor_string_map[(uint32_t) vendor]; + } + if (vendor_string == NULL) { + strncpy(package_name, normalized_brand_string, CPUINFO_PACKAGE_NAME_MAX); + package_name[CPUINFO_PACKAGE_NAME_MAX - 1] = '\0'; + return 0; + } else { + snprintf(package_name, CPUINFO_PACKAGE_NAME_MAX, + "%s %s", vendor_string, normalized_brand_string); + return (uint32_t) strlen(vendor_string) + 1; + } +} diff --git a/3rdparty/cpuinfo/src/x86/topology.c b/3rdparty/cpuinfo/src/x86/topology.c new file mode 100644 index 000000000..0e83d4683 --- /dev/null +++ b/3rdparty/cpuinfo/src/x86/topology.c @@ -0,0 +1,127 @@ +#include +#include + +#include +#include +#include +#include +#include + + +enum topology_type { + topology_type_invalid = 0, + topology_type_smt = 1, + topology_type_core = 2, +}; + +void cpuinfo_x86_detect_topology( + uint32_t max_base_index, + uint32_t max_extended_index, + struct cpuid_regs leaf1, + struct cpuinfo_x86_topology* topology) +{ + /* + * HTT: indicates multi-core/hyper-threading support on this core. + * - Intel, AMD: edx[bit 28] in basic info. + */ + const bool htt = !!(leaf1.edx & UINT32_C(0x10000000)); + + uint32_t apic_id = 0; + if (htt) { + apic_id = leaf1.ebx >> 24; + bool amd_cmp_legacy = false; + if (max_extended_index >= UINT32_C(0x80000001)) { + const struct cpuid_regs leaf0x80000001 = cpuid(UINT32_C(0x80000001)); + /* + * CmpLegacy: core multi-processing legacy mode. + * - AMD: ecx[bit 1] in extended info (reserved bit on Intel CPUs). + */ + amd_cmp_legacy = !!(leaf0x80000001.ecx & UINT32_C(0x00000002)); + } + if (amd_cmp_legacy) { + if (max_extended_index >= UINT32_C(0x80000008)) { + const struct cpuid_regs leaf0x80000008 = cpuid(UINT32_C(0x80000008)); + /* + * NC: number of physical cores - 1. The number of cores in the processor is NC+1. + * - AMD: ecx[bits 0-7] in leaf 0x80000008 (reserved zero bits on Intel CPUs). + */ + const uint32_t cores_per_processor = 1 + (leaf0x80000008.ecx & UINT32_C(0x000000FF)); + topology->core_bits_length = bit_length(cores_per_processor); + cpuinfo_log_debug("HTT: APIC ID = %08"PRIx32", cores per processor = %"PRIu32, apic_id, cores_per_processor); + } else { + /* + * LogicalProcessorCount: the number of cores per processor. + * - AMD: ebx[bits 16-23] in basic info (different interpretation on Intel CPUs). + */ + const uint32_t cores_per_processor = (leaf1.ebx >> 16) & UINT32_C(0x000000FF); + if (cores_per_processor != 0) { + topology->core_bits_length = bit_length(cores_per_processor); + } + cpuinfo_log_debug("HTT: APIC ID = %08"PRIx32", cores per processor = %"PRIu32, apic_id, cores_per_processor); + } + } else { + /* + * Maximum number of addressable IDs for logical processors in this physical package. + * - Intel: ebx[bits 16-23] in basic info (different interpretation on AMD CPUs). + */ + const uint32_t logical_processors = (leaf1.ebx >> 16) & UINT32_C(0x000000FF); + if (logical_processors != 0) { + const uint32_t log2_max_logical_processors = bit_length(logical_processors); + const uint32_t log2_max_threads_per_core = log2_max_logical_processors - topology->core_bits_length; + topology->core_bits_offset = log2_max_threads_per_core; + topology->thread_bits_length = log2_max_threads_per_core; + } + cpuinfo_log_debug("HTT: APIC ID = %08"PRIx32", logical processors = %"PRIu32, apic_id, logical_processors); + } + } + + /* + * x2APIC: indicated support for x2APIC feature. + * - Intel: ecx[bit 21] in basic info (reserved bit on AMD CPUs). + */ + const bool x2apic = !!(leaf1.ecx & UINT32_C(0x00200000)); + if (x2apic && (max_base_index >= UINT32_C(0xB))) { + uint32_t level = 0; + uint32_t type; + uint32_t total_shift = 0; + topology->thread_bits_offset = topology->thread_bits_length = 0; + topology->core_bits_offset = topology->core_bits_length = 0; + do { + const struct cpuid_regs leafB = cpuidex(UINT32_C(0xB), level); + type = (leafB.ecx >> 8) & UINT32_C(0x000000FF); + const uint32_t level_shift = leafB.eax & UINT32_C(0x0000001F); + const uint32_t x2apic_id = leafB.edx; + apic_id = x2apic_id; + switch (type) { + case topology_type_invalid: + break; + case topology_type_smt: + cpuinfo_log_debug("x2 level %"PRIu32": APIC ID = %08"PRIx32", " + "type SMT, shift %"PRIu32", total shift %"PRIu32, + level, apic_id, level_shift, total_shift); + topology->thread_bits_offset = total_shift; + topology->thread_bits_length = level_shift; + break; + case topology_type_core: + cpuinfo_log_debug("x2 level %"PRIu32": APIC ID = %08"PRIx32", " + "type core, shift %"PRIu32", total shift %"PRIu32, + level, apic_id, level_shift, total_shift); + topology->core_bits_offset = total_shift; + topology->core_bits_length = level_shift; + break; + default: + cpuinfo_log_warning("unexpected topology type %"PRIu32" (offset %"PRIu32", length %"PRIu32") " + "reported in leaf 0x0000000B is ignored", type, total_shift, level_shift); + break; + } + total_shift += level_shift; + level += 1; + } while (type != 0); + cpuinfo_log_debug("x2APIC ID 0x%08"PRIx32", " + "SMT offset %"PRIu32" length %"PRIu32", core offset %"PRIu32" length %"PRIu32, apic_id, + topology->thread_bits_offset, topology->thread_bits_length, + topology->core_bits_offset, topology->core_bits_length); + } + + topology->apic_id = apic_id; +} diff --git a/3rdparty/cpuinfo/src/x86/uarch.c b/3rdparty/cpuinfo/src/x86/uarch.c new file mode 100644 index 000000000..370549942 --- /dev/null +++ b/3rdparty/cpuinfo/src/x86/uarch.c @@ -0,0 +1,241 @@ +#include + +#include +#include + + +enum cpuinfo_uarch cpuinfo_x86_decode_uarch( + enum cpuinfo_vendor vendor, + const struct cpuinfo_x86_model_info* model_info) +{ + switch (vendor) { + case cpuinfo_vendor_intel: + switch (model_info->family) { +#if CPUINFO_ARCH_X86 + case 0x05: + switch (model_info->model) { + case 0x01: // Pentium (60, 66) + case 0x02: // Pentium (75, 90, 100, 120, 133, 150, 166, 200) + case 0x03: // Pentium OverDrive for Intel486-based systems + case 0x04: // Pentium MMX + return cpuinfo_uarch_p5; + case 0x09: + return cpuinfo_uarch_quark; + } + break; +#endif /* CPUINFO_ARCH_X86 */ + case 0x06: + switch (model_info->model) { + /* Mainstream cores */ +#if CPUINFO_ARCH_X86 + case 0x01: // Pentium Pro + case 0x03: // Pentium II (Klamath) and Pentium II Overdrive + case 0x05: // Pentium II (Deschutes, Tonga), Pentium II Celeron (Covington), Pentium II Xeon (Drake) + case 0x06: // Pentium II (Dixon), Pentium II Celeron (Mendocino) + case 0x07: // Pentium III (Katmai), Pentium III Xeon (Tanner) + case 0x08: // Pentium III (Coppermine), Pentium II Celeron (Coppermine-128), Pentium III Xeon (Cascades) + case 0x0A: // Pentium III Xeon (Cascades-2MB) + case 0x0B: // Pentium III (Tualatin), Pentium III Celeron (Tualatin-256) + return cpuinfo_uarch_p6; + case 0x09: // Pentium M (Banias), Pentium M Celeron (Banias-0, Banias-512) + case 0x0D: // Pentium M (Dothan), Pentium M Celeron (Dothan-512, Dothan-1024) + case 0x15: // Intel 80579 (Tolapai) + return cpuinfo_uarch_dothan; + case 0x0E: // Core Solo/Duo (Yonah), Pentium Dual-Core T2xxx (Yonah), Celeron M (Yonah-512, Yonah-1024), Dual-Core Xeon (Sossaman) + return cpuinfo_uarch_yonah; +#endif /* CPUINFO_ARCH_X86 */ + case 0x0F: // Core 2 Duo (Conroe, Conroe-2M, Merom), Core 2 Quad (Tigerton), Xeon (Woodcrest, Clovertown, Kentsfield) + case 0x16: // Celeron (Conroe-L, Merom-L), Core 2 Duo (Merom) + return cpuinfo_uarch_conroe; + case 0x17: // Core 2 Duo (Penryn-3M), Core 2 Quad (Yorkfield), Core 2 Extreme (Yorkfield), Xeon (Harpertown), Pentium Dual-Core (Penryn) + case 0x1D: // Xeon (Dunnington) + return cpuinfo_uarch_penryn; + case 0x1A: // Core iX (Bloomfield), Xeon (Gainestown) + case 0x1E: // Core iX (Lynnfield, Clarksfield) + case 0x1F: // Core iX (Havendale) + case 0x2E: // Xeon (Beckton) + case 0x25: // Core iX (Clarkdale) + case 0x2C: // Core iX (Gulftown), Xeon (Gulftown) + case 0x2F: // Xeon (Eagleton) + return cpuinfo_uarch_nehalem; + case 0x2A: // Core iX (Sandy Bridge) + case 0x2D: // Core iX (Sandy Bridge-E), Xeon (Sandy Bridge EP/EX) + return cpuinfo_uarch_sandy_bridge; + case 0x3A: // Core iX (Ivy Bridge) + case 0x3E: // Ivy Bridge-E + return cpuinfo_uarch_ivy_bridge; + case 0x3C: + case 0x3F: // Haswell-E + case 0x45: // Haswell ULT + case 0x46: // Haswell with eDRAM + return cpuinfo_uarch_haswell; + case 0x3D: // Broadwell-U + case 0x47: // Broadwell-H + case 0x4F: // Broadwell-E + case 0x56: // Broadwell-DE + return cpuinfo_uarch_broadwell; + case 0x4E: // Sky Lake Client Y/U + case 0x55: // Sky/Cascade/Cooper Lake Server + case 0x5E: // Sky Lake Client DT/H/S + case 0x8E: // Kaby/Whiskey/Amber/Comet Lake Y/U + case 0x9E: // Kaby/Coffee Lake DT/H/S + case 0xA5: // Comet Lake H/S + case 0xA6: // Comet Lake U/Y + return cpuinfo_uarch_sky_lake; + case 0x66: // Cannon Lake (Core i3-8121U) + return cpuinfo_uarch_palm_cove; + case 0x6A: // Ice Lake-DE + case 0x6C: // Ice Lake-SP + case 0x7D: // Ice Lake-Y + case 0x7E: // Ice Lake-U + return cpuinfo_uarch_sunny_cove; + + /* Low-power cores */ + case 0x1C: // Diamondville, Silverthorne, Pineview + case 0x26: // Tunnel Creek + return cpuinfo_uarch_bonnell; + case 0x27: // Medfield + case 0x35: // Cloverview + case 0x36: // Cedarview, Centerton + return cpuinfo_uarch_saltwell; + case 0x37: // Bay Trail + case 0x4A: // Merrifield + case 0x4D: // Avoton, Rangeley + case 0x5A: // Moorefield + case 0x5D: // SoFIA + return cpuinfo_uarch_silvermont; + case 0x4C: // Braswell, Cherry Trail + case 0x75: // Spreadtrum SC9853I-IA + return cpuinfo_uarch_airmont; + case 0x5C: // Apollo Lake + case 0x5F: // Denverton + return cpuinfo_uarch_goldmont; + case 0x7A: // Gemini Lake + return cpuinfo_uarch_goldmont_plus; + + /* Knights-series cores */ + case 0x57: + return cpuinfo_uarch_knights_landing; + case 0x85: + return cpuinfo_uarch_knights_mill; + } + break; + case 0x0F: + switch (model_info->model) { + case 0x00: // Pentium 4 Xeon (Foster) + case 0x01: // Pentium 4 Celeron (Willamette-128), Pentium 4 Xeon (Foster, Foster MP) + case 0x02: // Pentium 4 (Northwood), Pentium 4 EE (Gallatin), Pentium 4 Celeron (Northwood-128, Northwood-256), Pentium 4 Xeon (Gallatin DP, Prestonia) + return cpuinfo_uarch_willamette; + break; + case 0x03: // Pentium 4 (Prescott), Pentium 4 Xeon (Nocona) + case 0x04: // Pentium 4 (Prescott-2M), Pentium 4 EE (Prescott-2M), Pentium D (Smithfield), Celeron D (Prescott-256), Pentium 4 Xeon (Cranford, Irwindale, Paxville) + case 0x06: // Pentium 4 (Cedar Mill), Pentium D EE (Presler), Celeron D (Cedar Mill), Pentium 4 Xeon (Dempsey, Tulsa) + return cpuinfo_uarch_prescott; + } + break; + } + break; + case cpuinfo_vendor_amd: + switch (model_info->family) { +#if CPUINFO_ARCH_X86 + case 0x5: + switch (model_info->model) { + case 0x00: + case 0x01: + case 0x02: + return cpuinfo_uarch_k5; + case 0x06: + case 0x07: + case 0x08: + case 0x0D: + return cpuinfo_uarch_k6; + case 0x0A: + return cpuinfo_uarch_geode; + } + break; + case 0x6: + return cpuinfo_uarch_k7; +#endif /* CPUINFO_ARCH_X86 */ + case 0xF: // Opteron, Athlon 64, Sempron + case 0x11: // Turion + return cpuinfo_uarch_k8; + case 0x10: // Opteron, Phenom, Athlon, Sempron + case 0x12: // Llano APU + return cpuinfo_uarch_k10; + case 0x14: + return cpuinfo_uarch_bobcat; + case 0x15: + switch (model_info->model) { + case 0x00: // Engineering samples + case 0x01: // Zambezi, Interlagos + return cpuinfo_uarch_bulldozer; + case 0x02: // Vishera + case 0x10: // Trinity + case 0x13: // Richland + return cpuinfo_uarch_piledriver; + case 0x38: // Godavari + case 0x30: // Kaveri + return cpuinfo_uarch_steamroller; + case 0x60: // Carrizo + case 0x65: // Bristol Ridge + case 0x70: // Stoney Ridge + return cpuinfo_uarch_excavator; + default: + switch (model_info->extended_model) { + case 0x0: + return cpuinfo_uarch_bulldozer; + case 0x1: // No L3 cache + case 0x2: // With L3 cache + return cpuinfo_uarch_piledriver; + case 0x3: // With L3 cache + case 0x4: // No L3 cache + return cpuinfo_uarch_steamroller; + } + break; + } + break; + case 0x16: + if (model_info->model >= 0x03) { + return cpuinfo_uarch_puma; + } else { + return cpuinfo_uarch_jaguar; + } + case 0x17: + switch (model_info->model) { + case 0x01: // 14 nm Naples, Whitehaven, Summit Ridge, Snowy Owl + case 0x08: // 12 nm Pinnacle Ridge + case 0x11: // 14 nm Raven Ridge, Great Horned Owl + case 0x18: // 12 nm Picasso + return cpuinfo_uarch_zen; + case 0x31: // Rome, Castle Peak + case 0x60: // Renoir + case 0x68: // Lucienne + case 0x71: // Matisse + case 0x90: // Van Gogh + case 0x98: // Mero + return cpuinfo_uarch_zen2; + } + break; + case 0x19: + switch (model_info->model) { + case 0x01: // Genesis + case 0x21: // Vermeer + case 0x30: // Badami, Trento + case 0x40: // Rembrandt + case 0x50: // Cezanne + return cpuinfo_uarch_zen3; + } + break; + } + break; + case cpuinfo_vendor_hygon: + switch (model_info->family) { + case 0x00: + return cpuinfo_uarch_dhyana; + } + break; + default: + break; + } + return cpuinfo_uarch_unknown; +} diff --git a/3rdparty/cpuinfo/src/x86/vendor.c b/3rdparty/cpuinfo/src/x86/vendor.c new file mode 100644 index 000000000..bad50fa95 --- /dev/null +++ b/3rdparty/cpuinfo/src/x86/vendor.c @@ -0,0 +1,189 @@ +#include + +#include +#include + + +/* Intel vendor string: "GenuineIntel" */ +#define Genu UINT32_C(0x756E6547) +#define ineI UINT32_C(0x49656E69) +#define ntel UINT32_C(0x6C65746E) + +/* AMD vendor strings: "AuthenticAMD", "AMDisbetter!", "AMD ISBETTER" */ +#define Auth UINT32_C(0x68747541) +#define enti UINT32_C(0x69746E65) +#define cAMD UINT32_C(0x444D4163) +#define AMDi UINT32_C(0x69444D41) +#define sbet UINT32_C(0x74656273) +#define ter UINT32_C(0x21726574) +#define AMD UINT32_C(0x20444D41) +#define ISBE UINT32_C(0x45425349) +#define TTER UINT32_C(0x52455454) + +/* VIA (Centaur) vendor strings: "CentaurHauls", "VIA VIA VIA " */ +#define Cent UINT32_C(0x746E6543) +#define aurH UINT32_C(0x48727561) +#define auls UINT32_C(0x736C7561) +#define VIA UINT32_C(0x20414956) + +/* Hygon vendor string: "HygonGenuine" */ +#define Hygo UINT32_C(0x6F677948) +#define nGen UINT32_C(0x6E65476E) +#define uine UINT32_C(0x656E6975) + +/* Transmeta vendor strings: "GenuineTMx86", "TransmetaCPU" */ +#define ineT UINT32_C(0x54656E69) +#define Mx86 UINT32_C(0x3638784D) +#define Tran UINT32_C(0x6E617254) +#define smet UINT32_C(0x74656D73) +#define aCPU UINT32_C(0x55504361) + +/* Cyrix vendor string: "CyrixInstead" */ +#define Cyri UINT32_C(0x69727943) +#define xIns UINT32_C(0x736E4978) +#define tead UINT32_C(0x64616574) + +/* Rise vendor string: "RiseRiseRise" */ +#define Rise UINT32_C(0x65736952) + +/* NSC vendor string: "Geode by NSC" */ +#define Geod UINT32_C(0x646F6547) +#define e_by UINT32_C(0x79622065) +#define NSC UINT32_C(0x43534E20) + +/* SiS vendor string: "SiS SiS SiS " */ +#define SiS UINT32_C(0x20536953) + +/* NexGen vendor string: "NexGenDriven" */ +#define NexG UINT32_C(0x4778654E) +#define enDr UINT32_C(0x72446E65) +#define iven UINT32_C(0x6E657669) + +/* UMC vendor string: "UMC UMC UMC " */ +#define UMC UINT32_C(0x20434D55) + +/* RDC vendor string: "Genuine RDC" */ +#define ine UINT32_C(0x20656E69) +#define RDC UINT32_C(0x43445220) + +/* D&MP vendor string: "Vortex86 SoC" */ +#define Vort UINT32_C(0x74726F56) +#define ex86 UINT32_C(0x36387865) +#define SoC UINT32_C(0x436F5320) + + +enum cpuinfo_vendor cpuinfo_x86_decode_vendor(uint32_t ebx, uint32_t ecx, uint32_t edx) { + switch (ebx) { + case Genu: + switch (edx) { + case ineI: + if (ecx == ntel) { + /* "GenuineIntel" */ + return cpuinfo_vendor_intel; + } + break; +#if CPUINFO_ARCH_X86 + case ineT: + if (ecx == Mx86) { + /* "GenuineTMx86" */ + return cpuinfo_vendor_transmeta; + } + break; + case ine: + if (ecx == RDC) { + /* "Genuine RDC" */ + return cpuinfo_vendor_rdc; + } + break; +#endif + } + break; + case Auth: + if (edx == enti && ecx == cAMD) { + /* "AuthenticAMD" */ + return cpuinfo_vendor_amd; + } + break; + case Cent: + if (edx == aurH && ecx == auls) { + /* "CentaurHauls" */ + return cpuinfo_vendor_via; + } + break; + case Hygo: + if (edx == nGen && ecx == uine) { + /* "HygonGenuine" */ + return cpuinfo_vendor_hygon; + } + break; +#if CPUINFO_ARCH_X86 + case AMDi: + if (edx == sbet && ecx == ter) { + /* "AMDisbetter!" */ + return cpuinfo_vendor_amd; + } + break; + case AMD: + if (edx == ISBE && ecx == TTER) { + /* "AMD ISBETTER" */ + return cpuinfo_vendor_amd; + } + break; + case VIA: + if (edx == VIA && ecx == VIA) { + /* "VIA VIA VIA " */ + return cpuinfo_vendor_via; + } + break; + case Tran: + if (edx == smet && ecx == aCPU) { + /* "TransmetaCPU" */ + return cpuinfo_vendor_transmeta; + } + break; + case Cyri: + if (edx == xIns && ecx == tead) { + /* "CyrixInstead" */ + return cpuinfo_vendor_cyrix; + } + break; + case Rise: + if (edx == Rise && ecx == Rise) { + /* "RiseRiseRise" */ + return cpuinfo_vendor_rise; + } + break; + case Geod: + if (edx == e_by && ecx == NSC) { + /* "Geode by NSC" */ + return cpuinfo_vendor_nsc; + } + break; + case SiS: + if (edx == SiS && ecx == SiS) { + /* "SiS SiS SiS " */ + return cpuinfo_vendor_sis; + } + break; + case NexG: + if (edx == enDr && ecx == iven) { + /* "NexGenDriven" */ + return cpuinfo_vendor_nexgen; + } + break; + case UMC: + if (edx == UMC && ecx == UMC) { + /* "UMC UMC UMC " */ + return cpuinfo_vendor_umc; + } + break; + case Vort: + if (edx == ex86 && ecx == SoC) { + /* "Vortex86 SoC" */ + return cpuinfo_vendor_dmp; + } + break; +#endif + } + return cpuinfo_vendor_unknown; +} diff --git a/3rdparty/cpuinfo/src/x86/windows/api.h b/3rdparty/cpuinfo/src/x86/windows/api.h new file mode 100644 index 000000000..33d917e0d --- /dev/null +++ b/3rdparty/cpuinfo/src/x86/windows/api.h @@ -0,0 +1,41 @@ +#pragma once + +#include +#include +#include + +#include +#include + +struct cpuinfo_arm_linux_processor { + /** + * Minimum processor ID on the package which includes this logical processor. + * This value can serve as an ID for the cluster of logical processors: it is the + * same for all logical processors on the same package. + */ + uint32_t package_leader_id; + /** + * Minimum processor ID on the core which includes this logical processor. + * This value can serve as an ID for the cluster of logical processors: it is the + * same for all logical processors on the same package. + */ + /** + * Number of logical processors in the package. + */ + uint32_t package_processor_count; + /** + * Maximum frequency, in kHZ. + * The value is parsed from /sys/devices/system/cpu/cpu/cpufreq/cpuinfo_max_freq + * If failed to read or parse the file, the value is 0. + */ + uint32_t max_frequency; + /** + * Minimum frequency, in kHZ. + * The value is parsed from /sys/devices/system/cpu/cpu/cpufreq/cpuinfo_min_freq + * If failed to read or parse the file, the value is 0. + */ + uint32_t min_frequency; + /** Linux processor ID */ + uint32_t system_processor_id; + uint32_t flags; +}; diff --git a/3rdparty/cpuinfo/src/x86/windows/init.c b/3rdparty/cpuinfo/src/x86/windows/init.c new file mode 100644 index 000000000..274075c02 --- /dev/null +++ b/3rdparty/cpuinfo/src/x86/windows/init.c @@ -0,0 +1,634 @@ +#include +#include +#include +#include + +#include +#include +#include +#include + +#include + +#ifdef __GNUC__ + #define CPUINFO_ALLOCA __builtin_alloca +#else + #define CPUINFO_ALLOCA _alloca +#endif + + +static inline uint32_t bit_mask(uint32_t bits) { + return (UINT32_C(1) << bits) - UINT32_C(1); +} + +static inline uint32_t low_index_from_kaffinity(KAFFINITY kaffinity) { + #if defined(_M_X64) || defined(_M_AMD64) + unsigned long index; + _BitScanForward64(&index, (unsigned __int64) kaffinity); + return (uint32_t) index; + #elif defined(_M_IX86) + unsigned long index; + _BitScanForward(&index, (unsigned long) kaffinity); + return (uint32_t) index; + #else + #error Platform-specific implementation required + #endif +} + +static void cpuinfo_x86_count_caches( + uint32_t processors_count, + const struct cpuinfo_processor* processors, + const struct cpuinfo_x86_processor* x86_processor, + uint32_t* l1i_count_ptr, + uint32_t* l1d_count_ptr, + uint32_t* l2_count_ptr, + uint32_t* l3_count_ptr, + uint32_t* l4_count_ptr) +{ + uint32_t l1i_count = 0, l1d_count = 0, l2_count = 0, l3_count = 0, l4_count = 0; + uint32_t last_l1i_id = UINT32_MAX, last_l1d_id = UINT32_MAX; + uint32_t last_l2_id = UINT32_MAX, last_l3_id = UINT32_MAX, last_l4_id = UINT32_MAX; + for (uint32_t i = 0; i < processors_count; i++) { + const uint32_t apic_id = processors[i].apic_id; + cpuinfo_log_debug("APID ID %"PRIu32": logical processor %"PRIu32, apic_id, i); + + if (x86_processor->cache.l1i.size != 0) { + const uint32_t l1i_id = apic_id & ~bit_mask(x86_processor->cache.l1i.apic_bits); + if (l1i_id != last_l1i_id) { + last_l1i_id = l1i_id; + l1i_count++; + } + } + if (x86_processor->cache.l1d.size != 0) { + const uint32_t l1d_id = apic_id & ~bit_mask(x86_processor->cache.l1d.apic_bits); + if (l1d_id != last_l1d_id) { + last_l1d_id = l1d_id; + l1d_count++; + } + } + if (x86_processor->cache.l2.size != 0) { + const uint32_t l2_id = apic_id & ~bit_mask(x86_processor->cache.l2.apic_bits); + if (l2_id != last_l2_id) { + last_l2_id = l2_id; + l2_count++; + } + } + if (x86_processor->cache.l3.size != 0) { + const uint32_t l3_id = apic_id & ~bit_mask(x86_processor->cache.l3.apic_bits); + if (l3_id != last_l3_id) { + last_l3_id = l3_id; + l3_count++; + } + } + if (x86_processor->cache.l4.size != 0) { + const uint32_t l4_id = apic_id & ~bit_mask(x86_processor->cache.l4.apic_bits); + if (l4_id != last_l4_id) { + last_l4_id = l4_id; + l4_count++; + } + } + } + *l1i_count_ptr = l1i_count; + *l1d_count_ptr = l1d_count; + *l2_count_ptr = l2_count; + *l3_count_ptr = l3_count; + *l4_count_ptr = l4_count; +} + +static bool cpuinfo_x86_windows_is_wine(void) { + HMODULE ntdll = GetModuleHandleW(L"ntdll.dll"); + if (ntdll == NULL) { + return false; + } + + return GetProcAddress(ntdll, "wine_get_version") != NULL; +} + +BOOL CALLBACK cpuinfo_x86_windows_init(PINIT_ONCE init_once, PVOID parameter, PVOID* context) { + struct cpuinfo_processor* processors = NULL; + struct cpuinfo_core* cores = NULL; + struct cpuinfo_cluster* clusters = NULL; + struct cpuinfo_package* packages = NULL; + struct cpuinfo_cache* l1i = NULL; + struct cpuinfo_cache* l1d = NULL; + struct cpuinfo_cache* l2 = NULL; + struct cpuinfo_cache* l3 = NULL; + struct cpuinfo_cache* l4 = NULL; + PSYSTEM_LOGICAL_PROCESSOR_INFORMATION_EX processor_infos = NULL; + + HANDLE heap = GetProcessHeap(); + const bool is_wine = cpuinfo_x86_windows_is_wine(); + + struct cpuinfo_x86_processor x86_processor; + ZeroMemory(&x86_processor, sizeof(x86_processor)); + cpuinfo_x86_init_processor(&x86_processor); + char brand_string[48]; + cpuinfo_x86_normalize_brand_string(x86_processor.brand_string, brand_string); + + const uint32_t thread_bits_mask = bit_mask(x86_processor.topology.thread_bits_length); + const uint32_t core_bits_mask = bit_mask(x86_processor.topology.core_bits_length); + const uint32_t package_bits_offset = max( + x86_processor.topology.thread_bits_offset + x86_processor.topology.thread_bits_length, + x86_processor.topology.core_bits_offset + x86_processor.topology.core_bits_length); + + /* WINE doesn't implement GetMaximumProcessorGroupCount and aborts when calling it */ + const uint32_t max_group_count = is_wine ? 1 : (uint32_t) GetMaximumProcessorGroupCount(); + cpuinfo_log_debug("detected %"PRIu32" processor groups", max_group_count); + + uint32_t processors_count = 0; + uint32_t* processors_per_group = (uint32_t*) CPUINFO_ALLOCA(max_group_count * sizeof(uint32_t)); + for (uint32_t i = 0; i < max_group_count; i++) { + processors_per_group[i] = GetMaximumProcessorCount((WORD) i); + cpuinfo_log_debug("detected %"PRIu32" processors in group %"PRIu32, + processors_per_group[i], i); + processors_count += processors_per_group[i]; + } + + uint32_t* processors_before_group = (uint32_t*) CPUINFO_ALLOCA(max_group_count * sizeof(uint32_t)); + for (uint32_t i = 0, count = 0; i < max_group_count; i++) { + processors_before_group[i] = count; + cpuinfo_log_debug("detected %"PRIu32" processors before group %"PRIu32, + processors_before_group[i], i); + count += processors_per_group[i]; + } + + processors = HeapAlloc(heap, HEAP_ZERO_MEMORY, processors_count * sizeof(struct cpuinfo_processor)); + if (processors == NULL) { + cpuinfo_log_error("failed to allocate %zu bytes for descriptions of %"PRIu32" logical processors", + processors_count * sizeof(struct cpuinfo_processor), processors_count); + goto cleanup; + } + + DWORD cores_info_size = 0; + if (GetLogicalProcessorInformationEx(RelationProcessorCore, NULL, &cores_info_size) == FALSE) { + const DWORD last_error = GetLastError(); + if (last_error != ERROR_INSUFFICIENT_BUFFER) { + cpuinfo_log_error("failed to query size of processor cores information: error %"PRIu32, + (uint32_t) last_error); + goto cleanup; + } + } + + DWORD packages_info_size = 0; + if (GetLogicalProcessorInformationEx(RelationProcessorPackage, NULL, &packages_info_size) == FALSE) { + const DWORD last_error = GetLastError(); + if (last_error != ERROR_INSUFFICIENT_BUFFER) { + cpuinfo_log_error("failed to query size of processor packages information: error %"PRIu32, + (uint32_t) last_error); + goto cleanup; + } + } + + DWORD max_info_size = max(cores_info_size, packages_info_size); + + processor_infos = HeapAlloc(heap, 0, max_info_size); + if (processor_infos == NULL) { + cpuinfo_log_error("failed to allocate %"PRIu32" bytes for logical processor information", + (uint32_t) max_info_size); + goto cleanup; + } + + if (GetLogicalProcessorInformationEx(RelationProcessorPackage, processor_infos, &max_info_size) == FALSE) { + cpuinfo_log_error("failed to query processor packages information: error %"PRIu32, + (uint32_t) GetLastError()); + goto cleanup; + } + + uint32_t packages_count = 0; + PSYSTEM_LOGICAL_PROCESSOR_INFORMATION_EX packages_info_end = + (PSYSTEM_LOGICAL_PROCESSOR_INFORMATION_EX) ((uintptr_t) processor_infos + packages_info_size); + for (PSYSTEM_LOGICAL_PROCESSOR_INFORMATION_EX package_info = processor_infos; + package_info < packages_info_end; + package_info = (PSYSTEM_LOGICAL_PROCESSOR_INFORMATION_EX) ((uintptr_t) package_info + package_info->Size)) + { + if (package_info->Relationship != RelationProcessorPackage) { + cpuinfo_log_warning("unexpected processor info type (%"PRIu32") for processor package information", + (uint32_t) package_info->Relationship); + continue; + } + + /* We assume that packages are reported in APIC order */ + const uint32_t package_id = packages_count++; + /* Reconstruct package part of APIC ID */ + const uint32_t package_apic_id = package_id << package_bits_offset; + /* Iterate processor groups and set the package part of APIC ID */ + for (uint32_t i = 0; i < package_info->Processor.GroupCount; i++) { + const uint32_t group_id = package_info->Processor.GroupMask[i].Group; + /* Global index of the first logical processor belonging to this group */ + const uint32_t group_processors_start = processors_before_group[group_id]; + /* Bitmask representing processors in this group belonging to this package */ + KAFFINITY group_processors_mask = package_info->Processor.GroupMask[i].Mask; + while (group_processors_mask != 0) { + const uint32_t group_processor_id = low_index_from_kaffinity(group_processors_mask); + const uint32_t processor_id = group_processors_start + group_processor_id; + processors[processor_id].package = (const struct cpuinfo_package*) NULL + package_id; + processors[processor_id].windows_group_id = (uint16_t) group_id; + processors[processor_id].windows_processor_id = (uint16_t) group_processor_id; + processors[processor_id].apic_id = package_apic_id; + + /* Reset the lowest bit in affinity mask */ + group_processors_mask &= (group_processors_mask - 1); + } + } + } + + max_info_size = max(cores_info_size, packages_info_size); + if (GetLogicalProcessorInformationEx(RelationProcessorCore, processor_infos, &max_info_size) == FALSE) { + cpuinfo_log_error("failed to query processor cores information: error %"PRIu32, + (uint32_t) GetLastError()); + goto cleanup; + } + + uint32_t cores_count = 0; + /* Index (among all cores) of the the first core on the current package */ + uint32_t package_core_start = 0; + uint32_t current_package_apic_id = 0; + PSYSTEM_LOGICAL_PROCESSOR_INFORMATION_EX cores_info_end = + (PSYSTEM_LOGICAL_PROCESSOR_INFORMATION_EX) ((uintptr_t) processor_infos + cores_info_size); + for (PSYSTEM_LOGICAL_PROCESSOR_INFORMATION_EX core_info = processor_infos; + core_info < cores_info_end; + core_info = (PSYSTEM_LOGICAL_PROCESSOR_INFORMATION_EX) ((uintptr_t) core_info + core_info->Size)) + { + if (core_info->Relationship != RelationProcessorCore) { + cpuinfo_log_warning("unexpected processor info type (%"PRIu32") for processor core information", + (uint32_t) core_info->Relationship); + continue; + } + + /* We assume that cores and logical processors are reported in APIC order */ + const uint32_t core_id = cores_count++; + uint32_t smt_id = 0; + /* Reconstruct core part of APIC ID */ + const uint32_t core_apic_id = (core_id & core_bits_mask) << x86_processor.topology.core_bits_offset; + /* Iterate processor groups and set the core & SMT parts of APIC ID */ + for (uint32_t i = 0; i < core_info->Processor.GroupCount; i++) { + const uint32_t group_id = core_info->Processor.GroupMask[i].Group; + /* Global index of the first logical processor belonging to this group */ + const uint32_t group_processors_start = processors_before_group[group_id]; + /* Bitmask representing processors in this group belonging to this package */ + KAFFINITY group_processors_mask = core_info->Processor.GroupMask[i].Mask; + while (group_processors_mask != 0) { + const uint32_t group_processor_id = low_index_from_kaffinity(group_processors_mask); + const uint32_t processor_id = group_processors_start + group_processor_id; + + /* Check if this is the first core on a new package */ + if (processors[processor_id].apic_id != current_package_apic_id) { + package_core_start = core_id; + current_package_apic_id = processors[processor_id].apic_id; + } + /* Core ID w.r.t package */ + const uint32_t package_core_id = core_id - package_core_start; + + /* Update APIC ID with core and SMT parts */ + processors[processor_id].apic_id |= + ((smt_id & thread_bits_mask) << x86_processor.topology.thread_bits_offset) | + ((package_core_id & core_bits_mask) << x86_processor.topology.core_bits_offset); + cpuinfo_log_debug("reconstructed APIC ID 0x%08"PRIx32" for processor %"PRIu32" in group %"PRIu32, + processors[processor_id].apic_id, group_processor_id, group_id); + + /* Set SMT ID (assume logical processors within the core are reported in APIC order) */ + processors[processor_id].smt_id = smt_id++; + processors[processor_id].core = (const struct cpuinfo_core*) NULL + core_id; + + /* Reset the lowest bit in affinity mask */ + group_processors_mask &= (group_processors_mask - 1); + } + } + } + + cores = HeapAlloc(heap, HEAP_ZERO_MEMORY, cores_count * sizeof(struct cpuinfo_core)); + if (cores == NULL) { + cpuinfo_log_error("failed to allocate %zu bytes for descriptions of %"PRIu32" cores", + cores_count * sizeof(struct cpuinfo_core), cores_count); + goto cleanup; + } + + clusters = HeapAlloc(heap, HEAP_ZERO_MEMORY, packages_count * sizeof(struct cpuinfo_cluster)); + if (clusters == NULL) { + cpuinfo_log_error("failed to allocate %zu bytes for descriptions of %"PRIu32" core clusters", + packages_count * sizeof(struct cpuinfo_cluster), packages_count); + goto cleanup; + } + + packages = HeapAlloc(heap, HEAP_ZERO_MEMORY, packages_count * sizeof(struct cpuinfo_package)); + if (packages == NULL) { + cpuinfo_log_error("failed to allocate %zu bytes for descriptions of %"PRIu32" physical packages", + packages_count * sizeof(struct cpuinfo_package), packages_count); + goto cleanup; + } + + for (uint32_t i = processors_count; i != 0; i--) { + const uint32_t processor_id = i - 1; + struct cpuinfo_processor* processor = processors + processor_id; + + /* Adjust core and package pointers for all logical processors */ + struct cpuinfo_core* core = + (struct cpuinfo_core*) ((uintptr_t) cores + (uintptr_t) processor->core); + processor->core = core; + struct cpuinfo_cluster* cluster = + (struct cpuinfo_cluster*) ((uintptr_t) clusters + (uintptr_t) processor->cluster); + processor->cluster = cluster; + struct cpuinfo_package* package = + (struct cpuinfo_package*) ((uintptr_t) packages + (uintptr_t) processor->package); + processor->package = package; + + /* This can be overwritten by lower-index processors on the same package */ + package->processor_start = processor_id; + package->processor_count += 1; + + /* This can be overwritten by lower-index processors on the same cluster */ + cluster->processor_start = processor_id; + cluster->processor_count += 1; + + /* This can be overwritten by lower-index processors on the same core*/ + core->processor_start = processor_id; + core->processor_count += 1; + } + + /* Set vendor/uarch/CPUID information for cores */ + for (uint32_t i = cores_count; i != 0; i--) { + const uint32_t global_core_id = i - 1; + struct cpuinfo_core* core = cores + global_core_id; + const struct cpuinfo_processor* processor = processors + core->processor_start; + struct cpuinfo_package* package = (struct cpuinfo_package*) processor->package; + struct cpuinfo_cluster* cluster = (struct cpuinfo_cluster*) processor->cluster; + + core->cluster = cluster; + core->package = package; + core->core_id = core_bits_mask & + (processor->apic_id >> x86_processor.topology.core_bits_offset); + core->vendor = x86_processor.vendor; + core->uarch = x86_processor.uarch; + core->cpuid = x86_processor.cpuid; + + /* This can be overwritten by lower-index cores on the same cluster/package */ + cluster->core_start = global_core_id; + cluster->core_count += 1; + package->core_start = global_core_id; + package->core_count += 1; + } + + for (uint32_t i = 0; i < packages_count; i++) { + struct cpuinfo_package* package = packages + i; + struct cpuinfo_cluster* cluster = clusters + i; + + cluster->package = package; + cluster->vendor = cores[cluster->core_start].vendor; + cluster->uarch = cores[cluster->core_start].uarch; + cluster->cpuid = cores[cluster->core_start].cpuid; + package->cluster_start = i; + package->cluster_count = 1; + cpuinfo_x86_format_package_name(x86_processor.vendor, brand_string, package->name); + } + + /* Count caches */ + uint32_t l1i_count, l1d_count, l2_count, l3_count, l4_count; + cpuinfo_x86_count_caches(processors_count, processors, &x86_processor, + &l1i_count, &l1d_count, &l2_count, &l3_count, &l4_count); + + /* Allocate cache descriptions */ + if (l1i_count != 0) { + l1i = HeapAlloc(heap, HEAP_ZERO_MEMORY, l1i_count * sizeof(struct cpuinfo_cache)); + if (l1i == NULL) { + cpuinfo_log_error("failed to allocate %zu bytes for descriptions of %"PRIu32" L1I caches", + l1i_count * sizeof(struct cpuinfo_cache), l1i_count); + goto cleanup; + } + } + if (l1d_count != 0) { + l1d = HeapAlloc(heap, HEAP_ZERO_MEMORY, l1d_count * sizeof(struct cpuinfo_cache)); + if (l1d == NULL) { + cpuinfo_log_error("failed to allocate %zu bytes for descriptions of %"PRIu32" L1D caches", + l1d_count * sizeof(struct cpuinfo_cache), l1d_count); + goto cleanup; + } + } + if (l2_count != 0) { + l2 = HeapAlloc(heap, HEAP_ZERO_MEMORY, l2_count * sizeof(struct cpuinfo_cache)); + if (l2 == NULL) { + cpuinfo_log_error("failed to allocate %zu bytes for descriptions of %"PRIu32" L2 caches", + l2_count * sizeof(struct cpuinfo_cache), l2_count); + goto cleanup; + } + } + if (l3_count != 0) { + l3 = HeapAlloc(heap, HEAP_ZERO_MEMORY, l3_count * sizeof(struct cpuinfo_cache)); + if (l3 == NULL) { + cpuinfo_log_error("failed to allocate %zu bytes for descriptions of %"PRIu32" L3 caches", + l3_count * sizeof(struct cpuinfo_cache), l3_count); + goto cleanup; + } + } + if (l4_count != 0) { + l4 = HeapAlloc(heap, HEAP_ZERO_MEMORY, l4_count * sizeof(struct cpuinfo_cache)); + if (l4 == NULL) { + cpuinfo_log_error("failed to allocate %zu bytes for descriptions of %"PRIu32" L4 caches", + l4_count * sizeof(struct cpuinfo_cache), l4_count); + goto cleanup; + } + } + + /* Set cache information */ + uint32_t l1i_index = UINT32_MAX, l1d_index = UINT32_MAX, l2_index = UINT32_MAX, l3_index = UINT32_MAX, l4_index = UINT32_MAX; + uint32_t last_l1i_id = UINT32_MAX, last_l1d_id = UINT32_MAX; + uint32_t last_l2_id = UINT32_MAX, last_l3_id = UINT32_MAX, last_l4_id = UINT32_MAX; + for (uint32_t i = 0; i < processors_count; i++) { + const uint32_t apic_id = processors[i].apic_id; + + if (x86_processor.cache.l1i.size != 0) { + const uint32_t l1i_id = apic_id & ~bit_mask(x86_processor.cache.l1i.apic_bits); + processors[i].cache.l1i = &l1i[l1i_index]; + if (l1i_id != last_l1i_id) { + /* new cache */ + last_l1i_id = l1i_id; + l1i[++l1i_index] = (struct cpuinfo_cache) { + .size = x86_processor.cache.l1i.size, + .associativity = x86_processor.cache.l1i.associativity, + .sets = x86_processor.cache.l1i.sets, + .partitions = x86_processor.cache.l1i.partitions, + .line_size = x86_processor.cache.l1i.line_size, + .flags = x86_processor.cache.l1i.flags, + .processor_start = i, + .processor_count = 1, + }; + } else { + /* another processor sharing the same cache */ + l1i[l1i_index].processor_count += 1; + } + processors[i].cache.l1i = &l1i[l1i_index]; + } else { + /* reset cache id */ + last_l1i_id = UINT32_MAX; + } + if (x86_processor.cache.l1d.size != 0) { + const uint32_t l1d_id = apic_id & ~bit_mask(x86_processor.cache.l1d.apic_bits); + processors[i].cache.l1d = &l1d[l1d_index]; + if (l1d_id != last_l1d_id) { + /* new cache */ + last_l1d_id = l1d_id; + l1d[++l1d_index] = (struct cpuinfo_cache) { + .size = x86_processor.cache.l1d.size, + .associativity = x86_processor.cache.l1d.associativity, + .sets = x86_processor.cache.l1d.sets, + .partitions = x86_processor.cache.l1d.partitions, + .line_size = x86_processor.cache.l1d.line_size, + .flags = x86_processor.cache.l1d.flags, + .processor_start = i, + .processor_count = 1, + }; + } else { + /* another processor sharing the same cache */ + l1d[l1d_index].processor_count += 1; + } + processors[i].cache.l1d = &l1d[l1d_index]; + } else { + /* reset cache id */ + last_l1d_id = UINT32_MAX; + } + if (x86_processor.cache.l2.size != 0) { + const uint32_t l2_id = apic_id & ~bit_mask(x86_processor.cache.l2.apic_bits); + processors[i].cache.l2 = &l2[l2_index]; + if (l2_id != last_l2_id) { + /* new cache */ + last_l2_id = l2_id; + l2[++l2_index] = (struct cpuinfo_cache) { + .size = x86_processor.cache.l2.size, + .associativity = x86_processor.cache.l2.associativity, + .sets = x86_processor.cache.l2.sets, + .partitions = x86_processor.cache.l2.partitions, + .line_size = x86_processor.cache.l2.line_size, + .flags = x86_processor.cache.l2.flags, + .processor_start = i, + .processor_count = 1, + }; + } else { + /* another processor sharing the same cache */ + l2[l2_index].processor_count += 1; + } + processors[i].cache.l2 = &l2[l2_index]; + } else { + /* reset cache id */ + last_l2_id = UINT32_MAX; + } + if (x86_processor.cache.l3.size != 0) { + const uint32_t l3_id = apic_id & ~bit_mask(x86_processor.cache.l3.apic_bits); + processors[i].cache.l3 = &l3[l3_index]; + if (l3_id != last_l3_id) { + /* new cache */ + last_l3_id = l3_id; + l3[++l3_index] = (struct cpuinfo_cache) { + .size = x86_processor.cache.l3.size, + .associativity = x86_processor.cache.l3.associativity, + .sets = x86_processor.cache.l3.sets, + .partitions = x86_processor.cache.l3.partitions, + .line_size = x86_processor.cache.l3.line_size, + .flags = x86_processor.cache.l3.flags, + .processor_start = i, + .processor_count = 1, + }; + } else { + /* another processor sharing the same cache */ + l3[l3_index].processor_count += 1; + } + processors[i].cache.l3 = &l3[l3_index]; + } else { + /* reset cache id */ + last_l3_id = UINT32_MAX; + } + if (x86_processor.cache.l4.size != 0) { + const uint32_t l4_id = apic_id & ~bit_mask(x86_processor.cache.l4.apic_bits); + processors[i].cache.l4 = &l4[l4_index]; + if (l4_id != last_l4_id) { + /* new cache */ + last_l4_id = l4_id; + l4[++l4_index] = (struct cpuinfo_cache) { + .size = x86_processor.cache.l4.size, + .associativity = x86_processor.cache.l4.associativity, + .sets = x86_processor.cache.l4.sets, + .partitions = x86_processor.cache.l4.partitions, + .line_size = x86_processor.cache.l4.line_size, + .flags = x86_processor.cache.l4.flags, + .processor_start = i, + .processor_count = 1, + }; + } else { + /* another processor sharing the same cache */ + l4[l4_index].processor_count += 1; + } + processors[i].cache.l4 = &l4[l4_index]; + } else { + /* reset cache id */ + last_l4_id = UINT32_MAX; + } + } + + + /* Commit changes */ + cpuinfo_processors = processors; + cpuinfo_cores = cores; + cpuinfo_clusters = clusters; + cpuinfo_packages = packages; + cpuinfo_cache[cpuinfo_cache_level_1i] = l1i; + cpuinfo_cache[cpuinfo_cache_level_1d] = l1d; + cpuinfo_cache[cpuinfo_cache_level_2] = l2; + cpuinfo_cache[cpuinfo_cache_level_3] = l3; + cpuinfo_cache[cpuinfo_cache_level_4] = l4; + + cpuinfo_processors_count = processors_count; + cpuinfo_cores_count = cores_count; + cpuinfo_clusters_count = packages_count; + cpuinfo_packages_count = packages_count; + cpuinfo_cache_count[cpuinfo_cache_level_1i] = l1i_count; + cpuinfo_cache_count[cpuinfo_cache_level_1d] = l1d_count; + cpuinfo_cache_count[cpuinfo_cache_level_2] = l2_count; + cpuinfo_cache_count[cpuinfo_cache_level_3] = l3_count; + cpuinfo_cache_count[cpuinfo_cache_level_4] = l4_count; + cpuinfo_max_cache_size = cpuinfo_compute_max_cache_size(&processors[0]); + + cpuinfo_global_uarch = (struct cpuinfo_uarch_info) { + .uarch = x86_processor.uarch, + .cpuid = x86_processor.cpuid, + .processor_count = processors_count, + .core_count = cores_count, + }; + + MemoryBarrier(); + + cpuinfo_is_initialized = true; + + processors = NULL; + cores = NULL; + clusters = NULL; + packages = NULL; + l1i = l1d = l2 = l3 = l4 = NULL; + +cleanup: + if (processors != NULL) { + HeapFree(heap, 0, processors); + } + if (cores != NULL) { + HeapFree(heap, 0, cores); + } + if (clusters != NULL) { + HeapFree(heap, 0, clusters); + } + if (packages != NULL) { + HeapFree(heap, 0, packages); + } + if (l1i != NULL) { + HeapFree(heap, 0, l1i); + } + if (l1d != NULL) { + HeapFree(heap, 0, l1d); + } + if (l2 != NULL) { + HeapFree(heap, 0, l2); + } + if (l3 != NULL) { + HeapFree(heap, 0, l3); + } + if (l4 != NULL) { + HeapFree(heap, 0, l4); + } + return TRUE; +} diff --git a/3rdparty/ffmpeg-core b/3rdparty/ffmpeg-core new file mode 160000 index 000000000..94dde08c8 --- /dev/null +++ b/3rdparty/ffmpeg-core @@ -0,0 +1 @@ +Subproject commit 94dde08c8a9e4271a93a2a7e4159e9fb05d30c0a diff --git a/3rdparty/fmt b/3rdparty/fmt new file mode 160000 index 000000000..11ddbcb78 --- /dev/null +++ b/3rdparty/fmt @@ -0,0 +1 @@ +Subproject commit 11ddbcb7898d2d3445d431a54814367b21dee6ad diff --git a/3rdparty/magic_enum b/3rdparty/magic_enum new file mode 160000 index 000000000..1384769c6 --- /dev/null +++ b/3rdparty/magic_enum @@ -0,0 +1 @@ +Subproject commit 1384769c66bd16ec9bb1353f45fe8ec8ccc12dbd diff --git a/3rdparty/nlohmann_json b/3rdparty/nlohmann_json new file mode 160000 index 000000000..272411c5e --- /dev/null +++ b/3rdparty/nlohmann_json @@ -0,0 +1 @@ +Subproject commit 272411c5e6ea45919af7673524e74e60c62116df diff --git a/3rdparty/spdlog b/3rdparty/spdlog new file mode 160000 index 000000000..8671ca4d4 --- /dev/null +++ b/3rdparty/spdlog @@ -0,0 +1 @@ +Subproject commit 8671ca4d492c8ee1cdfd3dd88afb9f88dd268178 diff --git a/3rdparty/stb/LICENSE b/3rdparty/stb/LICENSE new file mode 100644 index 000000000..a77ae91f3 --- /dev/null +++ b/3rdparty/stb/LICENSE @@ -0,0 +1,37 @@ +This software is available under 2 licenses -- choose whichever you prefer. +------------------------------------------------------------------------------ +ALTERNATIVE A - MIT License +Copyright (c) 2017 Sean Barrett +Permission is hereby granted, free of charge, to any person obtaining a copy of +this software and associated documentation files (the "Software"), to deal in +the Software without restriction, including without limitation the rights to +use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies +of the Software, and to permit persons to whom the Software is furnished to do +so, subject to the following conditions: +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. +------------------------------------------------------------------------------ +ALTERNATIVE B - Public Domain (www.unlicense.org) +This is free and unencumbered software released into the public domain. +Anyone is free to copy, modify, publish, use, compile, sell, or distribute this +software, either in source code form or as a compiled binary, for any purpose, +commercial or non-commercial, and by any means. +In jurisdictions that recognize copyright laws, the author or authors of this +software dedicate any and all copyright interest in the software to the public +domain. We make this dedication for the benefit of the public at large and to +the detriment of our heirs and successors. We intend this dedication to be an +overt act of relinquishment in perpetuity of all present and future rights to +this software under copyright law. +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN +ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/3rdparty/stb/stb_image.h b/3rdparty/stb/stb_image.h new file mode 100644 index 000000000..d60371b95 --- /dev/null +++ b/3rdparty/stb/stb_image.h @@ -0,0 +1,7897 @@ +/* stb_image - v2.27 - public domain image loader - http://nothings.org/stb + no warranty implied; use at your own risk + + Do this: + #define STB_IMAGE_IMPLEMENTATION + before you include this file in *one* C or C++ file to create the implementation. + + // i.e. it should look like this: + #include ... + #include ... + #include ... + #define STB_IMAGE_IMPLEMENTATION + #include "stb_image.h" + + You can #define STBI_ASSERT(x) before the #include to avoid using assert.h. + And #define STBI_MALLOC, STBI_REALLOC, and STBI_FREE to avoid using malloc,realloc,free + + + QUICK NOTES: + Primarily of interest to game developers and other people who can + avoid problematic images and only need the trivial interface + + JPEG baseline & progressive (12 bpc/arithmetic not supported, same as stock IJG lib) + PNG 1/2/4/8/16-bit-per-channel + + TGA (not sure what subset, if a subset) + BMP non-1bpp, non-RLE + PSD (composited view only, no extra channels, 8/16 bit-per-channel) + + GIF (*comp always reports as 4-channel) + HDR (radiance rgbE format) + PIC (Softimage PIC) + PNM (PPM and PGM binary only) + + Animated GIF still needs a proper API, but here's one way to do it: + http://gist.github.com/urraka/685d9a6340b26b830d49 + + - decode from memory or through FILE (define STBI_NO_STDIO to remove code) + - decode from arbitrary I/O callbacks + - SIMD acceleration on x86/x64 (SSE2) and ARM (NEON) + + Full documentation under "DOCUMENTATION" below. + + +LICENSE + + See end of file for license information. + +RECENT REVISION HISTORY: + + 2.27 (2021-07-11) document stbi_info better, 16-bit PNM support, bug fixes + 2.26 (2020-07-13) many minor fixes + 2.25 (2020-02-02) fix warnings + 2.24 (2020-02-02) fix warnings; thread-local failure_reason and flip_vertically + 2.23 (2019-08-11) fix clang static analysis warning + 2.22 (2019-03-04) gif fixes, fix warnings + 2.21 (2019-02-25) fix typo in comment + 2.20 (2019-02-07) support utf8 filenames in Windows; fix warnings and platform ifdefs + 2.19 (2018-02-11) fix warning + 2.18 (2018-01-30) fix warnings + 2.17 (2018-01-29) bugfix, 1-bit BMP, 16-bitness query, fix warnings + 2.16 (2017-07-23) all functions have 16-bit variants; optimizations; bugfixes + 2.15 (2017-03-18) fix png-1,2,4; all Imagenet JPGs; no runtime SSE detection on GCC + 2.14 (2017-03-03) remove deprecated STBI_JPEG_OLD; fixes for Imagenet JPGs + 2.13 (2016-12-04) experimental 16-bit API, only for PNG so far; fixes + 2.12 (2016-04-02) fix typo in 2.11 PSD fix that caused crashes + 2.11 (2016-04-02) 16-bit PNGS; enable SSE2 in non-gcc x64 + RGB-format JPEG; remove white matting in PSD; + allocate large structures on the stack; + correct channel count for PNG & BMP + 2.10 (2016-01-22) avoid warning introduced in 2.09 + 2.09 (2016-01-16) 16-bit TGA; comments in PNM files; STBI_REALLOC_SIZED + + See end of file for full revision history. + + + ============================ Contributors ========================= + + Image formats Extensions, features + Sean Barrett (jpeg, png, bmp) Jetro Lauha (stbi_info) + Nicolas Schulz (hdr, psd) Martin "SpartanJ" Golini (stbi_info) + Jonathan Dummer (tga) James "moose2000" Brown (iPhone PNG) + Jean-Marc Lienher (gif) Ben "Disch" Wenger (io callbacks) + Tom Seddon (pic) Omar Cornut (1/2/4-bit PNG) + Thatcher Ulrich (psd) Nicolas Guillemot (vertical flip) + Ken Miller (pgm, ppm) Richard Mitton (16-bit PSD) + github:urraka (animated gif) Junggon Kim (PNM comments) + Christopher Forseth (animated gif) Daniel Gibson (16-bit TGA) + socks-the-fox (16-bit PNG) + Jeremy Sawicki (handle all ImageNet JPGs) + Optimizations & bugfixes Mikhail Morozov (1-bit BMP) + Fabian "ryg" Giesen Anael Seghezzi (is-16-bit query) + Arseny Kapoulkine Simon Breuss (16-bit PNM) + John-Mark Allen + Carmelo J Fdez-Aguera + + Bug & warning fixes + Marc LeBlanc David Woo Guillaume George Martins Mozeiko + Christpher Lloyd Jerry Jansson Joseph Thomson Blazej Dariusz Roszkowski + Phil Jordan Dave Moore Roy Eltham + Hayaki Saito Nathan Reed Won Chun + Luke Graham Johan Duparc Nick Verigakis the Horde3D community + Thomas Ruf Ronny Chevalier github:rlyeh + Janez Zemva John Bartholomew Michal Cichon github:romigrou + Jonathan Blow Ken Hamada Tero Hanninen github:svdijk + Eugene Golushkov Laurent Gomila Cort Stratton github:snagar + Aruelien Pocheville Sergio Gonzalez Thibault Reuille github:Zelex + Cass Everitt Ryamond Barbiero github:grim210 + Paul Du Bois Engin Manap Aldo Culquicondor github:sammyhw + Philipp Wiesemann Dale Weiler Oriol Ferrer Mesia github:phprus + Josh Tobin Matthew Gregan github:poppolopoppo + Julian Raschke Gregory Mullen Christian Floisand github:darealshinji + Baldur Karlsson Kevin Schmidt JR Smith github:Michaelangel007 + Brad Weinberger Matvey Cherevko github:mosra + Luca Sas Alexander Veselov Zack Middleton [reserved] + Ryan C. Gordon [reserved] [reserved] + DO NOT ADD YOUR NAME HERE + + Jacko Dirks + + To add your name to the credits, pick a random blank space in the middle and fill it. + 80% of merge conflicts on stb PRs are due to people adding their name at the end + of the credits. +*/ + +#ifndef STBI_INCLUDE_STB_IMAGE_H +#define STBI_INCLUDE_STB_IMAGE_H + +// DOCUMENTATION +// +// Limitations: +// - no 12-bit-per-channel JPEG +// - no JPEGs with arithmetic coding +// - GIF always returns *comp=4 +// +// Basic usage (see HDR discussion below for HDR usage): +// int x,y,n; +// unsigned char *data = stbi_load(filename, &x, &y, &n, 0); +// // ... process data if not NULL ... +// // ... x = width, y = height, n = # 8-bit components per pixel ... +// // ... replace '0' with '1'..'4' to force that many components per pixel +// // ... but 'n' will always be the number that it would have been if you said 0 +// stbi_image_free(data) +// +// Standard parameters: +// int *x -- outputs image width in pixels +// int *y -- outputs image height in pixels +// int *channels_in_file -- outputs # of image components in image file +// int desired_channels -- if non-zero, # of image components requested in result +// +// The return value from an image loader is an 'unsigned char *' which points +// to the pixel data, or NULL on an allocation failure or if the image is +// corrupt or invalid. The pixel data consists of *y scanlines of *x pixels, +// with each pixel consisting of N interleaved 8-bit components; the first +// pixel pointed to is top-left-most in the image. There is no padding between +// image scanlines or between pixels, regardless of format. The number of +// components N is 'desired_channels' if desired_channels is non-zero, or +// *channels_in_file otherwise. If desired_channels is non-zero, +// *channels_in_file has the number of components that _would_ have been +// output otherwise. E.g. if you set desired_channels to 4, you will always +// get RGBA output, but you can check *channels_in_file to see if it's trivially +// opaque because e.g. there were only 3 channels in the source image. +// +// An output image with N components has the following components interleaved +// in this order in each pixel: +// +// N=#comp components +// 1 grey +// 2 grey, alpha +// 3 red, green, blue +// 4 red, green, blue, alpha +// +// If image loading fails for any reason, the return value will be NULL, +// and *x, *y, *channels_in_file will be unchanged. The function +// stbi_failure_reason() can be queried for an extremely brief, end-user +// unfriendly explanation of why the load failed. Define STBI_NO_FAILURE_STRINGS +// to avoid compiling these strings at all, and STBI_FAILURE_USERMSG to get slightly +// more user-friendly ones. +// +// Paletted PNG, BMP, GIF, and PIC images are automatically depalettized. +// +// To query the width, height and component count of an image without having to +// decode the full file, you can use the stbi_info family of functions: +// +// int x,y,n,ok; +// ok = stbi_info(filename, &x, &y, &n); +// // returns ok=1 and sets x, y, n if image is a supported format, +// // 0 otherwise. +// +// Note that stb_image pervasively uses ints in its public API for sizes, +// including sizes of memory buffers. This is now part of the API and thus +// hard to change without causing breakage. As a result, the various image +// loaders all have certain limits on image size; these differ somewhat +// by format but generally boil down to either just under 2GB or just under +// 1GB. When the decoded image would be larger than this, stb_image decoding +// will fail. +// +// Additionally, stb_image will reject image files that have any of their +// dimensions set to a larger value than the configurable STBI_MAX_DIMENSIONS, +// which defaults to 2**24 = 16777216 pixels. Due to the above memory limit, +// the only way to have an image with such dimensions load correctly +// is for it to have a rather extreme aspect ratio. Either way, the +// assumption here is that such larger images are likely to be malformed +// or malicious. If you do need to load an image with individual dimensions +// larger than that, and it still fits in the overall size limit, you can +// #define STBI_MAX_DIMENSIONS on your own to be something larger. +// +// =========================================================================== +// +// UNICODE: +// +// If compiling for Windows and you wish to use Unicode filenames, compile +// with +// #define STBI_WINDOWS_UTF8 +// and pass utf8-encoded filenames. Call stbi_convert_wchar_to_utf8 to convert +// Windows wchar_t filenames to utf8. +// +// =========================================================================== +// +// Philosophy +// +// stb libraries are designed with the following priorities: +// +// 1. easy to use +// 2. easy to maintain +// 3. good performance +// +// Sometimes I let "good performance" creep up in priority over "easy to maintain", +// and for best performance I may provide less-easy-to-use APIs that give higher +// performance, in addition to the easy-to-use ones. Nevertheless, it's important +// to keep in mind that from the standpoint of you, a client of this library, +// all you care about is #1 and #3, and stb libraries DO NOT emphasize #3 above all. +// +// Some secondary priorities arise directly from the first two, some of which +// provide more explicit reasons why performance can't be emphasized. +// +// - Portable ("ease of use") +// - Small source code footprint ("easy to maintain") +// - No dependencies ("ease of use") +// +// =========================================================================== +// +// I/O callbacks +// +// I/O callbacks allow you to read from arbitrary sources, like packaged +// files or some other source. Data read from callbacks are processed +// through a small internal buffer (currently 128 bytes) to try to reduce +// overhead. +// +// The three functions you must define are "read" (reads some bytes of data), +// "skip" (skips some bytes of data), "eof" (reports if the stream is at the end). +// +// =========================================================================== +// +// SIMD support +// +// The JPEG decoder will try to automatically use SIMD kernels on x86 when +// supported by the compiler. For ARM Neon support, you must explicitly +// request it. +// +// (The old do-it-yourself SIMD API is no longer supported in the current +// code.) +// +// On x86, SSE2 will automatically be used when available based on a run-time +// test; if not, the generic C versions are used as a fall-back. On ARM targets, +// the typical path is to have separate builds for NEON and non-NEON devices +// (at least this is true for iOS and Android). Therefore, the NEON support is +// toggled by a build flag: define STBI_NEON to get NEON loops. +// +// If for some reason you do not want to use any of SIMD code, or if +// you have issues compiling it, you can disable it entirely by +// defining STBI_NO_SIMD. +// +// =========================================================================== +// +// HDR image support (disable by defining STBI_NO_HDR) +// +// stb_image supports loading HDR images in general, and currently the Radiance +// .HDR file format specifically. You can still load any file through the existing +// interface; if you attempt to load an HDR file, it will be automatically remapped +// to LDR, assuming gamma 2.2 and an arbitrary scale factor defaulting to 1; +// both of these constants can be reconfigured through this interface: +// +// stbi_hdr_to_ldr_gamma(2.2f); +// stbi_hdr_to_ldr_scale(1.0f); +// +// (note, do not use _inverse_ constants; stbi_image will invert them +// appropriately). +// +// Additionally, there is a new, parallel interface for loading files as +// (linear) floats to preserve the full dynamic range: +// +// float *data = stbi_loadf(filename, &x, &y, &n, 0); +// +// If you load LDR images through this interface, those images will +// be promoted to floating point values, run through the inverse of +// constants corresponding to the above: +// +// stbi_ldr_to_hdr_scale(1.0f); +// stbi_ldr_to_hdr_gamma(2.2f); +// +// Finally, given a filename (or an open file or memory block--see header +// file for details) containing image data, you can query for the "most +// appropriate" interface to use (that is, whether the image is HDR or +// not), using: +// +// stbi_is_hdr(char *filename); +// +// =========================================================================== +// +// iPhone PNG support: +// +// We optionally support converting iPhone-formatted PNGs (which store +// premultiplied BGRA) back to RGB, even though they're internally encoded +// differently. To enable this conversion, call +// stbi_convert_iphone_png_to_rgb(1). +// +// Call stbi_set_unpremultiply_on_load(1) as well to force a divide per +// pixel to remove any premultiplied alpha *only* if the image file explicitly +// says there's premultiplied data (currently only happens in iPhone images, +// and only if iPhone convert-to-rgb processing is on). +// +// =========================================================================== +// +// ADDITIONAL CONFIGURATION +// +// - You can suppress implementation of any of the decoders to reduce +// your code footprint by #defining one or more of the following +// symbols before creating the implementation. +// +// STBI_NO_JPEG +// STBI_NO_PNG +// STBI_NO_BMP +// STBI_NO_PSD +// STBI_NO_TGA +// STBI_NO_GIF +// STBI_NO_HDR +// STBI_NO_PIC +// STBI_NO_PNM (.ppm and .pgm) +// +// - You can request *only* certain decoders and suppress all other ones +// (this will be more forward-compatible, as addition of new decoders +// doesn't require you to disable them explicitly): +// +// STBI_ONLY_JPEG +// STBI_ONLY_PNG +// STBI_ONLY_BMP +// STBI_ONLY_PSD +// STBI_ONLY_TGA +// STBI_ONLY_GIF +// STBI_ONLY_HDR +// STBI_ONLY_PIC +// STBI_ONLY_PNM (.ppm and .pgm) +// +// - If you use STBI_NO_PNG (or _ONLY_ without PNG), and you still +// want the zlib decoder to be available, #define STBI_SUPPORT_ZLIB +// +// - If you define STBI_MAX_DIMENSIONS, stb_image will reject images greater +// than that size (in either width or height) without further processing. +// This is to let programs in the wild set an upper bound to prevent +// denial-of-service attacks on untrusted data, as one could generate a +// valid image of gigantic dimensions and force stb_image to allocate a +// huge block of memory and spend disproportionate time decoding it. By +// default this is set to (1 << 24), which is 16777216, but that's still +// very big. + +#ifndef STBI_NO_STDIO +#include +#endif // STBI_NO_STDIO + +#define STBI_VERSION 1 + +enum +{ + STBI_default = 0, // only used for desired_channels + + STBI_grey = 1, + STBI_grey_alpha = 2, + STBI_rgb = 3, + STBI_rgb_alpha = 4 +}; + +#include +typedef unsigned char stbi_uc; +typedef unsigned short stbi_us; + +#ifdef __cplusplus +extern "C" { +#endif + +#ifndef STBIDEF +#ifdef STB_IMAGE_STATIC +#define STBIDEF static +#else +#define STBIDEF extern +#endif +#endif + +////////////////////////////////////////////////////////////////////////////// +// +// PRIMARY API - works on images of any type +// + +// +// load image by filename, open file, or memory buffer +// + +typedef struct +{ + int (*read) (void *user,char *data,int size); // fill 'data' with 'size' bytes. return number of bytes actually read + void (*skip) (void *user,int n); // skip the next 'n' bytes, or 'unget' the last -n bytes if negative + int (*eof) (void *user); // returns nonzero if we are at end of file/data +} stbi_io_callbacks; + +//////////////////////////////////// +// +// 8-bits-per-channel interface +// + +STBIDEF stbi_uc *stbi_load_from_memory (stbi_uc const *buffer, int len , int *x, int *y, int *channels_in_file, int desired_channels); +STBIDEF stbi_uc *stbi_load_from_callbacks(stbi_io_callbacks const *clbk , void *user, int *x, int *y, int *channels_in_file, int desired_channels); + +#ifndef STBI_NO_STDIO +STBIDEF stbi_uc *stbi_load (char const *filename, int *x, int *y, int *channels_in_file, int desired_channels); +STBIDEF stbi_uc *stbi_load_from_file (FILE *f, int *x, int *y, int *channels_in_file, int desired_channels); +// for stbi_load_from_file, file pointer is left pointing immediately after image +#endif + +#ifndef STBI_NO_GIF +STBIDEF stbi_uc *stbi_load_gif_from_memory(stbi_uc const *buffer, int len, int **delays, int *x, int *y, int *z, int *comp, int req_comp); +#endif + +#ifdef STBI_WINDOWS_UTF8 +STBIDEF int stbi_convert_wchar_to_utf8(char *buffer, size_t bufferlen, const wchar_t* input); +#endif + +//////////////////////////////////// +// +// 16-bits-per-channel interface +// + +STBIDEF stbi_us *stbi_load_16_from_memory (stbi_uc const *buffer, int len, int *x, int *y, int *channels_in_file, int desired_channels); +STBIDEF stbi_us *stbi_load_16_from_callbacks(stbi_io_callbacks const *clbk, void *user, int *x, int *y, int *channels_in_file, int desired_channels); + +#ifndef STBI_NO_STDIO +STBIDEF stbi_us *stbi_load_16 (char const *filename, int *x, int *y, int *channels_in_file, int desired_channels); +STBIDEF stbi_us *stbi_load_from_file_16(FILE *f, int *x, int *y, int *channels_in_file, int desired_channels); +#endif + +//////////////////////////////////// +// +// float-per-channel interface +// +#ifndef STBI_NO_LINEAR + STBIDEF float *stbi_loadf_from_memory (stbi_uc const *buffer, int len, int *x, int *y, int *channels_in_file, int desired_channels); + STBIDEF float *stbi_loadf_from_callbacks (stbi_io_callbacks const *clbk, void *user, int *x, int *y, int *channels_in_file, int desired_channels); + + #ifndef STBI_NO_STDIO + STBIDEF float *stbi_loadf (char const *filename, int *x, int *y, int *channels_in_file, int desired_channels); + STBIDEF float *stbi_loadf_from_file (FILE *f, int *x, int *y, int *channels_in_file, int desired_channels); + #endif +#endif + +#ifndef STBI_NO_HDR + STBIDEF void stbi_hdr_to_ldr_gamma(float gamma); + STBIDEF void stbi_hdr_to_ldr_scale(float scale); +#endif // STBI_NO_HDR + +#ifndef STBI_NO_LINEAR + STBIDEF void stbi_ldr_to_hdr_gamma(float gamma); + STBIDEF void stbi_ldr_to_hdr_scale(float scale); +#endif // STBI_NO_LINEAR + +// stbi_is_hdr is always defined, but always returns false if STBI_NO_HDR +STBIDEF int stbi_is_hdr_from_callbacks(stbi_io_callbacks const *clbk, void *user); +STBIDEF int stbi_is_hdr_from_memory(stbi_uc const *buffer, int len); +#ifndef STBI_NO_STDIO +STBIDEF int stbi_is_hdr (char const *filename); +STBIDEF int stbi_is_hdr_from_file(FILE *f); +#endif // STBI_NO_STDIO + + +// get a VERY brief reason for failure +// on most compilers (and ALL modern mainstream compilers) this is threadsafe +STBIDEF const char *stbi_failure_reason (void); + +// free the loaded image -- this is just free() +STBIDEF void stbi_image_free (void *retval_from_stbi_load); + +// get image dimensions & components without fully decoding +STBIDEF int stbi_info_from_memory(stbi_uc const *buffer, int len, int *x, int *y, int *comp); +STBIDEF int stbi_info_from_callbacks(stbi_io_callbacks const *clbk, void *user, int *x, int *y, int *comp); +STBIDEF int stbi_is_16_bit_from_memory(stbi_uc const *buffer, int len); +STBIDEF int stbi_is_16_bit_from_callbacks(stbi_io_callbacks const *clbk, void *user); + +#ifndef STBI_NO_STDIO +STBIDEF int stbi_info (char const *filename, int *x, int *y, int *comp); +STBIDEF int stbi_info_from_file (FILE *f, int *x, int *y, int *comp); +STBIDEF int stbi_is_16_bit (char const *filename); +STBIDEF int stbi_is_16_bit_from_file(FILE *f); +#endif + + + +// for image formats that explicitly notate that they have premultiplied alpha, +// we just return the colors as stored in the file. set this flag to force +// unpremultiplication. results are undefined if the unpremultiply overflow. +STBIDEF void stbi_set_unpremultiply_on_load(int flag_true_if_should_unpremultiply); + +// indicate whether we should process iphone images back to canonical format, +// or just pass them through "as-is" +STBIDEF void stbi_convert_iphone_png_to_rgb(int flag_true_if_should_convert); + +// flip the image vertically, so the first pixel in the output array is the bottom left +STBIDEF void stbi_set_flip_vertically_on_load(int flag_true_if_should_flip); + +// as above, but only applies to images loaded on the thread that calls the function +// this function is only available if your compiler supports thread-local variables; +// calling it will fail to link if your compiler doesn't +STBIDEF void stbi_set_unpremultiply_on_load_thread(int flag_true_if_should_unpremultiply); +STBIDEF void stbi_convert_iphone_png_to_rgb_thread(int flag_true_if_should_convert); +STBIDEF void stbi_set_flip_vertically_on_load_thread(int flag_true_if_should_flip); + +// ZLIB client - used by PNG, available for other purposes + +STBIDEF char *stbi_zlib_decode_malloc_guesssize(const char *buffer, int len, int initial_size, int *outlen); +STBIDEF char *stbi_zlib_decode_malloc_guesssize_headerflag(const char *buffer, int len, int initial_size, int *outlen, int parse_header); +STBIDEF char *stbi_zlib_decode_malloc(const char *buffer, int len, int *outlen); +STBIDEF int stbi_zlib_decode_buffer(char *obuffer, int olen, const char *ibuffer, int ilen); + +STBIDEF char *stbi_zlib_decode_noheader_malloc(const char *buffer, int len, int *outlen); +STBIDEF int stbi_zlib_decode_noheader_buffer(char *obuffer, int olen, const char *ibuffer, int ilen); + + +#ifdef __cplusplus +} +#endif + +// +// +//// end header file ///////////////////////////////////////////////////// +#endif // STBI_INCLUDE_STB_IMAGE_H + +#ifdef STB_IMAGE_IMPLEMENTATION + +#if defined(STBI_ONLY_JPEG) || defined(STBI_ONLY_PNG) || defined(STBI_ONLY_BMP) \ + || defined(STBI_ONLY_TGA) || defined(STBI_ONLY_GIF) || defined(STBI_ONLY_PSD) \ + || defined(STBI_ONLY_HDR) || defined(STBI_ONLY_PIC) || defined(STBI_ONLY_PNM) \ + || defined(STBI_ONLY_ZLIB) + #ifndef STBI_ONLY_JPEG + #define STBI_NO_JPEG + #endif + #ifndef STBI_ONLY_PNG + #define STBI_NO_PNG + #endif + #ifndef STBI_ONLY_BMP + #define STBI_NO_BMP + #endif + #ifndef STBI_ONLY_PSD + #define STBI_NO_PSD + #endif + #ifndef STBI_ONLY_TGA + #define STBI_NO_TGA + #endif + #ifndef STBI_ONLY_GIF + #define STBI_NO_GIF + #endif + #ifndef STBI_ONLY_HDR + #define STBI_NO_HDR + #endif + #ifndef STBI_ONLY_PIC + #define STBI_NO_PIC + #endif + #ifndef STBI_ONLY_PNM + #define STBI_NO_PNM + #endif +#endif + +#if defined(STBI_NO_PNG) && !defined(STBI_SUPPORT_ZLIB) && !defined(STBI_NO_ZLIB) +#define STBI_NO_ZLIB +#endif + + +#include +#include // ptrdiff_t on osx +#include +#include +#include + +#if !defined(STBI_NO_LINEAR) || !defined(STBI_NO_HDR) +#include // ldexp, pow +#endif + +#ifndef STBI_NO_STDIO +#include +#endif + +#ifndef STBI_ASSERT +#include +#define STBI_ASSERT(x) assert(x) +#endif + +#ifdef __cplusplus +#define STBI_EXTERN extern "C" +#else +#define STBI_EXTERN extern +#endif + + +#ifndef _MSC_VER + #ifdef __cplusplus + #define stbi_inline inline + #else + #define stbi_inline + #endif +#else + #define stbi_inline __forceinline +#endif + +#ifndef STBI_NO_THREAD_LOCALS + #if defined(__cplusplus) && __cplusplus >= 201103L + #define STBI_THREAD_LOCAL thread_local + #elif defined(__GNUC__) && __GNUC__ < 5 + #define STBI_THREAD_LOCAL __thread + #elif defined(_MSC_VER) + #define STBI_THREAD_LOCAL __declspec(thread) + #elif defined (__STDC_VERSION__) && __STDC_VERSION__ >= 201112L && !defined(__STDC_NO_THREADS__) + #define STBI_THREAD_LOCAL _Thread_local + #endif + + #ifndef STBI_THREAD_LOCAL + #if defined(__GNUC__) + #define STBI_THREAD_LOCAL __thread + #endif + #endif +#endif + +#ifdef _MSC_VER +typedef unsigned short stbi__uint16; +typedef signed short stbi__int16; +typedef unsigned int stbi__uint32; +typedef signed int stbi__int32; +#else +#include +typedef uint16_t stbi__uint16; +typedef int16_t stbi__int16; +typedef uint32_t stbi__uint32; +typedef int32_t stbi__int32; +#endif + +// should produce compiler error if size is wrong +typedef unsigned char validate_uint32[sizeof(stbi__uint32)==4 ? 1 : -1]; + +#ifdef _MSC_VER +#define STBI_NOTUSED(v) (void)(v) +#else +#define STBI_NOTUSED(v) (void)sizeof(v) +#endif + +#ifdef _MSC_VER +#define STBI_HAS_LROTL +#endif + +#ifdef STBI_HAS_LROTL + #define stbi_lrot(x,y) _lrotl(x,y) +#else + #define stbi_lrot(x,y) (((x) << (y)) | ((x) >> (-(y) & 31))) +#endif + +#if defined(STBI_MALLOC) && defined(STBI_FREE) && (defined(STBI_REALLOC) || defined(STBI_REALLOC_SIZED)) +// ok +#elif !defined(STBI_MALLOC) && !defined(STBI_FREE) && !defined(STBI_REALLOC) && !defined(STBI_REALLOC_SIZED) +// ok +#else +#error "Must define all or none of STBI_MALLOC, STBI_FREE, and STBI_REALLOC (or STBI_REALLOC_SIZED)." +#endif + +#ifndef STBI_MALLOC +#define STBI_MALLOC(sz) malloc(sz) +#define STBI_REALLOC(p,newsz) realloc(p,newsz) +#define STBI_FREE(p) free(p) +#endif + +#ifndef STBI_REALLOC_SIZED +#define STBI_REALLOC_SIZED(p,oldsz,newsz) STBI_REALLOC(p,newsz) +#endif + +// x86/x64 detection +#if defined(__x86_64__) || defined(_M_X64) +#define STBI__X64_TARGET +#elif defined(__i386) || defined(_M_IX86) +#define STBI__X86_TARGET +#endif + +#if defined(__GNUC__) && defined(STBI__X86_TARGET) && !defined(__SSE2__) && !defined(STBI_NO_SIMD) +// gcc doesn't support sse2 intrinsics unless you compile with -msse2, +// which in turn means it gets to use SSE2 everywhere. This is unfortunate, +// but previous attempts to provide the SSE2 functions with runtime +// detection caused numerous issues. The way architecture extensions are +// exposed in GCC/Clang is, sadly, not really suited for one-file libs. +// New behavior: if compiled with -msse2, we use SSE2 without any +// detection; if not, we don't use it at all. +#define STBI_NO_SIMD +#endif + +#if defined(__MINGW32__) && defined(STBI__X86_TARGET) && !defined(STBI_MINGW_ENABLE_SSE2) && !defined(STBI_NO_SIMD) +// Note that __MINGW32__ doesn't actually mean 32-bit, so we have to avoid STBI__X64_TARGET +// +// 32-bit MinGW wants ESP to be 16-byte aligned, but this is not in the +// Windows ABI and VC++ as well as Windows DLLs don't maintain that invariant. +// As a result, enabling SSE2 on 32-bit MinGW is dangerous when not +// simultaneously enabling "-mstackrealign". +// +// See https://github.com/nothings/stb/issues/81 for more information. +// +// So default to no SSE2 on 32-bit MinGW. If you've read this far and added +// -mstackrealign to your build settings, feel free to #define STBI_MINGW_ENABLE_SSE2. +#define STBI_NO_SIMD +#endif + +#if !defined(STBI_NO_SIMD) && (defined(STBI__X86_TARGET) || defined(STBI__X64_TARGET)) +#define STBI_SSE2 +#include + +#ifdef _MSC_VER + +#if _MSC_VER >= 1400 // not VC6 +#include // __cpuid +static int stbi__cpuid3(void) +{ + int info[4]; + __cpuid(info,1); + return info[3]; +} +#else +static int stbi__cpuid3(void) +{ + int res; + __asm { + mov eax,1 + cpuid + mov res,edx + } + return res; +} +#endif + +#define STBI_SIMD_ALIGN(type, name) __declspec(align(16)) type name + +#if !defined(STBI_NO_JPEG) && defined(STBI_SSE2) +static int stbi__sse2_available(void) +{ + int info3 = stbi__cpuid3(); + return ((info3 >> 26) & 1) != 0; +} +#endif + +#else // assume GCC-style if not VC++ +#define STBI_SIMD_ALIGN(type, name) type name __attribute__((aligned(16))) + +#if !defined(STBI_NO_JPEG) && defined(STBI_SSE2) +static int stbi__sse2_available(void) +{ + // If we're even attempting to compile this on GCC/Clang, that means + // -msse2 is on, which means the compiler is allowed to use SSE2 + // instructions at will, and so are we. + return 1; +} +#endif + +#endif +#endif + +// ARM NEON +#if defined(STBI_NO_SIMD) && defined(STBI_NEON) +#undef STBI_NEON +#endif + +#ifdef STBI_NEON +#include +#ifdef _MSC_VER +#define STBI_SIMD_ALIGN(type, name) __declspec(align(16)) type name +#else +#define STBI_SIMD_ALIGN(type, name) type name __attribute__((aligned(16))) +#endif +#endif + +#ifndef STBI_SIMD_ALIGN +#define STBI_SIMD_ALIGN(type, name) type name +#endif + +#ifndef STBI_MAX_DIMENSIONS +#define STBI_MAX_DIMENSIONS (1 << 24) +#endif + +/////////////////////////////////////////////// +// +// stbi__context struct and start_xxx functions + +// stbi__context structure is our basic context used by all images, so it +// contains all the IO context, plus some basic image information +typedef struct +{ + stbi__uint32 img_x, img_y; + int img_n, img_out_n; + + stbi_io_callbacks io; + void *io_user_data; + + int read_from_callbacks; + int buflen; + stbi_uc buffer_start[128]; + int callback_already_read; + + stbi_uc *img_buffer, *img_buffer_end; + stbi_uc *img_buffer_original, *img_buffer_original_end; +} stbi__context; + + +static void stbi__refill_buffer(stbi__context *s); + +// initialize a memory-decode context +static void stbi__start_mem(stbi__context *s, stbi_uc const *buffer, int len) +{ + s->io.read = NULL; + s->read_from_callbacks = 0; + s->callback_already_read = 0; + s->img_buffer = s->img_buffer_original = (stbi_uc *) buffer; + s->img_buffer_end = s->img_buffer_original_end = (stbi_uc *) buffer+len; +} + +// initialize a callback-based context +static void stbi__start_callbacks(stbi__context *s, stbi_io_callbacks *c, void *user) +{ + s->io = *c; + s->io_user_data = user; + s->buflen = sizeof(s->buffer_start); + s->read_from_callbacks = 1; + s->callback_already_read = 0; + s->img_buffer = s->img_buffer_original = s->buffer_start; + stbi__refill_buffer(s); + s->img_buffer_original_end = s->img_buffer_end; +} + +#ifndef STBI_NO_STDIO + +static int stbi__stdio_read(void *user, char *data, int size) +{ + return (int) fread(data,1,size,(FILE*) user); +} + +static void stbi__stdio_skip(void *user, int n) +{ + int ch; + fseek((FILE*) user, n, SEEK_CUR); + ch = fgetc((FILE*) user); /* have to read a byte to reset feof()'s flag */ + if (ch != EOF) { + ungetc(ch, (FILE *) user); /* push byte back onto stream if valid. */ + } +} + +static int stbi__stdio_eof(void *user) +{ + return feof((FILE*) user) || ferror((FILE *) user); +} + +static stbi_io_callbacks stbi__stdio_callbacks = +{ + stbi__stdio_read, + stbi__stdio_skip, + stbi__stdio_eof, +}; + +static void stbi__start_file(stbi__context *s, FILE *f) +{ + stbi__start_callbacks(s, &stbi__stdio_callbacks, (void *) f); +} + +//static void stop_file(stbi__context *s) { } + +#endif // !STBI_NO_STDIO + +static void stbi__rewind(stbi__context *s) +{ + // conceptually rewind SHOULD rewind to the beginning of the stream, + // but we just rewind to the beginning of the initial buffer, because + // we only use it after doing 'test', which only ever looks at at most 92 bytes + s->img_buffer = s->img_buffer_original; + s->img_buffer_end = s->img_buffer_original_end; +} + +enum +{ + STBI_ORDER_RGB, + STBI_ORDER_BGR +}; + +typedef struct +{ + int bits_per_channel; + int num_channels; + int channel_order; +} stbi__result_info; + +#ifndef STBI_NO_JPEG +static int stbi__jpeg_test(stbi__context *s); +static void *stbi__jpeg_load(stbi__context *s, int *x, int *y, int *comp, int req_comp, stbi__result_info *ri); +static int stbi__jpeg_info(stbi__context *s, int *x, int *y, int *comp); +#endif + +#ifndef STBI_NO_PNG +static int stbi__png_test(stbi__context *s); +static void *stbi__png_load(stbi__context *s, int *x, int *y, int *comp, int req_comp, stbi__result_info *ri); +static int stbi__png_info(stbi__context *s, int *x, int *y, int *comp); +static int stbi__png_is16(stbi__context *s); +#endif + +#ifndef STBI_NO_BMP +static int stbi__bmp_test(stbi__context *s); +static void *stbi__bmp_load(stbi__context *s, int *x, int *y, int *comp, int req_comp, stbi__result_info *ri); +static int stbi__bmp_info(stbi__context *s, int *x, int *y, int *comp); +#endif + +#ifndef STBI_NO_TGA +static int stbi__tga_test(stbi__context *s); +static void *stbi__tga_load(stbi__context *s, int *x, int *y, int *comp, int req_comp, stbi__result_info *ri); +static int stbi__tga_info(stbi__context *s, int *x, int *y, int *comp); +#endif + +#ifndef STBI_NO_PSD +static int stbi__psd_test(stbi__context *s); +static void *stbi__psd_load(stbi__context *s, int *x, int *y, int *comp, int req_comp, stbi__result_info *ri, int bpc); +static int stbi__psd_info(stbi__context *s, int *x, int *y, int *comp); +static int stbi__psd_is16(stbi__context *s); +#endif + +#ifndef STBI_NO_HDR +static int stbi__hdr_test(stbi__context *s); +static float *stbi__hdr_load(stbi__context *s, int *x, int *y, int *comp, int req_comp, stbi__result_info *ri); +static int stbi__hdr_info(stbi__context *s, int *x, int *y, int *comp); +#endif + +#ifndef STBI_NO_PIC +static int stbi__pic_test(stbi__context *s); +static void *stbi__pic_load(stbi__context *s, int *x, int *y, int *comp, int req_comp, stbi__result_info *ri); +static int stbi__pic_info(stbi__context *s, int *x, int *y, int *comp); +#endif + +#ifndef STBI_NO_GIF +static int stbi__gif_test(stbi__context *s); +static void *stbi__gif_load(stbi__context *s, int *x, int *y, int *comp, int req_comp, stbi__result_info *ri); +static void *stbi__load_gif_main(stbi__context *s, int **delays, int *x, int *y, int *z, int *comp, int req_comp); +static int stbi__gif_info(stbi__context *s, int *x, int *y, int *comp); +#endif + +#ifndef STBI_NO_PNM +static int stbi__pnm_test(stbi__context *s); +static void *stbi__pnm_load(stbi__context *s, int *x, int *y, int *comp, int req_comp, stbi__result_info *ri); +static int stbi__pnm_info(stbi__context *s, int *x, int *y, int *comp); +static int stbi__pnm_is16(stbi__context *s); +#endif + +static +#ifdef STBI_THREAD_LOCAL +STBI_THREAD_LOCAL +#endif +const char *stbi__g_failure_reason; + +STBIDEF const char *stbi_failure_reason(void) +{ + return stbi__g_failure_reason; +} + +#ifndef STBI_NO_FAILURE_STRINGS +static int stbi__err(const char *str) +{ + stbi__g_failure_reason = str; + return 0; +} +#endif + +static void *stbi__malloc(size_t size) +{ + return STBI_MALLOC(size); +} + +// stb_image uses ints pervasively, including for offset calculations. +// therefore the largest decoded image size we can support with the +// current code, even on 64-bit targets, is INT_MAX. this is not a +// significant limitation for the intended use case. +// +// we do, however, need to make sure our size calculations don't +// overflow. hence a few helper functions for size calculations that +// multiply integers together, making sure that they're non-negative +// and no overflow occurs. + +// return 1 if the sum is valid, 0 on overflow. +// negative terms are considered invalid. +static int stbi__addsizes_valid(int a, int b) +{ + if (b < 0) return 0; + // now 0 <= b <= INT_MAX, hence also + // 0 <= INT_MAX - b <= INTMAX. + // And "a + b <= INT_MAX" (which might overflow) is the + // same as a <= INT_MAX - b (no overflow) + return a <= INT_MAX - b; +} + +// returns 1 if the product is valid, 0 on overflow. +// negative factors are considered invalid. +static int stbi__mul2sizes_valid(int a, int b) +{ + if (a < 0 || b < 0) return 0; + if (b == 0) return 1; // mul-by-0 is always safe + // portable way to check for no overflows in a*b + return a <= INT_MAX/b; +} + +#if !defined(STBI_NO_JPEG) || !defined(STBI_NO_PNG) || !defined(STBI_NO_TGA) || !defined(STBI_NO_HDR) +// returns 1 if "a*b + add" has no negative terms/factors and doesn't overflow +static int stbi__mad2sizes_valid(int a, int b, int add) +{ + return stbi__mul2sizes_valid(a, b) && stbi__addsizes_valid(a*b, add); +} +#endif + +// returns 1 if "a*b*c + add" has no negative terms/factors and doesn't overflow +static int stbi__mad3sizes_valid(int a, int b, int c, int add) +{ + return stbi__mul2sizes_valid(a, b) && stbi__mul2sizes_valid(a*b, c) && + stbi__addsizes_valid(a*b*c, add); +} + +// returns 1 if "a*b*c*d + add" has no negative terms/factors and doesn't overflow +#if !defined(STBI_NO_LINEAR) || !defined(STBI_NO_HDR) || !defined(STBI_NO_PNM) +static int stbi__mad4sizes_valid(int a, int b, int c, int d, int add) +{ + return stbi__mul2sizes_valid(a, b) && stbi__mul2sizes_valid(a*b, c) && + stbi__mul2sizes_valid(a*b*c, d) && stbi__addsizes_valid(a*b*c*d, add); +} +#endif + +#if !defined(STBI_NO_JPEG) || !defined(STBI_NO_PNG) || !defined(STBI_NO_TGA) || !defined(STBI_NO_HDR) +// mallocs with size overflow checking +static void *stbi__malloc_mad2(int a, int b, int add) +{ + if (!stbi__mad2sizes_valid(a, b, add)) return NULL; + return stbi__malloc(a*b + add); +} +#endif + +static void *stbi__malloc_mad3(int a, int b, int c, int add) +{ + if (!stbi__mad3sizes_valid(a, b, c, add)) return NULL; + return stbi__malloc(a*b*c + add); +} + +#if !defined(STBI_NO_LINEAR) || !defined(STBI_NO_HDR) || !defined(STBI_NO_PNM) +static void *stbi__malloc_mad4(int a, int b, int c, int d, int add) +{ + if (!stbi__mad4sizes_valid(a, b, c, d, add)) return NULL; + return stbi__malloc(a*b*c*d + add); +} +#endif + +// stbi__err - error +// stbi__errpf - error returning pointer to float +// stbi__errpuc - error returning pointer to unsigned char + +#ifdef STBI_NO_FAILURE_STRINGS + #define stbi__err(x,y) 0 +#elif defined(STBI_FAILURE_USERMSG) + #define stbi__err(x,y) stbi__err(y) +#else + #define stbi__err(x,y) stbi__err(x) +#endif + +#define stbi__errpf(x,y) ((float *)(size_t) (stbi__err(x,y)?NULL:NULL)) +#define stbi__errpuc(x,y) ((unsigned char *)(size_t) (stbi__err(x,y)?NULL:NULL)) + +STBIDEF void stbi_image_free(void *retval_from_stbi_load) +{ + STBI_FREE(retval_from_stbi_load); +} + +#ifndef STBI_NO_LINEAR +static float *stbi__ldr_to_hdr(stbi_uc *data, int x, int y, int comp); +#endif + +#ifndef STBI_NO_HDR +static stbi_uc *stbi__hdr_to_ldr(float *data, int x, int y, int comp); +#endif + +static int stbi__vertically_flip_on_load_global = 0; + +STBIDEF void stbi_set_flip_vertically_on_load(int flag_true_if_should_flip) +{ + stbi__vertically_flip_on_load_global = flag_true_if_should_flip; +} + +#ifndef STBI_THREAD_LOCAL +#define stbi__vertically_flip_on_load stbi__vertically_flip_on_load_global +#else +static STBI_THREAD_LOCAL int stbi__vertically_flip_on_load_local, stbi__vertically_flip_on_load_set; + +STBIDEF void stbi_set_flip_vertically_on_load_thread(int flag_true_if_should_flip) +{ + stbi__vertically_flip_on_load_local = flag_true_if_should_flip; + stbi__vertically_flip_on_load_set = 1; +} + +#define stbi__vertically_flip_on_load (stbi__vertically_flip_on_load_set \ + ? stbi__vertically_flip_on_load_local \ + : stbi__vertically_flip_on_load_global) +#endif // STBI_THREAD_LOCAL + +static void *stbi__load_main(stbi__context *s, int *x, int *y, int *comp, int req_comp, stbi__result_info *ri, int bpc) +{ + memset(ri, 0, sizeof(*ri)); // make sure it's initialized if we add new fields + ri->bits_per_channel = 8; // default is 8 so most paths don't have to be changed + ri->channel_order = STBI_ORDER_RGB; // all current input & output are this, but this is here so we can add BGR order + ri->num_channels = 0; + + // test the formats with a very explicit header first (at least a FOURCC + // or distinctive magic number first) + #ifndef STBI_NO_PNG + if (stbi__png_test(s)) return stbi__png_load(s,x,y,comp,req_comp, ri); + #endif + #ifndef STBI_NO_BMP + if (stbi__bmp_test(s)) return stbi__bmp_load(s,x,y,comp,req_comp, ri); + #endif + #ifndef STBI_NO_GIF + if (stbi__gif_test(s)) return stbi__gif_load(s,x,y,comp,req_comp, ri); + #endif + #ifndef STBI_NO_PSD + if (stbi__psd_test(s)) return stbi__psd_load(s,x,y,comp,req_comp, ri, bpc); + #else + STBI_NOTUSED(bpc); + #endif + #ifndef STBI_NO_PIC + if (stbi__pic_test(s)) return stbi__pic_load(s,x,y,comp,req_comp, ri); + #endif + + // then the formats that can end up attempting to load with just 1 or 2 + // bytes matching expectations; these are prone to false positives, so + // try them later + #ifndef STBI_NO_JPEG + if (stbi__jpeg_test(s)) return stbi__jpeg_load(s,x,y,comp,req_comp, ri); + #endif + #ifndef STBI_NO_PNM + if (stbi__pnm_test(s)) return stbi__pnm_load(s,x,y,comp,req_comp, ri); + #endif + + #ifndef STBI_NO_HDR + if (stbi__hdr_test(s)) { + float *hdr = stbi__hdr_load(s, x,y,comp,req_comp, ri); + return stbi__hdr_to_ldr(hdr, *x, *y, req_comp ? req_comp : *comp); + } + #endif + + #ifndef STBI_NO_TGA + // test tga last because it's a crappy test! + if (stbi__tga_test(s)) + return stbi__tga_load(s,x,y,comp,req_comp, ri); + #endif + + return stbi__errpuc("unknown image type", "Image not of any known type, or corrupt"); +} + +static stbi_uc *stbi__convert_16_to_8(stbi__uint16 *orig, int w, int h, int channels) +{ + int i; + int img_len = w * h * channels; + stbi_uc *reduced; + + reduced = (stbi_uc *) stbi__malloc(img_len); + if (reduced == NULL) return stbi__errpuc("outofmem", "Out of memory"); + + for (i = 0; i < img_len; ++i) + reduced[i] = (stbi_uc)((orig[i] >> 8) & 0xFF); // top half of each byte is sufficient approx of 16->8 bit scaling + + STBI_FREE(orig); + return reduced; +} + +static stbi__uint16 *stbi__convert_8_to_16(stbi_uc *orig, int w, int h, int channels) +{ + int i; + int img_len = w * h * channels; + stbi__uint16 *enlarged; + + enlarged = (stbi__uint16 *) stbi__malloc(img_len*2); + if (enlarged == NULL) return (stbi__uint16 *) stbi__errpuc("outofmem", "Out of memory"); + + for (i = 0; i < img_len; ++i) + enlarged[i] = (stbi__uint16)((orig[i] << 8) + orig[i]); // replicate to high and low byte, maps 0->0, 255->0xffff + + STBI_FREE(orig); + return enlarged; +} + +static void stbi__vertical_flip(void *image, int w, int h, int bytes_per_pixel) +{ + int row; + size_t bytes_per_row = (size_t)w * bytes_per_pixel; + stbi_uc temp[2048]; + stbi_uc *bytes = (stbi_uc *)image; + + for (row = 0; row < (h>>1); row++) { + stbi_uc *row0 = bytes + row*bytes_per_row; + stbi_uc *row1 = bytes + (h - row - 1)*bytes_per_row; + // swap row0 with row1 + size_t bytes_left = bytes_per_row; + while (bytes_left) { + size_t bytes_copy = (bytes_left < sizeof(temp)) ? bytes_left : sizeof(temp); + memcpy(temp, row0, bytes_copy); + memcpy(row0, row1, bytes_copy); + memcpy(row1, temp, bytes_copy); + row0 += bytes_copy; + row1 += bytes_copy; + bytes_left -= bytes_copy; + } + } +} + +#ifndef STBI_NO_GIF +static void stbi__vertical_flip_slices(void *image, int w, int h, int z, int bytes_per_pixel) +{ + int slice; + int slice_size = w * h * bytes_per_pixel; + + stbi_uc *bytes = (stbi_uc *)image; + for (slice = 0; slice < z; ++slice) { + stbi__vertical_flip(bytes, w, h, bytes_per_pixel); + bytes += slice_size; + } +} +#endif + +static unsigned char *stbi__load_and_postprocess_8bit(stbi__context *s, int *x, int *y, int *comp, int req_comp) +{ + stbi__result_info ri; + void *result = stbi__load_main(s, x, y, comp, req_comp, &ri, 8); + + if (result == NULL) + return NULL; + + // it is the responsibility of the loaders to make sure we get either 8 or 16 bit. + STBI_ASSERT(ri.bits_per_channel == 8 || ri.bits_per_channel == 16); + + if (ri.bits_per_channel != 8) { + result = stbi__convert_16_to_8((stbi__uint16 *) result, *x, *y, req_comp == 0 ? *comp : req_comp); + ri.bits_per_channel = 8; + } + + // @TODO: move stbi__convert_format to here + + if (stbi__vertically_flip_on_load) { + int channels = req_comp ? req_comp : *comp; + stbi__vertical_flip(result, *x, *y, channels * sizeof(stbi_uc)); + } + + return (unsigned char *) result; +} + +static stbi__uint16 *stbi__load_and_postprocess_16bit(stbi__context *s, int *x, int *y, int *comp, int req_comp) +{ + stbi__result_info ri; + void *result = stbi__load_main(s, x, y, comp, req_comp, &ri, 16); + + if (result == NULL) + return NULL; + + // it is the responsibility of the loaders to make sure we get either 8 or 16 bit. + STBI_ASSERT(ri.bits_per_channel == 8 || ri.bits_per_channel == 16); + + if (ri.bits_per_channel != 16) { + result = stbi__convert_8_to_16((stbi_uc *) result, *x, *y, req_comp == 0 ? *comp : req_comp); + ri.bits_per_channel = 16; + } + + // @TODO: move stbi__convert_format16 to here + // @TODO: special case RGB-to-Y (and RGBA-to-YA) for 8-bit-to-16-bit case to keep more precision + + if (stbi__vertically_flip_on_load) { + int channels = req_comp ? req_comp : *comp; + stbi__vertical_flip(result, *x, *y, channels * sizeof(stbi__uint16)); + } + + return (stbi__uint16 *) result; +} + +#if !defined(STBI_NO_HDR) && !defined(STBI_NO_LINEAR) +static void stbi__float_postprocess(float *result, int *x, int *y, int *comp, int req_comp) +{ + if (stbi__vertically_flip_on_load && result != NULL) { + int channels = req_comp ? req_comp : *comp; + stbi__vertical_flip(result, *x, *y, channels * sizeof(float)); + } +} +#endif + +#ifndef STBI_NO_STDIO + +#if defined(_WIN32) && defined(STBI_WINDOWS_UTF8) +STBI_EXTERN __declspec(dllimport) int __stdcall MultiByteToWideChar(unsigned int cp, unsigned long flags, const char *str, int cbmb, wchar_t *widestr, int cchwide); +STBI_EXTERN __declspec(dllimport) int __stdcall WideCharToMultiByte(unsigned int cp, unsigned long flags, const wchar_t *widestr, int cchwide, char *str, int cbmb, const char *defchar, int *used_default); +#endif + +#if defined(_WIN32) && defined(STBI_WINDOWS_UTF8) +STBIDEF int stbi_convert_wchar_to_utf8(char *buffer, size_t bufferlen, const wchar_t* input) +{ + return WideCharToMultiByte(65001 /* UTF8 */, 0, input, -1, buffer, (int) bufferlen, NULL, NULL); +} +#endif + +static FILE *stbi__fopen(char const *filename, char const *mode) +{ + FILE *f; +#if defined(_WIN32) && defined(STBI_WINDOWS_UTF8) + wchar_t wMode[64]; + wchar_t wFilename[1024]; + if (0 == MultiByteToWideChar(65001 /* UTF8 */, 0, filename, -1, wFilename, sizeof(wFilename)/sizeof(*wFilename))) + return 0; + + if (0 == MultiByteToWideChar(65001 /* UTF8 */, 0, mode, -1, wMode, sizeof(wMode)/sizeof(*wMode))) + return 0; + +#if defined(_MSC_VER) && _MSC_VER >= 1400 + if (0 != _wfopen_s(&f, wFilename, wMode)) + f = 0; +#else + f = _wfopen(wFilename, wMode); +#endif + +#elif defined(_MSC_VER) && _MSC_VER >= 1400 + if (0 != fopen_s(&f, filename, mode)) + f=0; +#else + f = fopen(filename, mode); +#endif + return f; +} + + +STBIDEF stbi_uc *stbi_load(char const *filename, int *x, int *y, int *comp, int req_comp) +{ + FILE *f = stbi__fopen(filename, "rb"); + unsigned char *result; + if (!f) return stbi__errpuc("can't fopen", "Unable to open file"); + result = stbi_load_from_file(f,x,y,comp,req_comp); + fclose(f); + return result; +} + +STBIDEF stbi_uc *stbi_load_from_file(FILE *f, int *x, int *y, int *comp, int req_comp) +{ + unsigned char *result; + stbi__context s; + stbi__start_file(&s,f); + result = stbi__load_and_postprocess_8bit(&s,x,y,comp,req_comp); + if (result) { + // need to 'unget' all the characters in the IO buffer + fseek(f, - (int) (s.img_buffer_end - s.img_buffer), SEEK_CUR); + } + return result; +} + +STBIDEF stbi__uint16 *stbi_load_from_file_16(FILE *f, int *x, int *y, int *comp, int req_comp) +{ + stbi__uint16 *result; + stbi__context s; + stbi__start_file(&s,f); + result = stbi__load_and_postprocess_16bit(&s,x,y,comp,req_comp); + if (result) { + // need to 'unget' all the characters in the IO buffer + fseek(f, - (int) (s.img_buffer_end - s.img_buffer), SEEK_CUR); + } + return result; +} + +STBIDEF stbi_us *stbi_load_16(char const *filename, int *x, int *y, int *comp, int req_comp) +{ + FILE *f = stbi__fopen(filename, "rb"); + stbi__uint16 *result; + if (!f) return (stbi_us *) stbi__errpuc("can't fopen", "Unable to open file"); + result = stbi_load_from_file_16(f,x,y,comp,req_comp); + fclose(f); + return result; +} + + +#endif //!STBI_NO_STDIO + +STBIDEF stbi_us *stbi_load_16_from_memory(stbi_uc const *buffer, int len, int *x, int *y, int *channels_in_file, int desired_channels) +{ + stbi__context s; + stbi__start_mem(&s,buffer,len); + return stbi__load_and_postprocess_16bit(&s,x,y,channels_in_file,desired_channels); +} + +STBIDEF stbi_us *stbi_load_16_from_callbacks(stbi_io_callbacks const *clbk, void *user, int *x, int *y, int *channels_in_file, int desired_channels) +{ + stbi__context s; + stbi__start_callbacks(&s, (stbi_io_callbacks *)clbk, user); + return stbi__load_and_postprocess_16bit(&s,x,y,channels_in_file,desired_channels); +} + +STBIDEF stbi_uc *stbi_load_from_memory(stbi_uc const *buffer, int len, int *x, int *y, int *comp, int req_comp) +{ + stbi__context s; + stbi__start_mem(&s,buffer,len); + return stbi__load_and_postprocess_8bit(&s,x,y,comp,req_comp); +} + +STBIDEF stbi_uc *stbi_load_from_callbacks(stbi_io_callbacks const *clbk, void *user, int *x, int *y, int *comp, int req_comp) +{ + stbi__context s; + stbi__start_callbacks(&s, (stbi_io_callbacks *) clbk, user); + return stbi__load_and_postprocess_8bit(&s,x,y,comp,req_comp); +} + +#ifndef STBI_NO_GIF +STBIDEF stbi_uc *stbi_load_gif_from_memory(stbi_uc const *buffer, int len, int **delays, int *x, int *y, int *z, int *comp, int req_comp) +{ + unsigned char *result; + stbi__context s; + stbi__start_mem(&s,buffer,len); + + result = (unsigned char*) stbi__load_gif_main(&s, delays, x, y, z, comp, req_comp); + if (stbi__vertically_flip_on_load) { + stbi__vertical_flip_slices( result, *x, *y, *z, *comp ); + } + + return result; +} +#endif + +#ifndef STBI_NO_LINEAR +static float *stbi__loadf_main(stbi__context *s, int *x, int *y, int *comp, int req_comp) +{ + unsigned char *data; + #ifndef STBI_NO_HDR + if (stbi__hdr_test(s)) { + stbi__result_info ri; + float *hdr_data = stbi__hdr_load(s,x,y,comp,req_comp, &ri); + if (hdr_data) + stbi__float_postprocess(hdr_data,x,y,comp,req_comp); + return hdr_data; + } + #endif + data = stbi__load_and_postprocess_8bit(s, x, y, comp, req_comp); + if (data) + return stbi__ldr_to_hdr(data, *x, *y, req_comp ? req_comp : *comp); + return stbi__errpf("unknown image type", "Image not of any known type, or corrupt"); +} + +STBIDEF float *stbi_loadf_from_memory(stbi_uc const *buffer, int len, int *x, int *y, int *comp, int req_comp) +{ + stbi__context s; + stbi__start_mem(&s,buffer,len); + return stbi__loadf_main(&s,x,y,comp,req_comp); +} + +STBIDEF float *stbi_loadf_from_callbacks(stbi_io_callbacks const *clbk, void *user, int *x, int *y, int *comp, int req_comp) +{ + stbi__context s; + stbi__start_callbacks(&s, (stbi_io_callbacks *) clbk, user); + return stbi__loadf_main(&s,x,y,comp,req_comp); +} + +#ifndef STBI_NO_STDIO +STBIDEF float *stbi_loadf(char const *filename, int *x, int *y, int *comp, int req_comp) +{ + float *result; + FILE *f = stbi__fopen(filename, "rb"); + if (!f) return stbi__errpf("can't fopen", "Unable to open file"); + result = stbi_loadf_from_file(f,x,y,comp,req_comp); + fclose(f); + return result; +} + +STBIDEF float *stbi_loadf_from_file(FILE *f, int *x, int *y, int *comp, int req_comp) +{ + stbi__context s; + stbi__start_file(&s,f); + return stbi__loadf_main(&s,x,y,comp,req_comp); +} +#endif // !STBI_NO_STDIO + +#endif // !STBI_NO_LINEAR + +// these is-hdr-or-not is defined independent of whether STBI_NO_LINEAR is +// defined, for API simplicity; if STBI_NO_LINEAR is defined, it always +// reports false! + +STBIDEF int stbi_is_hdr_from_memory(stbi_uc const *buffer, int len) +{ + #ifndef STBI_NO_HDR + stbi__context s; + stbi__start_mem(&s,buffer,len); + return stbi__hdr_test(&s); + #else + STBI_NOTUSED(buffer); + STBI_NOTUSED(len); + return 0; + #endif +} + +#ifndef STBI_NO_STDIO +STBIDEF int stbi_is_hdr (char const *filename) +{ + FILE *f = stbi__fopen(filename, "rb"); + int result=0; + if (f) { + result = stbi_is_hdr_from_file(f); + fclose(f); + } + return result; +} + +STBIDEF int stbi_is_hdr_from_file(FILE *f) +{ + #ifndef STBI_NO_HDR + long pos = ftell(f); + int res; + stbi__context s; + stbi__start_file(&s,f); + res = stbi__hdr_test(&s); + fseek(f, pos, SEEK_SET); + return res; + #else + STBI_NOTUSED(f); + return 0; + #endif +} +#endif // !STBI_NO_STDIO + +STBIDEF int stbi_is_hdr_from_callbacks(stbi_io_callbacks const *clbk, void *user) +{ + #ifndef STBI_NO_HDR + stbi__context s; + stbi__start_callbacks(&s, (stbi_io_callbacks *) clbk, user); + return stbi__hdr_test(&s); + #else + STBI_NOTUSED(clbk); + STBI_NOTUSED(user); + return 0; + #endif +} + +#ifndef STBI_NO_LINEAR +static float stbi__l2h_gamma=2.2f, stbi__l2h_scale=1.0f; + +STBIDEF void stbi_ldr_to_hdr_gamma(float gamma) { stbi__l2h_gamma = gamma; } +STBIDEF void stbi_ldr_to_hdr_scale(float scale) { stbi__l2h_scale = scale; } +#endif + +static float stbi__h2l_gamma_i=1.0f/2.2f, stbi__h2l_scale_i=1.0f; + +STBIDEF void stbi_hdr_to_ldr_gamma(float gamma) { stbi__h2l_gamma_i = 1/gamma; } +STBIDEF void stbi_hdr_to_ldr_scale(float scale) { stbi__h2l_scale_i = 1/scale; } + + +////////////////////////////////////////////////////////////////////////////// +// +// Common code used by all image loaders +// + +enum +{ + STBI__SCAN_load=0, + STBI__SCAN_type, + STBI__SCAN_header +}; + +static void stbi__refill_buffer(stbi__context *s) +{ + int n = (s->io.read)(s->io_user_data,(char*)s->buffer_start,s->buflen); + s->callback_already_read += (int) (s->img_buffer - s->img_buffer_original); + if (n == 0) { + // at end of file, treat same as if from memory, but need to handle case + // where s->img_buffer isn't pointing to safe memory, e.g. 0-byte file + s->read_from_callbacks = 0; + s->img_buffer = s->buffer_start; + s->img_buffer_end = s->buffer_start+1; + *s->img_buffer = 0; + } else { + s->img_buffer = s->buffer_start; + s->img_buffer_end = s->buffer_start + n; + } +} + +stbi_inline static stbi_uc stbi__get8(stbi__context *s) +{ + if (s->img_buffer < s->img_buffer_end) + return *s->img_buffer++; + if (s->read_from_callbacks) { + stbi__refill_buffer(s); + return *s->img_buffer++; + } + return 0; +} + +#if defined(STBI_NO_JPEG) && defined(STBI_NO_HDR) && defined(STBI_NO_PIC) && defined(STBI_NO_PNM) +// nothing +#else +stbi_inline static int stbi__at_eof(stbi__context *s) +{ + if (s->io.read) { + if (!(s->io.eof)(s->io_user_data)) return 0; + // if feof() is true, check if buffer = end + // special case: we've only got the special 0 character at the end + if (s->read_from_callbacks == 0) return 1; + } + + return s->img_buffer >= s->img_buffer_end; +} +#endif + +#if defined(STBI_NO_JPEG) && defined(STBI_NO_PNG) && defined(STBI_NO_BMP) && defined(STBI_NO_PSD) && defined(STBI_NO_TGA) && defined(STBI_NO_GIF) && defined(STBI_NO_PIC) +// nothing +#else +static void stbi__skip(stbi__context *s, int n) +{ + if (n == 0) return; // already there! + if (n < 0) { + s->img_buffer = s->img_buffer_end; + return; + } + if (s->io.read) { + int blen = (int) (s->img_buffer_end - s->img_buffer); + if (blen < n) { + s->img_buffer = s->img_buffer_end; + (s->io.skip)(s->io_user_data, n - blen); + return; + } + } + s->img_buffer += n; +} +#endif + +#if defined(STBI_NO_PNG) && defined(STBI_NO_TGA) && defined(STBI_NO_HDR) && defined(STBI_NO_PNM) +// nothing +#else +static int stbi__getn(stbi__context *s, stbi_uc *buffer, int n) +{ + if (s->io.read) { + int blen = (int) (s->img_buffer_end - s->img_buffer); + if (blen < n) { + int res, count; + + memcpy(buffer, s->img_buffer, blen); + + count = (s->io.read)(s->io_user_data, (char*) buffer + blen, n - blen); + res = (count == (n-blen)); + s->img_buffer = s->img_buffer_end; + return res; + } + } + + if (s->img_buffer+n <= s->img_buffer_end) { + memcpy(buffer, s->img_buffer, n); + s->img_buffer += n; + return 1; + } else + return 0; +} +#endif + +#if defined(STBI_NO_JPEG) && defined(STBI_NO_PNG) && defined(STBI_NO_PSD) && defined(STBI_NO_PIC) +// nothing +#else +static int stbi__get16be(stbi__context *s) +{ + int z = stbi__get8(s); + return (z << 8) + stbi__get8(s); +} +#endif + +#if defined(STBI_NO_PNG) && defined(STBI_NO_PSD) && defined(STBI_NO_PIC) +// nothing +#else +static stbi__uint32 stbi__get32be(stbi__context *s) +{ + stbi__uint32 z = stbi__get16be(s); + return (z << 16) + stbi__get16be(s); +} +#endif + +#if defined(STBI_NO_BMP) && defined(STBI_NO_TGA) && defined(STBI_NO_GIF) +// nothing +#else +static int stbi__get16le(stbi__context *s) +{ + int z = stbi__get8(s); + return z + (stbi__get8(s) << 8); +} +#endif + +#ifndef STBI_NO_BMP +static stbi__uint32 stbi__get32le(stbi__context *s) +{ + stbi__uint32 z = stbi__get16le(s); + z += (stbi__uint32)stbi__get16le(s) << 16; + return z; +} +#endif + +#define STBI__BYTECAST(x) ((stbi_uc) ((x) & 255)) // truncate int to byte without warnings + +#if defined(STBI_NO_JPEG) && defined(STBI_NO_PNG) && defined(STBI_NO_BMP) && defined(STBI_NO_PSD) && defined(STBI_NO_TGA) && defined(STBI_NO_GIF) && defined(STBI_NO_PIC) && defined(STBI_NO_PNM) +// nothing +#else +////////////////////////////////////////////////////////////////////////////// +// +// generic converter from built-in img_n to req_comp +// individual types do this automatically as much as possible (e.g. jpeg +// does all cases internally since it needs to colorspace convert anyway, +// and it never has alpha, so very few cases ). png can automatically +// interleave an alpha=255 channel, but falls back to this for other cases +// +// assume data buffer is malloced, so malloc a new one and free that one +// only failure mode is malloc failing + +static stbi_uc stbi__compute_y(int r, int g, int b) +{ + return (stbi_uc) (((r*77) + (g*150) + (29*b)) >> 8); +} +#endif + +#if defined(STBI_NO_PNG) && defined(STBI_NO_BMP) && defined(STBI_NO_PSD) && defined(STBI_NO_TGA) && defined(STBI_NO_GIF) && defined(STBI_NO_PIC) && defined(STBI_NO_PNM) +// nothing +#else +static unsigned char *stbi__convert_format(unsigned char *data, int img_n, int req_comp, unsigned int x, unsigned int y) +{ + int i,j; + unsigned char *good; + + if (req_comp == img_n) return data; + STBI_ASSERT(req_comp >= 1 && req_comp <= 4); + + good = (unsigned char *) stbi__malloc_mad3(req_comp, x, y, 0); + if (good == NULL) { + STBI_FREE(data); + return stbi__errpuc("outofmem", "Out of memory"); + } + + for (j=0; j < (int) y; ++j) { + unsigned char *src = data + j * x * img_n ; + unsigned char *dest = good + j * x * req_comp; + + #define STBI__COMBO(a,b) ((a)*8+(b)) + #define STBI__CASE(a,b) case STBI__COMBO(a,b): for(i=x-1; i >= 0; --i, src += a, dest += b) + // convert source image with img_n components to one with req_comp components; + // avoid switch per pixel, so use switch per scanline and massive macros + switch (STBI__COMBO(img_n, req_comp)) { + STBI__CASE(1,2) { dest[0]=src[0]; dest[1]=255; } break; + STBI__CASE(1,3) { dest[0]=dest[1]=dest[2]=src[0]; } break; + STBI__CASE(1,4) { dest[0]=dest[1]=dest[2]=src[0]; dest[3]=255; } break; + STBI__CASE(2,1) { dest[0]=src[0]; } break; + STBI__CASE(2,3) { dest[0]=dest[1]=dest[2]=src[0]; } break; + STBI__CASE(2,4) { dest[0]=dest[1]=dest[2]=src[0]; dest[3]=src[1]; } break; + STBI__CASE(3,4) { dest[0]=src[0];dest[1]=src[1];dest[2]=src[2];dest[3]=255; } break; + STBI__CASE(3,1) { dest[0]=stbi__compute_y(src[0],src[1],src[2]); } break; + STBI__CASE(3,2) { dest[0]=stbi__compute_y(src[0],src[1],src[2]); dest[1] = 255; } break; + STBI__CASE(4,1) { dest[0]=stbi__compute_y(src[0],src[1],src[2]); } break; + STBI__CASE(4,2) { dest[0]=stbi__compute_y(src[0],src[1],src[2]); dest[1] = src[3]; } break; + STBI__CASE(4,3) { dest[0]=src[0];dest[1]=src[1];dest[2]=src[2]; } break; + default: STBI_ASSERT(0); STBI_FREE(data); STBI_FREE(good); return stbi__errpuc("unsupported", "Unsupported format conversion"); + } + #undef STBI__CASE + } + + STBI_FREE(data); + return good; +} +#endif + +#if defined(STBI_NO_PNG) && defined(STBI_NO_PSD) +// nothing +#else +static stbi__uint16 stbi__compute_y_16(int r, int g, int b) +{ + return (stbi__uint16) (((r*77) + (g*150) + (29*b)) >> 8); +} +#endif + +#if defined(STBI_NO_PNG) && defined(STBI_NO_PSD) +// nothing +#else +static stbi__uint16 *stbi__convert_format16(stbi__uint16 *data, int img_n, int req_comp, unsigned int x, unsigned int y) +{ + int i,j; + stbi__uint16 *good; + + if (req_comp == img_n) return data; + STBI_ASSERT(req_comp >= 1 && req_comp <= 4); + + good = (stbi__uint16 *) stbi__malloc(req_comp * x * y * 2); + if (good == NULL) { + STBI_FREE(data); + return (stbi__uint16 *) stbi__errpuc("outofmem", "Out of memory"); + } + + for (j=0; j < (int) y; ++j) { + stbi__uint16 *src = data + j * x * img_n ; + stbi__uint16 *dest = good + j * x * req_comp; + + #define STBI__COMBO(a,b) ((a)*8+(b)) + #define STBI__CASE(a,b) case STBI__COMBO(a,b): for(i=x-1; i >= 0; --i, src += a, dest += b) + // convert source image with img_n components to one with req_comp components; + // avoid switch per pixel, so use switch per scanline and massive macros + switch (STBI__COMBO(img_n, req_comp)) { + STBI__CASE(1,2) { dest[0]=src[0]; dest[1]=0xffff; } break; + STBI__CASE(1,3) { dest[0]=dest[1]=dest[2]=src[0]; } break; + STBI__CASE(1,4) { dest[0]=dest[1]=dest[2]=src[0]; dest[3]=0xffff; } break; + STBI__CASE(2,1) { dest[0]=src[0]; } break; + STBI__CASE(2,3) { dest[0]=dest[1]=dest[2]=src[0]; } break; + STBI__CASE(2,4) { dest[0]=dest[1]=dest[2]=src[0]; dest[3]=src[1]; } break; + STBI__CASE(3,4) { dest[0]=src[0];dest[1]=src[1];dest[2]=src[2];dest[3]=0xffff; } break; + STBI__CASE(3,1) { dest[0]=stbi__compute_y_16(src[0],src[1],src[2]); } break; + STBI__CASE(3,2) { dest[0]=stbi__compute_y_16(src[0],src[1],src[2]); dest[1] = 0xffff; } break; + STBI__CASE(4,1) { dest[0]=stbi__compute_y_16(src[0],src[1],src[2]); } break; + STBI__CASE(4,2) { dest[0]=stbi__compute_y_16(src[0],src[1],src[2]); dest[1] = src[3]; } break; + STBI__CASE(4,3) { dest[0]=src[0];dest[1]=src[1];dest[2]=src[2]; } break; + default: STBI_ASSERT(0); STBI_FREE(data); STBI_FREE(good); return (stbi__uint16*) stbi__errpuc("unsupported", "Unsupported format conversion"); + } + #undef STBI__CASE + } + + STBI_FREE(data); + return good; +} +#endif + +#ifndef STBI_NO_LINEAR +static float *stbi__ldr_to_hdr(stbi_uc *data, int x, int y, int comp) +{ + int i,k,n; + float *output; + if (!data) return NULL; + output = (float *) stbi__malloc_mad4(x, y, comp, sizeof(float), 0); + if (output == NULL) { STBI_FREE(data); return stbi__errpf("outofmem", "Out of memory"); } + // compute number of non-alpha components + if (comp & 1) n = comp; else n = comp-1; + for (i=0; i < x*y; ++i) { + for (k=0; k < n; ++k) { + output[i*comp + k] = (float) (pow(data[i*comp+k]/255.0f, stbi__l2h_gamma) * stbi__l2h_scale); + } + } + if (n < comp) { + for (i=0; i < x*y; ++i) { + output[i*comp + n] = data[i*comp + n]/255.0f; + } + } + STBI_FREE(data); + return output; +} +#endif + +#ifndef STBI_NO_HDR +#define stbi__float2int(x) ((int) (x)) +static stbi_uc *stbi__hdr_to_ldr(float *data, int x, int y, int comp) +{ + int i,k,n; + stbi_uc *output; + if (!data) return NULL; + output = (stbi_uc *) stbi__malloc_mad3(x, y, comp, 0); + if (output == NULL) { STBI_FREE(data); return stbi__errpuc("outofmem", "Out of memory"); } + // compute number of non-alpha components + if (comp & 1) n = comp; else n = comp-1; + for (i=0; i < x*y; ++i) { + for (k=0; k < n; ++k) { + float z = (float) pow(data[i*comp+k]*stbi__h2l_scale_i, stbi__h2l_gamma_i) * 255 + 0.5f; + if (z < 0) z = 0; + if (z > 255) z = 255; + output[i*comp + k] = (stbi_uc) stbi__float2int(z); + } + if (k < comp) { + float z = data[i*comp+k] * 255 + 0.5f; + if (z < 0) z = 0; + if (z > 255) z = 255; + output[i*comp + k] = (stbi_uc) stbi__float2int(z); + } + } + STBI_FREE(data); + return output; +} +#endif + +////////////////////////////////////////////////////////////////////////////// +// +// "baseline" JPEG/JFIF decoder +// +// simple implementation +// - doesn't support delayed output of y-dimension +// - simple interface (only one output format: 8-bit interleaved RGB) +// - doesn't try to recover corrupt jpegs +// - doesn't allow partial loading, loading multiple at once +// - still fast on x86 (copying globals into locals doesn't help x86) +// - allocates lots of intermediate memory (full size of all components) +// - non-interleaved case requires this anyway +// - allows good upsampling (see next) +// high-quality +// - upsampled channels are bilinearly interpolated, even across blocks +// - quality integer IDCT derived from IJG's 'slow' +// performance +// - fast huffman; reasonable integer IDCT +// - some SIMD kernels for common paths on targets with SSE2/NEON +// - uses a lot of intermediate memory, could cache poorly + +#ifndef STBI_NO_JPEG + +// huffman decoding acceleration +#define FAST_BITS 9 // larger handles more cases; smaller stomps less cache + +typedef struct +{ + stbi_uc fast[1 << FAST_BITS]; + // weirdly, repacking this into AoS is a 10% speed loss, instead of a win + stbi__uint16 code[256]; + stbi_uc values[256]; + stbi_uc size[257]; + unsigned int maxcode[18]; + int delta[17]; // old 'firstsymbol' - old 'firstcode' +} stbi__huffman; + +typedef struct +{ + stbi__context *s; + stbi__huffman huff_dc[4]; + stbi__huffman huff_ac[4]; + stbi__uint16 dequant[4][64]; + stbi__int16 fast_ac[4][1 << FAST_BITS]; + +// sizes for components, interleaved MCUs + int img_h_max, img_v_max; + int img_mcu_x, img_mcu_y; + int img_mcu_w, img_mcu_h; + +// definition of jpeg image component + struct + { + int id; + int h,v; + int tq; + int hd,ha; + int dc_pred; + + int x,y,w2,h2; + stbi_uc *data; + void *raw_data, *raw_coeff; + stbi_uc *linebuf; + short *coeff; // progressive only + int coeff_w, coeff_h; // number of 8x8 coefficient blocks + } img_comp[4]; + + stbi__uint32 code_buffer; // jpeg entropy-coded buffer + int code_bits; // number of valid bits + unsigned char marker; // marker seen while filling entropy buffer + int nomore; // flag if we saw a marker so must stop + + int progressive; + int spec_start; + int spec_end; + int succ_high; + int succ_low; + int eob_run; + int jfif; + int app14_color_transform; // Adobe APP14 tag + int rgb; + + int scan_n, order[4]; + int restart_interval, todo; + +// kernels + void (*idct_block_kernel)(stbi_uc *out, int out_stride, short data[64]); + void (*YCbCr_to_RGB_kernel)(stbi_uc *out, const stbi_uc *y, const stbi_uc *pcb, const stbi_uc *pcr, int count, int step); + stbi_uc *(*resample_row_hv_2_kernel)(stbi_uc *out, stbi_uc *in_near, stbi_uc *in_far, int w, int hs); +} stbi__jpeg; + +static int stbi__build_huffman(stbi__huffman *h, int *count) +{ + int i,j,k=0; + unsigned int code; + // build size list for each symbol (from JPEG spec) + for (i=0; i < 16; ++i) + for (j=0; j < count[i]; ++j) + h->size[k++] = (stbi_uc) (i+1); + h->size[k] = 0; + + // compute actual symbols (from jpeg spec) + code = 0; + k = 0; + for(j=1; j <= 16; ++j) { + // compute delta to add to code to compute symbol id + h->delta[j] = k - code; + if (h->size[k] == j) { + while (h->size[k] == j) + h->code[k++] = (stbi__uint16) (code++); + if (code-1 >= (1u << j)) return stbi__err("bad code lengths","Corrupt JPEG"); + } + // compute largest code + 1 for this size, preshifted as needed later + h->maxcode[j] = code << (16-j); + code <<= 1; + } + h->maxcode[j] = 0xffffffff; + + // build non-spec acceleration table; 255 is flag for not-accelerated + memset(h->fast, 255, 1 << FAST_BITS); + for (i=0; i < k; ++i) { + int s = h->size[i]; + if (s <= FAST_BITS) { + int c = h->code[i] << (FAST_BITS-s); + int m = 1 << (FAST_BITS-s); + for (j=0; j < m; ++j) { + h->fast[c+j] = (stbi_uc) i; + } + } + } + return 1; +} + +// build a table that decodes both magnitude and value of small ACs in +// one go. +static void stbi__build_fast_ac(stbi__int16 *fast_ac, stbi__huffman *h) +{ + int i; + for (i=0; i < (1 << FAST_BITS); ++i) { + stbi_uc fast = h->fast[i]; + fast_ac[i] = 0; + if (fast < 255) { + int rs = h->values[fast]; + int run = (rs >> 4) & 15; + int magbits = rs & 15; + int len = h->size[fast]; + + if (magbits && len + magbits <= FAST_BITS) { + // magnitude code followed by receive_extend code + int k = ((i << len) & ((1 << FAST_BITS) - 1)) >> (FAST_BITS - magbits); + int m = 1 << (magbits - 1); + if (k < m) k += (~0U << magbits) + 1; + // if the result is small enough, we can fit it in fast_ac table + if (k >= -128 && k <= 127) + fast_ac[i] = (stbi__int16) ((k * 256) + (run * 16) + (len + magbits)); + } + } + } +} + +static void stbi__grow_buffer_unsafe(stbi__jpeg *j) +{ + do { + unsigned int b = j->nomore ? 0 : stbi__get8(j->s); + if (b == 0xff) { + int c = stbi__get8(j->s); + while (c == 0xff) c = stbi__get8(j->s); // consume fill bytes + if (c != 0) { + j->marker = (unsigned char) c; + j->nomore = 1; + return; + } + } + j->code_buffer |= b << (24 - j->code_bits); + j->code_bits += 8; + } while (j->code_bits <= 24); +} + +// (1 << n) - 1 +static const stbi__uint32 stbi__bmask[17]={0,1,3,7,15,31,63,127,255,511,1023,2047,4095,8191,16383,32767,65535}; + +// decode a jpeg huffman value from the bitstream +stbi_inline static int stbi__jpeg_huff_decode(stbi__jpeg *j, stbi__huffman *h) +{ + unsigned int temp; + int c,k; + + if (j->code_bits < 16) stbi__grow_buffer_unsafe(j); + + // look at the top FAST_BITS and determine what symbol ID it is, + // if the code is <= FAST_BITS + c = (j->code_buffer >> (32 - FAST_BITS)) & ((1 << FAST_BITS)-1); + k = h->fast[c]; + if (k < 255) { + int s = h->size[k]; + if (s > j->code_bits) + return -1; + j->code_buffer <<= s; + j->code_bits -= s; + return h->values[k]; + } + + // naive test is to shift the code_buffer down so k bits are + // valid, then test against maxcode. To speed this up, we've + // preshifted maxcode left so that it has (16-k) 0s at the + // end; in other words, regardless of the number of bits, it + // wants to be compared against something shifted to have 16; + // that way we don't need to shift inside the loop. + temp = j->code_buffer >> 16; + for (k=FAST_BITS+1 ; ; ++k) + if (temp < h->maxcode[k]) + break; + if (k == 17) { + // error! code not found + j->code_bits -= 16; + return -1; + } + + if (k > j->code_bits) + return -1; + + // convert the huffman code to the symbol id + c = ((j->code_buffer >> (32 - k)) & stbi__bmask[k]) + h->delta[k]; + STBI_ASSERT((((j->code_buffer) >> (32 - h->size[c])) & stbi__bmask[h->size[c]]) == h->code[c]); + + // convert the id to a symbol + j->code_bits -= k; + j->code_buffer <<= k; + return h->values[c]; +} + +// bias[n] = (-1<code_bits < n) stbi__grow_buffer_unsafe(j); + + sgn = j->code_buffer >> 31; // sign bit always in MSB; 0 if MSB clear (positive), 1 if MSB set (negative) + k = stbi_lrot(j->code_buffer, n); + j->code_buffer = k & ~stbi__bmask[n]; + k &= stbi__bmask[n]; + j->code_bits -= n; + return k + (stbi__jbias[n] & (sgn - 1)); +} + +// get some unsigned bits +stbi_inline static int stbi__jpeg_get_bits(stbi__jpeg *j, int n) +{ + unsigned int k; + if (j->code_bits < n) stbi__grow_buffer_unsafe(j); + k = stbi_lrot(j->code_buffer, n); + j->code_buffer = k & ~stbi__bmask[n]; + k &= stbi__bmask[n]; + j->code_bits -= n; + return k; +} + +stbi_inline static int stbi__jpeg_get_bit(stbi__jpeg *j) +{ + unsigned int k; + if (j->code_bits < 1) stbi__grow_buffer_unsafe(j); + k = j->code_buffer; + j->code_buffer <<= 1; + --j->code_bits; + return k & 0x80000000; +} + +// given a value that's at position X in the zigzag stream, +// where does it appear in the 8x8 matrix coded as row-major? +static const stbi_uc stbi__jpeg_dezigzag[64+15] = +{ + 0, 1, 8, 16, 9, 2, 3, 10, + 17, 24, 32, 25, 18, 11, 4, 5, + 12, 19, 26, 33, 40, 48, 41, 34, + 27, 20, 13, 6, 7, 14, 21, 28, + 35, 42, 49, 56, 57, 50, 43, 36, + 29, 22, 15, 23, 30, 37, 44, 51, + 58, 59, 52, 45, 38, 31, 39, 46, + 53, 60, 61, 54, 47, 55, 62, 63, + // let corrupt input sample past end + 63, 63, 63, 63, 63, 63, 63, 63, + 63, 63, 63, 63, 63, 63, 63 +}; + +// decode one 64-entry block-- +static int stbi__jpeg_decode_block(stbi__jpeg *j, short data[64], stbi__huffman *hdc, stbi__huffman *hac, stbi__int16 *fac, int b, stbi__uint16 *dequant) +{ + int diff,dc,k; + int t; + + if (j->code_bits < 16) stbi__grow_buffer_unsafe(j); + t = stbi__jpeg_huff_decode(j, hdc); + if (t < 0 || t > 15) return stbi__err("bad huffman code","Corrupt JPEG"); + + // 0 all the ac values now so we can do it 32-bits at a time + memset(data,0,64*sizeof(data[0])); + + diff = t ? stbi__extend_receive(j, t) : 0; + dc = j->img_comp[b].dc_pred + diff; + j->img_comp[b].dc_pred = dc; + data[0] = (short) (dc * dequant[0]); + + // decode AC components, see JPEG spec + k = 1; + do { + unsigned int zig; + int c,r,s; + if (j->code_bits < 16) stbi__grow_buffer_unsafe(j); + c = (j->code_buffer >> (32 - FAST_BITS)) & ((1 << FAST_BITS)-1); + r = fac[c]; + if (r) { // fast-AC path + k += (r >> 4) & 15; // run + s = r & 15; // combined length + j->code_buffer <<= s; + j->code_bits -= s; + // decode into unzigzag'd location + zig = stbi__jpeg_dezigzag[k++]; + data[zig] = (short) ((r >> 8) * dequant[zig]); + } else { + int rs = stbi__jpeg_huff_decode(j, hac); + if (rs < 0) return stbi__err("bad huffman code","Corrupt JPEG"); + s = rs & 15; + r = rs >> 4; + if (s == 0) { + if (rs != 0xf0) break; // end block + k += 16; + } else { + k += r; + // decode into unzigzag'd location + zig = stbi__jpeg_dezigzag[k++]; + data[zig] = (short) (stbi__extend_receive(j,s) * dequant[zig]); + } + } + } while (k < 64); + return 1; +} + +static int stbi__jpeg_decode_block_prog_dc(stbi__jpeg *j, short data[64], stbi__huffman *hdc, int b) +{ + int diff,dc; + int t; + if (j->spec_end != 0) return stbi__err("can't merge dc and ac", "Corrupt JPEG"); + + if (j->code_bits < 16) stbi__grow_buffer_unsafe(j); + + if (j->succ_high == 0) { + // first scan for DC coefficient, must be first + memset(data,0,64*sizeof(data[0])); // 0 all the ac values now + t = stbi__jpeg_huff_decode(j, hdc); + if (t < 0 || t > 15) return stbi__err("can't merge dc and ac", "Corrupt JPEG"); + diff = t ? stbi__extend_receive(j, t) : 0; + + dc = j->img_comp[b].dc_pred + diff; + j->img_comp[b].dc_pred = dc; + data[0] = (short) (dc * (1 << j->succ_low)); + } else { + // refinement scan for DC coefficient + if (stbi__jpeg_get_bit(j)) + data[0] += (short) (1 << j->succ_low); + } + return 1; +} + +// @OPTIMIZE: store non-zigzagged during the decode passes, +// and only de-zigzag when dequantizing +static int stbi__jpeg_decode_block_prog_ac(stbi__jpeg *j, short data[64], stbi__huffman *hac, stbi__int16 *fac) +{ + int k; + if (j->spec_start == 0) return stbi__err("can't merge dc and ac", "Corrupt JPEG"); + + if (j->succ_high == 0) { + int shift = j->succ_low; + + if (j->eob_run) { + --j->eob_run; + return 1; + } + + k = j->spec_start; + do { + unsigned int zig; + int c,r,s; + if (j->code_bits < 16) stbi__grow_buffer_unsafe(j); + c = (j->code_buffer >> (32 - FAST_BITS)) & ((1 << FAST_BITS)-1); + r = fac[c]; + if (r) { // fast-AC path + k += (r >> 4) & 15; // run + s = r & 15; // combined length + j->code_buffer <<= s; + j->code_bits -= s; + zig = stbi__jpeg_dezigzag[k++]; + data[zig] = (short) ((r >> 8) * (1 << shift)); + } else { + int rs = stbi__jpeg_huff_decode(j, hac); + if (rs < 0) return stbi__err("bad huffman code","Corrupt JPEG"); + s = rs & 15; + r = rs >> 4; + if (s == 0) { + if (r < 15) { + j->eob_run = (1 << r); + if (r) + j->eob_run += stbi__jpeg_get_bits(j, r); + --j->eob_run; + break; + } + k += 16; + } else { + k += r; + zig = stbi__jpeg_dezigzag[k++]; + data[zig] = (short) (stbi__extend_receive(j,s) * (1 << shift)); + } + } + } while (k <= j->spec_end); + } else { + // refinement scan for these AC coefficients + + short bit = (short) (1 << j->succ_low); + + if (j->eob_run) { + --j->eob_run; + for (k = j->spec_start; k <= j->spec_end; ++k) { + short *p = &data[stbi__jpeg_dezigzag[k]]; + if (*p != 0) + if (stbi__jpeg_get_bit(j)) + if ((*p & bit)==0) { + if (*p > 0) + *p += bit; + else + *p -= bit; + } + } + } else { + k = j->spec_start; + do { + int r,s; + int rs = stbi__jpeg_huff_decode(j, hac); // @OPTIMIZE see if we can use the fast path here, advance-by-r is so slow, eh + if (rs < 0) return stbi__err("bad huffman code","Corrupt JPEG"); + s = rs & 15; + r = rs >> 4; + if (s == 0) { + if (r < 15) { + j->eob_run = (1 << r) - 1; + if (r) + j->eob_run += stbi__jpeg_get_bits(j, r); + r = 64; // force end of block + } else { + // r=15 s=0 should write 16 0s, so we just do + // a run of 15 0s and then write s (which is 0), + // so we don't have to do anything special here + } + } else { + if (s != 1) return stbi__err("bad huffman code", "Corrupt JPEG"); + // sign bit + if (stbi__jpeg_get_bit(j)) + s = bit; + else + s = -bit; + } + + // advance by r + while (k <= j->spec_end) { + short *p = &data[stbi__jpeg_dezigzag[k++]]; + if (*p != 0) { + if (stbi__jpeg_get_bit(j)) + if ((*p & bit)==0) { + if (*p > 0) + *p += bit; + else + *p -= bit; + } + } else { + if (r == 0) { + *p = (short) s; + break; + } + --r; + } + } + } while (k <= j->spec_end); + } + } + return 1; +} + +// take a -128..127 value and stbi__clamp it and convert to 0..255 +stbi_inline static stbi_uc stbi__clamp(int x) +{ + // trick to use a single test to catch both cases + if ((unsigned int) x > 255) { + if (x < 0) return 0; + if (x > 255) return 255; + } + return (stbi_uc) x; +} + +#define stbi__f2f(x) ((int) (((x) * 4096 + 0.5))) +#define stbi__fsh(x) ((x) * 4096) + +// derived from jidctint -- DCT_ISLOW +#define STBI__IDCT_1D(s0,s1,s2,s3,s4,s5,s6,s7) \ + int t0,t1,t2,t3,p1,p2,p3,p4,p5,x0,x1,x2,x3; \ + p2 = s2; \ + p3 = s6; \ + p1 = (p2+p3) * stbi__f2f(0.5411961f); \ + t2 = p1 + p3*stbi__f2f(-1.847759065f); \ + t3 = p1 + p2*stbi__f2f( 0.765366865f); \ + p2 = s0; \ + p3 = s4; \ + t0 = stbi__fsh(p2+p3); \ + t1 = stbi__fsh(p2-p3); \ + x0 = t0+t3; \ + x3 = t0-t3; \ + x1 = t1+t2; \ + x2 = t1-t2; \ + t0 = s7; \ + t1 = s5; \ + t2 = s3; \ + t3 = s1; \ + p3 = t0+t2; \ + p4 = t1+t3; \ + p1 = t0+t3; \ + p2 = t1+t2; \ + p5 = (p3+p4)*stbi__f2f( 1.175875602f); \ + t0 = t0*stbi__f2f( 0.298631336f); \ + t1 = t1*stbi__f2f( 2.053119869f); \ + t2 = t2*stbi__f2f( 3.072711026f); \ + t3 = t3*stbi__f2f( 1.501321110f); \ + p1 = p5 + p1*stbi__f2f(-0.899976223f); \ + p2 = p5 + p2*stbi__f2f(-2.562915447f); \ + p3 = p3*stbi__f2f(-1.961570560f); \ + p4 = p4*stbi__f2f(-0.390180644f); \ + t3 += p1+p4; \ + t2 += p2+p3; \ + t1 += p2+p4; \ + t0 += p1+p3; + +static void stbi__idct_block(stbi_uc *out, int out_stride, short data[64]) +{ + int i,val[64],*v=val; + stbi_uc *o; + short *d = data; + + // columns + for (i=0; i < 8; ++i,++d, ++v) { + // if all zeroes, shortcut -- this avoids dequantizing 0s and IDCTing + if (d[ 8]==0 && d[16]==0 && d[24]==0 && d[32]==0 + && d[40]==0 && d[48]==0 && d[56]==0) { + // no shortcut 0 seconds + // (1|2|3|4|5|6|7)==0 0 seconds + // all separate -0.047 seconds + // 1 && 2|3 && 4|5 && 6|7: -0.047 seconds + int dcterm = d[0]*4; + v[0] = v[8] = v[16] = v[24] = v[32] = v[40] = v[48] = v[56] = dcterm; + } else { + STBI__IDCT_1D(d[ 0],d[ 8],d[16],d[24],d[32],d[40],d[48],d[56]) + // constants scaled things up by 1<<12; let's bring them back + // down, but keep 2 extra bits of precision + x0 += 512; x1 += 512; x2 += 512; x3 += 512; + v[ 0] = (x0+t3) >> 10; + v[56] = (x0-t3) >> 10; + v[ 8] = (x1+t2) >> 10; + v[48] = (x1-t2) >> 10; + v[16] = (x2+t1) >> 10; + v[40] = (x2-t1) >> 10; + v[24] = (x3+t0) >> 10; + v[32] = (x3-t0) >> 10; + } + } + + for (i=0, v=val, o=out; i < 8; ++i,v+=8,o+=out_stride) { + // no fast case since the first 1D IDCT spread components out + STBI__IDCT_1D(v[0],v[1],v[2],v[3],v[4],v[5],v[6],v[7]) + // constants scaled things up by 1<<12, plus we had 1<<2 from first + // loop, plus horizontal and vertical each scale by sqrt(8) so together + // we've got an extra 1<<3, so 1<<17 total we need to remove. + // so we want to round that, which means adding 0.5 * 1<<17, + // aka 65536. Also, we'll end up with -128 to 127 that we want + // to encode as 0..255 by adding 128, so we'll add that before the shift + x0 += 65536 + (128<<17); + x1 += 65536 + (128<<17); + x2 += 65536 + (128<<17); + x3 += 65536 + (128<<17); + // tried computing the shifts into temps, or'ing the temps to see + // if any were out of range, but that was slower + o[0] = stbi__clamp((x0+t3) >> 17); + o[7] = stbi__clamp((x0-t3) >> 17); + o[1] = stbi__clamp((x1+t2) >> 17); + o[6] = stbi__clamp((x1-t2) >> 17); + o[2] = stbi__clamp((x2+t1) >> 17); + o[5] = stbi__clamp((x2-t1) >> 17); + o[3] = stbi__clamp((x3+t0) >> 17); + o[4] = stbi__clamp((x3-t0) >> 17); + } +} + +#ifdef STBI_SSE2 +// sse2 integer IDCT. not the fastest possible implementation but it +// produces bit-identical results to the generic C version so it's +// fully "transparent". +static void stbi__idct_simd(stbi_uc *out, int out_stride, short data[64]) +{ + // This is constructed to match our regular (generic) integer IDCT exactly. + __m128i row0, row1, row2, row3, row4, row5, row6, row7; + __m128i tmp; + + // dot product constant: even elems=x, odd elems=y + #define dct_const(x,y) _mm_setr_epi16((x),(y),(x),(y),(x),(y),(x),(y)) + + // out(0) = c0[even]*x + c0[odd]*y (c0, x, y 16-bit, out 32-bit) + // out(1) = c1[even]*x + c1[odd]*y + #define dct_rot(out0,out1, x,y,c0,c1) \ + __m128i c0##lo = _mm_unpacklo_epi16((x),(y)); \ + __m128i c0##hi = _mm_unpackhi_epi16((x),(y)); \ + __m128i out0##_l = _mm_madd_epi16(c0##lo, c0); \ + __m128i out0##_h = _mm_madd_epi16(c0##hi, c0); \ + __m128i out1##_l = _mm_madd_epi16(c0##lo, c1); \ + __m128i out1##_h = _mm_madd_epi16(c0##hi, c1) + + // out = in << 12 (in 16-bit, out 32-bit) + #define dct_widen(out, in) \ + __m128i out##_l = _mm_srai_epi32(_mm_unpacklo_epi16(_mm_setzero_si128(), (in)), 4); \ + __m128i out##_h = _mm_srai_epi32(_mm_unpackhi_epi16(_mm_setzero_si128(), (in)), 4) + + // wide add + #define dct_wadd(out, a, b) \ + __m128i out##_l = _mm_add_epi32(a##_l, b##_l); \ + __m128i out##_h = _mm_add_epi32(a##_h, b##_h) + + // wide sub + #define dct_wsub(out, a, b) \ + __m128i out##_l = _mm_sub_epi32(a##_l, b##_l); \ + __m128i out##_h = _mm_sub_epi32(a##_h, b##_h) + + // butterfly a/b, add bias, then shift by "s" and pack + #define dct_bfly32o(out0, out1, a,b,bias,s) \ + { \ + __m128i abiased_l = _mm_add_epi32(a##_l, bias); \ + __m128i abiased_h = _mm_add_epi32(a##_h, bias); \ + dct_wadd(sum, abiased, b); \ + dct_wsub(dif, abiased, b); \ + out0 = _mm_packs_epi32(_mm_srai_epi32(sum_l, s), _mm_srai_epi32(sum_h, s)); \ + out1 = _mm_packs_epi32(_mm_srai_epi32(dif_l, s), _mm_srai_epi32(dif_h, s)); \ + } + + // 8-bit interleave step (for transposes) + #define dct_interleave8(a, b) \ + tmp = a; \ + a = _mm_unpacklo_epi8(a, b); \ + b = _mm_unpackhi_epi8(tmp, b) + + // 16-bit interleave step (for transposes) + #define dct_interleave16(a, b) \ + tmp = a; \ + a = _mm_unpacklo_epi16(a, b); \ + b = _mm_unpackhi_epi16(tmp, b) + + #define dct_pass(bias,shift) \ + { \ + /* even part */ \ + dct_rot(t2e,t3e, row2,row6, rot0_0,rot0_1); \ + __m128i sum04 = _mm_add_epi16(row0, row4); \ + __m128i dif04 = _mm_sub_epi16(row0, row4); \ + dct_widen(t0e, sum04); \ + dct_widen(t1e, dif04); \ + dct_wadd(x0, t0e, t3e); \ + dct_wsub(x3, t0e, t3e); \ + dct_wadd(x1, t1e, t2e); \ + dct_wsub(x2, t1e, t2e); \ + /* odd part */ \ + dct_rot(y0o,y2o, row7,row3, rot2_0,rot2_1); \ + dct_rot(y1o,y3o, row5,row1, rot3_0,rot3_1); \ + __m128i sum17 = _mm_add_epi16(row1, row7); \ + __m128i sum35 = _mm_add_epi16(row3, row5); \ + dct_rot(y4o,y5o, sum17,sum35, rot1_0,rot1_1); \ + dct_wadd(x4, y0o, y4o); \ + dct_wadd(x5, y1o, y5o); \ + dct_wadd(x6, y2o, y5o); \ + dct_wadd(x7, y3o, y4o); \ + dct_bfly32o(row0,row7, x0,x7,bias,shift); \ + dct_bfly32o(row1,row6, x1,x6,bias,shift); \ + dct_bfly32o(row2,row5, x2,x5,bias,shift); \ + dct_bfly32o(row3,row4, x3,x4,bias,shift); \ + } + + __m128i rot0_0 = dct_const(stbi__f2f(0.5411961f), stbi__f2f(0.5411961f) + stbi__f2f(-1.847759065f)); + __m128i rot0_1 = dct_const(stbi__f2f(0.5411961f) + stbi__f2f( 0.765366865f), stbi__f2f(0.5411961f)); + __m128i rot1_0 = dct_const(stbi__f2f(1.175875602f) + stbi__f2f(-0.899976223f), stbi__f2f(1.175875602f)); + __m128i rot1_1 = dct_const(stbi__f2f(1.175875602f), stbi__f2f(1.175875602f) + stbi__f2f(-2.562915447f)); + __m128i rot2_0 = dct_const(stbi__f2f(-1.961570560f) + stbi__f2f( 0.298631336f), stbi__f2f(-1.961570560f)); + __m128i rot2_1 = dct_const(stbi__f2f(-1.961570560f), stbi__f2f(-1.961570560f) + stbi__f2f( 3.072711026f)); + __m128i rot3_0 = dct_const(stbi__f2f(-0.390180644f) + stbi__f2f( 2.053119869f), stbi__f2f(-0.390180644f)); + __m128i rot3_1 = dct_const(stbi__f2f(-0.390180644f), stbi__f2f(-0.390180644f) + stbi__f2f( 1.501321110f)); + + // rounding biases in column/row passes, see stbi__idct_block for explanation. + __m128i bias_0 = _mm_set1_epi32(512); + __m128i bias_1 = _mm_set1_epi32(65536 + (128<<17)); + + // load + row0 = _mm_load_si128((const __m128i *) (data + 0*8)); + row1 = _mm_load_si128((const __m128i *) (data + 1*8)); + row2 = _mm_load_si128((const __m128i *) (data + 2*8)); + row3 = _mm_load_si128((const __m128i *) (data + 3*8)); + row4 = _mm_load_si128((const __m128i *) (data + 4*8)); + row5 = _mm_load_si128((const __m128i *) (data + 5*8)); + row6 = _mm_load_si128((const __m128i *) (data + 6*8)); + row7 = _mm_load_si128((const __m128i *) (data + 7*8)); + + // column pass + dct_pass(bias_0, 10); + + { + // 16bit 8x8 transpose pass 1 + dct_interleave16(row0, row4); + dct_interleave16(row1, row5); + dct_interleave16(row2, row6); + dct_interleave16(row3, row7); + + // transpose pass 2 + dct_interleave16(row0, row2); + dct_interleave16(row1, row3); + dct_interleave16(row4, row6); + dct_interleave16(row5, row7); + + // transpose pass 3 + dct_interleave16(row0, row1); + dct_interleave16(row2, row3); + dct_interleave16(row4, row5); + dct_interleave16(row6, row7); + } + + // row pass + dct_pass(bias_1, 17); + + { + // pack + __m128i p0 = _mm_packus_epi16(row0, row1); // a0a1a2a3...a7b0b1b2b3...b7 + __m128i p1 = _mm_packus_epi16(row2, row3); + __m128i p2 = _mm_packus_epi16(row4, row5); + __m128i p3 = _mm_packus_epi16(row6, row7); + + // 8bit 8x8 transpose pass 1 + dct_interleave8(p0, p2); // a0e0a1e1... + dct_interleave8(p1, p3); // c0g0c1g1... + + // transpose pass 2 + dct_interleave8(p0, p1); // a0c0e0g0... + dct_interleave8(p2, p3); // b0d0f0h0... + + // transpose pass 3 + dct_interleave8(p0, p2); // a0b0c0d0... + dct_interleave8(p1, p3); // a4b4c4d4... + + // store + _mm_storel_epi64((__m128i *) out, p0); out += out_stride; + _mm_storel_epi64((__m128i *) out, _mm_shuffle_epi32(p0, 0x4e)); out += out_stride; + _mm_storel_epi64((__m128i *) out, p2); out += out_stride; + _mm_storel_epi64((__m128i *) out, _mm_shuffle_epi32(p2, 0x4e)); out += out_stride; + _mm_storel_epi64((__m128i *) out, p1); out += out_stride; + _mm_storel_epi64((__m128i *) out, _mm_shuffle_epi32(p1, 0x4e)); out += out_stride; + _mm_storel_epi64((__m128i *) out, p3); out += out_stride; + _mm_storel_epi64((__m128i *) out, _mm_shuffle_epi32(p3, 0x4e)); + } + +#undef dct_const +#undef dct_rot +#undef dct_widen +#undef dct_wadd +#undef dct_wsub +#undef dct_bfly32o +#undef dct_interleave8 +#undef dct_interleave16 +#undef dct_pass +} + +#endif // STBI_SSE2 + +#ifdef STBI_NEON + +// NEON integer IDCT. should produce bit-identical +// results to the generic C version. +static void stbi__idct_simd(stbi_uc *out, int out_stride, short data[64]) +{ + int16x8_t row0, row1, row2, row3, row4, row5, row6, row7; + + int16x4_t rot0_0 = vdup_n_s16(stbi__f2f(0.5411961f)); + int16x4_t rot0_1 = vdup_n_s16(stbi__f2f(-1.847759065f)); + int16x4_t rot0_2 = vdup_n_s16(stbi__f2f( 0.765366865f)); + int16x4_t rot1_0 = vdup_n_s16(stbi__f2f( 1.175875602f)); + int16x4_t rot1_1 = vdup_n_s16(stbi__f2f(-0.899976223f)); + int16x4_t rot1_2 = vdup_n_s16(stbi__f2f(-2.562915447f)); + int16x4_t rot2_0 = vdup_n_s16(stbi__f2f(-1.961570560f)); + int16x4_t rot2_1 = vdup_n_s16(stbi__f2f(-0.390180644f)); + int16x4_t rot3_0 = vdup_n_s16(stbi__f2f( 0.298631336f)); + int16x4_t rot3_1 = vdup_n_s16(stbi__f2f( 2.053119869f)); + int16x4_t rot3_2 = vdup_n_s16(stbi__f2f( 3.072711026f)); + int16x4_t rot3_3 = vdup_n_s16(stbi__f2f( 1.501321110f)); + +#define dct_long_mul(out, inq, coeff) \ + int32x4_t out##_l = vmull_s16(vget_low_s16(inq), coeff); \ + int32x4_t out##_h = vmull_s16(vget_high_s16(inq), coeff) + +#define dct_long_mac(out, acc, inq, coeff) \ + int32x4_t out##_l = vmlal_s16(acc##_l, vget_low_s16(inq), coeff); \ + int32x4_t out##_h = vmlal_s16(acc##_h, vget_high_s16(inq), coeff) + +#define dct_widen(out, inq) \ + int32x4_t out##_l = vshll_n_s16(vget_low_s16(inq), 12); \ + int32x4_t out##_h = vshll_n_s16(vget_high_s16(inq), 12) + +// wide add +#define dct_wadd(out, a, b) \ + int32x4_t out##_l = vaddq_s32(a##_l, b##_l); \ + int32x4_t out##_h = vaddq_s32(a##_h, b##_h) + +// wide sub +#define dct_wsub(out, a, b) \ + int32x4_t out##_l = vsubq_s32(a##_l, b##_l); \ + int32x4_t out##_h = vsubq_s32(a##_h, b##_h) + +// butterfly a/b, then shift using "shiftop" by "s" and pack +#define dct_bfly32o(out0,out1, a,b,shiftop,s) \ + { \ + dct_wadd(sum, a, b); \ + dct_wsub(dif, a, b); \ + out0 = vcombine_s16(shiftop(sum_l, s), shiftop(sum_h, s)); \ + out1 = vcombine_s16(shiftop(dif_l, s), shiftop(dif_h, s)); \ + } + +#define dct_pass(shiftop, shift) \ + { \ + /* even part */ \ + int16x8_t sum26 = vaddq_s16(row2, row6); \ + dct_long_mul(p1e, sum26, rot0_0); \ + dct_long_mac(t2e, p1e, row6, rot0_1); \ + dct_long_mac(t3e, p1e, row2, rot0_2); \ + int16x8_t sum04 = vaddq_s16(row0, row4); \ + int16x8_t dif04 = vsubq_s16(row0, row4); \ + dct_widen(t0e, sum04); \ + dct_widen(t1e, dif04); \ + dct_wadd(x0, t0e, t3e); \ + dct_wsub(x3, t0e, t3e); \ + dct_wadd(x1, t1e, t2e); \ + dct_wsub(x2, t1e, t2e); \ + /* odd part */ \ + int16x8_t sum15 = vaddq_s16(row1, row5); \ + int16x8_t sum17 = vaddq_s16(row1, row7); \ + int16x8_t sum35 = vaddq_s16(row3, row5); \ + int16x8_t sum37 = vaddq_s16(row3, row7); \ + int16x8_t sumodd = vaddq_s16(sum17, sum35); \ + dct_long_mul(p5o, sumodd, rot1_0); \ + dct_long_mac(p1o, p5o, sum17, rot1_1); \ + dct_long_mac(p2o, p5o, sum35, rot1_2); \ + dct_long_mul(p3o, sum37, rot2_0); \ + dct_long_mul(p4o, sum15, rot2_1); \ + dct_wadd(sump13o, p1o, p3o); \ + dct_wadd(sump24o, p2o, p4o); \ + dct_wadd(sump23o, p2o, p3o); \ + dct_wadd(sump14o, p1o, p4o); \ + dct_long_mac(x4, sump13o, row7, rot3_0); \ + dct_long_mac(x5, sump24o, row5, rot3_1); \ + dct_long_mac(x6, sump23o, row3, rot3_2); \ + dct_long_mac(x7, sump14o, row1, rot3_3); \ + dct_bfly32o(row0,row7, x0,x7,shiftop,shift); \ + dct_bfly32o(row1,row6, x1,x6,shiftop,shift); \ + dct_bfly32o(row2,row5, x2,x5,shiftop,shift); \ + dct_bfly32o(row3,row4, x3,x4,shiftop,shift); \ + } + + // load + row0 = vld1q_s16(data + 0*8); + row1 = vld1q_s16(data + 1*8); + row2 = vld1q_s16(data + 2*8); + row3 = vld1q_s16(data + 3*8); + row4 = vld1q_s16(data + 4*8); + row5 = vld1q_s16(data + 5*8); + row6 = vld1q_s16(data + 6*8); + row7 = vld1q_s16(data + 7*8); + + // add DC bias + row0 = vaddq_s16(row0, vsetq_lane_s16(1024, vdupq_n_s16(0), 0)); + + // column pass + dct_pass(vrshrn_n_s32, 10); + + // 16bit 8x8 transpose + { +// these three map to a single VTRN.16, VTRN.32, and VSWP, respectively. +// whether compilers actually get this is another story, sadly. +#define dct_trn16(x, y) { int16x8x2_t t = vtrnq_s16(x, y); x = t.val[0]; y = t.val[1]; } +#define dct_trn32(x, y) { int32x4x2_t t = vtrnq_s32(vreinterpretq_s32_s16(x), vreinterpretq_s32_s16(y)); x = vreinterpretq_s16_s32(t.val[0]); y = vreinterpretq_s16_s32(t.val[1]); } +#define dct_trn64(x, y) { int16x8_t x0 = x; int16x8_t y0 = y; x = vcombine_s16(vget_low_s16(x0), vget_low_s16(y0)); y = vcombine_s16(vget_high_s16(x0), vget_high_s16(y0)); } + + // pass 1 + dct_trn16(row0, row1); // a0b0a2b2a4b4a6b6 + dct_trn16(row2, row3); + dct_trn16(row4, row5); + dct_trn16(row6, row7); + + // pass 2 + dct_trn32(row0, row2); // a0b0c0d0a4b4c4d4 + dct_trn32(row1, row3); + dct_trn32(row4, row6); + dct_trn32(row5, row7); + + // pass 3 + dct_trn64(row0, row4); // a0b0c0d0e0f0g0h0 + dct_trn64(row1, row5); + dct_trn64(row2, row6); + dct_trn64(row3, row7); + +#undef dct_trn16 +#undef dct_trn32 +#undef dct_trn64 + } + + // row pass + // vrshrn_n_s32 only supports shifts up to 16, we need + // 17. so do a non-rounding shift of 16 first then follow + // up with a rounding shift by 1. + dct_pass(vshrn_n_s32, 16); + + { + // pack and round + uint8x8_t p0 = vqrshrun_n_s16(row0, 1); + uint8x8_t p1 = vqrshrun_n_s16(row1, 1); + uint8x8_t p2 = vqrshrun_n_s16(row2, 1); + uint8x8_t p3 = vqrshrun_n_s16(row3, 1); + uint8x8_t p4 = vqrshrun_n_s16(row4, 1); + uint8x8_t p5 = vqrshrun_n_s16(row5, 1); + uint8x8_t p6 = vqrshrun_n_s16(row6, 1); + uint8x8_t p7 = vqrshrun_n_s16(row7, 1); + + // again, these can translate into one instruction, but often don't. +#define dct_trn8_8(x, y) { uint8x8x2_t t = vtrn_u8(x, y); x = t.val[0]; y = t.val[1]; } +#define dct_trn8_16(x, y) { uint16x4x2_t t = vtrn_u16(vreinterpret_u16_u8(x), vreinterpret_u16_u8(y)); x = vreinterpret_u8_u16(t.val[0]); y = vreinterpret_u8_u16(t.val[1]); } +#define dct_trn8_32(x, y) { uint32x2x2_t t = vtrn_u32(vreinterpret_u32_u8(x), vreinterpret_u32_u8(y)); x = vreinterpret_u8_u32(t.val[0]); y = vreinterpret_u8_u32(t.val[1]); } + + // sadly can't use interleaved stores here since we only write + // 8 bytes to each scan line! + + // 8x8 8-bit transpose pass 1 + dct_trn8_8(p0, p1); + dct_trn8_8(p2, p3); + dct_trn8_8(p4, p5); + dct_trn8_8(p6, p7); + + // pass 2 + dct_trn8_16(p0, p2); + dct_trn8_16(p1, p3); + dct_trn8_16(p4, p6); + dct_trn8_16(p5, p7); + + // pass 3 + dct_trn8_32(p0, p4); + dct_trn8_32(p1, p5); + dct_trn8_32(p2, p6); + dct_trn8_32(p3, p7); + + // store + vst1_u8(out, p0); out += out_stride; + vst1_u8(out, p1); out += out_stride; + vst1_u8(out, p2); out += out_stride; + vst1_u8(out, p3); out += out_stride; + vst1_u8(out, p4); out += out_stride; + vst1_u8(out, p5); out += out_stride; + vst1_u8(out, p6); out += out_stride; + vst1_u8(out, p7); + +#undef dct_trn8_8 +#undef dct_trn8_16 +#undef dct_trn8_32 + } + +#undef dct_long_mul +#undef dct_long_mac +#undef dct_widen +#undef dct_wadd +#undef dct_wsub +#undef dct_bfly32o +#undef dct_pass +} + +#endif // STBI_NEON + +#define STBI__MARKER_none 0xff +// if there's a pending marker from the entropy stream, return that +// otherwise, fetch from the stream and get a marker. if there's no +// marker, return 0xff, which is never a valid marker value +static stbi_uc stbi__get_marker(stbi__jpeg *j) +{ + stbi_uc x; + if (j->marker != STBI__MARKER_none) { x = j->marker; j->marker = STBI__MARKER_none; return x; } + x = stbi__get8(j->s); + if (x != 0xff) return STBI__MARKER_none; + while (x == 0xff) + x = stbi__get8(j->s); // consume repeated 0xff fill bytes + return x; +} + +// in each scan, we'll have scan_n components, and the order +// of the components is specified by order[] +#define STBI__RESTART(x) ((x) >= 0xd0 && (x) <= 0xd7) + +// after a restart interval, stbi__jpeg_reset the entropy decoder and +// the dc prediction +static void stbi__jpeg_reset(stbi__jpeg *j) +{ + j->code_bits = 0; + j->code_buffer = 0; + j->nomore = 0; + j->img_comp[0].dc_pred = j->img_comp[1].dc_pred = j->img_comp[2].dc_pred = j->img_comp[3].dc_pred = 0; + j->marker = STBI__MARKER_none; + j->todo = j->restart_interval ? j->restart_interval : 0x7fffffff; + j->eob_run = 0; + // no more than 1<<31 MCUs if no restart_interal? that's plenty safe, + // since we don't even allow 1<<30 pixels +} + +static int stbi__parse_entropy_coded_data(stbi__jpeg *z) +{ + stbi__jpeg_reset(z); + if (!z->progressive) { + if (z->scan_n == 1) { + int i,j; + STBI_SIMD_ALIGN(short, data[64]); + int n = z->order[0]; + // non-interleaved data, we just need to process one block at a time, + // in trivial scanline order + // number of blocks to do just depends on how many actual "pixels" this + // component has, independent of interleaved MCU blocking and such + int w = (z->img_comp[n].x+7) >> 3; + int h = (z->img_comp[n].y+7) >> 3; + for (j=0; j < h; ++j) { + for (i=0; i < w; ++i) { + int ha = z->img_comp[n].ha; + if (!stbi__jpeg_decode_block(z, data, z->huff_dc+z->img_comp[n].hd, z->huff_ac+ha, z->fast_ac[ha], n, z->dequant[z->img_comp[n].tq])) return 0; + z->idct_block_kernel(z->img_comp[n].data+z->img_comp[n].w2*j*8+i*8, z->img_comp[n].w2, data); + // every data block is an MCU, so countdown the restart interval + if (--z->todo <= 0) { + if (z->code_bits < 24) stbi__grow_buffer_unsafe(z); + // if it's NOT a restart, then just bail, so we get corrupt data + // rather than no data + if (!STBI__RESTART(z->marker)) return 1; + stbi__jpeg_reset(z); + } + } + } + return 1; + } else { // interleaved + int i,j,k,x,y; + STBI_SIMD_ALIGN(short, data[64]); + for (j=0; j < z->img_mcu_y; ++j) { + for (i=0; i < z->img_mcu_x; ++i) { + // scan an interleaved mcu... process scan_n components in order + for (k=0; k < z->scan_n; ++k) { + int n = z->order[k]; + // scan out an mcu's worth of this component; that's just determined + // by the basic H and V specified for the component + for (y=0; y < z->img_comp[n].v; ++y) { + for (x=0; x < z->img_comp[n].h; ++x) { + int x2 = (i*z->img_comp[n].h + x)*8; + int y2 = (j*z->img_comp[n].v + y)*8; + int ha = z->img_comp[n].ha; + if (!stbi__jpeg_decode_block(z, data, z->huff_dc+z->img_comp[n].hd, z->huff_ac+ha, z->fast_ac[ha], n, z->dequant[z->img_comp[n].tq])) return 0; + z->idct_block_kernel(z->img_comp[n].data+z->img_comp[n].w2*y2+x2, z->img_comp[n].w2, data); + } + } + } + // after all interleaved components, that's an interleaved MCU, + // so now count down the restart interval + if (--z->todo <= 0) { + if (z->code_bits < 24) stbi__grow_buffer_unsafe(z); + if (!STBI__RESTART(z->marker)) return 1; + stbi__jpeg_reset(z); + } + } + } + return 1; + } + } else { + if (z->scan_n == 1) { + int i,j; + int n = z->order[0]; + // non-interleaved data, we just need to process one block at a time, + // in trivial scanline order + // number of blocks to do just depends on how many actual "pixels" this + // component has, independent of interleaved MCU blocking and such + int w = (z->img_comp[n].x+7) >> 3; + int h = (z->img_comp[n].y+7) >> 3; + for (j=0; j < h; ++j) { + for (i=0; i < w; ++i) { + short *data = z->img_comp[n].coeff + 64 * (i + j * z->img_comp[n].coeff_w); + if (z->spec_start == 0) { + if (!stbi__jpeg_decode_block_prog_dc(z, data, &z->huff_dc[z->img_comp[n].hd], n)) + return 0; + } else { + int ha = z->img_comp[n].ha; + if (!stbi__jpeg_decode_block_prog_ac(z, data, &z->huff_ac[ha], z->fast_ac[ha])) + return 0; + } + // every data block is an MCU, so countdown the restart interval + if (--z->todo <= 0) { + if (z->code_bits < 24) stbi__grow_buffer_unsafe(z); + if (!STBI__RESTART(z->marker)) return 1; + stbi__jpeg_reset(z); + } + } + } + return 1; + } else { // interleaved + int i,j,k,x,y; + for (j=0; j < z->img_mcu_y; ++j) { + for (i=0; i < z->img_mcu_x; ++i) { + // scan an interleaved mcu... process scan_n components in order + for (k=0; k < z->scan_n; ++k) { + int n = z->order[k]; + // scan out an mcu's worth of this component; that's just determined + // by the basic H and V specified for the component + for (y=0; y < z->img_comp[n].v; ++y) { + for (x=0; x < z->img_comp[n].h; ++x) { + int x2 = (i*z->img_comp[n].h + x); + int y2 = (j*z->img_comp[n].v + y); + short *data = z->img_comp[n].coeff + 64 * (x2 + y2 * z->img_comp[n].coeff_w); + if (!stbi__jpeg_decode_block_prog_dc(z, data, &z->huff_dc[z->img_comp[n].hd], n)) + return 0; + } + } + } + // after all interleaved components, that's an interleaved MCU, + // so now count down the restart interval + if (--z->todo <= 0) { + if (z->code_bits < 24) stbi__grow_buffer_unsafe(z); + if (!STBI__RESTART(z->marker)) return 1; + stbi__jpeg_reset(z); + } + } + } + return 1; + } + } +} + +static void stbi__jpeg_dequantize(short *data, stbi__uint16 *dequant) +{ + int i; + for (i=0; i < 64; ++i) + data[i] *= dequant[i]; +} + +static void stbi__jpeg_finish(stbi__jpeg *z) +{ + if (z->progressive) { + // dequantize and idct the data + int i,j,n; + for (n=0; n < z->s->img_n; ++n) { + int w = (z->img_comp[n].x+7) >> 3; + int h = (z->img_comp[n].y+7) >> 3; + for (j=0; j < h; ++j) { + for (i=0; i < w; ++i) { + short *data = z->img_comp[n].coeff + 64 * (i + j * z->img_comp[n].coeff_w); + stbi__jpeg_dequantize(data, z->dequant[z->img_comp[n].tq]); + z->idct_block_kernel(z->img_comp[n].data+z->img_comp[n].w2*j*8+i*8, z->img_comp[n].w2, data); + } + } + } + } +} + +static int stbi__process_marker(stbi__jpeg *z, int m) +{ + int L; + switch (m) { + case STBI__MARKER_none: // no marker found + return stbi__err("expected marker","Corrupt JPEG"); + + case 0xDD: // DRI - specify restart interval + if (stbi__get16be(z->s) != 4) return stbi__err("bad DRI len","Corrupt JPEG"); + z->restart_interval = stbi__get16be(z->s); + return 1; + + case 0xDB: // DQT - define quantization table + L = stbi__get16be(z->s)-2; + while (L > 0) { + int q = stbi__get8(z->s); + int p = q >> 4, sixteen = (p != 0); + int t = q & 15,i; + if (p != 0 && p != 1) return stbi__err("bad DQT type","Corrupt JPEG"); + if (t > 3) return stbi__err("bad DQT table","Corrupt JPEG"); + + for (i=0; i < 64; ++i) + z->dequant[t][stbi__jpeg_dezigzag[i]] = (stbi__uint16)(sixteen ? stbi__get16be(z->s) : stbi__get8(z->s)); + L -= (sixteen ? 129 : 65); + } + return L==0; + + case 0xC4: // DHT - define huffman table + L = stbi__get16be(z->s)-2; + while (L > 0) { + stbi_uc *v; + int sizes[16],i,n=0; + int q = stbi__get8(z->s); + int tc = q >> 4; + int th = q & 15; + if (tc > 1 || th > 3) return stbi__err("bad DHT header","Corrupt JPEG"); + for (i=0; i < 16; ++i) { + sizes[i] = stbi__get8(z->s); + n += sizes[i]; + } + L -= 17; + if (tc == 0) { + if (!stbi__build_huffman(z->huff_dc+th, sizes)) return 0; + v = z->huff_dc[th].values; + } else { + if (!stbi__build_huffman(z->huff_ac+th, sizes)) return 0; + v = z->huff_ac[th].values; + } + for (i=0; i < n; ++i) + v[i] = stbi__get8(z->s); + if (tc != 0) + stbi__build_fast_ac(z->fast_ac[th], z->huff_ac + th); + L -= n; + } + return L==0; + } + + // check for comment block or APP blocks + if ((m >= 0xE0 && m <= 0xEF) || m == 0xFE) { + L = stbi__get16be(z->s); + if (L < 2) { + if (m == 0xFE) + return stbi__err("bad COM len","Corrupt JPEG"); + else + return stbi__err("bad APP len","Corrupt JPEG"); + } + L -= 2; + + if (m == 0xE0 && L >= 5) { // JFIF APP0 segment + static const unsigned char tag[5] = {'J','F','I','F','\0'}; + int ok = 1; + int i; + for (i=0; i < 5; ++i) + if (stbi__get8(z->s) != tag[i]) + ok = 0; + L -= 5; + if (ok) + z->jfif = 1; + } else if (m == 0xEE && L >= 12) { // Adobe APP14 segment + static const unsigned char tag[6] = {'A','d','o','b','e','\0'}; + int ok = 1; + int i; + for (i=0; i < 6; ++i) + if (stbi__get8(z->s) != tag[i]) + ok = 0; + L -= 6; + if (ok) { + stbi__get8(z->s); // version + stbi__get16be(z->s); // flags0 + stbi__get16be(z->s); // flags1 + z->app14_color_transform = stbi__get8(z->s); // color transform + L -= 6; + } + } + + stbi__skip(z->s, L); + return 1; + } + + return stbi__err("unknown marker","Corrupt JPEG"); +} + +// after we see SOS +static int stbi__process_scan_header(stbi__jpeg *z) +{ + int i; + int Ls = stbi__get16be(z->s); + z->scan_n = stbi__get8(z->s); + if (z->scan_n < 1 || z->scan_n > 4 || z->scan_n > (int) z->s->img_n) return stbi__err("bad SOS component count","Corrupt JPEG"); + if (Ls != 6+2*z->scan_n) return stbi__err("bad SOS len","Corrupt JPEG"); + for (i=0; i < z->scan_n; ++i) { + int id = stbi__get8(z->s), which; + int q = stbi__get8(z->s); + for (which = 0; which < z->s->img_n; ++which) + if (z->img_comp[which].id == id) + break; + if (which == z->s->img_n) return 0; // no match + z->img_comp[which].hd = q >> 4; if (z->img_comp[which].hd > 3) return stbi__err("bad DC huff","Corrupt JPEG"); + z->img_comp[which].ha = q & 15; if (z->img_comp[which].ha > 3) return stbi__err("bad AC huff","Corrupt JPEG"); + z->order[i] = which; + } + + { + int aa; + z->spec_start = stbi__get8(z->s); + z->spec_end = stbi__get8(z->s); // should be 63, but might be 0 + aa = stbi__get8(z->s); + z->succ_high = (aa >> 4); + z->succ_low = (aa & 15); + if (z->progressive) { + if (z->spec_start > 63 || z->spec_end > 63 || z->spec_start > z->spec_end || z->succ_high > 13 || z->succ_low > 13) + return stbi__err("bad SOS", "Corrupt JPEG"); + } else { + if (z->spec_start != 0) return stbi__err("bad SOS","Corrupt JPEG"); + if (z->succ_high != 0 || z->succ_low != 0) return stbi__err("bad SOS","Corrupt JPEG"); + z->spec_end = 63; + } + } + + return 1; +} + +static int stbi__free_jpeg_components(stbi__jpeg *z, int ncomp, int why) +{ + int i; + for (i=0; i < ncomp; ++i) { + if (z->img_comp[i].raw_data) { + STBI_FREE(z->img_comp[i].raw_data); + z->img_comp[i].raw_data = NULL; + z->img_comp[i].data = NULL; + } + if (z->img_comp[i].raw_coeff) { + STBI_FREE(z->img_comp[i].raw_coeff); + z->img_comp[i].raw_coeff = 0; + z->img_comp[i].coeff = 0; + } + if (z->img_comp[i].linebuf) { + STBI_FREE(z->img_comp[i].linebuf); + z->img_comp[i].linebuf = NULL; + } + } + return why; +} + +static int stbi__process_frame_header(stbi__jpeg *z, int scan) +{ + stbi__context *s = z->s; + int Lf,p,i,q, h_max=1,v_max=1,c; + Lf = stbi__get16be(s); if (Lf < 11) return stbi__err("bad SOF len","Corrupt JPEG"); // JPEG + p = stbi__get8(s); if (p != 8) return stbi__err("only 8-bit","JPEG format not supported: 8-bit only"); // JPEG baseline + s->img_y = stbi__get16be(s); if (s->img_y == 0) return stbi__err("no header height", "JPEG format not supported: delayed height"); // Legal, but we don't handle it--but neither does IJG + s->img_x = stbi__get16be(s); if (s->img_x == 0) return stbi__err("0 width","Corrupt JPEG"); // JPEG requires + if (s->img_y > STBI_MAX_DIMENSIONS) return stbi__err("too large","Very large image (corrupt?)"); + if (s->img_x > STBI_MAX_DIMENSIONS) return stbi__err("too large","Very large image (corrupt?)"); + c = stbi__get8(s); + if (c != 3 && c != 1 && c != 4) return stbi__err("bad component count","Corrupt JPEG"); + s->img_n = c; + for (i=0; i < c; ++i) { + z->img_comp[i].data = NULL; + z->img_comp[i].linebuf = NULL; + } + + if (Lf != 8+3*s->img_n) return stbi__err("bad SOF len","Corrupt JPEG"); + + z->rgb = 0; + for (i=0; i < s->img_n; ++i) { + static const unsigned char rgb[3] = { 'R', 'G', 'B' }; + z->img_comp[i].id = stbi__get8(s); + if (s->img_n == 3 && z->img_comp[i].id == rgb[i]) + ++z->rgb; + q = stbi__get8(s); + z->img_comp[i].h = (q >> 4); if (!z->img_comp[i].h || z->img_comp[i].h > 4) return stbi__err("bad H","Corrupt JPEG"); + z->img_comp[i].v = q & 15; if (!z->img_comp[i].v || z->img_comp[i].v > 4) return stbi__err("bad V","Corrupt JPEG"); + z->img_comp[i].tq = stbi__get8(s); if (z->img_comp[i].tq > 3) return stbi__err("bad TQ","Corrupt JPEG"); + } + + if (scan != STBI__SCAN_load) return 1; + + if (!stbi__mad3sizes_valid(s->img_x, s->img_y, s->img_n, 0)) return stbi__err("too large", "Image too large to decode"); + + for (i=0; i < s->img_n; ++i) { + if (z->img_comp[i].h > h_max) h_max = z->img_comp[i].h; + if (z->img_comp[i].v > v_max) v_max = z->img_comp[i].v; + } + + // check that plane subsampling factors are integer ratios; our resamplers can't deal with fractional ratios + // and I've never seen a non-corrupted JPEG file actually use them + for (i=0; i < s->img_n; ++i) { + if (h_max % z->img_comp[i].h != 0) return stbi__err("bad H","Corrupt JPEG"); + if (v_max % z->img_comp[i].v != 0) return stbi__err("bad V","Corrupt JPEG"); + } + + // compute interleaved mcu info + z->img_h_max = h_max; + z->img_v_max = v_max; + z->img_mcu_w = h_max * 8; + z->img_mcu_h = v_max * 8; + // these sizes can't be more than 17 bits + z->img_mcu_x = (s->img_x + z->img_mcu_w-1) / z->img_mcu_w; + z->img_mcu_y = (s->img_y + z->img_mcu_h-1) / z->img_mcu_h; + + for (i=0; i < s->img_n; ++i) { + // number of effective pixels (e.g. for non-interleaved MCU) + z->img_comp[i].x = (s->img_x * z->img_comp[i].h + h_max-1) / h_max; + z->img_comp[i].y = (s->img_y * z->img_comp[i].v + v_max-1) / v_max; + // to simplify generation, we'll allocate enough memory to decode + // the bogus oversized data from using interleaved MCUs and their + // big blocks (e.g. a 16x16 iMCU on an image of width 33); we won't + // discard the extra data until colorspace conversion + // + // img_mcu_x, img_mcu_y: <=17 bits; comp[i].h and .v are <=4 (checked earlier) + // so these muls can't overflow with 32-bit ints (which we require) + z->img_comp[i].w2 = z->img_mcu_x * z->img_comp[i].h * 8; + z->img_comp[i].h2 = z->img_mcu_y * z->img_comp[i].v * 8; + z->img_comp[i].coeff = 0; + z->img_comp[i].raw_coeff = 0; + z->img_comp[i].linebuf = NULL; + z->img_comp[i].raw_data = stbi__malloc_mad2(z->img_comp[i].w2, z->img_comp[i].h2, 15); + if (z->img_comp[i].raw_data == NULL) + return stbi__free_jpeg_components(z, i+1, stbi__err("outofmem", "Out of memory")); + // align blocks for idct using mmx/sse + z->img_comp[i].data = (stbi_uc*) (((size_t) z->img_comp[i].raw_data + 15) & ~15); + if (z->progressive) { + // w2, h2 are multiples of 8 (see above) + z->img_comp[i].coeff_w = z->img_comp[i].w2 / 8; + z->img_comp[i].coeff_h = z->img_comp[i].h2 / 8; + z->img_comp[i].raw_coeff = stbi__malloc_mad3(z->img_comp[i].w2, z->img_comp[i].h2, sizeof(short), 15); + if (z->img_comp[i].raw_coeff == NULL) + return stbi__free_jpeg_components(z, i+1, stbi__err("outofmem", "Out of memory")); + z->img_comp[i].coeff = (short*) (((size_t) z->img_comp[i].raw_coeff + 15) & ~15); + } + } + + return 1; +} + +// use comparisons since in some cases we handle more than one case (e.g. SOF) +#define stbi__DNL(x) ((x) == 0xdc) +#define stbi__SOI(x) ((x) == 0xd8) +#define stbi__EOI(x) ((x) == 0xd9) +#define stbi__SOF(x) ((x) == 0xc0 || (x) == 0xc1 || (x) == 0xc2) +#define stbi__SOS(x) ((x) == 0xda) + +#define stbi__SOF_progressive(x) ((x) == 0xc2) + +static int stbi__decode_jpeg_header(stbi__jpeg *z, int scan) +{ + int m; + z->jfif = 0; + z->app14_color_transform = -1; // valid values are 0,1,2 + z->marker = STBI__MARKER_none; // initialize cached marker to empty + m = stbi__get_marker(z); + if (!stbi__SOI(m)) return stbi__err("no SOI","Corrupt JPEG"); + if (scan == STBI__SCAN_type) return 1; + m = stbi__get_marker(z); + while (!stbi__SOF(m)) { + if (!stbi__process_marker(z,m)) return 0; + m = stbi__get_marker(z); + while (m == STBI__MARKER_none) { + // some files have extra padding after their blocks, so ok, we'll scan + if (stbi__at_eof(z->s)) return stbi__err("no SOF", "Corrupt JPEG"); + m = stbi__get_marker(z); + } + } + z->progressive = stbi__SOF_progressive(m); + if (!stbi__process_frame_header(z, scan)) return 0; + return 1; +} + +// decode image to YCbCr format +static int stbi__decode_jpeg_image(stbi__jpeg *j) +{ + int m; + for (m = 0; m < 4; m++) { + j->img_comp[m].raw_data = NULL; + j->img_comp[m].raw_coeff = NULL; + } + j->restart_interval = 0; + if (!stbi__decode_jpeg_header(j, STBI__SCAN_load)) return 0; + m = stbi__get_marker(j); + while (!stbi__EOI(m)) { + if (stbi__SOS(m)) { + if (!stbi__process_scan_header(j)) return 0; + if (!stbi__parse_entropy_coded_data(j)) return 0; + if (j->marker == STBI__MARKER_none ) { + // handle 0s at the end of image data from IP Kamera 9060 + while (!stbi__at_eof(j->s)) { + int x = stbi__get8(j->s); + if (x == 255) { + j->marker = stbi__get8(j->s); + break; + } + } + // if we reach eof without hitting a marker, stbi__get_marker() below will fail and we'll eventually return 0 + } + } else if (stbi__DNL(m)) { + int Ld = stbi__get16be(j->s); + stbi__uint32 NL = stbi__get16be(j->s); + if (Ld != 4) return stbi__err("bad DNL len", "Corrupt JPEG"); + if (NL != j->s->img_y) return stbi__err("bad DNL height", "Corrupt JPEG"); + } else { + if (!stbi__process_marker(j, m)) return 0; + } + m = stbi__get_marker(j); + } + if (j->progressive) + stbi__jpeg_finish(j); + return 1; +} + +// static jfif-centered resampling (across block boundaries) + +typedef stbi_uc *(*resample_row_func)(stbi_uc *out, stbi_uc *in0, stbi_uc *in1, + int w, int hs); + +#define stbi__div4(x) ((stbi_uc) ((x) >> 2)) + +static stbi_uc *resample_row_1(stbi_uc *out, stbi_uc *in_near, stbi_uc *in_far, int w, int hs) +{ + STBI_NOTUSED(out); + STBI_NOTUSED(in_far); + STBI_NOTUSED(w); + STBI_NOTUSED(hs); + return in_near; +} + +static stbi_uc* stbi__resample_row_v_2(stbi_uc *out, stbi_uc *in_near, stbi_uc *in_far, int w, int hs) +{ + // need to generate two samples vertically for every one in input + int i; + STBI_NOTUSED(hs); + for (i=0; i < w; ++i) + out[i] = stbi__div4(3*in_near[i] + in_far[i] + 2); + return out; +} + +static stbi_uc* stbi__resample_row_h_2(stbi_uc *out, stbi_uc *in_near, stbi_uc *in_far, int w, int hs) +{ + // need to generate two samples horizontally for every one in input + int i; + stbi_uc *input = in_near; + + if (w == 1) { + // if only one sample, can't do any interpolation + out[0] = out[1] = input[0]; + return out; + } + + out[0] = input[0]; + out[1] = stbi__div4(input[0]*3 + input[1] + 2); + for (i=1; i < w-1; ++i) { + int n = 3*input[i]+2; + out[i*2+0] = stbi__div4(n+input[i-1]); + out[i*2+1] = stbi__div4(n+input[i+1]); + } + out[i*2+0] = stbi__div4(input[w-2]*3 + input[w-1] + 2); + out[i*2+1] = input[w-1]; + + STBI_NOTUSED(in_far); + STBI_NOTUSED(hs); + + return out; +} + +#define stbi__div16(x) ((stbi_uc) ((x) >> 4)) + +static stbi_uc *stbi__resample_row_hv_2(stbi_uc *out, stbi_uc *in_near, stbi_uc *in_far, int w, int hs) +{ + // need to generate 2x2 samples for every one in input + int i,t0,t1; + if (w == 1) { + out[0] = out[1] = stbi__div4(3*in_near[0] + in_far[0] + 2); + return out; + } + + t1 = 3*in_near[0] + in_far[0]; + out[0] = stbi__div4(t1+2); + for (i=1; i < w; ++i) { + t0 = t1; + t1 = 3*in_near[i]+in_far[i]; + out[i*2-1] = stbi__div16(3*t0 + t1 + 8); + out[i*2 ] = stbi__div16(3*t1 + t0 + 8); + } + out[w*2-1] = stbi__div4(t1+2); + + STBI_NOTUSED(hs); + + return out; +} + +#if defined(STBI_SSE2) || defined(STBI_NEON) +static stbi_uc *stbi__resample_row_hv_2_simd(stbi_uc *out, stbi_uc *in_near, stbi_uc *in_far, int w, int hs) +{ + // need to generate 2x2 samples for every one in input + int i=0,t0,t1; + + if (w == 1) { + out[0] = out[1] = stbi__div4(3*in_near[0] + in_far[0] + 2); + return out; + } + + t1 = 3*in_near[0] + in_far[0]; + // process groups of 8 pixels for as long as we can. + // note we can't handle the last pixel in a row in this loop + // because we need to handle the filter boundary conditions. + for (; i < ((w-1) & ~7); i += 8) { +#if defined(STBI_SSE2) + // load and perform the vertical filtering pass + // this uses 3*x + y = 4*x + (y - x) + __m128i zero = _mm_setzero_si128(); + __m128i farb = _mm_loadl_epi64((__m128i *) (in_far + i)); + __m128i nearb = _mm_loadl_epi64((__m128i *) (in_near + i)); + __m128i farw = _mm_unpacklo_epi8(farb, zero); + __m128i nearw = _mm_unpacklo_epi8(nearb, zero); + __m128i diff = _mm_sub_epi16(farw, nearw); + __m128i nears = _mm_slli_epi16(nearw, 2); + __m128i curr = _mm_add_epi16(nears, diff); // current row + + // horizontal filter works the same based on shifted vers of current + // row. "prev" is current row shifted right by 1 pixel; we need to + // insert the previous pixel value (from t1). + // "next" is current row shifted left by 1 pixel, with first pixel + // of next block of 8 pixels added in. + __m128i prv0 = _mm_slli_si128(curr, 2); + __m128i nxt0 = _mm_srli_si128(curr, 2); + __m128i prev = _mm_insert_epi16(prv0, t1, 0); + __m128i next = _mm_insert_epi16(nxt0, 3*in_near[i+8] + in_far[i+8], 7); + + // horizontal filter, polyphase implementation since it's convenient: + // even pixels = 3*cur + prev = cur*4 + (prev - cur) + // odd pixels = 3*cur + next = cur*4 + (next - cur) + // note the shared term. + __m128i bias = _mm_set1_epi16(8); + __m128i curs = _mm_slli_epi16(curr, 2); + __m128i prvd = _mm_sub_epi16(prev, curr); + __m128i nxtd = _mm_sub_epi16(next, curr); + __m128i curb = _mm_add_epi16(curs, bias); + __m128i even = _mm_add_epi16(prvd, curb); + __m128i odd = _mm_add_epi16(nxtd, curb); + + // interleave even and odd pixels, then undo scaling. + __m128i int0 = _mm_unpacklo_epi16(even, odd); + __m128i int1 = _mm_unpackhi_epi16(even, odd); + __m128i de0 = _mm_srli_epi16(int0, 4); + __m128i de1 = _mm_srli_epi16(int1, 4); + + // pack and write output + __m128i outv = _mm_packus_epi16(de0, de1); + _mm_storeu_si128((__m128i *) (out + i*2), outv); +#elif defined(STBI_NEON) + // load and perform the vertical filtering pass + // this uses 3*x + y = 4*x + (y - x) + uint8x8_t farb = vld1_u8(in_far + i); + uint8x8_t nearb = vld1_u8(in_near + i); + int16x8_t diff = vreinterpretq_s16_u16(vsubl_u8(farb, nearb)); + int16x8_t nears = vreinterpretq_s16_u16(vshll_n_u8(nearb, 2)); + int16x8_t curr = vaddq_s16(nears, diff); // current row + + // horizontal filter works the same based on shifted vers of current + // row. "prev" is current row shifted right by 1 pixel; we need to + // insert the previous pixel value (from t1). + // "next" is current row shifted left by 1 pixel, with first pixel + // of next block of 8 pixels added in. + int16x8_t prv0 = vextq_s16(curr, curr, 7); + int16x8_t nxt0 = vextq_s16(curr, curr, 1); + int16x8_t prev = vsetq_lane_s16(t1, prv0, 0); + int16x8_t next = vsetq_lane_s16(3*in_near[i+8] + in_far[i+8], nxt0, 7); + + // horizontal filter, polyphase implementation since it's convenient: + // even pixels = 3*cur + prev = cur*4 + (prev - cur) + // odd pixels = 3*cur + next = cur*4 + (next - cur) + // note the shared term. + int16x8_t curs = vshlq_n_s16(curr, 2); + int16x8_t prvd = vsubq_s16(prev, curr); + int16x8_t nxtd = vsubq_s16(next, curr); + int16x8_t even = vaddq_s16(curs, prvd); + int16x8_t odd = vaddq_s16(curs, nxtd); + + // undo scaling and round, then store with even/odd phases interleaved + uint8x8x2_t o; + o.val[0] = vqrshrun_n_s16(even, 4); + o.val[1] = vqrshrun_n_s16(odd, 4); + vst2_u8(out + i*2, o); +#endif + + // "previous" value for next iter + t1 = 3*in_near[i+7] + in_far[i+7]; + } + + t0 = t1; + t1 = 3*in_near[i] + in_far[i]; + out[i*2] = stbi__div16(3*t1 + t0 + 8); + + for (++i; i < w; ++i) { + t0 = t1; + t1 = 3*in_near[i]+in_far[i]; + out[i*2-1] = stbi__div16(3*t0 + t1 + 8); + out[i*2 ] = stbi__div16(3*t1 + t0 + 8); + } + out[w*2-1] = stbi__div4(t1+2); + + STBI_NOTUSED(hs); + + return out; +} +#endif + +static stbi_uc *stbi__resample_row_generic(stbi_uc *out, stbi_uc *in_near, stbi_uc *in_far, int w, int hs) +{ + // resample with nearest-neighbor + int i,j; + STBI_NOTUSED(in_far); + for (i=0; i < w; ++i) + for (j=0; j < hs; ++j) + out[i*hs+j] = in_near[i]; + return out; +} + +// this is a reduced-precision calculation of YCbCr-to-RGB introduced +// to make sure the code produces the same results in both SIMD and scalar +#define stbi__float2fixed(x) (((int) ((x) * 4096.0f + 0.5f)) << 8) +static void stbi__YCbCr_to_RGB_row(stbi_uc *out, const stbi_uc *y, const stbi_uc *pcb, const stbi_uc *pcr, int count, int step) +{ + int i; + for (i=0; i < count; ++i) { + int y_fixed = (y[i] << 20) + (1<<19); // rounding + int r,g,b; + int cr = pcr[i] - 128; + int cb = pcb[i] - 128; + r = y_fixed + cr* stbi__float2fixed(1.40200f); + g = y_fixed + (cr*-stbi__float2fixed(0.71414f)) + ((cb*-stbi__float2fixed(0.34414f)) & 0xffff0000); + b = y_fixed + cb* stbi__float2fixed(1.77200f); + r >>= 20; + g >>= 20; + b >>= 20; + if ((unsigned) r > 255) { if (r < 0) r = 0; else r = 255; } + if ((unsigned) g > 255) { if (g < 0) g = 0; else g = 255; } + if ((unsigned) b > 255) { if (b < 0) b = 0; else b = 255; } + out[0] = (stbi_uc)r; + out[1] = (stbi_uc)g; + out[2] = (stbi_uc)b; + out[3] = 255; + out += step; + } +} + +#if defined(STBI_SSE2) || defined(STBI_NEON) +static void stbi__YCbCr_to_RGB_simd(stbi_uc *out, stbi_uc const *y, stbi_uc const *pcb, stbi_uc const *pcr, int count, int step) +{ + int i = 0; + +#ifdef STBI_SSE2 + // step == 3 is pretty ugly on the final interleave, and i'm not convinced + // it's useful in practice (you wouldn't use it for textures, for example). + // so just accelerate step == 4 case. + if (step == 4) { + // this is a fairly straightforward implementation and not super-optimized. + __m128i signflip = _mm_set1_epi8(-0x80); + __m128i cr_const0 = _mm_set1_epi16( (short) ( 1.40200f*4096.0f+0.5f)); + __m128i cr_const1 = _mm_set1_epi16( - (short) ( 0.71414f*4096.0f+0.5f)); + __m128i cb_const0 = _mm_set1_epi16( - (short) ( 0.34414f*4096.0f+0.5f)); + __m128i cb_const1 = _mm_set1_epi16( (short) ( 1.77200f*4096.0f+0.5f)); + __m128i y_bias = _mm_set1_epi8((char) (unsigned char) 128); + __m128i xw = _mm_set1_epi16(255); // alpha channel + + for (; i+7 < count; i += 8) { + // load + __m128i y_bytes = _mm_loadl_epi64((__m128i *) (y+i)); + __m128i cr_bytes = _mm_loadl_epi64((__m128i *) (pcr+i)); + __m128i cb_bytes = _mm_loadl_epi64((__m128i *) (pcb+i)); + __m128i cr_biased = _mm_xor_si128(cr_bytes, signflip); // -128 + __m128i cb_biased = _mm_xor_si128(cb_bytes, signflip); // -128 + + // unpack to short (and left-shift cr, cb by 8) + __m128i yw = _mm_unpacklo_epi8(y_bias, y_bytes); + __m128i crw = _mm_unpacklo_epi8(_mm_setzero_si128(), cr_biased); + __m128i cbw = _mm_unpacklo_epi8(_mm_setzero_si128(), cb_biased); + + // color transform + __m128i yws = _mm_srli_epi16(yw, 4); + __m128i cr0 = _mm_mulhi_epi16(cr_const0, crw); + __m128i cb0 = _mm_mulhi_epi16(cb_const0, cbw); + __m128i cb1 = _mm_mulhi_epi16(cbw, cb_const1); + __m128i cr1 = _mm_mulhi_epi16(crw, cr_const1); + __m128i rws = _mm_add_epi16(cr0, yws); + __m128i gwt = _mm_add_epi16(cb0, yws); + __m128i bws = _mm_add_epi16(yws, cb1); + __m128i gws = _mm_add_epi16(gwt, cr1); + + // descale + __m128i rw = _mm_srai_epi16(rws, 4); + __m128i bw = _mm_srai_epi16(bws, 4); + __m128i gw = _mm_srai_epi16(gws, 4); + + // back to byte, set up for transpose + __m128i brb = _mm_packus_epi16(rw, bw); + __m128i gxb = _mm_packus_epi16(gw, xw); + + // transpose to interleave channels + __m128i t0 = _mm_unpacklo_epi8(brb, gxb); + __m128i t1 = _mm_unpackhi_epi8(brb, gxb); + __m128i o0 = _mm_unpacklo_epi16(t0, t1); + __m128i o1 = _mm_unpackhi_epi16(t0, t1); + + // store + _mm_storeu_si128((__m128i *) (out + 0), o0); + _mm_storeu_si128((__m128i *) (out + 16), o1); + out += 32; + } + } +#endif + +#ifdef STBI_NEON + // in this version, step=3 support would be easy to add. but is there demand? + if (step == 4) { + // this is a fairly straightforward implementation and not super-optimized. + uint8x8_t signflip = vdup_n_u8(0x80); + int16x8_t cr_const0 = vdupq_n_s16( (short) ( 1.40200f*4096.0f+0.5f)); + int16x8_t cr_const1 = vdupq_n_s16( - (short) ( 0.71414f*4096.0f+0.5f)); + int16x8_t cb_const0 = vdupq_n_s16( - (short) ( 0.34414f*4096.0f+0.5f)); + int16x8_t cb_const1 = vdupq_n_s16( (short) ( 1.77200f*4096.0f+0.5f)); + + for (; i+7 < count; i += 8) { + // load + uint8x8_t y_bytes = vld1_u8(y + i); + uint8x8_t cr_bytes = vld1_u8(pcr + i); + uint8x8_t cb_bytes = vld1_u8(pcb + i); + int8x8_t cr_biased = vreinterpret_s8_u8(vsub_u8(cr_bytes, signflip)); + int8x8_t cb_biased = vreinterpret_s8_u8(vsub_u8(cb_bytes, signflip)); + + // expand to s16 + int16x8_t yws = vreinterpretq_s16_u16(vshll_n_u8(y_bytes, 4)); + int16x8_t crw = vshll_n_s8(cr_biased, 7); + int16x8_t cbw = vshll_n_s8(cb_biased, 7); + + // color transform + int16x8_t cr0 = vqdmulhq_s16(crw, cr_const0); + int16x8_t cb0 = vqdmulhq_s16(cbw, cb_const0); + int16x8_t cr1 = vqdmulhq_s16(crw, cr_const1); + int16x8_t cb1 = vqdmulhq_s16(cbw, cb_const1); + int16x8_t rws = vaddq_s16(yws, cr0); + int16x8_t gws = vaddq_s16(vaddq_s16(yws, cb0), cr1); + int16x8_t bws = vaddq_s16(yws, cb1); + + // undo scaling, round, convert to byte + uint8x8x4_t o; + o.val[0] = vqrshrun_n_s16(rws, 4); + o.val[1] = vqrshrun_n_s16(gws, 4); + o.val[2] = vqrshrun_n_s16(bws, 4); + o.val[3] = vdup_n_u8(255); + + // store, interleaving r/g/b/a + vst4_u8(out, o); + out += 8*4; + } + } +#endif + + for (; i < count; ++i) { + int y_fixed = (y[i] << 20) + (1<<19); // rounding + int r,g,b; + int cr = pcr[i] - 128; + int cb = pcb[i] - 128; + r = y_fixed + cr* stbi__float2fixed(1.40200f); + g = y_fixed + cr*-stbi__float2fixed(0.71414f) + ((cb*-stbi__float2fixed(0.34414f)) & 0xffff0000); + b = y_fixed + cb* stbi__float2fixed(1.77200f); + r >>= 20; + g >>= 20; + b >>= 20; + if ((unsigned) r > 255) { if (r < 0) r = 0; else r = 255; } + if ((unsigned) g > 255) { if (g < 0) g = 0; else g = 255; } + if ((unsigned) b > 255) { if (b < 0) b = 0; else b = 255; } + out[0] = (stbi_uc)r; + out[1] = (stbi_uc)g; + out[2] = (stbi_uc)b; + out[3] = 255; + out += step; + } +} +#endif + +// set up the kernels +static void stbi__setup_jpeg(stbi__jpeg *j) +{ + j->idct_block_kernel = stbi__idct_block; + j->YCbCr_to_RGB_kernel = stbi__YCbCr_to_RGB_row; + j->resample_row_hv_2_kernel = stbi__resample_row_hv_2; + +#ifdef STBI_SSE2 + if (stbi__sse2_available()) { + j->idct_block_kernel = stbi__idct_simd; + j->YCbCr_to_RGB_kernel = stbi__YCbCr_to_RGB_simd; + j->resample_row_hv_2_kernel = stbi__resample_row_hv_2_simd; + } +#endif + +#ifdef STBI_NEON + j->idct_block_kernel = stbi__idct_simd; + j->YCbCr_to_RGB_kernel = stbi__YCbCr_to_RGB_simd; + j->resample_row_hv_2_kernel = stbi__resample_row_hv_2_simd; +#endif +} + +// clean up the temporary component buffers +static void stbi__cleanup_jpeg(stbi__jpeg *j) +{ + stbi__free_jpeg_components(j, j->s->img_n, 0); +} + +typedef struct +{ + resample_row_func resample; + stbi_uc *line0,*line1; + int hs,vs; // expansion factor in each axis + int w_lores; // horizontal pixels pre-expansion + int ystep; // how far through vertical expansion we are + int ypos; // which pre-expansion row we're on +} stbi__resample; + +// fast 0..255 * 0..255 => 0..255 rounded multiplication +static stbi_uc stbi__blinn_8x8(stbi_uc x, stbi_uc y) +{ + unsigned int t = x*y + 128; + return (stbi_uc) ((t + (t >>8)) >> 8); +} + +static stbi_uc *load_jpeg_image(stbi__jpeg *z, int *out_x, int *out_y, int *comp, int req_comp) +{ + int n, decode_n, is_rgb; + z->s->img_n = 0; // make stbi__cleanup_jpeg safe + + // validate req_comp + if (req_comp < 0 || req_comp > 4) return stbi__errpuc("bad req_comp", "Internal error"); + + // load a jpeg image from whichever source, but leave in YCbCr format + if (!stbi__decode_jpeg_image(z)) { stbi__cleanup_jpeg(z); return NULL; } + + // determine actual number of components to generate + n = req_comp ? req_comp : z->s->img_n >= 3 ? 3 : 1; + + is_rgb = z->s->img_n == 3 && (z->rgb == 3 || (z->app14_color_transform == 0 && !z->jfif)); + + if (z->s->img_n == 3 && n < 3 && !is_rgb) + decode_n = 1; + else + decode_n = z->s->img_n; + + // nothing to do if no components requested; check this now to avoid + // accessing uninitialized coutput[0] later + if (decode_n <= 0) { stbi__cleanup_jpeg(z); return NULL; } + + // resample and color-convert + { + int k; + unsigned int i,j; + stbi_uc *output; + stbi_uc *coutput[4] = { NULL, NULL, NULL, NULL }; + + stbi__resample res_comp[4]; + + for (k=0; k < decode_n; ++k) { + stbi__resample *r = &res_comp[k]; + + // allocate line buffer big enough for upsampling off the edges + // with upsample factor of 4 + z->img_comp[k].linebuf = (stbi_uc *) stbi__malloc(z->s->img_x + 3); + if (!z->img_comp[k].linebuf) { stbi__cleanup_jpeg(z); return stbi__errpuc("outofmem", "Out of memory"); } + + r->hs = z->img_h_max / z->img_comp[k].h; + r->vs = z->img_v_max / z->img_comp[k].v; + r->ystep = r->vs >> 1; + r->w_lores = (z->s->img_x + r->hs-1) / r->hs; + r->ypos = 0; + r->line0 = r->line1 = z->img_comp[k].data; + + if (r->hs == 1 && r->vs == 1) r->resample = resample_row_1; + else if (r->hs == 1 && r->vs == 2) r->resample = stbi__resample_row_v_2; + else if (r->hs == 2 && r->vs == 1) r->resample = stbi__resample_row_h_2; + else if (r->hs == 2 && r->vs == 2) r->resample = z->resample_row_hv_2_kernel; + else r->resample = stbi__resample_row_generic; + } + + // can't error after this so, this is safe + output = (stbi_uc *) stbi__malloc_mad3(n, z->s->img_x, z->s->img_y, 1); + if (!output) { stbi__cleanup_jpeg(z); return stbi__errpuc("outofmem", "Out of memory"); } + + // now go ahead and resample + for (j=0; j < z->s->img_y; ++j) { + stbi_uc *out = output + n * z->s->img_x * j; + for (k=0; k < decode_n; ++k) { + stbi__resample *r = &res_comp[k]; + int y_bot = r->ystep >= (r->vs >> 1); + coutput[k] = r->resample(z->img_comp[k].linebuf, + y_bot ? r->line1 : r->line0, + y_bot ? r->line0 : r->line1, + r->w_lores, r->hs); + if (++r->ystep >= r->vs) { + r->ystep = 0; + r->line0 = r->line1; + if (++r->ypos < z->img_comp[k].y) + r->line1 += z->img_comp[k].w2; + } + } + if (n >= 3) { + stbi_uc *y = coutput[0]; + if (z->s->img_n == 3) { + if (is_rgb) { + for (i=0; i < z->s->img_x; ++i) { + out[0] = y[i]; + out[1] = coutput[1][i]; + out[2] = coutput[2][i]; + out[3] = 255; + out += n; + } + } else { + z->YCbCr_to_RGB_kernel(out, y, coutput[1], coutput[2], z->s->img_x, n); + } + } else if (z->s->img_n == 4) { + if (z->app14_color_transform == 0) { // CMYK + for (i=0; i < z->s->img_x; ++i) { + stbi_uc m = coutput[3][i]; + out[0] = stbi__blinn_8x8(coutput[0][i], m); + out[1] = stbi__blinn_8x8(coutput[1][i], m); + out[2] = stbi__blinn_8x8(coutput[2][i], m); + out[3] = 255; + out += n; + } + } else if (z->app14_color_transform == 2) { // YCCK + z->YCbCr_to_RGB_kernel(out, y, coutput[1], coutput[2], z->s->img_x, n); + for (i=0; i < z->s->img_x; ++i) { + stbi_uc m = coutput[3][i]; + out[0] = stbi__blinn_8x8(255 - out[0], m); + out[1] = stbi__blinn_8x8(255 - out[1], m); + out[2] = stbi__blinn_8x8(255 - out[2], m); + out += n; + } + } else { // YCbCr + alpha? Ignore the fourth channel for now + z->YCbCr_to_RGB_kernel(out, y, coutput[1], coutput[2], z->s->img_x, n); + } + } else + for (i=0; i < z->s->img_x; ++i) { + out[0] = out[1] = out[2] = y[i]; + out[3] = 255; // not used if n==3 + out += n; + } + } else { + if (is_rgb) { + if (n == 1) + for (i=0; i < z->s->img_x; ++i) + *out++ = stbi__compute_y(coutput[0][i], coutput[1][i], coutput[2][i]); + else { + for (i=0; i < z->s->img_x; ++i, out += 2) { + out[0] = stbi__compute_y(coutput[0][i], coutput[1][i], coutput[2][i]); + out[1] = 255; + } + } + } else if (z->s->img_n == 4 && z->app14_color_transform == 0) { + for (i=0; i < z->s->img_x; ++i) { + stbi_uc m = coutput[3][i]; + stbi_uc r = stbi__blinn_8x8(coutput[0][i], m); + stbi_uc g = stbi__blinn_8x8(coutput[1][i], m); + stbi_uc b = stbi__blinn_8x8(coutput[2][i], m); + out[0] = stbi__compute_y(r, g, b); + out[1] = 255; + out += n; + } + } else if (z->s->img_n == 4 && z->app14_color_transform == 2) { + for (i=0; i < z->s->img_x; ++i) { + out[0] = stbi__blinn_8x8(255 - coutput[0][i], coutput[3][i]); + out[1] = 255; + out += n; + } + } else { + stbi_uc *y = coutput[0]; + if (n == 1) + for (i=0; i < z->s->img_x; ++i) out[i] = y[i]; + else + for (i=0; i < z->s->img_x; ++i) { *out++ = y[i]; *out++ = 255; } + } + } + } + stbi__cleanup_jpeg(z); + *out_x = z->s->img_x; + *out_y = z->s->img_y; + if (comp) *comp = z->s->img_n >= 3 ? 3 : 1; // report original components, not output + return output; + } +} + +static void *stbi__jpeg_load(stbi__context *s, int *x, int *y, int *comp, int req_comp, stbi__result_info *ri) +{ + unsigned char* result; + stbi__jpeg* j = (stbi__jpeg*) stbi__malloc(sizeof(stbi__jpeg)); + if (!j) return stbi__errpuc("outofmem", "Out of memory"); + STBI_NOTUSED(ri); + j->s = s; + stbi__setup_jpeg(j); + result = load_jpeg_image(j, x,y,comp,req_comp); + STBI_FREE(j); + return result; +} + +static int stbi__jpeg_test(stbi__context *s) +{ + int r; + stbi__jpeg* j = (stbi__jpeg*)stbi__malloc(sizeof(stbi__jpeg)); + if (!j) return stbi__err("outofmem", "Out of memory"); + j->s = s; + stbi__setup_jpeg(j); + r = stbi__decode_jpeg_header(j, STBI__SCAN_type); + stbi__rewind(s); + STBI_FREE(j); + return r; +} + +static int stbi__jpeg_info_raw(stbi__jpeg *j, int *x, int *y, int *comp) +{ + if (!stbi__decode_jpeg_header(j, STBI__SCAN_header)) { + stbi__rewind( j->s ); + return 0; + } + if (x) *x = j->s->img_x; + if (y) *y = j->s->img_y; + if (comp) *comp = j->s->img_n >= 3 ? 3 : 1; + return 1; +} + +static int stbi__jpeg_info(stbi__context *s, int *x, int *y, int *comp) +{ + int result; + stbi__jpeg* j = (stbi__jpeg*) (stbi__malloc(sizeof(stbi__jpeg))); + if (!j) return stbi__err("outofmem", "Out of memory"); + j->s = s; + result = stbi__jpeg_info_raw(j, x, y, comp); + STBI_FREE(j); + return result; +} +#endif + +// public domain zlib decode v0.2 Sean Barrett 2006-11-18 +// simple implementation +// - all input must be provided in an upfront buffer +// - all output is written to a single output buffer (can malloc/realloc) +// performance +// - fast huffman + +#ifndef STBI_NO_ZLIB + +// fast-way is faster to check than jpeg huffman, but slow way is slower +#define STBI__ZFAST_BITS 9 // accelerate all cases in default tables +#define STBI__ZFAST_MASK ((1 << STBI__ZFAST_BITS) - 1) +#define STBI__ZNSYMS 288 // number of symbols in literal/length alphabet + +// zlib-style huffman encoding +// (jpegs packs from left, zlib from right, so can't share code) +typedef struct +{ + stbi__uint16 fast[1 << STBI__ZFAST_BITS]; + stbi__uint16 firstcode[16]; + int maxcode[17]; + stbi__uint16 firstsymbol[16]; + stbi_uc size[STBI__ZNSYMS]; + stbi__uint16 value[STBI__ZNSYMS]; +} stbi__zhuffman; + +stbi_inline static int stbi__bitreverse16(int n) +{ + n = ((n & 0xAAAA) >> 1) | ((n & 0x5555) << 1); + n = ((n & 0xCCCC) >> 2) | ((n & 0x3333) << 2); + n = ((n & 0xF0F0) >> 4) | ((n & 0x0F0F) << 4); + n = ((n & 0xFF00) >> 8) | ((n & 0x00FF) << 8); + return n; +} + +stbi_inline static int stbi__bit_reverse(int v, int bits) +{ + STBI_ASSERT(bits <= 16); + // to bit reverse n bits, reverse 16 and shift + // e.g. 11 bits, bit reverse and shift away 5 + return stbi__bitreverse16(v) >> (16-bits); +} + +static int stbi__zbuild_huffman(stbi__zhuffman *z, const stbi_uc *sizelist, int num) +{ + int i,k=0; + int code, next_code[16], sizes[17]; + + // DEFLATE spec for generating codes + memset(sizes, 0, sizeof(sizes)); + memset(z->fast, 0, sizeof(z->fast)); + for (i=0; i < num; ++i) + ++sizes[sizelist[i]]; + sizes[0] = 0; + for (i=1; i < 16; ++i) + if (sizes[i] > (1 << i)) + return stbi__err("bad sizes", "Corrupt PNG"); + code = 0; + for (i=1; i < 16; ++i) { + next_code[i] = code; + z->firstcode[i] = (stbi__uint16) code; + z->firstsymbol[i] = (stbi__uint16) k; + code = (code + sizes[i]); + if (sizes[i]) + if (code-1 >= (1 << i)) return stbi__err("bad codelengths","Corrupt PNG"); + z->maxcode[i] = code << (16-i); // preshift for inner loop + code <<= 1; + k += sizes[i]; + } + z->maxcode[16] = 0x10000; // sentinel + for (i=0; i < num; ++i) { + int s = sizelist[i]; + if (s) { + int c = next_code[s] - z->firstcode[s] + z->firstsymbol[s]; + stbi__uint16 fastv = (stbi__uint16) ((s << 9) | i); + z->size [c] = (stbi_uc ) s; + z->value[c] = (stbi__uint16) i; + if (s <= STBI__ZFAST_BITS) { + int j = stbi__bit_reverse(next_code[s],s); + while (j < (1 << STBI__ZFAST_BITS)) { + z->fast[j] = fastv; + j += (1 << s); + } + } + ++next_code[s]; + } + } + return 1; +} + +// zlib-from-memory implementation for PNG reading +// because PNG allows splitting the zlib stream arbitrarily, +// and it's annoying structurally to have PNG call ZLIB call PNG, +// we require PNG read all the IDATs and combine them into a single +// memory buffer + +typedef struct +{ + stbi_uc *zbuffer, *zbuffer_end; + int num_bits; + stbi__uint32 code_buffer; + + char *zout; + char *zout_start; + char *zout_end; + int z_expandable; + + stbi__zhuffman z_length, z_distance; +} stbi__zbuf; + +stbi_inline static int stbi__zeof(stbi__zbuf *z) +{ + return (z->zbuffer >= z->zbuffer_end); +} + +stbi_inline static stbi_uc stbi__zget8(stbi__zbuf *z) +{ + return stbi__zeof(z) ? 0 : *z->zbuffer++; +} + +static void stbi__fill_bits(stbi__zbuf *z) +{ + do { + if (z->code_buffer >= (1U << z->num_bits)) { + z->zbuffer = z->zbuffer_end; /* treat this as EOF so we fail. */ + return; + } + z->code_buffer |= (unsigned int) stbi__zget8(z) << z->num_bits; + z->num_bits += 8; + } while (z->num_bits <= 24); +} + +stbi_inline static unsigned int stbi__zreceive(stbi__zbuf *z, int n) +{ + unsigned int k; + if (z->num_bits < n) stbi__fill_bits(z); + k = z->code_buffer & ((1 << n) - 1); + z->code_buffer >>= n; + z->num_bits -= n; + return k; +} + +static int stbi__zhuffman_decode_slowpath(stbi__zbuf *a, stbi__zhuffman *z) +{ + int b,s,k; + // not resolved by fast table, so compute it the slow way + // use jpeg approach, which requires MSbits at top + k = stbi__bit_reverse(a->code_buffer, 16); + for (s=STBI__ZFAST_BITS+1; ; ++s) + if (k < z->maxcode[s]) + break; + if (s >= 16) return -1; // invalid code! + // code size is s, so: + b = (k >> (16-s)) - z->firstcode[s] + z->firstsymbol[s]; + if (b >= STBI__ZNSYMS) return -1; // some data was corrupt somewhere! + if (z->size[b] != s) return -1; // was originally an assert, but report failure instead. + a->code_buffer >>= s; + a->num_bits -= s; + return z->value[b]; +} + +stbi_inline static int stbi__zhuffman_decode(stbi__zbuf *a, stbi__zhuffman *z) +{ + int b,s; + if (a->num_bits < 16) { + if (stbi__zeof(a)) { + return -1; /* report error for unexpected end of data. */ + } + stbi__fill_bits(a); + } + b = z->fast[a->code_buffer & STBI__ZFAST_MASK]; + if (b) { + s = b >> 9; + a->code_buffer >>= s; + a->num_bits -= s; + return b & 511; + } + return stbi__zhuffman_decode_slowpath(a, z); +} + +static int stbi__zexpand(stbi__zbuf *z, char *zout, int n) // need to make room for n bytes +{ + char *q; + unsigned int cur, limit, old_limit; + z->zout = zout; + if (!z->z_expandable) return stbi__err("output buffer limit","Corrupt PNG"); + cur = (unsigned int) (z->zout - z->zout_start); + limit = old_limit = (unsigned) (z->zout_end - z->zout_start); + if (UINT_MAX - cur < (unsigned) n) return stbi__err("outofmem", "Out of memory"); + while (cur + n > limit) { + if(limit > UINT_MAX / 2) return stbi__err("outofmem", "Out of memory"); + limit *= 2; + } + q = (char *) STBI_REALLOC_SIZED(z->zout_start, old_limit, limit); + STBI_NOTUSED(old_limit); + if (q == NULL) return stbi__err("outofmem", "Out of memory"); + z->zout_start = q; + z->zout = q + cur; + z->zout_end = q + limit; + return 1; +} + +static const int stbi__zlength_base[31] = { + 3,4,5,6,7,8,9,10,11,13, + 15,17,19,23,27,31,35,43,51,59, + 67,83,99,115,131,163,195,227,258,0,0 }; + +static const int stbi__zlength_extra[31]= +{ 0,0,0,0,0,0,0,0,1,1,1,1,2,2,2,2,3,3,3,3,4,4,4,4,5,5,5,5,0,0,0 }; + +static const int stbi__zdist_base[32] = { 1,2,3,4,5,7,9,13,17,25,33,49,65,97,129,193, +257,385,513,769,1025,1537,2049,3073,4097,6145,8193,12289,16385,24577,0,0}; + +static const int stbi__zdist_extra[32] = +{ 0,0,0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,8,8,9,9,10,10,11,11,12,12,13,13}; + +static int stbi__parse_huffman_block(stbi__zbuf *a) +{ + char *zout = a->zout; + for(;;) { + int z = stbi__zhuffman_decode(a, &a->z_length); + if (z < 256) { + if (z < 0) return stbi__err("bad huffman code","Corrupt PNG"); // error in huffman codes + if (zout >= a->zout_end) { + if (!stbi__zexpand(a, zout, 1)) return 0; + zout = a->zout; + } + *zout++ = (char) z; + } else { + stbi_uc *p; + int len,dist; + if (z == 256) { + a->zout = zout; + return 1; + } + z -= 257; + len = stbi__zlength_base[z]; + if (stbi__zlength_extra[z]) len += stbi__zreceive(a, stbi__zlength_extra[z]); + z = stbi__zhuffman_decode(a, &a->z_distance); + if (z < 0) return stbi__err("bad huffman code","Corrupt PNG"); + dist = stbi__zdist_base[z]; + if (stbi__zdist_extra[z]) dist += stbi__zreceive(a, stbi__zdist_extra[z]); + if (zout - a->zout_start < dist) return stbi__err("bad dist","Corrupt PNG"); + if (zout + len > a->zout_end) { + if (!stbi__zexpand(a, zout, len)) return 0; + zout = a->zout; + } + p = (stbi_uc *) (zout - dist); + if (dist == 1) { // run of one byte; common in images. + stbi_uc v = *p; + if (len) { do *zout++ = v; while (--len); } + } else { + if (len) { do *zout++ = *p++; while (--len); } + } + } + } +} + +static int stbi__compute_huffman_codes(stbi__zbuf *a) +{ + static const stbi_uc length_dezigzag[19] = { 16,17,18,0,8,7,9,6,10,5,11,4,12,3,13,2,14,1,15 }; + stbi__zhuffman z_codelength; + stbi_uc lencodes[286+32+137];//padding for maximum single op + stbi_uc codelength_sizes[19]; + int i,n; + + int hlit = stbi__zreceive(a,5) + 257; + int hdist = stbi__zreceive(a,5) + 1; + int hclen = stbi__zreceive(a,4) + 4; + int ntot = hlit + hdist; + + memset(codelength_sizes, 0, sizeof(codelength_sizes)); + for (i=0; i < hclen; ++i) { + int s = stbi__zreceive(a,3); + codelength_sizes[length_dezigzag[i]] = (stbi_uc) s; + } + if (!stbi__zbuild_huffman(&z_codelength, codelength_sizes, 19)) return 0; + + n = 0; + while (n < ntot) { + int c = stbi__zhuffman_decode(a, &z_codelength); + if (c < 0 || c >= 19) return stbi__err("bad codelengths", "Corrupt PNG"); + if (c < 16) + lencodes[n++] = (stbi_uc) c; + else { + stbi_uc fill = 0; + if (c == 16) { + c = stbi__zreceive(a,2)+3; + if (n == 0) return stbi__err("bad codelengths", "Corrupt PNG"); + fill = lencodes[n-1]; + } else if (c == 17) { + c = stbi__zreceive(a,3)+3; + } else if (c == 18) { + c = stbi__zreceive(a,7)+11; + } else { + return stbi__err("bad codelengths", "Corrupt PNG"); + } + if (ntot - n < c) return stbi__err("bad codelengths", "Corrupt PNG"); + memset(lencodes+n, fill, c); + n += c; + } + } + if (n != ntot) return stbi__err("bad codelengths","Corrupt PNG"); + if (!stbi__zbuild_huffman(&a->z_length, lencodes, hlit)) return 0; + if (!stbi__zbuild_huffman(&a->z_distance, lencodes+hlit, hdist)) return 0; + return 1; +} + +static int stbi__parse_uncompressed_block(stbi__zbuf *a) +{ + stbi_uc header[4]; + int len,nlen,k; + if (a->num_bits & 7) + stbi__zreceive(a, a->num_bits & 7); // discard + // drain the bit-packed data into header + k = 0; + while (a->num_bits > 0) { + header[k++] = (stbi_uc) (a->code_buffer & 255); // suppress MSVC run-time check + a->code_buffer >>= 8; + a->num_bits -= 8; + } + if (a->num_bits < 0) return stbi__err("zlib corrupt","Corrupt PNG"); + // now fill header the normal way + while (k < 4) + header[k++] = stbi__zget8(a); + len = header[1] * 256 + header[0]; + nlen = header[3] * 256 + header[2]; + if (nlen != (len ^ 0xffff)) return stbi__err("zlib corrupt","Corrupt PNG"); + if (a->zbuffer + len > a->zbuffer_end) return stbi__err("read past buffer","Corrupt PNG"); + if (a->zout + len > a->zout_end) + if (!stbi__zexpand(a, a->zout, len)) return 0; + memcpy(a->zout, a->zbuffer, len); + a->zbuffer += len; + a->zout += len; + return 1; +} + +static int stbi__parse_zlib_header(stbi__zbuf *a) +{ + int cmf = stbi__zget8(a); + int cm = cmf & 15; + /* int cinfo = cmf >> 4; */ + int flg = stbi__zget8(a); + if (stbi__zeof(a)) return stbi__err("bad zlib header","Corrupt PNG"); // zlib spec + if ((cmf*256+flg) % 31 != 0) return stbi__err("bad zlib header","Corrupt PNG"); // zlib spec + if (flg & 32) return stbi__err("no preset dict","Corrupt PNG"); // preset dictionary not allowed in png + if (cm != 8) return stbi__err("bad compression","Corrupt PNG"); // DEFLATE required for png + // window = 1 << (8 + cinfo)... but who cares, we fully buffer output + return 1; +} + +static const stbi_uc stbi__zdefault_length[STBI__ZNSYMS] = +{ + 8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8, 8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8, + 8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8, 8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8, + 8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8, 8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8, + 8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8, 8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8, + 8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8, 9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9, + 9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9, 9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9, + 9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9, 9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9, + 9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9, 9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9, + 7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7, 7,7,7,7,7,7,7,7,8,8,8,8,8,8,8,8 +}; +static const stbi_uc stbi__zdefault_distance[32] = +{ + 5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5 +}; +/* +Init algorithm: +{ + int i; // use <= to match clearly with spec + for (i=0; i <= 143; ++i) stbi__zdefault_length[i] = 8; + for ( ; i <= 255; ++i) stbi__zdefault_length[i] = 9; + for ( ; i <= 279; ++i) stbi__zdefault_length[i] = 7; + for ( ; i <= 287; ++i) stbi__zdefault_length[i] = 8; + + for (i=0; i <= 31; ++i) stbi__zdefault_distance[i] = 5; +} +*/ + +static int stbi__parse_zlib(stbi__zbuf *a, int parse_header) +{ + int final, type; + if (parse_header) + if (!stbi__parse_zlib_header(a)) return 0; + a->num_bits = 0; + a->code_buffer = 0; + do { + final = stbi__zreceive(a,1); + type = stbi__zreceive(a,2); + if (type == 0) { + if (!stbi__parse_uncompressed_block(a)) return 0; + } else if (type == 3) { + return 0; + } else { + if (type == 1) { + // use fixed code lengths + if (!stbi__zbuild_huffman(&a->z_length , stbi__zdefault_length , STBI__ZNSYMS)) return 0; + if (!stbi__zbuild_huffman(&a->z_distance, stbi__zdefault_distance, 32)) return 0; + } else { + if (!stbi__compute_huffman_codes(a)) return 0; + } + if (!stbi__parse_huffman_block(a)) return 0; + } + } while (!final); + return 1; +} + +static int stbi__do_zlib(stbi__zbuf *a, char *obuf, int olen, int exp, int parse_header) +{ + a->zout_start = obuf; + a->zout = obuf; + a->zout_end = obuf + olen; + a->z_expandable = exp; + + return stbi__parse_zlib(a, parse_header); +} + +STBIDEF char *stbi_zlib_decode_malloc_guesssize(const char *buffer, int len, int initial_size, int *outlen) +{ + stbi__zbuf a; + char *p = (char *) stbi__malloc(initial_size); + if (p == NULL) return NULL; + a.zbuffer = (stbi_uc *) buffer; + a.zbuffer_end = (stbi_uc *) buffer + len; + if (stbi__do_zlib(&a, p, initial_size, 1, 1)) { + if (outlen) *outlen = (int) (a.zout - a.zout_start); + return a.zout_start; + } else { + STBI_FREE(a.zout_start); + return NULL; + } +} + +STBIDEF char *stbi_zlib_decode_malloc(char const *buffer, int len, int *outlen) +{ + return stbi_zlib_decode_malloc_guesssize(buffer, len, 16384, outlen); +} + +STBIDEF char *stbi_zlib_decode_malloc_guesssize_headerflag(const char *buffer, int len, int initial_size, int *outlen, int parse_header) +{ + stbi__zbuf a; + char *p = (char *) stbi__malloc(initial_size); + if (p == NULL) return NULL; + a.zbuffer = (stbi_uc *) buffer; + a.zbuffer_end = (stbi_uc *) buffer + len; + if (stbi__do_zlib(&a, p, initial_size, 1, parse_header)) { + if (outlen) *outlen = (int) (a.zout - a.zout_start); + return a.zout_start; + } else { + STBI_FREE(a.zout_start); + return NULL; + } +} + +STBIDEF int stbi_zlib_decode_buffer(char *obuffer, int olen, char const *ibuffer, int ilen) +{ + stbi__zbuf a; + a.zbuffer = (stbi_uc *) ibuffer; + a.zbuffer_end = (stbi_uc *) ibuffer + ilen; + if (stbi__do_zlib(&a, obuffer, olen, 0, 1)) + return (int) (a.zout - a.zout_start); + else + return -1; +} + +STBIDEF char *stbi_zlib_decode_noheader_malloc(char const *buffer, int len, int *outlen) +{ + stbi__zbuf a; + char *p = (char *) stbi__malloc(16384); + if (p == NULL) return NULL; + a.zbuffer = (stbi_uc *) buffer; + a.zbuffer_end = (stbi_uc *) buffer+len; + if (stbi__do_zlib(&a, p, 16384, 1, 0)) { + if (outlen) *outlen = (int) (a.zout - a.zout_start); + return a.zout_start; + } else { + STBI_FREE(a.zout_start); + return NULL; + } +} + +STBIDEF int stbi_zlib_decode_noheader_buffer(char *obuffer, int olen, const char *ibuffer, int ilen) +{ + stbi__zbuf a; + a.zbuffer = (stbi_uc *) ibuffer; + a.zbuffer_end = (stbi_uc *) ibuffer + ilen; + if (stbi__do_zlib(&a, obuffer, olen, 0, 0)) + return (int) (a.zout - a.zout_start); + else + return -1; +} +#endif + +// public domain "baseline" PNG decoder v0.10 Sean Barrett 2006-11-18 +// simple implementation +// - only 8-bit samples +// - no CRC checking +// - allocates lots of intermediate memory +// - avoids problem of streaming data between subsystems +// - avoids explicit window management +// performance +// - uses stb_zlib, a PD zlib implementation with fast huffman decoding + +#ifndef STBI_NO_PNG +typedef struct +{ + stbi__uint32 length; + stbi__uint32 type; +} stbi__pngchunk; + +static stbi__pngchunk stbi__get_chunk_header(stbi__context *s) +{ + stbi__pngchunk c; + c.length = stbi__get32be(s); + c.type = stbi__get32be(s); + return c; +} + +static int stbi__check_png_header(stbi__context *s) +{ + static const stbi_uc png_sig[8] = { 137,80,78,71,13,10,26,10 }; + int i; + for (i=0; i < 8; ++i) + if (stbi__get8(s) != png_sig[i]) return stbi__err("bad png sig","Not a PNG"); + return 1; +} + +typedef struct +{ + stbi__context *s; + stbi_uc *idata, *expanded, *out; + int depth; +} stbi__png; + + +enum { + STBI__F_none=0, + STBI__F_sub=1, + STBI__F_up=2, + STBI__F_avg=3, + STBI__F_paeth=4, + // synthetic filters used for first scanline to avoid needing a dummy row of 0s + STBI__F_avg_first, + STBI__F_paeth_first +}; + +static stbi_uc first_row_filter[5] = +{ + STBI__F_none, + STBI__F_sub, + STBI__F_none, + STBI__F_avg_first, + STBI__F_paeth_first +}; + +static int stbi__paeth(int a, int b, int c) +{ + int p = a + b - c; + int pa = abs(p-a); + int pb = abs(p-b); + int pc = abs(p-c); + if (pa <= pb && pa <= pc) return a; + if (pb <= pc) return b; + return c; +} + +static const stbi_uc stbi__depth_scale_table[9] = { 0, 0xff, 0x55, 0, 0x11, 0,0,0, 0x01 }; + +// create the png data from post-deflated data +static int stbi__create_png_image_raw(stbi__png *a, stbi_uc *raw, stbi__uint32 raw_len, int out_n, stbi__uint32 x, stbi__uint32 y, int depth, int color) +{ + int bytes = (depth == 16? 2 : 1); + stbi__context *s = a->s; + stbi__uint32 i,j,stride = x*out_n*bytes; + stbi__uint32 img_len, img_width_bytes; + int k; + int img_n = s->img_n; // copy it into a local for later + + int output_bytes = out_n*bytes; + int filter_bytes = img_n*bytes; + int width = x; + + STBI_ASSERT(out_n == s->img_n || out_n == s->img_n+1); + a->out = (stbi_uc *) stbi__malloc_mad3(x, y, output_bytes, 0); // extra bytes to write off the end into + if (!a->out) return stbi__err("outofmem", "Out of memory"); + + if (!stbi__mad3sizes_valid(img_n, x, depth, 7)) return stbi__err("too large", "Corrupt PNG"); + img_width_bytes = (((img_n * x * depth) + 7) >> 3); + img_len = (img_width_bytes + 1) * y; + + // we used to check for exact match between raw_len and img_len on non-interlaced PNGs, + // but issue #276 reported a PNG in the wild that had extra data at the end (all zeros), + // so just check for raw_len < img_len always. + if (raw_len < img_len) return stbi__err("not enough pixels","Corrupt PNG"); + + for (j=0; j < y; ++j) { + stbi_uc *cur = a->out + stride*j; + stbi_uc *prior; + int filter = *raw++; + + if (filter > 4) + return stbi__err("invalid filter","Corrupt PNG"); + + if (depth < 8) { + if (img_width_bytes > x) return stbi__err("invalid width","Corrupt PNG"); + cur += x*out_n - img_width_bytes; // store output to the rightmost img_len bytes, so we can decode in place + filter_bytes = 1; + width = img_width_bytes; + } + prior = cur - stride; // bugfix: need to compute this after 'cur +=' computation above + + // if first row, use special filter that doesn't sample previous row + if (j == 0) filter = first_row_filter[filter]; + + // handle first byte explicitly + for (k=0; k < filter_bytes; ++k) { + switch (filter) { + case STBI__F_none : cur[k] = raw[k]; break; + case STBI__F_sub : cur[k] = raw[k]; break; + case STBI__F_up : cur[k] = STBI__BYTECAST(raw[k] + prior[k]); break; + case STBI__F_avg : cur[k] = STBI__BYTECAST(raw[k] + (prior[k]>>1)); break; + case STBI__F_paeth : cur[k] = STBI__BYTECAST(raw[k] + stbi__paeth(0,prior[k],0)); break; + case STBI__F_avg_first : cur[k] = raw[k]; break; + case STBI__F_paeth_first: cur[k] = raw[k]; break; + } + } + + if (depth == 8) { + if (img_n != out_n) + cur[img_n] = 255; // first pixel + raw += img_n; + cur += out_n; + prior += out_n; + } else if (depth == 16) { + if (img_n != out_n) { + cur[filter_bytes] = 255; // first pixel top byte + cur[filter_bytes+1] = 255; // first pixel bottom byte + } + raw += filter_bytes; + cur += output_bytes; + prior += output_bytes; + } else { + raw += 1; + cur += 1; + prior += 1; + } + + // this is a little gross, so that we don't switch per-pixel or per-component + if (depth < 8 || img_n == out_n) { + int nk = (width - 1)*filter_bytes; + #define STBI__CASE(f) \ + case f: \ + for (k=0; k < nk; ++k) + switch (filter) { + // "none" filter turns into a memcpy here; make that explicit. + case STBI__F_none: memcpy(cur, raw, nk); break; + STBI__CASE(STBI__F_sub) { cur[k] = STBI__BYTECAST(raw[k] + cur[k-filter_bytes]); } break; + STBI__CASE(STBI__F_up) { cur[k] = STBI__BYTECAST(raw[k] + prior[k]); } break; + STBI__CASE(STBI__F_avg) { cur[k] = STBI__BYTECAST(raw[k] + ((prior[k] + cur[k-filter_bytes])>>1)); } break; + STBI__CASE(STBI__F_paeth) { cur[k] = STBI__BYTECAST(raw[k] + stbi__paeth(cur[k-filter_bytes],prior[k],prior[k-filter_bytes])); } break; + STBI__CASE(STBI__F_avg_first) { cur[k] = STBI__BYTECAST(raw[k] + (cur[k-filter_bytes] >> 1)); } break; + STBI__CASE(STBI__F_paeth_first) { cur[k] = STBI__BYTECAST(raw[k] + stbi__paeth(cur[k-filter_bytes],0,0)); } break; + } + #undef STBI__CASE + raw += nk; + } else { + STBI_ASSERT(img_n+1 == out_n); + #define STBI__CASE(f) \ + case f: \ + for (i=x-1; i >= 1; --i, cur[filter_bytes]=255,raw+=filter_bytes,cur+=output_bytes,prior+=output_bytes) \ + for (k=0; k < filter_bytes; ++k) + switch (filter) { + STBI__CASE(STBI__F_none) { cur[k] = raw[k]; } break; + STBI__CASE(STBI__F_sub) { cur[k] = STBI__BYTECAST(raw[k] + cur[k- output_bytes]); } break; + STBI__CASE(STBI__F_up) { cur[k] = STBI__BYTECAST(raw[k] + prior[k]); } break; + STBI__CASE(STBI__F_avg) { cur[k] = STBI__BYTECAST(raw[k] + ((prior[k] + cur[k- output_bytes])>>1)); } break; + STBI__CASE(STBI__F_paeth) { cur[k] = STBI__BYTECAST(raw[k] + stbi__paeth(cur[k- output_bytes],prior[k],prior[k- output_bytes])); } break; + STBI__CASE(STBI__F_avg_first) { cur[k] = STBI__BYTECAST(raw[k] + (cur[k- output_bytes] >> 1)); } break; + STBI__CASE(STBI__F_paeth_first) { cur[k] = STBI__BYTECAST(raw[k] + stbi__paeth(cur[k- output_bytes],0,0)); } break; + } + #undef STBI__CASE + + // the loop above sets the high byte of the pixels' alpha, but for + // 16 bit png files we also need the low byte set. we'll do that here. + if (depth == 16) { + cur = a->out + stride*j; // start at the beginning of the row again + for (i=0; i < x; ++i,cur+=output_bytes) { + cur[filter_bytes+1] = 255; + } + } + } + } + + // we make a separate pass to expand bits to pixels; for performance, + // this could run two scanlines behind the above code, so it won't + // intefere with filtering but will still be in the cache. + if (depth < 8) { + for (j=0; j < y; ++j) { + stbi_uc *cur = a->out + stride*j; + stbi_uc *in = a->out + stride*j + x*out_n - img_width_bytes; + // unpack 1/2/4-bit into a 8-bit buffer. allows us to keep the common 8-bit path optimal at minimal cost for 1/2/4-bit + // png guarante byte alignment, if width is not multiple of 8/4/2 we'll decode dummy trailing data that will be skipped in the later loop + stbi_uc scale = (color == 0) ? stbi__depth_scale_table[depth] : 1; // scale grayscale values to 0..255 range + + // note that the final byte might overshoot and write more data than desired. + // we can allocate enough data that this never writes out of memory, but it + // could also overwrite the next scanline. can it overwrite non-empty data + // on the next scanline? yes, consider 1-pixel-wide scanlines with 1-bit-per-pixel. + // so we need to explicitly clamp the final ones + + if (depth == 4) { + for (k=x*img_n; k >= 2; k-=2, ++in) { + *cur++ = scale * ((*in >> 4) ); + *cur++ = scale * ((*in ) & 0x0f); + } + if (k > 0) *cur++ = scale * ((*in >> 4) ); + } else if (depth == 2) { + for (k=x*img_n; k >= 4; k-=4, ++in) { + *cur++ = scale * ((*in >> 6) ); + *cur++ = scale * ((*in >> 4) & 0x03); + *cur++ = scale * ((*in >> 2) & 0x03); + *cur++ = scale * ((*in ) & 0x03); + } + if (k > 0) *cur++ = scale * ((*in >> 6) ); + if (k > 1) *cur++ = scale * ((*in >> 4) & 0x03); + if (k > 2) *cur++ = scale * ((*in >> 2) & 0x03); + } else if (depth == 1) { + for (k=x*img_n; k >= 8; k-=8, ++in) { + *cur++ = scale * ((*in >> 7) ); + *cur++ = scale * ((*in >> 6) & 0x01); + *cur++ = scale * ((*in >> 5) & 0x01); + *cur++ = scale * ((*in >> 4) & 0x01); + *cur++ = scale * ((*in >> 3) & 0x01); + *cur++ = scale * ((*in >> 2) & 0x01); + *cur++ = scale * ((*in >> 1) & 0x01); + *cur++ = scale * ((*in ) & 0x01); + } + if (k > 0) *cur++ = scale * ((*in >> 7) ); + if (k > 1) *cur++ = scale * ((*in >> 6) & 0x01); + if (k > 2) *cur++ = scale * ((*in >> 5) & 0x01); + if (k > 3) *cur++ = scale * ((*in >> 4) & 0x01); + if (k > 4) *cur++ = scale * ((*in >> 3) & 0x01); + if (k > 5) *cur++ = scale * ((*in >> 2) & 0x01); + if (k > 6) *cur++ = scale * ((*in >> 1) & 0x01); + } + if (img_n != out_n) { + int q; + // insert alpha = 255 + cur = a->out + stride*j; + if (img_n == 1) { + for (q=x-1; q >= 0; --q) { + cur[q*2+1] = 255; + cur[q*2+0] = cur[q]; + } + } else { + STBI_ASSERT(img_n == 3); + for (q=x-1; q >= 0; --q) { + cur[q*4+3] = 255; + cur[q*4+2] = cur[q*3+2]; + cur[q*4+1] = cur[q*3+1]; + cur[q*4+0] = cur[q*3+0]; + } + } + } + } + } else if (depth == 16) { + // force the image data from big-endian to platform-native. + // this is done in a separate pass due to the decoding relying + // on the data being untouched, but could probably be done + // per-line during decode if care is taken. + stbi_uc *cur = a->out; + stbi__uint16 *cur16 = (stbi__uint16*)cur; + + for(i=0; i < x*y*out_n; ++i,cur16++,cur+=2) { + *cur16 = (cur[0] << 8) | cur[1]; + } + } + + return 1; +} + +static int stbi__create_png_image(stbi__png *a, stbi_uc *image_data, stbi__uint32 image_data_len, int out_n, int depth, int color, int interlaced) +{ + int bytes = (depth == 16 ? 2 : 1); + int out_bytes = out_n * bytes; + stbi_uc *final; + int p; + if (!interlaced) + return stbi__create_png_image_raw(a, image_data, image_data_len, out_n, a->s->img_x, a->s->img_y, depth, color); + + // de-interlacing + final = (stbi_uc *) stbi__malloc_mad3(a->s->img_x, a->s->img_y, out_bytes, 0); + if (!final) return stbi__err("outofmem", "Out of memory"); + for (p=0; p < 7; ++p) { + int xorig[] = { 0,4,0,2,0,1,0 }; + int yorig[] = { 0,0,4,0,2,0,1 }; + int xspc[] = { 8,8,4,4,2,2,1 }; + int yspc[] = { 8,8,8,4,4,2,2 }; + int i,j,x,y; + // pass1_x[4] = 0, pass1_x[5] = 1, pass1_x[12] = 1 + x = (a->s->img_x - xorig[p] + xspc[p]-1) / xspc[p]; + y = (a->s->img_y - yorig[p] + yspc[p]-1) / yspc[p]; + if (x && y) { + stbi__uint32 img_len = ((((a->s->img_n * x * depth) + 7) >> 3) + 1) * y; + if (!stbi__create_png_image_raw(a, image_data, image_data_len, out_n, x, y, depth, color)) { + STBI_FREE(final); + return 0; + } + for (j=0; j < y; ++j) { + for (i=0; i < x; ++i) { + int out_y = j*yspc[p]+yorig[p]; + int out_x = i*xspc[p]+xorig[p]; + memcpy(final + out_y*a->s->img_x*out_bytes + out_x*out_bytes, + a->out + (j*x+i)*out_bytes, out_bytes); + } + } + STBI_FREE(a->out); + image_data += img_len; + image_data_len -= img_len; + } + } + a->out = final; + + return 1; +} + +static int stbi__compute_transparency(stbi__png *z, stbi_uc tc[3], int out_n) +{ + stbi__context *s = z->s; + stbi__uint32 i, pixel_count = s->img_x * s->img_y; + stbi_uc *p = z->out; + + // compute color-based transparency, assuming we've + // already got 255 as the alpha value in the output + STBI_ASSERT(out_n == 2 || out_n == 4); + + if (out_n == 2) { + for (i=0; i < pixel_count; ++i) { + p[1] = (p[0] == tc[0] ? 0 : 255); + p += 2; + } + } else { + for (i=0; i < pixel_count; ++i) { + if (p[0] == tc[0] && p[1] == tc[1] && p[2] == tc[2]) + p[3] = 0; + p += 4; + } + } + return 1; +} + +static int stbi__compute_transparency16(stbi__png *z, stbi__uint16 tc[3], int out_n) +{ + stbi__context *s = z->s; + stbi__uint32 i, pixel_count = s->img_x * s->img_y; + stbi__uint16 *p = (stbi__uint16*) z->out; + + // compute color-based transparency, assuming we've + // already got 65535 as the alpha value in the output + STBI_ASSERT(out_n == 2 || out_n == 4); + + if (out_n == 2) { + for (i = 0; i < pixel_count; ++i) { + p[1] = (p[0] == tc[0] ? 0 : 65535); + p += 2; + } + } else { + for (i = 0; i < pixel_count; ++i) { + if (p[0] == tc[0] && p[1] == tc[1] && p[2] == tc[2]) + p[3] = 0; + p += 4; + } + } + return 1; +} + +static int stbi__expand_png_palette(stbi__png *a, stbi_uc *palette, int len, int pal_img_n) +{ + stbi__uint32 i, pixel_count = a->s->img_x * a->s->img_y; + stbi_uc *p, *temp_out, *orig = a->out; + + p = (stbi_uc *) stbi__malloc_mad2(pixel_count, pal_img_n, 0); + if (p == NULL) return stbi__err("outofmem", "Out of memory"); + + // between here and free(out) below, exitting would leak + temp_out = p; + + if (pal_img_n == 3) { + for (i=0; i < pixel_count; ++i) { + int n = orig[i]*4; + p[0] = palette[n ]; + p[1] = palette[n+1]; + p[2] = palette[n+2]; + p += 3; + } + } else { + for (i=0; i < pixel_count; ++i) { + int n = orig[i]*4; + p[0] = palette[n ]; + p[1] = palette[n+1]; + p[2] = palette[n+2]; + p[3] = palette[n+3]; + p += 4; + } + } + STBI_FREE(a->out); + a->out = temp_out; + + STBI_NOTUSED(len); + + return 1; +} + +static int stbi__unpremultiply_on_load_global = 0; +static int stbi__de_iphone_flag_global = 0; + +STBIDEF void stbi_set_unpremultiply_on_load(int flag_true_if_should_unpremultiply) +{ + stbi__unpremultiply_on_load_global = flag_true_if_should_unpremultiply; +} + +STBIDEF void stbi_convert_iphone_png_to_rgb(int flag_true_if_should_convert) +{ + stbi__de_iphone_flag_global = flag_true_if_should_convert; +} + +#ifndef STBI_THREAD_LOCAL +#define stbi__unpremultiply_on_load stbi__unpremultiply_on_load_global +#define stbi__de_iphone_flag stbi__de_iphone_flag_global +#else +static STBI_THREAD_LOCAL int stbi__unpremultiply_on_load_local, stbi__unpremultiply_on_load_set; +static STBI_THREAD_LOCAL int stbi__de_iphone_flag_local, stbi__de_iphone_flag_set; + +STBIDEF void stbi__unpremultiply_on_load_thread(int flag_true_if_should_unpremultiply) +{ + stbi__unpremultiply_on_load_local = flag_true_if_should_unpremultiply; + stbi__unpremultiply_on_load_set = 1; +} + +STBIDEF void stbi_convert_iphone_png_to_rgb_thread(int flag_true_if_should_convert) +{ + stbi__de_iphone_flag_local = flag_true_if_should_convert; + stbi__de_iphone_flag_set = 1; +} + +#define stbi__unpremultiply_on_load (stbi__unpremultiply_on_load_set \ + ? stbi__unpremultiply_on_load_local \ + : stbi__unpremultiply_on_load_global) +#define stbi__de_iphone_flag (stbi__de_iphone_flag_set \ + ? stbi__de_iphone_flag_local \ + : stbi__de_iphone_flag_global) +#endif // STBI_THREAD_LOCAL + +static void stbi__de_iphone(stbi__png *z) +{ + stbi__context *s = z->s; + stbi__uint32 i, pixel_count = s->img_x * s->img_y; + stbi_uc *p = z->out; + + if (s->img_out_n == 3) { // convert bgr to rgb + for (i=0; i < pixel_count; ++i) { + stbi_uc t = p[0]; + p[0] = p[2]; + p[2] = t; + p += 3; + } + } else { + STBI_ASSERT(s->img_out_n == 4); + if (stbi__unpremultiply_on_load) { + // convert bgr to rgb and unpremultiply + for (i=0; i < pixel_count; ++i) { + stbi_uc a = p[3]; + stbi_uc t = p[0]; + if (a) { + stbi_uc half = a / 2; + p[0] = (p[2] * 255 + half) / a; + p[1] = (p[1] * 255 + half) / a; + p[2] = ( t * 255 + half) / a; + } else { + p[0] = p[2]; + p[2] = t; + } + p += 4; + } + } else { + // convert bgr to rgb + for (i=0; i < pixel_count; ++i) { + stbi_uc t = p[0]; + p[0] = p[2]; + p[2] = t; + p += 4; + } + } + } +} + +#define STBI__PNG_TYPE(a,b,c,d) (((unsigned) (a) << 24) + ((unsigned) (b) << 16) + ((unsigned) (c) << 8) + (unsigned) (d)) + +static int stbi__parse_png_file(stbi__png *z, int scan, int req_comp) +{ + stbi_uc palette[1024], pal_img_n=0; + stbi_uc has_trans=0, tc[3]={0}; + stbi__uint16 tc16[3]; + stbi__uint32 ioff=0, idata_limit=0, i, pal_len=0; + int first=1,k,interlace=0, color=0, is_iphone=0; + stbi__context *s = z->s; + + z->expanded = NULL; + z->idata = NULL; + z->out = NULL; + + if (!stbi__check_png_header(s)) return 0; + + if (scan == STBI__SCAN_type) return 1; + + for (;;) { + stbi__pngchunk c = stbi__get_chunk_header(s); + switch (c.type) { + case STBI__PNG_TYPE('C','g','B','I'): + is_iphone = 1; + stbi__skip(s, c.length); + break; + case STBI__PNG_TYPE('I','H','D','R'): { + int comp,filter; + if (!first) return stbi__err("multiple IHDR","Corrupt PNG"); + first = 0; + if (c.length != 13) return stbi__err("bad IHDR len","Corrupt PNG"); + s->img_x = stbi__get32be(s); + s->img_y = stbi__get32be(s); + if (s->img_y > STBI_MAX_DIMENSIONS) return stbi__err("too large","Very large image (corrupt?)"); + if (s->img_x > STBI_MAX_DIMENSIONS) return stbi__err("too large","Very large image (corrupt?)"); + z->depth = stbi__get8(s); if (z->depth != 1 && z->depth != 2 && z->depth != 4 && z->depth != 8 && z->depth != 16) return stbi__err("1/2/4/8/16-bit only","PNG not supported: 1/2/4/8/16-bit only"); + color = stbi__get8(s); if (color > 6) return stbi__err("bad ctype","Corrupt PNG"); + if (color == 3 && z->depth == 16) return stbi__err("bad ctype","Corrupt PNG"); + if (color == 3) pal_img_n = 3; else if (color & 1) return stbi__err("bad ctype","Corrupt PNG"); + comp = stbi__get8(s); if (comp) return stbi__err("bad comp method","Corrupt PNG"); + filter= stbi__get8(s); if (filter) return stbi__err("bad filter method","Corrupt PNG"); + interlace = stbi__get8(s); if (interlace>1) return stbi__err("bad interlace method","Corrupt PNG"); + if (!s->img_x || !s->img_y) return stbi__err("0-pixel image","Corrupt PNG"); + if (!pal_img_n) { + s->img_n = (color & 2 ? 3 : 1) + (color & 4 ? 1 : 0); + if ((1 << 30) / s->img_x / s->img_n < s->img_y) return stbi__err("too large", "Image too large to decode"); + if (scan == STBI__SCAN_header) return 1; + } else { + // if paletted, then pal_n is our final components, and + // img_n is # components to decompress/filter. + s->img_n = 1; + if ((1 << 30) / s->img_x / 4 < s->img_y) return stbi__err("too large","Corrupt PNG"); + // if SCAN_header, have to scan to see if we have a tRNS + } + break; + } + + case STBI__PNG_TYPE('P','L','T','E'): { + if (first) return stbi__err("first not IHDR", "Corrupt PNG"); + if (c.length > 256*3) return stbi__err("invalid PLTE","Corrupt PNG"); + pal_len = c.length / 3; + if (pal_len * 3 != c.length) return stbi__err("invalid PLTE","Corrupt PNG"); + for (i=0; i < pal_len; ++i) { + palette[i*4+0] = stbi__get8(s); + palette[i*4+1] = stbi__get8(s); + palette[i*4+2] = stbi__get8(s); + palette[i*4+3] = 255; + } + break; + } + + case STBI__PNG_TYPE('t','R','N','S'): { + if (first) return stbi__err("first not IHDR", "Corrupt PNG"); + if (z->idata) return stbi__err("tRNS after IDAT","Corrupt PNG"); + if (pal_img_n) { + if (scan == STBI__SCAN_header) { s->img_n = 4; return 1; } + if (pal_len == 0) return stbi__err("tRNS before PLTE","Corrupt PNG"); + if (c.length > pal_len) return stbi__err("bad tRNS len","Corrupt PNG"); + pal_img_n = 4; + for (i=0; i < c.length; ++i) + palette[i*4+3] = stbi__get8(s); + } else { + if (!(s->img_n & 1)) return stbi__err("tRNS with alpha","Corrupt PNG"); + if (c.length != (stbi__uint32) s->img_n*2) return stbi__err("bad tRNS len","Corrupt PNG"); + has_trans = 1; + if (z->depth == 16) { + for (k = 0; k < s->img_n; ++k) tc16[k] = (stbi__uint16)stbi__get16be(s); // copy the values as-is + } else { + for (k = 0; k < s->img_n; ++k) tc[k] = (stbi_uc)(stbi__get16be(s) & 255) * stbi__depth_scale_table[z->depth]; // non 8-bit images will be larger + } + } + break; + } + + case STBI__PNG_TYPE('I','D','A','T'): { + if (first) return stbi__err("first not IHDR", "Corrupt PNG"); + if (pal_img_n && !pal_len) return stbi__err("no PLTE","Corrupt PNG"); + if (scan == STBI__SCAN_header) { s->img_n = pal_img_n; return 1; } + if ((int)(ioff + c.length) < (int)ioff) return 0; + if (ioff + c.length > idata_limit) { + stbi__uint32 idata_limit_old = idata_limit; + stbi_uc *p; + if (idata_limit == 0) idata_limit = c.length > 4096 ? c.length : 4096; + while (ioff + c.length > idata_limit) + idata_limit *= 2; + STBI_NOTUSED(idata_limit_old); + p = (stbi_uc *) STBI_REALLOC_SIZED(z->idata, idata_limit_old, idata_limit); if (p == NULL) return stbi__err("outofmem", "Out of memory"); + z->idata = p; + } + if (!stbi__getn(s, z->idata+ioff,c.length)) return stbi__err("outofdata","Corrupt PNG"); + ioff += c.length; + break; + } + + case STBI__PNG_TYPE('I','E','N','D'): { + stbi__uint32 raw_len, bpl; + if (first) return stbi__err("first not IHDR", "Corrupt PNG"); + if (scan != STBI__SCAN_load) return 1; + if (z->idata == NULL) return stbi__err("no IDAT","Corrupt PNG"); + // initial guess for decoded data size to avoid unnecessary reallocs + bpl = (s->img_x * z->depth + 7) / 8; // bytes per line, per component + raw_len = bpl * s->img_y * s->img_n /* pixels */ + s->img_y /* filter mode per row */; + z->expanded = (stbi_uc *) stbi_zlib_decode_malloc_guesssize_headerflag((char *) z->idata, ioff, raw_len, (int *) &raw_len, !is_iphone); + if (z->expanded == NULL) return 0; // zlib should set error + STBI_FREE(z->idata); z->idata = NULL; + if ((req_comp == s->img_n+1 && req_comp != 3 && !pal_img_n) || has_trans) + s->img_out_n = s->img_n+1; + else + s->img_out_n = s->img_n; + if (!stbi__create_png_image(z, z->expanded, raw_len, s->img_out_n, z->depth, color, interlace)) return 0; + if (has_trans) { + if (z->depth == 16) { + if (!stbi__compute_transparency16(z, tc16, s->img_out_n)) return 0; + } else { + if (!stbi__compute_transparency(z, tc, s->img_out_n)) return 0; + } + } + if (is_iphone && stbi__de_iphone_flag && s->img_out_n > 2) + stbi__de_iphone(z); + if (pal_img_n) { + // pal_img_n == 3 or 4 + s->img_n = pal_img_n; // record the actual colors we had + s->img_out_n = pal_img_n; + if (req_comp >= 3) s->img_out_n = req_comp; + if (!stbi__expand_png_palette(z, palette, pal_len, s->img_out_n)) + return 0; + } else if (has_trans) { + // non-paletted image with tRNS -> source image has (constant) alpha + ++s->img_n; + } + STBI_FREE(z->expanded); z->expanded = NULL; + // end of PNG chunk, read and skip CRC + stbi__get32be(s); + return 1; + } + + default: + // if critical, fail + if (first) return stbi__err("first not IHDR", "Corrupt PNG"); + if ((c.type & (1 << 29)) == 0) { + #ifndef STBI_NO_FAILURE_STRINGS + // not threadsafe + static char invalid_chunk[] = "XXXX PNG chunk not known"; + invalid_chunk[0] = STBI__BYTECAST(c.type >> 24); + invalid_chunk[1] = STBI__BYTECAST(c.type >> 16); + invalid_chunk[2] = STBI__BYTECAST(c.type >> 8); + invalid_chunk[3] = STBI__BYTECAST(c.type >> 0); + #endif + return stbi__err(invalid_chunk, "PNG not supported: unknown PNG chunk type"); + } + stbi__skip(s, c.length); + break; + } + // end of PNG chunk, read and skip CRC + stbi__get32be(s); + } +} + +static void *stbi__do_png(stbi__png *p, int *x, int *y, int *n, int req_comp, stbi__result_info *ri) +{ + void *result=NULL; + if (req_comp < 0 || req_comp > 4) return stbi__errpuc("bad req_comp", "Internal error"); + if (stbi__parse_png_file(p, STBI__SCAN_load, req_comp)) { + if (p->depth <= 8) + ri->bits_per_channel = 8; + else if (p->depth == 16) + ri->bits_per_channel = 16; + else + return stbi__errpuc("bad bits_per_channel", "PNG not supported: unsupported color depth"); + result = p->out; + p->out = NULL; + if (req_comp && req_comp != p->s->img_out_n) { + if (ri->bits_per_channel == 8) + result = stbi__convert_format((unsigned char *) result, p->s->img_out_n, req_comp, p->s->img_x, p->s->img_y); + else + result = stbi__convert_format16((stbi__uint16 *) result, p->s->img_out_n, req_comp, p->s->img_x, p->s->img_y); + p->s->img_out_n = req_comp; + if (result == NULL) return result; + } + *x = p->s->img_x; + *y = p->s->img_y; + if (n) *n = p->s->img_n; + } + STBI_FREE(p->out); p->out = NULL; + STBI_FREE(p->expanded); p->expanded = NULL; + STBI_FREE(p->idata); p->idata = NULL; + + return result; +} + +static void *stbi__png_load(stbi__context *s, int *x, int *y, int *comp, int req_comp, stbi__result_info *ri) +{ + stbi__png p; + p.s = s; + return stbi__do_png(&p, x,y,comp,req_comp, ri); +} + +static int stbi__png_test(stbi__context *s) +{ + int r; + r = stbi__check_png_header(s); + stbi__rewind(s); + return r; +} + +static int stbi__png_info_raw(stbi__png *p, int *x, int *y, int *comp) +{ + if (!stbi__parse_png_file(p, STBI__SCAN_header, 0)) { + stbi__rewind( p->s ); + return 0; + } + if (x) *x = p->s->img_x; + if (y) *y = p->s->img_y; + if (comp) *comp = p->s->img_n; + return 1; +} + +static int stbi__png_info(stbi__context *s, int *x, int *y, int *comp) +{ + stbi__png p; + p.s = s; + return stbi__png_info_raw(&p, x, y, comp); +} + +static int stbi__png_is16(stbi__context *s) +{ + stbi__png p; + p.s = s; + if (!stbi__png_info_raw(&p, NULL, NULL, NULL)) + return 0; + if (p.depth != 16) { + stbi__rewind(p.s); + return 0; + } + return 1; +} +#endif + +// Microsoft/Windows BMP image + +#ifndef STBI_NO_BMP +static int stbi__bmp_test_raw(stbi__context *s) +{ + int r; + int sz; + if (stbi__get8(s) != 'B') return 0; + if (stbi__get8(s) != 'M') return 0; + stbi__get32le(s); // discard filesize + stbi__get16le(s); // discard reserved + stbi__get16le(s); // discard reserved + stbi__get32le(s); // discard data offset + sz = stbi__get32le(s); + r = (sz == 12 || sz == 40 || sz == 56 || sz == 108 || sz == 124); + return r; +} + +static int stbi__bmp_test(stbi__context *s) +{ + int r = stbi__bmp_test_raw(s); + stbi__rewind(s); + return r; +} + + +// returns 0..31 for the highest set bit +static int stbi__high_bit(unsigned int z) +{ + int n=0; + if (z == 0) return -1; + if (z >= 0x10000) { n += 16; z >>= 16; } + if (z >= 0x00100) { n += 8; z >>= 8; } + if (z >= 0x00010) { n += 4; z >>= 4; } + if (z >= 0x00004) { n += 2; z >>= 2; } + if (z >= 0x00002) { n += 1;/* >>= 1;*/ } + return n; +} + +static int stbi__bitcount(unsigned int a) +{ + a = (a & 0x55555555) + ((a >> 1) & 0x55555555); // max 2 + a = (a & 0x33333333) + ((a >> 2) & 0x33333333); // max 4 + a = (a + (a >> 4)) & 0x0f0f0f0f; // max 8 per 4, now 8 bits + a = (a + (a >> 8)); // max 16 per 8 bits + a = (a + (a >> 16)); // max 32 per 8 bits + return a & 0xff; +} + +// extract an arbitrarily-aligned N-bit value (N=bits) +// from v, and then make it 8-bits long and fractionally +// extend it to full full range. +static int stbi__shiftsigned(unsigned int v, int shift, int bits) +{ + static unsigned int mul_table[9] = { + 0, + 0xff/*0b11111111*/, 0x55/*0b01010101*/, 0x49/*0b01001001*/, 0x11/*0b00010001*/, + 0x21/*0b00100001*/, 0x41/*0b01000001*/, 0x81/*0b10000001*/, 0x01/*0b00000001*/, + }; + static unsigned int shift_table[9] = { + 0, 0,0,1,0,2,4,6,0, + }; + if (shift < 0) + v <<= -shift; + else + v >>= shift; + STBI_ASSERT(v < 256); + v >>= (8-bits); + STBI_ASSERT(bits >= 0 && bits <= 8); + return (int) ((unsigned) v * mul_table[bits]) >> shift_table[bits]; +} + +typedef struct +{ + int bpp, offset, hsz; + unsigned int mr,mg,mb,ma, all_a; + int extra_read; +} stbi__bmp_data; + +static int stbi__bmp_set_mask_defaults(stbi__bmp_data *info, int compress) +{ + // BI_BITFIELDS specifies masks explicitly, don't override + if (compress == 3) + return 1; + + if (compress == 0) { + if (info->bpp == 16) { + info->mr = 31u << 10; + info->mg = 31u << 5; + info->mb = 31u << 0; + } else if (info->bpp == 32) { + info->mr = 0xffu << 16; + info->mg = 0xffu << 8; + info->mb = 0xffu << 0; + info->ma = 0xffu << 24; + info->all_a = 0; // if all_a is 0 at end, then we loaded alpha channel but it was all 0 + } else { + // otherwise, use defaults, which is all-0 + info->mr = info->mg = info->mb = info->ma = 0; + } + return 1; + } + return 0; // error +} + +static void *stbi__bmp_parse_header(stbi__context *s, stbi__bmp_data *info) +{ + int hsz; + if (stbi__get8(s) != 'B' || stbi__get8(s) != 'M') return stbi__errpuc("not BMP", "Corrupt BMP"); + stbi__get32le(s); // discard filesize + stbi__get16le(s); // discard reserved + stbi__get16le(s); // discard reserved + info->offset = stbi__get32le(s); + info->hsz = hsz = stbi__get32le(s); + info->mr = info->mg = info->mb = info->ma = 0; + info->extra_read = 14; + + if (info->offset < 0) return stbi__errpuc("bad BMP", "bad BMP"); + + if (hsz != 12 && hsz != 40 && hsz != 56 && hsz != 108 && hsz != 124) return stbi__errpuc("unknown BMP", "BMP type not supported: unknown"); + if (hsz == 12) { + s->img_x = stbi__get16le(s); + s->img_y = stbi__get16le(s); + } else { + s->img_x = stbi__get32le(s); + s->img_y = stbi__get32le(s); + } + if (stbi__get16le(s) != 1) return stbi__errpuc("bad BMP", "bad BMP"); + info->bpp = stbi__get16le(s); + if (hsz != 12) { + int compress = stbi__get32le(s); + if (compress == 1 || compress == 2) return stbi__errpuc("BMP RLE", "BMP type not supported: RLE"); + if (compress >= 4) return stbi__errpuc("BMP JPEG/PNG", "BMP type not supported: unsupported compression"); // this includes PNG/JPEG modes + if (compress == 3 && info->bpp != 16 && info->bpp != 32) return stbi__errpuc("bad BMP", "bad BMP"); // bitfields requires 16 or 32 bits/pixel + stbi__get32le(s); // discard sizeof + stbi__get32le(s); // discard hres + stbi__get32le(s); // discard vres + stbi__get32le(s); // discard colorsused + stbi__get32le(s); // discard max important + if (hsz == 40 || hsz == 56) { + if (hsz == 56) { + stbi__get32le(s); + stbi__get32le(s); + stbi__get32le(s); + stbi__get32le(s); + } + if (info->bpp == 16 || info->bpp == 32) { + if (compress == 0) { + stbi__bmp_set_mask_defaults(info, compress); + } else if (compress == 3) { + info->mr = stbi__get32le(s); + info->mg = stbi__get32le(s); + info->mb = stbi__get32le(s); + info->extra_read += 12; + // not documented, but generated by photoshop and handled by mspaint + if (info->mr == info->mg && info->mg == info->mb) { + // ?!?!? + return stbi__errpuc("bad BMP", "bad BMP"); + } + } else + return stbi__errpuc("bad BMP", "bad BMP"); + } + } else { + // V4/V5 header + int i; + if (hsz != 108 && hsz != 124) + return stbi__errpuc("bad BMP", "bad BMP"); + info->mr = stbi__get32le(s); + info->mg = stbi__get32le(s); + info->mb = stbi__get32le(s); + info->ma = stbi__get32le(s); + if (compress != 3) // override mr/mg/mb unless in BI_BITFIELDS mode, as per docs + stbi__bmp_set_mask_defaults(info, compress); + stbi__get32le(s); // discard color space + for (i=0; i < 12; ++i) + stbi__get32le(s); // discard color space parameters + if (hsz == 124) { + stbi__get32le(s); // discard rendering intent + stbi__get32le(s); // discard offset of profile data + stbi__get32le(s); // discard size of profile data + stbi__get32le(s); // discard reserved + } + } + } + return (void *) 1; +} + + +static void *stbi__bmp_load(stbi__context *s, int *x, int *y, int *comp, int req_comp, stbi__result_info *ri) +{ + stbi_uc *out; + unsigned int mr=0,mg=0,mb=0,ma=0, all_a; + stbi_uc pal[256][4]; + int psize=0,i,j,width; + int flip_vertically, pad, target; + stbi__bmp_data info; + STBI_NOTUSED(ri); + + info.all_a = 255; + if (stbi__bmp_parse_header(s, &info) == NULL) + return NULL; // error code already set + + flip_vertically = ((int) s->img_y) > 0; + s->img_y = abs((int) s->img_y); + + if (s->img_y > STBI_MAX_DIMENSIONS) return stbi__errpuc("too large","Very large image (corrupt?)"); + if (s->img_x > STBI_MAX_DIMENSIONS) return stbi__errpuc("too large","Very large image (corrupt?)"); + + mr = info.mr; + mg = info.mg; + mb = info.mb; + ma = info.ma; + all_a = info.all_a; + + if (info.hsz == 12) { + if (info.bpp < 24) + psize = (info.offset - info.extra_read - 24) / 3; + } else { + if (info.bpp < 16) + psize = (info.offset - info.extra_read - info.hsz) >> 2; + } + if (psize == 0) { + if (info.offset != s->callback_already_read + (s->img_buffer - s->img_buffer_original)) { + return stbi__errpuc("bad offset", "Corrupt BMP"); + } + } + + if (info.bpp == 24 && ma == 0xff000000) + s->img_n = 3; + else + s->img_n = ma ? 4 : 3; + if (req_comp && req_comp >= 3) // we can directly decode 3 or 4 + target = req_comp; + else + target = s->img_n; // if they want monochrome, we'll post-convert + + // sanity-check size + if (!stbi__mad3sizes_valid(target, s->img_x, s->img_y, 0)) + return stbi__errpuc("too large", "Corrupt BMP"); + + out = (stbi_uc *) stbi__malloc_mad3(target, s->img_x, s->img_y, 0); + if (!out) return stbi__errpuc("outofmem", "Out of memory"); + if (info.bpp < 16) { + int z=0; + if (psize == 0 || psize > 256) { STBI_FREE(out); return stbi__errpuc("invalid", "Corrupt BMP"); } + for (i=0; i < psize; ++i) { + pal[i][2] = stbi__get8(s); + pal[i][1] = stbi__get8(s); + pal[i][0] = stbi__get8(s); + if (info.hsz != 12) stbi__get8(s); + pal[i][3] = 255; + } + stbi__skip(s, info.offset - info.extra_read - info.hsz - psize * (info.hsz == 12 ? 3 : 4)); + if (info.bpp == 1) width = (s->img_x + 7) >> 3; + else if (info.bpp == 4) width = (s->img_x + 1) >> 1; + else if (info.bpp == 8) width = s->img_x; + else { STBI_FREE(out); return stbi__errpuc("bad bpp", "Corrupt BMP"); } + pad = (-width)&3; + if (info.bpp == 1) { + for (j=0; j < (int) s->img_y; ++j) { + int bit_offset = 7, v = stbi__get8(s); + for (i=0; i < (int) s->img_x; ++i) { + int color = (v>>bit_offset)&0x1; + out[z++] = pal[color][0]; + out[z++] = pal[color][1]; + out[z++] = pal[color][2]; + if (target == 4) out[z++] = 255; + if (i+1 == (int) s->img_x) break; + if((--bit_offset) < 0) { + bit_offset = 7; + v = stbi__get8(s); + } + } + stbi__skip(s, pad); + } + } else { + for (j=0; j < (int) s->img_y; ++j) { + for (i=0; i < (int) s->img_x; i += 2) { + int v=stbi__get8(s),v2=0; + if (info.bpp == 4) { + v2 = v & 15; + v >>= 4; + } + out[z++] = pal[v][0]; + out[z++] = pal[v][1]; + out[z++] = pal[v][2]; + if (target == 4) out[z++] = 255; + if (i+1 == (int) s->img_x) break; + v = (info.bpp == 8) ? stbi__get8(s) : v2; + out[z++] = pal[v][0]; + out[z++] = pal[v][1]; + out[z++] = pal[v][2]; + if (target == 4) out[z++] = 255; + } + stbi__skip(s, pad); + } + } + } else { + int rshift=0,gshift=0,bshift=0,ashift=0,rcount=0,gcount=0,bcount=0,acount=0; + int z = 0; + int easy=0; + stbi__skip(s, info.offset - info.extra_read - info.hsz); + if (info.bpp == 24) width = 3 * s->img_x; + else if (info.bpp == 16) width = 2*s->img_x; + else /* bpp = 32 and pad = 0 */ width=0; + pad = (-width) & 3; + if (info.bpp == 24) { + easy = 1; + } else if (info.bpp == 32) { + if (mb == 0xff && mg == 0xff00 && mr == 0x00ff0000 && ma == 0xff000000) + easy = 2; + } + if (!easy) { + if (!mr || !mg || !mb) { STBI_FREE(out); return stbi__errpuc("bad masks", "Corrupt BMP"); } + // right shift amt to put high bit in position #7 + rshift = stbi__high_bit(mr)-7; rcount = stbi__bitcount(mr); + gshift = stbi__high_bit(mg)-7; gcount = stbi__bitcount(mg); + bshift = stbi__high_bit(mb)-7; bcount = stbi__bitcount(mb); + ashift = stbi__high_bit(ma)-7; acount = stbi__bitcount(ma); + if (rcount > 8 || gcount > 8 || bcount > 8 || acount > 8) { STBI_FREE(out); return stbi__errpuc("bad masks", "Corrupt BMP"); } + } + for (j=0; j < (int) s->img_y; ++j) { + if (easy) { + for (i=0; i < (int) s->img_x; ++i) { + unsigned char a; + out[z+2] = stbi__get8(s); + out[z+1] = stbi__get8(s); + out[z+0] = stbi__get8(s); + z += 3; + a = (easy == 2 ? stbi__get8(s) : 255); + all_a |= a; + if (target == 4) out[z++] = a; + } + } else { + int bpp = info.bpp; + for (i=0; i < (int) s->img_x; ++i) { + stbi__uint32 v = (bpp == 16 ? (stbi__uint32) stbi__get16le(s) : stbi__get32le(s)); + unsigned int a; + out[z++] = STBI__BYTECAST(stbi__shiftsigned(v & mr, rshift, rcount)); + out[z++] = STBI__BYTECAST(stbi__shiftsigned(v & mg, gshift, gcount)); + out[z++] = STBI__BYTECAST(stbi__shiftsigned(v & mb, bshift, bcount)); + a = (ma ? stbi__shiftsigned(v & ma, ashift, acount) : 255); + all_a |= a; + if (target == 4) out[z++] = STBI__BYTECAST(a); + } + } + stbi__skip(s, pad); + } + } + + // if alpha channel is all 0s, replace with all 255s + if (target == 4 && all_a == 0) + for (i=4*s->img_x*s->img_y-1; i >= 0; i -= 4) + out[i] = 255; + + if (flip_vertically) { + stbi_uc t; + for (j=0; j < (int) s->img_y>>1; ++j) { + stbi_uc *p1 = out + j *s->img_x*target; + stbi_uc *p2 = out + (s->img_y-1-j)*s->img_x*target; + for (i=0; i < (int) s->img_x*target; ++i) { + t = p1[i]; p1[i] = p2[i]; p2[i] = t; + } + } + } + + if (req_comp && req_comp != target) { + out = stbi__convert_format(out, target, req_comp, s->img_x, s->img_y); + if (out == NULL) return out; // stbi__convert_format frees input on failure + } + + *x = s->img_x; + *y = s->img_y; + if (comp) *comp = s->img_n; + return out; +} +#endif + +// Targa Truevision - TGA +// by Jonathan Dummer +#ifndef STBI_NO_TGA +// returns STBI_rgb or whatever, 0 on error +static int stbi__tga_get_comp(int bits_per_pixel, int is_grey, int* is_rgb16) +{ + // only RGB or RGBA (incl. 16bit) or grey allowed + if (is_rgb16) *is_rgb16 = 0; + switch(bits_per_pixel) { + case 8: return STBI_grey; + case 16: if(is_grey) return STBI_grey_alpha; + // fallthrough + case 15: if(is_rgb16) *is_rgb16 = 1; + return STBI_rgb; + case 24: // fallthrough + case 32: return bits_per_pixel/8; + default: return 0; + } +} + +static int stbi__tga_info(stbi__context *s, int *x, int *y, int *comp) +{ + int tga_w, tga_h, tga_comp, tga_image_type, tga_bits_per_pixel, tga_colormap_bpp; + int sz, tga_colormap_type; + stbi__get8(s); // discard Offset + tga_colormap_type = stbi__get8(s); // colormap type + if( tga_colormap_type > 1 ) { + stbi__rewind(s); + return 0; // only RGB or indexed allowed + } + tga_image_type = stbi__get8(s); // image type + if ( tga_colormap_type == 1 ) { // colormapped (paletted) image + if (tga_image_type != 1 && tga_image_type != 9) { + stbi__rewind(s); + return 0; + } + stbi__skip(s,4); // skip index of first colormap entry and number of entries + sz = stbi__get8(s); // check bits per palette color entry + if ( (sz != 8) && (sz != 15) && (sz != 16) && (sz != 24) && (sz != 32) ) { + stbi__rewind(s); + return 0; + } + stbi__skip(s,4); // skip image x and y origin + tga_colormap_bpp = sz; + } else { // "normal" image w/o colormap - only RGB or grey allowed, +/- RLE + if ( (tga_image_type != 2) && (tga_image_type != 3) && (tga_image_type != 10) && (tga_image_type != 11) ) { + stbi__rewind(s); + return 0; // only RGB or grey allowed, +/- RLE + } + stbi__skip(s,9); // skip colormap specification and image x/y origin + tga_colormap_bpp = 0; + } + tga_w = stbi__get16le(s); + if( tga_w < 1 ) { + stbi__rewind(s); + return 0; // test width + } + tga_h = stbi__get16le(s); + if( tga_h < 1 ) { + stbi__rewind(s); + return 0; // test height + } + tga_bits_per_pixel = stbi__get8(s); // bits per pixel + stbi__get8(s); // ignore alpha bits + if (tga_colormap_bpp != 0) { + if((tga_bits_per_pixel != 8) && (tga_bits_per_pixel != 16)) { + // when using a colormap, tga_bits_per_pixel is the size of the indexes + // I don't think anything but 8 or 16bit indexes makes sense + stbi__rewind(s); + return 0; + } + tga_comp = stbi__tga_get_comp(tga_colormap_bpp, 0, NULL); + } else { + tga_comp = stbi__tga_get_comp(tga_bits_per_pixel, (tga_image_type == 3) || (tga_image_type == 11), NULL); + } + if(!tga_comp) { + stbi__rewind(s); + return 0; + } + if (x) *x = tga_w; + if (y) *y = tga_h; + if (comp) *comp = tga_comp; + return 1; // seems to have passed everything +} + +static int stbi__tga_test(stbi__context *s) +{ + int res = 0; + int sz, tga_color_type; + stbi__get8(s); // discard Offset + tga_color_type = stbi__get8(s); // color type + if ( tga_color_type > 1 ) goto errorEnd; // only RGB or indexed allowed + sz = stbi__get8(s); // image type + if ( tga_color_type == 1 ) { // colormapped (paletted) image + if (sz != 1 && sz != 9) goto errorEnd; // colortype 1 demands image type 1 or 9 + stbi__skip(s,4); // skip index of first colormap entry and number of entries + sz = stbi__get8(s); // check bits per palette color entry + if ( (sz != 8) && (sz != 15) && (sz != 16) && (sz != 24) && (sz != 32) ) goto errorEnd; + stbi__skip(s,4); // skip image x and y origin + } else { // "normal" image w/o colormap + if ( (sz != 2) && (sz != 3) && (sz != 10) && (sz != 11) ) goto errorEnd; // only RGB or grey allowed, +/- RLE + stbi__skip(s,9); // skip colormap specification and image x/y origin + } + if ( stbi__get16le(s) < 1 ) goto errorEnd; // test width + if ( stbi__get16le(s) < 1 ) goto errorEnd; // test height + sz = stbi__get8(s); // bits per pixel + if ( (tga_color_type == 1) && (sz != 8) && (sz != 16) ) goto errorEnd; // for colormapped images, bpp is size of an index + if ( (sz != 8) && (sz != 15) && (sz != 16) && (sz != 24) && (sz != 32) ) goto errorEnd; + + res = 1; // if we got this far, everything's good and we can return 1 instead of 0 + +errorEnd: + stbi__rewind(s); + return res; +} + +// read 16bit value and convert to 24bit RGB +static void stbi__tga_read_rgb16(stbi__context *s, stbi_uc* out) +{ + stbi__uint16 px = (stbi__uint16)stbi__get16le(s); + stbi__uint16 fiveBitMask = 31; + // we have 3 channels with 5bits each + int r = (px >> 10) & fiveBitMask; + int g = (px >> 5) & fiveBitMask; + int b = px & fiveBitMask; + // Note that this saves the data in RGB(A) order, so it doesn't need to be swapped later + out[0] = (stbi_uc)((r * 255)/31); + out[1] = (stbi_uc)((g * 255)/31); + out[2] = (stbi_uc)((b * 255)/31); + + // some people claim that the most significant bit might be used for alpha + // (possibly if an alpha-bit is set in the "image descriptor byte") + // but that only made 16bit test images completely translucent.. + // so let's treat all 15 and 16bit TGAs as RGB with no alpha. +} + +static void *stbi__tga_load(stbi__context *s, int *x, int *y, int *comp, int req_comp, stbi__result_info *ri) +{ + // read in the TGA header stuff + int tga_offset = stbi__get8(s); + int tga_indexed = stbi__get8(s); + int tga_image_type = stbi__get8(s); + int tga_is_RLE = 0; + int tga_palette_start = stbi__get16le(s); + int tga_palette_len = stbi__get16le(s); + int tga_palette_bits = stbi__get8(s); + int tga_x_origin = stbi__get16le(s); + int tga_y_origin = stbi__get16le(s); + int tga_width = stbi__get16le(s); + int tga_height = stbi__get16le(s); + int tga_bits_per_pixel = stbi__get8(s); + int tga_comp, tga_rgb16=0; + int tga_inverted = stbi__get8(s); + // int tga_alpha_bits = tga_inverted & 15; // the 4 lowest bits - unused (useless?) + // image data + unsigned char *tga_data; + unsigned char *tga_palette = NULL; + int i, j; + unsigned char raw_data[4] = {0}; + int RLE_count = 0; + int RLE_repeating = 0; + int read_next_pixel = 1; + STBI_NOTUSED(ri); + STBI_NOTUSED(tga_x_origin); // @TODO + STBI_NOTUSED(tga_y_origin); // @TODO + + if (tga_height > STBI_MAX_DIMENSIONS) return stbi__errpuc("too large","Very large image (corrupt?)"); + if (tga_width > STBI_MAX_DIMENSIONS) return stbi__errpuc("too large","Very large image (corrupt?)"); + + // do a tiny bit of precessing + if ( tga_image_type >= 8 ) + { + tga_image_type -= 8; + tga_is_RLE = 1; + } + tga_inverted = 1 - ((tga_inverted >> 5) & 1); + + // If I'm paletted, then I'll use the number of bits from the palette + if ( tga_indexed ) tga_comp = stbi__tga_get_comp(tga_palette_bits, 0, &tga_rgb16); + else tga_comp = stbi__tga_get_comp(tga_bits_per_pixel, (tga_image_type == 3), &tga_rgb16); + + if(!tga_comp) // shouldn't really happen, stbi__tga_test() should have ensured basic consistency + return stbi__errpuc("bad format", "Can't find out TGA pixelformat"); + + // tga info + *x = tga_width; + *y = tga_height; + if (comp) *comp = tga_comp; + + if (!stbi__mad3sizes_valid(tga_width, tga_height, tga_comp, 0)) + return stbi__errpuc("too large", "Corrupt TGA"); + + tga_data = (unsigned char*)stbi__malloc_mad3(tga_width, tga_height, tga_comp, 0); + if (!tga_data) return stbi__errpuc("outofmem", "Out of memory"); + + // skip to the data's starting position (offset usually = 0) + stbi__skip(s, tga_offset ); + + if ( !tga_indexed && !tga_is_RLE && !tga_rgb16 ) { + for (i=0; i < tga_height; ++i) { + int row = tga_inverted ? tga_height -i - 1 : i; + stbi_uc *tga_row = tga_data + row*tga_width*tga_comp; + stbi__getn(s, tga_row, tga_width * tga_comp); + } + } else { + // do I need to load a palette? + if ( tga_indexed) + { + if (tga_palette_len == 0) { /* you have to have at least one entry! */ + STBI_FREE(tga_data); + return stbi__errpuc("bad palette", "Corrupt TGA"); + } + + // any data to skip? (offset usually = 0) + stbi__skip(s, tga_palette_start ); + // load the palette + tga_palette = (unsigned char*)stbi__malloc_mad2(tga_palette_len, tga_comp, 0); + if (!tga_palette) { + STBI_FREE(tga_data); + return stbi__errpuc("outofmem", "Out of memory"); + } + if (tga_rgb16) { + stbi_uc *pal_entry = tga_palette; + STBI_ASSERT(tga_comp == STBI_rgb); + for (i=0; i < tga_palette_len; ++i) { + stbi__tga_read_rgb16(s, pal_entry); + pal_entry += tga_comp; + } + } else if (!stbi__getn(s, tga_palette, tga_palette_len * tga_comp)) { + STBI_FREE(tga_data); + STBI_FREE(tga_palette); + return stbi__errpuc("bad palette", "Corrupt TGA"); + } + } + // load the data + for (i=0; i < tga_width * tga_height; ++i) + { + // if I'm in RLE mode, do I need to get a RLE stbi__pngchunk? + if ( tga_is_RLE ) + { + if ( RLE_count == 0 ) + { + // yep, get the next byte as a RLE command + int RLE_cmd = stbi__get8(s); + RLE_count = 1 + (RLE_cmd & 127); + RLE_repeating = RLE_cmd >> 7; + read_next_pixel = 1; + } else if ( !RLE_repeating ) + { + read_next_pixel = 1; + } + } else + { + read_next_pixel = 1; + } + // OK, if I need to read a pixel, do it now + if ( read_next_pixel ) + { + // load however much data we did have + if ( tga_indexed ) + { + // read in index, then perform the lookup + int pal_idx = (tga_bits_per_pixel == 8) ? stbi__get8(s) : stbi__get16le(s); + if ( pal_idx >= tga_palette_len ) { + // invalid index + pal_idx = 0; + } + pal_idx *= tga_comp; + for (j = 0; j < tga_comp; ++j) { + raw_data[j] = tga_palette[pal_idx+j]; + } + } else if(tga_rgb16) { + STBI_ASSERT(tga_comp == STBI_rgb); + stbi__tga_read_rgb16(s, raw_data); + } else { + // read in the data raw + for (j = 0; j < tga_comp; ++j) { + raw_data[j] = stbi__get8(s); + } + } + // clear the reading flag for the next pixel + read_next_pixel = 0; + } // end of reading a pixel + + // copy data + for (j = 0; j < tga_comp; ++j) + tga_data[i*tga_comp+j] = raw_data[j]; + + // in case we're in RLE mode, keep counting down + --RLE_count; + } + // do I need to invert the image? + if ( tga_inverted ) + { + for (j = 0; j*2 < tga_height; ++j) + { + int index1 = j * tga_width * tga_comp; + int index2 = (tga_height - 1 - j) * tga_width * tga_comp; + for (i = tga_width * tga_comp; i > 0; --i) + { + unsigned char temp = tga_data[index1]; + tga_data[index1] = tga_data[index2]; + tga_data[index2] = temp; + ++index1; + ++index2; + } + } + } + // clear my palette, if I had one + if ( tga_palette != NULL ) + { + STBI_FREE( tga_palette ); + } + } + + // swap RGB - if the source data was RGB16, it already is in the right order + if (tga_comp >= 3 && !tga_rgb16) + { + unsigned char* tga_pixel = tga_data; + for (i=0; i < tga_width * tga_height; ++i) + { + unsigned char temp = tga_pixel[0]; + tga_pixel[0] = tga_pixel[2]; + tga_pixel[2] = temp; + tga_pixel += tga_comp; + } + } + + // convert to target component count + if (req_comp && req_comp != tga_comp) + tga_data = stbi__convert_format(tga_data, tga_comp, req_comp, tga_width, tga_height); + + // the things I do to get rid of an error message, and yet keep + // Microsoft's C compilers happy... [8^( + tga_palette_start = tga_palette_len = tga_palette_bits = + tga_x_origin = tga_y_origin = 0; + STBI_NOTUSED(tga_palette_start); + // OK, done + return tga_data; +} +#endif + +// ************************************************************************************************* +// Photoshop PSD loader -- PD by Thatcher Ulrich, integration by Nicolas Schulz, tweaked by STB + +#ifndef STBI_NO_PSD +static int stbi__psd_test(stbi__context *s) +{ + int r = (stbi__get32be(s) == 0x38425053); + stbi__rewind(s); + return r; +} + +static int stbi__psd_decode_rle(stbi__context *s, stbi_uc *p, int pixelCount) +{ + int count, nleft, len; + + count = 0; + while ((nleft = pixelCount - count) > 0) { + len = stbi__get8(s); + if (len == 128) { + // No-op. + } else if (len < 128) { + // Copy next len+1 bytes literally. + len++; + if (len > nleft) return 0; // corrupt data + count += len; + while (len) { + *p = stbi__get8(s); + p += 4; + len--; + } + } else if (len > 128) { + stbi_uc val; + // Next -len+1 bytes in the dest are replicated from next source byte. + // (Interpret len as a negative 8-bit int.) + len = 257 - len; + if (len > nleft) return 0; // corrupt data + val = stbi__get8(s); + count += len; + while (len) { + *p = val; + p += 4; + len--; + } + } + } + + return 1; +} + +static void *stbi__psd_load(stbi__context *s, int *x, int *y, int *comp, int req_comp, stbi__result_info *ri, int bpc) +{ + int pixelCount; + int channelCount, compression; + int channel, i; + int bitdepth; + int w,h; + stbi_uc *out; + STBI_NOTUSED(ri); + + // Check identifier + if (stbi__get32be(s) != 0x38425053) // "8BPS" + return stbi__errpuc("not PSD", "Corrupt PSD image"); + + // Check file type version. + if (stbi__get16be(s) != 1) + return stbi__errpuc("wrong version", "Unsupported version of PSD image"); + + // Skip 6 reserved bytes. + stbi__skip(s, 6 ); + + // Read the number of channels (R, G, B, A, etc). + channelCount = stbi__get16be(s); + if (channelCount < 0 || channelCount > 16) + return stbi__errpuc("wrong channel count", "Unsupported number of channels in PSD image"); + + // Read the rows and columns of the image. + h = stbi__get32be(s); + w = stbi__get32be(s); + + if (h > STBI_MAX_DIMENSIONS) return stbi__errpuc("too large","Very large image (corrupt?)"); + if (w > STBI_MAX_DIMENSIONS) return stbi__errpuc("too large","Very large image (corrupt?)"); + + // Make sure the depth is 8 bits. + bitdepth = stbi__get16be(s); + if (bitdepth != 8 && bitdepth != 16) + return stbi__errpuc("unsupported bit depth", "PSD bit depth is not 8 or 16 bit"); + + // Make sure the color mode is RGB. + // Valid options are: + // 0: Bitmap + // 1: Grayscale + // 2: Indexed color + // 3: RGB color + // 4: CMYK color + // 7: Multichannel + // 8: Duotone + // 9: Lab color + if (stbi__get16be(s) != 3) + return stbi__errpuc("wrong color format", "PSD is not in RGB color format"); + + // Skip the Mode Data. (It's the palette for indexed color; other info for other modes.) + stbi__skip(s,stbi__get32be(s) ); + + // Skip the image resources. (resolution, pen tool paths, etc) + stbi__skip(s, stbi__get32be(s) ); + + // Skip the reserved data. + stbi__skip(s, stbi__get32be(s) ); + + // Find out if the data is compressed. + // Known values: + // 0: no compression + // 1: RLE compressed + compression = stbi__get16be(s); + if (compression > 1) + return stbi__errpuc("bad compression", "PSD has an unknown compression format"); + + // Check size + if (!stbi__mad3sizes_valid(4, w, h, 0)) + return stbi__errpuc("too large", "Corrupt PSD"); + + // Create the destination image. + + if (!compression && bitdepth == 16 && bpc == 16) { + out = (stbi_uc *) stbi__malloc_mad3(8, w, h, 0); + ri->bits_per_channel = 16; + } else + out = (stbi_uc *) stbi__malloc(4 * w*h); + + if (!out) return stbi__errpuc("outofmem", "Out of memory"); + pixelCount = w*h; + + // Initialize the data to zero. + //memset( out, 0, pixelCount * 4 ); + + // Finally, the image data. + if (compression) { + // RLE as used by .PSD and .TIFF + // Loop until you get the number of unpacked bytes you are expecting: + // Read the next source byte into n. + // If n is between 0 and 127 inclusive, copy the next n+1 bytes literally. + // Else if n is between -127 and -1 inclusive, copy the next byte -n+1 times. + // Else if n is 128, noop. + // Endloop + + // The RLE-compressed data is preceded by a 2-byte data count for each row in the data, + // which we're going to just skip. + stbi__skip(s, h * channelCount * 2 ); + + // Read the RLE data by channel. + for (channel = 0; channel < 4; channel++) { + stbi_uc *p; + + p = out+channel; + if (channel >= channelCount) { + // Fill this channel with default data. + for (i = 0; i < pixelCount; i++, p += 4) + *p = (channel == 3 ? 255 : 0); + } else { + // Read the RLE data. + if (!stbi__psd_decode_rle(s, p, pixelCount)) { + STBI_FREE(out); + return stbi__errpuc("corrupt", "bad RLE data"); + } + } + } + + } else { + // We're at the raw image data. It's each channel in order (Red, Green, Blue, Alpha, ...) + // where each channel consists of an 8-bit (or 16-bit) value for each pixel in the image. + + // Read the data by channel. + for (channel = 0; channel < 4; channel++) { + if (channel >= channelCount) { + // Fill this channel with default data. + if (bitdepth == 16 && bpc == 16) { + stbi__uint16 *q = ((stbi__uint16 *) out) + channel; + stbi__uint16 val = channel == 3 ? 65535 : 0; + for (i = 0; i < pixelCount; i++, q += 4) + *q = val; + } else { + stbi_uc *p = out+channel; + stbi_uc val = channel == 3 ? 255 : 0; + for (i = 0; i < pixelCount; i++, p += 4) + *p = val; + } + } else { + if (ri->bits_per_channel == 16) { // output bpc + stbi__uint16 *q = ((stbi__uint16 *) out) + channel; + for (i = 0; i < pixelCount; i++, q += 4) + *q = (stbi__uint16) stbi__get16be(s); + } else { + stbi_uc *p = out+channel; + if (bitdepth == 16) { // input bpc + for (i = 0; i < pixelCount; i++, p += 4) + *p = (stbi_uc) (stbi__get16be(s) >> 8); + } else { + for (i = 0; i < pixelCount; i++, p += 4) + *p = stbi__get8(s); + } + } + } + } + } + + // remove weird white matte from PSD + if (channelCount >= 4) { + if (ri->bits_per_channel == 16) { + for (i=0; i < w*h; ++i) { + stbi__uint16 *pixel = (stbi__uint16 *) out + 4*i; + if (pixel[3] != 0 && pixel[3] != 65535) { + float a = pixel[3] / 65535.0f; + float ra = 1.0f / a; + float inv_a = 65535.0f * (1 - ra); + pixel[0] = (stbi__uint16) (pixel[0]*ra + inv_a); + pixel[1] = (stbi__uint16) (pixel[1]*ra + inv_a); + pixel[2] = (stbi__uint16) (pixel[2]*ra + inv_a); + } + } + } else { + for (i=0; i < w*h; ++i) { + unsigned char *pixel = out + 4*i; + if (pixel[3] != 0 && pixel[3] != 255) { + float a = pixel[3] / 255.0f; + float ra = 1.0f / a; + float inv_a = 255.0f * (1 - ra); + pixel[0] = (unsigned char) (pixel[0]*ra + inv_a); + pixel[1] = (unsigned char) (pixel[1]*ra + inv_a); + pixel[2] = (unsigned char) (pixel[2]*ra + inv_a); + } + } + } + } + + // convert to desired output format + if (req_comp && req_comp != 4) { + if (ri->bits_per_channel == 16) + out = (stbi_uc *) stbi__convert_format16((stbi__uint16 *) out, 4, req_comp, w, h); + else + out = stbi__convert_format(out, 4, req_comp, w, h); + if (out == NULL) return out; // stbi__convert_format frees input on failure + } + + if (comp) *comp = 4; + *y = h; + *x = w; + + return out; +} +#endif + +// ************************************************************************************************* +// Softimage PIC loader +// by Tom Seddon +// +// See http://softimage.wiki.softimage.com/index.php/INFO:_PIC_file_format +// See http://ozviz.wasp.uwa.edu.au/~pbourke/dataformats/softimagepic/ + +#ifndef STBI_NO_PIC +static int stbi__pic_is4(stbi__context *s,const char *str) +{ + int i; + for (i=0; i<4; ++i) + if (stbi__get8(s) != (stbi_uc)str[i]) + return 0; + + return 1; +} + +static int stbi__pic_test_core(stbi__context *s) +{ + int i; + + if (!stbi__pic_is4(s,"\x53\x80\xF6\x34")) + return 0; + + for(i=0;i<84;++i) + stbi__get8(s); + + if (!stbi__pic_is4(s,"PICT")) + return 0; + + return 1; +} + +typedef struct +{ + stbi_uc size,type,channel; +} stbi__pic_packet; + +static stbi_uc *stbi__readval(stbi__context *s, int channel, stbi_uc *dest) +{ + int mask=0x80, i; + + for (i=0; i<4; ++i, mask>>=1) { + if (channel & mask) { + if (stbi__at_eof(s)) return stbi__errpuc("bad file","PIC file too short"); + dest[i]=stbi__get8(s); + } + } + + return dest; +} + +static void stbi__copyval(int channel,stbi_uc *dest,const stbi_uc *src) +{ + int mask=0x80,i; + + for (i=0;i<4; ++i, mask>>=1) + if (channel&mask) + dest[i]=src[i]; +} + +static stbi_uc *stbi__pic_load_core(stbi__context *s,int width,int height,int *comp, stbi_uc *result) +{ + int act_comp=0,num_packets=0,y,chained; + stbi__pic_packet packets[10]; + + // this will (should...) cater for even some bizarre stuff like having data + // for the same channel in multiple packets. + do { + stbi__pic_packet *packet; + + if (num_packets==sizeof(packets)/sizeof(packets[0])) + return stbi__errpuc("bad format","too many packets"); + + packet = &packets[num_packets++]; + + chained = stbi__get8(s); + packet->size = stbi__get8(s); + packet->type = stbi__get8(s); + packet->channel = stbi__get8(s); + + act_comp |= packet->channel; + + if (stbi__at_eof(s)) return stbi__errpuc("bad file","file too short (reading packets)"); + if (packet->size != 8) return stbi__errpuc("bad format","packet isn't 8bpp"); + } while (chained); + + *comp = (act_comp & 0x10 ? 4 : 3); // has alpha channel? + + for(y=0; ytype) { + default: + return stbi__errpuc("bad format","packet has bad compression type"); + + case 0: {//uncompressed + int x; + + for(x=0;xchannel,dest)) + return 0; + break; + } + + case 1://Pure RLE + { + int left=width, i; + + while (left>0) { + stbi_uc count,value[4]; + + count=stbi__get8(s); + if (stbi__at_eof(s)) return stbi__errpuc("bad file","file too short (pure read count)"); + + if (count > left) + count = (stbi_uc) left; + + if (!stbi__readval(s,packet->channel,value)) return 0; + + for(i=0; ichannel,dest,value); + left -= count; + } + } + break; + + case 2: {//Mixed RLE + int left=width; + while (left>0) { + int count = stbi__get8(s), i; + if (stbi__at_eof(s)) return stbi__errpuc("bad file","file too short (mixed read count)"); + + if (count >= 128) { // Repeated + stbi_uc value[4]; + + if (count==128) + count = stbi__get16be(s); + else + count -= 127; + if (count > left) + return stbi__errpuc("bad file","scanline overrun"); + + if (!stbi__readval(s,packet->channel,value)) + return 0; + + for(i=0;ichannel,dest,value); + } else { // Raw + ++count; + if (count>left) return stbi__errpuc("bad file","scanline overrun"); + + for(i=0;ichannel,dest)) + return 0; + } + left-=count; + } + break; + } + } + } + } + + return result; +} + +static void *stbi__pic_load(stbi__context *s,int *px,int *py,int *comp,int req_comp, stbi__result_info *ri) +{ + stbi_uc *result; + int i, x,y, internal_comp; + STBI_NOTUSED(ri); + + if (!comp) comp = &internal_comp; + + for (i=0; i<92; ++i) + stbi__get8(s); + + x = stbi__get16be(s); + y = stbi__get16be(s); + + if (y > STBI_MAX_DIMENSIONS) return stbi__errpuc("too large","Very large image (corrupt?)"); + if (x > STBI_MAX_DIMENSIONS) return stbi__errpuc("too large","Very large image (corrupt?)"); + + if (stbi__at_eof(s)) return stbi__errpuc("bad file","file too short (pic header)"); + if (!stbi__mad3sizes_valid(x, y, 4, 0)) return stbi__errpuc("too large", "PIC image too large to decode"); + + stbi__get32be(s); //skip `ratio' + stbi__get16be(s); //skip `fields' + stbi__get16be(s); //skip `pad' + + // intermediate buffer is RGBA + result = (stbi_uc *) stbi__malloc_mad3(x, y, 4, 0); + if (!result) return stbi__errpuc("outofmem", "Out of memory"); + memset(result, 0xff, x*y*4); + + if (!stbi__pic_load_core(s,x,y,comp, result)) { + STBI_FREE(result); + result=0; + } + *px = x; + *py = y; + if (req_comp == 0) req_comp = *comp; + result=stbi__convert_format(result,4,req_comp,x,y); + + return result; +} + +static int stbi__pic_test(stbi__context *s) +{ + int r = stbi__pic_test_core(s); + stbi__rewind(s); + return r; +} +#endif + +// ************************************************************************************************* +// GIF loader -- public domain by Jean-Marc Lienher -- simplified/shrunk by stb + +#ifndef STBI_NO_GIF +typedef struct +{ + stbi__int16 prefix; + stbi_uc first; + stbi_uc suffix; +} stbi__gif_lzw; + +typedef struct +{ + int w,h; + stbi_uc *out; // output buffer (always 4 components) + stbi_uc *background; // The current "background" as far as a gif is concerned + stbi_uc *history; + int flags, bgindex, ratio, transparent, eflags; + stbi_uc pal[256][4]; + stbi_uc lpal[256][4]; + stbi__gif_lzw codes[8192]; + stbi_uc *color_table; + int parse, step; + int lflags; + int start_x, start_y; + int max_x, max_y; + int cur_x, cur_y; + int line_size; + int delay; +} stbi__gif; + +static int stbi__gif_test_raw(stbi__context *s) +{ + int sz; + if (stbi__get8(s) != 'G' || stbi__get8(s) != 'I' || stbi__get8(s) != 'F' || stbi__get8(s) != '8') return 0; + sz = stbi__get8(s); + if (sz != '9' && sz != '7') return 0; + if (stbi__get8(s) != 'a') return 0; + return 1; +} + +static int stbi__gif_test(stbi__context *s) +{ + int r = stbi__gif_test_raw(s); + stbi__rewind(s); + return r; +} + +static void stbi__gif_parse_colortable(stbi__context *s, stbi_uc pal[256][4], int num_entries, int transp) +{ + int i; + for (i=0; i < num_entries; ++i) { + pal[i][2] = stbi__get8(s); + pal[i][1] = stbi__get8(s); + pal[i][0] = stbi__get8(s); + pal[i][3] = transp == i ? 0 : 255; + } +} + +static int stbi__gif_header(stbi__context *s, stbi__gif *g, int *comp, int is_info) +{ + stbi_uc version; + if (stbi__get8(s) != 'G' || stbi__get8(s) != 'I' || stbi__get8(s) != 'F' || stbi__get8(s) != '8') + return stbi__err("not GIF", "Corrupt GIF"); + + version = stbi__get8(s); + if (version != '7' && version != '9') return stbi__err("not GIF", "Corrupt GIF"); + if (stbi__get8(s) != 'a') return stbi__err("not GIF", "Corrupt GIF"); + + stbi__g_failure_reason = ""; + g->w = stbi__get16le(s); + g->h = stbi__get16le(s); + g->flags = stbi__get8(s); + g->bgindex = stbi__get8(s); + g->ratio = stbi__get8(s); + g->transparent = -1; + + if (g->w > STBI_MAX_DIMENSIONS) return stbi__err("too large","Very large image (corrupt?)"); + if (g->h > STBI_MAX_DIMENSIONS) return stbi__err("too large","Very large image (corrupt?)"); + + if (comp != 0) *comp = 4; // can't actually tell whether it's 3 or 4 until we parse the comments + + if (is_info) return 1; + + if (g->flags & 0x80) + stbi__gif_parse_colortable(s,g->pal, 2 << (g->flags & 7), -1); + + return 1; +} + +static int stbi__gif_info_raw(stbi__context *s, int *x, int *y, int *comp) +{ + stbi__gif* g = (stbi__gif*) stbi__malloc(sizeof(stbi__gif)); + if (!g) return stbi__err("outofmem", "Out of memory"); + if (!stbi__gif_header(s, g, comp, 1)) { + STBI_FREE(g); + stbi__rewind( s ); + return 0; + } + if (x) *x = g->w; + if (y) *y = g->h; + STBI_FREE(g); + return 1; +} + +static void stbi__out_gif_code(stbi__gif *g, stbi__uint16 code) +{ + stbi_uc *p, *c; + int idx; + + // recurse to decode the prefixes, since the linked-list is backwards, + // and working backwards through an interleaved image would be nasty + if (g->codes[code].prefix >= 0) + stbi__out_gif_code(g, g->codes[code].prefix); + + if (g->cur_y >= g->max_y) return; + + idx = g->cur_x + g->cur_y; + p = &g->out[idx]; + g->history[idx / 4] = 1; + + c = &g->color_table[g->codes[code].suffix * 4]; + if (c[3] > 128) { // don't render transparent pixels; + p[0] = c[2]; + p[1] = c[1]; + p[2] = c[0]; + p[3] = c[3]; + } + g->cur_x += 4; + + if (g->cur_x >= g->max_x) { + g->cur_x = g->start_x; + g->cur_y += g->step; + + while (g->cur_y >= g->max_y && g->parse > 0) { + g->step = (1 << g->parse) * g->line_size; + g->cur_y = g->start_y + (g->step >> 1); + --g->parse; + } + } +} + +static stbi_uc *stbi__process_gif_raster(stbi__context *s, stbi__gif *g) +{ + stbi_uc lzw_cs; + stbi__int32 len, init_code; + stbi__uint32 first; + stbi__int32 codesize, codemask, avail, oldcode, bits, valid_bits, clear; + stbi__gif_lzw *p; + + lzw_cs = stbi__get8(s); + if (lzw_cs > 12) return NULL; + clear = 1 << lzw_cs; + first = 1; + codesize = lzw_cs + 1; + codemask = (1 << codesize) - 1; + bits = 0; + valid_bits = 0; + for (init_code = 0; init_code < clear; init_code++) { + g->codes[init_code].prefix = -1; + g->codes[init_code].first = (stbi_uc) init_code; + g->codes[init_code].suffix = (stbi_uc) init_code; + } + + // support no starting clear code + avail = clear+2; + oldcode = -1; + + len = 0; + for(;;) { + if (valid_bits < codesize) { + if (len == 0) { + len = stbi__get8(s); // start new block + if (len == 0) + return g->out; + } + --len; + bits |= (stbi__int32) stbi__get8(s) << valid_bits; + valid_bits += 8; + } else { + stbi__int32 code = bits & codemask; + bits >>= codesize; + valid_bits -= codesize; + // @OPTIMIZE: is there some way we can accelerate the non-clear path? + if (code == clear) { // clear code + codesize = lzw_cs + 1; + codemask = (1 << codesize) - 1; + avail = clear + 2; + oldcode = -1; + first = 0; + } else if (code == clear + 1) { // end of stream code + stbi__skip(s, len); + while ((len = stbi__get8(s)) > 0) + stbi__skip(s,len); + return g->out; + } else if (code <= avail) { + if (first) { + return stbi__errpuc("no clear code", "Corrupt GIF"); + } + + if (oldcode >= 0) { + p = &g->codes[avail++]; + if (avail > 8192) { + return stbi__errpuc("too many codes", "Corrupt GIF"); + } + + p->prefix = (stbi__int16) oldcode; + p->first = g->codes[oldcode].first; + p->suffix = (code == avail) ? p->first : g->codes[code].first; + } else if (code == avail) + return stbi__errpuc("illegal code in raster", "Corrupt GIF"); + + stbi__out_gif_code(g, (stbi__uint16) code); + + if ((avail & codemask) == 0 && avail <= 0x0FFF) { + codesize++; + codemask = (1 << codesize) - 1; + } + + oldcode = code; + } else { + return stbi__errpuc("illegal code in raster", "Corrupt GIF"); + } + } + } +} + +// this function is designed to support animated gifs, although stb_image doesn't support it +// two back is the image from two frames ago, used for a very specific disposal format +static stbi_uc *stbi__gif_load_next(stbi__context *s, stbi__gif *g, int *comp, int req_comp, stbi_uc *two_back) +{ + int dispose; + int first_frame; + int pi; + int pcount; + STBI_NOTUSED(req_comp); + + // on first frame, any non-written pixels get the background colour (non-transparent) + first_frame = 0; + if (g->out == 0) { + if (!stbi__gif_header(s, g, comp,0)) return 0; // stbi__g_failure_reason set by stbi__gif_header + if (!stbi__mad3sizes_valid(4, g->w, g->h, 0)) + return stbi__errpuc("too large", "GIF image is too large"); + pcount = g->w * g->h; + g->out = (stbi_uc *) stbi__malloc(4 * pcount); + g->background = (stbi_uc *) stbi__malloc(4 * pcount); + g->history = (stbi_uc *) stbi__malloc(pcount); + if (!g->out || !g->background || !g->history) + return stbi__errpuc("outofmem", "Out of memory"); + + // image is treated as "transparent" at the start - ie, nothing overwrites the current background; + // background colour is only used for pixels that are not rendered first frame, after that "background" + // color refers to the color that was there the previous frame. + memset(g->out, 0x00, 4 * pcount); + memset(g->background, 0x00, 4 * pcount); // state of the background (starts transparent) + memset(g->history, 0x00, pcount); // pixels that were affected previous frame + first_frame = 1; + } else { + // second frame - how do we dispose of the previous one? + dispose = (g->eflags & 0x1C) >> 2; + pcount = g->w * g->h; + + if ((dispose == 3) && (two_back == 0)) { + dispose = 2; // if I don't have an image to revert back to, default to the old background + } + + if (dispose == 3) { // use previous graphic + for (pi = 0; pi < pcount; ++pi) { + if (g->history[pi]) { + memcpy( &g->out[pi * 4], &two_back[pi * 4], 4 ); + } + } + } else if (dispose == 2) { + // restore what was changed last frame to background before that frame; + for (pi = 0; pi < pcount; ++pi) { + if (g->history[pi]) { + memcpy( &g->out[pi * 4], &g->background[pi * 4], 4 ); + } + } + } else { + // This is a non-disposal case eithe way, so just + // leave the pixels as is, and they will become the new background + // 1: do not dispose + // 0: not specified. + } + + // background is what out is after the undoing of the previou frame; + memcpy( g->background, g->out, 4 * g->w * g->h ); + } + + // clear my history; + memset( g->history, 0x00, g->w * g->h ); // pixels that were affected previous frame + + for (;;) { + int tag = stbi__get8(s); + switch (tag) { + case 0x2C: /* Image Descriptor */ + { + stbi__int32 x, y, w, h; + stbi_uc *o; + + x = stbi__get16le(s); + y = stbi__get16le(s); + w = stbi__get16le(s); + h = stbi__get16le(s); + if (((x + w) > (g->w)) || ((y + h) > (g->h))) + return stbi__errpuc("bad Image Descriptor", "Corrupt GIF"); + + g->line_size = g->w * 4; + g->start_x = x * 4; + g->start_y = y * g->line_size; + g->max_x = g->start_x + w * 4; + g->max_y = g->start_y + h * g->line_size; + g->cur_x = g->start_x; + g->cur_y = g->start_y; + + // if the width of the specified rectangle is 0, that means + // we may not see *any* pixels or the image is malformed; + // to make sure this is caught, move the current y down to + // max_y (which is what out_gif_code checks). + if (w == 0) + g->cur_y = g->max_y; + + g->lflags = stbi__get8(s); + + if (g->lflags & 0x40) { + g->step = 8 * g->line_size; // first interlaced spacing + g->parse = 3; + } else { + g->step = g->line_size; + g->parse = 0; + } + + if (g->lflags & 0x80) { + stbi__gif_parse_colortable(s,g->lpal, 2 << (g->lflags & 7), g->eflags & 0x01 ? g->transparent : -1); + g->color_table = (stbi_uc *) g->lpal; + } else if (g->flags & 0x80) { + g->color_table = (stbi_uc *) g->pal; + } else + return stbi__errpuc("missing color table", "Corrupt GIF"); + + o = stbi__process_gif_raster(s, g); + if (!o) return NULL; + + // if this was the first frame, + pcount = g->w * g->h; + if (first_frame && (g->bgindex > 0)) { + // if first frame, any pixel not drawn to gets the background color + for (pi = 0; pi < pcount; ++pi) { + if (g->history[pi] == 0) { + g->pal[g->bgindex][3] = 255; // just in case it was made transparent, undo that; It will be reset next frame if need be; + memcpy( &g->out[pi * 4], &g->pal[g->bgindex], 4 ); + } + } + } + + return o; + } + + case 0x21: // Comment Extension. + { + int len; + int ext = stbi__get8(s); + if (ext == 0xF9) { // Graphic Control Extension. + len = stbi__get8(s); + if (len == 4) { + g->eflags = stbi__get8(s); + g->delay = 10 * stbi__get16le(s); // delay - 1/100th of a second, saving as 1/1000ths. + + // unset old transparent + if (g->transparent >= 0) { + g->pal[g->transparent][3] = 255; + } + if (g->eflags & 0x01) { + g->transparent = stbi__get8(s); + if (g->transparent >= 0) { + g->pal[g->transparent][3] = 0; + } + } else { + // don't need transparent + stbi__skip(s, 1); + g->transparent = -1; + } + } else { + stbi__skip(s, len); + break; + } + } + while ((len = stbi__get8(s)) != 0) { + stbi__skip(s, len); + } + break; + } + + case 0x3B: // gif stream termination code + return (stbi_uc *) s; // using '1' causes warning on some compilers + + default: + return stbi__errpuc("unknown code", "Corrupt GIF"); + } + } +} + +static void *stbi__load_gif_main_outofmem(stbi__gif *g, stbi_uc *out, int **delays) +{ + STBI_FREE(g->out); + STBI_FREE(g->history); + STBI_FREE(g->background); + + if (out) STBI_FREE(out); + if (delays && *delays) STBI_FREE(*delays); + return stbi__errpuc("outofmem", "Out of memory"); +} + +static void *stbi__load_gif_main(stbi__context *s, int **delays, int *x, int *y, int *z, int *comp, int req_comp) +{ + if (stbi__gif_test(s)) { + int layers = 0; + stbi_uc *u = 0; + stbi_uc *out = 0; + stbi_uc *two_back = 0; + stbi__gif g; + int stride; + int out_size = 0; + int delays_size = 0; + + STBI_NOTUSED(out_size); + STBI_NOTUSED(delays_size); + + memset(&g, 0, sizeof(g)); + if (delays) { + *delays = 0; + } + + do { + u = stbi__gif_load_next(s, &g, comp, req_comp, two_back); + if (u == (stbi_uc *) s) u = 0; // end of animated gif marker + + if (u) { + *x = g.w; + *y = g.h; + ++layers; + stride = g.w * g.h * 4; + + if (out) { + void *tmp = (stbi_uc*) STBI_REALLOC_SIZED( out, out_size, layers * stride ); + if (!tmp) + return stbi__load_gif_main_outofmem(&g, out, delays); + else { + out = (stbi_uc*) tmp; + out_size = layers * stride; + } + + if (delays) { + int *new_delays = (int*) STBI_REALLOC_SIZED( *delays, delays_size, sizeof(int) * layers ); + if (!new_delays) + return stbi__load_gif_main_outofmem(&g, out, delays); + *delays = new_delays; + delays_size = layers * sizeof(int); + } + } else { + out = (stbi_uc*)stbi__malloc( layers * stride ); + if (!out) + return stbi__load_gif_main_outofmem(&g, out, delays); + out_size = layers * stride; + if (delays) { + *delays = (int*) stbi__malloc( layers * sizeof(int) ); + if (!*delays) + return stbi__load_gif_main_outofmem(&g, out, delays); + delays_size = layers * sizeof(int); + } + } + memcpy( out + ((layers - 1) * stride), u, stride ); + if (layers >= 2) { + two_back = out - 2 * stride; + } + + if (delays) { + (*delays)[layers - 1U] = g.delay; + } + } + } while (u != 0); + + // free temp buffer; + STBI_FREE(g.out); + STBI_FREE(g.history); + STBI_FREE(g.background); + + // do the final conversion after loading everything; + if (req_comp && req_comp != 4) + out = stbi__convert_format(out, 4, req_comp, layers * g.w, g.h); + + *z = layers; + return out; + } else { + return stbi__errpuc("not GIF", "Image was not as a gif type."); + } +} + +static void *stbi__gif_load(stbi__context *s, int *x, int *y, int *comp, int req_comp, stbi__result_info *ri) +{ + stbi_uc *u = 0; + stbi__gif g; + memset(&g, 0, sizeof(g)); + STBI_NOTUSED(ri); + + u = stbi__gif_load_next(s, &g, comp, req_comp, 0); + if (u == (stbi_uc *) s) u = 0; // end of animated gif marker + if (u) { + *x = g.w; + *y = g.h; + + // moved conversion to after successful load so that the same + // can be done for multiple frames. + if (req_comp && req_comp != 4) + u = stbi__convert_format(u, 4, req_comp, g.w, g.h); + } else if (g.out) { + // if there was an error and we allocated an image buffer, free it! + STBI_FREE(g.out); + } + + // free buffers needed for multiple frame loading; + STBI_FREE(g.history); + STBI_FREE(g.background); + + return u; +} + +static int stbi__gif_info(stbi__context *s, int *x, int *y, int *comp) +{ + return stbi__gif_info_raw(s,x,y,comp); +} +#endif + +// ************************************************************************************************* +// Radiance RGBE HDR loader +// originally by Nicolas Schulz +#ifndef STBI_NO_HDR +static int stbi__hdr_test_core(stbi__context *s, const char *signature) +{ + int i; + for (i=0; signature[i]; ++i) + if (stbi__get8(s) != signature[i]) + return 0; + stbi__rewind(s); + return 1; +} + +static int stbi__hdr_test(stbi__context* s) +{ + int r = stbi__hdr_test_core(s, "#?RADIANCE\n"); + stbi__rewind(s); + if(!r) { + r = stbi__hdr_test_core(s, "#?RGBE\n"); + stbi__rewind(s); + } + return r; +} + +#define STBI__HDR_BUFLEN 1024 +static char *stbi__hdr_gettoken(stbi__context *z, char *buffer) +{ + int len=0; + char c = '\0'; + + c = (char) stbi__get8(z); + + while (!stbi__at_eof(z) && c != '\n') { + buffer[len++] = c; + if (len == STBI__HDR_BUFLEN-1) { + // flush to end of line + while (!stbi__at_eof(z) && stbi__get8(z) != '\n') + ; + break; + } + c = (char) stbi__get8(z); + } + + buffer[len] = 0; + return buffer; +} + +static void stbi__hdr_convert(float *output, stbi_uc *input, int req_comp) +{ + if ( input[3] != 0 ) { + float f1; + // Exponent + f1 = (float) ldexp(1.0f, input[3] - (int)(128 + 8)); + if (req_comp <= 2) + output[0] = (input[0] + input[1] + input[2]) * f1 / 3; + else { + output[0] = input[0] * f1; + output[1] = input[1] * f1; + output[2] = input[2] * f1; + } + if (req_comp == 2) output[1] = 1; + if (req_comp == 4) output[3] = 1; + } else { + switch (req_comp) { + case 4: output[3] = 1; /* fallthrough */ + case 3: output[0] = output[1] = output[2] = 0; + break; + case 2: output[1] = 1; /* fallthrough */ + case 1: output[0] = 0; + break; + } + } +} + +static float *stbi__hdr_load(stbi__context *s, int *x, int *y, int *comp, int req_comp, stbi__result_info *ri) +{ + char buffer[STBI__HDR_BUFLEN]; + char *token; + int valid = 0; + int width, height; + stbi_uc *scanline; + float *hdr_data; + int len; + unsigned char count, value; + int i, j, k, c1,c2, z; + const char *headerToken; + STBI_NOTUSED(ri); + + // Check identifier + headerToken = stbi__hdr_gettoken(s,buffer); + if (strcmp(headerToken, "#?RADIANCE") != 0 && strcmp(headerToken, "#?RGBE") != 0) + return stbi__errpf("not HDR", "Corrupt HDR image"); + + // Parse header + for(;;) { + token = stbi__hdr_gettoken(s,buffer); + if (token[0] == 0) break; + if (strcmp(token, "FORMAT=32-bit_rle_rgbe") == 0) valid = 1; + } + + if (!valid) return stbi__errpf("unsupported format", "Unsupported HDR format"); + + // Parse width and height + // can't use sscanf() if we're not using stdio! + token = stbi__hdr_gettoken(s,buffer); + if (strncmp(token, "-Y ", 3)) return stbi__errpf("unsupported data layout", "Unsupported HDR format"); + token += 3; + height = (int) strtol(token, &token, 10); + while (*token == ' ') ++token; + if (strncmp(token, "+X ", 3)) return stbi__errpf("unsupported data layout", "Unsupported HDR format"); + token += 3; + width = (int) strtol(token, NULL, 10); + + if (height > STBI_MAX_DIMENSIONS) return stbi__errpf("too large","Very large image (corrupt?)"); + if (width > STBI_MAX_DIMENSIONS) return stbi__errpf("too large","Very large image (corrupt?)"); + + *x = width; + *y = height; + + if (comp) *comp = 3; + if (req_comp == 0) req_comp = 3; + + if (!stbi__mad4sizes_valid(width, height, req_comp, sizeof(float), 0)) + return stbi__errpf("too large", "HDR image is too large"); + + // Read data + hdr_data = (float *) stbi__malloc_mad4(width, height, req_comp, sizeof(float), 0); + if (!hdr_data) + return stbi__errpf("outofmem", "Out of memory"); + + // Load image data + // image data is stored as some number of sca + if ( width < 8 || width >= 32768) { + // Read flat data + for (j=0; j < height; ++j) { + for (i=0; i < width; ++i) { + stbi_uc rgbe[4]; + main_decode_loop: + stbi__getn(s, rgbe, 4); + stbi__hdr_convert(hdr_data + j * width * req_comp + i * req_comp, rgbe, req_comp); + } + } + } else { + // Read RLE-encoded data + scanline = NULL; + + for (j = 0; j < height; ++j) { + c1 = stbi__get8(s); + c2 = stbi__get8(s); + len = stbi__get8(s); + if (c1 != 2 || c2 != 2 || (len & 0x80)) { + // not run-length encoded, so we have to actually use THIS data as a decoded + // pixel (note this can't be a valid pixel--one of RGB must be >= 128) + stbi_uc rgbe[4]; + rgbe[0] = (stbi_uc) c1; + rgbe[1] = (stbi_uc) c2; + rgbe[2] = (stbi_uc) len; + rgbe[3] = (stbi_uc) stbi__get8(s); + stbi__hdr_convert(hdr_data, rgbe, req_comp); + i = 1; + j = 0; + STBI_FREE(scanline); + goto main_decode_loop; // yes, this makes no sense + } + len <<= 8; + len |= stbi__get8(s); + if (len != width) { STBI_FREE(hdr_data); STBI_FREE(scanline); return stbi__errpf("invalid decoded scanline length", "corrupt HDR"); } + if (scanline == NULL) { + scanline = (stbi_uc *) stbi__malloc_mad2(width, 4, 0); + if (!scanline) { + STBI_FREE(hdr_data); + return stbi__errpf("outofmem", "Out of memory"); + } + } + + for (k = 0; k < 4; ++k) { + int nleft; + i = 0; + while ((nleft = width - i) > 0) { + count = stbi__get8(s); + if (count > 128) { + // Run + value = stbi__get8(s); + count -= 128; + if (count > nleft) { STBI_FREE(hdr_data); STBI_FREE(scanline); return stbi__errpf("corrupt", "bad RLE data in HDR"); } + for (z = 0; z < count; ++z) + scanline[i++ * 4 + k] = value; + } else { + // Dump + if (count > nleft) { STBI_FREE(hdr_data); STBI_FREE(scanline); return stbi__errpf("corrupt", "bad RLE data in HDR"); } + for (z = 0; z < count; ++z) + scanline[i++ * 4 + k] = stbi__get8(s); + } + } + } + for (i=0; i < width; ++i) + stbi__hdr_convert(hdr_data+(j*width + i)*req_comp, scanline + i*4, req_comp); + } + if (scanline) + STBI_FREE(scanline); + } + + return hdr_data; +} + +static int stbi__hdr_info(stbi__context *s, int *x, int *y, int *comp) +{ + char buffer[STBI__HDR_BUFLEN]; + char *token; + int valid = 0; + int dummy; + + if (!x) x = &dummy; + if (!y) y = &dummy; + if (!comp) comp = &dummy; + + if (stbi__hdr_test(s) == 0) { + stbi__rewind( s ); + return 0; + } + + for(;;) { + token = stbi__hdr_gettoken(s,buffer); + if (token[0] == 0) break; + if (strcmp(token, "FORMAT=32-bit_rle_rgbe") == 0) valid = 1; + } + + if (!valid) { + stbi__rewind( s ); + return 0; + } + token = stbi__hdr_gettoken(s,buffer); + if (strncmp(token, "-Y ", 3)) { + stbi__rewind( s ); + return 0; + } + token += 3; + *y = (int) strtol(token, &token, 10); + while (*token == ' ') ++token; + if (strncmp(token, "+X ", 3)) { + stbi__rewind( s ); + return 0; + } + token += 3; + *x = (int) strtol(token, NULL, 10); + *comp = 3; + return 1; +} +#endif // STBI_NO_HDR + +#ifndef STBI_NO_BMP +static int stbi__bmp_info(stbi__context *s, int *x, int *y, int *comp) +{ + void *p; + stbi__bmp_data info; + + info.all_a = 255; + p = stbi__bmp_parse_header(s, &info); + if (p == NULL) { + stbi__rewind( s ); + return 0; + } + if (x) *x = s->img_x; + if (y) *y = s->img_y; + if (comp) { + if (info.bpp == 24 && info.ma == 0xff000000) + *comp = 3; + else + *comp = info.ma ? 4 : 3; + } + return 1; +} +#endif + +#ifndef STBI_NO_PSD +static int stbi__psd_info(stbi__context *s, int *x, int *y, int *comp) +{ + int channelCount, dummy, depth; + if (!x) x = &dummy; + if (!y) y = &dummy; + if (!comp) comp = &dummy; + if (stbi__get32be(s) != 0x38425053) { + stbi__rewind( s ); + return 0; + } + if (stbi__get16be(s) != 1) { + stbi__rewind( s ); + return 0; + } + stbi__skip(s, 6); + channelCount = stbi__get16be(s); + if (channelCount < 0 || channelCount > 16) { + stbi__rewind( s ); + return 0; + } + *y = stbi__get32be(s); + *x = stbi__get32be(s); + depth = stbi__get16be(s); + if (depth != 8 && depth != 16) { + stbi__rewind( s ); + return 0; + } + if (stbi__get16be(s) != 3) { + stbi__rewind( s ); + return 0; + } + *comp = 4; + return 1; +} + +static int stbi__psd_is16(stbi__context *s) +{ + int channelCount, depth; + if (stbi__get32be(s) != 0x38425053) { + stbi__rewind( s ); + return 0; + } + if (stbi__get16be(s) != 1) { + stbi__rewind( s ); + return 0; + } + stbi__skip(s, 6); + channelCount = stbi__get16be(s); + if (channelCount < 0 || channelCount > 16) { + stbi__rewind( s ); + return 0; + } + STBI_NOTUSED(stbi__get32be(s)); + STBI_NOTUSED(stbi__get32be(s)); + depth = stbi__get16be(s); + if (depth != 16) { + stbi__rewind( s ); + return 0; + } + return 1; +} +#endif + +#ifndef STBI_NO_PIC +static int stbi__pic_info(stbi__context *s, int *x, int *y, int *comp) +{ + int act_comp=0,num_packets=0,chained,dummy; + stbi__pic_packet packets[10]; + + if (!x) x = &dummy; + if (!y) y = &dummy; + if (!comp) comp = &dummy; + + if (!stbi__pic_is4(s,"\x53\x80\xF6\x34")) { + stbi__rewind(s); + return 0; + } + + stbi__skip(s, 88); + + *x = stbi__get16be(s); + *y = stbi__get16be(s); + if (stbi__at_eof(s)) { + stbi__rewind( s); + return 0; + } + if ( (*x) != 0 && (1 << 28) / (*x) < (*y)) { + stbi__rewind( s ); + return 0; + } + + stbi__skip(s, 8); + + do { + stbi__pic_packet *packet; + + if (num_packets==sizeof(packets)/sizeof(packets[0])) + return 0; + + packet = &packets[num_packets++]; + chained = stbi__get8(s); + packet->size = stbi__get8(s); + packet->type = stbi__get8(s); + packet->channel = stbi__get8(s); + act_comp |= packet->channel; + + if (stbi__at_eof(s)) { + stbi__rewind( s ); + return 0; + } + if (packet->size != 8) { + stbi__rewind( s ); + return 0; + } + } while (chained); + + *comp = (act_comp & 0x10 ? 4 : 3); + + return 1; +} +#endif + +// ************************************************************************************************* +// Portable Gray Map and Portable Pixel Map loader +// by Ken Miller +// +// PGM: http://netpbm.sourceforge.net/doc/pgm.html +// PPM: http://netpbm.sourceforge.net/doc/ppm.html +// +// Known limitations: +// Does not support comments in the header section +// Does not support ASCII image data (formats P2 and P3) + +#ifndef STBI_NO_PNM + +static int stbi__pnm_test(stbi__context *s) +{ + char p, t; + p = (char) stbi__get8(s); + t = (char) stbi__get8(s); + if (p != 'P' || (t != '5' && t != '6')) { + stbi__rewind( s ); + return 0; + } + return 1; +} + +static void *stbi__pnm_load(stbi__context *s, int *x, int *y, int *comp, int req_comp, stbi__result_info *ri) +{ + stbi_uc *out; + STBI_NOTUSED(ri); + + ri->bits_per_channel = stbi__pnm_info(s, (int *)&s->img_x, (int *)&s->img_y, (int *)&s->img_n); + if (ri->bits_per_channel == 0) + return 0; + + if (s->img_y > STBI_MAX_DIMENSIONS) return stbi__errpuc("too large","Very large image (corrupt?)"); + if (s->img_x > STBI_MAX_DIMENSIONS) return stbi__errpuc("too large","Very large image (corrupt?)"); + + *x = s->img_x; + *y = s->img_y; + if (comp) *comp = s->img_n; + + if (!stbi__mad4sizes_valid(s->img_n, s->img_x, s->img_y, ri->bits_per_channel / 8, 0)) + return stbi__errpuc("too large", "PNM too large"); + + out = (stbi_uc *) stbi__malloc_mad4(s->img_n, s->img_x, s->img_y, ri->bits_per_channel / 8, 0); + if (!out) return stbi__errpuc("outofmem", "Out of memory"); + stbi__getn(s, out, s->img_n * s->img_x * s->img_y * (ri->bits_per_channel / 8)); + + if (req_comp && req_comp != s->img_n) { + out = stbi__convert_format(out, s->img_n, req_comp, s->img_x, s->img_y); + if (out == NULL) return out; // stbi__convert_format frees input on failure + } + return out; +} + +static int stbi__pnm_isspace(char c) +{ + return c == ' ' || c == '\t' || c == '\n' || c == '\v' || c == '\f' || c == '\r'; +} + +static void stbi__pnm_skip_whitespace(stbi__context *s, char *c) +{ + for (;;) { + while (!stbi__at_eof(s) && stbi__pnm_isspace(*c)) + *c = (char) stbi__get8(s); + + if (stbi__at_eof(s) || *c != '#') + break; + + while (!stbi__at_eof(s) && *c != '\n' && *c != '\r' ) + *c = (char) stbi__get8(s); + } +} + +static int stbi__pnm_isdigit(char c) +{ + return c >= '0' && c <= '9'; +} + +static int stbi__pnm_getinteger(stbi__context *s, char *c) +{ + int value = 0; + + while (!stbi__at_eof(s) && stbi__pnm_isdigit(*c)) { + value = value*10 + (*c - '0'); + *c = (char) stbi__get8(s); + } + + return value; +} + +static int stbi__pnm_info(stbi__context *s, int *x, int *y, int *comp) +{ + int maxv, dummy; + char c, p, t; + + if (!x) x = &dummy; + if (!y) y = &dummy; + if (!comp) comp = &dummy; + + stbi__rewind(s); + + // Get identifier + p = (char) stbi__get8(s); + t = (char) stbi__get8(s); + if (p != 'P' || (t != '5' && t != '6')) { + stbi__rewind(s); + return 0; + } + + *comp = (t == '6') ? 3 : 1; // '5' is 1-component .pgm; '6' is 3-component .ppm + + c = (char) stbi__get8(s); + stbi__pnm_skip_whitespace(s, &c); + + *x = stbi__pnm_getinteger(s, &c); // read width + stbi__pnm_skip_whitespace(s, &c); + + *y = stbi__pnm_getinteger(s, &c); // read height + stbi__pnm_skip_whitespace(s, &c); + + maxv = stbi__pnm_getinteger(s, &c); // read max value + if (maxv > 65535) + return stbi__err("max value > 65535", "PPM image supports only 8-bit and 16-bit images"); + else if (maxv > 255) + return 16; + else + return 8; +} + +static int stbi__pnm_is16(stbi__context *s) +{ + if (stbi__pnm_info(s, NULL, NULL, NULL) == 16) + return 1; + return 0; +} +#endif + +static int stbi__info_main(stbi__context *s, int *x, int *y, int *comp) +{ + #ifndef STBI_NO_JPEG + if (stbi__jpeg_info(s, x, y, comp)) return 1; + #endif + + #ifndef STBI_NO_PNG + if (stbi__png_info(s, x, y, comp)) return 1; + #endif + + #ifndef STBI_NO_GIF + if (stbi__gif_info(s, x, y, comp)) return 1; + #endif + + #ifndef STBI_NO_BMP + if (stbi__bmp_info(s, x, y, comp)) return 1; + #endif + + #ifndef STBI_NO_PSD + if (stbi__psd_info(s, x, y, comp)) return 1; + #endif + + #ifndef STBI_NO_PIC + if (stbi__pic_info(s, x, y, comp)) return 1; + #endif + + #ifndef STBI_NO_PNM + if (stbi__pnm_info(s, x, y, comp)) return 1; + #endif + + #ifndef STBI_NO_HDR + if (stbi__hdr_info(s, x, y, comp)) return 1; + #endif + + // test tga last because it's a crappy test! + #ifndef STBI_NO_TGA + if (stbi__tga_info(s, x, y, comp)) + return 1; + #endif + return stbi__err("unknown image type", "Image not of any known type, or corrupt"); +} + +static int stbi__is_16_main(stbi__context *s) +{ + #ifndef STBI_NO_PNG + if (stbi__png_is16(s)) return 1; + #endif + + #ifndef STBI_NO_PSD + if (stbi__psd_is16(s)) return 1; + #endif + + #ifndef STBI_NO_PNM + if (stbi__pnm_is16(s)) return 1; + #endif + return 0; +} + +#ifndef STBI_NO_STDIO +STBIDEF int stbi_info(char const *filename, int *x, int *y, int *comp) +{ + FILE *f = stbi__fopen(filename, "rb"); + int result; + if (!f) return stbi__err("can't fopen", "Unable to open file"); + result = stbi_info_from_file(f, x, y, comp); + fclose(f); + return result; +} + +STBIDEF int stbi_info_from_file(FILE *f, int *x, int *y, int *comp) +{ + int r; + stbi__context s; + long pos = ftell(f); + stbi__start_file(&s, f); + r = stbi__info_main(&s,x,y,comp); + fseek(f,pos,SEEK_SET); + return r; +} + +STBIDEF int stbi_is_16_bit(char const *filename) +{ + FILE *f = stbi__fopen(filename, "rb"); + int result; + if (!f) return stbi__err("can't fopen", "Unable to open file"); + result = stbi_is_16_bit_from_file(f); + fclose(f); + return result; +} + +STBIDEF int stbi_is_16_bit_from_file(FILE *f) +{ + int r; + stbi__context s; + long pos = ftell(f); + stbi__start_file(&s, f); + r = stbi__is_16_main(&s); + fseek(f,pos,SEEK_SET); + return r; +} +#endif // !STBI_NO_STDIO + +STBIDEF int stbi_info_from_memory(stbi_uc const *buffer, int len, int *x, int *y, int *comp) +{ + stbi__context s; + stbi__start_mem(&s,buffer,len); + return stbi__info_main(&s,x,y,comp); +} + +STBIDEF int stbi_info_from_callbacks(stbi_io_callbacks const *c, void *user, int *x, int *y, int *comp) +{ + stbi__context s; + stbi__start_callbacks(&s, (stbi_io_callbacks *) c, user); + return stbi__info_main(&s,x,y,comp); +} + +STBIDEF int stbi_is_16_bit_from_memory(stbi_uc const *buffer, int len) +{ + stbi__context s; + stbi__start_mem(&s,buffer,len); + return stbi__is_16_main(&s); +} + +STBIDEF int stbi_is_16_bit_from_callbacks(stbi_io_callbacks const *c, void *user) +{ + stbi__context s; + stbi__start_callbacks(&s, (stbi_io_callbacks *) c, user); + return stbi__is_16_main(&s); +} + +#endif // STB_IMAGE_IMPLEMENTATION + +/* + revision history: + 2.20 (2019-02-07) support utf8 filenames in Windows; fix warnings and platform ifdefs + 2.19 (2018-02-11) fix warning + 2.18 (2018-01-30) fix warnings + 2.17 (2018-01-29) change sbti__shiftsigned to avoid clang -O2 bug + 1-bit BMP + *_is_16_bit api + avoid warnings + 2.16 (2017-07-23) all functions have 16-bit variants; + STBI_NO_STDIO works again; + compilation fixes; + fix rounding in unpremultiply; + optimize vertical flip; + disable raw_len validation; + documentation fixes + 2.15 (2017-03-18) fix png-1,2,4 bug; now all Imagenet JPGs decode; + warning fixes; disable run-time SSE detection on gcc; + uniform handling of optional "return" values; + thread-safe initialization of zlib tables + 2.14 (2017-03-03) remove deprecated STBI_JPEG_OLD; fixes for Imagenet JPGs + 2.13 (2016-11-29) add 16-bit API, only supported for PNG right now + 2.12 (2016-04-02) fix typo in 2.11 PSD fix that caused crashes + 2.11 (2016-04-02) allocate large structures on the stack + remove white matting for transparent PSD + fix reported channel count for PNG & BMP + re-enable SSE2 in non-gcc 64-bit + support RGB-formatted JPEG + read 16-bit PNGs (only as 8-bit) + 2.10 (2016-01-22) avoid warning introduced in 2.09 by STBI_REALLOC_SIZED + 2.09 (2016-01-16) allow comments in PNM files + 16-bit-per-pixel TGA (not bit-per-component) + info() for TGA could break due to .hdr handling + info() for BMP to shares code instead of sloppy parse + can use STBI_REALLOC_SIZED if allocator doesn't support realloc + code cleanup + 2.08 (2015-09-13) fix to 2.07 cleanup, reading RGB PSD as RGBA + 2.07 (2015-09-13) fix compiler warnings + partial animated GIF support + limited 16-bpc PSD support + #ifdef unused functions + bug with < 92 byte PIC,PNM,HDR,TGA + 2.06 (2015-04-19) fix bug where PSD returns wrong '*comp' value + 2.05 (2015-04-19) fix bug in progressive JPEG handling, fix warning + 2.04 (2015-04-15) try to re-enable SIMD on MinGW 64-bit + 2.03 (2015-04-12) extra corruption checking (mmozeiko) + stbi_set_flip_vertically_on_load (nguillemot) + fix NEON support; fix mingw support + 2.02 (2015-01-19) fix incorrect assert, fix warning + 2.01 (2015-01-17) fix various warnings; suppress SIMD on gcc 32-bit without -msse2 + 2.00b (2014-12-25) fix STBI_MALLOC in progressive JPEG + 2.00 (2014-12-25) optimize JPG, including x86 SSE2 & NEON SIMD (ryg) + progressive JPEG (stb) + PGM/PPM support (Ken Miller) + STBI_MALLOC,STBI_REALLOC,STBI_FREE + GIF bugfix -- seemingly never worked + STBI_NO_*, STBI_ONLY_* + 1.48 (2014-12-14) fix incorrectly-named assert() + 1.47 (2014-12-14) 1/2/4-bit PNG support, both direct and paletted (Omar Cornut & stb) + optimize PNG (ryg) + fix bug in interlaced PNG with user-specified channel count (stb) + 1.46 (2014-08-26) + fix broken tRNS chunk (colorkey-style transparency) in non-paletted PNG + 1.45 (2014-08-16) + fix MSVC-ARM internal compiler error by wrapping malloc + 1.44 (2014-08-07) + various warning fixes from Ronny Chevalier + 1.43 (2014-07-15) + fix MSVC-only compiler problem in code changed in 1.42 + 1.42 (2014-07-09) + don't define _CRT_SECURE_NO_WARNINGS (affects user code) + fixes to stbi__cleanup_jpeg path + added STBI_ASSERT to avoid requiring assert.h + 1.41 (2014-06-25) + fix search&replace from 1.36 that messed up comments/error messages + 1.40 (2014-06-22) + fix gcc struct-initialization warning + 1.39 (2014-06-15) + fix to TGA optimization when req_comp != number of components in TGA; + fix to GIF loading because BMP wasn't rewinding (whoops, no GIFs in my test suite) + add support for BMP version 5 (more ignored fields) + 1.38 (2014-06-06) + suppress MSVC warnings on integer casts truncating values + fix accidental rename of 'skip' field of I/O + 1.37 (2014-06-04) + remove duplicate typedef + 1.36 (2014-06-03) + convert to header file single-file library + if de-iphone isn't set, load iphone images color-swapped instead of returning NULL + 1.35 (2014-05-27) + various warnings + fix broken STBI_SIMD path + fix bug where stbi_load_from_file no longer left file pointer in correct place + fix broken non-easy path for 32-bit BMP (possibly never used) + TGA optimization by Arseny Kapoulkine + 1.34 (unknown) + use STBI_NOTUSED in stbi__resample_row_generic(), fix one more leak in tga failure case + 1.33 (2011-07-14) + make stbi_is_hdr work in STBI_NO_HDR (as specified), minor compiler-friendly improvements + 1.32 (2011-07-13) + support for "info" function for all supported filetypes (SpartanJ) + 1.31 (2011-06-20) + a few more leak fixes, bug in PNG handling (SpartanJ) + 1.30 (2011-06-11) + added ability to load files via callbacks to accomidate custom input streams (Ben Wenger) + removed deprecated format-specific test/load functions + removed support for installable file formats (stbi_loader) -- would have been broken for IO callbacks anyway + error cases in bmp and tga give messages and don't leak (Raymond Barbiero, grisha) + fix inefficiency in decoding 32-bit BMP (David Woo) + 1.29 (2010-08-16) + various warning fixes from Aurelien Pocheville + 1.28 (2010-08-01) + fix bug in GIF palette transparency (SpartanJ) + 1.27 (2010-08-01) + cast-to-stbi_uc to fix warnings + 1.26 (2010-07-24) + fix bug in file buffering for PNG reported by SpartanJ + 1.25 (2010-07-17) + refix trans_data warning (Won Chun) + 1.24 (2010-07-12) + perf improvements reading from files on platforms with lock-heavy fgetc() + minor perf improvements for jpeg + deprecated type-specific functions so we'll get feedback if they're needed + attempt to fix trans_data warning (Won Chun) + 1.23 fixed bug in iPhone support + 1.22 (2010-07-10) + removed image *writing* support + stbi_info support from Jetro Lauha + GIF support from Jean-Marc Lienher + iPhone PNG-extensions from James Brown + warning-fixes from Nicolas Schulz and Janez Zemva (i.stbi__err. Janez (U+017D)emva) + 1.21 fix use of 'stbi_uc' in header (reported by jon blow) + 1.20 added support for Softimage PIC, by Tom Seddon + 1.19 bug in interlaced PNG corruption check (found by ryg) + 1.18 (2008-08-02) + fix a threading bug (local mutable static) + 1.17 support interlaced PNG + 1.16 major bugfix - stbi__convert_format converted one too many pixels + 1.15 initialize some fields for thread safety + 1.14 fix threadsafe conversion bug + header-file-only version (#define STBI_HEADER_FILE_ONLY before including) + 1.13 threadsafe + 1.12 const qualifiers in the API + 1.11 Support installable IDCT, colorspace conversion routines + 1.10 Fixes for 64-bit (don't use "unsigned long") + optimized upsampling by Fabian "ryg" Giesen + 1.09 Fix format-conversion for PSD code (bad global variables!) + 1.08 Thatcher Ulrich's PSD code integrated by Nicolas Schulz + 1.07 attempt to fix C++ warning/errors again + 1.06 attempt to fix C++ warning/errors again + 1.05 fix TGA loading to return correct *comp and use good luminance calc + 1.04 default float alpha is 1, not 255; use 'void *' for stbi_image_free + 1.03 bugfixes to STBI_NO_STDIO, STBI_NO_HDR + 1.02 support for (subset of) HDR files, float interface for preferred access to them + 1.01 fix bug: possible bug in handling right-side up bmps... not sure + fix bug: the stbi__bmp_load() and stbi__tga_load() functions didn't work at all + 1.00 interface to zlib that skips zlib header + 0.99 correct handling of alpha in palette + 0.98 TGA loader by lonesock; dynamically add loaders (untested) + 0.97 jpeg errors on too large a file; also catch another malloc failure + 0.96 fix detection of invalid v value - particleman@mollyrocket forum + 0.95 during header scan, seek to markers in case of padding + 0.94 STBI_NO_STDIO to disable stdio usage; rename all #defines the same + 0.93 handle jpegtran output; verbose errors + 0.92 read 4,8,16,24,32-bit BMP files of several formats + 0.91 output 24-bit Windows 3.0 BMP files + 0.90 fix a few more warnings; bump version number to approach 1.0 + 0.61 bugfixes due to Marc LeBlanc, Christopher Lloyd + 0.60 fix compiling as c++ + 0.59 fix warnings: merge Dave Moore's -Wall fixes + 0.58 fix bug: zlib uncompressed mode len/nlen was wrong endian + 0.57 fix bug: jpg last huffman symbol before marker was >9 bits but less than 16 available + 0.56 fix bug: zlib uncompressed mode len vs. nlen + 0.55 fix bug: restart_interval not initialized to 0 + 0.54 allow NULL for 'int *comp' + 0.53 fix bug in png 3->4; speedup png decoding + 0.52 png handles req_comp=3,4 directly; minor cleanup; jpeg comments + 0.51 obey req_comp requests, 1-component jpegs return as 1-component, + on 'test' only check type, not whether we support this variant + 0.50 (2006-11-19) + first released version +*/ + + +/* +------------------------------------------------------------------------------ +This software is available under 2 licenses -- choose whichever you prefer. +------------------------------------------------------------------------------ +ALTERNATIVE A - MIT License +Copyright (c) 2017 Sean Barrett +Permission is hereby granted, free of charge, to any person obtaining a copy of +this software and associated documentation files (the "Software"), to deal in +the Software without restriction, including without limitation the rights to +use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies +of the Software, and to permit persons to whom the Software is furnished to do +so, subject to the following conditions: +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. +------------------------------------------------------------------------------ +ALTERNATIVE B - Public Domain (www.unlicense.org) +This is free and unencumbered software released into the public domain. +Anyone is free to copy, modify, publish, use, compile, sell, or distribute this +software, either in source code form or as a compiled binary, for any purpose, +commercial or non-commercial, and by any means. +In jurisdictions that recognize copyright laws, the author or authors of this +software dedicate any and all copyright interest in the software to the public +domain. We make this dedication for the benefit of the public at large and to +the detriment of our heirs and successors. We intend this dedication to be an +overt act of relinquishment in perpetuity of all present and future rights to +this software under copyright law. +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN +ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +------------------------------------------------------------------------------ +*/ diff --git a/3rdparty/stb/stb_truetype.h b/3rdparty/stb/stb_truetype.h new file mode 100644 index 000000000..90a5c2e2b --- /dev/null +++ b/3rdparty/stb/stb_truetype.h @@ -0,0 +1,5079 @@ +// stb_truetype.h - v1.26 - public domain +// authored from 2009-2021 by Sean Barrett / RAD Game Tools +// +// ======================================================================= +// +// NO SECURITY GUARANTEE -- DO NOT USE THIS ON UNTRUSTED FONT FILES +// +// This library does no range checking of the offsets found in the file, +// meaning an attacker can use it to read arbitrary memory. +// +// ======================================================================= +// +// This library processes TrueType files: +// parse files +// extract glyph metrics +// extract glyph shapes +// render glyphs to one-channel bitmaps with antialiasing (box filter) +// render glyphs to one-channel SDF bitmaps (signed-distance field/function) +// +// Todo: +// non-MS cmaps +// crashproof on bad data +// hinting? (no longer patented) +// cleartype-style AA? +// optimize: use simple memory allocator for intermediates +// optimize: build edge-list directly from curves +// optimize: rasterize directly from curves? +// +// ADDITIONAL CONTRIBUTORS +// +// Mikko Mononen: compound shape support, more cmap formats +// Tor Andersson: kerning, subpixel rendering +// Dougall Johnson: OpenType / Type 2 font handling +// Daniel Ribeiro Maciel: basic GPOS-based kerning +// +// Misc other: +// Ryan Gordon +// Simon Glass +// github:IntellectualKitty +// Imanol Celaya +// Daniel Ribeiro Maciel +// +// Bug/warning reports/fixes: +// "Zer" on mollyrocket Fabian "ryg" Giesen github:NiLuJe +// Cass Everitt Martins Mozeiko github:aloucks +// stoiko (Haemimont Games) Cap Petschulat github:oyvindjam +// Brian Hook Omar Cornut github:vassvik +// Walter van Niftrik Ryan Griege +// David Gow Peter LaValle +// David Given Sergey Popov +// Ivan-Assen Ivanov Giumo X. Clanjor +// Anthony Pesch Higor Euripedes +// Johan Duparc Thomas Fields +// Hou Qiming Derek Vinyard +// Rob Loach Cort Stratton +// Kenney Phillis Jr. Brian Costabile +// Ken Voskuil (kaesve) Yakov Galka +// +// VERSION HISTORY +// +// 1.26 (2021-08-28) fix broken rasterizer +// 1.25 (2021-07-11) many fixes +// 1.24 (2020-02-05) fix warning +// 1.23 (2020-02-02) query SVG data for glyphs; query whole kerning table (but only kern not GPOS) +// 1.22 (2019-08-11) minimize missing-glyph duplication; fix kerning if both 'GPOS' and 'kern' are defined +// 1.21 (2019-02-25) fix warning +// 1.20 (2019-02-07) PackFontRange skips missing codepoints; GetScaleFontVMetrics() +// 1.19 (2018-02-11) GPOS kerning, STBTT_fmod +// 1.18 (2018-01-29) add missing function +// 1.17 (2017-07-23) make more arguments const; doc fix +// 1.16 (2017-07-12) SDF support +// 1.15 (2017-03-03) make more arguments const +// 1.14 (2017-01-16) num-fonts-in-TTC function +// 1.13 (2017-01-02) support OpenType fonts, certain Apple fonts +// 1.12 (2016-10-25) suppress warnings about casting away const with -Wcast-qual +// 1.11 (2016-04-02) fix unused-variable warning +// 1.10 (2016-04-02) user-defined fabs(); rare memory leak; remove duplicate typedef +// 1.09 (2016-01-16) warning fix; avoid crash on outofmem; use allocation userdata properly +// 1.08 (2015-09-13) document stbtt_Rasterize(); fixes for vertical & horizontal edges +// 1.07 (2015-08-01) allow PackFontRanges to accept arrays of sparse codepoints; +// variant PackFontRanges to pack and render in separate phases; +// fix stbtt_GetFontOFfsetForIndex (never worked for non-0 input?); +// fixed an assert() bug in the new rasterizer +// replace assert() with STBTT_assert() in new rasterizer +// +// Full history can be found at the end of this file. +// +// LICENSE +// +// See end of file for license information. +// +// USAGE +// +// Include this file in whatever places need to refer to it. In ONE C/C++ +// file, write: +// #define STB_TRUETYPE_IMPLEMENTATION +// before the #include of this file. This expands out the actual +// implementation into that C/C++ file. +// +// To make the implementation private to the file that generates the implementation, +// #define STBTT_STATIC +// +// Simple 3D API (don't ship this, but it's fine for tools and quick start) +// stbtt_BakeFontBitmap() -- bake a font to a bitmap for use as texture +// stbtt_GetBakedQuad() -- compute quad to draw for a given char +// +// Improved 3D API (more shippable): +// #include "stb_rect_pack.h" -- optional, but you really want it +// stbtt_PackBegin() +// stbtt_PackSetOversampling() -- for improved quality on small fonts +// stbtt_PackFontRanges() -- pack and renders +// stbtt_PackEnd() +// stbtt_GetPackedQuad() +// +// "Load" a font file from a memory buffer (you have to keep the buffer loaded) +// stbtt_InitFont() +// stbtt_GetFontOffsetForIndex() -- indexing for TTC font collections +// stbtt_GetNumberOfFonts() -- number of fonts for TTC font collections +// +// Render a unicode codepoint to a bitmap +// stbtt_GetCodepointBitmap() -- allocates and returns a bitmap +// stbtt_MakeCodepointBitmap() -- renders into bitmap you provide +// stbtt_GetCodepointBitmapBox() -- how big the bitmap must be +// +// Character advance/positioning +// stbtt_GetCodepointHMetrics() +// stbtt_GetFontVMetrics() +// stbtt_GetFontVMetricsOS2() +// stbtt_GetCodepointKernAdvance() +// +// Starting with version 1.06, the rasterizer was replaced with a new, +// faster and generally-more-precise rasterizer. The new rasterizer more +// accurately measures pixel coverage for anti-aliasing, except in the case +// where multiple shapes overlap, in which case it overestimates the AA pixel +// coverage. Thus, anti-aliasing of intersecting shapes may look wrong. If +// this turns out to be a problem, you can re-enable the old rasterizer with +// #define STBTT_RASTERIZER_VERSION 1 +// which will incur about a 15% speed hit. +// +// ADDITIONAL DOCUMENTATION +// +// Immediately after this block comment are a series of sample programs. +// +// After the sample programs is the "header file" section. This section +// includes documentation for each API function. +// +// Some important concepts to understand to use this library: +// +// Codepoint +// Characters are defined by unicode codepoints, e.g. 65 is +// uppercase A, 231 is lowercase c with a cedilla, 0x7e30 is +// the hiragana for "ma". +// +// Glyph +// A visual character shape (every codepoint is rendered as +// some glyph) +// +// Glyph index +// A font-specific integer ID representing a glyph +// +// Baseline +// Glyph shapes are defined relative to a baseline, which is the +// bottom of uppercase characters. Characters extend both above +// and below the baseline. +// +// Current Point +// As you draw text to the screen, you keep track of a "current point" +// which is the origin of each character. The current point's vertical +// position is the baseline. Even "baked fonts" use this model. +// +// Vertical Font Metrics +// The vertical qualities of the font, used to vertically position +// and space the characters. See docs for stbtt_GetFontVMetrics. +// +// Font Size in Pixels or Points +// The preferred interface for specifying font sizes in stb_truetype +// is to specify how tall the font's vertical extent should be in pixels. +// If that sounds good enough, skip the next paragraph. +// +// Most font APIs instead use "points", which are a common typographic +// measurement for describing font size, defined as 72 points per inch. +// stb_truetype provides a point API for compatibility. However, true +// "per inch" conventions don't make much sense on computer displays +// since different monitors have different number of pixels per +// inch. For example, Windows traditionally uses a convention that +// there are 96 pixels per inch, thus making 'inch' measurements have +// nothing to do with inches, and thus effectively defining a point to +// be 1.333 pixels. Additionally, the TrueType font data provides +// an explicit scale factor to scale a given font's glyphs to points, +// but the author has observed that this scale factor is often wrong +// for non-commercial fonts, thus making fonts scaled in points +// according to the TrueType spec incoherently sized in practice. +// +// DETAILED USAGE: +// +// Scale: +// Select how high you want the font to be, in points or pixels. +// Call ScaleForPixelHeight or ScaleForMappingEmToPixels to compute +// a scale factor SF that will be used by all other functions. +// +// Baseline: +// You need to select a y-coordinate that is the baseline of where +// your text will appear. Call GetFontBoundingBox to get the baseline-relative +// bounding box for all characters. SF*-y0 will be the distance in pixels +// that the worst-case character could extend above the baseline, so if +// you want the top edge of characters to appear at the top of the +// screen where y=0, then you would set the baseline to SF*-y0. +// +// Current point: +// Set the current point where the first character will appear. The +// first character could extend left of the current point; this is font +// dependent. You can either choose a current point that is the leftmost +// point and hope, or add some padding, or check the bounding box or +// left-side-bearing of the first character to be displayed and set +// the current point based on that. +// +// Displaying a character: +// Compute the bounding box of the character. It will contain signed values +// relative to . I.e. if it returns x0,y0,x1,y1, +// then the character should be displayed in the rectangle from +// to = 32 && *text < 128) { + stbtt_aligned_quad q; + stbtt_GetBakedQuad(cdata, 512,512, *text-32, &x,&y,&q,1);//1=opengl & d3d10+,0=d3d9 + glTexCoord2f(q.s0,q.t0); glVertex2f(q.x0,q.y0); + glTexCoord2f(q.s1,q.t0); glVertex2f(q.x1,q.y0); + glTexCoord2f(q.s1,q.t1); glVertex2f(q.x1,q.y1); + glTexCoord2f(q.s0,q.t1); glVertex2f(q.x0,q.y1); + } + ++text; + } + glEnd(); +} +#endif +// +// +////////////////////////////////////////////////////////////////////////////// +// +// Complete program (this compiles): get a single bitmap, print as ASCII art +// +#if 0 +#include +#define STB_TRUETYPE_IMPLEMENTATION // force following include to generate implementation +#include "stb_truetype.h" + +char ttf_buffer[1<<25]; + +int main(int argc, char **argv) +{ + stbtt_fontinfo font; + unsigned char *bitmap; + int w,h,i,j,c = (argc > 1 ? atoi(argv[1]) : 'a'), s = (argc > 2 ? atoi(argv[2]) : 20); + + fread(ttf_buffer, 1, 1<<25, fopen(argc > 3 ? argv[3] : "c:/windows/fonts/arialbd.ttf", "rb")); + + stbtt_InitFont(&font, ttf_buffer, stbtt_GetFontOffsetForIndex(ttf_buffer,0)); + bitmap = stbtt_GetCodepointBitmap(&font, 0,stbtt_ScaleForPixelHeight(&font, s), c, &w, &h, 0,0); + + for (j=0; j < h; ++j) { + for (i=0; i < w; ++i) + putchar(" .:ioVM@"[bitmap[j*w+i]>>5]); + putchar('\n'); + } + return 0; +} +#endif +// +// Output: +// +// .ii. +// @@@@@@. +// V@Mio@@o +// :i. V@V +// :oM@@M +// :@@@MM@M +// @@o o@M +// :@@. M@M +// @@@o@@@@ +// :M@@V:@@. +// +////////////////////////////////////////////////////////////////////////////// +// +// Complete program: print "Hello World!" banner, with bugs +// +#if 0 +char buffer[24<<20]; +unsigned char screen[20][79]; + +int main(int arg, char **argv) +{ + stbtt_fontinfo font; + int i,j,ascent,baseline,ch=0; + float scale, xpos=2; // leave a little padding in case the character extends left + char *text = "Heljo World!"; // intentionally misspelled to show 'lj' brokenness + + fread(buffer, 1, 1000000, fopen("c:/windows/fonts/arialbd.ttf", "rb")); + stbtt_InitFont(&font, buffer, 0); + + scale = stbtt_ScaleForPixelHeight(&font, 15); + stbtt_GetFontVMetrics(&font, &ascent,0,0); + baseline = (int) (ascent*scale); + + while (text[ch]) { + int advance,lsb,x0,y0,x1,y1; + float x_shift = xpos - (float) floor(xpos); + stbtt_GetCodepointHMetrics(&font, text[ch], &advance, &lsb); + stbtt_GetCodepointBitmapBoxSubpixel(&font, text[ch], scale,scale,x_shift,0, &x0,&y0,&x1,&y1); + stbtt_MakeCodepointBitmapSubpixel(&font, &screen[baseline + y0][(int) xpos + x0], x1-x0,y1-y0, 79, scale,scale,x_shift,0, text[ch]); + // note that this stomps the old data, so where character boxes overlap (e.g. 'lj') it's wrong + // because this API is really for baking character bitmaps into textures. if you want to render + // a sequence of characters, you really need to render each bitmap to a temp buffer, then + // "alpha blend" that into the working buffer + xpos += (advance * scale); + if (text[ch+1]) + xpos += scale*stbtt_GetCodepointKernAdvance(&font, text[ch],text[ch+1]); + ++ch; + } + + for (j=0; j < 20; ++j) { + for (i=0; i < 78; ++i) + putchar(" .:ioVM@"[screen[j][i]>>5]); + putchar('\n'); + } + + return 0; +} +#endif + + +////////////////////////////////////////////////////////////////////////////// +////////////////////////////////////////////////////////////////////////////// +//// +//// INTEGRATION WITH YOUR CODEBASE +//// +//// The following sections allow you to supply alternate definitions +//// of C library functions used by stb_truetype, e.g. if you don't +//// link with the C runtime library. + +#ifdef STB_TRUETYPE_IMPLEMENTATION + // #define your own (u)stbtt_int8/16/32 before including to override this + #ifndef stbtt_uint8 + typedef unsigned char stbtt_uint8; + typedef signed char stbtt_int8; + typedef unsigned short stbtt_uint16; + typedef signed short stbtt_int16; + typedef unsigned int stbtt_uint32; + typedef signed int stbtt_int32; + #endif + + typedef char stbtt__check_size32[sizeof(stbtt_int32)==4 ? 1 : -1]; + typedef char stbtt__check_size16[sizeof(stbtt_int16)==2 ? 1 : -1]; + + // e.g. #define your own STBTT_ifloor/STBTT_iceil() to avoid math.h + #ifndef STBTT_ifloor + #include + #define STBTT_ifloor(x) ((int) floor(x)) + #define STBTT_iceil(x) ((int) ceil(x)) + #endif + + #ifndef STBTT_sqrt + #include + #define STBTT_sqrt(x) sqrt(x) + #define STBTT_pow(x,y) pow(x,y) + #endif + + #ifndef STBTT_fmod + #include + #define STBTT_fmod(x,y) fmod(x,y) + #endif + + #ifndef STBTT_cos + #include + #define STBTT_cos(x) cos(x) + #define STBTT_acos(x) acos(x) + #endif + + #ifndef STBTT_fabs + #include + #define STBTT_fabs(x) fabs(x) + #endif + + // #define your own functions "STBTT_malloc" / "STBTT_free" to avoid malloc.h + #ifndef STBTT_malloc + #include + #define STBTT_malloc(x,u) ((void)(u),malloc(x)) + #define STBTT_free(x,u) ((void)(u),free(x)) + #endif + + #ifndef STBTT_assert + #include + #define STBTT_assert(x) assert(x) + #endif + + #ifndef STBTT_strlen + #include + #define STBTT_strlen(x) strlen(x) + #endif + + #ifndef STBTT_memcpy + #include + #define STBTT_memcpy memcpy + #define STBTT_memset memset + #endif +#endif + +/////////////////////////////////////////////////////////////////////////////// +/////////////////////////////////////////////////////////////////////////////// +//// +//// INTERFACE +//// +//// + +#ifndef __STB_INCLUDE_STB_TRUETYPE_H__ +#define __STB_INCLUDE_STB_TRUETYPE_H__ + +#ifdef STBTT_STATIC +#define STBTT_DEF static +#else +#define STBTT_DEF extern +#endif + +#ifdef __cplusplus +extern "C" { +#endif + +// private structure +typedef struct +{ + unsigned char *data; + int cursor; + int size; +} stbtt__buf; + +////////////////////////////////////////////////////////////////////////////// +// +// TEXTURE BAKING API +// +// If you use this API, you only have to call two functions ever. +// + +typedef struct +{ + unsigned short x0,y0,x1,y1; // coordinates of bbox in bitmap + float xoff,yoff,xadvance; +} stbtt_bakedchar; + +STBTT_DEF int stbtt_BakeFontBitmap(const unsigned char *data, int offset, // font location (use offset=0 for plain .ttf) + float pixel_height, // height of font in pixels + unsigned char *pixels, int pw, int ph, // bitmap to be filled in + int first_char, int num_chars, // characters to bake + stbtt_bakedchar *chardata); // you allocate this, it's num_chars long +// if return is positive, the first unused row of the bitmap +// if return is negative, returns the negative of the number of characters that fit +// if return is 0, no characters fit and no rows were used +// This uses a very crappy packing. + +typedef struct +{ + float x0,y0,s0,t0; // top-left + float x1,y1,s1,t1; // bottom-right +} stbtt_aligned_quad; + +STBTT_DEF void stbtt_GetBakedQuad(const stbtt_bakedchar *chardata, int pw, int ph, // same data as above + int char_index, // character to display + float *xpos, float *ypos, // pointers to current position in screen pixel space + stbtt_aligned_quad *q, // output: quad to draw + int opengl_fillrule); // true if opengl fill rule; false if DX9 or earlier +// Call GetBakedQuad with char_index = 'character - first_char', and it +// creates the quad you need to draw and advances the current position. +// +// The coordinate system used assumes y increases downwards. +// +// Characters will extend both above and below the current position; +// see discussion of "BASELINE" above. +// +// It's inefficient; you might want to c&p it and optimize it. + +STBTT_DEF void stbtt_GetScaledFontVMetrics(const unsigned char *fontdata, int index, float size, float *ascent, float *descent, float *lineGap); +// Query the font vertical metrics without having to create a font first. + + +////////////////////////////////////////////////////////////////////////////// +// +// NEW TEXTURE BAKING API +// +// This provides options for packing multiple fonts into one atlas, not +// perfectly but better than nothing. + +typedef struct +{ + unsigned short x0,y0,x1,y1; // coordinates of bbox in bitmap + float xoff,yoff,xadvance; + float xoff2,yoff2; +} stbtt_packedchar; + +typedef struct stbtt_pack_context stbtt_pack_context; +typedef struct stbtt_fontinfo stbtt_fontinfo; +#ifndef STB_RECT_PACK_VERSION +typedef struct stbrp_rect stbrp_rect; +#endif + +STBTT_DEF int stbtt_PackBegin(stbtt_pack_context *spc, unsigned char *pixels, int width, int height, int stride_in_bytes, int padding, void *alloc_context); +// Initializes a packing context stored in the passed-in stbtt_pack_context. +// Future calls using this context will pack characters into the bitmap passed +// in here: a 1-channel bitmap that is width * height. stride_in_bytes is +// the distance from one row to the next (or 0 to mean they are packed tightly +// together). "padding" is the amount of padding to leave between each +// character (normally you want '1' for bitmaps you'll use as textures with +// bilinear filtering). +// +// Returns 0 on failure, 1 on success. + +STBTT_DEF void stbtt_PackEnd (stbtt_pack_context *spc); +// Cleans up the packing context and frees all memory. + +#define STBTT_POINT_SIZE(x) (-(x)) + +STBTT_DEF int stbtt_PackFontRange(stbtt_pack_context *spc, const unsigned char *fontdata, int font_index, float font_size, + int first_unicode_char_in_range, int num_chars_in_range, stbtt_packedchar *chardata_for_range); +// Creates character bitmaps from the font_index'th font found in fontdata (use +// font_index=0 if you don't know what that is). It creates num_chars_in_range +// bitmaps for characters with unicode values starting at first_unicode_char_in_range +// and increasing. Data for how to render them is stored in chardata_for_range; +// pass these to stbtt_GetPackedQuad to get back renderable quads. +// +// font_size is the full height of the character from ascender to descender, +// as computed by stbtt_ScaleForPixelHeight. To use a point size as computed +// by stbtt_ScaleForMappingEmToPixels, wrap the point size in STBTT_POINT_SIZE() +// and pass that result as 'font_size': +// ..., 20 , ... // font max minus min y is 20 pixels tall +// ..., STBTT_POINT_SIZE(20), ... // 'M' is 20 pixels tall + +typedef struct +{ + float font_size; + int first_unicode_codepoint_in_range; // if non-zero, then the chars are continuous, and this is the first codepoint + int *array_of_unicode_codepoints; // if non-zero, then this is an array of unicode codepoints + int num_chars; + stbtt_packedchar *chardata_for_range; // output + unsigned char h_oversample, v_oversample; // don't set these, they're used internally +} stbtt_pack_range; + +STBTT_DEF int stbtt_PackFontRanges(stbtt_pack_context *spc, const unsigned char *fontdata, int font_index, stbtt_pack_range *ranges, int num_ranges); +// Creates character bitmaps from multiple ranges of characters stored in +// ranges. This will usually create a better-packed bitmap than multiple +// calls to stbtt_PackFontRange. Note that you can call this multiple +// times within a single PackBegin/PackEnd. + +STBTT_DEF void stbtt_PackSetOversampling(stbtt_pack_context *spc, unsigned int h_oversample, unsigned int v_oversample); +// Oversampling a font increases the quality by allowing higher-quality subpixel +// positioning, and is especially valuable at smaller text sizes. +// +// This function sets the amount of oversampling for all following calls to +// stbtt_PackFontRange(s) or stbtt_PackFontRangesGatherRects for a given +// pack context. The default (no oversampling) is achieved by h_oversample=1 +// and v_oversample=1. The total number of pixels required is +// h_oversample*v_oversample larger than the default; for example, 2x2 +// oversampling requires 4x the storage of 1x1. For best results, render +// oversampled textures with bilinear filtering. Look at the readme in +// stb/tests/oversample for information about oversampled fonts +// +// To use with PackFontRangesGather etc., you must set it before calls +// call to PackFontRangesGatherRects. + +STBTT_DEF void stbtt_PackSetSkipMissingCodepoints(stbtt_pack_context *spc, int skip); +// If skip != 0, this tells stb_truetype to skip any codepoints for which +// there is no corresponding glyph. If skip=0, which is the default, then +// codepoints without a glyph recived the font's "missing character" glyph, +// typically an empty box by convention. + +STBTT_DEF void stbtt_GetPackedQuad(const stbtt_packedchar *chardata, int pw, int ph, // same data as above + int char_index, // character to display + float *xpos, float *ypos, // pointers to current position in screen pixel space + stbtt_aligned_quad *q, // output: quad to draw + int align_to_integer); + +STBTT_DEF int stbtt_PackFontRangesGatherRects(stbtt_pack_context *spc, const stbtt_fontinfo *info, stbtt_pack_range *ranges, int num_ranges, stbrp_rect *rects); +STBTT_DEF void stbtt_PackFontRangesPackRects(stbtt_pack_context *spc, stbrp_rect *rects, int num_rects); +STBTT_DEF int stbtt_PackFontRangesRenderIntoRects(stbtt_pack_context *spc, const stbtt_fontinfo *info, stbtt_pack_range *ranges, int num_ranges, stbrp_rect *rects); +// Calling these functions in sequence is roughly equivalent to calling +// stbtt_PackFontRanges(). If you more control over the packing of multiple +// fonts, or if you want to pack custom data into a font texture, take a look +// at the source to of stbtt_PackFontRanges() and create a custom version +// using these functions, e.g. call GatherRects multiple times, +// building up a single array of rects, then call PackRects once, +// then call RenderIntoRects repeatedly. This may result in a +// better packing than calling PackFontRanges multiple times +// (or it may not). + +// this is an opaque structure that you shouldn't mess with which holds +// all the context needed from PackBegin to PackEnd. +struct stbtt_pack_context { + void *user_allocator_context; + void *pack_info; + int width; + int height; + int stride_in_bytes; + int padding; + int skip_missing; + unsigned int h_oversample, v_oversample; + unsigned char *pixels; + void *nodes; +}; + +////////////////////////////////////////////////////////////////////////////// +// +// FONT LOADING +// +// + +STBTT_DEF int stbtt_GetNumberOfFonts(const unsigned char *data); +// This function will determine the number of fonts in a font file. TrueType +// collection (.ttc) files may contain multiple fonts, while TrueType font +// (.ttf) files only contain one font. The number of fonts can be used for +// indexing with the previous function where the index is between zero and one +// less than the total fonts. If an error occurs, -1 is returned. + +STBTT_DEF int stbtt_GetFontOffsetForIndex(const unsigned char *data, int index); +// Each .ttf/.ttc file may have more than one font. Each font has a sequential +// index number starting from 0. Call this function to get the font offset for +// a given index; it returns -1 if the index is out of range. A regular .ttf +// file will only define one font and it always be at offset 0, so it will +// return '0' for index 0, and -1 for all other indices. + +// The following structure is defined publicly so you can declare one on +// the stack or as a global or etc, but you should treat it as opaque. +struct stbtt_fontinfo +{ + void * userdata; + unsigned char * data; // pointer to .ttf file + int fontstart; // offset of start of font + + int numGlyphs; // number of glyphs, needed for range checking + + int loca,head,glyf,hhea,hmtx,kern,gpos,svg; // table locations as offset from start of .ttf + int index_map; // a cmap mapping for our chosen character encoding + int indexToLocFormat; // format needed to map from glyph index to glyph + + stbtt__buf cff; // cff font data + stbtt__buf charstrings; // the charstring index + stbtt__buf gsubrs; // global charstring subroutines index + stbtt__buf subrs; // private charstring subroutines index + stbtt__buf fontdicts; // array of font dicts + stbtt__buf fdselect; // map from glyph to fontdict +}; + +STBTT_DEF int stbtt_InitFont(stbtt_fontinfo *info, const unsigned char *data, int offset); +// Given an offset into the file that defines a font, this function builds +// the necessary cached info for the rest of the system. You must allocate +// the stbtt_fontinfo yourself, and stbtt_InitFont will fill it out. You don't +// need to do anything special to free it, because the contents are pure +// value data with no additional data structures. Returns 0 on failure. + + +////////////////////////////////////////////////////////////////////////////// +// +// CHARACTER TO GLYPH-INDEX CONVERSIOn + +STBTT_DEF int stbtt_FindGlyphIndex(const stbtt_fontinfo *info, int unicode_codepoint); +// If you're going to perform multiple operations on the same character +// and you want a speed-up, call this function with the character you're +// going to process, then use glyph-based functions instead of the +// codepoint-based functions. +// Returns 0 if the character codepoint is not defined in the font. + + +////////////////////////////////////////////////////////////////////////////// +// +// CHARACTER PROPERTIES +// + +STBTT_DEF float stbtt_ScaleForPixelHeight(const stbtt_fontinfo *info, float pixels); +// computes a scale factor to produce a font whose "height" is 'pixels' tall. +// Height is measured as the distance from the highest ascender to the lowest +// descender; in other words, it's equivalent to calling stbtt_GetFontVMetrics +// and computing: +// scale = pixels / (ascent - descent) +// so if you prefer to measure height by the ascent only, use a similar calculation. + +STBTT_DEF float stbtt_ScaleForMappingEmToPixels(const stbtt_fontinfo *info, float pixels); +// computes a scale factor to produce a font whose EM size is mapped to +// 'pixels' tall. This is probably what traditional APIs compute, but +// I'm not positive. + +STBTT_DEF void stbtt_GetFontVMetrics(const stbtt_fontinfo *info, int *ascent, int *descent, int *lineGap); +// ascent is the coordinate above the baseline the font extends; descent +// is the coordinate below the baseline the font extends (i.e. it is typically negative) +// lineGap is the spacing between one row's descent and the next row's ascent... +// so you should advance the vertical position by "*ascent - *descent + *lineGap" +// these are expressed in unscaled coordinates, so you must multiply by +// the scale factor for a given size + +STBTT_DEF int stbtt_GetFontVMetricsOS2(const stbtt_fontinfo *info, int *typoAscent, int *typoDescent, int *typoLineGap); +// analogous to GetFontVMetrics, but returns the "typographic" values from the OS/2 +// table (specific to MS/Windows TTF files). +// +// Returns 1 on success (table present), 0 on failure. + +STBTT_DEF void stbtt_GetFontBoundingBox(const stbtt_fontinfo *info, int *x0, int *y0, int *x1, int *y1); +// the bounding box around all possible characters + +STBTT_DEF void stbtt_GetCodepointHMetrics(const stbtt_fontinfo *info, int codepoint, int *advanceWidth, int *leftSideBearing); +// leftSideBearing is the offset from the current horizontal position to the left edge of the character +// advanceWidth is the offset from the current horizontal position to the next horizontal position +// these are expressed in unscaled coordinates + +STBTT_DEF int stbtt_GetCodepointKernAdvance(const stbtt_fontinfo *info, int ch1, int ch2); +// an additional amount to add to the 'advance' value between ch1 and ch2 + +STBTT_DEF int stbtt_GetCodepointBox(const stbtt_fontinfo *info, int codepoint, int *x0, int *y0, int *x1, int *y1); +// Gets the bounding box of the visible part of the glyph, in unscaled coordinates + +STBTT_DEF void stbtt_GetGlyphHMetrics(const stbtt_fontinfo *info, int glyph_index, int *advanceWidth, int *leftSideBearing); +STBTT_DEF int stbtt_GetGlyphKernAdvance(const stbtt_fontinfo *info, int glyph1, int glyph2); +STBTT_DEF int stbtt_GetGlyphBox(const stbtt_fontinfo *info, int glyph_index, int *x0, int *y0, int *x1, int *y1); +// as above, but takes one or more glyph indices for greater efficiency + +typedef struct stbtt_kerningentry +{ + int glyph1; // use stbtt_FindGlyphIndex + int glyph2; + int advance; +} stbtt_kerningentry; + +STBTT_DEF int stbtt_GetKerningTableLength(const stbtt_fontinfo *info); +STBTT_DEF int stbtt_GetKerningTable(const stbtt_fontinfo *info, stbtt_kerningentry* table, int table_length); +// Retrieves a complete list of all of the kerning pairs provided by the font +// stbtt_GetKerningTable never writes more than table_length entries and returns how many entries it did write. +// The table will be sorted by (a.glyph1 == b.glyph1)?(a.glyph2 < b.glyph2):(a.glyph1 < b.glyph1) + +////////////////////////////////////////////////////////////////////////////// +// +// GLYPH SHAPES (you probably don't need these, but they have to go before +// the bitmaps for C declaration-order reasons) +// + +#ifndef STBTT_vmove // you can predefine these to use different values (but why?) + enum { + STBTT_vmove=1, + STBTT_vline, + STBTT_vcurve, + STBTT_vcubic + }; +#endif + +#ifndef stbtt_vertex // you can predefine this to use different values + // (we share this with other code at RAD) + #define stbtt_vertex_type short // can't use stbtt_int16 because that's not visible in the header file + typedef struct + { + stbtt_vertex_type x,y,cx,cy,cx1,cy1; + unsigned char type,padding; + } stbtt_vertex; +#endif + +STBTT_DEF int stbtt_IsGlyphEmpty(const stbtt_fontinfo *info, int glyph_index); +// returns non-zero if nothing is drawn for this glyph + +STBTT_DEF int stbtt_GetCodepointShape(const stbtt_fontinfo *info, int unicode_codepoint, stbtt_vertex **vertices); +STBTT_DEF int stbtt_GetGlyphShape(const stbtt_fontinfo *info, int glyph_index, stbtt_vertex **vertices); +// returns # of vertices and fills *vertices with the pointer to them +// these are expressed in "unscaled" coordinates +// +// The shape is a series of contours. Each one starts with +// a STBTT_moveto, then consists of a series of mixed +// STBTT_lineto and STBTT_curveto segments. A lineto +// draws a line from previous endpoint to its x,y; a curveto +// draws a quadratic bezier from previous endpoint to +// its x,y, using cx,cy as the bezier control point. + +STBTT_DEF void stbtt_FreeShape(const stbtt_fontinfo *info, stbtt_vertex *vertices); +// frees the data allocated above + +STBTT_DEF unsigned char *stbtt_FindSVGDoc(const stbtt_fontinfo *info, int gl); +STBTT_DEF int stbtt_GetCodepointSVG(const stbtt_fontinfo *info, int unicode_codepoint, const char **svg); +STBTT_DEF int stbtt_GetGlyphSVG(const stbtt_fontinfo *info, int gl, const char **svg); +// fills svg with the character's SVG data. +// returns data size or 0 if SVG not found. + +////////////////////////////////////////////////////////////////////////////// +// +// BITMAP RENDERING +// + +STBTT_DEF void stbtt_FreeBitmap(unsigned char *bitmap, void *userdata); +// frees the bitmap allocated below + +STBTT_DEF unsigned char *stbtt_GetCodepointBitmap(const stbtt_fontinfo *info, float scale_x, float scale_y, int codepoint, int *width, int *height, int *xoff, int *yoff); +// allocates a large-enough single-channel 8bpp bitmap and renders the +// specified character/glyph at the specified scale into it, with +// antialiasing. 0 is no coverage (transparent), 255 is fully covered (opaque). +// *width & *height are filled out with the width & height of the bitmap, +// which is stored left-to-right, top-to-bottom. +// +// xoff/yoff are the offset it pixel space from the glyph origin to the top-left of the bitmap + +STBTT_DEF unsigned char *stbtt_GetCodepointBitmapSubpixel(const stbtt_fontinfo *info, float scale_x, float scale_y, float shift_x, float shift_y, int codepoint, int *width, int *height, int *xoff, int *yoff); +// the same as stbtt_GetCodepoitnBitmap, but you can specify a subpixel +// shift for the character + +STBTT_DEF void stbtt_MakeCodepointBitmap(const stbtt_fontinfo *info, unsigned char *output, int out_w, int out_h, int out_stride, float scale_x, float scale_y, int codepoint); +// the same as stbtt_GetCodepointBitmap, but you pass in storage for the bitmap +// in the form of 'output', with row spacing of 'out_stride' bytes. the bitmap +// is clipped to out_w/out_h bytes. Call stbtt_GetCodepointBitmapBox to get the +// width and height and positioning info for it first. + +STBTT_DEF void stbtt_MakeCodepointBitmapSubpixel(const stbtt_fontinfo *info, unsigned char *output, int out_w, int out_h, int out_stride, float scale_x, float scale_y, float shift_x, float shift_y, int codepoint); +// same as stbtt_MakeCodepointBitmap, but you can specify a subpixel +// shift for the character + +STBTT_DEF void stbtt_MakeCodepointBitmapSubpixelPrefilter(const stbtt_fontinfo *info, unsigned char *output, int out_w, int out_h, int out_stride, float scale_x, float scale_y, float shift_x, float shift_y, int oversample_x, int oversample_y, float *sub_x, float *sub_y, int codepoint); +// same as stbtt_MakeCodepointBitmapSubpixel, but prefiltering +// is performed (see stbtt_PackSetOversampling) + +STBTT_DEF void stbtt_GetCodepointBitmapBox(const stbtt_fontinfo *font, int codepoint, float scale_x, float scale_y, int *ix0, int *iy0, int *ix1, int *iy1); +// get the bbox of the bitmap centered around the glyph origin; so the +// bitmap width is ix1-ix0, height is iy1-iy0, and location to place +// the bitmap top left is (leftSideBearing*scale,iy0). +// (Note that the bitmap uses y-increases-down, but the shape uses +// y-increases-up, so CodepointBitmapBox and CodepointBox are inverted.) + +STBTT_DEF void stbtt_GetCodepointBitmapBoxSubpixel(const stbtt_fontinfo *font, int codepoint, float scale_x, float scale_y, float shift_x, float shift_y, int *ix0, int *iy0, int *ix1, int *iy1); +// same as stbtt_GetCodepointBitmapBox, but you can specify a subpixel +// shift for the character + +// the following functions are equivalent to the above functions, but operate +// on glyph indices instead of Unicode codepoints (for efficiency) +STBTT_DEF unsigned char *stbtt_GetGlyphBitmap(const stbtt_fontinfo *info, float scale_x, float scale_y, int glyph, int *width, int *height, int *xoff, int *yoff); +STBTT_DEF unsigned char *stbtt_GetGlyphBitmapSubpixel(const stbtt_fontinfo *info, float scale_x, float scale_y, float shift_x, float shift_y, int glyph, int *width, int *height, int *xoff, int *yoff); +STBTT_DEF void stbtt_MakeGlyphBitmap(const stbtt_fontinfo *info, unsigned char *output, int out_w, int out_h, int out_stride, float scale_x, float scale_y, int glyph); +STBTT_DEF void stbtt_MakeGlyphBitmapSubpixel(const stbtt_fontinfo *info, unsigned char *output, int out_w, int out_h, int out_stride, float scale_x, float scale_y, float shift_x, float shift_y, int glyph); +STBTT_DEF void stbtt_MakeGlyphBitmapSubpixelPrefilter(const stbtt_fontinfo *info, unsigned char *output, int out_w, int out_h, int out_stride, float scale_x, float scale_y, float shift_x, float shift_y, int oversample_x, int oversample_y, float *sub_x, float *sub_y, int glyph); +STBTT_DEF void stbtt_GetGlyphBitmapBox(const stbtt_fontinfo *font, int glyph, float scale_x, float scale_y, int *ix0, int *iy0, int *ix1, int *iy1); +STBTT_DEF void stbtt_GetGlyphBitmapBoxSubpixel(const stbtt_fontinfo *font, int glyph, float scale_x, float scale_y,float shift_x, float shift_y, int *ix0, int *iy0, int *ix1, int *iy1); + + +// @TODO: don't expose this structure +typedef struct +{ + int w,h,stride; + unsigned char *pixels; +} stbtt__bitmap; + +// rasterize a shape with quadratic beziers into a bitmap +STBTT_DEF void stbtt_Rasterize(stbtt__bitmap *result, // 1-channel bitmap to draw into + float flatness_in_pixels, // allowable error of curve in pixels + stbtt_vertex *vertices, // array of vertices defining shape + int num_verts, // number of vertices in above array + float scale_x, float scale_y, // scale applied to input vertices + float shift_x, float shift_y, // translation applied to input vertices + int x_off, int y_off, // another translation applied to input + int invert, // if non-zero, vertically flip shape + void *userdata); // context for to STBTT_MALLOC + +////////////////////////////////////////////////////////////////////////////// +// +// Signed Distance Function (or Field) rendering + +STBTT_DEF void stbtt_FreeSDF(unsigned char *bitmap, void *userdata); +// frees the SDF bitmap allocated below + +STBTT_DEF unsigned char * stbtt_GetGlyphSDF(const stbtt_fontinfo *info, float scale, int glyph, int padding, unsigned char onedge_value, float pixel_dist_scale, int *width, int *height, int *xoff, int *yoff); +STBTT_DEF unsigned char * stbtt_GetCodepointSDF(const stbtt_fontinfo *info, float scale, int codepoint, int padding, unsigned char onedge_value, float pixel_dist_scale, int *width, int *height, int *xoff, int *yoff); +// These functions compute a discretized SDF field for a single character, suitable for storing +// in a single-channel texture, sampling with bilinear filtering, and testing against +// larger than some threshold to produce scalable fonts. +// info -- the font +// scale -- controls the size of the resulting SDF bitmap, same as it would be creating a regular bitmap +// glyph/codepoint -- the character to generate the SDF for +// padding -- extra "pixels" around the character which are filled with the distance to the character (not 0), +// which allows effects like bit outlines +// onedge_value -- value 0-255 to test the SDF against to reconstruct the character (i.e. the isocontour of the character) +// pixel_dist_scale -- what value the SDF should increase by when moving one SDF "pixel" away from the edge (on the 0..255 scale) +// if positive, > onedge_value is inside; if negative, < onedge_value is inside +// width,height -- output height & width of the SDF bitmap (including padding) +// xoff,yoff -- output origin of the character +// return value -- a 2D array of bytes 0..255, width*height in size +// +// pixel_dist_scale & onedge_value are a scale & bias that allows you to make +// optimal use of the limited 0..255 for your application, trading off precision +// and special effects. SDF values outside the range 0..255 are clamped to 0..255. +// +// Example: +// scale = stbtt_ScaleForPixelHeight(22) +// padding = 5 +// onedge_value = 180 +// pixel_dist_scale = 180/5.0 = 36.0 +// +// This will create an SDF bitmap in which the character is about 22 pixels +// high but the whole bitmap is about 22+5+5=32 pixels high. To produce a filled +// shape, sample the SDF at each pixel and fill the pixel if the SDF value +// is greater than or equal to 180/255. (You'll actually want to antialias, +// which is beyond the scope of this example.) Additionally, you can compute +// offset outlines (e.g. to stroke the character border inside & outside, +// or only outside). For example, to fill outside the character up to 3 SDF +// pixels, you would compare against (180-36.0*3)/255 = 72/255. The above +// choice of variables maps a range from 5 pixels outside the shape to +// 2 pixels inside the shape to 0..255; this is intended primarily for apply +// outside effects only (the interior range is needed to allow proper +// antialiasing of the font at *smaller* sizes) +// +// The function computes the SDF analytically at each SDF pixel, not by e.g. +// building a higher-res bitmap and approximating it. In theory the quality +// should be as high as possible for an SDF of this size & representation, but +// unclear if this is true in practice (perhaps building a higher-res bitmap +// and computing from that can allow drop-out prevention). +// +// The algorithm has not been optimized at all, so expect it to be slow +// if computing lots of characters or very large sizes. + + + +////////////////////////////////////////////////////////////////////////////// +// +// Finding the right font... +// +// You should really just solve this offline, keep your own tables +// of what font is what, and don't try to get it out of the .ttf file. +// That's because getting it out of the .ttf file is really hard, because +// the names in the file can appear in many possible encodings, in many +// possible languages, and e.g. if you need a case-insensitive comparison, +// the details of that depend on the encoding & language in a complex way +// (actually underspecified in truetype, but also gigantic). +// +// But you can use the provided functions in two possible ways: +// stbtt_FindMatchingFont() will use *case-sensitive* comparisons on +// unicode-encoded names to try to find the font you want; +// you can run this before calling stbtt_InitFont() +// +// stbtt_GetFontNameString() lets you get any of the various strings +// from the file yourself and do your own comparisons on them. +// You have to have called stbtt_InitFont() first. + + +STBTT_DEF int stbtt_FindMatchingFont(const unsigned char *fontdata, const char *name, int flags); +// returns the offset (not index) of the font that matches, or -1 if none +// if you use STBTT_MACSTYLE_DONTCARE, use a font name like "Arial Bold". +// if you use any other flag, use a font name like "Arial"; this checks +// the 'macStyle' header field; i don't know if fonts set this consistently +#define STBTT_MACSTYLE_DONTCARE 0 +#define STBTT_MACSTYLE_BOLD 1 +#define STBTT_MACSTYLE_ITALIC 2 +#define STBTT_MACSTYLE_UNDERSCORE 4 +#define STBTT_MACSTYLE_NONE 8 // <= not same as 0, this makes us check the bitfield is 0 + +STBTT_DEF int stbtt_CompareUTF8toUTF16_bigendian(const char *s1, int len1, const char *s2, int len2); +// returns 1/0 whether the first string interpreted as utf8 is identical to +// the second string interpreted as big-endian utf16... useful for strings from next func + +STBTT_DEF const char *stbtt_GetFontNameString(const stbtt_fontinfo *font, int *length, int platformID, int encodingID, int languageID, int nameID); +// returns the string (which may be big-endian double byte, e.g. for unicode) +// and puts the length in bytes in *length. +// +// some of the values for the IDs are below; for more see the truetype spec: +// http://developer.apple.com/textfonts/TTRefMan/RM06/Chap6name.html +// http://www.microsoft.com/typography/otspec/name.htm + +enum { // platformID + STBTT_PLATFORM_ID_UNICODE =0, + STBTT_PLATFORM_ID_MAC =1, + STBTT_PLATFORM_ID_ISO =2, + STBTT_PLATFORM_ID_MICROSOFT =3 +}; + +enum { // encodingID for STBTT_PLATFORM_ID_UNICODE + STBTT_UNICODE_EID_UNICODE_1_0 =0, + STBTT_UNICODE_EID_UNICODE_1_1 =1, + STBTT_UNICODE_EID_ISO_10646 =2, + STBTT_UNICODE_EID_UNICODE_2_0_BMP=3, + STBTT_UNICODE_EID_UNICODE_2_0_FULL=4 +}; + +enum { // encodingID for STBTT_PLATFORM_ID_MICROSOFT + STBTT_MS_EID_SYMBOL =0, + STBTT_MS_EID_UNICODE_BMP =1, + STBTT_MS_EID_SHIFTJIS =2, + STBTT_MS_EID_UNICODE_FULL =10 +}; + +enum { // encodingID for STBTT_PLATFORM_ID_MAC; same as Script Manager codes + STBTT_MAC_EID_ROMAN =0, STBTT_MAC_EID_ARABIC =4, + STBTT_MAC_EID_JAPANESE =1, STBTT_MAC_EID_HEBREW =5, + STBTT_MAC_EID_CHINESE_TRAD =2, STBTT_MAC_EID_GREEK =6, + STBTT_MAC_EID_KOREAN =3, STBTT_MAC_EID_RUSSIAN =7 +}; + +enum { // languageID for STBTT_PLATFORM_ID_MICROSOFT; same as LCID... + // problematic because there are e.g. 16 english LCIDs and 16 arabic LCIDs + STBTT_MS_LANG_ENGLISH =0x0409, STBTT_MS_LANG_ITALIAN =0x0410, + STBTT_MS_LANG_CHINESE =0x0804, STBTT_MS_LANG_JAPANESE =0x0411, + STBTT_MS_LANG_DUTCH =0x0413, STBTT_MS_LANG_KOREAN =0x0412, + STBTT_MS_LANG_FRENCH =0x040c, STBTT_MS_LANG_RUSSIAN =0x0419, + STBTT_MS_LANG_GERMAN =0x0407, STBTT_MS_LANG_SPANISH =0x0409, + STBTT_MS_LANG_HEBREW =0x040d, STBTT_MS_LANG_SWEDISH =0x041D +}; + +enum { // languageID for STBTT_PLATFORM_ID_MAC + STBTT_MAC_LANG_ENGLISH =0 , STBTT_MAC_LANG_JAPANESE =11, + STBTT_MAC_LANG_ARABIC =12, STBTT_MAC_LANG_KOREAN =23, + STBTT_MAC_LANG_DUTCH =4 , STBTT_MAC_LANG_RUSSIAN =32, + STBTT_MAC_LANG_FRENCH =1 , STBTT_MAC_LANG_SPANISH =6 , + STBTT_MAC_LANG_GERMAN =2 , STBTT_MAC_LANG_SWEDISH =5 , + STBTT_MAC_LANG_HEBREW =10, STBTT_MAC_LANG_CHINESE_SIMPLIFIED =33, + STBTT_MAC_LANG_ITALIAN =3 , STBTT_MAC_LANG_CHINESE_TRAD =19 +}; + +#ifdef __cplusplus +} +#endif + +#endif // __STB_INCLUDE_STB_TRUETYPE_H__ + +/////////////////////////////////////////////////////////////////////////////// +/////////////////////////////////////////////////////////////////////////////// +//// +//// IMPLEMENTATION +//// +//// + +#ifdef STB_TRUETYPE_IMPLEMENTATION + +#ifndef STBTT_MAX_OVERSAMPLE +#define STBTT_MAX_OVERSAMPLE 8 +#endif + +#if STBTT_MAX_OVERSAMPLE > 255 +#error "STBTT_MAX_OVERSAMPLE cannot be > 255" +#endif + +typedef int stbtt__test_oversample_pow2[(STBTT_MAX_OVERSAMPLE & (STBTT_MAX_OVERSAMPLE-1)) == 0 ? 1 : -1]; + +#ifndef STBTT_RASTERIZER_VERSION +#define STBTT_RASTERIZER_VERSION 2 +#endif + +#ifdef _MSC_VER +#define STBTT__NOTUSED(v) (void)(v) +#else +#define STBTT__NOTUSED(v) (void)sizeof(v) +#endif + +////////////////////////////////////////////////////////////////////////// +// +// stbtt__buf helpers to parse data from file +// + +static stbtt_uint8 stbtt__buf_get8(stbtt__buf *b) +{ + if (b->cursor >= b->size) + return 0; + return b->data[b->cursor++]; +} + +static stbtt_uint8 stbtt__buf_peek8(stbtt__buf *b) +{ + if (b->cursor >= b->size) + return 0; + return b->data[b->cursor]; +} + +static void stbtt__buf_seek(stbtt__buf *b, int o) +{ + STBTT_assert(!(o > b->size || o < 0)); + b->cursor = (o > b->size || o < 0) ? b->size : o; +} + +static void stbtt__buf_skip(stbtt__buf *b, int o) +{ + stbtt__buf_seek(b, b->cursor + o); +} + +static stbtt_uint32 stbtt__buf_get(stbtt__buf *b, int n) +{ + stbtt_uint32 v = 0; + int i; + STBTT_assert(n >= 1 && n <= 4); + for (i = 0; i < n; i++) + v = (v << 8) | stbtt__buf_get8(b); + return v; +} + +static stbtt__buf stbtt__new_buf(const void *p, size_t size) +{ + stbtt__buf r; + STBTT_assert(size < 0x40000000); + r.data = (stbtt_uint8*) p; + r.size = (int) size; + r.cursor = 0; + return r; +} + +#define stbtt__buf_get16(b) stbtt__buf_get((b), 2) +#define stbtt__buf_get32(b) stbtt__buf_get((b), 4) + +static stbtt__buf stbtt__buf_range(const stbtt__buf *b, int o, int s) +{ + stbtt__buf r = stbtt__new_buf(NULL, 0); + if (o < 0 || s < 0 || o > b->size || s > b->size - o) return r; + r.data = b->data + o; + r.size = s; + return r; +} + +static stbtt__buf stbtt__cff_get_index(stbtt__buf *b) +{ + int count, start, offsize; + start = b->cursor; + count = stbtt__buf_get16(b); + if (count) { + offsize = stbtt__buf_get8(b); + STBTT_assert(offsize >= 1 && offsize <= 4); + stbtt__buf_skip(b, offsize * count); + stbtt__buf_skip(b, stbtt__buf_get(b, offsize) - 1); + } + return stbtt__buf_range(b, start, b->cursor - start); +} + +static stbtt_uint32 stbtt__cff_int(stbtt__buf *b) +{ + int b0 = stbtt__buf_get8(b); + if (b0 >= 32 && b0 <= 246) return b0 - 139; + else if (b0 >= 247 && b0 <= 250) return (b0 - 247)*256 + stbtt__buf_get8(b) + 108; + else if (b0 >= 251 && b0 <= 254) return -(b0 - 251)*256 - stbtt__buf_get8(b) - 108; + else if (b0 == 28) return stbtt__buf_get16(b); + else if (b0 == 29) return stbtt__buf_get32(b); + STBTT_assert(0); + return 0; +} + +static void stbtt__cff_skip_operand(stbtt__buf *b) { + int v, b0 = stbtt__buf_peek8(b); + STBTT_assert(b0 >= 28); + if (b0 == 30) { + stbtt__buf_skip(b, 1); + while (b->cursor < b->size) { + v = stbtt__buf_get8(b); + if ((v & 0xF) == 0xF || (v >> 4) == 0xF) + break; + } + } else { + stbtt__cff_int(b); + } +} + +static stbtt__buf stbtt__dict_get(stbtt__buf *b, int key) +{ + stbtt__buf_seek(b, 0); + while (b->cursor < b->size) { + int start = b->cursor, end, op; + while (stbtt__buf_peek8(b) >= 28) + stbtt__cff_skip_operand(b); + end = b->cursor; + op = stbtt__buf_get8(b); + if (op == 12) op = stbtt__buf_get8(b) | 0x100; + if (op == key) return stbtt__buf_range(b, start, end-start); + } + return stbtt__buf_range(b, 0, 0); +} + +static void stbtt__dict_get_ints(stbtt__buf *b, int key, int outcount, stbtt_uint32 *out) +{ + int i; + stbtt__buf operands = stbtt__dict_get(b, key); + for (i = 0; i < outcount && operands.cursor < operands.size; i++) + out[i] = stbtt__cff_int(&operands); +} + +static int stbtt__cff_index_count(stbtt__buf *b) +{ + stbtt__buf_seek(b, 0); + return stbtt__buf_get16(b); +} + +static stbtt__buf stbtt__cff_index_get(stbtt__buf b, int i) +{ + int count, offsize, start, end; + stbtt__buf_seek(&b, 0); + count = stbtt__buf_get16(&b); + offsize = stbtt__buf_get8(&b); + STBTT_assert(i >= 0 && i < count); + STBTT_assert(offsize >= 1 && offsize <= 4); + stbtt__buf_skip(&b, i*offsize); + start = stbtt__buf_get(&b, offsize); + end = stbtt__buf_get(&b, offsize); + return stbtt__buf_range(&b, 2+(count+1)*offsize+start, end - start); +} + +////////////////////////////////////////////////////////////////////////// +// +// accessors to parse data from file +// + +// on platforms that don't allow misaligned reads, if we want to allow +// truetype fonts that aren't padded to alignment, define ALLOW_UNALIGNED_TRUETYPE + +#define ttBYTE(p) (* (stbtt_uint8 *) (p)) +#define ttCHAR(p) (* (stbtt_int8 *) (p)) +#define ttFixed(p) ttLONG(p) + +static stbtt_uint16 ttUSHORT(stbtt_uint8 *p) { return p[0]*256 + p[1]; } +static stbtt_int16 ttSHORT(stbtt_uint8 *p) { return p[0]*256 + p[1]; } +static stbtt_uint32 ttULONG(stbtt_uint8 *p) { return (p[0]<<24) + (p[1]<<16) + (p[2]<<8) + p[3]; } +static stbtt_int32 ttLONG(stbtt_uint8 *p) { return (p[0]<<24) + (p[1]<<16) + (p[2]<<8) + p[3]; } + +#define stbtt_tag4(p,c0,c1,c2,c3) ((p)[0] == (c0) && (p)[1] == (c1) && (p)[2] == (c2) && (p)[3] == (c3)) +#define stbtt_tag(p,str) stbtt_tag4(p,str[0],str[1],str[2],str[3]) + +static int stbtt__isfont(stbtt_uint8 *font) +{ + // check the version number + if (stbtt_tag4(font, '1',0,0,0)) return 1; // TrueType 1 + if (stbtt_tag(font, "typ1")) return 1; // TrueType with type 1 font -- we don't support this! + if (stbtt_tag(font, "OTTO")) return 1; // OpenType with CFF + if (stbtt_tag4(font, 0,1,0,0)) return 1; // OpenType 1.0 + if (stbtt_tag(font, "true")) return 1; // Apple specification for TrueType fonts + return 0; +} + +// @OPTIMIZE: binary search +static stbtt_uint32 stbtt__find_table(stbtt_uint8 *data, stbtt_uint32 fontstart, const char *tag) +{ + stbtt_int32 num_tables = ttUSHORT(data+fontstart+4); + stbtt_uint32 tabledir = fontstart + 12; + stbtt_int32 i; + for (i=0; i < num_tables; ++i) { + stbtt_uint32 loc = tabledir + 16*i; + if (stbtt_tag(data+loc+0, tag)) + return ttULONG(data+loc+8); + } + return 0; +} + +static int stbtt_GetFontOffsetForIndex_internal(unsigned char *font_collection, int index) +{ + // if it's just a font, there's only one valid index + if (stbtt__isfont(font_collection)) + return index == 0 ? 0 : -1; + + // check if it's a TTC + if (stbtt_tag(font_collection, "ttcf")) { + // version 1? + if (ttULONG(font_collection+4) == 0x00010000 || ttULONG(font_collection+4) == 0x00020000) { + stbtt_int32 n = ttLONG(font_collection+8); + if (index >= n) + return -1; + return ttULONG(font_collection+12+index*4); + } + } + return -1; +} + +static int stbtt_GetNumberOfFonts_internal(unsigned char *font_collection) +{ + // if it's just a font, there's only one valid font + if (stbtt__isfont(font_collection)) + return 1; + + // check if it's a TTC + if (stbtt_tag(font_collection, "ttcf")) { + // version 1? + if (ttULONG(font_collection+4) == 0x00010000 || ttULONG(font_collection+4) == 0x00020000) { + return ttLONG(font_collection+8); + } + } + return 0; +} + +static stbtt__buf stbtt__get_subrs(stbtt__buf cff, stbtt__buf fontdict) +{ + stbtt_uint32 subrsoff = 0, private_loc[2] = { 0, 0 }; + stbtt__buf pdict; + stbtt__dict_get_ints(&fontdict, 18, 2, private_loc); + if (!private_loc[1] || !private_loc[0]) return stbtt__new_buf(NULL, 0); + pdict = stbtt__buf_range(&cff, private_loc[1], private_loc[0]); + stbtt__dict_get_ints(&pdict, 19, 1, &subrsoff); + if (!subrsoff) return stbtt__new_buf(NULL, 0); + stbtt__buf_seek(&cff, private_loc[1]+subrsoff); + return stbtt__cff_get_index(&cff); +} + +// since most people won't use this, find this table the first time it's needed +static int stbtt__get_svg(stbtt_fontinfo *info) +{ + stbtt_uint32 t; + if (info->svg < 0) { + t = stbtt__find_table(info->data, info->fontstart, "SVG "); + if (t) { + stbtt_uint32 offset = ttULONG(info->data + t + 2); + info->svg = t + offset; + } else { + info->svg = 0; + } + } + return info->svg; +} + +static int stbtt_InitFont_internal(stbtt_fontinfo *info, unsigned char *data, int fontstart) +{ + stbtt_uint32 cmap, t; + stbtt_int32 i,numTables; + + info->data = data; + info->fontstart = fontstart; + info->cff = stbtt__new_buf(NULL, 0); + + cmap = stbtt__find_table(data, fontstart, "cmap"); // required + info->loca = stbtt__find_table(data, fontstart, "loca"); // required + info->head = stbtt__find_table(data, fontstart, "head"); // required + info->glyf = stbtt__find_table(data, fontstart, "glyf"); // required + info->hhea = stbtt__find_table(data, fontstart, "hhea"); // required + info->hmtx = stbtt__find_table(data, fontstart, "hmtx"); // required + info->kern = stbtt__find_table(data, fontstart, "kern"); // not required + info->gpos = stbtt__find_table(data, fontstart, "GPOS"); // not required + + if (!cmap || !info->head || !info->hhea || !info->hmtx) + return 0; + if (info->glyf) { + // required for truetype + if (!info->loca) return 0; + } else { + // initialization for CFF / Type2 fonts (OTF) + stbtt__buf b, topdict, topdictidx; + stbtt_uint32 cstype = 2, charstrings = 0, fdarrayoff = 0, fdselectoff = 0; + stbtt_uint32 cff; + + cff = stbtt__find_table(data, fontstart, "CFF "); + if (!cff) return 0; + + info->fontdicts = stbtt__new_buf(NULL, 0); + info->fdselect = stbtt__new_buf(NULL, 0); + + // @TODO this should use size from table (not 512MB) + info->cff = stbtt__new_buf(data+cff, 512*1024*1024); + b = info->cff; + + // read the header + stbtt__buf_skip(&b, 2); + stbtt__buf_seek(&b, stbtt__buf_get8(&b)); // hdrsize + + // @TODO the name INDEX could list multiple fonts, + // but we just use the first one. + stbtt__cff_get_index(&b); // name INDEX + topdictidx = stbtt__cff_get_index(&b); + topdict = stbtt__cff_index_get(topdictidx, 0); + stbtt__cff_get_index(&b); // string INDEX + info->gsubrs = stbtt__cff_get_index(&b); + + stbtt__dict_get_ints(&topdict, 17, 1, &charstrings); + stbtt__dict_get_ints(&topdict, 0x100 | 6, 1, &cstype); + stbtt__dict_get_ints(&topdict, 0x100 | 36, 1, &fdarrayoff); + stbtt__dict_get_ints(&topdict, 0x100 | 37, 1, &fdselectoff); + info->subrs = stbtt__get_subrs(b, topdict); + + // we only support Type 2 charstrings + if (cstype != 2) return 0; + if (charstrings == 0) return 0; + + if (fdarrayoff) { + // looks like a CID font + if (!fdselectoff) return 0; + stbtt__buf_seek(&b, fdarrayoff); + info->fontdicts = stbtt__cff_get_index(&b); + info->fdselect = stbtt__buf_range(&b, fdselectoff, b.size-fdselectoff); + } + + stbtt__buf_seek(&b, charstrings); + info->charstrings = stbtt__cff_get_index(&b); + } + + t = stbtt__find_table(data, fontstart, "maxp"); + if (t) + info->numGlyphs = ttUSHORT(data+t+4); + else + info->numGlyphs = 0xffff; + + info->svg = -1; + + // find a cmap encoding table we understand *now* to avoid searching + // later. (todo: could make this installable) + // the same regardless of glyph. + numTables = ttUSHORT(data + cmap + 2); + info->index_map = 0; + for (i=0; i < numTables; ++i) { + stbtt_uint32 encoding_record = cmap + 4 + 8 * i; + // find an encoding we understand: + switch(ttUSHORT(data+encoding_record)) { + case STBTT_PLATFORM_ID_MICROSOFT: + switch (ttUSHORT(data+encoding_record+2)) { + case STBTT_MS_EID_UNICODE_BMP: + case STBTT_MS_EID_UNICODE_FULL: + // MS/Unicode + info->index_map = cmap + ttULONG(data+encoding_record+4); + break; + } + break; + case STBTT_PLATFORM_ID_UNICODE: + // Mac/iOS has these + // all the encodingIDs are unicode, so we don't bother to check it + info->index_map = cmap + ttULONG(data+encoding_record+4); + break; + } + } + if (info->index_map == 0) + return 0; + + info->indexToLocFormat = ttUSHORT(data+info->head + 50); + return 1; +} + +STBTT_DEF int stbtt_FindGlyphIndex(const stbtt_fontinfo *info, int unicode_codepoint) +{ + stbtt_uint8 *data = info->data; + stbtt_uint32 index_map = info->index_map; + + stbtt_uint16 format = ttUSHORT(data + index_map + 0); + if (format == 0) { // apple byte encoding + stbtt_int32 bytes = ttUSHORT(data + index_map + 2); + if (unicode_codepoint < bytes-6) + return ttBYTE(data + index_map + 6 + unicode_codepoint); + return 0; + } else if (format == 6) { + stbtt_uint32 first = ttUSHORT(data + index_map + 6); + stbtt_uint32 count = ttUSHORT(data + index_map + 8); + if ((stbtt_uint32) unicode_codepoint >= first && (stbtt_uint32) unicode_codepoint < first+count) + return ttUSHORT(data + index_map + 10 + (unicode_codepoint - first)*2); + return 0; + } else if (format == 2) { + STBTT_assert(0); // @TODO: high-byte mapping for japanese/chinese/korean + return 0; + } else if (format == 4) { // standard mapping for windows fonts: binary search collection of ranges + stbtt_uint16 segcount = ttUSHORT(data+index_map+6) >> 1; + stbtt_uint16 searchRange = ttUSHORT(data+index_map+8) >> 1; + stbtt_uint16 entrySelector = ttUSHORT(data+index_map+10); + stbtt_uint16 rangeShift = ttUSHORT(data+index_map+12) >> 1; + + // do a binary search of the segments + stbtt_uint32 endCount = index_map + 14; + stbtt_uint32 search = endCount; + + if (unicode_codepoint > 0xffff) + return 0; + + // they lie from endCount .. endCount + segCount + // but searchRange is the nearest power of two, so... + if (unicode_codepoint >= ttUSHORT(data + search + rangeShift*2)) + search += rangeShift*2; + + // now decrement to bias correctly to find smallest + search -= 2; + while (entrySelector) { + stbtt_uint16 end; + searchRange >>= 1; + end = ttUSHORT(data + search + searchRange*2); + if (unicode_codepoint > end) + search += searchRange*2; + --entrySelector; + } + search += 2; + + { + stbtt_uint16 offset, start, last; + stbtt_uint16 item = (stbtt_uint16) ((search - endCount) >> 1); + + start = ttUSHORT(data + index_map + 14 + segcount*2 + 2 + 2*item); + last = ttUSHORT(data + endCount + 2*item); + if (unicode_codepoint < start || unicode_codepoint > last) + return 0; + + offset = ttUSHORT(data + index_map + 14 + segcount*6 + 2 + 2*item); + if (offset == 0) + return (stbtt_uint16) (unicode_codepoint + ttSHORT(data + index_map + 14 + segcount*4 + 2 + 2*item)); + + return ttUSHORT(data + offset + (unicode_codepoint-start)*2 + index_map + 14 + segcount*6 + 2 + 2*item); + } + } else if (format == 12 || format == 13) { + stbtt_uint32 ngroups = ttULONG(data+index_map+12); + stbtt_int32 low,high; + low = 0; high = (stbtt_int32)ngroups; + // Binary search the right group. + while (low < high) { + stbtt_int32 mid = low + ((high-low) >> 1); // rounds down, so low <= mid < high + stbtt_uint32 start_char = ttULONG(data+index_map+16+mid*12); + stbtt_uint32 end_char = ttULONG(data+index_map+16+mid*12+4); + if ((stbtt_uint32) unicode_codepoint < start_char) + high = mid; + else if ((stbtt_uint32) unicode_codepoint > end_char) + low = mid+1; + else { + stbtt_uint32 start_glyph = ttULONG(data+index_map+16+mid*12+8); + if (format == 12) + return start_glyph + unicode_codepoint-start_char; + else // format == 13 + return start_glyph; + } + } + return 0; // not found + } + // @TODO + STBTT_assert(0); + return 0; +} + +STBTT_DEF int stbtt_GetCodepointShape(const stbtt_fontinfo *info, int unicode_codepoint, stbtt_vertex **vertices) +{ + return stbtt_GetGlyphShape(info, stbtt_FindGlyphIndex(info, unicode_codepoint), vertices); +} + +static void stbtt_setvertex(stbtt_vertex *v, stbtt_uint8 type, stbtt_int32 x, stbtt_int32 y, stbtt_int32 cx, stbtt_int32 cy) +{ + v->type = type; + v->x = (stbtt_int16) x; + v->y = (stbtt_int16) y; + v->cx = (stbtt_int16) cx; + v->cy = (stbtt_int16) cy; +} + +static int stbtt__GetGlyfOffset(const stbtt_fontinfo *info, int glyph_index) +{ + int g1,g2; + + STBTT_assert(!info->cff.size); + + if (glyph_index >= info->numGlyphs) return -1; // glyph index out of range + if (info->indexToLocFormat >= 2) return -1; // unknown index->glyph map format + + if (info->indexToLocFormat == 0) { + g1 = info->glyf + ttUSHORT(info->data + info->loca + glyph_index * 2) * 2; + g2 = info->glyf + ttUSHORT(info->data + info->loca + glyph_index * 2 + 2) * 2; + } else { + g1 = info->glyf + ttULONG (info->data + info->loca + glyph_index * 4); + g2 = info->glyf + ttULONG (info->data + info->loca + glyph_index * 4 + 4); + } + + return g1==g2 ? -1 : g1; // if length is 0, return -1 +} + +static int stbtt__GetGlyphInfoT2(const stbtt_fontinfo *info, int glyph_index, int *x0, int *y0, int *x1, int *y1); + +STBTT_DEF int stbtt_GetGlyphBox(const stbtt_fontinfo *info, int glyph_index, int *x0, int *y0, int *x1, int *y1) +{ + if (info->cff.size) { + stbtt__GetGlyphInfoT2(info, glyph_index, x0, y0, x1, y1); + } else { + int g = stbtt__GetGlyfOffset(info, glyph_index); + if (g < 0) return 0; + + if (x0) *x0 = ttSHORT(info->data + g + 2); + if (y0) *y0 = ttSHORT(info->data + g + 4); + if (x1) *x1 = ttSHORT(info->data + g + 6); + if (y1) *y1 = ttSHORT(info->data + g + 8); + } + return 1; +} + +STBTT_DEF int stbtt_GetCodepointBox(const stbtt_fontinfo *info, int codepoint, int *x0, int *y0, int *x1, int *y1) +{ + return stbtt_GetGlyphBox(info, stbtt_FindGlyphIndex(info,codepoint), x0,y0,x1,y1); +} + +STBTT_DEF int stbtt_IsGlyphEmpty(const stbtt_fontinfo *info, int glyph_index) +{ + stbtt_int16 numberOfContours; + int g; + if (info->cff.size) + return stbtt__GetGlyphInfoT2(info, glyph_index, NULL, NULL, NULL, NULL) == 0; + g = stbtt__GetGlyfOffset(info, glyph_index); + if (g < 0) return 1; + numberOfContours = ttSHORT(info->data + g); + return numberOfContours == 0; +} + +static int stbtt__close_shape(stbtt_vertex *vertices, int num_vertices, int was_off, int start_off, + stbtt_int32 sx, stbtt_int32 sy, stbtt_int32 scx, stbtt_int32 scy, stbtt_int32 cx, stbtt_int32 cy) +{ + if (start_off) { + if (was_off) + stbtt_setvertex(&vertices[num_vertices++], STBTT_vcurve, (cx+scx)>>1, (cy+scy)>>1, cx,cy); + stbtt_setvertex(&vertices[num_vertices++], STBTT_vcurve, sx,sy,scx,scy); + } else { + if (was_off) + stbtt_setvertex(&vertices[num_vertices++], STBTT_vcurve,sx,sy,cx,cy); + else + stbtt_setvertex(&vertices[num_vertices++], STBTT_vline,sx,sy,0,0); + } + return num_vertices; +} + +static int stbtt__GetGlyphShapeTT(const stbtt_fontinfo *info, int glyph_index, stbtt_vertex **pvertices) +{ + stbtt_int16 numberOfContours; + stbtt_uint8 *endPtsOfContours; + stbtt_uint8 *data = info->data; + stbtt_vertex *vertices=0; + int num_vertices=0; + int g = stbtt__GetGlyfOffset(info, glyph_index); + + *pvertices = NULL; + + if (g < 0) return 0; + + numberOfContours = ttSHORT(data + g); + + if (numberOfContours > 0) { + stbtt_uint8 flags=0,flagcount; + stbtt_int32 ins, i,j=0,m,n, next_move, was_off=0, off, start_off=0; + stbtt_int32 x,y,cx,cy,sx,sy, scx,scy; + stbtt_uint8 *points; + endPtsOfContours = (data + g + 10); + ins = ttUSHORT(data + g + 10 + numberOfContours * 2); + points = data + g + 10 + numberOfContours * 2 + 2 + ins; + + n = 1+ttUSHORT(endPtsOfContours + numberOfContours*2-2); + + m = n + 2*numberOfContours; // a loose bound on how many vertices we might need + vertices = (stbtt_vertex *) STBTT_malloc(m * sizeof(vertices[0]), info->userdata); + if (vertices == 0) + return 0; + + next_move = 0; + flagcount=0; + + // in first pass, we load uninterpreted data into the allocated array + // above, shifted to the end of the array so we won't overwrite it when + // we create our final data starting from the front + + off = m - n; // starting offset for uninterpreted data, regardless of how m ends up being calculated + + // first load flags + + for (i=0; i < n; ++i) { + if (flagcount == 0) { + flags = *points++; + if (flags & 8) + flagcount = *points++; + } else + --flagcount; + vertices[off+i].type = flags; + } + + // now load x coordinates + x=0; + for (i=0; i < n; ++i) { + flags = vertices[off+i].type; + if (flags & 2) { + stbtt_int16 dx = *points++; + x += (flags & 16) ? dx : -dx; // ??? + } else { + if (!(flags & 16)) { + x = x + (stbtt_int16) (points[0]*256 + points[1]); + points += 2; + } + } + vertices[off+i].x = (stbtt_int16) x; + } + + // now load y coordinates + y=0; + for (i=0; i < n; ++i) { + flags = vertices[off+i].type; + if (flags & 4) { + stbtt_int16 dy = *points++; + y += (flags & 32) ? dy : -dy; // ??? + } else { + if (!(flags & 32)) { + y = y + (stbtt_int16) (points[0]*256 + points[1]); + points += 2; + } + } + vertices[off+i].y = (stbtt_int16) y; + } + + // now convert them to our format + num_vertices=0; + sx = sy = cx = cy = scx = scy = 0; + for (i=0; i < n; ++i) { + flags = vertices[off+i].type; + x = (stbtt_int16) vertices[off+i].x; + y = (stbtt_int16) vertices[off+i].y; + + if (next_move == i) { + if (i != 0) + num_vertices = stbtt__close_shape(vertices, num_vertices, was_off, start_off, sx,sy,scx,scy,cx,cy); + + // now start the new one + start_off = !(flags & 1); + if (start_off) { + // if we start off with an off-curve point, then when we need to find a point on the curve + // where we can start, and we need to save some state for when we wraparound. + scx = x; + scy = y; + if (!(vertices[off+i+1].type & 1)) { + // next point is also a curve point, so interpolate an on-point curve + sx = (x + (stbtt_int32) vertices[off+i+1].x) >> 1; + sy = (y + (stbtt_int32) vertices[off+i+1].y) >> 1; + } else { + // otherwise just use the next point as our start point + sx = (stbtt_int32) vertices[off+i+1].x; + sy = (stbtt_int32) vertices[off+i+1].y; + ++i; // we're using point i+1 as the starting point, so skip it + } + } else { + sx = x; + sy = y; + } + stbtt_setvertex(&vertices[num_vertices++], STBTT_vmove,sx,sy,0,0); + was_off = 0; + next_move = 1 + ttUSHORT(endPtsOfContours+j*2); + ++j; + } else { + if (!(flags & 1)) { // if it's a curve + if (was_off) // two off-curve control points in a row means interpolate an on-curve midpoint + stbtt_setvertex(&vertices[num_vertices++], STBTT_vcurve, (cx+x)>>1, (cy+y)>>1, cx, cy); + cx = x; + cy = y; + was_off = 1; + } else { + if (was_off) + stbtt_setvertex(&vertices[num_vertices++], STBTT_vcurve, x,y, cx, cy); + else + stbtt_setvertex(&vertices[num_vertices++], STBTT_vline, x,y,0,0); + was_off = 0; + } + } + } + num_vertices = stbtt__close_shape(vertices, num_vertices, was_off, start_off, sx,sy,scx,scy,cx,cy); + } else if (numberOfContours < 0) { + // Compound shapes. + int more = 1; + stbtt_uint8 *comp = data + g + 10; + num_vertices = 0; + vertices = 0; + while (more) { + stbtt_uint16 flags, gidx; + int comp_num_verts = 0, i; + stbtt_vertex *comp_verts = 0, *tmp = 0; + float mtx[6] = {1,0,0,1,0,0}, m, n; + + flags = ttSHORT(comp); comp+=2; + gidx = ttSHORT(comp); comp+=2; + + if (flags & 2) { // XY values + if (flags & 1) { // shorts + mtx[4] = ttSHORT(comp); comp+=2; + mtx[5] = ttSHORT(comp); comp+=2; + } else { + mtx[4] = ttCHAR(comp); comp+=1; + mtx[5] = ttCHAR(comp); comp+=1; + } + } + else { + // @TODO handle matching point + STBTT_assert(0); + } + if (flags & (1<<3)) { // WE_HAVE_A_SCALE + mtx[0] = mtx[3] = ttSHORT(comp)/16384.0f; comp+=2; + mtx[1] = mtx[2] = 0; + } else if (flags & (1<<6)) { // WE_HAVE_AN_X_AND_YSCALE + mtx[0] = ttSHORT(comp)/16384.0f; comp+=2; + mtx[1] = mtx[2] = 0; + mtx[3] = ttSHORT(comp)/16384.0f; comp+=2; + } else if (flags & (1<<7)) { // WE_HAVE_A_TWO_BY_TWO + mtx[0] = ttSHORT(comp)/16384.0f; comp+=2; + mtx[1] = ttSHORT(comp)/16384.0f; comp+=2; + mtx[2] = ttSHORT(comp)/16384.0f; comp+=2; + mtx[3] = ttSHORT(comp)/16384.0f; comp+=2; + } + + // Find transformation scales. + m = (float) STBTT_sqrt(mtx[0]*mtx[0] + mtx[1]*mtx[1]); + n = (float) STBTT_sqrt(mtx[2]*mtx[2] + mtx[3]*mtx[3]); + + // Get indexed glyph. + comp_num_verts = stbtt_GetGlyphShape(info, gidx, &comp_verts); + if (comp_num_verts > 0) { + // Transform vertices. + for (i = 0; i < comp_num_verts; ++i) { + stbtt_vertex* v = &comp_verts[i]; + stbtt_vertex_type x,y; + x=v->x; y=v->y; + v->x = (stbtt_vertex_type)(m * (mtx[0]*x + mtx[2]*y + mtx[4])); + v->y = (stbtt_vertex_type)(n * (mtx[1]*x + mtx[3]*y + mtx[5])); + x=v->cx; y=v->cy; + v->cx = (stbtt_vertex_type)(m * (mtx[0]*x + mtx[2]*y + mtx[4])); + v->cy = (stbtt_vertex_type)(n * (mtx[1]*x + mtx[3]*y + mtx[5])); + } + // Append vertices. + tmp = (stbtt_vertex*)STBTT_malloc((num_vertices+comp_num_verts)*sizeof(stbtt_vertex), info->userdata); + if (!tmp) { + if (vertices) STBTT_free(vertices, info->userdata); + if (comp_verts) STBTT_free(comp_verts, info->userdata); + return 0; + } + if (num_vertices > 0 && vertices) STBTT_memcpy(tmp, vertices, num_vertices*sizeof(stbtt_vertex)); + STBTT_memcpy(tmp+num_vertices, comp_verts, comp_num_verts*sizeof(stbtt_vertex)); + if (vertices) STBTT_free(vertices, info->userdata); + vertices = tmp; + STBTT_free(comp_verts, info->userdata); + num_vertices += comp_num_verts; + } + // More components ? + more = flags & (1<<5); + } + } else { + // numberOfCounters == 0, do nothing + } + + *pvertices = vertices; + return num_vertices; +} + +typedef struct +{ + int bounds; + int started; + float first_x, first_y; + float x, y; + stbtt_int32 min_x, max_x, min_y, max_y; + + stbtt_vertex *pvertices; + int num_vertices; +} stbtt__csctx; + +#define STBTT__CSCTX_INIT(bounds) {bounds,0, 0,0, 0,0, 0,0,0,0, NULL, 0} + +static void stbtt__track_vertex(stbtt__csctx *c, stbtt_int32 x, stbtt_int32 y) +{ + if (x > c->max_x || !c->started) c->max_x = x; + if (y > c->max_y || !c->started) c->max_y = y; + if (x < c->min_x || !c->started) c->min_x = x; + if (y < c->min_y || !c->started) c->min_y = y; + c->started = 1; +} + +static void stbtt__csctx_v(stbtt__csctx *c, stbtt_uint8 type, stbtt_int32 x, stbtt_int32 y, stbtt_int32 cx, stbtt_int32 cy, stbtt_int32 cx1, stbtt_int32 cy1) +{ + if (c->bounds) { + stbtt__track_vertex(c, x, y); + if (type == STBTT_vcubic) { + stbtt__track_vertex(c, cx, cy); + stbtt__track_vertex(c, cx1, cy1); + } + } else { + stbtt_setvertex(&c->pvertices[c->num_vertices], type, x, y, cx, cy); + c->pvertices[c->num_vertices].cx1 = (stbtt_int16) cx1; + c->pvertices[c->num_vertices].cy1 = (stbtt_int16) cy1; + } + c->num_vertices++; +} + +static void stbtt__csctx_close_shape(stbtt__csctx *ctx) +{ + if (ctx->first_x != ctx->x || ctx->first_y != ctx->y) + stbtt__csctx_v(ctx, STBTT_vline, (int)ctx->first_x, (int)ctx->first_y, 0, 0, 0, 0); +} + +static void stbtt__csctx_rmove_to(stbtt__csctx *ctx, float dx, float dy) +{ + stbtt__csctx_close_shape(ctx); + ctx->first_x = ctx->x = ctx->x + dx; + ctx->first_y = ctx->y = ctx->y + dy; + stbtt__csctx_v(ctx, STBTT_vmove, (int)ctx->x, (int)ctx->y, 0, 0, 0, 0); +} + +static void stbtt__csctx_rline_to(stbtt__csctx *ctx, float dx, float dy) +{ + ctx->x += dx; + ctx->y += dy; + stbtt__csctx_v(ctx, STBTT_vline, (int)ctx->x, (int)ctx->y, 0, 0, 0, 0); +} + +static void stbtt__csctx_rccurve_to(stbtt__csctx *ctx, float dx1, float dy1, float dx2, float dy2, float dx3, float dy3) +{ + float cx1 = ctx->x + dx1; + float cy1 = ctx->y + dy1; + float cx2 = cx1 + dx2; + float cy2 = cy1 + dy2; + ctx->x = cx2 + dx3; + ctx->y = cy2 + dy3; + stbtt__csctx_v(ctx, STBTT_vcubic, (int)ctx->x, (int)ctx->y, (int)cx1, (int)cy1, (int)cx2, (int)cy2); +} + +static stbtt__buf stbtt__get_subr(stbtt__buf idx, int n) +{ + int count = stbtt__cff_index_count(&idx); + int bias = 107; + if (count >= 33900) + bias = 32768; + else if (count >= 1240) + bias = 1131; + n += bias; + if (n < 0 || n >= count) + return stbtt__new_buf(NULL, 0); + return stbtt__cff_index_get(idx, n); +} + +static stbtt__buf stbtt__cid_get_glyph_subrs(const stbtt_fontinfo *info, int glyph_index) +{ + stbtt__buf fdselect = info->fdselect; + int nranges, start, end, v, fmt, fdselector = -1, i; + + stbtt__buf_seek(&fdselect, 0); + fmt = stbtt__buf_get8(&fdselect); + if (fmt == 0) { + // untested + stbtt__buf_skip(&fdselect, glyph_index); + fdselector = stbtt__buf_get8(&fdselect); + } else if (fmt == 3) { + nranges = stbtt__buf_get16(&fdselect); + start = stbtt__buf_get16(&fdselect); + for (i = 0; i < nranges; i++) { + v = stbtt__buf_get8(&fdselect); + end = stbtt__buf_get16(&fdselect); + if (glyph_index >= start && glyph_index < end) { + fdselector = v; + break; + } + start = end; + } + } + if (fdselector == -1) stbtt__new_buf(NULL, 0); + return stbtt__get_subrs(info->cff, stbtt__cff_index_get(info->fontdicts, fdselector)); +} + +static int stbtt__run_charstring(const stbtt_fontinfo *info, int glyph_index, stbtt__csctx *c) +{ + int in_header = 1, maskbits = 0, subr_stack_height = 0, sp = 0, v, i, b0; + int has_subrs = 0, clear_stack; + float s[48]; + stbtt__buf subr_stack[10], subrs = info->subrs, b; + float f; + +#define STBTT__CSERR(s) (0) + + // this currently ignores the initial width value, which isn't needed if we have hmtx + b = stbtt__cff_index_get(info->charstrings, glyph_index); + while (b.cursor < b.size) { + i = 0; + clear_stack = 1; + b0 = stbtt__buf_get8(&b); + switch (b0) { + // @TODO implement hinting + case 0x13: // hintmask + case 0x14: // cntrmask + if (in_header) + maskbits += (sp / 2); // implicit "vstem" + in_header = 0; + stbtt__buf_skip(&b, (maskbits + 7) / 8); + break; + + case 0x01: // hstem + case 0x03: // vstem + case 0x12: // hstemhm + case 0x17: // vstemhm + maskbits += (sp / 2); + break; + + case 0x15: // rmoveto + in_header = 0; + if (sp < 2) return STBTT__CSERR("rmoveto stack"); + stbtt__csctx_rmove_to(c, s[sp-2], s[sp-1]); + break; + case 0x04: // vmoveto + in_header = 0; + if (sp < 1) return STBTT__CSERR("vmoveto stack"); + stbtt__csctx_rmove_to(c, 0, s[sp-1]); + break; + case 0x16: // hmoveto + in_header = 0; + if (sp < 1) return STBTT__CSERR("hmoveto stack"); + stbtt__csctx_rmove_to(c, s[sp-1], 0); + break; + + case 0x05: // rlineto + if (sp < 2) return STBTT__CSERR("rlineto stack"); + for (; i + 1 < sp; i += 2) + stbtt__csctx_rline_to(c, s[i], s[i+1]); + break; + + // hlineto/vlineto and vhcurveto/hvcurveto alternate horizontal and vertical + // starting from a different place. + + case 0x07: // vlineto + if (sp < 1) return STBTT__CSERR("vlineto stack"); + goto vlineto; + case 0x06: // hlineto + if (sp < 1) return STBTT__CSERR("hlineto stack"); + for (;;) { + if (i >= sp) break; + stbtt__csctx_rline_to(c, s[i], 0); + i++; + vlineto: + if (i >= sp) break; + stbtt__csctx_rline_to(c, 0, s[i]); + i++; + } + break; + + case 0x1F: // hvcurveto + if (sp < 4) return STBTT__CSERR("hvcurveto stack"); + goto hvcurveto; + case 0x1E: // vhcurveto + if (sp < 4) return STBTT__CSERR("vhcurveto stack"); + for (;;) { + if (i + 3 >= sp) break; + stbtt__csctx_rccurve_to(c, 0, s[i], s[i+1], s[i+2], s[i+3], (sp - i == 5) ? s[i + 4] : 0.0f); + i += 4; + hvcurveto: + if (i + 3 >= sp) break; + stbtt__csctx_rccurve_to(c, s[i], 0, s[i+1], s[i+2], (sp - i == 5) ? s[i+4] : 0.0f, s[i+3]); + i += 4; + } + break; + + case 0x08: // rrcurveto + if (sp < 6) return STBTT__CSERR("rcurveline stack"); + for (; i + 5 < sp; i += 6) + stbtt__csctx_rccurve_to(c, s[i], s[i+1], s[i+2], s[i+3], s[i+4], s[i+5]); + break; + + case 0x18: // rcurveline + if (sp < 8) return STBTT__CSERR("rcurveline stack"); + for (; i + 5 < sp - 2; i += 6) + stbtt__csctx_rccurve_to(c, s[i], s[i+1], s[i+2], s[i+3], s[i+4], s[i+5]); + if (i + 1 >= sp) return STBTT__CSERR("rcurveline stack"); + stbtt__csctx_rline_to(c, s[i], s[i+1]); + break; + + case 0x19: // rlinecurve + if (sp < 8) return STBTT__CSERR("rlinecurve stack"); + for (; i + 1 < sp - 6; i += 2) + stbtt__csctx_rline_to(c, s[i], s[i+1]); + if (i + 5 >= sp) return STBTT__CSERR("rlinecurve stack"); + stbtt__csctx_rccurve_to(c, s[i], s[i+1], s[i+2], s[i+3], s[i+4], s[i+5]); + break; + + case 0x1A: // vvcurveto + case 0x1B: // hhcurveto + if (sp < 4) return STBTT__CSERR("(vv|hh)curveto stack"); + f = 0.0; + if (sp & 1) { f = s[i]; i++; } + for (; i + 3 < sp; i += 4) { + if (b0 == 0x1B) + stbtt__csctx_rccurve_to(c, s[i], f, s[i+1], s[i+2], s[i+3], 0.0); + else + stbtt__csctx_rccurve_to(c, f, s[i], s[i+1], s[i+2], 0.0, s[i+3]); + f = 0.0; + } + break; + + case 0x0A: // callsubr + if (!has_subrs) { + if (info->fdselect.size) + subrs = stbtt__cid_get_glyph_subrs(info, glyph_index); + has_subrs = 1; + } + // FALLTHROUGH + case 0x1D: // callgsubr + if (sp < 1) return STBTT__CSERR("call(g|)subr stack"); + v = (int) s[--sp]; + if (subr_stack_height >= 10) return STBTT__CSERR("recursion limit"); + subr_stack[subr_stack_height++] = b; + b = stbtt__get_subr(b0 == 0x0A ? subrs : info->gsubrs, v); + if (b.size == 0) return STBTT__CSERR("subr not found"); + b.cursor = 0; + clear_stack = 0; + break; + + case 0x0B: // return + if (subr_stack_height <= 0) return STBTT__CSERR("return outside subr"); + b = subr_stack[--subr_stack_height]; + clear_stack = 0; + break; + + case 0x0E: // endchar + stbtt__csctx_close_shape(c); + return 1; + + case 0x0C: { // two-byte escape + float dx1, dx2, dx3, dx4, dx5, dx6, dy1, dy2, dy3, dy4, dy5, dy6; + float dx, dy; + int b1 = stbtt__buf_get8(&b); + switch (b1) { + // @TODO These "flex" implementations ignore the flex-depth and resolution, + // and always draw beziers. + case 0x22: // hflex + if (sp < 7) return STBTT__CSERR("hflex stack"); + dx1 = s[0]; + dx2 = s[1]; + dy2 = s[2]; + dx3 = s[3]; + dx4 = s[4]; + dx5 = s[5]; + dx6 = s[6]; + stbtt__csctx_rccurve_to(c, dx1, 0, dx2, dy2, dx3, 0); + stbtt__csctx_rccurve_to(c, dx4, 0, dx5, -dy2, dx6, 0); + break; + + case 0x23: // flex + if (sp < 13) return STBTT__CSERR("flex stack"); + dx1 = s[0]; + dy1 = s[1]; + dx2 = s[2]; + dy2 = s[3]; + dx3 = s[4]; + dy3 = s[5]; + dx4 = s[6]; + dy4 = s[7]; + dx5 = s[8]; + dy5 = s[9]; + dx6 = s[10]; + dy6 = s[11]; + //fd is s[12] + stbtt__csctx_rccurve_to(c, dx1, dy1, dx2, dy2, dx3, dy3); + stbtt__csctx_rccurve_to(c, dx4, dy4, dx5, dy5, dx6, dy6); + break; + + case 0x24: // hflex1 + if (sp < 9) return STBTT__CSERR("hflex1 stack"); + dx1 = s[0]; + dy1 = s[1]; + dx2 = s[2]; + dy2 = s[3]; + dx3 = s[4]; + dx4 = s[5]; + dx5 = s[6]; + dy5 = s[7]; + dx6 = s[8]; + stbtt__csctx_rccurve_to(c, dx1, dy1, dx2, dy2, dx3, 0); + stbtt__csctx_rccurve_to(c, dx4, 0, dx5, dy5, dx6, -(dy1+dy2+dy5)); + break; + + case 0x25: // flex1 + if (sp < 11) return STBTT__CSERR("flex1 stack"); + dx1 = s[0]; + dy1 = s[1]; + dx2 = s[2]; + dy2 = s[3]; + dx3 = s[4]; + dy3 = s[5]; + dx4 = s[6]; + dy4 = s[7]; + dx5 = s[8]; + dy5 = s[9]; + dx6 = dy6 = s[10]; + dx = dx1+dx2+dx3+dx4+dx5; + dy = dy1+dy2+dy3+dy4+dy5; + if (STBTT_fabs(dx) > STBTT_fabs(dy)) + dy6 = -dy; + else + dx6 = -dx; + stbtt__csctx_rccurve_to(c, dx1, dy1, dx2, dy2, dx3, dy3); + stbtt__csctx_rccurve_to(c, dx4, dy4, dx5, dy5, dx6, dy6); + break; + + default: + return STBTT__CSERR("unimplemented"); + } + } break; + + default: + if (b0 != 255 && b0 != 28 && b0 < 32) + return STBTT__CSERR("reserved operator"); + + // push immediate + if (b0 == 255) { + f = (float)(stbtt_int32)stbtt__buf_get32(&b) / 0x10000; + } else { + stbtt__buf_skip(&b, -1); + f = (float)(stbtt_int16)stbtt__cff_int(&b); + } + if (sp >= 48) return STBTT__CSERR("push stack overflow"); + s[sp++] = f; + clear_stack = 0; + break; + } + if (clear_stack) sp = 0; + } + return STBTT__CSERR("no endchar"); + +#undef STBTT__CSERR +} + +static int stbtt__GetGlyphShapeT2(const stbtt_fontinfo *info, int glyph_index, stbtt_vertex **pvertices) +{ + // runs the charstring twice, once to count and once to output (to avoid realloc) + stbtt__csctx count_ctx = STBTT__CSCTX_INIT(1); + stbtt__csctx output_ctx = STBTT__CSCTX_INIT(0); + if (stbtt__run_charstring(info, glyph_index, &count_ctx)) { + *pvertices = (stbtt_vertex*)STBTT_malloc(count_ctx.num_vertices*sizeof(stbtt_vertex), info->userdata); + output_ctx.pvertices = *pvertices; + if (stbtt__run_charstring(info, glyph_index, &output_ctx)) { + STBTT_assert(output_ctx.num_vertices == count_ctx.num_vertices); + return output_ctx.num_vertices; + } + } + *pvertices = NULL; + return 0; +} + +static int stbtt__GetGlyphInfoT2(const stbtt_fontinfo *info, int glyph_index, int *x0, int *y0, int *x1, int *y1) +{ + stbtt__csctx c = STBTT__CSCTX_INIT(1); + int r = stbtt__run_charstring(info, glyph_index, &c); + if (x0) *x0 = r ? c.min_x : 0; + if (y0) *y0 = r ? c.min_y : 0; + if (x1) *x1 = r ? c.max_x : 0; + if (y1) *y1 = r ? c.max_y : 0; + return r ? c.num_vertices : 0; +} + +STBTT_DEF int stbtt_GetGlyphShape(const stbtt_fontinfo *info, int glyph_index, stbtt_vertex **pvertices) +{ + if (!info->cff.size) + return stbtt__GetGlyphShapeTT(info, glyph_index, pvertices); + else + return stbtt__GetGlyphShapeT2(info, glyph_index, pvertices); +} + +STBTT_DEF void stbtt_GetGlyphHMetrics(const stbtt_fontinfo *info, int glyph_index, int *advanceWidth, int *leftSideBearing) +{ + stbtt_uint16 numOfLongHorMetrics = ttUSHORT(info->data+info->hhea + 34); + if (glyph_index < numOfLongHorMetrics) { + if (advanceWidth) *advanceWidth = ttSHORT(info->data + info->hmtx + 4*glyph_index); + if (leftSideBearing) *leftSideBearing = ttSHORT(info->data + info->hmtx + 4*glyph_index + 2); + } else { + if (advanceWidth) *advanceWidth = ttSHORT(info->data + info->hmtx + 4*(numOfLongHorMetrics-1)); + if (leftSideBearing) *leftSideBearing = ttSHORT(info->data + info->hmtx + 4*numOfLongHorMetrics + 2*(glyph_index - numOfLongHorMetrics)); + } +} + +STBTT_DEF int stbtt_GetKerningTableLength(const stbtt_fontinfo *info) +{ + stbtt_uint8 *data = info->data + info->kern; + + // we only look at the first table. it must be 'horizontal' and format 0. + if (!info->kern) + return 0; + if (ttUSHORT(data+2) < 1) // number of tables, need at least 1 + return 0; + if (ttUSHORT(data+8) != 1) // horizontal flag must be set in format + return 0; + + return ttUSHORT(data+10); +} + +STBTT_DEF int stbtt_GetKerningTable(const stbtt_fontinfo *info, stbtt_kerningentry* table, int table_length) +{ + stbtt_uint8 *data = info->data + info->kern; + int k, length; + + // we only look at the first table. it must be 'horizontal' and format 0. + if (!info->kern) + return 0; + if (ttUSHORT(data+2) < 1) // number of tables, need at least 1 + return 0; + if (ttUSHORT(data+8) != 1) // horizontal flag must be set in format + return 0; + + length = ttUSHORT(data+10); + if (table_length < length) + length = table_length; + + for (k = 0; k < length; k++) + { + table[k].glyph1 = ttUSHORT(data+18+(k*6)); + table[k].glyph2 = ttUSHORT(data+20+(k*6)); + table[k].advance = ttSHORT(data+22+(k*6)); + } + + return length; +} + +static int stbtt__GetGlyphKernInfoAdvance(const stbtt_fontinfo *info, int glyph1, int glyph2) +{ + stbtt_uint8 *data = info->data + info->kern; + stbtt_uint32 needle, straw; + int l, r, m; + + // we only look at the first table. it must be 'horizontal' and format 0. + if (!info->kern) + return 0; + if (ttUSHORT(data+2) < 1) // number of tables, need at least 1 + return 0; + if (ttUSHORT(data+8) != 1) // horizontal flag must be set in format + return 0; + + l = 0; + r = ttUSHORT(data+10) - 1; + needle = glyph1 << 16 | glyph2; + while (l <= r) { + m = (l + r) >> 1; + straw = ttULONG(data+18+(m*6)); // note: unaligned read + if (needle < straw) + r = m - 1; + else if (needle > straw) + l = m + 1; + else + return ttSHORT(data+22+(m*6)); + } + return 0; +} + +static stbtt_int32 stbtt__GetCoverageIndex(stbtt_uint8 *coverageTable, int glyph) +{ + stbtt_uint16 coverageFormat = ttUSHORT(coverageTable); + switch (coverageFormat) { + case 1: { + stbtt_uint16 glyphCount = ttUSHORT(coverageTable + 2); + + // Binary search. + stbtt_int32 l=0, r=glyphCount-1, m; + int straw, needle=glyph; + while (l <= r) { + stbtt_uint8 *glyphArray = coverageTable + 4; + stbtt_uint16 glyphID; + m = (l + r) >> 1; + glyphID = ttUSHORT(glyphArray + 2 * m); + straw = glyphID; + if (needle < straw) + r = m - 1; + else if (needle > straw) + l = m + 1; + else { + return m; + } + } + break; + } + + case 2: { + stbtt_uint16 rangeCount = ttUSHORT(coverageTable + 2); + stbtt_uint8 *rangeArray = coverageTable + 4; + + // Binary search. + stbtt_int32 l=0, r=rangeCount-1, m; + int strawStart, strawEnd, needle=glyph; + while (l <= r) { + stbtt_uint8 *rangeRecord; + m = (l + r) >> 1; + rangeRecord = rangeArray + 6 * m; + strawStart = ttUSHORT(rangeRecord); + strawEnd = ttUSHORT(rangeRecord + 2); + if (needle < strawStart) + r = m - 1; + else if (needle > strawEnd) + l = m + 1; + else { + stbtt_uint16 startCoverageIndex = ttUSHORT(rangeRecord + 4); + return startCoverageIndex + glyph - strawStart; + } + } + break; + } + + default: return -1; // unsupported + } + + return -1; +} + +static stbtt_int32 stbtt__GetGlyphClass(stbtt_uint8 *classDefTable, int glyph) +{ + stbtt_uint16 classDefFormat = ttUSHORT(classDefTable); + switch (classDefFormat) + { + case 1: { + stbtt_uint16 startGlyphID = ttUSHORT(classDefTable + 2); + stbtt_uint16 glyphCount = ttUSHORT(classDefTable + 4); + stbtt_uint8 *classDef1ValueArray = classDefTable + 6; + + if (glyph >= startGlyphID && glyph < startGlyphID + glyphCount) + return (stbtt_int32)ttUSHORT(classDef1ValueArray + 2 * (glyph - startGlyphID)); + break; + } + + case 2: { + stbtt_uint16 classRangeCount = ttUSHORT(classDefTable + 2); + stbtt_uint8 *classRangeRecords = classDefTable + 4; + + // Binary search. + stbtt_int32 l=0, r=classRangeCount-1, m; + int strawStart, strawEnd, needle=glyph; + while (l <= r) { + stbtt_uint8 *classRangeRecord; + m = (l + r) >> 1; + classRangeRecord = classRangeRecords + 6 * m; + strawStart = ttUSHORT(classRangeRecord); + strawEnd = ttUSHORT(classRangeRecord + 2); + if (needle < strawStart) + r = m - 1; + else if (needle > strawEnd) + l = m + 1; + else + return (stbtt_int32)ttUSHORT(classRangeRecord + 4); + } + break; + } + + default: + return -1; // Unsupported definition type, return an error. + } + + // "All glyphs not assigned to a class fall into class 0". (OpenType spec) + return 0; +} + +// Define to STBTT_assert(x) if you want to break on unimplemented formats. +#define STBTT_GPOS_TODO_assert(x) + +static stbtt_int32 stbtt__GetGlyphGPOSInfoAdvance(const stbtt_fontinfo *info, int glyph1, int glyph2) +{ + stbtt_uint16 lookupListOffset; + stbtt_uint8 *lookupList; + stbtt_uint16 lookupCount; + stbtt_uint8 *data; + stbtt_int32 i, sti; + + if (!info->gpos) return 0; + + data = info->data + info->gpos; + + if (ttUSHORT(data+0) != 1) return 0; // Major version 1 + if (ttUSHORT(data+2) != 0) return 0; // Minor version 0 + + lookupListOffset = ttUSHORT(data+8); + lookupList = data + lookupListOffset; + lookupCount = ttUSHORT(lookupList); + + for (i=0; i= pairSetCount) return 0; + + needle=glyph2; + r=pairValueCount-1; + l=0; + + // Binary search. + while (l <= r) { + stbtt_uint16 secondGlyph; + stbtt_uint8 *pairValue; + m = (l + r) >> 1; + pairValue = pairValueArray + (2 + valueRecordPairSizeInBytes) * m; + secondGlyph = ttUSHORT(pairValue); + straw = secondGlyph; + if (needle < straw) + r = m - 1; + else if (needle > straw) + l = m + 1; + else { + stbtt_int16 xAdvance = ttSHORT(pairValue + 2); + return xAdvance; + } + } + } else + return 0; + break; + } + + case 2: { + stbtt_uint16 valueFormat1 = ttUSHORT(table + 4); + stbtt_uint16 valueFormat2 = ttUSHORT(table + 6); + if (valueFormat1 == 4 && valueFormat2 == 0) { // Support more formats? + stbtt_uint16 classDef1Offset = ttUSHORT(table + 8); + stbtt_uint16 classDef2Offset = ttUSHORT(table + 10); + int glyph1class = stbtt__GetGlyphClass(table + classDef1Offset, glyph1); + int glyph2class = stbtt__GetGlyphClass(table + classDef2Offset, glyph2); + + stbtt_uint16 class1Count = ttUSHORT(table + 12); + stbtt_uint16 class2Count = ttUSHORT(table + 14); + stbtt_uint8 *class1Records, *class2Records; + stbtt_int16 xAdvance; + + if (glyph1class < 0 || glyph1class >= class1Count) return 0; // malformed + if (glyph2class < 0 || glyph2class >= class2Count) return 0; // malformed + + class1Records = table + 16; + class2Records = class1Records + 2 * (glyph1class * class2Count); + xAdvance = ttSHORT(class2Records + 2 * glyph2class); + return xAdvance; + } else + return 0; + break; + } + + default: + return 0; // Unsupported position format + } + } + } + + return 0; +} + +STBTT_DEF int stbtt_GetGlyphKernAdvance(const stbtt_fontinfo *info, int g1, int g2) +{ + int xAdvance = 0; + + if (info->gpos) + xAdvance += stbtt__GetGlyphGPOSInfoAdvance(info, g1, g2); + else if (info->kern) + xAdvance += stbtt__GetGlyphKernInfoAdvance(info, g1, g2); + + return xAdvance; +} + +STBTT_DEF int stbtt_GetCodepointKernAdvance(const stbtt_fontinfo *info, int ch1, int ch2) +{ + if (!info->kern && !info->gpos) // if no kerning table, don't waste time looking up both codepoint->glyphs + return 0; + return stbtt_GetGlyphKernAdvance(info, stbtt_FindGlyphIndex(info,ch1), stbtt_FindGlyphIndex(info,ch2)); +} + +STBTT_DEF void stbtt_GetCodepointHMetrics(const stbtt_fontinfo *info, int codepoint, int *advanceWidth, int *leftSideBearing) +{ + stbtt_GetGlyphHMetrics(info, stbtt_FindGlyphIndex(info,codepoint), advanceWidth, leftSideBearing); +} + +STBTT_DEF void stbtt_GetFontVMetrics(const stbtt_fontinfo *info, int *ascent, int *descent, int *lineGap) +{ + if (ascent ) *ascent = ttSHORT(info->data+info->hhea + 4); + if (descent) *descent = ttSHORT(info->data+info->hhea + 6); + if (lineGap) *lineGap = ttSHORT(info->data+info->hhea + 8); +} + +STBTT_DEF int stbtt_GetFontVMetricsOS2(const stbtt_fontinfo *info, int *typoAscent, int *typoDescent, int *typoLineGap) +{ + int tab = stbtt__find_table(info->data, info->fontstart, "OS/2"); + if (!tab) + return 0; + if (typoAscent ) *typoAscent = ttSHORT(info->data+tab + 68); + if (typoDescent) *typoDescent = ttSHORT(info->data+tab + 70); + if (typoLineGap) *typoLineGap = ttSHORT(info->data+tab + 72); + return 1; +} + +STBTT_DEF void stbtt_GetFontBoundingBox(const stbtt_fontinfo *info, int *x0, int *y0, int *x1, int *y1) +{ + *x0 = ttSHORT(info->data + info->head + 36); + *y0 = ttSHORT(info->data + info->head + 38); + *x1 = ttSHORT(info->data + info->head + 40); + *y1 = ttSHORT(info->data + info->head + 42); +} + +STBTT_DEF float stbtt_ScaleForPixelHeight(const stbtt_fontinfo *info, float height) +{ + int fheight = ttSHORT(info->data + info->hhea + 4) - ttSHORT(info->data + info->hhea + 6); + return (float) height / fheight; +} + +STBTT_DEF float stbtt_ScaleForMappingEmToPixels(const stbtt_fontinfo *info, float pixels) +{ + int unitsPerEm = ttUSHORT(info->data + info->head + 18); + return pixels / unitsPerEm; +} + +STBTT_DEF void stbtt_FreeShape(const stbtt_fontinfo *info, stbtt_vertex *v) +{ + STBTT_free(v, info->userdata); +} + +STBTT_DEF stbtt_uint8 *stbtt_FindSVGDoc(const stbtt_fontinfo *info, int gl) +{ + int i; + stbtt_uint8 *data = info->data; + stbtt_uint8 *svg_doc_list = data + stbtt__get_svg((stbtt_fontinfo *) info); + + int numEntries = ttUSHORT(svg_doc_list); + stbtt_uint8 *svg_docs = svg_doc_list + 2; + + for(i=0; i= ttUSHORT(svg_doc)) && (gl <= ttUSHORT(svg_doc + 2))) + return svg_doc; + } + return 0; +} + +STBTT_DEF int stbtt_GetGlyphSVG(const stbtt_fontinfo *info, int gl, const char **svg) +{ + stbtt_uint8 *data = info->data; + stbtt_uint8 *svg_doc; + + if (info->svg == 0) + return 0; + + svg_doc = stbtt_FindSVGDoc(info, gl); + if (svg_doc != NULL) { + *svg = (char *) data + info->svg + ttULONG(svg_doc + 4); + return ttULONG(svg_doc + 8); + } else { + return 0; + } +} + +STBTT_DEF int stbtt_GetCodepointSVG(const stbtt_fontinfo *info, int unicode_codepoint, const char **svg) +{ + return stbtt_GetGlyphSVG(info, stbtt_FindGlyphIndex(info, unicode_codepoint), svg); +} + +////////////////////////////////////////////////////////////////////////////// +// +// antialiasing software rasterizer +// + +STBTT_DEF void stbtt_GetGlyphBitmapBoxSubpixel(const stbtt_fontinfo *font, int glyph, float scale_x, float scale_y,float shift_x, float shift_y, int *ix0, int *iy0, int *ix1, int *iy1) +{ + int x0=0,y0=0,x1,y1; // =0 suppresses compiler warning + if (!stbtt_GetGlyphBox(font, glyph, &x0,&y0,&x1,&y1)) { + // e.g. space character + if (ix0) *ix0 = 0; + if (iy0) *iy0 = 0; + if (ix1) *ix1 = 0; + if (iy1) *iy1 = 0; + } else { + // move to integral bboxes (treating pixels as little squares, what pixels get touched)? + if (ix0) *ix0 = STBTT_ifloor( x0 * scale_x + shift_x); + if (iy0) *iy0 = STBTT_ifloor(-y1 * scale_y + shift_y); + if (ix1) *ix1 = STBTT_iceil ( x1 * scale_x + shift_x); + if (iy1) *iy1 = STBTT_iceil (-y0 * scale_y + shift_y); + } +} + +STBTT_DEF void stbtt_GetGlyphBitmapBox(const stbtt_fontinfo *font, int glyph, float scale_x, float scale_y, int *ix0, int *iy0, int *ix1, int *iy1) +{ + stbtt_GetGlyphBitmapBoxSubpixel(font, glyph, scale_x, scale_y,0.0f,0.0f, ix0, iy0, ix1, iy1); +} + +STBTT_DEF void stbtt_GetCodepointBitmapBoxSubpixel(const stbtt_fontinfo *font, int codepoint, float scale_x, float scale_y, float shift_x, float shift_y, int *ix0, int *iy0, int *ix1, int *iy1) +{ + stbtt_GetGlyphBitmapBoxSubpixel(font, stbtt_FindGlyphIndex(font,codepoint), scale_x, scale_y,shift_x,shift_y, ix0,iy0,ix1,iy1); +} + +STBTT_DEF void stbtt_GetCodepointBitmapBox(const stbtt_fontinfo *font, int codepoint, float scale_x, float scale_y, int *ix0, int *iy0, int *ix1, int *iy1) +{ + stbtt_GetCodepointBitmapBoxSubpixel(font, codepoint, scale_x, scale_y,0.0f,0.0f, ix0,iy0,ix1,iy1); +} + +////////////////////////////////////////////////////////////////////////////// +// +// Rasterizer + +typedef struct stbtt__hheap_chunk +{ + struct stbtt__hheap_chunk *next; +} stbtt__hheap_chunk; + +typedef struct stbtt__hheap +{ + struct stbtt__hheap_chunk *head; + void *first_free; + int num_remaining_in_head_chunk; +} stbtt__hheap; + +static void *stbtt__hheap_alloc(stbtt__hheap *hh, size_t size, void *userdata) +{ + if (hh->first_free) { + void *p = hh->first_free; + hh->first_free = * (void **) p; + return p; + } else { + if (hh->num_remaining_in_head_chunk == 0) { + int count = (size < 32 ? 2000 : size < 128 ? 800 : 100); + stbtt__hheap_chunk *c = (stbtt__hheap_chunk *) STBTT_malloc(sizeof(stbtt__hheap_chunk) + size * count, userdata); + if (c == NULL) + return NULL; + c->next = hh->head; + hh->head = c; + hh->num_remaining_in_head_chunk = count; + } + --hh->num_remaining_in_head_chunk; + return (char *) (hh->head) + sizeof(stbtt__hheap_chunk) + size * hh->num_remaining_in_head_chunk; + } +} + +static void stbtt__hheap_free(stbtt__hheap *hh, void *p) +{ + *(void **) p = hh->first_free; + hh->first_free = p; +} + +static void stbtt__hheap_cleanup(stbtt__hheap *hh, void *userdata) +{ + stbtt__hheap_chunk *c = hh->head; + while (c) { + stbtt__hheap_chunk *n = c->next; + STBTT_free(c, userdata); + c = n; + } +} + +typedef struct stbtt__edge { + float x0,y0, x1,y1; + int invert; +} stbtt__edge; + + +typedef struct stbtt__active_edge +{ + struct stbtt__active_edge *next; + #if STBTT_RASTERIZER_VERSION==1 + int x,dx; + float ey; + int direction; + #elif STBTT_RASTERIZER_VERSION==2 + float fx,fdx,fdy; + float direction; + float sy; + float ey; + #else + #error "Unrecognized value of STBTT_RASTERIZER_VERSION" + #endif +} stbtt__active_edge; + +#if STBTT_RASTERIZER_VERSION == 1 +#define STBTT_FIXSHIFT 10 +#define STBTT_FIX (1 << STBTT_FIXSHIFT) +#define STBTT_FIXMASK (STBTT_FIX-1) + +static stbtt__active_edge *stbtt__new_active(stbtt__hheap *hh, stbtt__edge *e, int off_x, float start_point, void *userdata) +{ + stbtt__active_edge *z = (stbtt__active_edge *) stbtt__hheap_alloc(hh, sizeof(*z), userdata); + float dxdy = (e->x1 - e->x0) / (e->y1 - e->y0); + STBTT_assert(z != NULL); + if (!z) return z; + + // round dx down to avoid overshooting + if (dxdy < 0) + z->dx = -STBTT_ifloor(STBTT_FIX * -dxdy); + else + z->dx = STBTT_ifloor(STBTT_FIX * dxdy); + + z->x = STBTT_ifloor(STBTT_FIX * e->x0 + z->dx * (start_point - e->y0)); // use z->dx so when we offset later it's by the same amount + z->x -= off_x * STBTT_FIX; + + z->ey = e->y1; + z->next = 0; + z->direction = e->invert ? 1 : -1; + return z; +} +#elif STBTT_RASTERIZER_VERSION == 2 +static stbtt__active_edge *stbtt__new_active(stbtt__hheap *hh, stbtt__edge *e, int off_x, float start_point, void *userdata) +{ + stbtt__active_edge *z = (stbtt__active_edge *) stbtt__hheap_alloc(hh, sizeof(*z), userdata); + float dxdy = (e->x1 - e->x0) / (e->y1 - e->y0); + STBTT_assert(z != NULL); + //STBTT_assert(e->y0 <= start_point); + if (!z) return z; + z->fdx = dxdy; + z->fdy = dxdy != 0.0f ? (1.0f/dxdy) : 0.0f; + z->fx = e->x0 + dxdy * (start_point - e->y0); + z->fx -= off_x; + z->direction = e->invert ? 1.0f : -1.0f; + z->sy = e->y0; + z->ey = e->y1; + z->next = 0; + return z; +} +#else +#error "Unrecognized value of STBTT_RASTERIZER_VERSION" +#endif + +#if STBTT_RASTERIZER_VERSION == 1 +// note: this routine clips fills that extend off the edges... ideally this +// wouldn't happen, but it could happen if the truetype glyph bounding boxes +// are wrong, or if the user supplies a too-small bitmap +static void stbtt__fill_active_edges(unsigned char *scanline, int len, stbtt__active_edge *e, int max_weight) +{ + // non-zero winding fill + int x0=0, w=0; + + while (e) { + if (w == 0) { + // if we're currently at zero, we need to record the edge start point + x0 = e->x; w += e->direction; + } else { + int x1 = e->x; w += e->direction; + // if we went to zero, we need to draw + if (w == 0) { + int i = x0 >> STBTT_FIXSHIFT; + int j = x1 >> STBTT_FIXSHIFT; + + if (i < len && j >= 0) { + if (i == j) { + // x0,x1 are the same pixel, so compute combined coverage + scanline[i] = scanline[i] + (stbtt_uint8) ((x1 - x0) * max_weight >> STBTT_FIXSHIFT); + } else { + if (i >= 0) // add antialiasing for x0 + scanline[i] = scanline[i] + (stbtt_uint8) (((STBTT_FIX - (x0 & STBTT_FIXMASK)) * max_weight) >> STBTT_FIXSHIFT); + else + i = -1; // clip + + if (j < len) // add antialiasing for x1 + scanline[j] = scanline[j] + (stbtt_uint8) (((x1 & STBTT_FIXMASK) * max_weight) >> STBTT_FIXSHIFT); + else + j = len; // clip + + for (++i; i < j; ++i) // fill pixels between x0 and x1 + scanline[i] = scanline[i] + (stbtt_uint8) max_weight; + } + } + } + } + + e = e->next; + } +} + +static void stbtt__rasterize_sorted_edges(stbtt__bitmap *result, stbtt__edge *e, int n, int vsubsample, int off_x, int off_y, void *userdata) +{ + stbtt__hheap hh = { 0, 0, 0 }; + stbtt__active_edge *active = NULL; + int y,j=0; + int max_weight = (255 / vsubsample); // weight per vertical scanline + int s; // vertical subsample index + unsigned char scanline_data[512], *scanline; + + if (result->w > 512) + scanline = (unsigned char *) STBTT_malloc(result->w, userdata); + else + scanline = scanline_data; + + y = off_y * vsubsample; + e[n].y0 = (off_y + result->h) * (float) vsubsample + 1; + + while (j < result->h) { + STBTT_memset(scanline, 0, result->w); + for (s=0; s < vsubsample; ++s) { + // find center of pixel for this scanline + float scan_y = y + 0.5f; + stbtt__active_edge **step = &active; + + // update all active edges; + // remove all active edges that terminate before the center of this scanline + while (*step) { + stbtt__active_edge * z = *step; + if (z->ey <= scan_y) { + *step = z->next; // delete from list + STBTT_assert(z->direction); + z->direction = 0; + stbtt__hheap_free(&hh, z); + } else { + z->x += z->dx; // advance to position for current scanline + step = &((*step)->next); // advance through list + } + } + + // resort the list if needed + for(;;) { + int changed=0; + step = &active; + while (*step && (*step)->next) { + if ((*step)->x > (*step)->next->x) { + stbtt__active_edge *t = *step; + stbtt__active_edge *q = t->next; + + t->next = q->next; + q->next = t; + *step = q; + changed = 1; + } + step = &(*step)->next; + } + if (!changed) break; + } + + // insert all edges that start before the center of this scanline -- omit ones that also end on this scanline + while (e->y0 <= scan_y) { + if (e->y1 > scan_y) { + stbtt__active_edge *z = stbtt__new_active(&hh, e, off_x, scan_y, userdata); + if (z != NULL) { + // find insertion point + if (active == NULL) + active = z; + else if (z->x < active->x) { + // insert at front + z->next = active; + active = z; + } else { + // find thing to insert AFTER + stbtt__active_edge *p = active; + while (p->next && p->next->x < z->x) + p = p->next; + // at this point, p->next->x is NOT < z->x + z->next = p->next; + p->next = z; + } + } + } + ++e; + } + + // now process all active edges in XOR fashion + if (active) + stbtt__fill_active_edges(scanline, result->w, active, max_weight); + + ++y; + } + STBTT_memcpy(result->pixels + j * result->stride, scanline, result->w); + ++j; + } + + stbtt__hheap_cleanup(&hh, userdata); + + if (scanline != scanline_data) + STBTT_free(scanline, userdata); +} + +#elif STBTT_RASTERIZER_VERSION == 2 + +// the edge passed in here does not cross the vertical line at x or the vertical line at x+1 +// (i.e. it has already been clipped to those) +static void stbtt__handle_clipped_edge(float *scanline, int x, stbtt__active_edge *e, float x0, float y0, float x1, float y1) +{ + if (y0 == y1) return; + STBTT_assert(y0 < y1); + STBTT_assert(e->sy <= e->ey); + if (y0 > e->ey) return; + if (y1 < e->sy) return; + if (y0 < e->sy) { + x0 += (x1-x0) * (e->sy - y0) / (y1-y0); + y0 = e->sy; + } + if (y1 > e->ey) { + x1 += (x1-x0) * (e->ey - y1) / (y1-y0); + y1 = e->ey; + } + + if (x0 == x) + STBTT_assert(x1 <= x+1); + else if (x0 == x+1) + STBTT_assert(x1 >= x); + else if (x0 <= x) + STBTT_assert(x1 <= x); + else if (x0 >= x+1) + STBTT_assert(x1 >= x+1); + else + STBTT_assert(x1 >= x && x1 <= x+1); + + if (x0 <= x && x1 <= x) + scanline[x] += e->direction * (y1-y0); + else if (x0 >= x+1 && x1 >= x+1) + ; + else { + STBTT_assert(x0 >= x && x0 <= x+1 && x1 >= x && x1 <= x+1); + scanline[x] += e->direction * (y1-y0) * (1-((x0-x)+(x1-x))/2); // coverage = 1 - average x position + } +} + +static float stbtt__sized_trapezoid_area(float height, float top_width, float bottom_width) +{ + STBTT_assert(top_width >= 0); + STBTT_assert(bottom_width >= 0); + return (top_width + bottom_width) / 2.0f * height; +} + +static float stbtt__position_trapezoid_area(float height, float tx0, float tx1, float bx0, float bx1) +{ + return stbtt__sized_trapezoid_area(height, tx1 - tx0, bx1 - bx0); +} + +static float stbtt__sized_triangle_area(float height, float width) +{ + return height * width / 2; +} + +static void stbtt__fill_active_edges_new(float *scanline, float *scanline_fill, int len, stbtt__active_edge *e, float y_top) +{ + float y_bottom = y_top+1; + + while (e) { + // brute force every pixel + + // compute intersection points with top & bottom + STBTT_assert(e->ey >= y_top); + + if (e->fdx == 0) { + float x0 = e->fx; + if (x0 < len) { + if (x0 >= 0) { + stbtt__handle_clipped_edge(scanline,(int) x0,e, x0,y_top, x0,y_bottom); + stbtt__handle_clipped_edge(scanline_fill-1,(int) x0+1,e, x0,y_top, x0,y_bottom); + } else { + stbtt__handle_clipped_edge(scanline_fill-1,0,e, x0,y_top, x0,y_bottom); + } + } + } else { + float x0 = e->fx; + float dx = e->fdx; + float xb = x0 + dx; + float x_top, x_bottom; + float sy0,sy1; + float dy = e->fdy; + STBTT_assert(e->sy <= y_bottom && e->ey >= y_top); + + // compute endpoints of line segment clipped to this scanline (if the + // line segment starts on this scanline. x0 is the intersection of the + // line with y_top, but that may be off the line segment. + if (e->sy > y_top) { + x_top = x0 + dx * (e->sy - y_top); + sy0 = e->sy; + } else { + x_top = x0; + sy0 = y_top; + } + if (e->ey < y_bottom) { + x_bottom = x0 + dx * (e->ey - y_top); + sy1 = e->ey; + } else { + x_bottom = xb; + sy1 = y_bottom; + } + + if (x_top >= 0 && x_bottom >= 0 && x_top < len && x_bottom < len) { + // from here on, we don't have to range check x values + + if ((int) x_top == (int) x_bottom) { + float height; + // simple case, only spans one pixel + int x = (int) x_top; + height = (sy1 - sy0) * e->direction; + STBTT_assert(x >= 0 && x < len); + scanline[x] += stbtt__position_trapezoid_area(height, x_top, x+1.0f, x_bottom, x+1.0f); + scanline_fill[x] += height; // everything right of this pixel is filled + } else { + int x,x1,x2; + float y_crossing, y_final, step, sign, area; + // covers 2+ pixels + if (x_top > x_bottom) { + // flip scanline vertically; signed area is the same + float t; + sy0 = y_bottom - (sy0 - y_top); + sy1 = y_bottom - (sy1 - y_top); + t = sy0, sy0 = sy1, sy1 = t; + t = x_bottom, x_bottom = x_top, x_top = t; + dx = -dx; + dy = -dy; + t = x0, x0 = xb, xb = t; + } + STBTT_assert(dy >= 0); + STBTT_assert(dx >= 0); + + x1 = (int) x_top; + x2 = (int) x_bottom; + // compute intersection with y axis at x1+1 + y_crossing = y_top + dy * (x1+1 - x0); + + // compute intersection with y axis at x2 + y_final = y_top + dy * (x2 - x0); + + // x1 x_top x2 x_bottom + // y_top +------|-----+------------+------------+--------|---+------------+ + // | | | | | | + // | | | | | | + // sy0 | Txxxxx|............|............|............|............| + // y_crossing | *xxxxx.......|............|............|............| + // | | xxxxx..|............|............|............| + // | | /- xx*xxxx........|............|............| + // | | dy < | xxxxxx..|............|............| + // y_final | | \- | xx*xxx.........|............| + // sy1 | | | | xxxxxB...|............| + // | | | | | | + // | | | | | | + // y_bottom +------------+------------+------------+------------+------------+ + // + // goal is to measure the area covered by '.' in each pixel + + // if x2 is right at the right edge of x1, y_crossing can blow up, github #1057 + // @TODO: maybe test against sy1 rather than y_bottom? + if (y_crossing > y_bottom) + y_crossing = y_bottom; + + sign = e->direction; + + // area of the rectangle covered from sy0..y_crossing + area = sign * (y_crossing-sy0); + + // area of the triangle (x_top,sy0), (x1+1,sy0), (x1+1,y_crossing) + scanline[x1] += stbtt__sized_triangle_area(area, x1+1 - x_top); + + // check if final y_crossing is blown up; no test case for this + if (y_final > y_bottom) { + y_final = y_bottom; + dy = (y_final - y_crossing ) / (x2 - (x1+1)); // if denom=0, y_final = y_crossing, so y_final <= y_bottom + } + + // in second pixel, area covered by line segment found in first pixel + // is always a rectangle 1 wide * the height of that line segment; this + // is exactly what the variable 'area' stores. it also gets a contribution + // from the line segment within it. the THIRD pixel will get the first + // pixel's rectangle contribution, the second pixel's rectangle contribution, + // and its own contribution. the 'own contribution' is the same in every pixel except + // the leftmost and rightmost, a trapezoid that slides down in each pixel. + // the second pixel's contribution to the third pixel will be the + // rectangle 1 wide times the height change in the second pixel, which is dy. + + step = sign * dy * 1; // dy is dy/dx, change in y for every 1 change in x, + // which multiplied by 1-pixel-width is how much pixel area changes for each step in x + // so the area advances by 'step' every time + + for (x = x1+1; x < x2; ++x) { + scanline[x] += area + step/2; // area of trapezoid is 1*step/2 + area += step; + } + STBTT_assert(STBTT_fabs(area) <= 1.01f); // accumulated error from area += step unless we round step down + STBTT_assert(sy1 > y_final-0.01f); + + // area covered in the last pixel is the rectangle from all the pixels to the left, + // plus the trapezoid filled by the line segment in this pixel all the way to the right edge + scanline[x2] += area + sign * stbtt__position_trapezoid_area(sy1-y_final, (float) x2, x2+1.0f, x_bottom, x2+1.0f); + + // the rest of the line is filled based on the total height of the line segment in this pixel + scanline_fill[x2] += sign * (sy1-sy0); + } + } else { + // if edge goes outside of box we're drawing, we require + // clipping logic. since this does not match the intended use + // of this library, we use a different, very slow brute + // force implementation + // note though that this does happen some of the time because + // x_top and x_bottom can be extrapolated at the top & bottom of + // the shape and actually lie outside the bounding box + int x; + for (x=0; x < len; ++x) { + // cases: + // + // there can be up to two intersections with the pixel. any intersection + // with left or right edges can be handled by splitting into two (or three) + // regions. intersections with top & bottom do not necessitate case-wise logic. + // + // the old way of doing this found the intersections with the left & right edges, + // then used some simple logic to produce up to three segments in sorted order + // from top-to-bottom. however, this had a problem: if an x edge was epsilon + // across the x border, then the corresponding y position might not be distinct + // from the other y segment, and it might ignored as an empty segment. to avoid + // that, we need to explicitly produce segments based on x positions. + + // rename variables to clearly-defined pairs + float y0 = y_top; + float x1 = (float) (x); + float x2 = (float) (x+1); + float x3 = xb; + float y3 = y_bottom; + + // x = e->x + e->dx * (y-y_top) + // (y-y_top) = (x - e->x) / e->dx + // y = (x - e->x) / e->dx + y_top + float y1 = (x - x0) / dx + y_top; + float y2 = (x+1 - x0) / dx + y_top; + + if (x0 < x1 && x3 > x2) { // three segments descending down-right + stbtt__handle_clipped_edge(scanline,x,e, x0,y0, x1,y1); + stbtt__handle_clipped_edge(scanline,x,e, x1,y1, x2,y2); + stbtt__handle_clipped_edge(scanline,x,e, x2,y2, x3,y3); + } else if (x3 < x1 && x0 > x2) { // three segments descending down-left + stbtt__handle_clipped_edge(scanline,x,e, x0,y0, x2,y2); + stbtt__handle_clipped_edge(scanline,x,e, x2,y2, x1,y1); + stbtt__handle_clipped_edge(scanline,x,e, x1,y1, x3,y3); + } else if (x0 < x1 && x3 > x1) { // two segments across x, down-right + stbtt__handle_clipped_edge(scanline,x,e, x0,y0, x1,y1); + stbtt__handle_clipped_edge(scanline,x,e, x1,y1, x3,y3); + } else if (x3 < x1 && x0 > x1) { // two segments across x, down-left + stbtt__handle_clipped_edge(scanline,x,e, x0,y0, x1,y1); + stbtt__handle_clipped_edge(scanline,x,e, x1,y1, x3,y3); + } else if (x0 < x2 && x3 > x2) { // two segments across x+1, down-right + stbtt__handle_clipped_edge(scanline,x,e, x0,y0, x2,y2); + stbtt__handle_clipped_edge(scanline,x,e, x2,y2, x3,y3); + } else if (x3 < x2 && x0 > x2) { // two segments across x+1, down-left + stbtt__handle_clipped_edge(scanline,x,e, x0,y0, x2,y2); + stbtt__handle_clipped_edge(scanline,x,e, x2,y2, x3,y3); + } else { // one segment + stbtt__handle_clipped_edge(scanline,x,e, x0,y0, x3,y3); + } + } + } + } + e = e->next; + } +} + +// directly AA rasterize edges w/o supersampling +static void stbtt__rasterize_sorted_edges(stbtt__bitmap *result, stbtt__edge *e, int n, int vsubsample, int off_x, int off_y, void *userdata) +{ + stbtt__hheap hh = { 0, 0, 0 }; + stbtt__active_edge *active = NULL; + int y,j=0, i; + float scanline_data[129], *scanline, *scanline2; + + STBTT__NOTUSED(vsubsample); + + if (result->w > 64) + scanline = (float *) STBTT_malloc((result->w*2+1) * sizeof(float), userdata); + else + scanline = scanline_data; + + scanline2 = scanline + result->w; + + y = off_y; + e[n].y0 = (float) (off_y + result->h) + 1; + + while (j < result->h) { + // find center of pixel for this scanline + float scan_y_top = y + 0.0f; + float scan_y_bottom = y + 1.0f; + stbtt__active_edge **step = &active; + + STBTT_memset(scanline , 0, result->w*sizeof(scanline[0])); + STBTT_memset(scanline2, 0, (result->w+1)*sizeof(scanline[0])); + + // update all active edges; + // remove all active edges that terminate before the top of this scanline + while (*step) { + stbtt__active_edge * z = *step; + if (z->ey <= scan_y_top) { + *step = z->next; // delete from list + STBTT_assert(z->direction); + z->direction = 0; + stbtt__hheap_free(&hh, z); + } else { + step = &((*step)->next); // advance through list + } + } + + // insert all edges that start before the bottom of this scanline + while (e->y0 <= scan_y_bottom) { + if (e->y0 != e->y1) { + stbtt__active_edge *z = stbtt__new_active(&hh, e, off_x, scan_y_top, userdata); + if (z != NULL) { + if (j == 0 && off_y != 0) { + if (z->ey < scan_y_top) { + // this can happen due to subpixel positioning and some kind of fp rounding error i think + z->ey = scan_y_top; + } + } + STBTT_assert(z->ey >= scan_y_top); // if we get really unlucky a tiny bit of an edge can be out of bounds + // insert at front + z->next = active; + active = z; + } + } + ++e; + } + + // now process all active edges + if (active) + stbtt__fill_active_edges_new(scanline, scanline2+1, result->w, active, scan_y_top); + + { + float sum = 0; + for (i=0; i < result->w; ++i) { + float k; + int m; + sum += scanline2[i]; + k = scanline[i] + sum; + k = (float) STBTT_fabs(k)*255 + 0.5f; + m = (int) k; + if (m > 255) m = 255; + result->pixels[j*result->stride + i] = (unsigned char) m; + } + } + // advance all the edges + step = &active; + while (*step) { + stbtt__active_edge *z = *step; + z->fx += z->fdx; // advance to position for current scanline + step = &((*step)->next); // advance through list + } + + ++y; + ++j; + } + + stbtt__hheap_cleanup(&hh, userdata); + + if (scanline != scanline_data) + STBTT_free(scanline, userdata); +} +#else +#error "Unrecognized value of STBTT_RASTERIZER_VERSION" +#endif + +#define STBTT__COMPARE(a,b) ((a)->y0 < (b)->y0) + +static void stbtt__sort_edges_ins_sort(stbtt__edge *p, int n) +{ + int i,j; + for (i=1; i < n; ++i) { + stbtt__edge t = p[i], *a = &t; + j = i; + while (j > 0) { + stbtt__edge *b = &p[j-1]; + int c = STBTT__COMPARE(a,b); + if (!c) break; + p[j] = p[j-1]; + --j; + } + if (i != j) + p[j] = t; + } +} + +static void stbtt__sort_edges_quicksort(stbtt__edge *p, int n) +{ + /* threshold for transitioning to insertion sort */ + while (n > 12) { + stbtt__edge t; + int c01,c12,c,m,i,j; + + /* compute median of three */ + m = n >> 1; + c01 = STBTT__COMPARE(&p[0],&p[m]); + c12 = STBTT__COMPARE(&p[m],&p[n-1]); + /* if 0 >= mid >= end, or 0 < mid < end, then use mid */ + if (c01 != c12) { + /* otherwise, we'll need to swap something else to middle */ + int z; + c = STBTT__COMPARE(&p[0],&p[n-1]); + /* 0>mid && midn => n; 0 0 */ + /* 0n: 0>n => 0; 0 n */ + z = (c == c12) ? 0 : n-1; + t = p[z]; + p[z] = p[m]; + p[m] = t; + } + /* now p[m] is the median-of-three */ + /* swap it to the beginning so it won't move around */ + t = p[0]; + p[0] = p[m]; + p[m] = t; + + /* partition loop */ + i=1; + j=n-1; + for(;;) { + /* handling of equality is crucial here */ + /* for sentinels & efficiency with duplicates */ + for (;;++i) { + if (!STBTT__COMPARE(&p[i], &p[0])) break; + } + for (;;--j) { + if (!STBTT__COMPARE(&p[0], &p[j])) break; + } + /* make sure we haven't crossed */ + if (i >= j) break; + t = p[i]; + p[i] = p[j]; + p[j] = t; + + ++i; + --j; + } + /* recurse on smaller side, iterate on larger */ + if (j < (n-i)) { + stbtt__sort_edges_quicksort(p,j); + p = p+i; + n = n-i; + } else { + stbtt__sort_edges_quicksort(p+i, n-i); + n = j; + } + } +} + +static void stbtt__sort_edges(stbtt__edge *p, int n) +{ + stbtt__sort_edges_quicksort(p, n); + stbtt__sort_edges_ins_sort(p, n); +} + +typedef struct +{ + float x,y; +} stbtt__point; + +static void stbtt__rasterize(stbtt__bitmap *result, stbtt__point *pts, int *wcount, int windings, float scale_x, float scale_y, float shift_x, float shift_y, int off_x, int off_y, int invert, void *userdata) +{ + float y_scale_inv = invert ? -scale_y : scale_y; + stbtt__edge *e; + int n,i,j,k,m; +#if STBTT_RASTERIZER_VERSION == 1 + int vsubsample = result->h < 8 ? 15 : 5; +#elif STBTT_RASTERIZER_VERSION == 2 + int vsubsample = 1; +#else + #error "Unrecognized value of STBTT_RASTERIZER_VERSION" +#endif + // vsubsample should divide 255 evenly; otherwise we won't reach full opacity + + // now we have to blow out the windings into explicit edge lists + n = 0; + for (i=0; i < windings; ++i) + n += wcount[i]; + + e = (stbtt__edge *) STBTT_malloc(sizeof(*e) * (n+1), userdata); // add an extra one as a sentinel + if (e == 0) return; + n = 0; + + m=0; + for (i=0; i < windings; ++i) { + stbtt__point *p = pts + m; + m += wcount[i]; + j = wcount[i]-1; + for (k=0; k < wcount[i]; j=k++) { + int a=k,b=j; + // skip the edge if horizontal + if (p[j].y == p[k].y) + continue; + // add edge from j to k to the list + e[n].invert = 0; + if (invert ? p[j].y > p[k].y : p[j].y < p[k].y) { + e[n].invert = 1; + a=j,b=k; + } + e[n].x0 = p[a].x * scale_x + shift_x; + e[n].y0 = (p[a].y * y_scale_inv + shift_y) * vsubsample; + e[n].x1 = p[b].x * scale_x + shift_x; + e[n].y1 = (p[b].y * y_scale_inv + shift_y) * vsubsample; + ++n; + } + } + + // now sort the edges by their highest point (should snap to integer, and then by x) + //STBTT_sort(e, n, sizeof(e[0]), stbtt__edge_compare); + stbtt__sort_edges(e, n); + + // now, traverse the scanlines and find the intersections on each scanline, use xor winding rule + stbtt__rasterize_sorted_edges(result, e, n, vsubsample, off_x, off_y, userdata); + + STBTT_free(e, userdata); +} + +static void stbtt__add_point(stbtt__point *points, int n, float x, float y) +{ + if (!points) return; // during first pass, it's unallocated + points[n].x = x; + points[n].y = y; +} + +// tessellate until threshold p is happy... @TODO warped to compensate for non-linear stretching +static int stbtt__tesselate_curve(stbtt__point *points, int *num_points, float x0, float y0, float x1, float y1, float x2, float y2, float objspace_flatness_squared, int n) +{ + // midpoint + float mx = (x0 + 2*x1 + x2)/4; + float my = (y0 + 2*y1 + y2)/4; + // versus directly drawn line + float dx = (x0+x2)/2 - mx; + float dy = (y0+y2)/2 - my; + if (n > 16) // 65536 segments on one curve better be enough! + return 1; + if (dx*dx+dy*dy > objspace_flatness_squared) { // half-pixel error allowed... need to be smaller if AA + stbtt__tesselate_curve(points, num_points, x0,y0, (x0+x1)/2.0f,(y0+y1)/2.0f, mx,my, objspace_flatness_squared,n+1); + stbtt__tesselate_curve(points, num_points, mx,my, (x1+x2)/2.0f,(y1+y2)/2.0f, x2,y2, objspace_flatness_squared,n+1); + } else { + stbtt__add_point(points, *num_points,x2,y2); + *num_points = *num_points+1; + } + return 1; +} + +static void stbtt__tesselate_cubic(stbtt__point *points, int *num_points, float x0, float y0, float x1, float y1, float x2, float y2, float x3, float y3, float objspace_flatness_squared, int n) +{ + // @TODO this "flatness" calculation is just made-up nonsense that seems to work well enough + float dx0 = x1-x0; + float dy0 = y1-y0; + float dx1 = x2-x1; + float dy1 = y2-y1; + float dx2 = x3-x2; + float dy2 = y3-y2; + float dx = x3-x0; + float dy = y3-y0; + float longlen = (float) (STBTT_sqrt(dx0*dx0+dy0*dy0)+STBTT_sqrt(dx1*dx1+dy1*dy1)+STBTT_sqrt(dx2*dx2+dy2*dy2)); + float shortlen = (float) STBTT_sqrt(dx*dx+dy*dy); + float flatness_squared = longlen*longlen-shortlen*shortlen; + + if (n > 16) // 65536 segments on one curve better be enough! + return; + + if (flatness_squared > objspace_flatness_squared) { + float x01 = (x0+x1)/2; + float y01 = (y0+y1)/2; + float x12 = (x1+x2)/2; + float y12 = (y1+y2)/2; + float x23 = (x2+x3)/2; + float y23 = (y2+y3)/2; + + float xa = (x01+x12)/2; + float ya = (y01+y12)/2; + float xb = (x12+x23)/2; + float yb = (y12+y23)/2; + + float mx = (xa+xb)/2; + float my = (ya+yb)/2; + + stbtt__tesselate_cubic(points, num_points, x0,y0, x01,y01, xa,ya, mx,my, objspace_flatness_squared,n+1); + stbtt__tesselate_cubic(points, num_points, mx,my, xb,yb, x23,y23, x3,y3, objspace_flatness_squared,n+1); + } else { + stbtt__add_point(points, *num_points,x3,y3); + *num_points = *num_points+1; + } +} + +// returns number of contours +static stbtt__point *stbtt_FlattenCurves(stbtt_vertex *vertices, int num_verts, float objspace_flatness, int **contour_lengths, int *num_contours, void *userdata) +{ + stbtt__point *points=0; + int num_points=0; + + float objspace_flatness_squared = objspace_flatness * objspace_flatness; + int i,n=0,start=0, pass; + + // count how many "moves" there are to get the contour count + for (i=0; i < num_verts; ++i) + if (vertices[i].type == STBTT_vmove) + ++n; + + *num_contours = n; + if (n == 0) return 0; + + *contour_lengths = (int *) STBTT_malloc(sizeof(**contour_lengths) * n, userdata); + + if (*contour_lengths == 0) { + *num_contours = 0; + return 0; + } + + // make two passes through the points so we don't need to realloc + for (pass=0; pass < 2; ++pass) { + float x=0,y=0; + if (pass == 1) { + points = (stbtt__point *) STBTT_malloc(num_points * sizeof(points[0]), userdata); + if (points == NULL) goto error; + } + num_points = 0; + n= -1; + for (i=0; i < num_verts; ++i) { + switch (vertices[i].type) { + case STBTT_vmove: + // start the next contour + if (n >= 0) + (*contour_lengths)[n] = num_points - start; + ++n; + start = num_points; + + x = vertices[i].x, y = vertices[i].y; + stbtt__add_point(points, num_points++, x,y); + break; + case STBTT_vline: + x = vertices[i].x, y = vertices[i].y; + stbtt__add_point(points, num_points++, x, y); + break; + case STBTT_vcurve: + stbtt__tesselate_curve(points, &num_points, x,y, + vertices[i].cx, vertices[i].cy, + vertices[i].x, vertices[i].y, + objspace_flatness_squared, 0); + x = vertices[i].x, y = vertices[i].y; + break; + case STBTT_vcubic: + stbtt__tesselate_cubic(points, &num_points, x,y, + vertices[i].cx, vertices[i].cy, + vertices[i].cx1, vertices[i].cy1, + vertices[i].x, vertices[i].y, + objspace_flatness_squared, 0); + x = vertices[i].x, y = vertices[i].y; + break; + } + } + (*contour_lengths)[n] = num_points - start; + } + + return points; +error: + STBTT_free(points, userdata); + STBTT_free(*contour_lengths, userdata); + *contour_lengths = 0; + *num_contours = 0; + return NULL; +} + +STBTT_DEF void stbtt_Rasterize(stbtt__bitmap *result, float flatness_in_pixels, stbtt_vertex *vertices, int num_verts, float scale_x, float scale_y, float shift_x, float shift_y, int x_off, int y_off, int invert, void *userdata) +{ + float scale = scale_x > scale_y ? scale_y : scale_x; + int winding_count = 0; + int *winding_lengths = NULL; + stbtt__point *windings = stbtt_FlattenCurves(vertices, num_verts, flatness_in_pixels / scale, &winding_lengths, &winding_count, userdata); + if (windings) { + stbtt__rasterize(result, windings, winding_lengths, winding_count, scale_x, scale_y, shift_x, shift_y, x_off, y_off, invert, userdata); + STBTT_free(winding_lengths, userdata); + STBTT_free(windings, userdata); + } +} + +STBTT_DEF void stbtt_FreeBitmap(unsigned char *bitmap, void *userdata) +{ + STBTT_free(bitmap, userdata); +} + +STBTT_DEF unsigned char *stbtt_GetGlyphBitmapSubpixel(const stbtt_fontinfo *info, float scale_x, float scale_y, float shift_x, float shift_y, int glyph, int *width, int *height, int *xoff, int *yoff) +{ + int ix0,iy0,ix1,iy1; + stbtt__bitmap gbm; + stbtt_vertex *vertices; + int num_verts = stbtt_GetGlyphShape(info, glyph, &vertices); + + if (scale_x == 0) scale_x = scale_y; + if (scale_y == 0) { + if (scale_x == 0) { + STBTT_free(vertices, info->userdata); + return NULL; + } + scale_y = scale_x; + } + + stbtt_GetGlyphBitmapBoxSubpixel(info, glyph, scale_x, scale_y, shift_x, shift_y, &ix0,&iy0,&ix1,&iy1); + + // now we get the size + gbm.w = (ix1 - ix0); + gbm.h = (iy1 - iy0); + gbm.pixels = NULL; // in case we error + + if (width ) *width = gbm.w; + if (height) *height = gbm.h; + if (xoff ) *xoff = ix0; + if (yoff ) *yoff = iy0; + + if (gbm.w && gbm.h) { + gbm.pixels = (unsigned char *) STBTT_malloc(gbm.w * gbm.h, info->userdata); + if (gbm.pixels) { + gbm.stride = gbm.w; + + stbtt_Rasterize(&gbm, 0.35f, vertices, num_verts, scale_x, scale_y, shift_x, shift_y, ix0, iy0, 1, info->userdata); + } + } + STBTT_free(vertices, info->userdata); + return gbm.pixels; +} + +STBTT_DEF unsigned char *stbtt_GetGlyphBitmap(const stbtt_fontinfo *info, float scale_x, float scale_y, int glyph, int *width, int *height, int *xoff, int *yoff) +{ + return stbtt_GetGlyphBitmapSubpixel(info, scale_x, scale_y, 0.0f, 0.0f, glyph, width, height, xoff, yoff); +} + +STBTT_DEF void stbtt_MakeGlyphBitmapSubpixel(const stbtt_fontinfo *info, unsigned char *output, int out_w, int out_h, int out_stride, float scale_x, float scale_y, float shift_x, float shift_y, int glyph) +{ + int ix0,iy0; + stbtt_vertex *vertices; + int num_verts = stbtt_GetGlyphShape(info, glyph, &vertices); + stbtt__bitmap gbm; + + stbtt_GetGlyphBitmapBoxSubpixel(info, glyph, scale_x, scale_y, shift_x, shift_y, &ix0,&iy0,0,0); + gbm.pixels = output; + gbm.w = out_w; + gbm.h = out_h; + gbm.stride = out_stride; + + if (gbm.w && gbm.h) + stbtt_Rasterize(&gbm, 0.35f, vertices, num_verts, scale_x, scale_y, shift_x, shift_y, ix0,iy0, 1, info->userdata); + + STBTT_free(vertices, info->userdata); +} + +STBTT_DEF void stbtt_MakeGlyphBitmap(const stbtt_fontinfo *info, unsigned char *output, int out_w, int out_h, int out_stride, float scale_x, float scale_y, int glyph) +{ + stbtt_MakeGlyphBitmapSubpixel(info, output, out_w, out_h, out_stride, scale_x, scale_y, 0.0f,0.0f, glyph); +} + +STBTT_DEF unsigned char *stbtt_GetCodepointBitmapSubpixel(const stbtt_fontinfo *info, float scale_x, float scale_y, float shift_x, float shift_y, int codepoint, int *width, int *height, int *xoff, int *yoff) +{ + return stbtt_GetGlyphBitmapSubpixel(info, scale_x, scale_y,shift_x,shift_y, stbtt_FindGlyphIndex(info,codepoint), width,height,xoff,yoff); +} + +STBTT_DEF void stbtt_MakeCodepointBitmapSubpixelPrefilter(const stbtt_fontinfo *info, unsigned char *output, int out_w, int out_h, int out_stride, float scale_x, float scale_y, float shift_x, float shift_y, int oversample_x, int oversample_y, float *sub_x, float *sub_y, int codepoint) +{ + stbtt_MakeGlyphBitmapSubpixelPrefilter(info, output, out_w, out_h, out_stride, scale_x, scale_y, shift_x, shift_y, oversample_x, oversample_y, sub_x, sub_y, stbtt_FindGlyphIndex(info,codepoint)); +} + +STBTT_DEF void stbtt_MakeCodepointBitmapSubpixel(const stbtt_fontinfo *info, unsigned char *output, int out_w, int out_h, int out_stride, float scale_x, float scale_y, float shift_x, float shift_y, int codepoint) +{ + stbtt_MakeGlyphBitmapSubpixel(info, output, out_w, out_h, out_stride, scale_x, scale_y, shift_x, shift_y, stbtt_FindGlyphIndex(info,codepoint)); +} + +STBTT_DEF unsigned char *stbtt_GetCodepointBitmap(const stbtt_fontinfo *info, float scale_x, float scale_y, int codepoint, int *width, int *height, int *xoff, int *yoff) +{ + return stbtt_GetCodepointBitmapSubpixel(info, scale_x, scale_y, 0.0f,0.0f, codepoint, width,height,xoff,yoff); +} + +STBTT_DEF void stbtt_MakeCodepointBitmap(const stbtt_fontinfo *info, unsigned char *output, int out_w, int out_h, int out_stride, float scale_x, float scale_y, int codepoint) +{ + stbtt_MakeCodepointBitmapSubpixel(info, output, out_w, out_h, out_stride, scale_x, scale_y, 0.0f,0.0f, codepoint); +} + +////////////////////////////////////////////////////////////////////////////// +// +// bitmap baking +// +// This is SUPER-CRAPPY packing to keep source code small + +static int stbtt_BakeFontBitmap_internal(unsigned char *data, int offset, // font location (use offset=0 for plain .ttf) + float pixel_height, // height of font in pixels + unsigned char *pixels, int pw, int ph, // bitmap to be filled in + int first_char, int num_chars, // characters to bake + stbtt_bakedchar *chardata) +{ + float scale; + int x,y,bottom_y, i; + stbtt_fontinfo f; + f.userdata = NULL; + if (!stbtt_InitFont(&f, data, offset)) + return -1; + STBTT_memset(pixels, 0, pw*ph); // background of 0 around pixels + x=y=1; + bottom_y = 1; + + scale = stbtt_ScaleForPixelHeight(&f, pixel_height); + + for (i=0; i < num_chars; ++i) { + int advance, lsb, x0,y0,x1,y1,gw,gh; + int g = stbtt_FindGlyphIndex(&f, first_char + i); + stbtt_GetGlyphHMetrics(&f, g, &advance, &lsb); + stbtt_GetGlyphBitmapBox(&f, g, scale,scale, &x0,&y0,&x1,&y1); + gw = x1-x0; + gh = y1-y0; + if (x + gw + 1 >= pw) + y = bottom_y, x = 1; // advance to next row + if (y + gh + 1 >= ph) // check if it fits vertically AFTER potentially moving to next row + return -i; + STBTT_assert(x+gw < pw); + STBTT_assert(y+gh < ph); + stbtt_MakeGlyphBitmap(&f, pixels+x+y*pw, gw,gh,pw, scale,scale, g); + chardata[i].x0 = (stbtt_int16) x; + chardata[i].y0 = (stbtt_int16) y; + chardata[i].x1 = (stbtt_int16) (x + gw); + chardata[i].y1 = (stbtt_int16) (y + gh); + chardata[i].xadvance = scale * advance; + chardata[i].xoff = (float) x0; + chardata[i].yoff = (float) y0; + x = x + gw + 1; + if (y+gh+1 > bottom_y) + bottom_y = y+gh+1; + } + return bottom_y; +} + +STBTT_DEF void stbtt_GetBakedQuad(const stbtt_bakedchar *chardata, int pw, int ph, int char_index, float *xpos, float *ypos, stbtt_aligned_quad *q, int opengl_fillrule) +{ + float d3d_bias = opengl_fillrule ? 0 : -0.5f; + float ipw = 1.0f / pw, iph = 1.0f / ph; + const stbtt_bakedchar *b = chardata + char_index; + int round_x = STBTT_ifloor((*xpos + b->xoff) + 0.5f); + int round_y = STBTT_ifloor((*ypos + b->yoff) + 0.5f); + + q->x0 = round_x + d3d_bias; + q->y0 = round_y + d3d_bias; + q->x1 = round_x + b->x1 - b->x0 + d3d_bias; + q->y1 = round_y + b->y1 - b->y0 + d3d_bias; + + q->s0 = b->x0 * ipw; + q->t0 = b->y0 * iph; + q->s1 = b->x1 * ipw; + q->t1 = b->y1 * iph; + + *xpos += b->xadvance; +} + +////////////////////////////////////////////////////////////////////////////// +// +// rectangle packing replacement routines if you don't have stb_rect_pack.h +// + +#ifndef STB_RECT_PACK_VERSION + +typedef int stbrp_coord; + +//////////////////////////////////////////////////////////////////////////////////// +// // +// // +// COMPILER WARNING ?!?!? // +// // +// // +// if you get a compile warning due to these symbols being defined more than // +// once, move #include "stb_rect_pack.h" before #include "stb_truetype.h" // +// // +//////////////////////////////////////////////////////////////////////////////////// + +typedef struct +{ + int width,height; + int x,y,bottom_y; +} stbrp_context; + +typedef struct +{ + unsigned char x; +} stbrp_node; + +struct stbrp_rect +{ + stbrp_coord x,y; + int id,w,h,was_packed; +}; + +static void stbrp_init_target(stbrp_context *con, int pw, int ph, stbrp_node *nodes, int num_nodes) +{ + con->width = pw; + con->height = ph; + con->x = 0; + con->y = 0; + con->bottom_y = 0; + STBTT__NOTUSED(nodes); + STBTT__NOTUSED(num_nodes); +} + +static void stbrp_pack_rects(stbrp_context *con, stbrp_rect *rects, int num_rects) +{ + int i; + for (i=0; i < num_rects; ++i) { + if (con->x + rects[i].w > con->width) { + con->x = 0; + con->y = con->bottom_y; + } + if (con->y + rects[i].h > con->height) + break; + rects[i].x = con->x; + rects[i].y = con->y; + rects[i].was_packed = 1; + con->x += rects[i].w; + if (con->y + rects[i].h > con->bottom_y) + con->bottom_y = con->y + rects[i].h; + } + for ( ; i < num_rects; ++i) + rects[i].was_packed = 0; +} +#endif + +////////////////////////////////////////////////////////////////////////////// +// +// bitmap baking +// +// This is SUPER-AWESOME (tm Ryan Gordon) packing using stb_rect_pack.h. If +// stb_rect_pack.h isn't available, it uses the BakeFontBitmap strategy. + +STBTT_DEF int stbtt_PackBegin(stbtt_pack_context *spc, unsigned char *pixels, int pw, int ph, int stride_in_bytes, int padding, void *alloc_context) +{ + stbrp_context *context = (stbrp_context *) STBTT_malloc(sizeof(*context) ,alloc_context); + int num_nodes = pw - padding; + stbrp_node *nodes = (stbrp_node *) STBTT_malloc(sizeof(*nodes ) * num_nodes,alloc_context); + + if (context == NULL || nodes == NULL) { + if (context != NULL) STBTT_free(context, alloc_context); + if (nodes != NULL) STBTT_free(nodes , alloc_context); + return 0; + } + + spc->user_allocator_context = alloc_context; + spc->width = pw; + spc->height = ph; + spc->pixels = pixels; + spc->pack_info = context; + spc->nodes = nodes; + spc->padding = padding; + spc->stride_in_bytes = stride_in_bytes != 0 ? stride_in_bytes : pw; + spc->h_oversample = 1; + spc->v_oversample = 1; + spc->skip_missing = 0; + + stbrp_init_target(context, pw-padding, ph-padding, nodes, num_nodes); + + if (pixels) + STBTT_memset(pixels, 0, pw*ph); // background of 0 around pixels + + return 1; +} + +STBTT_DEF void stbtt_PackEnd (stbtt_pack_context *spc) +{ + STBTT_free(spc->nodes , spc->user_allocator_context); + STBTT_free(spc->pack_info, spc->user_allocator_context); +} + +STBTT_DEF void stbtt_PackSetOversampling(stbtt_pack_context *spc, unsigned int h_oversample, unsigned int v_oversample) +{ + STBTT_assert(h_oversample <= STBTT_MAX_OVERSAMPLE); + STBTT_assert(v_oversample <= STBTT_MAX_OVERSAMPLE); + if (h_oversample <= STBTT_MAX_OVERSAMPLE) + spc->h_oversample = h_oversample; + if (v_oversample <= STBTT_MAX_OVERSAMPLE) + spc->v_oversample = v_oversample; +} + +STBTT_DEF void stbtt_PackSetSkipMissingCodepoints(stbtt_pack_context *spc, int skip) +{ + spc->skip_missing = skip; +} + +#define STBTT__OVER_MASK (STBTT_MAX_OVERSAMPLE-1) + +static void stbtt__h_prefilter(unsigned char *pixels, int w, int h, int stride_in_bytes, unsigned int kernel_width) +{ + unsigned char buffer[STBTT_MAX_OVERSAMPLE]; + int safe_w = w - kernel_width; + int j; + STBTT_memset(buffer, 0, STBTT_MAX_OVERSAMPLE); // suppress bogus warning from VS2013 -analyze + for (j=0; j < h; ++j) { + int i; + unsigned int total; + STBTT_memset(buffer, 0, kernel_width); + + total = 0; + + // make kernel_width a constant in common cases so compiler can optimize out the divide + switch (kernel_width) { + case 2: + for (i=0; i <= safe_w; ++i) { + total += pixels[i] - buffer[i & STBTT__OVER_MASK]; + buffer[(i+kernel_width) & STBTT__OVER_MASK] = pixels[i]; + pixels[i] = (unsigned char) (total / 2); + } + break; + case 3: + for (i=0; i <= safe_w; ++i) { + total += pixels[i] - buffer[i & STBTT__OVER_MASK]; + buffer[(i+kernel_width) & STBTT__OVER_MASK] = pixels[i]; + pixels[i] = (unsigned char) (total / 3); + } + break; + case 4: + for (i=0; i <= safe_w; ++i) { + total += pixels[i] - buffer[i & STBTT__OVER_MASK]; + buffer[(i+kernel_width) & STBTT__OVER_MASK] = pixels[i]; + pixels[i] = (unsigned char) (total / 4); + } + break; + case 5: + for (i=0; i <= safe_w; ++i) { + total += pixels[i] - buffer[i & STBTT__OVER_MASK]; + buffer[(i+kernel_width) & STBTT__OVER_MASK] = pixels[i]; + pixels[i] = (unsigned char) (total / 5); + } + break; + default: + for (i=0; i <= safe_w; ++i) { + total += pixels[i] - buffer[i & STBTT__OVER_MASK]; + buffer[(i+kernel_width) & STBTT__OVER_MASK] = pixels[i]; + pixels[i] = (unsigned char) (total / kernel_width); + } + break; + } + + for (; i < w; ++i) { + STBTT_assert(pixels[i] == 0); + total -= buffer[i & STBTT__OVER_MASK]; + pixels[i] = (unsigned char) (total / kernel_width); + } + + pixels += stride_in_bytes; + } +} + +static void stbtt__v_prefilter(unsigned char *pixels, int w, int h, int stride_in_bytes, unsigned int kernel_width) +{ + unsigned char buffer[STBTT_MAX_OVERSAMPLE]; + int safe_h = h - kernel_width; + int j; + STBTT_memset(buffer, 0, STBTT_MAX_OVERSAMPLE); // suppress bogus warning from VS2013 -analyze + for (j=0; j < w; ++j) { + int i; + unsigned int total; + STBTT_memset(buffer, 0, kernel_width); + + total = 0; + + // make kernel_width a constant in common cases so compiler can optimize out the divide + switch (kernel_width) { + case 2: + for (i=0; i <= safe_h; ++i) { + total += pixels[i*stride_in_bytes] - buffer[i & STBTT__OVER_MASK]; + buffer[(i+kernel_width) & STBTT__OVER_MASK] = pixels[i*stride_in_bytes]; + pixels[i*stride_in_bytes] = (unsigned char) (total / 2); + } + break; + case 3: + for (i=0; i <= safe_h; ++i) { + total += pixels[i*stride_in_bytes] - buffer[i & STBTT__OVER_MASK]; + buffer[(i+kernel_width) & STBTT__OVER_MASK] = pixels[i*stride_in_bytes]; + pixels[i*stride_in_bytes] = (unsigned char) (total / 3); + } + break; + case 4: + for (i=0; i <= safe_h; ++i) { + total += pixels[i*stride_in_bytes] - buffer[i & STBTT__OVER_MASK]; + buffer[(i+kernel_width) & STBTT__OVER_MASK] = pixels[i*stride_in_bytes]; + pixels[i*stride_in_bytes] = (unsigned char) (total / 4); + } + break; + case 5: + for (i=0; i <= safe_h; ++i) { + total += pixels[i*stride_in_bytes] - buffer[i & STBTT__OVER_MASK]; + buffer[(i+kernel_width) & STBTT__OVER_MASK] = pixels[i*stride_in_bytes]; + pixels[i*stride_in_bytes] = (unsigned char) (total / 5); + } + break; + default: + for (i=0; i <= safe_h; ++i) { + total += pixels[i*stride_in_bytes] - buffer[i & STBTT__OVER_MASK]; + buffer[(i+kernel_width) & STBTT__OVER_MASK] = pixels[i*stride_in_bytes]; + pixels[i*stride_in_bytes] = (unsigned char) (total / kernel_width); + } + break; + } + + for (; i < h; ++i) { + STBTT_assert(pixels[i*stride_in_bytes] == 0); + total -= buffer[i & STBTT__OVER_MASK]; + pixels[i*stride_in_bytes] = (unsigned char) (total / kernel_width); + } + + pixels += 1; + } +} + +static float stbtt__oversample_shift(int oversample) +{ + if (!oversample) + return 0.0f; + + // The prefilter is a box filter of width "oversample", + // which shifts phase by (oversample - 1)/2 pixels in + // oversampled space. We want to shift in the opposite + // direction to counter this. + return (float)-(oversample - 1) / (2.0f * (float)oversample); +} + +// rects array must be big enough to accommodate all characters in the given ranges +STBTT_DEF int stbtt_PackFontRangesGatherRects(stbtt_pack_context *spc, const stbtt_fontinfo *info, stbtt_pack_range *ranges, int num_ranges, stbrp_rect *rects) +{ + int i,j,k; + int missing_glyph_added = 0; + + k=0; + for (i=0; i < num_ranges; ++i) { + float fh = ranges[i].font_size; + float scale = fh > 0 ? stbtt_ScaleForPixelHeight(info, fh) : stbtt_ScaleForMappingEmToPixels(info, -fh); + ranges[i].h_oversample = (unsigned char) spc->h_oversample; + ranges[i].v_oversample = (unsigned char) spc->v_oversample; + for (j=0; j < ranges[i].num_chars; ++j) { + int x0,y0,x1,y1; + int codepoint = ranges[i].array_of_unicode_codepoints == NULL ? ranges[i].first_unicode_codepoint_in_range + j : ranges[i].array_of_unicode_codepoints[j]; + int glyph = stbtt_FindGlyphIndex(info, codepoint); + if (glyph == 0 && (spc->skip_missing || missing_glyph_added)) { + rects[k].w = rects[k].h = 0; + } else { + stbtt_GetGlyphBitmapBoxSubpixel(info,glyph, + scale * spc->h_oversample, + scale * spc->v_oversample, + 0,0, + &x0,&y0,&x1,&y1); + rects[k].w = (stbrp_coord) (x1-x0 + spc->padding + spc->h_oversample-1); + rects[k].h = (stbrp_coord) (y1-y0 + spc->padding + spc->v_oversample-1); + if (glyph == 0) + missing_glyph_added = 1; + } + ++k; + } + } + + return k; +} + +STBTT_DEF void stbtt_MakeGlyphBitmapSubpixelPrefilter(const stbtt_fontinfo *info, unsigned char *output, int out_w, int out_h, int out_stride, float scale_x, float scale_y, float shift_x, float shift_y, int prefilter_x, int prefilter_y, float *sub_x, float *sub_y, int glyph) +{ + stbtt_MakeGlyphBitmapSubpixel(info, + output, + out_w - (prefilter_x - 1), + out_h - (prefilter_y - 1), + out_stride, + scale_x, + scale_y, + shift_x, + shift_y, + glyph); + + if (prefilter_x > 1) + stbtt__h_prefilter(output, out_w, out_h, out_stride, prefilter_x); + + if (prefilter_y > 1) + stbtt__v_prefilter(output, out_w, out_h, out_stride, prefilter_y); + + *sub_x = stbtt__oversample_shift(prefilter_x); + *sub_y = stbtt__oversample_shift(prefilter_y); +} + +// rects array must be big enough to accommodate all characters in the given ranges +STBTT_DEF int stbtt_PackFontRangesRenderIntoRects(stbtt_pack_context *spc, const stbtt_fontinfo *info, stbtt_pack_range *ranges, int num_ranges, stbrp_rect *rects) +{ + int i,j,k, missing_glyph = -1, return_value = 1; + + // save current values + int old_h_over = spc->h_oversample; + int old_v_over = spc->v_oversample; + + k = 0; + for (i=0; i < num_ranges; ++i) { + float fh = ranges[i].font_size; + float scale = fh > 0 ? stbtt_ScaleForPixelHeight(info, fh) : stbtt_ScaleForMappingEmToPixels(info, -fh); + float recip_h,recip_v,sub_x,sub_y; + spc->h_oversample = ranges[i].h_oversample; + spc->v_oversample = ranges[i].v_oversample; + recip_h = 1.0f / spc->h_oversample; + recip_v = 1.0f / spc->v_oversample; + sub_x = stbtt__oversample_shift(spc->h_oversample); + sub_y = stbtt__oversample_shift(spc->v_oversample); + for (j=0; j < ranges[i].num_chars; ++j) { + stbrp_rect *r = &rects[k]; + if (r->was_packed && r->w != 0 && r->h != 0) { + stbtt_packedchar *bc = &ranges[i].chardata_for_range[j]; + int advance, lsb, x0,y0,x1,y1; + int codepoint = ranges[i].array_of_unicode_codepoints == NULL ? ranges[i].first_unicode_codepoint_in_range + j : ranges[i].array_of_unicode_codepoints[j]; + int glyph = stbtt_FindGlyphIndex(info, codepoint); + stbrp_coord pad = (stbrp_coord) spc->padding; + + // pad on left and top + r->x += pad; + r->y += pad; + r->w -= pad; + r->h -= pad; + stbtt_GetGlyphHMetrics(info, glyph, &advance, &lsb); + stbtt_GetGlyphBitmapBox(info, glyph, + scale * spc->h_oversample, + scale * spc->v_oversample, + &x0,&y0,&x1,&y1); + stbtt_MakeGlyphBitmapSubpixel(info, + spc->pixels + r->x + r->y*spc->stride_in_bytes, + r->w - spc->h_oversample+1, + r->h - spc->v_oversample+1, + spc->stride_in_bytes, + scale * spc->h_oversample, + scale * spc->v_oversample, + 0,0, + glyph); + + if (spc->h_oversample > 1) + stbtt__h_prefilter(spc->pixels + r->x + r->y*spc->stride_in_bytes, + r->w, r->h, spc->stride_in_bytes, + spc->h_oversample); + + if (spc->v_oversample > 1) + stbtt__v_prefilter(spc->pixels + r->x + r->y*spc->stride_in_bytes, + r->w, r->h, spc->stride_in_bytes, + spc->v_oversample); + + bc->x0 = (stbtt_int16) r->x; + bc->y0 = (stbtt_int16) r->y; + bc->x1 = (stbtt_int16) (r->x + r->w); + bc->y1 = (stbtt_int16) (r->y + r->h); + bc->xadvance = scale * advance; + bc->xoff = (float) x0 * recip_h + sub_x; + bc->yoff = (float) y0 * recip_v + sub_y; + bc->xoff2 = (x0 + r->w) * recip_h + sub_x; + bc->yoff2 = (y0 + r->h) * recip_v + sub_y; + + if (glyph == 0) + missing_glyph = j; + } else if (spc->skip_missing) { + return_value = 0; + } else if (r->was_packed && r->w == 0 && r->h == 0 && missing_glyph >= 0) { + ranges[i].chardata_for_range[j] = ranges[i].chardata_for_range[missing_glyph]; + } else { + return_value = 0; // if any fail, report failure + } + + ++k; + } + } + + // restore original values + spc->h_oversample = old_h_over; + spc->v_oversample = old_v_over; + + return return_value; +} + +STBTT_DEF void stbtt_PackFontRangesPackRects(stbtt_pack_context *spc, stbrp_rect *rects, int num_rects) +{ + stbrp_pack_rects((stbrp_context *) spc->pack_info, rects, num_rects); +} + +STBTT_DEF int stbtt_PackFontRanges(stbtt_pack_context *spc, const unsigned char *fontdata, int font_index, stbtt_pack_range *ranges, int num_ranges) +{ + stbtt_fontinfo info; + int i,j,n, return_value = 1; + //stbrp_context *context = (stbrp_context *) spc->pack_info; + stbrp_rect *rects; + + // flag all characters as NOT packed + for (i=0; i < num_ranges; ++i) + for (j=0; j < ranges[i].num_chars; ++j) + ranges[i].chardata_for_range[j].x0 = + ranges[i].chardata_for_range[j].y0 = + ranges[i].chardata_for_range[j].x1 = + ranges[i].chardata_for_range[j].y1 = 0; + + n = 0; + for (i=0; i < num_ranges; ++i) + n += ranges[i].num_chars; + + rects = (stbrp_rect *) STBTT_malloc(sizeof(*rects) * n, spc->user_allocator_context); + if (rects == NULL) + return 0; + + info.userdata = spc->user_allocator_context; + stbtt_InitFont(&info, fontdata, stbtt_GetFontOffsetForIndex(fontdata,font_index)); + + n = stbtt_PackFontRangesGatherRects(spc, &info, ranges, num_ranges, rects); + + stbtt_PackFontRangesPackRects(spc, rects, n); + + return_value = stbtt_PackFontRangesRenderIntoRects(spc, &info, ranges, num_ranges, rects); + + STBTT_free(rects, spc->user_allocator_context); + return return_value; +} + +STBTT_DEF int stbtt_PackFontRange(stbtt_pack_context *spc, const unsigned char *fontdata, int font_index, float font_size, + int first_unicode_codepoint_in_range, int num_chars_in_range, stbtt_packedchar *chardata_for_range) +{ + stbtt_pack_range range; + range.first_unicode_codepoint_in_range = first_unicode_codepoint_in_range; + range.array_of_unicode_codepoints = NULL; + range.num_chars = num_chars_in_range; + range.chardata_for_range = chardata_for_range; + range.font_size = font_size; + return stbtt_PackFontRanges(spc, fontdata, font_index, &range, 1); +} + +STBTT_DEF void stbtt_GetScaledFontVMetrics(const unsigned char *fontdata, int index, float size, float *ascent, float *descent, float *lineGap) +{ + int i_ascent, i_descent, i_lineGap; + float scale; + stbtt_fontinfo info; + stbtt_InitFont(&info, fontdata, stbtt_GetFontOffsetForIndex(fontdata, index)); + scale = size > 0 ? stbtt_ScaleForPixelHeight(&info, size) : stbtt_ScaleForMappingEmToPixels(&info, -size); + stbtt_GetFontVMetrics(&info, &i_ascent, &i_descent, &i_lineGap); + *ascent = (float) i_ascent * scale; + *descent = (float) i_descent * scale; + *lineGap = (float) i_lineGap * scale; +} + +STBTT_DEF void stbtt_GetPackedQuad(const stbtt_packedchar *chardata, int pw, int ph, int char_index, float *xpos, float *ypos, stbtt_aligned_quad *q, int align_to_integer) +{ + float ipw = 1.0f / pw, iph = 1.0f / ph; + const stbtt_packedchar *b = chardata + char_index; + + if (align_to_integer) { + float x = (float) STBTT_ifloor((*xpos + b->xoff) + 0.5f); + float y = (float) STBTT_ifloor((*ypos + b->yoff) + 0.5f); + q->x0 = x; + q->y0 = y; + q->x1 = x + b->xoff2 - b->xoff; + q->y1 = y + b->yoff2 - b->yoff; + } else { + q->x0 = *xpos + b->xoff; + q->y0 = *ypos + b->yoff; + q->x1 = *xpos + b->xoff2; + q->y1 = *ypos + b->yoff2; + } + + q->s0 = b->x0 * ipw; + q->t0 = b->y0 * iph; + q->s1 = b->x1 * ipw; + q->t1 = b->y1 * iph; + + *xpos += b->xadvance; +} + +////////////////////////////////////////////////////////////////////////////// +// +// sdf computation +// + +#define STBTT_min(a,b) ((a) < (b) ? (a) : (b)) +#define STBTT_max(a,b) ((a) < (b) ? (b) : (a)) + +static int stbtt__ray_intersect_bezier(float orig[2], float ray[2], float q0[2], float q1[2], float q2[2], float hits[2][2]) +{ + float q0perp = q0[1]*ray[0] - q0[0]*ray[1]; + float q1perp = q1[1]*ray[0] - q1[0]*ray[1]; + float q2perp = q2[1]*ray[0] - q2[0]*ray[1]; + float roperp = orig[1]*ray[0] - orig[0]*ray[1]; + + float a = q0perp - 2*q1perp + q2perp; + float b = q1perp - q0perp; + float c = q0perp - roperp; + + float s0 = 0., s1 = 0.; + int num_s = 0; + + if (a != 0.0) { + float discr = b*b - a*c; + if (discr > 0.0) { + float rcpna = -1 / a; + float d = (float) STBTT_sqrt(discr); + s0 = (b+d) * rcpna; + s1 = (b-d) * rcpna; + if (s0 >= 0.0 && s0 <= 1.0) + num_s = 1; + if (d > 0.0 && s1 >= 0.0 && s1 <= 1.0) { + if (num_s == 0) s0 = s1; + ++num_s; + } + } + } else { + // 2*b*s + c = 0 + // s = -c / (2*b) + s0 = c / (-2 * b); + if (s0 >= 0.0 && s0 <= 1.0) + num_s = 1; + } + + if (num_s == 0) + return 0; + else { + float rcp_len2 = 1 / (ray[0]*ray[0] + ray[1]*ray[1]); + float rayn_x = ray[0] * rcp_len2, rayn_y = ray[1] * rcp_len2; + + float q0d = q0[0]*rayn_x + q0[1]*rayn_y; + float q1d = q1[0]*rayn_x + q1[1]*rayn_y; + float q2d = q2[0]*rayn_x + q2[1]*rayn_y; + float rod = orig[0]*rayn_x + orig[1]*rayn_y; + + float q10d = q1d - q0d; + float q20d = q2d - q0d; + float q0rd = q0d - rod; + + hits[0][0] = q0rd + s0*(2.0f - 2.0f*s0)*q10d + s0*s0*q20d; + hits[0][1] = a*s0+b; + + if (num_s > 1) { + hits[1][0] = q0rd + s1*(2.0f - 2.0f*s1)*q10d + s1*s1*q20d; + hits[1][1] = a*s1+b; + return 2; + } else { + return 1; + } + } +} + +static int equal(float *a, float *b) +{ + return (a[0] == b[0] && a[1] == b[1]); +} + +static int stbtt__compute_crossings_x(float x, float y, int nverts, stbtt_vertex *verts) +{ + int i; + float orig[2], ray[2] = { 1, 0 }; + float y_frac; + int winding = 0; + + // make sure y never passes through a vertex of the shape + y_frac = (float) STBTT_fmod(y, 1.0f); + if (y_frac < 0.01f) + y += 0.01f; + else if (y_frac > 0.99f) + y -= 0.01f; + + orig[0] = x; + orig[1] = y; + + // test a ray from (-infinity,y) to (x,y) + for (i=0; i < nverts; ++i) { + if (verts[i].type == STBTT_vline) { + int x0 = (int) verts[i-1].x, y0 = (int) verts[i-1].y; + int x1 = (int) verts[i ].x, y1 = (int) verts[i ].y; + if (y > STBTT_min(y0,y1) && y < STBTT_max(y0,y1) && x > STBTT_min(x0,x1)) { + float x_inter = (y - y0) / (y1 - y0) * (x1-x0) + x0; + if (x_inter < x) + winding += (y0 < y1) ? 1 : -1; + } + } + if (verts[i].type == STBTT_vcurve) { + int x0 = (int) verts[i-1].x , y0 = (int) verts[i-1].y ; + int x1 = (int) verts[i ].cx, y1 = (int) verts[i ].cy; + int x2 = (int) verts[i ].x , y2 = (int) verts[i ].y ; + int ax = STBTT_min(x0,STBTT_min(x1,x2)), ay = STBTT_min(y0,STBTT_min(y1,y2)); + int by = STBTT_max(y0,STBTT_max(y1,y2)); + if (y > ay && y < by && x > ax) { + float q0[2],q1[2],q2[2]; + float hits[2][2]; + q0[0] = (float)x0; + q0[1] = (float)y0; + q1[0] = (float)x1; + q1[1] = (float)y1; + q2[0] = (float)x2; + q2[1] = (float)y2; + if (equal(q0,q1) || equal(q1,q2)) { + x0 = (int)verts[i-1].x; + y0 = (int)verts[i-1].y; + x1 = (int)verts[i ].x; + y1 = (int)verts[i ].y; + if (y > STBTT_min(y0,y1) && y < STBTT_max(y0,y1) && x > STBTT_min(x0,x1)) { + float x_inter = (y - y0) / (y1 - y0) * (x1-x0) + x0; + if (x_inter < x) + winding += (y0 < y1) ? 1 : -1; + } + } else { + int num_hits = stbtt__ray_intersect_bezier(orig, ray, q0, q1, q2, hits); + if (num_hits >= 1) + if (hits[0][0] < 0) + winding += (hits[0][1] < 0 ? -1 : 1); + if (num_hits >= 2) + if (hits[1][0] < 0) + winding += (hits[1][1] < 0 ? -1 : 1); + } + } + } + } + return winding; +} + +static float stbtt__cuberoot( float x ) +{ + if (x<0) + return -(float) STBTT_pow(-x,1.0f/3.0f); + else + return (float) STBTT_pow( x,1.0f/3.0f); +} + +// x^3 + a*x^2 + b*x + c = 0 +static int stbtt__solve_cubic(float a, float b, float c, float* r) +{ + float s = -a / 3; + float p = b - a*a / 3; + float q = a * (2*a*a - 9*b) / 27 + c; + float p3 = p*p*p; + float d = q*q + 4*p3 / 27; + if (d >= 0) { + float z = (float) STBTT_sqrt(d); + float u = (-q + z) / 2; + float v = (-q - z) / 2; + u = stbtt__cuberoot(u); + v = stbtt__cuberoot(v); + r[0] = s + u + v; + return 1; + } else { + float u = (float) STBTT_sqrt(-p/3); + float v = (float) STBTT_acos(-STBTT_sqrt(-27/p3) * q / 2) / 3; // p3 must be negative, since d is negative + float m = (float) STBTT_cos(v); + float n = (float) STBTT_cos(v-3.141592/2)*1.732050808f; + r[0] = s + u * 2 * m; + r[1] = s - u * (m + n); + r[2] = s - u * (m - n); + + //STBTT_assert( STBTT_fabs(((r[0]+a)*r[0]+b)*r[0]+c) < 0.05f); // these asserts may not be safe at all scales, though they're in bezier t parameter units so maybe? + //STBTT_assert( STBTT_fabs(((r[1]+a)*r[1]+b)*r[1]+c) < 0.05f); + //STBTT_assert( STBTT_fabs(((r[2]+a)*r[2]+b)*r[2]+c) < 0.05f); + return 3; + } +} + +STBTT_DEF unsigned char * stbtt_GetGlyphSDF(const stbtt_fontinfo *info, float scale, int glyph, int padding, unsigned char onedge_value, float pixel_dist_scale, int *width, int *height, int *xoff, int *yoff) +{ + float scale_x = scale, scale_y = scale; + int ix0,iy0,ix1,iy1; + int w,h; + unsigned char *data; + + if (scale == 0) return NULL; + + stbtt_GetGlyphBitmapBoxSubpixel(info, glyph, scale, scale, 0.0f,0.0f, &ix0,&iy0,&ix1,&iy1); + + // if empty, return NULL + if (ix0 == ix1 || iy0 == iy1) + return NULL; + + ix0 -= padding; + iy0 -= padding; + ix1 += padding; + iy1 += padding; + + w = (ix1 - ix0); + h = (iy1 - iy0); + + if (width ) *width = w; + if (height) *height = h; + if (xoff ) *xoff = ix0; + if (yoff ) *yoff = iy0; + + // invert for y-downwards bitmaps + scale_y = -scale_y; + + { + // distance from singular values (in the same units as the pixel grid) + const float eps = 1./1024, eps2 = eps*eps; + int x,y,i,j; + float *precompute; + stbtt_vertex *verts; + int num_verts = stbtt_GetGlyphShape(info, glyph, &verts); + data = (unsigned char *) STBTT_malloc(w * h, info->userdata); + precompute = (float *) STBTT_malloc(num_verts * sizeof(float), info->userdata); + + for (i=0,j=num_verts-1; i < num_verts; j=i++) { + if (verts[i].type == STBTT_vline) { + float x0 = verts[i].x*scale_x, y0 = verts[i].y*scale_y; + float x1 = verts[j].x*scale_x, y1 = verts[j].y*scale_y; + float dist = (float) STBTT_sqrt((x1-x0)*(x1-x0) + (y1-y0)*(y1-y0)); + precompute[i] = (dist < eps) ? 0.0f : 1.0f / dist; + } else if (verts[i].type == STBTT_vcurve) { + float x2 = verts[j].x *scale_x, y2 = verts[j].y *scale_y; + float x1 = verts[i].cx*scale_x, y1 = verts[i].cy*scale_y; + float x0 = verts[i].x *scale_x, y0 = verts[i].y *scale_y; + float bx = x0 - 2*x1 + x2, by = y0 - 2*y1 + y2; + float len2 = bx*bx + by*by; + if (len2 >= eps2) + precompute[i] = 1.0f / len2; + else + precompute[i] = 0.0f; + } else + precompute[i] = 0.0f; + } + + for (y=iy0; y < iy1; ++y) { + for (x=ix0; x < ix1; ++x) { + float val; + float min_dist = 999999.0f; + float sx = (float) x + 0.5f; + float sy = (float) y + 0.5f; + float x_gspace = (sx / scale_x); + float y_gspace = (sy / scale_y); + + int winding = stbtt__compute_crossings_x(x_gspace, y_gspace, num_verts, verts); // @OPTIMIZE: this could just be a rasterization, but needs to be line vs. non-tesselated curves so a new path + + for (i=0; i < num_verts; ++i) { + float x0 = verts[i].x*scale_x, y0 = verts[i].y*scale_y; + + if (verts[i].type == STBTT_vline && precompute[i] != 0.0f) { + float x1 = verts[i-1].x*scale_x, y1 = verts[i-1].y*scale_y; + + float dist,dist2 = (x0-sx)*(x0-sx) + (y0-sy)*(y0-sy); + if (dist2 < min_dist*min_dist) + min_dist = (float) STBTT_sqrt(dist2); + + // coarse culling against bbox + //if (sx > STBTT_min(x0,x1)-min_dist && sx < STBTT_max(x0,x1)+min_dist && + // sy > STBTT_min(y0,y1)-min_dist && sy < STBTT_max(y0,y1)+min_dist) + dist = (float) STBTT_fabs((x1-x0)*(y0-sy) - (y1-y0)*(x0-sx)) * precompute[i]; + STBTT_assert(i != 0); + if (dist < min_dist) { + // check position along line + // x' = x0 + t*(x1-x0), y' = y0 + t*(y1-y0) + // minimize (x'-sx)*(x'-sx)+(y'-sy)*(y'-sy) + float dx = x1-x0, dy = y1-y0; + float px = x0-sx, py = y0-sy; + // minimize (px+t*dx)^2 + (py+t*dy)^2 = px*px + 2*px*dx*t + t^2*dx*dx + py*py + 2*py*dy*t + t^2*dy*dy + // derivative: 2*px*dx + 2*py*dy + (2*dx*dx+2*dy*dy)*t, set to 0 and solve + float t = -(px*dx + py*dy) / (dx*dx + dy*dy); + if (t >= 0.0f && t <= 1.0f) + min_dist = dist; + } + } else if (verts[i].type == STBTT_vcurve) { + float x2 = verts[i-1].x *scale_x, y2 = verts[i-1].y *scale_y; + float x1 = verts[i ].cx*scale_x, y1 = verts[i ].cy*scale_y; + float box_x0 = STBTT_min(STBTT_min(x0,x1),x2); + float box_y0 = STBTT_min(STBTT_min(y0,y1),y2); + float box_x1 = STBTT_max(STBTT_max(x0,x1),x2); + float box_y1 = STBTT_max(STBTT_max(y0,y1),y2); + // coarse culling against bbox to avoid computing cubic unnecessarily + if (sx > box_x0-min_dist && sx < box_x1+min_dist && sy > box_y0-min_dist && sy < box_y1+min_dist) { + int num=0; + float ax = x1-x0, ay = y1-y0; + float bx = x0 - 2*x1 + x2, by = y0 - 2*y1 + y2; + float mx = x0 - sx, my = y0 - sy; + float res[3] = {0.f,0.f,0.f}; + float px,py,t,it,dist2; + float a_inv = precompute[i]; + if (a_inv == 0.0) { // if a_inv is 0, it's 2nd degree so use quadratic formula + float a = 3*(ax*bx + ay*by); + float b = 2*(ax*ax + ay*ay) + (mx*bx+my*by); + float c = mx*ax+my*ay; + if (STBTT_fabs(a) < eps2) { // if a is 0, it's linear + if (STBTT_fabs(b) >= eps2) { + res[num++] = -c/b; + } + } else { + float discriminant = b*b - 4*a*c; + if (discriminant < 0) + num = 0; + else { + float root = (float) STBTT_sqrt(discriminant); + res[0] = (-b - root)/(2*a); + res[1] = (-b + root)/(2*a); + num = 2; // don't bother distinguishing 1-solution case, as code below will still work + } + } + } else { + float b = 3*(ax*bx + ay*by) * a_inv; // could precompute this as it doesn't depend on sample point + float c = (2*(ax*ax + ay*ay) + (mx*bx+my*by)) * a_inv; + float d = (mx*ax+my*ay) * a_inv; + num = stbtt__solve_cubic(b, c, d, res); + } + dist2 = (x0-sx)*(x0-sx) + (y0-sy)*(y0-sy); + if (dist2 < min_dist*min_dist) + min_dist = (float) STBTT_sqrt(dist2); + + if (num >= 1 && res[0] >= 0.0f && res[0] <= 1.0f) { + t = res[0], it = 1.0f - t; + px = it*it*x0 + 2*t*it*x1 + t*t*x2; + py = it*it*y0 + 2*t*it*y1 + t*t*y2; + dist2 = (px-sx)*(px-sx) + (py-sy)*(py-sy); + if (dist2 < min_dist * min_dist) + min_dist = (float) STBTT_sqrt(dist2); + } + if (num >= 2 && res[1] >= 0.0f && res[1] <= 1.0f) { + t = res[1], it = 1.0f - t; + px = it*it*x0 + 2*t*it*x1 + t*t*x2; + py = it*it*y0 + 2*t*it*y1 + t*t*y2; + dist2 = (px-sx)*(px-sx) + (py-sy)*(py-sy); + if (dist2 < min_dist * min_dist) + min_dist = (float) STBTT_sqrt(dist2); + } + if (num >= 3 && res[2] >= 0.0f && res[2] <= 1.0f) { + t = res[2], it = 1.0f - t; + px = it*it*x0 + 2*t*it*x1 + t*t*x2; + py = it*it*y0 + 2*t*it*y1 + t*t*y2; + dist2 = (px-sx)*(px-sx) + (py-sy)*(py-sy); + if (dist2 < min_dist * min_dist) + min_dist = (float) STBTT_sqrt(dist2); + } + } + } + } + if (winding == 0) + min_dist = -min_dist; // if outside the shape, value is negative + val = onedge_value + pixel_dist_scale * min_dist; + if (val < 0) + val = 0; + else if (val > 255) + val = 255; + data[(y-iy0)*w+(x-ix0)] = (unsigned char) val; + } + } + STBTT_free(precompute, info->userdata); + STBTT_free(verts, info->userdata); + } + return data; +} + +STBTT_DEF unsigned char * stbtt_GetCodepointSDF(const stbtt_fontinfo *info, float scale, int codepoint, int padding, unsigned char onedge_value, float pixel_dist_scale, int *width, int *height, int *xoff, int *yoff) +{ + return stbtt_GetGlyphSDF(info, scale, stbtt_FindGlyphIndex(info, codepoint), padding, onedge_value, pixel_dist_scale, width, height, xoff, yoff); +} + +STBTT_DEF void stbtt_FreeSDF(unsigned char *bitmap, void *userdata) +{ + STBTT_free(bitmap, userdata); +} + +////////////////////////////////////////////////////////////////////////////// +// +// font name matching -- recommended not to use this +// + +// check if a utf8 string contains a prefix which is the utf16 string; if so return length of matching utf8 string +static stbtt_int32 stbtt__CompareUTF8toUTF16_bigendian_prefix(stbtt_uint8 *s1, stbtt_int32 len1, stbtt_uint8 *s2, stbtt_int32 len2) +{ + stbtt_int32 i=0; + + // convert utf16 to utf8 and compare the results while converting + while (len2) { + stbtt_uint16 ch = s2[0]*256 + s2[1]; + if (ch < 0x80) { + if (i >= len1) return -1; + if (s1[i++] != ch) return -1; + } else if (ch < 0x800) { + if (i+1 >= len1) return -1; + if (s1[i++] != 0xc0 + (ch >> 6)) return -1; + if (s1[i++] != 0x80 + (ch & 0x3f)) return -1; + } else if (ch >= 0xd800 && ch < 0xdc00) { + stbtt_uint32 c; + stbtt_uint16 ch2 = s2[2]*256 + s2[3]; + if (i+3 >= len1) return -1; + c = ((ch - 0xd800) << 10) + (ch2 - 0xdc00) + 0x10000; + if (s1[i++] != 0xf0 + (c >> 18)) return -1; + if (s1[i++] != 0x80 + ((c >> 12) & 0x3f)) return -1; + if (s1[i++] != 0x80 + ((c >> 6) & 0x3f)) return -1; + if (s1[i++] != 0x80 + ((c ) & 0x3f)) return -1; + s2 += 2; // plus another 2 below + len2 -= 2; + } else if (ch >= 0xdc00 && ch < 0xe000) { + return -1; + } else { + if (i+2 >= len1) return -1; + if (s1[i++] != 0xe0 + (ch >> 12)) return -1; + if (s1[i++] != 0x80 + ((ch >> 6) & 0x3f)) return -1; + if (s1[i++] != 0x80 + ((ch ) & 0x3f)) return -1; + } + s2 += 2; + len2 -= 2; + } + return i; +} + +static int stbtt_CompareUTF8toUTF16_bigendian_internal(char *s1, int len1, char *s2, int len2) +{ + return len1 == stbtt__CompareUTF8toUTF16_bigendian_prefix((stbtt_uint8*) s1, len1, (stbtt_uint8*) s2, len2); +} + +// returns results in whatever encoding you request... but note that 2-byte encodings +// will be BIG-ENDIAN... use stbtt_CompareUTF8toUTF16_bigendian() to compare +STBTT_DEF const char *stbtt_GetFontNameString(const stbtt_fontinfo *font, int *length, int platformID, int encodingID, int languageID, int nameID) +{ + stbtt_int32 i,count,stringOffset; + stbtt_uint8 *fc = font->data; + stbtt_uint32 offset = font->fontstart; + stbtt_uint32 nm = stbtt__find_table(fc, offset, "name"); + if (!nm) return NULL; + + count = ttUSHORT(fc+nm+2); + stringOffset = nm + ttUSHORT(fc+nm+4); + for (i=0; i < count; ++i) { + stbtt_uint32 loc = nm + 6 + 12 * i; + if (platformID == ttUSHORT(fc+loc+0) && encodingID == ttUSHORT(fc+loc+2) + && languageID == ttUSHORT(fc+loc+4) && nameID == ttUSHORT(fc+loc+6)) { + *length = ttUSHORT(fc+loc+8); + return (const char *) (fc+stringOffset+ttUSHORT(fc+loc+10)); + } + } + return NULL; +} + +static int stbtt__matchpair(stbtt_uint8 *fc, stbtt_uint32 nm, stbtt_uint8 *name, stbtt_int32 nlen, stbtt_int32 target_id, stbtt_int32 next_id) +{ + stbtt_int32 i; + stbtt_int32 count = ttUSHORT(fc+nm+2); + stbtt_int32 stringOffset = nm + ttUSHORT(fc+nm+4); + + for (i=0; i < count; ++i) { + stbtt_uint32 loc = nm + 6 + 12 * i; + stbtt_int32 id = ttUSHORT(fc+loc+6); + if (id == target_id) { + // find the encoding + stbtt_int32 platform = ttUSHORT(fc+loc+0), encoding = ttUSHORT(fc+loc+2), language = ttUSHORT(fc+loc+4); + + // is this a Unicode encoding? + if (platform == 0 || (platform == 3 && encoding == 1) || (platform == 3 && encoding == 10)) { + stbtt_int32 slen = ttUSHORT(fc+loc+8); + stbtt_int32 off = ttUSHORT(fc+loc+10); + + // check if there's a prefix match + stbtt_int32 matchlen = stbtt__CompareUTF8toUTF16_bigendian_prefix(name, nlen, fc+stringOffset+off,slen); + if (matchlen >= 0) { + // check for target_id+1 immediately following, with same encoding & language + if (i+1 < count && ttUSHORT(fc+loc+12+6) == next_id && ttUSHORT(fc+loc+12) == platform && ttUSHORT(fc+loc+12+2) == encoding && ttUSHORT(fc+loc+12+4) == language) { + slen = ttUSHORT(fc+loc+12+8); + off = ttUSHORT(fc+loc+12+10); + if (slen == 0) { + if (matchlen == nlen) + return 1; + } else if (matchlen < nlen && name[matchlen] == ' ') { + ++matchlen; + if (stbtt_CompareUTF8toUTF16_bigendian_internal((char*) (name+matchlen), nlen-matchlen, (char*)(fc+stringOffset+off),slen)) + return 1; + } + } else { + // if nothing immediately following + if (matchlen == nlen) + return 1; + } + } + } + + // @TODO handle other encodings + } + } + return 0; +} + +static int stbtt__matches(stbtt_uint8 *fc, stbtt_uint32 offset, stbtt_uint8 *name, stbtt_int32 flags) +{ + stbtt_int32 nlen = (stbtt_int32) STBTT_strlen((char *) name); + stbtt_uint32 nm,hd; + if (!stbtt__isfont(fc+offset)) return 0; + + // check italics/bold/underline flags in macStyle... + if (flags) { + hd = stbtt__find_table(fc, offset, "head"); + if ((ttUSHORT(fc+hd+44) & 7) != (flags & 7)) return 0; + } + + nm = stbtt__find_table(fc, offset, "name"); + if (!nm) return 0; + + if (flags) { + // if we checked the macStyle flags, then just check the family and ignore the subfamily + if (stbtt__matchpair(fc, nm, name, nlen, 16, -1)) return 1; + if (stbtt__matchpair(fc, nm, name, nlen, 1, -1)) return 1; + if (stbtt__matchpair(fc, nm, name, nlen, 3, -1)) return 1; + } else { + if (stbtt__matchpair(fc, nm, name, nlen, 16, 17)) return 1; + if (stbtt__matchpair(fc, nm, name, nlen, 1, 2)) return 1; + if (stbtt__matchpair(fc, nm, name, nlen, 3, -1)) return 1; + } + + return 0; +} + +static int stbtt_FindMatchingFont_internal(unsigned char *font_collection, char *name_utf8, stbtt_int32 flags) +{ + stbtt_int32 i; + for (i=0;;++i) { + stbtt_int32 off = stbtt_GetFontOffsetForIndex(font_collection, i); + if (off < 0) return off; + if (stbtt__matches((stbtt_uint8 *) font_collection, off, (stbtt_uint8*) name_utf8, flags)) + return off; + } +} + +#if defined(__GNUC__) || defined(__clang__) +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wcast-qual" +#endif + +STBTT_DEF int stbtt_BakeFontBitmap(const unsigned char *data, int offset, + float pixel_height, unsigned char *pixels, int pw, int ph, + int first_char, int num_chars, stbtt_bakedchar *chardata) +{ + return stbtt_BakeFontBitmap_internal((unsigned char *) data, offset, pixel_height, pixels, pw, ph, first_char, num_chars, chardata); +} + +STBTT_DEF int stbtt_GetFontOffsetForIndex(const unsigned char *data, int index) +{ + return stbtt_GetFontOffsetForIndex_internal((unsigned char *) data, index); +} + +STBTT_DEF int stbtt_GetNumberOfFonts(const unsigned char *data) +{ + return stbtt_GetNumberOfFonts_internal((unsigned char *) data); +} + +STBTT_DEF int stbtt_InitFont(stbtt_fontinfo *info, const unsigned char *data, int offset) +{ + return stbtt_InitFont_internal(info, (unsigned char *) data, offset); +} + +STBTT_DEF int stbtt_FindMatchingFont(const unsigned char *fontdata, const char *name, int flags) +{ + return stbtt_FindMatchingFont_internal((unsigned char *) fontdata, (char *) name, flags); +} + +STBTT_DEF int stbtt_CompareUTF8toUTF16_bigendian(const char *s1, int len1, const char *s2, int len2) +{ + return stbtt_CompareUTF8toUTF16_bigendian_internal((char *) s1, len1, (char *) s2, len2); +} + +#if defined(__GNUC__) || defined(__clang__) +#pragma GCC diagnostic pop +#endif + +#endif // STB_TRUETYPE_IMPLEMENTATION + + +// FULL VERSION HISTORY +// +// 1.25 (2021-07-11) many fixes +// 1.24 (2020-02-05) fix warning +// 1.23 (2020-02-02) query SVG data for glyphs; query whole kerning table (but only kern not GPOS) +// 1.22 (2019-08-11) minimize missing-glyph duplication; fix kerning if both 'GPOS' and 'kern' are defined +// 1.21 (2019-02-25) fix warning +// 1.20 (2019-02-07) PackFontRange skips missing codepoints; GetScaleFontVMetrics() +// 1.19 (2018-02-11) OpenType GPOS kerning (horizontal only), STBTT_fmod +// 1.18 (2018-01-29) add missing function +// 1.17 (2017-07-23) make more arguments const; doc fix +// 1.16 (2017-07-12) SDF support +// 1.15 (2017-03-03) make more arguments const +// 1.14 (2017-01-16) num-fonts-in-TTC function +// 1.13 (2017-01-02) support OpenType fonts, certain Apple fonts +// 1.12 (2016-10-25) suppress warnings about casting away const with -Wcast-qual +// 1.11 (2016-04-02) fix unused-variable warning +// 1.10 (2016-04-02) allow user-defined fabs() replacement +// fix memory leak if fontsize=0.0 +// fix warning from duplicate typedef +// 1.09 (2016-01-16) warning fix; avoid crash on outofmem; use alloc userdata for PackFontRanges +// 1.08 (2015-09-13) document stbtt_Rasterize(); fixes for vertical & horizontal edges +// 1.07 (2015-08-01) allow PackFontRanges to accept arrays of sparse codepoints; +// allow PackFontRanges to pack and render in separate phases; +// fix stbtt_GetFontOFfsetForIndex (never worked for non-0 input?); +// fixed an assert() bug in the new rasterizer +// replace assert() with STBTT_assert() in new rasterizer +// 1.06 (2015-07-14) performance improvements (~35% faster on x86 and x64 on test machine) +// also more precise AA rasterizer, except if shapes overlap +// remove need for STBTT_sort +// 1.05 (2015-04-15) fix misplaced definitions for STBTT_STATIC +// 1.04 (2015-04-15) typo in example +// 1.03 (2015-04-12) STBTT_STATIC, fix memory leak in new packing, various fixes +// 1.02 (2014-12-10) fix various warnings & compile issues w/ stb_rect_pack, C++ +// 1.01 (2014-12-08) fix subpixel position when oversampling to exactly match +// non-oversampled; STBTT_POINT_SIZE for packed case only +// 1.00 (2014-12-06) add new PackBegin etc. API, w/ support for oversampling +// 0.99 (2014-09-18) fix multiple bugs with subpixel rendering (ryg) +// 0.9 (2014-08-07) support certain mac/iOS fonts without an MS platformID +// 0.8b (2014-07-07) fix a warning +// 0.8 (2014-05-25) fix a few more warnings +// 0.7 (2013-09-25) bugfix: subpixel glyph bug fixed in 0.5 had come back +// 0.6c (2012-07-24) improve documentation +// 0.6b (2012-07-20) fix a few more warnings +// 0.6 (2012-07-17) fix warnings; added stbtt_ScaleForMappingEmToPixels, +// stbtt_GetFontBoundingBox, stbtt_IsGlyphEmpty +// 0.5 (2011-12-09) bugfixes: +// subpixel glyph renderer computed wrong bounding box +// first vertex of shape can be off-curve (FreeSans) +// 0.4b (2011-12-03) fixed an error in the font baking example +// 0.4 (2011-12-01) kerning, subpixel rendering (tor) +// bugfixes for: +// codepoint-to-glyph conversion using table fmt=12 +// codepoint-to-glyph conversion using table fmt=4 +// stbtt_GetBakedQuad with non-square texture (Zer) +// updated Hello World! sample to use kerning and subpixel +// fixed some warnings +// 0.3 (2009-06-24) cmap fmt=12, compound shapes (MM) +// userdata, malloc-from-userdata, non-zero fill (stb) +// 0.2 (2009-03-11) Fix unsigned/signed char warnings +// 0.1 (2009-03-09) First public release +// + +/* +------------------------------------------------------------------------------ +This software is available under 2 licenses -- choose whichever you prefer. +------------------------------------------------------------------------------ +ALTERNATIVE A - MIT License +Copyright (c) 2017 Sean Barrett +Permission is hereby granted, free of charge, to any person obtaining a copy of +this software and associated documentation files (the "Software"), to deal in +the Software without restriction, including without limitation the rights to +use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies +of the Software, and to permit persons to whom the Software is furnished to do +so, subject to the following conditions: +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. +------------------------------------------------------------------------------ +ALTERNATIVE B - Public Domain (www.unlicense.org) +This is free and unencumbered software released into the public domain. +Anyone is free to copy, modify, publish, use, compile, sell, or distribute this +software, either in source code form or as a compiled binary, for any purpose, +commercial or non-commercial, and by any means. +In jurisdictions that recognize copyright laws, the author or authors of this +software dedicate any and all copyright interest in the software to the public +domain. We make this dedication for the benefit of the public at large and to +the detriment of our heirs and successors. We intend this dedication to be an +overt act of relinquishment in perpetuity of all present and future rights to +this software under copyright law. +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN +ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +------------------------------------------------------------------------------ +*/ diff --git a/3rdparty/tracy b/3rdparty/tracy new file mode 160000 index 000000000..00c079cec --- /dev/null +++ b/3rdparty/tracy @@ -0,0 +1 @@ +Subproject commit 00c079cec91e6a374519aa5719a073e7f4539c9d diff --git a/3rdparty/winpthread/CMakeLists.txt b/3rdparty/winpthread/CMakeLists.txt new file mode 100644 index 000000000..4b7c9c6e9 --- /dev/null +++ b/3rdparty/winpthread/CMakeLists.txt @@ -0,0 +1,8 @@ +if (MSVC AND CLANG) + +add_library(winpthread SHARED IMPORTED GLOBAL) + +set_target_properties(winpthread PROPERTIES IMPORTED_IMPLIB "${CMAKE_CURRENT_SOURCE_DIR}/lib/libwinpthread.dll.a") + +endif() + diff --git a/3rdparty/winpthread/COPYING b/3rdparty/winpthread/COPYING new file mode 100644 index 000000000..350770123 --- /dev/null +++ b/3rdparty/winpthread/COPYING @@ -0,0 +1,57 @@ +Copyright (c) 2011 mingw-w64 project + +Permission is hereby granted, free of charge, to any person obtaining a +copy of this software and associated documentation files (the "Software"), +to deal in the Software without restriction, including without limitation +the rights to use, copy, modify, merge, publish, distribute, sublicense, +and/or sell copies of the Software, and to permit persons to whom the +Software is furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +DEALINGS IN THE SOFTWARE. + + +/* + * Parts of this library are derived by: + * + * Posix Threads library for Microsoft Windows + * + * Use at own risk, there is no implied warranty to this code. + * It uses undocumented features of Microsoft Windows that can change + * at any time in the future. + * + * (C) 2010 Lockless Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without modification, + * are permitted provided that the following conditions are met: + * + * + * * Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * * Neither the name of Lockless Inc. nor the names of its contributors may be + * used to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AN + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, + * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, + * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF + * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE + * OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED + * OF THE POSSIBILITY OF SUCH DAMAGE. + */ diff --git a/3rdparty/winpthread/bin/libwinpthread-1.dll b/3rdparty/winpthread/bin/libwinpthread-1.dll new file mode 100644 index 000000000..f4ebd6381 Binary files /dev/null and b/3rdparty/winpthread/bin/libwinpthread-1.dll differ diff --git a/3rdparty/winpthread/include/pthread.h b/3rdparty/winpthread/include/pthread.h new file mode 100644 index 000000000..4c2affd36 --- /dev/null +++ b/3rdparty/winpthread/include/pthread.h @@ -0,0 +1,692 @@ +/* + Copyright (c) 2011-2016 mingw-w64 project + + Permission is hereby granted, free of charge, to any person obtaining a + copy of this software and associated documentation files (the "Software"), + to deal in the Software without restriction, including without limitation + the rights to use, copy, modify, merge, publish, distribute, sublicense, + and/or sell copies of the Software, and to permit persons to whom the + Software is furnished to do so, subject to the following conditions: + + The above copyright notice and this permission notice shall be included in + all copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + DEALINGS IN THE SOFTWARE. +*/ + +/* + * Parts of this library are derived by: + * + * Posix Threads library for Microsoft Windows + * + * Use at own risk, there is no implied warranty to this code. + * It uses undocumented features of Microsoft Windows that can change + * at any time in the future. + * + * (C) 2010 Lockless Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without modification, + * are permitted provided that the following conditions are met: + * + * + * * Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * * Neither the name of Lockless Inc. nor the names of its contributors may be + * used to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AN + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, + * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, + * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF + * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE + * OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED + * OF THE POSSIBILITY OF SUCH DAMAGE. + */ +#ifndef WIN_PTHREADS_H +#define WIN_PTHREADS_H + +#include +#include +#include + +#include +#include +#include + +#include + +#include "pthread_compat.h" + +#ifdef __cplusplus +extern "C" { +#endif + +#define __WINPTHREADS_VERSION_MAJOR 0 +#define __WINPTHREADS_VERSION_MINOR 5 +#define __WINPTHREADS_VERSION_PATCHLEVEL 0 + +/* MSB 8-bit major version, 8-bit minor version, 16-bit patch level. */ +#define __WINPTHREADS_VERSION 0x00050000 + +#if defined DLL_EXPORT +#ifdef IN_WINPTHREAD +#define WINPTHREAD_API __declspec(dllexport) +#else +#define WINPTHREAD_API __declspec(dllimport) +#endif +#else +#define WINPTHREAD_API +#endif + +/* #define WINPTHREAD_DBG 1 */ + +/* Compatibility stuff: */ +#define RWLS_PER_THREAD 8 + +/* Error-codes. */ +#ifndef ETIMEDOUT +#define ETIMEDOUT 138 +#endif +#ifndef ENOTSUP +#define ENOTSUP 129 +#endif +#ifndef EWOULDBLOCK +#define EWOULDBLOCK 140 +#endif + +/* pthread specific defines. */ + +#define PTHREAD_CANCEL_DISABLE 0 +#define PTHREAD_CANCEL_ENABLE 0x01 + +#define PTHREAD_CANCEL_DEFERRED 0 +#define PTHREAD_CANCEL_ASYNCHRONOUS 0x02 + +#define PTHREAD_CREATE_JOINABLE 0 +#define PTHREAD_CREATE_DETACHED 0x04 + +#define PTHREAD_EXPLICIT_SCHED 0 +#define PTHREAD_INHERIT_SCHED 0x08 + +#define PTHREAD_SCOPE_PROCESS 0 +#define PTHREAD_SCOPE_SYSTEM 0x10 + +#define PTHREAD_DEFAULT_ATTR (PTHREAD_CANCEL_ENABLE) + +#define PTHREAD_CANCELED ((void *) (intptr_t) 0xDEADBEEF) + +#define _PTHREAD_NULL_THREAD ((pthread_t) 0) + +#define PTHREAD_ONCE_INIT 0 + +#define PTHREAD_DESTRUCTOR_ITERATIONS 256 +#define PTHREAD_KEYS_MAX (1<<20) + +#define PTHREAD_MUTEX_NORMAL 0 +#define PTHREAD_MUTEX_ERRORCHECK 1 +#define PTHREAD_MUTEX_RECURSIVE 2 +#define PTHREAD_MUTEX_DEFAULT PTHREAD_MUTEX_NORMAL + +#define PTHREAD_MUTEX_SHARED 1 +#define PTHREAD_MUTEX_PRIVATE 0 + +#define PTHREAD_PRIO_NONE 0 +#define PTHREAD_PRIO_INHERIT 8 +#define PTHREAD_PRIO_PROTECT 16 +#define PTHREAD_PRIO_MULT 32 +#define PTHREAD_PROCESS_SHARED 1 +#define PTHREAD_PROCESS_PRIVATE 0 + +#define PTHREAD_MUTEX_FAST_NP PTHREAD_MUTEX_NORMAL +#define PTHREAD_MUTEX_TIMED_NP PTHREAD_MUTEX_FAST_NP +#define PTHREAD_MUTEX_ADAPTIVE_NP PTHREAD_MUTEX_FAST_NP +#define PTHREAD_MUTEX_ERRORCHECK_NP PTHREAD_MUTEX_ERRORCHECK +#define PTHREAD_MUTEX_RECURSIVE_NP PTHREAD_MUTEX_RECURSIVE + +void * WINPTHREAD_API pthread_timechange_handler_np(void * dummy); +int WINPTHREAD_API pthread_delay_np (const struct timespec *interval); +int WINPTHREAD_API pthread_num_processors_np(void); +int WINPTHREAD_API pthread_set_num_processors_np(int n); + +#define PTHREAD_BARRIER_SERIAL_THREAD 1 + +/* maximum number of times a read lock may be obtained */ +#define MAX_READ_LOCKS (INT_MAX - 1) + +/* No fork() in windows - so ignore this */ +#define pthread_atfork(F1,F2,F3) 0 + +/* unsupported stuff: */ +#define pthread_mutex_getprioceiling(M, P) ENOTSUP +#define pthread_mutex_setprioceiling(M, P) ENOTSUP +#define pthread_getcpuclockid(T, C) ENOTSUP +#define pthread_attr_getguardsize(A, S) ENOTSUP +#define pthread_attr_setgaurdsize(A, S) ENOTSUP + +typedef long pthread_once_t; +typedef unsigned pthread_mutexattr_t; +typedef unsigned pthread_key_t; +typedef void *pthread_barrierattr_t; +typedef int pthread_condattr_t; +typedef int pthread_rwlockattr_t; + +/* +struct _pthread_v; + +typedef struct pthread_t { + struct _pthread_v *p; + int x; +} pthread_t; +*/ + +typedef uintptr_t pthread_t; + +typedef struct _pthread_cleanup _pthread_cleanup; +struct _pthread_cleanup +{ + void (*func)(void *); + void *arg; + _pthread_cleanup *next; +}; + +#define pthread_cleanup_push(F, A)\ +{\ + const _pthread_cleanup _pthread_cup = {(F), (A), *pthread_getclean()};\ + __sync_synchronize();\ + *pthread_getclean() = (_pthread_cleanup *) &_pthread_cup;\ + __sync_synchronize() + +/* Note that if async cancelling is used, then there is a race here */ +#define pthread_cleanup_pop(E)\ + (*pthread_getclean() = _pthread_cup.next, ((E) ? (_pthread_cup.func((pthread_once_t *)_pthread_cup.arg)) : (void)0));} + +/* Windows doesn't have this, so declare it ourselves. */ +#ifndef _TIMESPEC_DEFINED +#define _TIMESPEC_DEFINED +struct timespec { + time_t tv_sec; /* Seconds */ + long tv_nsec; /* Nanoseconds */ +}; + +struct itimerspec { + struct timespec it_interval; /* Timer period */ + struct timespec it_value; /* Timer expiration */ +}; +#endif + +#ifndef SCHED_OTHER +/* Some POSIX realtime extensions, mostly stubbed */ +#define SCHED_OTHER 0 +#define SCHED_FIFO 1 +#define SCHED_RR 2 +#define SCHED_MIN SCHED_OTHER +#define SCHED_MAX SCHED_RR + +struct sched_param { + int sched_priority; +}; + +int WINPTHREAD_API sched_yield(void); +int WINPTHREAD_API sched_get_priority_min(int pol); +int WINPTHREAD_API sched_get_priority_max(int pol); +int WINPTHREAD_API sched_getscheduler(pid_t pid); +int WINPTHREAD_API sched_setscheduler(pid_t pid, int pol, const struct sched_param *param); + +#endif + +typedef struct pthread_attr_t pthread_attr_t; +struct pthread_attr_t +{ + unsigned p_state; + void *stack; + size_t s_size; + struct sched_param param; +}; + +int WINPTHREAD_API pthread_attr_setschedparam(pthread_attr_t *attr, const struct sched_param *param); +int WINPTHREAD_API pthread_attr_getschedparam(const pthread_attr_t *attr, struct sched_param *param); +int WINPTHREAD_API pthread_getschedparam(pthread_t thread, int *pol, struct sched_param *param); +int WINPTHREAD_API pthread_setschedparam(pthread_t thread, int pol, const struct sched_param *param); +int WINPTHREAD_API pthread_attr_setschedpolicy (pthread_attr_t *attr, int pol); +int WINPTHREAD_API pthread_attr_getschedpolicy (const pthread_attr_t *attr, int *pol); + +/* synchronization objects */ +typedef intptr_t pthread_spinlock_t; +typedef intptr_t pthread_mutex_t; +typedef intptr_t pthread_cond_t; +typedef intptr_t pthread_rwlock_t; +typedef void *pthread_barrier_t; + +#define PTHREAD_MUTEX_NORMAL 0 +#define PTHREAD_MUTEX_ERRORCHECK 1 +#define PTHREAD_MUTEX_RECURSIVE 2 + +#define GENERIC_INITIALIZER -1 +#define GENERIC_ERRORCHECK_INITIALIZER -2 +#define GENERIC_RECURSIVE_INITIALIZER -3 +#define GENERIC_NORMAL_INITIALIZER -1 +#define PTHREAD_MUTEX_INITIALIZER (pthread_mutex_t)GENERIC_INITIALIZER +#define PTHREAD_RECURSIVE_MUTEX_INITIALIZER (pthread_mutex_t)GENERIC_RECURSIVE_INITIALIZER +#define PTHREAD_ERRORCHECK_MUTEX_INITIALIZER (pthread_mutex_t)GENERIC_ERRORCHECK_INITIALIZER +#define PTHREAD_NORMAL_MUTEX_INITIALIZER (pthread_mutex_t)GENERIC_NORMAL_INITIALIZER +#define PTHREAD_DEFAULT_MUTEX_INITIALIZER PTHREAD_NORMAL_MUTEX_INITIALIZER +#define PTHREAD_COND_INITIALIZER (pthread_cond_t)GENERIC_INITIALIZER +#define PTHREAD_RWLOCK_INITIALIZER (pthread_rwlock_t)GENERIC_INITIALIZER +#define PTHREAD_SPINLOCK_INITIALIZER (pthread_spinlock_t)GENERIC_INITIALIZER + +extern void WINPTHREAD_API (**_pthread_key_dest)(void *); +int WINPTHREAD_API pthread_key_create(pthread_key_t *key, void (* dest)(void *)); +int WINPTHREAD_API pthread_key_delete(pthread_key_t key); +void * WINPTHREAD_API pthread_getspecific(pthread_key_t key); +int WINPTHREAD_API pthread_setspecific(pthread_key_t key, const void *value); + +pthread_t WINPTHREAD_API pthread_self(void); +int WINPTHREAD_API pthread_once(pthread_once_t *o, void (*func)(void)); +void WINPTHREAD_API pthread_testcancel(void); +int WINPTHREAD_API pthread_equal(pthread_t t1, pthread_t t2); +void WINPTHREAD_API pthread_tls_init(void); +void WINPTHREAD_API _pthread_cleanup_dest(pthread_t t); +int WINPTHREAD_API pthread_get_concurrency(int *val); +int WINPTHREAD_API pthread_set_concurrency(int val); +void WINPTHREAD_API pthread_exit(void *res); +void WINPTHREAD_API _pthread_invoke_cancel(void); +int WINPTHREAD_API pthread_cancel(pthread_t t); +int WINPTHREAD_API pthread_kill(pthread_t t, int sig); +unsigned WINPTHREAD_API _pthread_get_state(const pthread_attr_t *attr, unsigned flag); +int WINPTHREAD_API _pthread_set_state(pthread_attr_t *attr, unsigned flag, unsigned val); +int WINPTHREAD_API pthread_setcancelstate(int state, int *oldstate); +int WINPTHREAD_API pthread_setcanceltype(int type, int *oldtype); +int WINPTHREAD_API pthread_create_wrapper(void *args); +int WINPTHREAD_API pthread_create(pthread_t *th, const pthread_attr_t *attr, void *(* func)(void *), void *arg); +int WINPTHREAD_API pthread_join(pthread_t t, void **res); +int WINPTHREAD_API pthread_detach(pthread_t t); +int WINPTHREAD_API pthread_setname_np(pthread_t thread, const char *name); +int WINPTHREAD_API pthread_getname_np(pthread_t thread, char *name, size_t len); + + +int WINPTHREAD_API pthread_rwlock_init(pthread_rwlock_t *rwlock_, const pthread_rwlockattr_t *attr); +int WINPTHREAD_API pthread_rwlock_wrlock(pthread_rwlock_t *l); +int WINPTHREAD_API pthread_rwlock_timedwrlock(pthread_rwlock_t *rwlock, const struct timespec *ts); +int WINPTHREAD_API pthread_rwlock_rdlock(pthread_rwlock_t *l); +int WINPTHREAD_API pthread_rwlock_timedrdlock(pthread_rwlock_t *l, const struct timespec *ts); +int WINPTHREAD_API pthread_rwlock_unlock(pthread_rwlock_t *l); +int WINPTHREAD_API pthread_rwlock_tryrdlock(pthread_rwlock_t *l); +int WINPTHREAD_API pthread_rwlock_trywrlock(pthread_rwlock_t *l); +int WINPTHREAD_API pthread_rwlock_destroy (pthread_rwlock_t *l); + +int WINPTHREAD_API pthread_cond_init(pthread_cond_t *cv, const pthread_condattr_t *a); +int WINPTHREAD_API pthread_cond_destroy(pthread_cond_t *cv); +int WINPTHREAD_API pthread_cond_signal (pthread_cond_t *cv); +int WINPTHREAD_API pthread_cond_broadcast (pthread_cond_t *cv); +int WINPTHREAD_API pthread_cond_wait (pthread_cond_t *cv, pthread_mutex_t *external_mutex); +int WINPTHREAD_API pthread_cond_timedwait(pthread_cond_t *cv, pthread_mutex_t *external_mutex, const struct timespec *t); +int WINPTHREAD_API pthread_cond_timedwait_relative_np(pthread_cond_t *cv, pthread_mutex_t *external_mutex, const struct timespec *t); + +int WINPTHREAD_API pthread_mutex_lock(pthread_mutex_t *m); +int WINPTHREAD_API pthread_mutex_timedlock(pthread_mutex_t *m, const struct timespec *ts); +int WINPTHREAD_API pthread_mutex_unlock(pthread_mutex_t *m); +int WINPTHREAD_API pthread_mutex_trylock(pthread_mutex_t *m); +int WINPTHREAD_API pthread_mutex_init(pthread_mutex_t *m, const pthread_mutexattr_t *a); +int WINPTHREAD_API pthread_mutex_destroy(pthread_mutex_t *m); + +int WINPTHREAD_API pthread_barrier_destroy(pthread_barrier_t *b); +int WINPTHREAD_API pthread_barrier_init(pthread_barrier_t *b, const void *attr, unsigned int count); +int WINPTHREAD_API pthread_barrier_wait(pthread_barrier_t *b); + +int WINPTHREAD_API pthread_spin_init(pthread_spinlock_t *l, int pshared); +int WINPTHREAD_API pthread_spin_destroy(pthread_spinlock_t *l); +/* No-fair spinlock due to lack of knowledge of thread number. */ +int WINPTHREAD_API pthread_spin_lock(pthread_spinlock_t *l); +int WINPTHREAD_API pthread_spin_trylock(pthread_spinlock_t *l); +int WINPTHREAD_API pthread_spin_unlock(pthread_spinlock_t *l); + +int WINPTHREAD_API pthread_attr_init(pthread_attr_t *attr); +int WINPTHREAD_API pthread_attr_destroy(pthread_attr_t *attr); +int WINPTHREAD_API pthread_attr_setdetachstate(pthread_attr_t *a, int flag); +int WINPTHREAD_API pthread_attr_getdetachstate(const pthread_attr_t *a, int *flag); +int WINPTHREAD_API pthread_attr_setinheritsched(pthread_attr_t *a, int flag); +int WINPTHREAD_API pthread_attr_getinheritsched(const pthread_attr_t *a, int *flag); +int WINPTHREAD_API pthread_attr_setscope(pthread_attr_t *a, int flag); +int WINPTHREAD_API pthread_attr_getscope(const pthread_attr_t *a, int *flag); +int WINPTHREAD_API pthread_attr_getstackaddr(const pthread_attr_t *attr, void **stack); +int WINPTHREAD_API pthread_attr_setstackaddr(pthread_attr_t *attr, void *stack); +int WINPTHREAD_API pthread_attr_getstacksize(const pthread_attr_t *attr, size_t *size); +int WINPTHREAD_API pthread_attr_setstacksize(pthread_attr_t *attr, size_t size); + +int WINPTHREAD_API pthread_mutexattr_init(pthread_mutexattr_t *a); +int WINPTHREAD_API pthread_mutexattr_destroy(pthread_mutexattr_t *a); +int WINPTHREAD_API pthread_mutexattr_gettype(const pthread_mutexattr_t *a, int *type); +int WINPTHREAD_API pthread_mutexattr_settype(pthread_mutexattr_t *a, int type); +int WINPTHREAD_API pthread_mutexattr_getpshared(const pthread_mutexattr_t *a, int *type); +int WINPTHREAD_API pthread_mutexattr_setpshared(pthread_mutexattr_t * a, int type); +int WINPTHREAD_API pthread_mutexattr_getprotocol(const pthread_mutexattr_t *a, int *type); +int WINPTHREAD_API pthread_mutexattr_setprotocol(pthread_mutexattr_t *a, int type); +int WINPTHREAD_API pthread_mutexattr_getprioceiling(const pthread_mutexattr_t *a, int * prio); +int WINPTHREAD_API pthread_mutexattr_setprioceiling(pthread_mutexattr_t *a, int prio); +int WINPTHREAD_API pthread_getconcurrency(void); +int WINPTHREAD_API pthread_setconcurrency(int new_level); + +int WINPTHREAD_API pthread_condattr_destroy(pthread_condattr_t *a); +int WINPTHREAD_API pthread_condattr_init(pthread_condattr_t *a); +int WINPTHREAD_API pthread_condattr_getpshared(const pthread_condattr_t *a, int *s); +int WINPTHREAD_API pthread_condattr_setpshared(pthread_condattr_t *a, int s); + +#ifndef __clockid_t_defined +typedef int clockid_t; +#define __clockid_t_defined 1 +#endif /* __clockid_t_defined */ + +int WINPTHREAD_API pthread_condattr_getclock (const pthread_condattr_t *attr, + clockid_t *clock_id); +int WINPTHREAD_API pthread_condattr_setclock(pthread_condattr_t *attr, + clockid_t clock_id); +int WINPTHREAD_API __pthread_clock_nanosleep(clockid_t clock_id, int flags, const struct timespec *rqtp, struct timespec *rmtp); + +int WINPTHREAD_API pthread_barrierattr_init(void **attr); +int WINPTHREAD_API pthread_barrierattr_destroy(void **attr); +int WINPTHREAD_API pthread_barrierattr_setpshared(void **attr, int s); +int WINPTHREAD_API pthread_barrierattr_getpshared(void **attr, int *s); + +/* Private extensions for analysis and internal use. */ +struct _pthread_cleanup ** WINPTHREAD_API pthread_getclean (void); +void * WINPTHREAD_API pthread_gethandle (pthread_t t); +void * WINPTHREAD_API pthread_getevent (); + +unsigned long long WINPTHREAD_API _pthread_rel_time_in_ms(const struct timespec *ts); +unsigned long long WINPTHREAD_API _pthread_time_in_ms(void); +unsigned long long WINPTHREAD_API _pthread_time_in_ms_from_timespec(const struct timespec *ts); +int WINPTHREAD_API _pthread_tryjoin (pthread_t t, void **res); +int WINPTHREAD_API pthread_rwlockattr_destroy(pthread_rwlockattr_t *a); +int WINPTHREAD_API pthread_rwlockattr_getpshared(pthread_rwlockattr_t *a, int *s); +int WINPTHREAD_API pthread_rwlockattr_init(pthread_rwlockattr_t *a); +int WINPTHREAD_API pthread_rwlockattr_setpshared(pthread_rwlockattr_t *a, int s); + +#ifndef SIG_BLOCK +#define SIG_BLOCK 0 +#endif +#ifndef SIG_UNBLOCK +#define SIG_UNBLOCK 1 +#endif +#ifndef SIG_SETMASK +#define SIG_SETMASK 2 +#endif + +#include + +#undef _POSIX_THREAD_DESTRUCTOR_ITERATIONS +#define _POSIX_THREAD_DESTRUCTOR_ITERATIONS PTHREAD_DESTRUCTOR_ITERATIONS + +#undef _POSIX_THREAD_KEYS_MAX +#define _POSIX_THREAD_KEYS_MAX PTHREAD_KEYS_MAX + +#undef PTHREAD_THREADS_MAX +#define PTHREAD_THREADS_MAX 2019 + +#undef _POSIX_SEM_NSEMS_MAX +#define _POSIX_SEM_NSEMS_MAX 256 + +#undef SEM_NSEMS_MAX +#define SEM_NSEMS_MAX 1024 + +/* Wrap cancellation points. */ +#ifdef __WINPTRHEAD_ENABLE_WRAP_API +#define accept(...) (pthread_testcancel(), accept(__VA_ARGS__)) +#define aio_suspend(...) (pthread_testcancel(), aio_suspend(__VA_ARGS__)) +#define clock_nanosleep(...) (pthread_testcancel(), clock_nanosleep(__VA_ARGS__)) +#define close(...) (pthread_testcancel(), close(__VA_ARGS__)) +#define connect(...) (pthread_testcancel(), connect(__VA_ARGS__)) +#define creat(...) (pthread_testcancel(), creat(__VA_ARGS__)) +#define fcntl(...) (pthread_testcancel(), fcntl(__VA_ARGS__)) +#define fdatasync(...) (pthread_testcancel(), fdatasync(__VA_ARGS__)) +#define fsync(...) (pthread_testcancel(), fsync(__VA_ARGS__)) +#define getmsg(...) (pthread_testcancel(), getmsg(__VA_ARGS__)) +#define getpmsg(...) (pthread_testcancel(), getpmsg(__VA_ARGS__)) +#define lockf(...) (pthread_testcancel(), lockf(__VA_ARGS__)) +#define mg_receive(...) (pthread_testcancel(), mg_receive(__VA_ARGS__)) +#define mg_send(...) (pthread_testcancel(), mg_send(__VA_ARGS__)) +#define mg_timedreceive(...) (pthread_testcancel(), mg_timedreceive(__VA_ARGS__)) +#define mg_timessend(...) (pthread_testcancel(), mg_timedsend(__VA_ARGS__)) +#define msgrcv(...) (pthread_testcancel(), msgrecv(__VA_ARGS__)) +#define msgsnd(...) (pthread_testcancel(), msgsnd(__VA_ARGS__)) +#define msync(...) (pthread_testcancel(), msync(__VA_ARGS__)) +#define nanosleep(...) (pthread_testcancel(), nanosleep(__VA_ARGS__)) +#define open(...) (pthread_testcancel(), open(__VA_ARGS__)) +#define pause(...) (pthread_testcancel(), pause(__VA_ARGS__)) +#define poll(...) (pthread_testcancel(), poll(__VA_ARGS__)) +#define pread(...) (pthread_testcancel(), pread(__VA_ARGS__)) +#define pselect(...) (pthread_testcancel(), pselect(__VA_ARGS__)) +#define putmsg(...) (pthread_testcancel(), putmsg(__VA_ARGS__)) +#define putpmsg(...) (pthread_testcancel(), putpmsg(__VA_ARGS__)) +#define pwrite(...) (pthread_testcancel(), pwrite(__VA_ARGS__)) +#define read(...) (pthread_testcancel(), read(__VA_ARGS__)) +#define readv(...) (pthread_testcancel(), readv(__VA_ARGS__)) +#define recv(...) (pthread_testcancel(), recv(__VA_ARGS__)) +#define recvfrom(...) (pthread_testcancel(), recvfrom(__VA_ARGS__)) +#define recvmsg(...) (pthread_testcancel(), recvmsg(__VA_ARGS__)) +#define select(...) (pthread_testcancel(), select(__VA_ARGS__)) +#define sem_timedwait(...) (pthread_testcancel(), sem_timedwait(__VA_ARGS__)) +#define sem_wait(...) (pthread_testcancel(), sem_wait(__VA_ARGS__)) +#define send(...) (pthread_testcancel(), send(__VA_ARGS__)) +#define sendmsg(...) (pthread_testcancel(), sendmsg(__VA_ARGS__)) +#define sendto(...) (pthread_testcancel(), sendto(__VA_ARGS__)) +#define sigpause(...) (pthread_testcancel(), sigpause(__VA_ARGS__)) +#define sigsuspend(...) (pthread_testcancel(), sigsuspend(__VA_ARGS__)) +#define sigwait(...) (pthread_testcancel(), sigwait(__VA_ARGS__)) +#define sigwaitinfo(...) (pthread_testcancel(), sigwaitinfo(__VA_ARGS__)) +#define sleep(...) (pthread_testcancel(), sleep(__VA_ARGS__)) +//#define Sleep(...) (pthread_testcancel(), Sleep(__VA_ARGS__)) +#define system(...) (pthread_testcancel(), system(__VA_ARGS__)) +#define access(...) (pthread_testcancel(), access(__VA_ARGS__)) +#define asctime(...) (pthread_testcancel(), asctime(__VA_ARGS__)) +#define catclose(...) (pthread_testcancel(), catclose(__VA_ARGS__)) +#define catgets(...) (pthread_testcancel(), catgets(__VA_ARGS__)) +#define catopen(...) (pthread_testcancel(), catopen(__VA_ARGS__)) +#define closedir(...) (pthread_testcancel(), closedir(__VA_ARGS__)) +#define closelog(...) (pthread_testcancel(), closelog(__VA_ARGS__)) +#define ctermid(...) (pthread_testcancel(), ctermid(__VA_ARGS__)) +#define ctime(...) (pthread_testcancel(), ctime(__VA_ARGS__)) +#define dbm_close(...) (pthread_testcancel(), dbm_close(__VA_ARGS__)) +#define dbm_delete(...) (pthread_testcancel(), dbm_delete(__VA_ARGS__)) +#define dbm_fetch(...) (pthread_testcancel(), dbm_fetch(__VA_ARGS__)) +#define dbm_nextkey(...) (pthread_testcancel(), dbm_nextkey(__VA_ARGS__)) +#define dbm_open(...) (pthread_testcancel(), dbm_open(__VA_ARGS__)) +#define dbm_store(...) (pthread_testcancel(), dbm_store(__VA_ARGS__)) +#define dlclose(...) (pthread_testcancel(), dlclose(__VA_ARGS__)) +#define dlopen(...) (pthread_testcancel(), dlopen(__VA_ARGS__)) +#define endgrent(...) (pthread_testcancel(), endgrent(__VA_ARGS__)) +#define endhostent(...) (pthread_testcancel(), endhostent(__VA_ARGS__)) +#define endnetent(...) (pthread_testcancel(), endnetent(__VA_ARGS__)) +#define endprotoent(...) (pthread_testcancel(), endprotoend(__VA_ARGS__)) +#define endpwent(...) (pthread_testcancel(), endpwent(__VA_ARGS__)) +#define endservent(...) (pthread_testcancel(), endservent(__VA_ARGS__)) +#define endutxent(...) (pthread_testcancel(), endutxent(__VA_ARGS__)) +#define fclose(...) (pthread_testcancel(), fclose(__VA_ARGS__)) +#define fflush(...) (pthread_testcancel(), fflush(__VA_ARGS__)) +#define fgetc(...) (pthread_testcancel(), fgetc(__VA_ARGS__)) +#define fgetpos(...) (pthread_testcancel(), fgetpos(__VA_ARGS__)) +#define fgets(...) (pthread_testcancel(), fgets(__VA_ARGS__)) +#define fgetwc(...) (pthread_testcancel(), fgetwc(__VA_ARGS__)) +#define fgetws(...) (pthread_testcancel(), fgetws(__VA_ARGS__)) +#define fmtmsg(...) (pthread_testcancel(), fmtmsg(__VA_ARGS__)) +#define fopen(...) (pthread_testcancel(), fopen(__VA_ARGS__)) +#define fpathconf(...) (pthread_testcancel(), fpathconf(__VA_ARGS__)) +#define fprintf(...) (pthread_testcancel(), fprintf(__VA_ARGS__)) +#define fputc(...) (pthread_testcancel(), fputc(__VA_ARGS__)) +#define fputs(...) (pthread_testcancel(), fputs(__VA_ARGS__)) +#define fputwc(...) (pthread_testcancel(), fputwc(__VA_ARGS__)) +#define fputws(...) (pthread_testcancel(), fputws(__VA_ARGS__)) +#define fread(...) (pthread_testcancel(), fread(__VA_ARGS__)) +#define freopen(...) (pthread_testcancel(), freopen(__VA_ARGS__)) +#define fscanf(...) (pthread_testcancel(), fscanf(__VA_ARGS__)) +#define fseek(...) (pthread_testcancel(), fseek(__VA_ARGS__)) +#define fseeko(...) (pthread_testcancel(), fseeko(__VA_ARGS__)) +#define fsetpos(...) (pthread_testcancel(), fsetpos(__VA_ARGS__)) +#define fstat(...) (pthread_testcancel(), fstat(__VA_ARGS__)) +#define ftell(...) (pthread_testcancel(), ftell(__VA_ARGS__)) +#define ftello(...) (pthread_testcancel(), ftello(__VA_ARGS__)) +#define ftw(...) (pthread_testcancel(), ftw(__VA_ARGS__)) +#define fwprintf(...) (pthread_testcancel(), fwprintf(__VA_ARGS__)) +#define fwrite(...) (pthread_testcancel(), fwrite(__VA_ARGS__)) +#define fwscanf(...) (pthread_testcancel(), fwscanf(__VA_ARGS__)) +#define getaddrinfo(...) (pthread_testcancel(), getaddrinfo(__VA_ARGS__)) +#define getc(...) (pthread_testcancel(), getc(__VA_ARGS__)) +#define getc_unlocked(...) (pthread_testcancel(), getc_unlocked(__VA_ARGS__)) +#define getchar(...) (pthread_testcancel(), getchar(__VA_ARGS__)) +#define getchar_unlocked(...) (pthread_testcancel(), getchar_unlocked(__VA_ARGS__)) +#define getcwd(...) (pthread_testcancel(), getcwd(__VA_ARGS__)) +#define getdate(...) (pthread_testcancel(), getdate(__VA_ARGS__)) +#define getgrent(...) (pthread_testcancel(), getgrent(__VA_ARGS__)) +#define getgrgid(...) (pthread_testcancel(), getgrgid(__VA_ARGS__)) +#define getgrgid_r(...) (pthread_testcancel(), getgrgid_r(__VA_ARGS__)) +#define gergrnam(...) (pthread_testcancel(), getgrnam(__VA_ARGS__)) +#define getgrnam_r(...) (pthread_testcancel(), getgrnam_r(__VA_ARGS__)) +#define gethostbyaddr(...) (pthread_testcancel(), gethostbyaddr(__VA_ARGS__)) +#define gethostbyname(...) (pthread_testcancel(), gethostbyname(__VA_ARGS__)) +#define gethostent(...) (pthread_testcancel(), gethostent(__VA_ARGS__)) +#define gethostid(...) (pthread_testcancel(), gethostid(__VA_ARGS__)) +#define gethostname(...) (pthread_testcancel(), gethostname(__VA_ARGS__)) +#define getlogin(...) (pthread_testcancel(), getlogin(__VA_ARGS__)) +#define getlogin_r(...) (pthread_testcancel(), getlogin_r(__VA_ARGS__)) +#define getnameinfo(...) (pthread_testcancel(), getnameinfo(__VA_ARGS__)) +#define getnetbyaddr(...) (pthread_testcancel(), getnetbyaddr(__VA_ARGS__)) +#define getnetbyname(...) (pthread_testcancel(), getnetbyname(__VA_ARGS__)) +#define getnetent(...) (pthread_testcancel(), getnetent(__VA_ARGS__)) +#define getopt(...) (pthread_testcancel(), getopt(__VA_ARGS__)) +#define getprotobyname(...) (pthread_testcancel(), getprotobyname(__VA_ARGS__)) +#define getprotobynumber(...) (pthread_testcancel(), getprotobynumber(__VA_ARGS__)) +#define getprotoent(...) (pthread_testcancel(), getprotoent(__VA_ARGS__)) +#define getpwent(...) (pthread_testcancel(), getpwent(__VA_ARGS__)) +#define getpwnam(...) (pthread_testcancel(), getpwnam(__VA_ARGS__)) +#define getpwnam_r(...) (pthread_testcancel(), getpwnam_r(__VA_ARGS__)) +#define getpwuid(...) (pthread_testcancel(), getpwuid(__VA_ARGS__)) +#define getpwuid_r(...) (pthread_testcancel(), getpwuid_r(__VA_ARGS__)) +#define gets(...) (pthread_testcancel(), gets(__VA_ARGS__)) +#define getservbyname(...) (pthread_testcancel(), getservbyname(__VA_ARGS__)) +#define getservbyport(...) (pthread_testcancel(), getservbyport(__VA_ARGS__)) +#define getservent(...) (pthread_testcancel(), getservent(__VA_ARGS__)) +#define getutxent(...) (pthread_testcancel(), getutxent(__VA_ARGS__)) +#define getutxid(...) (pthread_testcancel(), getutxid(__VA_ARGS__)) +#define getutxline(...) (pthread_testcancel(), getutxline(__VA_ARGS__)) +#undef getwc +#define getwc(...) (pthread_testcancel(), getwc(__VA_ARGS__)) +#undef getwchar +#define getwchar(...) (pthread_testcancel(), getwchar(__VA_ARGS__)) +#define getwd(...) (pthread_testcancel(), getwd(__VA_ARGS__)) +#define glob(...) (pthread_testcancel(), glob(__VA_ARGS__)) +#define iconv_close(...) (pthread_testcancel(), iconv_close(__VA_ARGS__)) +#define iconv_open(...) (pthread_testcancel(), iconv_open(__VA_ARGS__)) +#define ioctl(...) (pthread_testcancel(), ioctl(__VA_ARGS__)) +#define link(...) (pthread_testcancel(), link(__VA_ARGS__)) +#define localtime(...) (pthread_testcancel(), localtime(__VA_ARGS__)) +#define lseek(...) (pthread_testcancel(), lseek(__VA_ARGS__)) +#define lstat(...) (pthread_testcancel(), lstat(__VA_ARGS__)) +#define mkstemp(...) (pthread_testcancel(), mkstemp(__VA_ARGS__)) +#define nftw(...) (pthread_testcancel(), nftw(__VA_ARGS__)) +#define opendir(...) (pthread_testcancel(), opendir(__VA_ARGS__)) +#define openlog(...) (pthread_testcancel(), openlog(__VA_ARGS__)) +#define pathconf(...) (pthread_testcancel(), pathconf(__VA_ARGS__)) +#define pclose(...) (pthread_testcancel(), pclose(__VA_ARGS__)) +#define perror(...) (pthread_testcancel(), perror(__VA_ARGS__)) +#define popen(...) (pthread_testcancel(), popen(__VA_ARGS__)) +#define posix_fadvise(...) (pthread_testcancel(), posix_fadvise(__VA_ARGS__)) +#define posix_fallocate(...) (pthread_testcancel(), posix_fallocate(__VA_ARGS__)) +#define posix_madvise(...) (pthread_testcancel(), posix_madvise(__VA_ARGS__)) +#define posix_openpt(...) (pthread_testcancel(), posix_openpt(__VA_ARGS__)) +#define posix_spawn(...) (pthread_testcancel(), posix_spawn(__VA_ARGS__)) +#define posix_spawnp(...) (pthread_testcancel(), posix_spawnp(__VA_ARGS__)) +#define posix_trace_clear(...) (pthread_testcancel(), posix_trace_clear(__VA_ARGS__)) +#define posix_trace_close(...) (pthread_testcancel(), posix_trace_close(__VA_ARGS__)) +#define posix_trace_create(...) (pthread_testcancel(), posix_trace_create(__VA_ARGS__)) +#define posix_trace_create_withlog(...) (pthread_testcancel(), posix_trace_create_withlog(__VA_ARGS__)) +#define posix_trace_eventtypelist_getne(...) (pthread_testcancel(), posix_trace_eventtypelist_getne(__VA_ARGS__)) +#define posix_trace_eventtypelist_rewin(...) (pthread_testcancel(), posix_trace_eventtypelist_rewin(__VA_ARGS__)) +#define posix_trace_flush(...) (pthread_testcancel(), posix_trace_flush(__VA_ARGS__)) +#define posix_trace_get_attr(...) (pthread_testcancel(), posix_trace_get_attr(__VA_ARGS__)) +#define posix_trace_get_filter(...) (pthread_testcancel(), posix_trace_get_filter(__VA_ARGS__)) +#define posix_trace_get_status(...) (pthread_testcancel(), posix_trace_get_status(__VA_ARGS__)) +#define posix_trace_getnext_event(...) (pthread_testcancel(), posix_trace_getnext_event(__VA_ARGS__)) +#define posix_trace_open(...) (pthread_testcancel(), posix_trace_open(__VA_ARGS__)) +#define posix_trace_rewind(...) (pthread_testcancel(), posix_trace_rewind(__VA_ARGS__)) +#define posix_trace_setfilter(...) (pthread_testcancel(), posix_trace_setfilter(__VA_ARGS__)) +#define posix_trace_shutdown(...) (pthread_testcancel(), posix_trace_shutdown(__VA_ARGS__)) +#define posix_trace_timedgetnext_event(...) (pthread_testcancel(), posix_trace_timedgetnext_event(__VA_ARGS__)) +#define posix_typed_mem_open(...) (pthread_testcancel(), posix_typed_mem_open(__VA_ARGS__)) +#define printf(...) (pthread_testcancel(), printf(__VA_ARGS__)) +#define putc(...) (pthread_testcancel(), putc(__VA_ARGS__)) +#define putc_unlocked(...) (pthread_testcancel(), putc_unlocked(__VA_ARGS__)) +#define putchar(...) (pthread_testcancel(), putchar(__VA_ARGS__)) +#define putchar_unlocked(...) (pthread_testcancel(), putchar_unlocked(__VA_ARGS__)) +#define puts(...) (pthread_testcancel(), puts(__VA_ARGS__)) +#define pututxline(...) (pthread_testcancel(), pututxline(__VA_ARGS__)) +#undef putwc +#define putwc(...) (pthread_testcancel(), putwc(__VA_ARGS__)) +#undef putwchar +#define putwchar(...) (pthread_testcancel(), putwchar(__VA_ARGS__)) +#define readdir(...) (pthread_testcancel(), readdir(__VA_ARSG__)) +#define readdir_r(...) (pthread_testcancel(), readdir_r(__VA_ARGS__)) +#define remove(...) (pthread_testcancel(), remove(__VA_ARGS__)) +#define rename(...) (pthread_testcancel(), rename(__VA_ARGS__)) +#define rewind(...) (pthread_testcancel(), rewind(__VA_ARGS__)) +#define rewinddir(...) (pthread_testcancel(), rewinddir(__VA_ARGS__)) +#define scanf(...) (pthread_testcancel(), scanf(__VA_ARGS__)) +#define seekdir(...) (pthread_testcancel(), seekdir(__VA_ARGS__)) +#define semop(...) (pthread_testcancel(), semop(__VA_ARGS__)) +#define setgrent(...) (pthread_testcancel(), setgrent(__VA_ARGS__)) +#define sethostent(...) (pthread_testcancel(), sethostemt(__VA_ARGS__)) +#define setnetent(...) (pthread_testcancel(), setnetent(__VA_ARGS__)) +#define setprotoent(...) (pthread_testcancel(), setprotoent(__VA_ARGS__)) +#define setpwent(...) (pthread_testcancel(), setpwent(__VA_ARGS__)) +#define setservent(...) (pthread_testcancel(), setservent(__VA_ARGS__)) +#define setutxent(...) (pthread_testcancel(), setutxent(__VA_ARGS__)) +#define stat(...) (pthread_testcancel(), stat(__VA_ARGS__)) +#define strerror(...) (pthread_testcancel(), strerror(__VA_ARGS__)) +#define strerror_r(...) (pthread_testcancel(), strerror_r(__VA_ARGS__)) +#define strftime(...) (pthread_testcancel(), strftime(__VA_ARGS__)) +#define symlink(...) (pthread_testcancel(), symlink(__VA_ARGS__)) +#define sync(...) (pthread_testcancel(), sync(__VA_ARGS__)) +#define syslog(...) (pthread_testcancel(), syslog(__VA_ARGS__)) +#define tmpfile(...) (pthread_testcancel(), tmpfile(__VA_ARGS__)) +#define tmpnam(...) (pthread_testcancel(), tmpnam(__VA_ARGS__)) +#define ttyname(...) (pthread_testcancel(), ttyname(__VA_ARGS__)) +#define ttyname_r(...) (pthread_testcancel(), ttyname_r(__VA_ARGS__)) +#define tzset(...) (pthread_testcancel(), tzset(__VA_ARGS__)) +#define ungetc(...) (pthread_testcancel(), ungetc(__VA_ARGS__)) +#define ungetwc(...) (pthread_testcancel(), ungetwc(__VA_ARGS__)) +#define unlink(...) (pthread_testcancel(), unlink(__VA_ARGS__)) +#define vfprintf(...) (pthread_testcancel(), vfprintf(__VA_ARGS__)) +#define vfwprintf(...) (pthread_testcancel(), vfwprintf(__VA_ARGS__)) +#define vprintf(...) (pthread_testcancel(), vprintf(__VA_ARGS__)) +#define vwprintf(...) (pthread_testcancel(), vwprintf(__VA_ARGS__)) +#define wcsftime(...) (pthread_testcancel(), wcsftime(__VA_ARGS__)) +#define wordexp(...) (pthread_testcancel(), wordexp(__VA_ARGS__)) +#define wprintf(...) (pthread_testcancel(), wprintf(__VA_ARGS__)) +#define wscanf(...) (pthread_testcancel(), wscanf(__VA_ARGS__)) +#endif + +/* We deal here with a gcc issue for posix threading on Windows. + We would need to change here gcc's gthr-posix.h header, but this + got rejected. So we deal it within this header. */ +#ifdef _GTHREAD_USE_MUTEX_INIT_FUNC +#undef _GTHREAD_USE_MUTEX_INIT_FUNC +#endif +#define _GTHREAD_USE_MUTEX_INIT_FUNC 1 + +#ifdef __cplusplus +} +#endif + +#endif /* WIN_PTHREADS_H */ diff --git a/3rdparty/winpthread/include/pthread_compat.h b/3rdparty/winpthread/include/pthread_compat.h new file mode 100644 index 000000000..63f5f495f --- /dev/null +++ b/3rdparty/winpthread/include/pthread_compat.h @@ -0,0 +1,86 @@ +/* + Copyright (c) 2011-2016 mingw-w64 project + + Permission is hereby granted, free of charge, to any person obtaining a + copy of this software and associated documentation files (the "Software"), + to deal in the Software without restriction, including without limitation + the rights to use, copy, modify, merge, publish, distribute, sublicense, + and/or sell copies of the Software, and to permit persons to whom the + Software is furnished to do so, subject to the following conditions: + + The above copyright notice and this permission notice shall be included in + all copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + DEALINGS IN THE SOFTWARE. +*/ + +/* + * Parts of this library are derived by: + * + * Posix Threads library for Microsoft Windows + * + * Use at own risk, there is no implied warranty to this code. + * It uses undocumented features of Microsoft Windows that can change + * at any time in the future. + * + * (C) 2010 Lockless Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without modification, + * are permitted provided that the following conditions are met: + * + * + * * Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * * Neither the name of Lockless Inc. nor the names of its contributors may be + * used to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AN + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, + * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, + * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF + * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE + * OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED + * OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef WIN_PTHREADS_PTHREAD_COMPAT_H +#define WIN_PTHREADS_PTHREAD_COMPAT_H + +#ifdef __GNUC__ + +#define WINPTHREADS_INLINE inline +#define WINPTHREADS_ATTRIBUTE(X) __attribute__(X) +#define WINPTHREADS_SECTION(X) __section__(X) + +#elif _MSC_VER + +#include "pthread_time.h" + +#ifdef _WIN64 +typedef __int64 pid_t; +#else +typedef int pid_t; +#endif +typedef int clockid_t; + +#define WINPTHREADS_INLINE __inline +#define WINPTHREADS_ATTRIBUTE(X) __declspec X +#define WINPTHREADS_SECTION(X) allocate(X) + +#endif + +#endif diff --git a/3rdparty/winpthread/include/pthread_signal.h b/3rdparty/winpthread/include/pthread_signal.h new file mode 100644 index 000000000..42a50ae06 --- /dev/null +++ b/3rdparty/winpthread/include/pthread_signal.h @@ -0,0 +1,29 @@ +/* + Copyright (c) 2013-2016 mingw-w64 project + + Permission is hereby granted, free of charge, to any person obtaining a + copy of this software and associated documentation files (the "Software"), + to deal in the Software without restriction, including without limitation + the rights to use, copy, modify, merge, publish, distribute, sublicense, + and/or sell copies of the Software, and to permit persons to whom the + Software is furnished to do so, subject to the following conditions: + + The above copyright notice and this permission notice shall be included in + all copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + DEALINGS IN THE SOFTWARE. +*/ + +#ifndef WIN_PTHREADS_SIGNAL_H +#define WIN_PTHREADS_SIGNAL_H + +/* Windows has rudimentary signals support. */ +#define pthread_sigmask(H, S1, S2) 0 + +#endif /* WIN_PTHREADS_SIGNAL_H */ diff --git a/3rdparty/winpthread/include/pthread_time.h b/3rdparty/winpthread/include/pthread_time.h new file mode 100644 index 000000000..7a7538c39 --- /dev/null +++ b/3rdparty/winpthread/include/pthread_time.h @@ -0,0 +1,98 @@ +/* + Copyright (c) 2011-2016 mingw-w64 project + + Permission is hereby granted, free of charge, to any person obtaining a + copy of this software and associated documentation files (the "Software"), + to deal in the Software without restriction, including without limitation + the rights to use, copy, modify, merge, publish, distribute, sublicense, + and/or sell copies of the Software, and to permit persons to whom the + Software is furnished to do so, subject to the following conditions: + + The above copyright notice and this permission notice shall be included in + all copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + DEALINGS IN THE SOFTWARE. +*/ + +#include + +#ifndef WIN_PTHREADS_TIME_H +#define WIN_PTHREADS_TIME_H + +/* Posix timers are supported */ +#ifndef _POSIX_TIMERS +#define _POSIX_TIMERS 200809L +#endif + +/* Monotonic clocks are available. */ +#ifndef _POSIX_MONOTONIC_CLOCK +#define _POSIX_MONOTONIC_CLOCK 200809L +#endif + +/* CPU-time clocks are available. */ +#ifndef _POSIX_CPUTIME +#define _POSIX_CPUTIME 200809L +#endif + +/* Clock support in threads are available. */ +#ifndef _POSIX_THREAD_CPUTIME +#define _POSIX_THREAD_CPUTIME 200809L +#endif + +#ifndef __clockid_t_defined +typedef int clockid_t; +#define __clockid_t_defined 1 +#endif /* __clockid_t_defined */ + +#ifndef TIMER_ABSTIME +#define TIMER_ABSTIME 1 +#endif + +#ifndef CLOCK_REALTIME +#define CLOCK_REALTIME 0 +#endif + +#ifndef CLOCK_MONOTONIC +#define CLOCK_MONOTONIC 1 +#endif + +#ifndef CLOCK_PROCESS_CPUTIME_ID +#define CLOCK_PROCESS_CPUTIME_ID 2 +#endif + +#ifndef CLOCK_THREAD_CPUTIME_ID +#define CLOCK_THREAD_CPUTIME_ID 3 +#endif + +#ifdef __cplusplus +extern "C" { +#endif + +/* Make sure we provide default for WINPTHREAD_API, if not defined. */ +#pragma push_macro("WINPTHREAD_API") +#ifndef WINPTHREAD_API +#define WINPTHREAD_API +#endif + +/* These should really be dllimport'ed if using winpthread dll */ +int __cdecl WINPTHREAD_API nanosleep(const struct timespec *request, struct timespec *remain); + +int __cdecl WINPTHREAD_API clock_nanosleep(clockid_t clock_id, int flags, const struct timespec *request, struct timespec *remain); +int __cdecl WINPTHREAD_API clock_getres(clockid_t clock_id, struct timespec *res); +int __cdecl WINPTHREAD_API clock_gettime(clockid_t clock_id, struct timespec *tp); +int __cdecl WINPTHREAD_API clock_settime(clockid_t clock_id, const struct timespec *tp); + +#pragma pop_macro("WINPTHREAD_API") + +#ifdef __cplusplus +} +#endif + +#endif /* WIN_PTHREADS_TIME_H */ + diff --git a/3rdparty/winpthread/include/pthread_unistd.h b/3rdparty/winpthread/include/pthread_unistd.h new file mode 100644 index 000000000..646935344 --- /dev/null +++ b/3rdparty/winpthread/include/pthread_unistd.h @@ -0,0 +1,192 @@ +/* + Copyright (c) 2011-2016 mingw-w64 project + + Permission is hereby granted, free of charge, to any person obtaining a + copy of this software and associated documentation files (the "Software"), + to deal in the Software without restriction, including without limitation + the rights to use, copy, modify, merge, publish, distribute, sublicense, + and/or sell copies of the Software, and to permit persons to whom the + Software is furnished to do so, subject to the following conditions: + + The above copyright notice and this permission notice shall be included in + all copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + DEALINGS IN THE SOFTWARE. +*/ + +#ifndef WIN_PTHREADS_UNISTD_H +#define WIN_PTHREADS_UNISTD_H + +/* Set defines described by the POSIX Threads Extension (1003.1c-1995) */ +/* _SC_THREADS + Basic support for POSIX threads is available. The functions + + pthread_atfork(), + pthread_attr_destroy(), + pthread_attr_getdetachstate(), + pthread_attr_getschedparam(), + pthread_attr_init(), + pthread_attr_setdetachstate(), + pthread_attr_setschedparam(), + pthread_cancel(), + pthread_cleanup_push(), + pthread_cleanup_pop(), + pthread_cond_broadcast(), + pthread_cond_destroy(), + pthread_cond_init(), + pthread_cond_signal(), + pthread_cond_timedwait(), + pthread_cond_wait(), + pthread_condattr_destroy(), + pthread_condattr_init(), + pthread_create(), + pthread_detach(), + pthread_equal(), + pthread_exit(), + pthread_getspecific(), + pthread_join(, + pthread_key_create(), + pthread_key_delete(), + pthread_mutex_destroy(), + pthread_mutex_init(), + pthread_mutex_lock(), + pthread_mutex_trylock(), + pthread_mutex_unlock(), + pthread_mutexattr_destroy(), + pthread_mutexattr_init(), + pthread_once(), + pthread_rwlock_destroy(), + pthread_rwlock_init(), + pthread_rwlock_rdlock(), + pthread_rwlock_tryrdlock(), + pthread_rwlock_trywrlock(), + pthread_rwlock_unlock(), + pthread_rwlock_wrlock(), + pthread_rwlockattr_destroy(), + pthread_rwlockattr_init(), + pthread_self(), + pthread_setcancelstate(), + pthread_setcanceltype(), + pthread_setspecific(), + pthread_testcancel() + + are present. */ +#undef _POSIX_THREADS +#define _POSIX_THREADS 200112L + +/* _SC_READER_WRITER_LOCKS + This option implies the _POSIX_THREADS option. Conversely, under + POSIX 1003.1-2001 the _POSIX_THREADS option implies this option. + + The functions + pthread_rwlock_destroy(), + pthread_rwlock_init(), + pthread_rwlock_rdlock(), + pthread_rwlock_tryrdlock(), + pthread_rwlock_trywrlock(), + pthread_rwlock_unlock(), + pthread_rwlock_wrlock(), + pthread_rwlockattr_destroy(), + pthread_rwlockattr_init() + + are present. +*/ +#undef _POSIX_READER_WRITER_LOCKS +#define _POSIX_READER_WRITER_LOCKS 200112L + +/* _SC_SPIN_LOCKS + This option implies the _POSIX_THREADS and _POSIX_THREAD_SAFE_FUNCTIONS + options. The functions + + pthread_spin_destroy(), + pthread_spin_init(), + pthread_spin_lock(), + pthread_spin_trylock(), + pthread_spin_unlock() + + are present. */ +#undef _POSIX_SPIN_LOCKS +#define _POSIX_SPIN_LOCKS 200112L + +/* _SC_BARRIERS + This option implies the _POSIX_THREADS and _POSIX_THREAD_SAFE_FUNCTIONS + options. The functions + + pthread_barrier_destroy(), + pthread_barrier_init(), + pthread_barrier_wait(), + pthread_barrierattr_destroy(), + pthread_barrierattr_init() + + are present. +*/ +#undef _POSIX_BARRIERS +#define _POSIX_BARRIERS 200112L + +/* _SC_TIMEOUTS + The functions + + mq_timedreceive(), - not supported + mq_timedsend(), - not supported + posix_trace_timedgetnext_event(), - not supported + pthread_mutex_timedlock(), + pthread_rwlock_timedrdlock(), + pthread_rwlock_timedwrlock(), + sem_timedwait(), + + are present. */ +#undef _POSIX_TIMEOUTS +#define _POSIX_TIMEOUTS 200112L + +/* _SC_TIMERS - not supported + The functions + + clock_getres(), + clock_gettime(), + clock_settime(), + nanosleep(), + timer_create(), + timer_delete(), + timer_gettime(), + timer_getoverrun(), + timer_settime() + + are present. */ +/* #undef _POSIX_TIMERS */ + +/* _SC_CLOCK_SELECTION + This option implies the _POSIX_TIMERS option. The functions + + pthread_condattr_getclock(), + pthread_condattr_setclock(), + clock_nanosleep() + + are present. +*/ +#undef _POSIX_CLOCK_SELECTION +#define _POSIX_CLOCK_SELECTION 200112 + +/* _SC_SEMAPHORES + The include file is present. The functions + + sem_close(), + sem_destroy(), + sem_getvalue(), + sem_init(), + sem_open(), + sem_post(), + sem_trywait(), + sem_unlink(), + sem_wait() + + are present. */ +#undef _POSIX_SEMAPHORES +#define _POSIX_SEMAPHORES 200112 + +#endif /* WIN_PTHREADS_UNISTD_H */ diff --git a/3rdparty/winpthread/include/sched.h b/3rdparty/winpthread/include/sched.h new file mode 100644 index 000000000..e77bf4cc3 --- /dev/null +++ b/3rdparty/winpthread/include/sched.h @@ -0,0 +1,79 @@ +/* + Copyright (c) 2011-2016 mingw-w64 project + + Permission is hereby granted, free of charge, to any person obtaining a + copy of this software and associated documentation files (the "Software"), + to deal in the Software without restriction, including without limitation + the rights to use, copy, modify, merge, publish, distribute, sublicense, + and/or sell copies of the Software, and to permit persons to whom the + Software is furnished to do so, subject to the following conditions: + + The above copyright notice and this permission notice shall be included in + all copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + DEALINGS IN THE SOFTWARE. +*/ + +#include +#include +#include + +#include +#include +#include + +#include + +#ifndef WIN_PTHREADS_SCHED_H +#define WIN_PTHREADS_SCHED_H + +#ifndef SCHED_OTHER +/* Some POSIX realtime extensions, mostly stubbed */ +#define SCHED_OTHER 0 +#define SCHED_FIFO 1 +#define SCHED_RR 2 +#define SCHED_MIN SCHED_OTHER +#define SCHED_MAX SCHED_RR + +struct sched_param { + int sched_priority; +}; + +#ifdef __cplusplus +extern "C" { +#endif + +#if defined DLL_EXPORT && !defined (WINPTHREAD_EXPORT_ALL_DEBUG) +#ifdef IN_WINPTHREAD +#define WINPTHREAD_SCHED_API __declspec(dllexport) +#else +#define WINPTHREAD_SCHED_API __declspec(dllimport) +#endif +#else +#define WINPTHREAD_SCHED_API +#endif + +int WINPTHREAD_SCHED_API sched_yield(void); +int WINPTHREAD_SCHED_API sched_get_priority_min(int pol); +int WINPTHREAD_SCHED_API sched_get_priority_max(int pol); +int WINPTHREAD_SCHED_API sched_getscheduler(pid_t pid); +int WINPTHREAD_SCHED_API sched_setscheduler(pid_t pid, int pol, const struct sched_param *param); + +#ifdef __cplusplus +} +#endif + +#endif + +#ifndef sched_rr_get_interval +#define sched_rr_get_interval(_p, _i) \ + ( errno = ENOTSUP, (int) -1 ) +#endif + +#endif /* WIN_PTHREADS_SCHED_H */ diff --git a/3rdparty/winpthread/include/semaphore.h b/3rdparty/winpthread/include/semaphore.h new file mode 100644 index 000000000..14cb70371 --- /dev/null +++ b/3rdparty/winpthread/include/semaphore.h @@ -0,0 +1,81 @@ +/* + Copyright (c) 2011-2016 mingw-w64 project + + Permission is hereby granted, free of charge, to any person obtaining a + copy of this software and associated documentation files (the "Software"), + to deal in the Software without restriction, including without limitation + the rights to use, copy, modify, merge, publish, distribute, sublicense, + and/or sell copies of the Software, and to permit persons to whom the + Software is furnished to do so, subject to the following conditions: + + The above copyright notice and this permission notice shall be included in + all copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + DEALINGS IN THE SOFTWARE. +*/ + +#ifndef WIN_PTHREADS_SEMAPHORE_H +#define WIN_PTHREADS_SEMAPHORE_H + +#ifdef __cplusplus +extern "C" { +#endif + +#if defined DLL_EXPORT && !defined (WINPTHREAD_EXPORT_ALL_DEBUG) +#ifdef IN_WINPTHREAD +#define WINPTHREAD_SEMA_API __declspec(dllexport) +#else +#define WINPTHREAD_SEMA_API __declspec(dllimport) +#endif +#else +#define WINPTHREAD_SEMA_API +#endif + +/* Set this to 0 to disable it */ +#define USE_SEM_CriticalSection_SpinCount 100 + +#define SEM_VALUE_MAX INT_MAX + +#ifndef _MODE_T_ +#define _MODE_T_ +typedef unsigned short mode_t; +#endif + +typedef void *sem_t; + +#define SEM_FAILED NULL + +int WINPTHREAD_SEMA_API sem_init(sem_t * sem, int pshared, unsigned int value); + +int WINPTHREAD_SEMA_API sem_destroy(sem_t *sem); + +int WINPTHREAD_SEMA_API sem_trywait(sem_t *sem); + +int WINPTHREAD_SEMA_API sem_wait(sem_t *sem); + +int WINPTHREAD_SEMA_API sem_timedwait(sem_t * sem, const struct timespec *t); + +int WINPTHREAD_SEMA_API sem_post(sem_t *sem); + +int WINPTHREAD_SEMA_API sem_post_multiple(sem_t *sem, int count); + +/* yes, it returns a semaphore (or SEM_FAILED) */ +sem_t * WINPTHREAD_SEMA_API sem_open(const char * name, int oflag, mode_t mode, unsigned int value); + +int WINPTHREAD_SEMA_API sem_close(sem_t * sem); + +int WINPTHREAD_SEMA_API sem_unlink(const char * name); + +int WINPTHREAD_SEMA_API sem_getvalue(sem_t * sem, int * sval); + +#ifdef __cplusplus +} +#endif + +#endif /* WIN_PTHREADS_SEMAPHORE_H */ diff --git a/3rdparty/winpthread/lib/libwinpthread.dll.a b/3rdparty/winpthread/lib/libwinpthread.dll.a new file mode 100644 index 000000000..008444159 Binary files /dev/null and b/3rdparty/winpthread/lib/libwinpthread.dll.a differ diff --git a/3rdparty/xxHash b/3rdparty/xxHash new file mode 160000 index 000000000..e573d4d2a --- /dev/null +++ b/3rdparty/xxHash @@ -0,0 +1 @@ +Subproject commit e573d4d2aaeaba0f3e5a0a9a54144a1f2b4b56e7 diff --git a/LICENSE b/LICENSE new file mode 100644 index 000000000..ecbc05937 --- /dev/null +++ b/LICENSE @@ -0,0 +1,339 @@ + GNU GENERAL PUBLIC LICENSE + Version 2, June 1991 + + Copyright (C) 1989, 1991 Free Software Foundation, Inc., + 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + Everyone is permitted to copy and distribute verbatim copies + of this license document, but changing it is not allowed. + + Preamble + + The licenses for most software are designed to take away your +freedom to share and change it. By contrast, the GNU General Public +License is intended to guarantee your freedom to share and change free +software--to make sure the software is free for all its users. This +General Public License applies to most of the Free Software +Foundation's software and to any other program whose authors commit to +using it. (Some other Free Software Foundation software is covered by +the GNU Lesser General Public License instead.) You can apply it to +your programs, too. + + When we speak of free software, we are referring to freedom, not +price. Our General Public Licenses are designed to make sure that you +have the freedom to distribute copies of free software (and charge for +this service if you wish), that you receive source code or can get it +if you want it, that you can change the software or use pieces of it +in new free programs; and that you know you can do these things. + + To protect your rights, we need to make restrictions that forbid +anyone to deny you these rights or to ask you to surrender the rights. +These restrictions translate to certain responsibilities for you if you +distribute copies of the software, or if you modify it. + + For example, if you distribute copies of such a program, whether +gratis or for a fee, you must give the recipients all the rights that +you have. You must make sure that they, too, receive or can get the +source code. And you must show them these terms so they know their +rights. + + We protect your rights with two steps: (1) copyright the software, and +(2) offer you this license which gives you legal permission to copy, +distribute and/or modify the software. + + Also, for each author's protection and ours, we want to make certain +that everyone understands that there is no warranty for this free +software. If the software is modified by someone else and passed on, we +want its recipients to know that what they have is not the original, so +that any problems introduced by others will not reflect on the original +authors' reputations. + + Finally, any free program is threatened constantly by software +patents. We wish to avoid the danger that redistributors of a free +program will individually obtain patent licenses, in effect making the +program proprietary. To prevent this, we have made it clear that any +patent must be licensed for everyone's free use or not licensed at all. + + The precise terms and conditions for copying, distribution and +modification follow. + + GNU GENERAL PUBLIC LICENSE + TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION + + 0. This License applies to any program or other work which contains +a notice placed by the copyright holder saying it may be distributed +under the terms of this General Public License. The "Program", below, +refers to any such program or work, and a "work based on the Program" +means either the Program or any derivative work under copyright law: +that is to say, a work containing the Program or a portion of it, +either verbatim or with modifications and/or translated into another +language. (Hereinafter, translation is included without limitation in +the term "modification".) Each licensee is addressed as "you". + +Activities other than copying, distribution and modification are not +covered by this License; they are outside its scope. The act of +running the Program is not restricted, and the output from the Program +is covered only if its contents constitute a work based on the +Program (independent of having been made by running the Program). +Whether that is true depends on what the Program does. + + 1. You may copy and distribute verbatim copies of the Program's +source code as you receive it, in any medium, provided that you +conspicuously and appropriately publish on each copy an appropriate +copyright notice and disclaimer of warranty; keep intact all the +notices that refer to this License and to the absence of any warranty; +and give any other recipients of the Program a copy of this License +along with the Program. + +You may charge a fee for the physical act of transferring a copy, and +you may at your option offer warranty protection in exchange for a fee. + + 2. You may modify your copy or copies of the Program or any portion +of it, thus forming a work based on the Program, and copy and +distribute such modifications or work under the terms of Section 1 +above, provided that you also meet all of these conditions: + + a) You must cause the modified files to carry prominent notices + stating that you changed the files and the date of any change. + + b) You must cause any work that you distribute or publish, that in + whole or in part contains or is derived from the Program or any + part thereof, to be licensed as a whole at no charge to all third + parties under the terms of this License. + + c) If the modified program normally reads commands interactively + when run, you must cause it, when started running for such + interactive use in the most ordinary way, to print or display an + announcement including an appropriate copyright notice and a + notice that there is no warranty (or else, saying that you provide + a warranty) and that users may redistribute the program under + these conditions, and telling the user how to view a copy of this + License. (Exception: if the Program itself is interactive but + does not normally print such an announcement, your work based on + the Program is not required to print an announcement.) + +These requirements apply to the modified work as a whole. If +identifiable sections of that work are not derived from the Program, +and can be reasonably considered independent and separate works in +themselves, then this License, and its terms, do not apply to those +sections when you distribute them as separate works. But when you +distribute the same sections as part of a whole which is a work based +on the Program, the distribution of the whole must be on the terms of +this License, whose permissions for other licensees extend to the +entire whole, and thus to each and every part regardless of who wrote it. + +Thus, it is not the intent of this section to claim rights or contest +your rights to work written entirely by you; rather, the intent is to +exercise the right to control the distribution of derivative or +collective works based on the Program. + +In addition, mere aggregation of another work not based on the Program +with the Program (or with a work based on the Program) on a volume of +a storage or distribution medium does not bring the other work under +the scope of this License. + + 3. You may copy and distribute the Program (or a work based on it, +under Section 2) in object code or executable form under the terms of +Sections 1 and 2 above provided that you also do one of the following: + + a) Accompany it with the complete corresponding machine-readable + source code, which must be distributed under the terms of Sections + 1 and 2 above on a medium customarily used for software interchange; or, + + b) Accompany it with a written offer, valid for at least three + years, to give any third party, for a charge no more than your + cost of physically performing source distribution, a complete + machine-readable copy of the corresponding source code, to be + distributed under the terms of Sections 1 and 2 above on a medium + customarily used for software interchange; or, + + c) Accompany it with the information you received as to the offer + to distribute corresponding source code. (This alternative is + allowed only for noncommercial distribution and only if you + received the program in object code or executable form with such + an offer, in accord with Subsection b above.) + +The source code for a work means the preferred form of the work for +making modifications to it. For an executable work, complete source +code means all the source code for all modules it contains, plus any +associated interface definition files, plus the scripts used to +control compilation and installation of the executable. However, as a +special exception, the source code distributed need not include +anything that is normally distributed (in either source or binary +form) with the major components (compiler, kernel, and so on) of the +operating system on which the executable runs, unless that component +itself accompanies the executable. + +If distribution of executable or object code is made by offering +access to copy from a designated place, then offering equivalent +access to copy the source code from the same place counts as +distribution of the source code, even though third parties are not +compelled to copy the source along with the object code. + + 4. You may not copy, modify, sublicense, or distribute the Program +except as expressly provided under this License. Any attempt +otherwise to copy, modify, sublicense or distribute the Program is +void, and will automatically terminate your rights under this License. +However, parties who have received copies, or rights, from you under +this License will not have their licenses terminated so long as such +parties remain in full compliance. + + 5. You are not required to accept this License, since you have not +signed it. However, nothing else grants you permission to modify or +distribute the Program or its derivative works. These actions are +prohibited by law if you do not accept this License. Therefore, by +modifying or distributing the Program (or any work based on the +Program), you indicate your acceptance of this License to do so, and +all its terms and conditions for copying, distributing or modifying +the Program or works based on it. + + 6. Each time you redistribute the Program (or any work based on the +Program), the recipient automatically receives a license from the +original licensor to copy, distribute or modify the Program subject to +these terms and conditions. You may not impose any further +restrictions on the recipients' exercise of the rights granted herein. +You are not responsible for enforcing compliance by third parties to +this License. + + 7. If, as a consequence of a court judgment or allegation of patent +infringement or for any other reason (not limited to patent issues), +conditions are imposed on you (whether by court order, agreement or +otherwise) that contradict the conditions of this License, they do not +excuse you from the conditions of this License. If you cannot +distribute so as to satisfy simultaneously your obligations under this +License and any other pertinent obligations, then as a consequence you +may not distribute the Program at all. For example, if a patent +license would not permit royalty-free redistribution of the Program by +all those who receive copies directly or indirectly through you, then +the only way you could satisfy both it and this License would be to +refrain entirely from distribution of the Program. + +If any portion of this section is held invalid or unenforceable under +any particular circumstance, the balance of the section is intended to +apply and the section as a whole is intended to apply in other +circumstances. + +It is not the purpose of this section to induce you to infringe any +patents or other property right claims or to contest validity of any +such claims; this section has the sole purpose of protecting the +integrity of the free software distribution system, which is +implemented by public license practices. Many people have made +generous contributions to the wide range of software distributed +through that system in reliance on consistent application of that +system; it is up to the author/donor to decide if he or she is willing +to distribute software through any other system and a licensee cannot +impose that choice. + +This section is intended to make thoroughly clear what is believed to +be a consequence of the rest of this License. + + 8. If the distribution and/or use of the Program is restricted in +certain countries either by patents or by copyrighted interfaces, the +original copyright holder who places the Program under this License +may add an explicit geographical distribution limitation excluding +those countries, so that distribution is permitted only in or among +countries not thus excluded. In such case, this License incorporates +the limitation as if written in the body of this License. + + 9. The Free Software Foundation may publish revised and/or new versions +of the General Public License from time to time. Such new versions will +be similar in spirit to the present version, but may differ in detail to +address new problems or concerns. + +Each version is given a distinguishing version number. If the Program +specifies a version number of this License which applies to it and "any +later version", you have the option of following the terms and conditions +either of that version or of any later version published by the Free +Software Foundation. If the Program does not specify a version number of +this License, you may choose any version ever published by the Free Software +Foundation. + + 10. If you wish to incorporate parts of the Program into other free +programs whose distribution conditions are different, write to the author +to ask for permission. For software which is copyrighted by the Free +Software Foundation, write to the Free Software Foundation; we sometimes +make exceptions for this. Our decision will be guided by the two goals +of preserving the free status of all derivatives of our free software and +of promoting the sharing and reuse of software generally. + + NO WARRANTY + + 11. BECAUSE THE PROGRAM IS LICENSED FREE OF CHARGE, THERE IS NO WARRANTY +FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW. EXCEPT WHEN +OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES +PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED +OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. THE ENTIRE RISK AS +TO THE QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH YOU. SHOULD THE +PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING, +REPAIR OR CORRECTION. + + 12. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING +WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY AND/OR +REDISTRIBUTE THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, +INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING +OUT OF THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED +TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY +YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER +PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE +POSSIBILITY OF SUCH DAMAGES. + + END OF TERMS AND CONDITIONS + + How to Apply These Terms to Your New Programs + + If you develop a new program, and you want it to be of the greatest +possible use to the public, the best way to achieve this is to make it +free software which everyone can redistribute and change under these terms. + + To do so, attach the following notices to the program. It is safest +to attach them to the start of each source file to most effectively +convey the exclusion of warranty; and each file should have at least +the "copyright" line and a pointer to where the full notice is found. + + + Copyright (C) + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 2 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License along + with this program; if not, write to the Free Software Foundation, Inc., + 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +Also add information on how to contact you by electronic and paper mail. + +If the program is interactive, make it output a short notice like this +when it starts in an interactive mode: + + Gnomovision version 69, Copyright (C) year name of author + Gnomovision comes with ABSOLUTELY NO WARRANTY; for details type `show w'. + This is free software, and you are welcome to redistribute it + under certain conditions; type `show c' for details. + +The hypothetical commands `show w' and `show c' should show the appropriate +parts of the General Public License. Of course, the commands you use may +be called something other than `show w' and `show c'; they could even be +mouse-clicks or menu items--whatever suits your program. + +You should also get your employer (if you work as a programmer) or your +school, if any, to sign a "copyright disclaimer" for the program, if +necessary. Here is a sample; alter the names: + + Yoyodyne, Inc., hereby disclaims all copyright interest in the program + `Gnomovision' (which makes passes at compilers) written by James Hacker. + + , 1 April 1989 + Ty Coon, President of Vice + +This General Public License does not permit incorporating your program into +proprietary programs. If your program is a subroutine library, you may +consider it more useful to permit linking proprietary applications with the +library. If this is what you want to do, use the GNU Lesser General +Public License instead of this License. \ No newline at end of file diff --git a/LICENSES/Kyty-MIT.txt b/LICENSES/Kyty-MIT.txt new file mode 100644 index 000000000..0867cf0cb --- /dev/null +++ b/LICENSES/Kyty-MIT.txt @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2021 InoriRus + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. \ No newline at end of file diff --git a/README.md b/README.md new file mode 100644 index 000000000..429b99533 --- /dev/null +++ b/README.md @@ -0,0 +1,190 @@ +# KytyPS5 + +[![Platform](https://img.shields.io/badge/platform-Windows%20x64-0078D4.svg)](#system-requirements) +[![Status](https://img.shields.io/badge/status-early%20development-orange.svg)](#current-status) +[![License](https://img.shields.io/badge/license-GPL--2.0-blue.svg)](LICENSE) + +KytyPS5 is a free and open-source PlayStation 5 emulator written in C++ for Windows. It is based on +a heavily modified version of [Kyty](https://github.com/InoriRus/Kyty). The project is in an early +stage of development, so compatibility is limited and behavior may change significantly between +builds. + +> [!IMPORTANT] +> KytyPS5 is not affiliated with Sony Interactive Entertainment or PlayStation. The project does +> not distribute games or copyrighted system software. Use only game files that you have obtained +> legally. + +## Current Status + +KytyPS5 can boot 2D games and a selection of 3D games, including titles built with Unreal Engine +4/5, Unity, and custom engines. No external low-level emulation modules are currently required. + +Development is focused on compatibility and boot reliability. + +Linux support is planned, but Windows is the only supported platform at this time. + +## Bugs and Issues + +The project is in an early stage, so please be mindful when opening new issues. Expect crashes, +graphical glitches, low compatibility, and poor performance. + +## Screenshots + + + + + + + + + + +
+ Disgaea 6
+ Disgaea 6 running in KytyPS5 + 		+ Dreaming Sarah
+ Dreaming Sarah running in KytyPS5 +
+ Minecraft Legends
+ Minecraft Legends running in KytyPS5 + 		+ SILENT HILL: The Short Message
+ SILENT HILL: The Short Message running in KytyPS5 +
+ +## Contributing + +Testing games and submitting detailed bug reports are useful ways to contribute. Search existing +issues first, then use the **Game Emulation Bug Report** template and attach the complete log file. + +Code contributions should be focused, build successfully on Windows, and include relevant tests +where practical. Because KytyPS5 is still evolving quickly, consider opening an issue before +starting a large change. + +## Developer Information + +The PS5 graphics architecture is based on AMD RDNA 2. Use AMD's +[RDNA 2 Instruction Set Architecture Reference Guide (document 70648)](https://docs.amd.com/v/u/en-US/rdna2-shader-instruction-set-architecture) +as the primary instruction-encoding reference when working on shader decoding and recompilation. + +Important areas of the codebase: + +- [`src/graphics/shader/recompiler`](src/graphics/shader/recompiler) — instruction decoding, + intermediate representation, control flow, resource tracking, and SPIR-V emission +- [`src/graphics/guest_gpu`](src/graphics/guest_gpu) — PS5 (Prospero) GPU formats and command processing +- [`src/graphics/host_gpu`](src/graphics/host_gpu) — Vulkan host backend and resource management +- [`tests`](tests) — focused memory, shader, and resource-tracking regression tests + +The renderer targets Vulkan 1.3. Keep shader changes aligned with both the RDNA 2 ISA semantics and +the Vulkan/SPIR-V validation rules. + +## Building + +### System requirements + +- Windows 10 version 1803 +- A 64-bit x86 processor +- A Vulkan 1.3-capable GPU with current drivers + +### Build requirements + +- Git +- CMake 3.12 or newer +- Ninja +- Visual Studio 2022 or Build Tools 2022 with the **Desktop development with C++** workload and + **C++ Clang tools for Windows** component +- Qt 6 for MSVC 2022 64-bit, including Concurrent, Network, and Widgets +- Vulkan SDK 1.3 or newer + +The Microsoft C++ compiler (`cl.exe`) is not supported; use `clang-cl`. + +Open an **x64 Native Tools Command Prompt for Visual Studio 2022** (or the equivalent Developer +PowerShell), change to the repository root, and initialize the dependencies: + +```powershell +git submodule update --init --recursive +``` + +Configure the project. Replace the Qt path with the version installed on your system: + +```powershell +cmake -S src -B _Build/windows -G Ninja -DCMAKE_BUILD_TYPE=Release -DCMAKE_C_COMPILER=clang-cl -DCMAKE_CXX_COMPILER=clang-cl -DCMAKE_PREFIX_PATH="C:/Qt/6.x.x/msvc2022_64" +``` + +Build the launcher and stage a runnable installation: + +```powershell +cmake --build _Build/windows --target launcher +cmake --install _Build/windows --prefix _Build/windows/install +``` + +The finished application and its runtime dependencies will be placed in +`_Build/windows/install`. + +### Visual Studio Code + +A ready-made Visual Studio Code setup is included in [`.vscode`](.vscode). It configures CMake +Tools to build the project with Ninja and `clang-cl` and provides launch profiles for both +`launcher.exe` and `kyty_emulator.exe`. + +Before using it: + +1. Install the **CMake Tools** and **C/C++** extensions in Visual Studio Code. +2. Update `CMAKE_PREFIX_PATH` in [`.vscode/settings.json`](.vscode/settings.json) to point to your + Qt 6 MSVC installation. +3. Update the `--game` path in [`.vscode/launch.json`](.vscode/launch.json) for the + **Debug kyty_emulator** profile. +4. Open the repository in an x64 Visual Studio developer environment, configure the CMake project, + and select a launch profile from **Run and Debug**. + +## Running + +Update your graphics driver before reporting rendering problems. + +To use the graphical launcher: + +```powershell +.\_Build\windows\install\launcher.exe +``` + +On first launch, add one or more game folders in the global settings. The launcher searches those +folders recursively for game directories containing `eboot.bin`. Select a detected game and run it +from the game list. + +The emulator can also be started directly with a legally obtained game directory or ELF file: + +```powershell +.\_Build\windows\install\kyty_emulator.exe --game "D:\Games\ExampleGame" +``` + +Run `kyty_emulator.exe --help` to see the available graphics, logging, validation, profiling, and +debugging options. + +### AI Use + +AI tools may be used for research, reverse engineering, and development assistance. Contributors +must fully understand, review, and test all code they submit and remain responsible for its +correctness. Repository communication, including pull-request descriptions, code comments, and +issue comments, must come from the human contributor rather than an autonomous AI agent. + +Pull requests that include AI-assisted or AI-generated work should disclose the scope of the AI +involvement and describe the human review and testing performed before submission. Unverified or +untested generated changes may be closed without review. + +## License + +KytyPS5 is licensed under the [GNU General Public License version 2](LICENSE) +(`GPL-2.0-only`). + +This project is based on the original [Kyty](https://github.com/InoriRus/Kyty), which was released +under the MIT License. Kyty's original copyright and license notice are preserved in +[`LICENSES/Kyty-MIT.txt`](LICENSES/Kyty-MIT.txt). Third-party components remain subject to the +licenses included with those components. + +## Special Thanks + +- [InoriRus/Kyty](https://github.com/InoriRus/Kyty) — KytyPS5 is based on a heavily modified version + of the original Kyty project. +- [shadps4-emu/shadPS4](https://github.com/shadps4-emu/shadPS4) — reference for memory-model + understanding and the AVPlayer implementation. diff --git a/docs/screenshots/.gitkeep b/docs/screenshots/.gitkeep new file mode 100644 index 000000000..8b1378917 --- /dev/null +++ b/docs/screenshots/.gitkeep @@ -0,0 +1 @@ + diff --git a/docs/screenshots/ps5-01.png b/docs/screenshots/ps5-01.png new file mode 100644 index 000000000..1152e3253 Binary files /dev/null and b/docs/screenshots/ps5-01.png differ diff --git a/docs/screenshots/ps5-03.png b/docs/screenshots/ps5-03.png new file mode 100644 index 000000000..f5da76374 Binary files /dev/null and b/docs/screenshots/ps5-03.png differ diff --git a/docs/screenshots/ps5-04.png b/docs/screenshots/ps5-04.png new file mode 100644 index 000000000..4032efce9 Binary files /dev/null and b/docs/screenshots/ps5-04.png differ diff --git a/docs/screenshots/ps5-05.png b/docs/screenshots/ps5-05.png new file mode 100644 index 000000000..068f9109e Binary files /dev/null and b/docs/screenshots/ps5-05.png differ diff --git a/src/.clang-format b/src/.clang-format new file mode 100644 index 000000000..5b5dfd5c9 --- /dev/null +++ b/src/.clang-format @@ -0,0 +1,200 @@ +--- +Language: Cpp +# BasedOnStyle: LLVM +AccessModifierOffset: -4 +AlignAfterOpenBracket: Align +AlignConsecutiveMacros: Consecutive +AlignConsecutiveAssignments: Consecutive +AlignConsecutiveBitFields: Consecutive +AlignConsecutiveDeclarations: Consecutive +AlignEscapedNewlines: Right +AlignOperands: Align +AlignTrailingComments: true +AllowAllArgumentsOnNextLine: true +AllowAllConstructorInitializersOnNextLine: true +AllowAllParametersOfDeclarationOnNextLine: true +AllowShortEnumsOnASingleLine: true +AllowShortBlocksOnASingleLine: Never +AllowShortCaseLabelsOnASingleLine: true +AllowShortFunctionsOnASingleLine: Inline +AllowShortLambdasOnASingleLine: All +AllowShortIfStatementsOnASingleLine: WithoutElse +AllowShortLoopsOnASingleLine: false +AlwaysBreakAfterDefinitionReturnType: None +AlwaysBreakAfterReturnType: None +AlwaysBreakBeforeMultilineStrings: false +AlwaysBreakTemplateDeclarations: Yes +AttributeMacros: + - __capability +BinPackArguments: true +BinPackParameters: true +BraceWrapping: + AfterCaseLabel: false + AfterClass: false + AfterControlStatement: Never + AfterEnum: false + AfterFunction: false + AfterNamespace: false + AfterObjCDeclaration: true + AfterStruct: false + AfterUnion: false + AfterExternBlock: false + BeforeCatch: false + BeforeElse: false + BeforeLambdaBody: true + BeforeWhile: false + IndentBraces: false + SplitEmptyFunction: true + SplitEmptyRecord: true + SplitEmptyNamespace: true +BreakBeforeBinaryOperators: None +BreakBeforeConceptDeclarations: true +BreakBeforeBraces: Attach +BreakBeforeInheritanceComma: false +BreakInheritanceList: BeforeColon +BreakBeforeTernaryOperators: true +BreakConstructorInitializersBeforeComma: false +BreakConstructorInitializers: BeforeColon +BreakAfterJavaFieldAnnotations: false +BreakStringLiterals: true +ColumnLimit: 100 +CommentPragmas: '^ IWYU pragma:' +CompactNamespaces: false +ConstructorInitializerAllOnOneLineOrOnePerLine: false +ConstructorInitializerIndentWidth: 4 +ContinuationIndentWidth: 4 +Cpp11BracedListStyle: true +DeriveLineEnding: true +DerivePointerAlignment: false +DisableFormat: false +EmptyLineBeforeAccessModifier: LogicalBlock +ExperimentalAutoDetectBinPacking: false +FixNamespaceComments: true +ForEachMacros: + - foreach + - Q_FOREACH + - BOOST_FOREACH + - FOR + - FOR_HASH +StatementAttributeLikeMacros: + - Q_EMIT +IncludeBlocks: Regroup +IncludeCategories: + - Regex: '^"(Kyty)/' + Priority: 1 + SortPriority: 0 + CaseSensitive: false + - Regex: '^"(Emulator)/' + Priority: 2 + SortPriority: 0 + CaseSensitive: false + - Regex: '^"(gtest)/' + Priority: 4 + SortPriority: 0 + CaseSensitive: false + - Regex: '^' + Priority: 7 + SortPriority: 1 + CaseSensitive: false + - Regex: '^' + Priority: 7 + SortPriority: 2 + CaseSensitive: false + - Regex: '^' + Priority: 7 + SortPriority: 3 + CaseSensitive: false + - Regex: '^' + Priority: 7 + SortPriority: 4 + CaseSensitive: false + - Regex: '^<' + Priority: 7 + SortPriority: 5 + CaseSensitive: false + - Regex: '.*' + Priority: 3 + SortPriority: 0 + CaseSensitive: false +IncludeIsMainRegex: '$' +IncludeIsMainSourceRegex: '' +IndentCaseLabels: true +IndentCaseBlocks: false +IndentGotoLabels: true +IndentPPDirectives: None +IndentExternBlock: AfterExternBlock +IndentRequires: false +IndentWidth: 4 +IndentWrappedFunctionNames: false +InsertTrailingCommas: None +JavaScriptQuotes: Leave +JavaScriptWrapImports: true +KeepEmptyLinesAtTheStartOfBlocks: true +MacroBlockBegin: '' +MacroBlockEnd: '' +MaxEmptyLinesToKeep: 1 +NamespaceIndentation: None +ObjCBinPackProtocolList: Auto +ObjCBlockIndentWidth: 2 +ObjCBreakBeforeNestedBlockParam: true +ObjCSpaceAfterProperty: false +ObjCSpaceBeforeProtocolList: true +PenaltyBreakAssignment: 2 +PenaltyBreakBeforeFirstCallParameter: 19 +PenaltyBreakComment: 300 +PenaltyBreakFirstLessLess: 120 +PenaltyBreakString: 1000 +PenaltyBreakTemplateDeclaration: 10 +PenaltyExcessCharacter: 1000000 +PenaltyReturnTypeOnItsOwnLine: 60 +PenaltyIndentedWhitespace: 0 +PointerAlignment: Left +ReflowComments: true +SortIncludes: true +SortJavaStaticImport: Before +SortUsingDeclarations: true +SpaceAfterCStyleCast: false +SpaceAfterLogicalNot: false +SpaceAfterTemplateKeyword: true +SpaceBeforeAssignmentOperators: true +SpaceBeforeCaseColon: false +SpaceBeforeCpp11BracedList: true +SpaceBeforeCtorInitializerColon: false +SpaceBeforeInheritanceColon: false +SpaceBeforeParens: ControlStatements +SpaceAroundPointerQualifiers: Default +SpaceBeforeRangeBasedForLoopColon: false +SpaceInEmptyBlock: false +SpaceInEmptyParentheses: false +SpacesBeforeTrailingComments: 1 +SpacesInAngles: false +SpacesInConditionalStatement: false +SpacesInContainerLiterals: true +SpacesInCStyleCastParentheses: false +SpacesInParentheses: false +SpacesInSquareBrackets: false +SpaceBeforeSquareBrackets: false +BitFieldColonSpacing: Both +Standard: Latest +StatementMacros: + - Q_UNUSED + - QT_REQUIRE_VERSION +TabWidth: 4 +UseCRLF: true +UseTab: ForIndentation +WhitespaceSensitiveMacros: + - STRINGIZE + - PP_STRINGIZE + - BOOST_PP_STRINGIZE + - NS_SWIFT_NAME + - CF_SWIFT_NAME +... + diff --git a/src/.clang-tidy b/src/.clang-tidy new file mode 100644 index 000000000..658dc588f --- /dev/null +++ b/src/.clang-tidy @@ -0,0 +1,52 @@ +Checks: '-*,bugprone-*,cert-*,clang-analyzer-*,google-*,llvm-*,-llvm-header-guard,misc-*,modernize-*,-modernize-use-trailing-return-type,-modernize-avoid-c-arrays,performance-*,portability-*,readability-*,-readability-magic-numbers,-readability-uppercase-literal-suffix,cppcoreguidelines-*,-cppcoreguidelines-pro-type-vararg,-cppcoreguidelines-avoid-magic-numbers,-cppcoreguidelines-pro-bounds-pointer-arithmetic,-cppcoreguidelines-owning-memory,-cppcoreguidelines-pro-bounds-array-to-pointer-decay,-cppcoreguidelines-pro-type-reinterpret-cast,-cppcoreguidelines-avoid-c-arrays,hicpp-*,-hicpp-vararg,-hicpp-no-array-decay,-hicpp-avoid-c-arrays,-hicpp-uppercase-literal-suffix,-cppcoreguidelines-pro-bounds-constant-array-index,-cppcoreguidelines-pro-type-union-access,-cppcoreguidelines-pro-type-static-cast-downcast,-modernize-pass-by-value,-cert-dcl50-cpp,-misc-no-recursion,-cppcoreguidelines-avoid-non-const-global-variables,-performance-no-int-to-ptr,-readability-identifier-length,-bugprone-easily-swappable-parameters,-bugprone-throw-keyword-missing' +CheckOptions: + - key: readability-function-cognitive-complexity.Threshold + value: 50 + - key: readability-redundant-access-specifiers.CheckFirstDeclaration + value: 1 + - key: misc-non-private-member-variables-in-classes.IgnorePublicMemberVariables + value: 1 + - key: cppcoreguidelines-pro-type-member-init.IgnoreArrays + value: 1 + - key: cppcoreguidelines-macro-usage.AllowedRegexp + value: (KYTY_*)|(EXIT*)|(UT_*) + - key: readability-identifier-naming.ClassCase + value: CamelCase + - key: readability-identifier-naming.MemberCase + value: lower_case + - key: readability-identifier-naming.PrivateMemberPrefix + value: 'm_' + - key: readability-identifier-naming.ProtectedMemberPrefix + value: 'm_' + - key: readability-identifier-naming.PublicMethodCase + value: CamelCase + - key: readability-identifier-naming.PublicMemberCase + value: aNy_CasE + - key: readability-identifier-naming.StructCase + value: CamelCase + - key: readability-identifier-naming.ConstantCase + value: UPPER_CASE + - key: readability-identifier-naming.EnumConstantCase + value: CamelCase + - key: readability-identifier-naming.EnumCase + value: CamelCase + - key: readability-identifier-naming.TemplateParameterCase + value: CamelCase + - key: readability-identifier-naming.ValueTemplateParameterCase + value: CamelCase + - key: readability-identifier-naming.LocalConstantCase + value: lower_case + - key: readability-identifier-naming.GlobalConstantCase + value: lower_case + - key: readability-identifier-naming.GlobalConstantPrefix + value: 'g_' + - key: readability-identifier-naming.GlobalVariableCase + value: lower_case + - key: readability-identifier-naming.GlobalVariablePrefix + value: 'g_' + - key: readability-identifier-naming.VariableCase + value: lower_case + - key: readability-identifier-naming.ParameterCase + value: lower_case + - key: readability-identifier-naming.ConstexprVariableCase + value: UPPER_CASE diff --git a/src/.gitattributes b/src/.gitattributes new file mode 100644 index 000000000..a96362986 --- /dev/null +++ b/src/.gitattributes @@ -0,0 +1,3 @@ +doc/** filter=lfs diff=lfs merge=lfs -text + +*.inc linguist-language=C++ diff --git a/src/.gitignore b/src/.gitignore new file mode 100644 index 000000000..86a9657ca --- /dev/null +++ b/src/.gitignore @@ -0,0 +1 @@ +*.pro.user \ No newline at end of file diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt new file mode 100644 index 000000000..69c8f5855 --- /dev/null +++ b/src/CMakeLists.txt @@ -0,0 +1,338 @@ +cmake_minimum_required(VERSION 3.12...4.3) + +if(CMAKE_VERSION VERSION_GREATER_EQUAL 4.0) + set(CMAKE_POLICY_VERSION_MINIMUM 3.5 CACHE STRING "Minimum policy version for bundled dependencies") +endif() + +project(Kyty) + +if(CMAKE_SYSTEM_NAME MATCHES ".*Linux") + set(LINUX TRUE) +endif() + +if (NOT (WIN32 OR LINUX)) + message(FATAL_ERROR "only Windows and Linux builds are supported") +endif() + +set(CMAKE_CXX_STANDARD 20) +set(CMAKE_CXX_STANDARD_REQUIRED ON) +set(CMAKE_CXX_EXTENSIONS OFF) + +include(utils.cmake) + +set(KYTY_THIRD_PARTY_DIR "${CMAKE_CURRENT_SOURCE_DIR}/../3rdparty") + +set(CMAKE_WINDOWS_EXPORT_ALL_SYMBOLS ON) + +include(TestBigEndian) + +if(NOT CMAKE_SIZEOF_VOID_P EQUAL 8) + message(FATAL_ERROR "only 64-bit builds are supported") +endif() + +test_big_endian(BIG_ENDIAN) +if(BIG_ENDIAN) + set(KYTY_ENDIAN KYTY_ENDIAN_BIG) +else() + set(KYTY_ENDIAN KYTY_ENDIAN_LITTLE) +endif() + +if(CMAKE_BUILD_TYPE MATCHES Debug) + set(KYTY_BUILD KYTY_BUILD_DEBUG) +else() + set(KYTY_BUILD KYTY_BUILD_RELEASE) +endif() + +if(LINUX) + set(KYTY_PLATFORM KYTY_PLATFORM_LINUX) +else() + set(KYTY_PLATFORM KYTY_PLATFORM_WINDOWS) +endif() + +set(CLANG FALSE) +set(GCC FALSE) +set(KYTY_CLANG_CL FALSE) +get_filename_component(KYTY_CXX_COMPILER_NAME "${CMAKE_CXX_COMPILER}" NAME_WE) +string(TOLOWER "${KYTY_CXX_COMPILER_NAME}" KYTY_CXX_COMPILER_NAME) + +if(CMAKE_CXX_COMPILER_ID MATCHES "^(Apple)?Clang$") + set(CLANG TRUE) + set(KYTY_COMPILER CLANG) + if(WIN32 AND KYTY_CXX_COMPILER_NAME STREQUAL "clang-cl") + set(KYTY_CLANG_CL TRUE) + set(KYTY_LINKER LLD_LINK) + else() + set(KYTY_LINKER LLD) + endif() + if ((CMAKE_AR MATCHES "llvm") AND (CMAKE_CXX_COMPILER_VERSION VERSION_LESS 12.0.0) AND (CMAKE_CXX_COMPILER MATCHES "winlibs")) + string(REPLACE "llvm-ar.exe" "ar.exe" CMAKE_AR ${CMAKE_AR}) # llvm-ar.exe 11.0.0 from winlibs is broken + endif() +elseif(CMAKE_CXX_COMPILER_ID STREQUAL "GNU") + set(GCC TRUE) + set(KYTY_COMPILER GCC) + set(KYTY_LINKER LD) +else() + message(FATAL_ERROR "only Clang and GCC are supported; found ${CMAKE_CXX_COMPILER_ID}") +endif() + +set(KYTY_PROJECT_NAME "Emulator" CACHE STRING "Project name suffix") + +string(TOLOWER ${KYTY_COMPILER} KYTY_COMPILER_ID) +string(TOLOWER ${KYTY_LINKER} KYTY_LINKER_ID) + +if (CLANG AND (KYTY_LINKER STREQUAL LLD)) + set(KYTY_LD_OPTIONS "-fuse-ld=lld") +endif() + +if (KYTY_LINKER STREQUAL LD AND NOT LINUX) + set(KYTY_LD_OPTIONS "-Wl,--image-base=0x100000000000") +endif() + +project(Kyty${KYTY_PROJECT_NAME}${CMAKE_BUILD_TYPE}${KYTY_COMPILER} VERSION 0.2.2) + +include_directories( + ${KYTY_THIRD_PARTY_DIR}/gtest/include + ${KYTY_THIRD_PARTY_DIR}/gtest + ${KYTY_THIRD_PARTY_DIR}/fmt/include + ${KYTY_THIRD_PARTY_DIR}/magic_enum/include/magic_enum + ${PROJECT_BINARY_DIR} + ${CMAKE_CURRENT_SOURCE_DIR} +) + +set(KYTY_VERSION "${PROJECT_VERSION}") + +configure_file( + ${PROJECT_SOURCE_DIR}/cmake_config.h.in + ${PROJECT_BINARY_DIR}/cmake_config.h + ) + +find_package(Git) + +add_custom_target( KytyGitVersion + COMMAND ${CMAKE_COMMAND} + -D INPUT_FILE=${CMAKE_CURRENT_SOURCE_DIR}/kytyGitVersion.h.in + -D OUTPUT_FILE=${CMAKE_CURRENT_BINARY_DIR}/kytyGitVersion.h + -D GIT_EXECUTABLE=${GIT_EXECUTABLE} + -P ${CMAKE_CURRENT_SOURCE_DIR}/generate_version.cmake + COMMENT "Generate kytyGitVersion.h" +) + +if (CMAKE_CXX_COMPILER_VERSION VERSION_GREATER_EQUAL 12.0.0) + list(APPEND KYTY_IWYU + kyty_emulator + common + #launcher + ) + list(APPEND KYTY_CLANG_TYDY + kyty_emulator + #common + launcher + ) +endif() + +option(KYTY_BUILD_LAUNCHER "Build Qt launcher" ON) + +config_compiler_and_linker() + +add_subdirectory("${KYTY_THIRD_PARTY_DIR}" "${CMAKE_CURRENT_BINARY_DIR}/3rdparty") +add_subdirectory(common) + +file(GLOB kyty_emulator_src + libs/*.cpp + libs/*.h + graphics/*.cpp + graphics/*.h + graphics/guest_gpu/*.cpp + graphics/guest_gpu/*.h + graphics/guest_gpu/command_processor/*.cpp + graphics/guest_gpu/command_processor/*.h + graphics/host_gpu/*.cpp + graphics/host_gpu/*.h + graphics/host_gpu/renderer/*.cpp + graphics/host_gpu/renderer/*.h + graphics/shader/*.cpp + graphics/shader/*.h + graphics/shader/recompiler/*.cpp + graphics/shader/recompiler/*.h + graphics/shader/recompiler/shaderIR/*.cpp + graphics/shader/recompiler/shaderIR/*.h + graphics/shader/recompiler/spirvEmitter/*.cpp + graphics/shader/recompiler/spirvEmitter/*.h + graphics/host_gpu/objects/*.cpp + graphics/host_gpu/objects/*.h + graphics/presentation/*.cpp + graphics/presentation/*.h + graphics/presentation/window/*.cpp + graphics/presentation/window/*.h + kernel/*.cpp + kernel/*.h + loader/*.cpp + loader/*.h +) + +list(APPEND kyty_emulator_src + emulator.h + emulator.cpp +) +list(REMOVE_DUPLICATES kyty_emulator_src) + +if (KYTY_CLANG_CL) + set_source_files_properties(${kyty_emulator_src} PROPERTIES COMPILE_FLAGS "-Wno-pragma-pack -Wno-deprecated-declarations -D_TIMESPEC_DEFINED") +endif() + +set(kyty_emulator_link_libraries common vulkan-1 spirv-tools-opt spirv-tools SDL2-static xxhash FFmpeg::ffmpeg fmt::fmt nlohmann_json::nlohmann_json LibAtrac9) + +set(inc_headers + ${CMAKE_CURRENT_SOURCE_DIR} + ${KYTY_THIRD_PARTY_DIR}/SDL2/include + ${KYTY_THIRD_PARTY_DIR}/Vulkan-Headers/include + ${KYTY_THIRD_PARTY_DIR}/VulkanMemoryAllocator/include + ${KYTY_THIRD_PARTY_DIR}/SPIRV-Tools/include + ${KYTY_THIRD_PARTY_DIR}/fmt/include + ${KYTY_THIRD_PARTY_DIR}/xxhash + ${KYTY_THIRD_PARTY_DIR}/cpuinfo/include + ${KYTY_THIRD_PARTY_DIR}/stb +) + +if(DEFINED ENV{VULKAN_SDK}) + list(APPEND inc_headers $ENV{VULKAN_SDK}/Include) +endif() + +if (KYTY_CLANG_CL) + list(APPEND kyty_emulator_link_libraries winpthread) + list(APPEND inc_headers + ${KYTY_THIRD_PARTY_DIR}/winpthread/include + ) +endif() + +list(APPEND check_headers + ${CMAKE_CURRENT_SOURCE_DIR} +) + +function(add_kyty_full_emulator_test target source) + add_executable(${target} EXCLUDE_FROM_ALL ${source} ${kyty_emulator_src}) + target_link_libraries(${target} ${kyty_emulator_link_libraries}) + target_include_directories(${target} PRIVATE ${inc_headers}) + if (WIN32) + target_link_libraries(${target} iphlpapi) + endif() + if (KYTY_CLANG_CL) + target_link_libraries(${target} onecore) + add_custom_command(TARGET ${target} POST_BUILD COMMAND ${CMAKE_COMMAND} -E copy_if_different "${KYTY_THIRD_PARTY_DIR}/winpthread/bin/libwinpthread-1.dll" $/libwinpthread-1.dll) + endif() +endfunction() + +add_kyty_full_emulator_test(shader_cfg_tests ../tests/shaderCfgTests.cpp) + +add_executable(scalar_provenance_tests EXCLUDE_FROM_ALL + ../tests/ScalarProvenanceTests.cpp + graphics/host_gpu/hostMemory.cpp + graphics/shader/recompiler/ScalarProvenance.cpp + graphics/shader/recompiler/SrtWalker.cpp +) +target_link_libraries(scalar_provenance_tests fmt::fmt) +target_include_directories(scalar_provenance_tests PRIVATE ${inc_headers}) + +add_executable(page_manager_tests EXCLUDE_FROM_ALL + ../tests/PageManagerTests.cpp + graphics/host_gpu/pageManager.cpp +) +target_include_directories(page_manager_tests PRIVATE ${inc_headers}) + +add_executable(memory_tracker_tests EXCLUDE_FROM_ALL + ../tests/MemoryTrackerTests.cpp + graphics/host_gpu/pageManager.cpp + graphics/host_gpu/memoryTracker.cpp +) +target_link_libraries(memory_tracker_tests fmt::fmt common) +target_include_directories(memory_tracker_tests PRIVATE ${inc_headers}) +target_compile_definitions(memory_tracker_tests PRIVATE KYTY_MEMORY_TRACKER_TESTS=1) + +add_executable(shader_vertex_metadata_tests EXCLUDE_FROM_ALL + ../tests/ShaderVertexMetadataTests.cpp + graphics/host_gpu/hostMemory.cpp + graphics/shader/shaderVertexMetadata.cpp +) +target_include_directories(shader_vertex_metadata_tests PRIVATE ${inc_headers}) + +add_executable(shader_stage_runtime_tests EXCLUDE_FROM_ALL + ../tests/ShaderStageRuntimeTests.cpp + graphics/guest_gpu/gpu_format.cpp + graphics/host_gpu/hostMemory.cpp + graphics/shader/shaderStageRuntime.cpp + graphics/shader/recompiler/ResourceMaterialization.cpp + graphics/shader/recompiler/ScalarProvenance.cpp + graphics/shader/recompiler/SrtWalker.cpp +) +target_link_libraries(shader_stage_runtime_tests fmt::fmt) +target_include_directories(shader_stage_runtime_tests PRIVATE ${inc_headers}) + +add_executable(resource_tracking_tests EXCLUDE_FROM_ALL + ../tests/ResourceTrackingTests.cpp + graphics/guest_gpu/gpu_format.cpp + graphics/host_gpu/hostMemory.cpp + graphics/shader/recompiler/ScalarProvenance.cpp + graphics/shader/recompiler/SrtWalker.cpp + graphics/shader/recompiler/SrtPatcher.cpp + graphics/shader/recompiler/ResourceTracking.cpp + graphics/shader/recompiler/ResourceMaterialization.cpp + graphics/shader/recompiler/ShaderInfoCollection.cpp + graphics/shader/recompiler/BindingLayout.cpp +) +target_link_libraries(resource_tracking_tests fmt::fmt) +target_include_directories(resource_tracking_tests PRIVATE ${inc_headers}) + +add_executable(resource_mutex_tests EXCLUDE_FROM_ALL + ../tests/ResourceMutexTests.cpp + graphics/host_gpu/renderer/resourceMutex.cpp +) +target_link_libraries(resource_mutex_tests common) +target_include_directories(resource_mutex_tests PRIVATE ${inc_headers}) + +add_kyty_full_emulator_test(shader_recompiler_compute_tests ../tests/ShaderRecompilerComputeTests.cpp) + +add_kyty_full_emulator_test(virtual_memory_allocation_tests ../tests/VirtualMemoryAllocationTests.cpp) + + +add_executable(kyty_emulator main.cpp ${kyty_emulator_src}) + +target_link_libraries(kyty_emulator ${kyty_emulator_link_libraries}) +if (WIN32) + target_link_libraries(kyty_emulator opengl32) + target_link_libraries(kyty_emulator iphlpapi) + target_link_libraries(kyty_emulator setupapi) + target_link_libraries(kyty_emulator ws2_32) +endif() +if (KYTY_CLANG_CL) + target_link_libraries(kyty_emulator onecore) +endif() +if (CLANG AND NOT KYTY_CLANG_CL) + target_link_libraries(kyty_emulator pthread) +endif() +target_include_directories(kyty_emulator PRIVATE ${inc_headers}) + +clang_tidy_check(kyty_emulator "" "${check_headers}" "${inc_headers}") + +include_what_you_use(kyty_emulator "${inc_headers}") + +set(KYTY_EMULATOR_MAP_NAME "kyty_emulator_${KYTY_COMPILER_ID}_${KYTY_LINKER_ID}.map") + +set(KYTY_EMULATOR_MAP_LINK_PATH "${CMAKE_CURRENT_BINARY_DIR}/${KYTY_EMULATOR_MAP_NAME}") +set(KYTY_EMULATOR_PDB_LINK_PATH "${CMAKE_CURRENT_BINARY_DIR}/kyty_emulator.pdb") + +if(KYTY_CLANG_CL) + set_target_properties(kyty_emulator PROPERTIES LINK_FLAGS "/DYNAMICBASE:NO /DEBUG:FULL /PDB:${KYTY_EMULATOR_PDB_LINK_PATH} /lldmap:${KYTY_EMULATOR_MAP_LINK_PATH}") + add_custom_command(TARGET kyty_emulator POST_BUILD COMMAND ${CMAKE_COMMAND} -E copy_if_different "${KYTY_THIRD_PARTY_DIR}/winpthread/bin/libwinpthread-1.dll" $/libwinpthread-1.dll) +elseif(WIN32 OR LINUX) + set_target_properties(kyty_emulator PROPERTIES LINK_FLAGS "${KYTY_LD_OPTIONS} -Wl,-Map=${KYTY_EMULATOR_MAP_LINK_PATH}") +endif() + +add_dependencies(kyty_emulator KytyGitVersion) + +install(TARGETS kyty_emulator DESTINATION .) +if(KYTY_BUILD_LAUNCHER) + add_subdirectory(launcher) +endif() +if(KYTY_CLANG_CL) + install(FILES "${KYTY_THIRD_PARTY_DIR}/winpthread/bin/libwinpthread-1.dll" DESTINATION .) +endif() diff --git a/src/cmake_config.h.in b/src/cmake_config.h.in new file mode 100644 index 000000000..2c4182f13 --- /dev/null +++ b/src/cmake_config.h.in @@ -0,0 +1,8 @@ +#define KYTY_VERSION "@KYTY_VERSION@" +#define KYTY_ENDIAN @KYTY_ENDIAN@ +#define KYTY_PLATFORM @KYTY_PLATFORM@ +#define KYTY_COMPILER KYTY_COMPILER_@KYTY_COMPILER@ +#define KYTY_LINKER KYTY_LINKER_@KYTY_LINKER@ +#define KYTY_BUILD @KYTY_BUILD@ +#cmakedefine KYTY_FINAL + diff --git a/src/common/CMakeLists.txt b/src/common/CMakeLists.txt new file mode 100644 index 000000000..6e9358ff3 --- /dev/null +++ b/src/common/CMakeLists.txt @@ -0,0 +1,34 @@ +file(GLOB common_src CONFIGURE_DEPENDS + "*.cpp" + "logging/*.cpp" + "platform/*.cpp" +) + +file(GLOB_RECURSE common_headers CONFIGURE_DEPENDS "*.h") + +if(KYTY_CLANG_CL) + set_source_files_properties(${common_src} PROPERTIES COMPILE_FLAGS "-Wno-pragma-pack") +endif() + +add_library(common STATIC ${common_src} ${common_headers}) + +target_include_directories(common + PUBLIC ${PROJECT_SOURCE_DIR} + PRIVATE + ${KYTY_THIRD_PARTY_DIR}/cpuinfo/include +) + +target_link_libraries(common + PUBLIC + fmt::fmt + Tracy::TracyClient + PRIVATE + cpuinfo + spdlog::spdlog +) + +get_property(inc_headers TARGET common PROPERTY INCLUDE_DIRECTORIES) +set(check_headers ${CMAKE_CURRENT_SOURCE_DIR}) + +clang_tidy_check(common "" "${check_headers}" "${inc_headers}") +include_what_you_use(common "${inc_headers}") diff --git a/src/common/abi.h b/src/common/abi.h new file mode 100644 index 000000000..621ed6107 --- /dev/null +++ b/src/common/abi.h @@ -0,0 +1,12 @@ +#ifndef KYTY_COMMON_ABI_H_ +#define KYTY_COMMON_ABI_H_ + +#include "common/common.h" + +// NOLINTNEXTLINE(cppcoreguidelines-macro-usage) +#define KYTY_MS_ABI __attribute__((ms_abi)) + +// NOLINTNEXTLINE(cppcoreguidelines-macro-usage) +#define KYTY_SYSV_ABI __attribute__((sysv_abi)) + +#endif /* KYTY_COMMON_ABI_H_ */ diff --git a/src/common/assert.cpp b/src/common/assert.cpp new file mode 100644 index 000000000..4016f8d5e --- /dev/null +++ b/src/common/assert.cpp @@ -0,0 +1,65 @@ +#include "common/assert.h" + +#include "common/debug.h" +#include "common/logging/log.h" +#include "common/subsystems.h" + +#include +#include +#include +#include + +namespace Common { + +#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS && KYTY_BUILD == KYTY_BUILD_DEBUG && \ + KYTY_COMPILER == KYTY_COMPILER_CLANG +constexpr int PRINT_STACK_FROM = 4; +#else +constexpr int PRINT_STACK_FROM = 2; +#endif + +static std::string BuildFatalReport(const char* title, std::string_view text, const char* file, + int line) { + DebugStack stack; + DebugStack::Trace(&stack); + + std::string report = "--- Stack Trace ---\n"; + for (int i = PRINT_STACK_FROM; i < stack.depth; i++) { + report += fmt::format("[{}] {:016x}\n", i - PRINT_STACK_FROM, + static_cast(stack.GetAddr(i))); + } + report += fmt::format("{}\n{} in {}:{}\n", title, text, file, line); + return report; +} + +static int DbgReport(const char* title, std::string_view text, const char* file, int line) { + Log::WriteFatal(BuildFatalReport(title, text, file, line)); + SubsystemsListSingleton::Instance()->ShutdownAll(); + return 1; +} + +int DbgExitIfHandler(const char* expr, const char* file, int line) { + return DbgReport("--- Fatal Error ---", fmt::format("Error: condition ({}) is true", expr), + file, line); +} + +int DbgNotImplementedHandler(const char* expr, const char* file, int line) { + return DbgReport("--- Fatal Error ---", fmt::format("Not implemented ({})", expr), file, line); +} + +int DbgExitHandler(const char* file, int line, std::string_view text) { + Log::WriteFatal(BuildFatalReport("--- Error ---", text, file, line)); + return 1; +} + +int DbgExitHandler(const char* file, int line, fmt::text_style style, std::string_view text) { + Log::WriteFatal(style, BuildFatalReport("--- Error ---", text, file, line)); + return 1; +} + +void DbgExit(int status) { + std::fflush(nullptr); + std::_Exit(status); +} + +} // namespace Common diff --git a/src/common/assert.h b/src/common/assert.h new file mode 100644 index 000000000..0a05ffeb8 --- /dev/null +++ b/src/common/assert.h @@ -0,0 +1,67 @@ +#ifndef KYTY_COMMON_ASSERT_H_ +#define KYTY_COMMON_ASSERT_H_ + +#include "common/common.h" +#include "common/logging/log.h" + +#include +#include + +namespace Common { + +#ifdef __clang__ +int DbgExitHandler(char const* file, int line, std::string_view text) + __attribute__((analyzer_noreturn)); +int DbgExitHandler(char const* file, int line, fmt::text_style style, std::string_view text) + __attribute__((analyzer_noreturn)); +int DbgExitIfHandler(char const* expr, char const* file, int line) + __attribute__((analyzer_noreturn)); +int DbgNotImplementedHandler(char const* expr, char const* file, int line) + __attribute__((analyzer_noreturn)); +void DbgExit(int status) __attribute__((analyzer_noreturn)); +#else +int DbgExitHandler(char const* file, int line, std::string_view text); +int DbgExitHandler(char const* file, int line, fmt::text_style style, std::string_view text); +int DbgExitIfHandler(char const* expr, char const* file, int line); +int DbgNotImplementedHandler(char const* expr, char const* file, int line); +void DbgExit(int status); +#endif + +} // namespace Common + +#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS || KYTY_PLATFORM == KYTY_PLATFORM_LINUX +#define EXIT_HALT() (Common::DbgExit(321), 1) +#else +#define EXIT_HALT() (std::_Exit(321), 1) +#endif + +#ifndef KYTY_FINAL +#define EXIT_IF(x) \ + ((void)((x) && Common::DbgExitIfHandler(#x, __FILE__, __LINE__) != 0 && (EXIT_HALT(), 1) != 0)) +#else +#define EXIT_IF(x) \ + do { \ + constexpr bool kyty_exit_if_disabled = false && (x); \ + (void)kyty_exit_if_disabled; \ + } while (0) +#endif + +#define EXIT(...) \ + do { \ + ((void)(Common::DbgExitHandler(__FILE__, __LINE__, ::fmt::sprintf(__VA_ARGS__)) && \ + (EXIT_HALT(), 1))); \ + } while (0) + +#define EXIT_COLOR(style, ...) \ + do { \ + ((void)(Common::DbgExitHandler(__FILE__, __LINE__, (style), \ + ::fmt::sprintf(__VA_ARGS__)) && \ + (EXIT_HALT(), 1))); \ + } while (0) + +#define EXIT_NOT_IMPLEMENTED(x) \ + ((void)((x) && Common::DbgNotImplementedHandler(#x, __FILE__, __LINE__) != 0 && \ + (EXIT_HALT(), 1) != 0)) +#define KYTY_NOT_IMPLEMENTED EXIT_NOT_IMPLEMENTED(true) + +#endif /* KYTY_COMMON_ASSERT_H_ */ diff --git a/src/common/byteBuffer.h b/src/common/byteBuffer.h new file mode 100644 index 000000000..b53016e6a --- /dev/null +++ b/src/common/byteBuffer.h @@ -0,0 +1,165 @@ +#ifndef KYTY_COMMON_BYTEBUFFER_H_ +#define KYTY_COMMON_BYTEBUFFER_H_ + +#include "common/assert.h" +#include "common/common.h" + +#include +#include // IWYU pragma: export +#include +#include + +namespace Common { + +using Byte = std::byte; + +class ByteBuffer { +public: + using DataType = std::vector; + + static constexpr uint32_t INVALID_INDEX = static_cast(-1); + + using iterator = DataType::iterator; + using const_iterator = DataType::const_iterator; + + ByteBuffer() = default; + ByteBuffer(const ByteBuffer&) = default; + ByteBuffer(ByteBuffer&&) = default; + + ByteBuffer& operator=(const ByteBuffer&) = default; + ByteBuffer& operator=(ByteBuffer&&) = default; + + explicit ByteBuffer(uint32_t size): m_data(size) {} + + ByteBuffer(std::initializer_list list) { + for (const uint8_t& e: list) { + m_data.emplace_back(static_cast(e)); + } + } + + explicit ByteBuffer(const void* buf, uint32_t size) + : ByteBuffer(size) // @suppress("Ambiguous problem") + { + EXIT_IF(buf == nullptr && size != 0); + + if (size != 0) { + std::memcpy(m_data.data(), buf, size); + } + } + + [[nodiscard]] uint32_t Size() const { return static_cast(m_data.size()); } + [[nodiscard]] bool IsEmpty() const { return m_data.empty(); } + + void Clear() { m_data.clear(); } + + void Add(const Byte& val) { m_data.push_back(val); } + void Add(Byte&& val) { m_data.push_back(std::move(val)); } + + void Add(const Byte* val, uint32_t num) { + EXIT_IF(val == nullptr && num != 0); + m_data.insert(m_data.end(), val, val + num); + } + + void Add(const ByteBuffer& v) { + if (this == &v) { + const ByteBuffer copy(v); + Add(copy.GetDataConst(), copy.Size()); + return; + } + Add(v.GetDataConst(), v.Size()); + } + + void InsertAt(uint32_t index, const Byte& val) { + EXIT_IF(index > Size()); + m_data.insert(m_data.begin() + index, val); + } + + void InsertAt(uint32_t index, Byte&& val) { + EXIT_IF(index > Size()); + m_data.insert(m_data.begin() + index, std::move(val)); + } + + [[nodiscard]] uint32_t Find(const Byte& t) const { + const auto it = std::find(m_data.begin(), m_data.end(), t); + return (it == m_data.end() ? INVALID_INDEX + : static_cast(std::distance(m_data.begin(), it))); + } + + [[nodiscard]] bool Contains(const Byte& t) const { return Find(t) != INVALID_INDEX; } + + bool Remove(const Byte& t) { + const auto it = std::find(m_data.begin(), m_data.end(), t); + if (it == m_data.end()) { + return false; + } + m_data.erase(it); + return true; + } + + bool RemoveAt(uint32_t index, uint32_t count = 1) { + if (index >= Size() || count == 0) { + return false; + } + + const uint32_t erase_count = std::min(count, Size() - index); + m_data.erase(m_data.begin() + index, m_data.begin() + index + erase_count); + return true; + } + + [[nodiscard]] bool IndexValid(uint32_t index) const { return index < Size(); } + + Byte& operator[](uint32_t index) { + EXIT_IF(index >= Size()); + return m_data[index]; + } + + const Byte& operator[](uint32_t index) const { + EXIT_IF(index >= Size()); + return m_data[index]; + } + + [[nodiscard]] const Byte& At(uint32_t index) const { + EXIT_IF(index >= Size()); + return m_data[index]; + } + + Byte* GetData() { return m_data.data(); } + [[nodiscard]] const Byte* GetData() const { return m_data.data(); } + [[nodiscard]] const Byte* GetDataConst() const { return m_data.data(); } + + void Sort() { + std::sort(m_data.begin(), m_data.end(), [](Byte a, Byte b) { + return std::to_integer(a) < std::to_integer(b); + }); + } + + template + void Sort(OP&& comp_func) { + std::sort(m_data.begin(), m_data.end(), std::forward(comp_func)); + } + + bool operator==(const ByteBuffer& s) const { return m_data == s.m_data; } + bool operator!=(const ByteBuffer& s) const { return !(*this == s); } + + iterator begin() { return m_data.begin(); } // NOLINT(readability-identifier-naming) + iterator end() { return m_data.end(); } // NOLINT(readability-identifier-naming) + [[nodiscard]] const_iterator begin() const { + return m_data.begin(); + } // NOLINT(readability-identifier-naming) + [[nodiscard]] const_iterator end() const { + return m_data.end(); + } // NOLINT(readability-identifier-naming) + [[nodiscard]] const_iterator cbegin() const { + return m_data.cbegin(); + } // NOLINT(readability-identifier-naming) + [[nodiscard]] const_iterator cend() const { + return m_data.cend(); + } // NOLINT(readability-identifier-naming) + +private: + DataType m_data; +}; + +} // namespace Common + +#endif /* KYTY_COMMON_BYTEBUFFER_H_ */ diff --git a/src/common/common.h b/src/common/common.h new file mode 100644 index 000000000..248a95576 --- /dev/null +++ b/src/common/common.h @@ -0,0 +1,43 @@ +#ifndef KYTY_COMMON_COMMON_H_ +#define KYTY_COMMON_COMMON_H_ + +#include "common/config.h" // IWYU pragma: export + +#if __cplusplus < 201703L +#undef __cplusplus +#define __cplusplus 201703L +#endif + +#if defined(__MINGW32__) || defined(__MINGW64__) +// NOLINTNEXTLINE(cppcoreguidelines-macro-usage,cert-dcl51-cpp,cert-dcl37-c,bugprone-reserved-identifier) +#define __USE_MINGW_ANSI_STDIO 1 +#endif + +// IWYU pragma: begin_exports +#include +#include +#include +#include +// IWYU pragma: end_exports + +#define KYTY_CLASS_NO_COPY(name) \ +public: \ + name(const name&) = delete; /* NOLINT(bugprone-macro-parentheses) */ \ + name& operator=(const name&) = delete; /* NOLINT(bugprone-macro-parentheses) */ \ + name(name&&) noexcept = delete; /* NOLINT(bugprone-macro-parentheses) */ \ + name& operator=(name&&) noexcept = delete; /* NOLINT(bugprone-macro-parentheses) */ + +#define KYTY_CLASS_DEFAULT_COPY(name) \ +public: \ + name(const name&) = default; /* NOLINT(bugprone-macro-parentheses) */ \ + name& operator=(const name&) = default; /* NOLINT(bugprone-macro-parentheses) */ \ + name(name&&) noexcept = default; /* NOLINT(bugprone-macro-parentheses) */ \ + name& operator=(name&&) noexcept = default; /* NOLINT(bugprone-macro-parentheses) */ + +#if KYTY_COMPILER == KYTY_COMPILER_GCC +#define KYTY_FORMAT_PRINTF(a, b) __attribute__((format(gnu_printf, a, b))) +#elif KYTY_COMPILER == KYTY_COMPILER_CLANG +#define KYTY_FORMAT_PRINTF(a, b) __attribute__((format(printf, a, b))) +#endif + +#endif /* KYTY_COMMON_COMMON_H_ */ diff --git a/src/common/commonSubsystem.cpp b/src/common/commonSubsystem.cpp new file mode 100644 index 000000000..1733c4448 --- /dev/null +++ b/src/common/commonSubsystem.cpp @@ -0,0 +1,21 @@ +#include "common/commonSubsystem.h" + +#include "common/byteBuffer.h" // IWYU pragma: associated +#include "common/common.h" // IWYU pragma: associated +#include "common/debug.h" // IWYU pragma: associated +#include "common/hash.h" // IWYU pragma: associated +#include "common/magicEnum.h" // IWYU pragma: associated +#include "common/singleton.h" // IWYU pragma: associated +#include "common/virtualMemory.h" + +namespace Common { + +KYTY_SUBSYSTEM_INIT(Common) { + VirtualMemory::Init(); +} + +KYTY_SUBSYSTEM_UNEXPECTED_SHUTDOWN(Common) {} + +KYTY_SUBSYSTEM_DESTROY(Common) {} + +} // namespace Common diff --git a/src/common/commonSubsystem.h b/src/common/commonSubsystem.h new file mode 100644 index 000000000..a77416181 --- /dev/null +++ b/src/common/commonSubsystem.h @@ -0,0 +1,12 @@ +#ifndef KYTY_COMMON_COMMON_SUBSYSTEM_H_ +#define KYTY_COMMON_COMMON_SUBSYSTEM_H_ + +#include "common/subsystems.h" + +namespace Common { + +KYTY_SUBSYSTEM_DEFINE(Common); + +} // namespace Common + +#endif /* KYTY_COMMON_COMMON_SUBSYSTEM_H_ */ diff --git a/src/common/config.h b/src/common/config.h new file mode 100644 index 000000000..3e50c4466 --- /dev/null +++ b/src/common/config.h @@ -0,0 +1,22 @@ +#ifndef KYTY_COMMON_CONFIG_H_ +#define KYTY_COMMON_CONFIG_H_ + +#define KYTY_PLATFORM_WINDOWS 1 +#define KYTY_PLATFORM_LINUX 5 + +#define KYTY_COMPILER_GCC 1 +#define KYTY_COMPILER_CLANG 2 + +#define KYTY_LINKER_LD 1 +#define KYTY_LINKER_LLD 2 +#define KYTY_LINKER_LLD_LINK 3 + +#define KYTY_BUILD_DEBUG 1 +#define KYTY_BUILD_RELEASE 2 + +#define KYTY_ENDIAN_BIG 1 +#define KYTY_ENDIAN_LITTLE 2 + +#include "cmake_config.h" + +#endif /* KYTY_COMMON_CONFIG_H_ */ diff --git a/src/common/dateTime.cpp b/src/common/dateTime.cpp new file mode 100644 index 000000000..e9145efba --- /dev/null +++ b/src/common/dateTime.cpp @@ -0,0 +1,694 @@ +#include "common/dateTime.h" + +#include "common/assert.h" +#include "common/platform/sysTimer.h" +#include "common/stringUtils.h" + +#include + +namespace Common { + +template +constexpr T FloorDiv(T a, T b) { + if (b < 0) { + if (a < 0) { + return a / b; + } + b = -b; + a = -a; + } + + return (a - (a < 0 ? b - 1 : 0)) / b; +} + +static void YmdToJd(int year, int month, int day, jd_t* jd) { + jd_t t_year = year; + jd_t t_month = month; + jd_t t_day = day; + + if (t_year < 0) { + t_year++; + } + + jd_t a = FloorDiv(14 - t_month, 12); + jd_t y = t_year + 4800 - a; + jd_t m = t_month + 12 * a - 3; + *jd = t_day + FloorDiv(153 * m + 2, 5) + 365 * y + FloorDiv(y, 4) - + FloorDiv(y, 100) + FloorDiv(y, 400) - 32045; +} + +static void JdToYmd(int* year, int* month, int* day, jd_t jd) { + jd_t a = jd + 32044; + jd_t b = FloorDiv(4 * a + 3, 146097); + jd_t c = a - FloorDiv(146097 * b, 4); + jd_t d = FloorDiv(4 * c + 3, 1461); + jd_t e = c - FloorDiv(1461 * d, 4); + jd_t m = FloorDiv(5 * e + 2, 153); + + int t_day = e - FloorDiv(153 * m + 2, 5) + 1; + int t_month = m + 3 - 12 * FloorDiv(m, 10); + int t_year = 100 * b + d - 4800 + FloorDiv(m, 10); + + if (t_year <= 0) { + t_year--; + } + + if (year != nullptr) { + *year = t_year; + } + if (month != nullptr) { + *month = t_month; + } + if (day != nullptr) { + *day = t_day; + } +} + +static void HmsToMs(int hour, int minute, int second, int msec, int* ms) { + *ms = hour * 60 * 60 * 1000 + minute * 60 * 1000 + second * 1000 + msec; +} + +static void MsToHms(int* hour, int* minute, int* second, int* msec, int ms) { + if (hour != nullptr) { + *hour = ms / (60 * 60 * 1000); + } + if (minute != nullptr) { + *minute = (ms % (60 * 60 * 1000)) / (60 * 1000); + } + if (second != nullptr) { + *second = (ms / 1000) % 60; + } + if (msec != nullptr) { + *msec = ms % 1000; + } +} + +Date::Date(int year, int month, int day) { + Set(year, month, day); +} + +bool Date::IsValid(int year, int month, int day) { + if (year == 0) { + return false; + } + + return (day >= 1) && + (day <= DaysInMonth(month) || (day == 29 && month == 2 && IsLeapYear(year))); +} + +int Date::DaysInMonth() const { + if (IsInvalid()) { + return 0; + } + + int year = 0; + int month = 0; + JdToYmd(&year, &month, nullptr, m_jd); + + if (month == 2 && IsLeapYear(year)) { + return 29; + } + + return DaysInMonth(month); +} + +bool Date::IsLeapYear() const { + if (IsInvalid()) { + return false; + } + + int year = 0; + JdToYmd(&year, nullptr, nullptr, m_jd); + + return IsLeapYear(year); +} + +int Date::DaysInMonth(int month) { + static const int days[] = {0, 31, 28, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31}; + + if (month < 1 || month > 12) { + month = 0; + } + + return days[month]; +} + +void Date::Set(int year, int month, int day) { + if (IsValid(year, month, day)) { + YmdToJd(year, month, day, &m_jd); + } else { + m_jd = DATE_JD_INVALID; + } +} + +void Date::Get(int* year, int* month, int* day) const { + if (IsInvalid()) { + if (year != nullptr) { + *year = 0; + } + if (month != nullptr) { + *month = 0; + } + if (day != nullptr) { + *day = 0; + } + } else { + JdToYmd(year, month, day, m_jd); + } +} + +bool Date::IsLeapYear(int year) { + if (year < 1) { + year++; + } + + return ((year % 4) == 0 && (year % 100) != 0) || (year % 400) == 0; +} + +int Date::Year() const { + if (IsInvalid()) { + return 0; + } + + int year = 0; + JdToYmd(&year, nullptr, nullptr, m_jd); + + return year; +} + +int Date::Month() const { + if (IsInvalid()) { + return 0; + } + + int month = 0; + JdToYmd(nullptr, &month, nullptr, m_jd); + + return month; +} + +int Date::Day() const { + if (IsInvalid()) { + return 0; + } + + int day = 0; + JdToYmd(nullptr, nullptr, &day, m_jd); + + return day; +} + +int Date::DaysInYear() const { + if (IsInvalid()) { + return 0; + } + + return IsLeapYear() ? 366 : 365; +} + +int Date::DayOfWeek() const { + if (IsInvalid()) { + return 0; + } + + if (m_jd >= 0) { + return (m_jd % 7) + 1; + } + + return ((m_jd + 1) % 7) + 7; +} + +int Date::DayOfYear() const { + if (IsInvalid()) { + return 0; + } + + jd_t d = 0; + YmdToJd(Year(), 1, 1, &d); + + return m_jd - d + 1; +} + +int Date::DaysInYear(int year) { + return IsLeapYear(year) ? 366 : 365; +} + +// NOLINTNEXTLINE(readability-non-const-parameter) +static bool FormatDate(const Date* d, const std::string& f, size_t* out_i, std::string* out_r) { + static const char* month_short[] = {"Jan", "Feb", "Mar", "Apr", "May", "Jun", + "Jul", "Aug", "Sep", "Oct", "Nov", "Dec"}; + static const char* month_long[] = {"January", "February", "March", "April", + "May", "June", "July", "August", + "September", "October", "November", "December"}; + static const char* day_short[] = {"Mon", "Tue", "Wed", "Thu", "Fri", "Sat", "Sun"}; + static const char* day_long[] = {"Monday", "Tuesday", "Wednesday", "Thursday", + "Friday", "Saturday", "Sunday"}; + + auto& i = *out_i; + auto& r = *out_r; + if (f.compare(i, 4, "YYYY") == 0) { + int y = d->Year(); + EXIT_IF(y < 0 || y > 9999); + r += fmt::format("{:04d}", y); + i += 3; + } else if (f.compare(i, 3, "YYY") == 0) { + int y = d->Year(); + EXIT_IF(y < 0 || y > 9999); + r += Mid(fmt::format("{:04d}", y), 1); + i += 2; + } else if (f.compare(i, 2, "YY") == 0) { + int y = d->Year(); + EXIT_IF(y < 0 || y > 9999); + r += Mid(fmt::format("{:04d}", y), 2); + i += 1; + } else if (f.compare(i, 1, "Y") == 0) { + int y = d->Year(); + EXIT_IF(y < 0 || y > 9999); + r += Mid(fmt::format("{:04d}", y), 3); + } else if (f.compare(i, 1, "Q") == 0) { + r += fmt::format("{}", d->QuarterOfYear()); + } else if (f.compare(i, 2, "MM") == 0) { + r += fmt::format("{:02d}", d->Month()); + i += 1; + } else if (f.compare(i, 5, "MONTH") == 0) { + r += month_long[d->Month() - 1]; + i += 4; + } else if (f.compare(i, 3, "MON") == 0) { + r += month_short[d->Month() - 1]; + i += 2; + } else if (f.compare(i, 3, "DDD") == 0) { + r += fmt::format("{:03d}", d->DayOfYear()); + i += 2; + } else if (f.compare(i, 2, "DD") == 0) { + r += fmt::format("{:02d}", d->Day()); + i += 1; + } else if (f.compare(i, 2, "DY") == 0) { + r += day_short[d->DayOfWeek() - 1]; + i += 1; + } else if (f.compare(i, 3, "DAY") == 0) { + r += day_long[d->DayOfWeek() - 1]; + i += 2; + } else if (f.compare(i, 1, "D") == 0) { + r += fmt::format("{}", d->DayOfWeek()); + } else if (f.compare(i, 1, "J") == 0) { + r += fmt::format("{}", d->JulianDay()); + } else { + return false; + } + + return true; +} + +/** + * YYYY - 4-digit year + * YYY, YY, Y - Last 3, 2, or 1 digit(s) of year. + * Q - Quarter of year (1, 2, 3, 4; JAN-MAR = 1). + * MM - Month (01-12; JAN = 01). + * MON - Abbreviated name of month. + * MONTH - Name of month + * D - Day of week (1-7). + * DAY - Name of day. + * DY - Abbreviated name of day. + * DD - Day of month (01-31). + * DDD - Day of year (001-366). + * J - Julian day + */ +std::string Date::ToString(const char* format) const { + if (IsInvalid()) { + return ""; + } + + std::string r; + std::string f = format; + + for (size_t i = 0; i < f.size(); i++) { + if (!FormatDate(this, f, &i, &r)) { + r += f[i]; + } + } + + return r; +} + +int Date::QuarterOfYear() const { + if (IsInvalid()) { + return 0; + } + + return (Month() - 1) / 3 + 1; +} + +Date Date::FromMacros(const std::string& date) { + auto lst = Split(date, ' '); + + if (lst.size() != 3) { + return {}; + } + + static const char* month_str[] = {"Jan", "Feb", "Mar", "Apr", "May", "Jun", + "Jul", "Aug", "Sep", "Oct", "Nov", "Dec"}; + + int month = -1; + for (int i = 0; i < 12; i++) { + if (lst[0] == month_str[i]) { + month = i + 1; + break; + } + } + + if (month < 0) { + return {}; + } + + return Date(ToInt32(lst[2]), month, ToInt32(lst[1])); +} + +Time::Time(int hour, int minute, int second, int msec) { + Set(hour, minute, second, msec); +} + +void Time::Set(int hour, int minute, int second, int msec) { + if (IsValid(hour, minute, second, msec)) { + HmsToMs(hour, minute, second, msec, &m_ms); + } else { + m_ms = TIME_MS_INVALID; + } +} + +void Time::Get(int* hour, int* minute, int* second, int* msec) const { + if (IsInvalid()) { + if (hour != nullptr) { + *hour = -1; + } + if (minute != nullptr) { + *minute = -1; + } + if (second != nullptr) { + *second = -1; + } + if (msec != nullptr) { + *msec = -1; + } + } else { + MsToHms(hour, minute, second, msec, m_ms); + } +} + +bool Time::IsValid(int hour, int minute, int second, int msec) { + return (hour >= 0 && hour <= 23) && (minute >= 0 && minute <= 59) && + (second >= 0 && second <= 59) && (msec >= 0 && msec <= 999); +} + +int Time::Hour24() const { + if (IsInvalid()) { + return -1; + } + + int h = 0; + MsToHms(&h, nullptr, nullptr, nullptr, m_ms); + + return h; +} + +int Time::Hour12() const { + if (IsInvalid()) { + return -1; + } + + int h = 0; + MsToHms(&h, nullptr, nullptr, nullptr, m_ms); + + h = h % 12; + + return h == 0 ? 12 : h; +} + +int Time::Minute() const { + if (IsInvalid()) { + return -1; + } + + int m = 0; + MsToHms(nullptr, &m, nullptr, nullptr, m_ms); + + return m; +} + +int Time::Second() const { + if (IsInvalid()) { + return -1; + } + + int s = 0; + MsToHms(nullptr, nullptr, &s, nullptr, m_ms); + + return s; +} + +int Time::Msec() const { + if (IsInvalid()) { + return -1; + } + + int m = 0; + MsToHms(nullptr, nullptr, nullptr, &m, m_ms); + + return m; +} + +bool Time::IsAM() const { + if (IsInvalid()) { + return false; + } + + int h = 0; + MsToHms(&h, nullptr, nullptr, nullptr, m_ms); + + return h < 12; +} + +bool Time::IsPM() const { + if (IsInvalid()) { + return false; + } + + int h = 0; + MsToHms(&h, nullptr, nullptr, nullptr, m_ms); + + return h >= 12; +} + +// NOLINTNEXTLINE(readability-non-const-parameter) +static bool FormatTime(const Time* t, const std::string& f, size_t* out_i, std::string* out_r) { + auto& i = *out_i; + auto& r = *out_r; + if (f.compare(i, 4, "HH24") == 0) { + r += fmt::format("{:02d}", t->Hour24()); + i += 3; + } else if (f.compare(i, 4, "HH12") == 0) { + r += fmt::format("{:02d}", t->Hour12()); + i += 3; + } else if (f.compare(i, 2, "HH") == 0) { + r += fmt::format("{:02d}", t->Hour12()); + i += 1; + } else if (f.compare(i, 2, "MI") == 0) { + r += fmt::format("{:02d}", t->Minute()); + i += 1; + } else if (f.compare(i, 5, "SSSSS") == 0) { + r += fmt::format("{:05d}", t->MsecTotal() / 1000); + i += 4; + } else if (f.compare(i, 2, "SS") == 0) { + r += fmt::format("{:02d}", t->Second()); + i += 1; + } else if (f.compare(i, 3, "FFF") == 0) { + r += fmt::format("{:03d}", t->Msec()); + i += 2; + } else if (f.compare(i, 2, "AM") == 0) { + r += t->IsAM() ? "AM" : "PM"; + i += 1; + } else if (f.compare(i, 4, "A.M.") == 0) { + r += t->IsAM() ? "A.M." : "P.M."; + i += 3; + } else { + return false; + } + + return true; +} + +/** + * HH - Hour of day (1-12) + * HH12 - Hour of day (1-12) + * HH24 - Hour of day (0-23) + * MI - Minute (0-59) + * SS - Second (0-59) + * SSSSS - Seconds past midnight (0-86399) + * FFF - Milliseconds + * AM - AM or PM + * A.M. - A.M. or P.M. + */ +std::string Time::ToString(const char* format) const { + if (IsInvalid()) { + return ""; + } + + std::string r; + std::string f = format; + + for (size_t i = 0; i < f.size(); i++) { + if (!FormatTime(this, f, &i, &r)) { + r += f[i]; + } + } + + return r; +} + +Time Time::operator+(int secs) const { + int ms_secs = secs * 1000; + + EXIT_IF(ms_secs > TIME_MS_IN_DAY || ms_secs < -TIME_MS_IN_DAY); + + if (IsInvalid()) { + return {}; + } + + int r = m_ms + ms_secs; + + if (r < 0) { + r += TIME_MS_IN_DAY; + } + if (r >= TIME_MS_IN_DAY) { + r -= TIME_MS_IN_DAY; + } + + return Time(r); +} + +Time Time::operator-(int secs) const { + int ms_secs = secs * 1000; + + EXIT_IF(ms_secs > TIME_MS_IN_DAY || ms_secs < -TIME_MS_IN_DAY); + + if (IsInvalid()) { + return {}; + } + + int r = m_ms - ms_secs; + + if (r < 0) { + r += TIME_MS_IN_DAY; + } + if (r >= TIME_MS_IN_DAY) { + r -= TIME_MS_IN_DAY; + } + + return Time(r); +} + +Time Time::operator+=(int secs) { + int ms_secs = secs * 1000; + + EXIT_IF(ms_secs > TIME_MS_IN_DAY || ms_secs < -TIME_MS_IN_DAY); + + if (!IsInvalid()) { + m_ms += ms_secs; + + if (m_ms < 0) { + m_ms += TIME_MS_IN_DAY; + } + if (m_ms >= TIME_MS_IN_DAY) { + m_ms -= TIME_MS_IN_DAY; + } + } + + return *this; +} + +Time Time::operator-=(int secs) { + int ms_secs = secs * 1000; + + EXIT_IF(ms_secs > TIME_MS_IN_DAY || ms_secs < -TIME_MS_IN_DAY); + + if (!IsInvalid()) { + m_ms -= ms_secs; + + if (m_ms < 0) { + m_ms += TIME_MS_IN_DAY; + } + if (m_ms >= TIME_MS_IN_DAY) { + m_ms -= TIME_MS_IN_DAY; + } + } + + return *this; +} + +DateTime DateTime::FromSystem() { + SysTimeStruct t {}; + SysGetSystemTime(t); + + if (t.is_invalid) { + return {}; + } + + return DateTime(Date(t.Year, t.Month, t.Day), Time(t.Hour, t.Minute, t.Second, t.Milliseconds)); +} + +DateTime DateTime::FromSystemUTC() { + SysTimeStruct t {}; + SysGetSystemTimeUtc(t); + + if (t.is_invalid) { + return {}; + } + + return DateTime(Date(t.Year, t.Month, t.Day), Time(t.Hour, t.Minute, t.Second, t.Milliseconds)); +} + +/** + * YYYY - 4-digit year + * YYY, YY, Y - Last 3, 2, or 1 digit(s) of year. + * Q - Quarter of year (1, 2, 3, 4; JAN-MAR = 1). + * MM - Month (01-12; JAN = 01). + * MON - Abbreviated name of month. + * MONTH - Name of month + * D - Day of week (1-7). + * DAY - Name of day. + * DY - Abbreviated name of day. + * DD - Day of month (01-31). + * DDD - Day of year (001-366). + * J - Julian day + * HH - Hour of day (1-12) + * HH12 - Hour of day (1-12) + * HH24 - Hour of day (0-23) + * MI - Minute (0-59) + * SS - Second (0-59) + * SSSSS - Seconds past midnight (0-86399) + * FFF - Milliseconds + * AM - AM or PM + * A.M. - A.M. or P.M. + */ +std::string DateTime::ToString(const char* format) const { + if (IsInvalid()) { + return ""; + } + + std::string r; + std::string f = format; + + for (size_t i = 0; i < f.size(); i++) { + if (!FormatDate(&m_date, f, &i, &r) && !FormatTime(&m_time, f, &i, &r)) { + r += f[i]; + } + } + + return r; +} + +double DateTime::ToUnix() const { + return (static_cast(GetDate().JulianDay()) - 2440588.0) * 86400.0 + + static_cast(GetTime().MsecTotal()) / 1000.0; +} + +} // namespace Common diff --git a/src/common/dateTime.h b/src/common/dateTime.h new file mode 100644 index 000000000..ec6140e24 --- /dev/null +++ b/src/common/dateTime.h @@ -0,0 +1,183 @@ +#ifndef KYTY_COMMON_DATETIME_H_ +#define KYTY_COMMON_DATETIME_H_ + +#include "common/common.h" + +#include + +namespace Common { + +using jd_t = int32_t; + +constexpr jd_t DATE_JD_INVALID = (INT32_MIN); +constexpr int TIME_MS_INVALID = (-1); +constexpr int TIME_MS_IN_DAY = (24 * 3600 * 1000); + +constexpr int MONTH_JANUARY = 1; +constexpr int MONTH_FEBRUARY = 2; +constexpr int MONTH_MARCH = 3; +constexpr int MONTH_APRIL = 4; +constexpr int MONTH_MAY = 5; +constexpr int MONTH_JUNE = 6; +constexpr int MONTH_JULY = 7; +constexpr int MONTH_AUGUST = 8; +constexpr int MONTH_SEPTEMBER = 9; +constexpr int MONTH_OCTOBER = 10; +constexpr int MONTH_NOVEMBER = 11; +constexpr int MONTH_DECEMBER = 12; + +class Date final { +public: + Date() = default; + explicit Date(jd_t d): m_jd(d) {} + Date(const Date& d) = default; + Date(Date&& d) noexcept = default; + explicit Date(int year, int month, int day); // month in [1...12], day in [1...31] + ~Date() = default; + + static Date FromMacros(const std::string& date); + + void Set(int year, int month, int day); + void Set(jd_t jd) { this->m_jd = jd; } + void Get(int* year, int* month, int* day) const; + + [[nodiscard]] bool IsInvalid() const { return m_jd == DATE_JD_INVALID; } + [[nodiscard]] int DaysInMonth() const; + [[nodiscard]] int DaysInYear() const; + [[nodiscard]] bool IsLeapYear() const; + [[nodiscard]] int Year() const; + [[nodiscard]] int Month() const; + [[nodiscard]] int Day() const; + [[nodiscard]] jd_t JulianDay() const { return m_jd; } + [[nodiscard]] int DayOfWeek() const; + [[nodiscard]] int DayOfYear() const; + [[nodiscard]] int QuarterOfYear() const; + + std::string ToString(const char* format = "YYYY.MM.DD") const; + + static bool IsValid(int year, int month, int day); + static int DaysInMonth(int month); + static int DaysInYear(int year); + static bool IsLeapYear(int year); + + bool operator==(const Date& other) const { return m_jd == other.m_jd; } + bool operator!=(const Date& other) const { return m_jd != other.m_jd; } + bool operator<(const Date& other) const { return m_jd < other.m_jd; } + bool operator<=(const Date& other) const { return m_jd <= other.m_jd; } + bool operator>(const Date& other) const { return m_jd > other.m_jd; } + bool operator>=(const Date& other) const { return m_jd >= other.m_jd; } + + Date& operator=(const Date& other) = default; + Date& operator=(Date&& other) noexcept = default; + Date operator+(int days) const { return Date(m_jd + days); } + Date operator-(int days) const { return Date(m_jd - days); } + Date operator+=(int days) { + m_jd += days; + return *this; + } + Date operator-=(int days) { + m_jd -= days; + return *this; + } + +private: + jd_t m_jd {DATE_JD_INVALID}; +}; + +class Time final { +public: + Time() = default; + explicit Time(int msec): m_ms(msec) { + if (m_ms < 0 || m_ms >= TIME_MS_IN_DAY) { + m_ms = TIME_MS_INVALID; + } + } + Time(const Time& t) = default; + Time(Time&& t) noexcept = default; + explicit Time(int hour24, int minute, int second, int msec = 0); + ~Time() = default; + + void Set(int hour24, int minute, int second, int msec = 0); + void Set(int msec) { m_ms = msec; } + void Get(int* hour24, int* minute, int* second, int* msec = nullptr) const; + + [[nodiscard]] bool IsInvalid() const { return m_ms < 0 || m_ms >= TIME_MS_IN_DAY; } + [[nodiscard]] int Hour12() const; + [[nodiscard]] int Hour24() const; + [[nodiscard]] bool IsAM() const; + [[nodiscard]] bool IsPM() const; + [[nodiscard]] int Minute() const; + [[nodiscard]] int Second() const; + [[nodiscard]] int Msec() const; + [[nodiscard]] int MsecTotal() const { return m_ms; } + + std::string ToString(const char* format = "HH24:MI:SS") const; + + static bool IsValid(int hour, int minute, int second, int msec = 0); + + bool operator==(const Time& other) const { return m_ms == other.m_ms; } + bool operator!=(const Time& other) const { return m_ms != other.m_ms; } + bool operator<(const Time& other) const { return m_ms < other.m_ms; } + bool operator<=(const Time& other) const { return m_ms <= other.m_ms; } + bool operator>(const Time& other) const { return m_ms > other.m_ms; } + bool operator>=(const Time& other) const { return m_ms >= other.m_ms; } + + Time& operator=(const Time& other) = default; + Time& operator=(Time&& other) noexcept = default; + Time operator+(int secs) const; + Time operator-(int secs) const; + Time operator+=(int secs); + Time operator-=(int secs); + +private: + int m_ms {TIME_MS_INVALID}; +}; + +class DateTime final { +public: + DateTime() = default; + explicit DateTime(const Date& d): m_date(d) {} + explicit DateTime(const Time& t): m_time(t) {} + explicit DateTime(const Date& d, const Time& t): m_date(d), m_time(t) {} + DateTime(const DateTime& dt) = default; + DateTime(DateTime&& dt) noexcept = default; + ~DateTime() = default; + + static DateTime FromSystem(); + static DateTime FromSystemUTC(); + + [[nodiscard]] double ToUnix() const; + + [[nodiscard]] bool IsInvalid() const { return m_date.IsInvalid() || m_time.IsInvalid(); } + + void SetDate(const Date& d) { m_date = d; } + void SetTime(const Time& t) { m_time = t; } + Date& GetDate() { return m_date; } + Time& GetTime() { return m_time; } + [[nodiscard]] const Date& GetDate() const { return m_date; } + [[nodiscard]] const Time& GetTime() const { return m_time; } + + std::string ToString(const char* format = "YYYY.MM.DD HH24:MI:SS") const; + + DateTime& operator=(const DateTime& other) = default; + DateTime& operator=(DateTime&& other) noexcept = default; + + bool operator==(const DateTime& other) const { + return m_date == other.m_date && m_time == other.m_time; + } + bool operator!=(const DateTime& other) const { return !(*this == other); } + bool operator<(const DateTime& other) const { + return m_date < other.m_date || (m_date == other.m_date && m_time < other.m_time); + } + bool operator<=(const DateTime& other) const { return !(other < *this); } + bool operator>(const DateTime& other) const { return other < *this; } + bool operator>=(const DateTime& other) const { return !(*this < other); } + +private: + Date m_date; + Time m_time; +}; + +} // namespace Common + +#endif /* KYTY_COMMON_DATETIME_H_ */ diff --git a/src/common/debug.cpp b/src/common/debug.cpp new file mode 100644 index 000000000..fcdf27a33 --- /dev/null +++ b/src/common/debug.cpp @@ -0,0 +1,44 @@ +#include "common/debug.h" + +#include "common/platform/sysDbg.h" + +namespace Common { + +void DebugStack::Print(int from, [[maybe_unused]] bool with_name) const { + for (int i = from; i < depth; i++) { + printf("[%d] %016" PRIx64 "\n", i - from, static_cast(GetAddr(i))); + } +} + +void DebugStack::Trace(DebugStack* stack) { + stack->depth = DEBUG_MAX_STACK_DEPTH; + SysStackWalk(stack->stack.data(), &stack->depth); +} + +std::string Debug::GetCompiler() { +#if KYTY_COMPILER == KYTY_COMPILER_CLANG + return "clang"; +#elif KYTY_COMPILER == KYTY_COMPILER_GCC + return "gcc"; +#else + return "????"; +#endif +} + +std::string Debug::GetLinker() { +#if KYTY_LINKER == KYTY_LINKER_LD + return "ld"; +#elif KYTY_LINKER == KYTY_LINKER_LLD + return "lld"; +#elif KYTY_LINKER == KYTY_LINKER_LLD_LINK + return "lld_link"; +#else + return "??"; +#endif +} + +std::string Debug::GetBitness() { + return "64"; +} + +} // namespace Common diff --git a/src/common/debug.h b/src/common/debug.h new file mode 100644 index 000000000..5aee461dc --- /dev/null +++ b/src/common/debug.h @@ -0,0 +1,42 @@ +#ifndef KYTY_COMMON_DEBUG_H_ +#define KYTY_COMMON_DEBUG_H_ + +#include "common/common.h" +#include "common/stringUtils.h" + +#include + +namespace Common { + +#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS && KYTY_BUILD == KYTY_BUILD_DEBUG && \ + KYTY_COMPILER == KYTY_COMPILER_CLANG +constexpr int DEBUG_MAX_STACK_DEPTH = 20; +#else +constexpr int DEBUG_MAX_STACK_DEPTH = 15; +#endif + +namespace Debug { +std::string GetCompiler(); +std::string GetLinker(); +std::string GetBitness(); +} // namespace Debug + +struct DebugStack { + int depth {0}; + std::array stack {}; + + [[nodiscard]] uintptr_t GetAddr(int i) const { return reinterpret_cast(stack[i]); } + + void Print(int from, bool with_name = true) const; + + void CopyTo(DebugStack* s) const { + std::memcpy(s->stack.data(), stack.data(), sizeof(void*) * depth); + s->depth = depth; + } + + static void Trace(DebugStack* stack); +}; + +} // namespace Common + +#endif /* KYTY_COMMON_DEBUG_H_ */ diff --git a/src/common/emulatorConfig.cpp b/src/common/emulatorConfig.cpp new file mode 100644 index 000000000..06d5cdb35 --- /dev/null +++ b/src/common/emulatorConfig.cpp @@ -0,0 +1,104 @@ +#include "common/emulatorConfig.h" + +#include "common/assert.h" + +#include +#include + +namespace Config { + +static std::unique_ptr g_config; + +KYTY_SUBSYSTEM_INIT(Config) { + EXIT_IF(g_config != nullptr); + + g_config = std::make_unique(); +} + +KYTY_SUBSYSTEM_UNEXPECTED_SHUTDOWN(Config) {} + +KYTY_SUBSYSTEM_DESTROY(Config) {} + +void Load(const ConfigOptions& cfg) { + EXIT_IF(g_config == nullptr); + + *g_config = cfg; +} + +uint32_t GetScreenWidth() { + return g_config->screen_width; +} + +uint32_t GetScreenHeight() { + return g_config->screen_height; +} + +uint32_t GetVblankFrequency() { + return std::clamp(g_config->vblank_frequency, 30u, 360u); +} + +bool VulkanValidationEnabled() { + return g_config->vulkan_validation_enabled; +} + +bool ShaderValidationEnabled() { + return g_config->shader_validation_enabled; +} + +ShaderOptimizationType GetShaderOptimizationType() { + return g_config->shader_optimization_type; +} + +ShaderLogDirection GetShaderLogDirection() { + return g_config->shader_log_direction; +} + +std::filesystem::path GetShaderLogFolder() { + return g_config->shader_log_folder; +} + +bool CommandBufferDumpEnabled() { + return g_config->command_buffer_dump_enabled; +} + +std::filesystem::path GetCommandBufferDumpFolder() { + return g_config->command_buffer_dump_folder; +} + +bool GraphicsDebugDumpEnabled() { + return g_config->graphics_debug_dump_enabled; +} + +OutputDirection GetPrintfDirection() { + return g_config->printf_direction; +} + +std::filesystem::path GetPrintfOutputFile() { + return g_config->printf_output_file; +} + +ProfilerDirection GetProfilerDirection() { + return g_config->profiler_direction; +} + +bool SpirvDebugPrintfEnabled() { + return g_config->spirv_debug_printf_enabled; +} + +bool PipelineDumpEnabled() { + return g_config->pipeline_dump_enabled; +} + +std::filesystem::path GetPipelineDumpFolder() { + return g_config->pipeline_dump_folder; +} + +bool RenderDocEnabled() { + return g_config->renderdoc_enabled; +} + +bool NggRectlistDrawEnabled() { + return g_config->ngg_rectlist_draw_enabled; +} + +} // namespace Config diff --git a/src/common/emulatorConfig.h b/src/common/emulatorConfig.h new file mode 100644 index 000000000..82f614147 --- /dev/null +++ b/src/common/emulatorConfig.h @@ -0,0 +1,75 @@ +#ifndef KYTY_COMMON_EMULATOR_CONFIG_H_ +#define KYTY_COMMON_EMULATOR_CONFIG_H_ + +#include "common/common.h" +#include "common/subsystems.h" + +#include + +namespace Config { + +KYTY_SUBSYSTEM_DEFINE(Config); + +enum class ShaderOptimizationType { None, Size, Performance }; + +enum class ShaderLogDirection { Silent, Console, File }; + +enum class ProfilerDirection { None, Network }; + +enum class OutputDirection { Silent, Console, File }; + +struct ConfigOptions { + uint32_t screen_width = 1280; + uint32_t screen_height = 720; + uint32_t vblank_frequency = 60; + bool vulkan_validation_enabled = false; + bool shader_validation_enabled = false; + ShaderOptimizationType shader_optimization_type = ShaderOptimizationType::None; + ShaderLogDirection shader_log_direction = ShaderLogDirection::Silent; + std::filesystem::path shader_log_folder = "_Shaders"; + bool command_buffer_dump_enabled = false; + std::filesystem::path command_buffer_dump_folder = "_Buffers"; + bool graphics_debug_dump_enabled = false; + OutputDirection printf_direction = OutputDirection::Console; + std::filesystem::path printf_output_file = "_kyty.txt"; + ProfilerDirection profiler_direction = ProfilerDirection::None; + bool spirv_debug_printf_enabled = false; + bool pipeline_dump_enabled = false; + std::filesystem::path pipeline_dump_folder = "_Pipelines"; + bool renderdoc_enabled = false; + bool ngg_rectlist_draw_enabled = true; +}; + +void Load(const ConfigOptions& cfg); + +uint32_t GetScreenWidth(); +uint32_t GetScreenHeight(); +uint32_t GetVblankFrequency(); +bool VulkanValidationEnabled(); + +bool ShaderValidationEnabled(); +ShaderOptimizationType GetShaderOptimizationType(); +ShaderLogDirection GetShaderLogDirection(); +std::filesystem::path GetShaderLogFolder(); + +bool CommandBufferDumpEnabled(); +std::filesystem::path GetCommandBufferDumpFolder(); + +bool GraphicsDebugDumpEnabled(); + +OutputDirection GetPrintfDirection(); +std::filesystem::path GetPrintfOutputFile(); + +ProfilerDirection GetProfilerDirection(); + +bool SpirvDebugPrintfEnabled(); + +bool PipelineDumpEnabled(); +std::filesystem::path GetPipelineDumpFolder(); + +bool RenderDocEnabled(); +bool NggRectlistDrawEnabled(); + +} // namespace Config + +#endif /* KYTY_COMMON_EMULATOR_CONFIG_H_ */ diff --git a/src/common/file.cpp b/src/common/file.cpp new file mode 100644 index 000000000..80ec93dc6 --- /dev/null +++ b/src/common/file.cpp @@ -0,0 +1,561 @@ +#include "common/file.h" + +#include "common/assert.h" +#include "common/common.h" +#include "common/platform/sysFileIO.h" +#include "common/platform/sysTimer.h" +#include "common/stringUtils.h" + +#include +#include +#include +#include + +// IWYU pragma: no_include +// IWYU pragma: no_include +// IWYU pragma: no_include + +#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS +#ifdef CreateDirectory +#undef CreateDirectory +#endif +#ifdef DeleteFile +#undef DeleteFile +#endif +#ifdef CopyFile +#undef CopyFile +#endif +#ifdef MoveFile +#undef MoveFile +#endif +#endif + +namespace Common { + +namespace { + +SysFileTimeStruct DateTimeToFileTimeUtc(const DateTime& date_time) { + SysTimeStruct time {}; + time.is_invalid = false; + + int year = 0; + int month = 0; + int day = 0; + date_time.GetDate().Get(&year, &month, &day); + time.Year = year; + time.Month = month; + time.Day = day; + + int hour = 0; + int minute = 0; + int second = 0; + int milliseconds = 0; + date_time.GetTime().Get(&hour, &minute, &second, &milliseconds); + time.Hour = hour; + time.Minute = minute; + time.Second = second; + time.Milliseconds = milliseconds; + + SysFileTimeStruct file_time {}; + SysSystemToFileTimeUtc(time, file_time); + return file_time; +} + +} // namespace + +struct File::FilePrivate { + sys_file_t* f; +}; + +File::File(): m_p(std::make_unique()) { + m_p->f = nullptr; +} + +File::~File() { + EXIT_IF(m_p->f != nullptr); +} + +File::File(const std::filesystem::path& name): m_p(std::make_unique()) { + m_p->f = nullptr; + + Create(name); +} + +File::File(const std::filesystem::path& name, Mode mode): m_p(std::make_unique()) { + m_p->f = nullptr; + + Open(name, mode); +} + +bool File::IsInvalid() const { + return m_p->f == nullptr; +} + +bool File::Create(const std::filesystem::path& name) { + EXIT_IF(m_p->f != nullptr); + + m_file_name = name; + + m_p->f = SysFileCreate(name); + + if (SysFileIsError(*m_p->f)) { + Close(); + return false; + } + + return true; +} + +bool File::Open(const std::filesystem::path& name, Mode mode) { + EXIT_IF(m_p->f != nullptr); + + m_file_name = name; + + switch (mode) { + case Mode::Read: m_p->f = SysFileOpenR(name); break; + case Mode::Write: m_p->f = SysFileOpenW(name); break; + case Mode::ReadWrite: + case Mode::WriteRead: m_p->f = SysFileOpenRw(name); break; + } + + if (SysFileIsError(*m_p->f)) { + Close(); + return false; + } + + return true; +} + +bool File::OpenInMem(void* buf, uint32_t buf_size) { + EXIT_IF(m_p->f != nullptr); + + m_p->f = SysFileOpen(static_cast(buf), buf_size); + + if (SysFileIsError(*m_p->f)) { + Close(); + return false; + } + + return true; +} + +bool File::OpenInMem(ByteBuffer& buf) { + return OpenInMem(buf.GetData(), buf.Size()); +} + +bool File::CreateInMem() { + EXIT_IF(m_p->f != nullptr); + + m_p->f = SysFileCreate(); + + if (SysFileIsError(*m_p->f)) { + Close(); + return false; + } + + return true; +} + +void File::Close() { + if (m_p->f != nullptr) { + SysFileClose(m_p->f); + m_p->f = nullptr; + } +} + +uint64_t File::Size() const { + EXIT_IF(m_p->f == nullptr); + + if (m_p->f == nullptr) { + return 0; + } + + return SysFileSize(*m_p->f); +} + +uint64_t File::Remaining() const { + EXIT_IF(m_p->f == nullptr); + + return Size() - Tell(); +} + +uint64_t File::Size(const std::filesystem::path& name) { + return SysFileSize(name); +} + +bool File::Seek(uint64_t offset) { + EXIT_IF(m_p->f == nullptr); + + return SysFileSeek(*m_p->f, offset); +} + +bool File::Truncate(uint64_t size) { + EXIT_IF(m_p->f == nullptr); + + return SysFileTruncate(*m_p->f, size); +} + +bool File::Unlink() { + EXIT_IF(m_p->f == nullptr); + + return SysFileUnlink(*m_p->f, m_file_name); +} + +uint64_t File::Tell() const { + EXIT_IF(m_p->f == nullptr); + + return SysFileTell(*m_p->f); +} + +void File::Read(void* data, uint32_t size, uint32_t* bytes_read) { + EXIT_IF(m_p->f == nullptr); + + if (m_p->f != nullptr) { + SysFileRead(data, size, *m_p->f, bytes_read); + } +} + +void File::Write(const void* data, uint32_t size, uint32_t* bytes_written) { + EXIT_IF(m_p->f == nullptr); + + SysFileWrite(data, size, *m_p->f, bytes_written); +} + +static std::string FileVformat(const char* format, va_list args) { + va_list copy {}; + va_copy(copy, args); + const int size = std::vsnprintf(nullptr, 0, format, copy); + va_end(copy); + + if (size <= 0) { + return {}; + } + + std::vector buffer(static_cast(size) + 1u); + va_list copy2 {}; + va_copy(copy2, args); + std::vsnprintf(buffer.data(), buffer.size(), format, copy2); + va_end(copy2); + + return {buffer.data(), static_cast(size)}; +} + +void File::Printf(const char* format, ...) { + va_list args {}; + va_start(args, format); + + auto s = FileVformat(format, args); + + va_end(args); + + s = Common::ReplaceStr(s, "\n", "\r\n"); + Write(s.data(), static_cast(s.size())); +} + +bool File::IsDirectoryExisting(const std::filesystem::path& path) { + auto path_str = PathToGenericString(path); + return SysFileIsDirectoryExisting( + Common::EndsWith(path_str, "/") ? Common::RemoveLast(path_str, 1) : path_str); +} + +bool File::IsFileExisting(const std::filesystem::path& name) { + return SysFileIsFileExisting(name); +} + +bool File::CreateDirectory( + const std::filesystem::path& path) // @suppress("Member declaration not found") +{ + auto path_str = PathToGenericString(path); + return SysFileCreateDirectory(Common::EndsWith(path_str, "/") ? Common::RemoveLast(path_str, 1) + : path_str); +} + +bool File::DeleteDirectory(const std::filesystem::path& path) { + auto path_str = PathToGenericString(path); + return SysFileDeleteDirectory(Common::EndsWith(path_str, "/") ? Common::RemoveLast(path_str, 1) + : path_str); +} + +bool File::CreateDirectories(const std::filesystem::path& path) { + std::string real_path = Common::ReplaceChar(PathToGenericString(path), '\\', '/'); + + std::vector list = Common::Split(real_path, "/"); + + std::string p; + + for (uint32_t si = 0; si < list.size(); si++) { + const std::string& s = list[si]; + + if (si != 0 || Common::StartsWith(real_path, "/")) { + p += "/"; + } + + p += s; + + if (IsDirectoryExisting(p)) { + continue; + } + + if (!CreateDirectory(p)) // @suppress("Invalid arguments") + { + return false; + } + } + + return true; +} + +bool File::DeleteDirectories(const std::filesystem::path& path) { + std::string real_path = Common::ReplaceChar(PathToGenericString(path), '\\', '/'); + + std::vector list = Common::Split(real_path, "/"); + + std::string p; + std::vector list2; + + for (uint32_t si = 0; si < list.size(); si++) { + const std::string& s = list[si]; + + if (si != 0 || Common::StartsWith(real_path, "/")) { + p += "/"; + } + + p += s; + + list2.push_back(p); + } + + uint32_t num = list2.size(); + + for (uint32_t si = num - 1; si < num; si--) { + if (!DeleteDirectory(list2[si])) { + return false; + } + } + + return true; +} + +bool File::DeleteFile( + const std::filesystem::path& name) // @suppress("Member declaration not found") +{ + return SysFileDeleteFile(name); +} + +bool File::Flush() { + EXIT_IF(m_p->f == nullptr); + + return SysFileFlush(*m_p->f); +} + +ByteBuffer File::ReadWholeBuffer() { + EXIT_IF(IsInvalid()); + EXIT_IF(Tell() != 0); + + uint64_t s = Size(); + + EXIT_IF((s >> 32u) != 0); + + ByteBuffer buf(s); + + Read(buf.GetData(), s); + + return buf; +} + +ByteBuffer File::Read(uint32_t size) { + ByteBuffer buf(size); + uint32_t b = 0; + Read(buf.GetData(), size, &b); + buf.RemoveAt(b, size - b); + return buf; +} + +void File::Write(const ByteBuffer& buf, uint32_t* bytes_written) { + Write(buf.GetDataConst(), buf.Size(), bytes_written); +} + +DateTime File::GetLastAccessTimeUTC(const std::filesystem::path& name) { + SysTimeStruct t {}; + SysFileToSystemTimeUtc(SysFileGetLastAccessTimeUtc(name), t); + + if (t.is_invalid) { + return {}; + } + + return DateTime(Date(t.Year, t.Month, t.Day), Time(t.Hour, t.Minute, t.Second, t.Milliseconds)); +} + +DateTime File::GetLastWriteTimeUTC(const std::filesystem::path& name) { + SysTimeStruct t {}; + SysFileToSystemTimeUtc(SysFileGetLastWriteTimeUtc(name), t); + + if (t.is_invalid) { + return {}; + } + + return DateTime(Date(t.Year, t.Month, t.Day), Time(t.Hour, t.Minute, t.Second, t.Milliseconds)); +} + +void File::GetLastAccessAndWriteTimeUTC(const std::filesystem::path& name, DateTime* access, + DateTime* write) { + EXIT_IF(access == nullptr); + EXIT_IF(write == nullptr); + + SysTimeStruct at {}; + SysTimeStruct wt {}; + SysFileTimeStruct af {}; + SysFileTimeStruct wf {}; + + SysFileGetLastAccessAndWriteTimeUtc(name, af, wf); + + SysFileToSystemTimeUtc(af, at); + SysFileToSystemTimeUtc(wf, wt); + + if (at.is_invalid || wt.is_invalid) { + *access = DateTime(); + *write = DateTime(); + } else { + *access = DateTime(Date(at.Year, at.Month, at.Day), + Time(at.Hour, at.Minute, at.Second, at.Milliseconds)); + *write = DateTime(Date(wt.Year, wt.Month, wt.Day), + Time(wt.Hour, wt.Minute, wt.Second, wt.Milliseconds)); + } +} + +void File::GetLastAccessAndWriteTimeUTC(DateTime* access, DateTime* write) { + EXIT_IF(access == nullptr); + EXIT_IF(write == nullptr); + + EXIT_IF(m_p->f == nullptr); + + if (m_p->f == nullptr) { + *access = DateTime(); + *write = DateTime(); + return; + } + + SysTimeStruct at {}; + SysTimeStruct wt {}; + SysFileTimeStruct af {}; + SysFileTimeStruct wf {}; + + SysFileGetLastAccessAndWriteTimeUtc(*m_p->f, af, wf); + + SysFileToSystemTimeUtc(af, at); + SysFileToSystemTimeUtc(wf, wt); + + if (at.is_invalid || wt.is_invalid) { + *access = DateTime(); + *write = DateTime(); + } else { + *access = DateTime(Date(at.Year, at.Month, at.Day), + Time(at.Hour, at.Minute, at.Second, at.Milliseconds)); + *write = DateTime(Date(wt.Year, wt.Month, wt.Day), + Time(wt.Hour, wt.Minute, wt.Second, wt.Milliseconds)); + } +} + +bool File::SetLastAccessTimeUTC(const std::filesystem::path& name, const DateTime& dt) { + auto f = DateTimeToFileTimeUtc(dt); + return SysFileSetLastAccessTimeUtc(name, f); +} + +bool File::SetLastWriteTimeUTC(const std::filesystem::path& name, const DateTime& dt) { + auto f = DateTimeToFileTimeUtc(dt); + return SysFileSetLastWriteTimeUtc(name, f); +} + +bool File::SetLastAccessAndWriteTimeUTC(const std::filesystem::path& name, const DateTime& access, + const DateTime& write) { + auto af = DateTimeToFileTimeUtc(access); + auto wf = DateTimeToFileTimeUtc(write); + return SysFileSetLastAccessAndWriteTimeUtc(name, af, wf); +} + +std::vector File::FindFiles(const std::filesystem::path& path) { + std::vector files; + + SysFileFindFiles(path, files); + + auto path_str = PathToGenericString(path); + uint32_t len = static_cast(path_str.size()); + + if (!Common::EndsWith(path_str, "/") && !Common::EndsWith(path_str, "\\")) { + len++; + } + + std::vector ret; + ret.reserve(files.size()); + + for (const auto& f: files) { + File::FindInfo r {}; + + r.path_with_name = f.path_with_name; + r.rel_path_with_name = Common::Mid(PathToGenericString(f.path_with_name), len); + r.size = f.size; + + SysTimeStruct at {}; + SysTimeStruct wt {}; + + SysFileToSystemTimeUtc(f.last_access_time, at); + SysFileToSystemTimeUtc(f.last_write_time, wt); + + if (!at.is_invalid && !wt.is_invalid) { + r.last_access_time = DateTime(Date(at.Year, at.Month, at.Day), + Time(at.Hour, at.Minute, at.Second, at.Milliseconds)); + r.last_write_time = DateTime(Date(wt.Year, wt.Month, wt.Day), + Time(wt.Hour, wt.Minute, wt.Second, wt.Milliseconds)); + } + + // printf("%s, %s, %" PRIu64", %s, %s\n", r.path_with_name.c_str(), + // r.rel_path_with_name.c_str(), r.size, r.last_access_time.ToString().c_str(), + // r.last_write_time.ToString().c_str()); + + ret.push_back(std::move(r)); + } + + return ret; +} + +std::vector File::GetDirEntries(const std::filesystem::path& path) { + std::vector files; + + SysFileGetDents(path, files); + + std::vector ret; + ret.reserve(files.size()); + + for (const auto& f: files) { + File::DirEntry r {}; + + r.name = f.name; + r.is_file = f.is_file; + + ret.push_back(std::move(r)); + } + + return ret; +} + +bool File::CopyFile(const std::filesystem::path& src, + const std::filesystem::path& dst) // @suppress("Member declaration not found") +{ + return SysFileCopyFile(src, dst); +} + +bool File::MoveFile(const std::filesystem::path& src, + const std::filesystem::path& dst) // @suppress("Member declaration not found") +{ + if (IsFileExisting(dst)) { + DeleteFile(dst); // @suppress("Invalid arguments") + } + + return SysFileMoveFile(src, dst); +} + +void File::RemoveReadonly(const std::filesystem::path& name) { + SysFileRemoveReadonly(name); +} + +} // namespace Common diff --git a/src/common/file.h b/src/common/file.h new file mode 100644 index 000000000..bf95fca64 --- /dev/null +++ b/src/common/file.h @@ -0,0 +1,124 @@ +#ifndef KYTY_COMMON_FILE_H_ +#define KYTY_COMMON_FILE_H_ + +#include "common/byteBuffer.h" +#include "common/common.h" +#include "common/dateTime.h" + +#include +#include +#include +#include + +#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS +#ifdef CreateDirectory +#undef CreateDirectory +#endif +#ifdef DeleteFile +#undef DeleteFile +#endif +#ifdef CopyFile +#undef CopyFile +#endif +#ifdef MoveFile +#undef MoveFile +#endif +#endif + +namespace Common { + +// SUBSYSTEM_DEFINE(File); + +class File { +public: + enum class Mode { Read, Write, ReadWrite, WriteRead }; + + struct FindInfo { + std::filesystem::path path_with_name; + std::filesystem::path rel_path_with_name; + DateTime last_access_time; + DateTime last_write_time; + uint64_t size; + }; + + struct DirEntry { + std::string name; + bool is_file; + }; + + File(); + explicit File(const std::filesystem::path& name); + File(const std::filesystem::path& name, Mode mode); + virtual ~File(); + + bool Create(const std::filesystem::path& name); + bool Open(const std::filesystem::path& name, Mode mode); + bool OpenInMem(void* buf, uint32_t buf_size); + bool OpenInMem(ByteBuffer& buf); // NOLINT(google-runtime-references) + bool CreateInMem(); + + void Close(); + + bool Flush(); + + [[nodiscard]] uint64_t Size() const; + [[nodiscard]] uint64_t Remaining() const; + + bool Seek(uint64_t offset); + [[nodiscard]] uint64_t Tell() const; + + bool Truncate(uint64_t size); + bool Unlink(); + + [[nodiscard]] bool IsInvalid() const; + + [[nodiscard]] bool IsEOF() const { return Tell() >= Size(); } + + void GetLastAccessAndWriteTimeUTC(DateTime* access, DateTime* write); + + void Read(void* data, uint32_t size, uint32_t* bytes_read = nullptr); + ByteBuffer Read(uint32_t size); + void Write(const void* data, uint32_t size, uint32_t* bytes_written = nullptr); + void Write(const ByteBuffer& buf, uint32_t* bytes_written = nullptr); + + void Printf(const char* format, ...) KYTY_FORMAT_PRINTF(2, 3); + + ByteBuffer ReadWholeBuffer(); + + static uint64_t Size(const std::filesystem::path& name); + + static bool IsDirectoryExisting(const std::filesystem::path& path); + static bool IsFileExisting(const std::filesystem::path& name); + static bool CreateDirectory(const std::filesystem::path& path); + static bool CreateDirectories(const std::filesystem::path& path); + static bool DeleteDirectory(const std::filesystem::path& path); + static bool DeleteDirectories(const std::filesystem::path& path); + static bool DeleteFile(const std::filesystem::path& name); + + static DateTime GetLastAccessTimeUTC(const std::filesystem::path& name); + static DateTime GetLastWriteTimeUTC(const std::filesystem::path& name); + static void GetLastAccessAndWriteTimeUTC(const std::filesystem::path& name, DateTime* access, + DateTime* write); + static bool SetLastAccessTimeUTC(const std::filesystem::path& name, const DateTime& dt); + static bool SetLastWriteTimeUTC(const std::filesystem::path& name, const DateTime& dt); + static bool SetLastAccessAndWriteTimeUTC(const std::filesystem::path& name, + const DateTime& access, const DateTime& write); + + static std::vector FindFiles(const std::filesystem::path& path); + static std::vector GetDirEntries(const std::filesystem::path& path); + static bool CopyFile(const std::filesystem::path& src, const std::filesystem::path& dst); + static bool MoveFile(const std::filesystem::path& src, const std::filesystem::path& dst); + static void RemoveReadonly(const std::filesystem::path& name); + + KYTY_CLASS_NO_COPY(File); + +private: + struct FilePrivate; + + std::filesystem::path m_file_name; + std::unique_ptr m_p; +}; + +} // namespace Common + +#endif /* KYTY_COMMON_FILE_H_ */ diff --git a/src/common/hash.h b/src/common/hash.h new file mode 100644 index 000000000..085d6c475 --- /dev/null +++ b/src/common/hash.h @@ -0,0 +1,156 @@ +#ifndef KYTY_COMMON_HASH_H_ +#define KYTY_COMMON_HASH_H_ + +#include "common/common.h" + +namespace Common { + +inline uint32_t hash(const void* key, uint32_t key_len) { + uint32_t hash = 0; + + const auto* ptr = static_cast(key); + + while (key_len >= 4) { + hash += ptr[0]; + hash += (hash << 10u); + hash ^= (hash >> 6u); + hash += ptr[1]; + hash += (hash << 10u); + hash ^= (hash >> 6u); + hash += ptr[2]; + hash += (hash << 10u); + hash ^= (hash >> 6u); + hash += ptr[3]; + hash += (hash << 10u); + hash ^= (hash >> 6u); + + key_len -= 4; + ptr += 4; + } + + switch (key_len) { + case 3: + hash += ptr[2]; + hash += (hash << 10u); + hash ^= (hash >> 6u); + [[fallthrough]]; + case 2: + hash += ptr[1]; + hash += (hash << 10u); + hash ^= (hash >> 6u); + [[fallthrough]]; + case 1: + hash += ptr[0]; + hash += (hash << 10u); + hash ^= (hash >> 6u); + [[fallthrough]]; + default: break; + } + + hash += (hash << 3u); + hash ^= (hash >> 11u); + hash += (hash << 15u); + + return hash; +} + +inline uint32_t hash8(uint8_t key) { + uint32_t hash = 0; + + uint8_t* ptr = &key; + + hash += ptr[0]; + hash += (hash << 10u); + hash ^= (hash >> 6u); + + hash += (hash << 3u); + hash ^= (hash >> 11u); + hash += (hash << 15u); + + return hash; +} + +inline uint32_t hash16(uint16_t key) { + uint32_t hash = 0; + + auto* ptr = reinterpret_cast(&key); + + hash += ptr[0]; + hash += (hash << 10u); + hash ^= (hash >> 6u); + hash += ptr[1]; + hash += (hash << 10u); + hash ^= (hash >> 6u); + + hash += (hash << 3u); + hash ^= (hash >> 11u); + hash += (hash << 15u); + + return hash; +} + +inline uint32_t hash32(uint32_t key) { + uint32_t hash = 0; + + auto* ptr = reinterpret_cast(&key); + + hash += ptr[0]; + hash += (hash << 10u); + hash ^= (hash >> 6u); + hash += ptr[1]; + hash += (hash << 10u); + hash ^= (hash >> 6u); + hash += ptr[2]; + hash += (hash << 10u); + hash ^= (hash >> 6u); + hash += ptr[3]; + hash += (hash << 10u); + hash ^= (hash >> 6u); + + hash += (hash << 3u); + hash ^= (hash >> 11u); + hash += (hash << 15u); + + return hash; +} + +inline uint32_t hash64(uint64_t key) { + uint32_t hash = 0; + + auto* ptr = reinterpret_cast(&key); + + hash += ptr[0]; + hash += (hash << 10u); + hash ^= (hash >> 6u); + hash += ptr[1]; + hash += (hash << 10u); + hash ^= (hash >> 6u); + hash += ptr[2]; + hash += (hash << 10u); + hash ^= (hash >> 6u); + hash += ptr[3]; + hash += (hash << 10u); + hash ^= (hash >> 6u); + hash += ptr[4]; + hash += (hash << 10u); + hash ^= (hash >> 6u); + hash += ptr[5]; + hash += (hash << 10u); + hash ^= (hash >> 6u); + hash += ptr[6]; + hash += (hash << 10u); + hash ^= (hash >> 6u); + hash += ptr[7]; + hash += (hash << 10u); + hash ^= (hash >> 6u); + + hash += (hash << 3u); + hash ^= (hash >> 11u); + hash += (hash << 15u); + + return hash; +} + +} // namespace Common + +#endif /* KYTY_COMMON_HASH_H_ */ diff --git a/src/common/hostException.cpp b/src/common/hostException.cpp new file mode 100644 index 000000000..5976869e6 --- /dev/null +++ b/src/common/hostException.cpp @@ -0,0 +1,151 @@ +#include "common/hostException.h" + +#include +#include +#include + +#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS +#include // IWYU pragma: keep +#endif + +// IWYU pragma: no_include +// IWYU pragma: no_include +// IWYU pragma: no_include +// IWYU pragma: no_include +// IWYU pragma: no_include + +namespace Common::HostException { + +#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS + +static std::atomic g_handler {nullptr}; +static std::atomic_uint32_t g_install_state {0}; +static thread_local bool g_in_exception_filter = false; + +static_assert(decltype(g_handler)::is_always_lock_free); +static_assert(decltype(g_install_state)::is_always_lock_free); + +[[noreturn]] static void FailFast(const char* reason) noexcept { + std::fputs("HostException fail-fast: ", stderr); + std::fputs(reason != nullptr ? reason : "unspecified", stderr); + std::fputc('\n', stderr); + std::fflush(stderr); + TerminateProcess(GetCurrentProcess(), static_cast(EXCEPTION_NONCONTINUABLE_EXCEPTION)); + std::_Exit(321); +} + +class FilterScope final { +public: + FilterScope() noexcept { + if (g_in_exception_filter) { + FailFast("nested exception while resolving a host fault"); + } + g_in_exception_filter = true; + } + + ~FilterScope() { g_in_exception_filter = false; } + + KYTY_CLASS_NO_COPY(FilterScope); +}; + +static LONG WINAPI ExceptionFilter(PEXCEPTION_POINTERS exception) { + FilterScope filter_scope; + + auto* exception_record = exception->ExceptionRecord; + + if (exception_record->ExceptionCode == DBG_PRINTEXCEPTION_C || + exception_record->ExceptionCode == DBG_PRINTEXCEPTION_WIDE_C) { + return EXCEPTION_CONTINUE_SEARCH; + } + + if (exception_record->ExceptionCode == 0x406D1388) { + // Set a thread name. + return EXCEPTION_CONTINUE_EXECUTION; + } + + ExceptionInfo info {}; + info.exception_address = reinterpret_cast(exception_record->ExceptionAddress); + info.native_code = exception_record->ExceptionCode; + info.native_context = exception->ContextRecord; + + if (exception_record->ExceptionCode == EXCEPTION_ACCESS_VIOLATION) { + info.type = ExceptionType::AccessViolation; + switch (exception_record->ExceptionInformation[0]) { + case 0: info.access_violation_type = AccessViolationType::Read; break; + case 1: info.access_violation_type = AccessViolationType::Write; break; + case 8: info.access_violation_type = AccessViolationType::Execute; break; + default: info.access_violation_type = AccessViolationType::Unknown; break; + } + info.access_violation_vaddr = exception_record->ExceptionInformation[1]; + } else if (exception_record->ExceptionCode == EXCEPTION_ILLEGAL_INSTRUCTION) { + info.type = ExceptionType::IllegalInstruction; + } else { + printf("Unhandled win exception: code=0x%08" PRIx32 ", addr=0x%016" PRIx64 + ", rip=0x%016" PRIx64 ", rsp=0x%016" PRIx64 ", rbp=0x%016" PRIx64 "\n", + static_cast(exception_record->ExceptionCode), + reinterpret_cast(exception_record->ExceptionAddress), + exception->ContextRecord->Rip, exception->ContextRecord->Rsp, + exception->ContextRecord->Rbp); + return EXCEPTION_CONTINUE_SEARCH; + } + + info.rax = exception->ContextRecord->Rax; + info.rbx = exception->ContextRecord->Rbx; + info.rcx = exception->ContextRecord->Rcx; + info.rdx = exception->ContextRecord->Rdx; + info.rsi = exception->ContextRecord->Rsi; + info.rdi = exception->ContextRecord->Rdi; + info.rbp = exception->ContextRecord->Rbp; + info.rsp = exception->ContextRecord->Rsp; + info.r8 = exception->ContextRecord->R8; + info.r9 = exception->ContextRecord->R9; + info.r10 = exception->ContextRecord->R10; + info.r11 = exception->ContextRecord->R11; + info.r12 = exception->ContextRecord->R12; + info.r13 = exception->ContextRecord->R13; + info.r14 = exception->ContextRecord->R14; + info.r15 = exception->ContextRecord->R15; + + if (g_install_state.load(std::memory_order_acquire) == 0) { + FailFast("host exception handler is not installed"); + } + + const auto handler = g_handler.load(std::memory_order_acquire); + if (handler == nullptr) { + FailFast("host exception callback is null"); + } + + return handler(info) ? EXCEPTION_CONTINUE_EXECUTION : EXCEPTION_CONTINUE_SEARCH; +} + +#endif + +bool InstallHandler(Handler handler) { +#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS + if (handler == nullptr) { + return false; + } + + uint32_t expected_state = 0; + if (!g_install_state.compare_exchange_strong(expected_state, 1, std::memory_order_acq_rel)) { + return expected_state == 2 && g_handler.load(std::memory_order_acquire) == handler; + } + + g_handler.store(handler, std::memory_order_release); + + if (AddVectoredExceptionHandler(1, ExceptionFilter) == nullptr) { + g_handler.store(nullptr, std::memory_order_release); + g_install_state.store(0, std::memory_order_release); + printf("AddVectoredExceptionHandler() failed\n"); + return false; + } + + g_install_state.store(2, std::memory_order_release); + return true; +#else + (void)handler; + return false; +#endif +} + +} // namespace Common::HostException diff --git a/src/common/hostException.h b/src/common/hostException.h new file mode 100644 index 000000000..5540ffa4e --- /dev/null +++ b/src/common/hostException.h @@ -0,0 +1,44 @@ +#ifndef KYTY_COMMON_HOST_EXCEPTION_H_ +#define KYTY_COMMON_HOST_EXCEPTION_H_ + +#include "common/common.h" + +namespace Common::HostException { + +enum class ExceptionType { Unknown, AccessViolation, IllegalInstruction }; + +enum class AccessViolationType { Unknown, Read, Write, Execute }; + +struct ExceptionInfo { + ExceptionType type = ExceptionType::Unknown; + AccessViolationType access_violation_type = AccessViolationType::Unknown; + uint64_t access_violation_vaddr = 0; + uint64_t exception_address = 0; + uint64_t rax = 0; + uint64_t rbx = 0; + uint64_t rcx = 0; + uint64_t rdx = 0; + uint64_t rsi = 0; + uint64_t rdi = 0; + uint64_t rbp = 0; + uint64_t rsp = 0; + uint64_t r8 = 0; + uint64_t r9 = 0; + uint64_t r10 = 0; + uint64_t r11 = 0; + uint64_t r12 = 0; + uint64_t r13 = 0; + uint64_t r14 = 0; + uint64_t r15 = 0; + uint32_t native_code = 0; + // Platform-specific mutable context, valid only for the duration of the handler call. + void* native_context = nullptr; +}; + +using Handler = bool (*)(const ExceptionInfo&); + +bool InstallHandler(Handler handler); + +} // namespace Common::HostException + +#endif /* KYTY_COMMON_HOST_EXCEPTION_H_ */ diff --git a/src/common/logging/log.cpp b/src/common/logging/log.cpp new file mode 100644 index 000000000..a53b52672 --- /dev/null +++ b/src/common/logging/log.cpp @@ -0,0 +1,189 @@ + +#include "common/logging/log.h" + +#include "common/assert.h" +#include "common/emulatorConfig.h" +#include "common/stringUtils.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace { + +class RawFormatter final: public spdlog::formatter { +public: + void format(const spdlog::details::log_msg& msg, spdlog::memory_buf_t& dest) override { + const std::string_view payload(msg.payload.data(), msg.payload.size()); + dest.append(payload.data(), payload.data() + payload.size()); + } + + std::unique_ptr clone() const override { + return std::make_unique(); + } +}; + +std::shared_ptr MakeLogger(std::string name, spdlog::sink_ptr sink) { + sink->set_formatter(std::make_unique()); + + auto logger = std::make_shared(std::move(name), std::move(sink)); + logger->set_level(spdlog::level::trace); + return logger; +} + +std::shared_ptr MakeFileLogger(std::string name, + const std::filesystem::path& path) { + const auto parent = path.parent_path(); + if (!parent.empty()) { + std::filesystem::create_directories(parent); + } + + auto sink = + std::make_shared(Common::PathToString(path), true); + return MakeLogger(std::move(name), std::move(sink)); +} + +static bool HasStyle(fmt::text_style style) { + return style != fmt::text_style {}; +} + +void WriteStdout(std::string_view text, fmt::text_style style = {}) { + if (HasStyle(style)) { + fmt::print(stdout, style, "{}", text); + } else { + std::fwrite(text.data(), 1, text.size(), stdout); + } + std::fflush(stdout); +} + +} // namespace + +namespace Log { + +static bool g_initialized = false; +static Direction g_direction = Direction::Console; +static std::filesystem::path g_output_file; +static std::mutex g_logger_mutex; +static std::shared_ptr g_logger; + +void Flush() { + if (g_logger != nullptr) { + g_logger->flush(); + } +} + +static void SetupLogger() { + std::lock_guard lock(g_logger_mutex); + Flush(); + g_logger.reset(); + + switch (g_direction) { + case Direction::Silent: + g_logger = MakeLogger("kyty", std::make_shared()); + break; + case Direction::Console: + g_logger = MakeLogger("kyty", std::make_shared()); + break; + case Direction::File: + if (!g_output_file.empty()) { + g_logger = MakeFileLogger("kyty", g_output_file); + } + break; + } +} + +static std::shared_ptr GetLogger() { + std::lock_guard lock(g_logger_mutex); + return g_logger; +} + +static void WriteImpl(std::string_view text, fmt::text_style style = {}) { + if (text.empty()) { + return; + } + + if (!g_initialized) { + WriteStdout(text, style); + return; + } + + if (g_direction == Direction::Silent) { + return; + } + + if (auto logger = GetLogger()) { + if (g_direction == Direction::Console && HasStyle(style)) { + const auto styled = fmt::format(style, "{}", text); + logger->log(spdlog::level::info, spdlog::string_view_t(styled.data(), styled.size())); + } else { + logger->log(spdlog::level::info, spdlog::string_view_t(text.data(), text.size())); + } + } +} + +void WriteFatal(std::string_view text) { + if (g_direction == Direction::Silent || !g_initialized) { + WriteStdout(text); + } else { + WriteStdout(text); + WriteImpl(text); + } + Flush(); +} + +void WriteFatal(fmt::text_style style, std::string_view text) { + if (g_direction == Direction::Silent || !g_initialized) { + WriteStdout(text, style); + } else { + WriteStdout(text, style); + WriteImpl(text, style); + } + Flush(); +} + +KYTY_SUBSYSTEM_INIT(Log) { + g_initialized = true; + switch (Config::GetPrintfDirection()) { + case Config::OutputDirection::Silent: g_direction = Direction::Silent; break; + case Config::OutputDirection::Console: g_direction = Direction::Console; break; + case Config::OutputDirection::File: g_direction = Direction::File; break; + } + g_output_file = + (g_direction == Direction::File ? Config::GetPrintfOutputFile() : std::filesystem::path {}); + SetupLogger(); +} + +KYTY_SUBSYSTEM_UNEXPECTED_SHUTDOWN(Log) { + Flush(); + std::lock_guard lock(g_logger_mutex); + g_logger.reset(); +} + +KYTY_SUBSYSTEM_DESTROY(Log) { + Flush(); + std::lock_guard lock(g_logger_mutex); + g_logger.reset(); +} + +Direction GetDirection() { + EXIT_IF(!g_initialized); + return g_direction; +} + +void Write(std::string_view text) { + WriteImpl(text); +} + +void Write(fmt::text_style style, std::string_view text) { + WriteImpl(text, style); +} + +} // namespace Log diff --git a/src/common/logging/log.h b/src/common/logging/log.h new file mode 100644 index 000000000..babfb247a --- /dev/null +++ b/src/common/logging/log.h @@ -0,0 +1,48 @@ +#ifndef KYTY_COMMON_LOGGING_LOG_H_ +#define KYTY_COMMON_LOGGING_LOG_H_ + +#include "common/common.h" +#include "common/subsystems.h" + +#include +#include +#include + +namespace Log { + +KYTY_SUBSYSTEM_DEFINE(Log); + +enum class Direction { Silent, Console, File }; + +Direction GetDirection(); +void Write(std::string_view text); +void Write(fmt::text_style style, std::string_view text); +void WriteFatal(std::string_view text); +void WriteFatal(fmt::text_style style, std::string_view text); +void Flush(); + +namespace Color { + +inline constexpr auto Default = fmt::text_style {}; +inline constexpr auto Red = fmt::fg(fmt::terminal_color::red); +inline constexpr auto Green = fmt::fg(fmt::terminal_color::green); +inline constexpr auto Yellow = fmt::fg(fmt::terminal_color::yellow); +inline constexpr auto Magenta = fmt::fg(fmt::terminal_color::magenta); +inline constexpr auto Cyan = fmt::fg(fmt::terminal_color::cyan); +inline constexpr auto White = fmt::fg(fmt::terminal_color::white); +inline constexpr auto BrightRed = fmt::fg(fmt::terminal_color::bright_red); +inline constexpr auto BrightGreen = fmt::fg(fmt::terminal_color::bright_green); +inline constexpr auto BrightYellow = fmt::fg(fmt::terminal_color::bright_yellow); +inline constexpr auto BrightMagenta = fmt::fg(fmt::terminal_color::bright_magenta); +inline constexpr auto BrightWhite = fmt::fg(fmt::terminal_color::bright_white); + +} // namespace Color + +} // namespace Log + +// NOLINTNEXTLINE(cppcoreguidelines-macro-usage) +#define LOGF(...) ::Log::Write(::fmt::sprintf(__VA_ARGS__)) +// NOLINTNEXTLINE(cppcoreguidelines-macro-usage) +#define LOGF_COLOR(style, ...) ::Log::Write((style), ::fmt::sprintf(__VA_ARGS__)) + +#endif /* KYTY_COMMON_LOGGING_LOG_H_ */ diff --git a/src/common/magicEnum.h b/src/common/magicEnum.h new file mode 100644 index 000000000..3869d70b2 --- /dev/null +++ b/src/common/magicEnum.h @@ -0,0 +1,42 @@ +#ifndef KYTY_COMMON_MAGICENUM_H_ +#define KYTY_COMMON_MAGICENUM_H_ + +#include "common/common.h" +#include "common/stringUtils.h" +#include "magic_enum.hpp" // IWYU pragma: export + +namespace Common { + +template +inline std::string EnumName(E v) { + auto str = magic_enum::enum_name(v); + return std::string(str.data(), str.length()); +} + +template +inline std::string EnumName8(E v) { + auto str = magic_enum::enum_name(v); + return std::string(str.data(), str.length()); +} + +template +inline E EnumValue(const std::string& str, E default_value) { + auto v = magic_enum::enum_cast(str.c_str()); + if (v.has_value()) { + return v.value(); + } + return default_value; +} + +#define KYTY_ENUM_RANGE(e, mx, mn) \ + namespace magic_enum::customize { \ + template <> \ + struct enum_range { \ + static constexpr int min = (mx); \ + static constexpr int max = (mn); \ + }; \ + } + +} // namespace Common + +#endif /* KYTY_COMMON_MAGICENUM_H_ */ diff --git a/src/common/platform/sysDbg.h b/src/common/platform/sysDbg.h new file mode 100644 index 000000000..c8817535e --- /dev/null +++ b/src/common/platform/sysDbg.h @@ -0,0 +1,33 @@ +#ifndef KYTY_COMMON_PLATFORM_SYSDBG_H_ +#define KYTY_COMMON_PLATFORM_SYSDBG_H_ + +#include "common/common.h" + +// NOLINTNEXTLINE(readability-identifier-naming) +struct sys_dbg_stack_info_t { +#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS + uintptr_t addr; + + uintptr_t reserved_addr; + size_t reserved_size; + uintptr_t guard_addr; + size_t guard_size; + uintptr_t commited_addr; + size_t commited_size; + + size_t total_size; +#elif KYTY_PLATFORM == KYTY_PLATFORM_LINUX + uintptr_t code_addr; + uintptr_t addr; + uintptr_t commited_addr; + size_t commited_size; + size_t total_size; + size_t code_size; +#endif +}; + +void SysStackWalk(void** stack, int* depth); +void SysStackUsage(sys_dbg_stack_info_t& s); // NOLINT(google-runtime-references) +void SysStackUsagePrint(sys_dbg_stack_info_t& stack); // NOLINT(google-runtime-references) + +#endif /* KYTY_COMMON_PLATFORM_SYSDBG_H_ */ diff --git a/src/common/platform/sysFileIO.h b/src/common/platform/sysFileIO.h new file mode 100644 index 000000000..0ab47bbce --- /dev/null +++ b/src/common/platform/sysFileIO.h @@ -0,0 +1,86 @@ +#ifndef KYTY_COMMON_PLATFORM_SYSFILEIO_H_ +#define KYTY_COMMON_PLATFORM_SYSFILEIO_H_ + +#include "common/common.h" +#include "common/platform/sysTimer.h" + +#include +#include +#include + +// The native file representation is private to the platform implementation. +struct sys_file_t; // NOLINT(readability-identifier-naming) + +// NOLINTNEXTLINE(readability-identifier-naming) +enum sys_file_cache_type_t { + SYS_FILE_CACHE_AUTO = 0, // NOLINT(readability-identifier-naming) + SYS_FILE_CACHE_RANDOM_ACCESS = 1, // NOLINT(readability-identifier-naming) + SYS_FILE_CACHE_SEQUENTIAL_SCAN = 2 // NOLINT(readability-identifier-naming) +}; + +// NOLINTNEXTLINE(readability-identifier-naming) +struct sys_file_find_t { + std::filesystem::path path_with_name; + SysFileTimeStruct last_access_time; + SysFileTimeStruct last_write_time; + uint64_t size; +}; + +// NOLINTNEXTLINE(readability-identifier-naming) +struct sys_dir_entry_t { + std::string name; + bool is_file; +}; + +void SysFileRead(void* data, uint32_t size, sys_file_t& f, + uint32_t* bytes_read = nullptr); // NOLINT(google-runtime-references) +void SysFileWrite(const void* data, uint32_t size, sys_file_t& f, + uint32_t* bytes_written = nullptr); // NOLINT(google-runtime-references) +sys_file_t* SysFileCreate(const std::filesystem::path& file_name); +sys_file_t* SysFileOpenR(const std::filesystem::path& file_name, + sys_file_cache_type_t cache_type = SYS_FILE_CACHE_AUTO); +sys_file_t* SysFileOpenW(const std::filesystem::path& file_name, + sys_file_cache_type_t cache_type = SYS_FILE_CACHE_AUTO); +sys_file_t* SysFileOpen(uint8_t* buf, uint32_t buf_size); +sys_file_t* SysFileCreate(); +sys_file_t* SysFileOpenRw(const std::filesystem::path& file_name, + sys_file_cache_type_t cache_type = SYS_FILE_CACHE_AUTO); +void SysFileClose(sys_file_t* f); +uint64_t SysFileSize(sys_file_t& f); // NOLINT(google-runtime-references) +bool SysFileSeek(sys_file_t& f, uint64_t offset); // NOLINT(google-runtime-references) +uint64_t SysFileTell(sys_file_t& f); // NOLINT(google-runtime-references) +bool SysFileTruncate(sys_file_t& f, uint64_t size); // NOLINT(google-runtime-references) +bool SysFileUnlink(sys_file_t& f, + const std::filesystem::path& name); // NOLINT(google-runtime-references) +void SysFileWrite(uint32_t n, sys_file_t& f); // NOLINT(google-runtime-references) +uint64_t SysFileSize(const std::filesystem::path& file_name); +bool SysFileIsError(sys_file_t& f); // NOLINT(google-runtime-references) +bool SysFileIsDirectoryExisting(const std::filesystem::path& path); +bool SysFileIsFileExisting(const std::filesystem::path& name); +bool SysFileCreateDirectory(const std::filesystem::path& path); +bool SysFileDeleteDirectory(const std::filesystem::path& path); +bool SysFileDeleteFile(const std::filesystem::path& name); +bool SysFileFlush(sys_file_t& f); // NOLINT(google-runtime-references) +SysFileTimeStruct SysFileGetLastAccessTimeUtc(const std::filesystem::path& name); +SysFileTimeStruct SysFileGetLastWriteTimeUtc(const std::filesystem::path& name); +// NOLINTNEXTLINE(google-runtime-references) +void SysFileGetLastAccessAndWriteTimeUtc(const std::filesystem::path& name, SysFileTimeStruct& a, + SysFileTimeStruct& w); +// NOLINTNEXTLINE(google-runtime-references) +void SysFileGetLastAccessAndWriteTimeUtc(sys_file_t& f, SysFileTimeStruct& a, SysFileTimeStruct& w); +// NOLINTNEXTLINE(google-runtime-references) +bool SysFileSetLastAccessTimeUtc(const std::filesystem::path& name, SysFileTimeStruct& access); +// NOLINTNEXTLINE(google-runtime-references) +bool SysFileSetLastWriteTimeUtc(const std::filesystem::path& name, SysFileTimeStruct& write); +// NOLINTNEXTLINE(google-runtime-references) +bool SysFileSetLastAccessAndWriteTimeUtc(const std::filesystem::path& name, + SysFileTimeStruct& access, SysFileTimeStruct& write); +// NOLINTNEXTLINE(google-runtime-references) +void SysFileFindFiles(const std::filesystem::path& path, std::vector& out); +// NOLINTNEXTLINE(google-runtime-references) +void SysFileGetDents(const std::filesystem::path& path, std::vector& out); +bool SysFileCopyFile(const std::filesystem::path& src, const std::filesystem::path& dst); +bool SysFileMoveFile(const std::filesystem::path& src, const std::filesystem::path& dst); +void SysFileRemoveReadonly(const std::filesystem::path& name); + +#endif /* KYTY_COMMON_PLATFORM_SYSFILEIO_H_ */ diff --git a/src/common/platform/sysLinuxDbg.cpp b/src/common/platform/sysLinuxDbg.cpp new file mode 100644 index 000000000..11186980e --- /dev/null +++ b/src/common/platform/sysLinuxDbg.cpp @@ -0,0 +1,123 @@ +#include "common/common.h" + +#if KYTY_PLATFORM != KYTY_PLATFORM_LINUX +// #error "KYTY_PLATFORM != KYTY_PLATFORM_LINUX" +#else + +#include "common/platform/sysDbg.h" + +#include +#include +#include +#include +#include + +void SysStackWalk(void** /*stack*/, int* depth) { + *depth = 0; +} + +void SysStackUsagePrint(sys_dbg_stack_info_t& stack) { + printf("stack: (0x%" PRIx64 ", %" PRIu64 ")\n", static_cast(stack.commited_addr), + static_cast(stack.commited_size)); + printf("code: (0x%" PRIx64 ", %" PRIu64 ")\n", static_cast(stack.code_addr), + static_cast(stack.code_size)); +} + +void SysStackUsage(sys_dbg_stack_info_t& s) { + pid_t pid = getpid(); + + // printf("pid = %"I64"d\n", (int64_t)pid); + + [[maybe_unused]] int result = 0; + + char str[1024]; + char str2[1024]; + result = sprintf(str, "/proc/%d/exe", static_cast(pid)); + + ssize_t buff_len = 0; + if ((buff_len = readlink(str, str2, 1023)) == -1) { + return; + } + str2[buff_len] = '\0'; + const char* name = basename(str2); + + result = sprintf(str, "/proc/%d/maps", static_cast(pid)); + + memset(&s, 0, sizeof(sys_dbg_stack_info_t)); + + FILE* f = fopen(str, "r"); + + if (f == nullptr) { + return; + } + + // printf("&str = %"I64"x\n", (uint64_t)&str); + + uint64_t addr = 0; + uint64_t endaddr = 0; + [[maybe_unused]] uint64_t size = 0; + uint64_t offset = 0; + uint64_t inode = 0; + char permissions[8] = {}; + char device[8] = {}; + char filename[MAXPATHLEN] = {}; + + auto check_addr = reinterpret_cast(&f); + + while (true) { + if (feof(f) != 0) { + break; + } + + if (fgets(str, sizeof(str), f) == nullptr) { + break; + } + + filename[0] = 0; + permissions[0] = 0; + addr = 0; + size = 0; + + // printf("%s", str); + + // NOLINTNEXTLINE(cert-err34-c) + result = sscanf(str, "%" SCNx64 "-%" SCNx64 " %s %" SCNx64 " %s %" SCNx64 " %s", &addr, + &endaddr, permissions, &offset, device, &inode, filename); + + size = endaddr - addr; + + bool read = (strchr(permissions, 'r') != nullptr); + bool write = (strchr(permissions, 'w') != nullptr); + bool exec = (strchr(permissions, 'x') != nullptr); + + // printf("%016"I64"x, %"I64"d, %s, %d, %d\n", addr, size, filename, read, write); + + if (read && write && !exec && strncmp(filename, "[stack", 6) == 0) { + // printf("stack: %016"I64"x, %"I64"d\n", addr, size); + + if (check_addr >= addr && check_addr < addr + size) { + s.addr = addr; + s.total_size = size; + s.commited_addr = addr; + s.commited_size = size; + + if (s.code_addr != 0) { + break; + } + } + } + + if (read && !write && exec && strstr(filename, name) != nullptr) { + s.code_addr = addr; + s.code_size = size; + + if (s.addr != 0) { + break; + } + } + } + + result = fclose(f); +} + +#endif diff --git a/src/common/platform/sysLinuxFileIO.cpp b/src/common/platform/sysLinuxFileIO.cpp new file mode 100644 index 000000000..4df6ba1e6 --- /dev/null +++ b/src/common/platform/sysLinuxFileIO.cpp @@ -0,0 +1,557 @@ +#include "common/common.h" + +#if KYTY_PLATFORM != KYTY_PLATFORM_LINUX +// #error "KYTY_PLATFORM != KYTY_PLATFORM_LINUX" +#else + +#include "common/assert.h" +#include "common/platform/sysFileIO.h" +#include "common/platform/sysTimer.h" +#include "common/stringUtils.h" + +#include +#include +#include +#include +#include + +// NOLINTNEXTLINE(readability-identifier-naming) +enum sys_file_type_t { + SYS_FILE_ERROR, // NOLINT(readability-identifier-naming) + SYS_FILE_MEMORY_STAT, // NOLINT(readability-identifier-naming) + SYS_FILE_FILE, // NOLINT(readability-identifier-naming) + SYS_FILE_MEMORY_DYN // NOLINT(readability-identifier-naming) +}; + +// NOLINTNEXTLINE(readability-identifier-naming) +struct sys_file_mem_buf_t { + uint8_t* base; + uint8_t* ptr; + uint32_t size; +}; + +// NOLINTNEXTLINE(readability-identifier-naming) +struct sys_file_t { + sys_file_type_t type; + union { + FILE* f; + sys_file_mem_buf_t* buf; + }; +}; + +static std::filesystem::path get_internal_name(const std::filesystem::path& name) { + return name.is_absolute() ? name : (std::filesystem::path(".") / name); +} + +void SysFileRead(void* data, uint32_t size, sys_file_t& f, uint32_t* bytes_read) { + if (f.type == SYS_FILE_FILE) { + size_t w = fread(data, 1, size, f.f); + if (bytes_read != nullptr) { + *bytes_read = w; + } + } else if (f.type == SYS_FILE_MEMORY_STAT) { + uint32_t s = size; + if (f.buf->size != 0) { + uint32_t l = f.buf->size - (f.buf->ptr - f.buf->base); + if (s > l) { + s = l; + } + } + memcpy(data, f.buf->ptr, s); + f.buf->ptr += s; + if (bytes_read != nullptr) { + *bytes_read = s; + } + } else if (f.type == SYS_FILE_MEMORY_DYN) { + uint32_t s = size; + if (f.buf->size != 0) { + uint32_t l = f.buf->size - (f.buf->ptr - f.buf->base); + if (s > l) { + s = l; + } + } else { + s = 0; + } + memcpy(data, f.buf->ptr, s); + f.buf->ptr += s; + if (bytes_read != nullptr) { + *bytes_read = s; + } + } +} + +void SysFileWrite(const void* data, uint32_t size, sys_file_t& f, uint32_t* bytes_written) { + if (f.type == SYS_FILE_FILE) { + size_t w = fwrite(data, 1, size, f.f); + if (bytes_written != nullptr) { + *bytes_written = w; + } + } else if (f.type == SYS_FILE_MEMORY_STAT) { + uint32_t s = size; + if (f.buf->size != 0) { + uint32_t l = f.buf->size - (f.buf->ptr - f.buf->base); + if (s > l) { + s = l; + } + } + memcpy(f.buf->ptr, data, s); + f.buf->ptr += s; + if (bytes_written != nullptr) { + *bytes_written = s; + } + } else if (f.type == SYS_FILE_MEMORY_DYN) { + uint32_t pos = f.buf->ptr - f.buf->base; + if (f.buf->size < pos + size) { + f.buf->base = static_cast(std::realloc(f.buf->base, pos + size)); + f.buf->ptr = f.buf->base + pos; + f.buf->size = pos + size; + } + memcpy(f.buf->ptr, data, size); + f.buf->ptr += size; + if (bytes_written != nullptr) { + *bytes_written = size; + } + } +} + +void SysFileWrite(uint32_t n, sys_file_t& f) { + SysFileWrite(&n, 4, f); +} + +sys_file_t* SysFileCreate(const std::filesystem::path& file_name) { + auto* ret = new sys_file_t; + + auto real_name = get_internal_name(file_name); + auto real_name_str = real_name.string(); + + ret->f = fopen(real_name_str.c_str(), "w+"); + + if (ret->f == nullptr) { + printf("can't create file: %s\n", real_name_str.c_str()); + } + + ret->type = SYS_FILE_FILE; + + return ret; +} + +sys_file_t* SysFileOpenR(const std::filesystem::path& file_name, + sys_file_cache_type_t /*cache_type*/) { + auto* ret = new sys_file_t; + + ret->type = SYS_FILE_FILE; + + auto internal_name = get_internal_name(file_name); + auto internal_name_str = internal_name.string(); + + FILE* f = fopen(internal_name_str.c_str(), "r"); + + if (f == nullptr) { + ret->type = SYS_FILE_ERROR; + } + + ret->f = f; + + return ret; +} + +sys_file_t* SysFileOpen(uint8_t* buf, uint32_t buf_size) { + auto* ret = new sys_file_t; + + ret->type = SYS_FILE_MEMORY_STAT; + ret->buf = new sys_file_mem_buf_t; + ret->buf->base = buf; + ret->buf->ptr = buf; + ret->buf->size = buf_size; + + return ret; +} + +sys_file_t* SysFileCreate() { + auto* ret = new sys_file_t; + + ret->type = SYS_FILE_MEMORY_DYN; + ret->buf = new sys_file_mem_buf_t; + ret->buf->base = nullptr; + ret->buf->ptr = nullptr; + ret->buf->size = 0; + + return ret; +} + +sys_file_t* SysFileOpenW(const std::filesystem::path& file_name, + sys_file_cache_type_t /*cache_type*/) { + auto* ret = new sys_file_t; + + auto real_name = get_internal_name(file_name); + auto real_name_str = real_name.string(); + + FILE* f = fopen(real_name_str.c_str(), "r+"); + + if (f == nullptr) { + ret->type = SYS_FILE_ERROR; + } else { + ret->type = SYS_FILE_FILE; + } + + ret->f = f; + + return ret; +} + +sys_file_t* SysFileOpenRw(const std::filesystem::path& file_name, + sys_file_cache_type_t /*cache_type*/) { + auto* ret = new sys_file_t; + + auto real_name = get_internal_name(file_name); + auto real_name_str = real_name.string(); + + FILE* f = fopen(real_name_str.c_str(), "r+"); + + if (f == nullptr) { + ret->type = SYS_FILE_ERROR; + } else { + ret->type = SYS_FILE_FILE; + } + + ret->f = f; + + return ret; +} + +void SysFileClose(sys_file_t* f) { + [[maybe_unused]] int result = 0; + + if (f->type == SYS_FILE_FILE && f->f != nullptr) { + result = fclose(f->f); + } else if (f->type == SYS_FILE_MEMORY_STAT) { + delete f->buf; + } else if (f->type == SYS_FILE_MEMORY_DYN) { + std::free(f->buf->base); + delete f->buf; + } + + // f.type = SYS_FILE_ERROR; + delete f; +} + +uint64_t SysFileSize(sys_file_t& f) { + [[maybe_unused]] int result = 0; + + if (f.type == SYS_FILE_FILE) { + uint32_t pos = ftell(f.f); + result = fseek(f.f, 0, SEEK_END); + uint32_t size = ftell(f.f); + result = fseek(f.f, pos, SEEK_SET); + return size; + } + + if (f.type == SYS_FILE_MEMORY_STAT || f.type == SYS_FILE_MEMORY_DYN) { + return f.buf->size; + } + + return 0; +} + +uint64_t SysFileSize(const std::filesystem::path& file_name) { + sys_file_t* f = SysFileOpenR(file_name); + uint64_t size = SysFileSize(*f); + SysFileClose(f); + return size; +} + +bool SysFileTruncate(sys_file_t& /*f*/, uint64_t /*size*/) { + return false; +} + +bool SysFileUnlink(sys_file_t& /*f*/, const std::filesystem::path& name) { + return SysFileDeleteFile(name); +} + +bool SysFileSeek(sys_file_t& f, uint64_t offset) { + bool ok = true; + if (f.type == SYS_FILE_FILE) { + ok = (fseek(f.f, static_cast(offset), SEEK_SET) == 0); + // LARGE_INTEGER s; + // s.QuadPart = offset; + // SetFilePointerEx(f.handle, s, 0, FILE_BEGIN); + // printf("seek: %u\n", offset); + } else if (f.type == SYS_FILE_MEMORY_STAT || f.type == SYS_FILE_MEMORY_DYN) { + f.buf->ptr = f.buf->base + offset; + } + return ok; +} + +uint64_t SysFileTell(sys_file_t& f) { + if (f.type == SYS_FILE_FILE) { + return ftell(f.f); + } + + if (f.type == SYS_FILE_MEMORY_STAT || f.type == SYS_FILE_MEMORY_DYN) { + return f.buf->ptr - f.buf->base; + } + + return 0; +} + +bool SysFileIsError(sys_file_t& f) { + return f.type == SYS_FILE_ERROR || (f.type == SYS_FILE_FILE && f.f == nullptr); +} + +bool SysFileIsDirectoryExisting(const std::filesystem::path& path) { + auto real_name = get_internal_name(path); + auto real_name_str = real_name.string(); + + struct stat s {}; + + if (0 != stat(real_name_str.c_str(), &s)) { + return false; + } + + return S_ISDIR(s.st_mode); // NOLINT +} + +bool SysFileIsFileExisting(const std::filesystem::path& name) { + auto real_name = get_internal_name(name); + auto real_name_str = real_name.string(); + + struct stat s {}; + + if (0 != stat(real_name_str.c_str(), &s)) { + return false; + } + + return !S_ISDIR(s.st_mode); // NOLINT +} + +bool SysFileCreateDirectory(const std::filesystem::path& path) { + auto real_name = get_internal_name(path); + auto real_name_str = real_name.string(); + + mode_t m = S_IRWXU | S_IRWXG | S_IRWXO; // NOLINT + + int r = mkdir(real_name_str.c_str(), m); + + if (r != 0) { + int e = errno; + if (e == EEXIST) { + printf("mkdir(%s, %" PRIx32 ") failed: The named file exists\n", real_name_str.c_str(), + static_cast(m)); + } else { + printf("mkdir(%s, %" PRIx32 ") failed: %d\n", real_name_str.c_str(), + static_cast(m), e); + return false; + } + } + + return true; +} + +bool SysFileDeleteDirectory(const std::filesystem::path& path) { + auto real_name = get_internal_name(path); + auto real_name_str = real_name.string(); + + return 0 == remove(real_name_str.c_str()); +} + +bool SysFileDeleteFile(const std::filesystem::path& name) { + auto real_name = get_internal_name(name); + auto real_name_str = real_name.string(); + + return 0 == unlink(real_name_str.c_str()); +} + +bool SysFileFlush(sys_file_t& f) { + if (f.type == SYS_FILE_FILE && f.f != nullptr) { + return (fflush(f.f) == 0); + } + + return false; +} + +SysFileTimeStruct SysFileGetLastAccessTimeUtc(const std::filesystem::path& name) { + SysFileTimeStruct r {}; + + auto real_name = get_internal_name(name); + auto real_name_str = real_name.string(); + + struct stat s {}; + + if (0 != stat(real_name_str.c_str(), &s)) { + r.is_invalid = true; + } else { + r.is_invalid = false; + r.time = s.st_atime; + } + + return r; +} + +SysFileTimeStruct SysFileGetLastWriteTimeUtc(const std::filesystem::path& name) { + SysFileTimeStruct r {}; + + auto real_name = get_internal_name(name); + auto real_name_str = real_name.string(); + + struct stat s {}; + + if (0 != stat(real_name_str.c_str(), &s)) { + r.is_invalid = true; + } else { + r.is_invalid = false; + r.time = s.st_mtime; + } + + return r; +} + +void SysFileGetLastAccessAndWriteTimeUtc(const std::filesystem::path& name, SysFileTimeStruct& a, + SysFileTimeStruct& w) { + auto real_name = get_internal_name(name); + auto real_name_str = real_name.string(); + + struct stat s {}; + + if (0 != stat(real_name_str.c_str(), &s)) { + a.is_invalid = true; + w.is_invalid = true; + } else { + a.is_invalid = false; + w.is_invalid = false; + a.time = s.st_atime; + w.time = s.st_mtime; + } +} + +void SysFileGetLastAccessAndWriteTimeUtc(sys_file_t& /*f*/, SysFileTimeStruct& /*a*/, + SysFileTimeStruct& /*w*/) { + EXIT("not implemented\n"); +} + +bool SysFileSetLastAccessTimeUtc(const std::filesystem::path& name, SysFileTimeStruct& access) { + if (access.is_invalid) { + return false; + } + + auto real_name = get_internal_name(name); + auto real_name_str = real_name.string(); + + struct stat s {}; + + if (0 != stat(real_name_str.c_str(), &s)) { + return false; + } + + utimbuf times {}; + times.actime = access.time; + times.modtime = s.st_mtime; + + return !(0 != utime(real_name_str.c_str(), ×)); + + // if (0 != stat(n.data(), &s)) + // { + // return false; + // } + // + // times.actime += times.actime - s.st_atime; + // times.modtime += times.modtime - s.st_mtime; + // + // if (0 != utime(n.data(), ×)) + // { + // return false; + // } +} + +bool SysFileSetLastWriteTimeUtc(const std::filesystem::path& name, SysFileTimeStruct& write) { + if (write.is_invalid) { + return false; + } + + auto real_name = get_internal_name(name); + auto real_name_str = real_name.string(); + + struct stat s {}; + + if (0 != stat(real_name_str.c_str(), &s)) { + return false; + } + + utimbuf times {}; + times.actime = s.st_atime; + times.modtime = write.time; + + return !(0 != utime(real_name_str.c_str(), ×)); + + // if (0 != stat(n.data(), &s)) + // { + // return false; + // } + // + // times.actime += times.actime - s.st_atime; + // times.modtime += times.modtime - s.st_mtime; + // + // if (0 != utime(n.data(), ×)) + // { + // return false; + // } +} + +bool SysFileSetLastAccessAndWriteTimeUtc(const std::filesystem::path& name, + SysFileTimeStruct& access, SysFileTimeStruct& write) { + if (access.is_invalid || write.is_invalid) { + return false; + } + + auto real_name = get_internal_name(name); + auto real_name_str = real_name.string(); + + struct stat s {}; + + if (0 != stat(real_name_str.c_str(), &s)) { + return false; + } + + utimbuf times {}; + times.actime = access.time; + times.modtime = write.time; + + return !(0 != utime(real_name_str.c_str(), ×)); + + // if (0 != stat(n.data(), &s)) + // { + // return false; + // } + // + // times.actime += times.actime - s.st_atime; + // times.modtime += times.modtime - s.st_mtime; + // + // if (0 != utime(n.data(), ×)) + // { + // return false; + // } +} + +void SysFileFindFiles(const std::filesystem::path& /*path*/, + std::vector& /*out*/) { + EXIT("not implemented\n"); +} + +void SysFileGetDents(const std::filesystem::path& /*path*/, std::vector& /*out*/) { + EXIT("not implemented\n"); +} + +bool SysFileCopyFile(const std::filesystem::path& /*src*/, const std::filesystem::path& /*dst*/) { + EXIT("not implemented\n"); + return false; +} + +bool SysFileMoveFile(const std::filesystem::path& /*src*/, const std::filesystem::path& /*dst*/) { + EXIT("not implemented\n"); + return false; +} + +void SysFileRemoveReadonly(const std::filesystem::path& /*name*/) { + EXIT("not implemented\n"); +} + +#endif diff --git a/src/common/platform/sysLinuxVirtual.cpp b/src/common/platform/sysLinuxVirtual.cpp new file mode 100644 index 000000000..eadcacd11 --- /dev/null +++ b/src/common/platform/sysLinuxVirtual.cpp @@ -0,0 +1,451 @@ +#include "common/common.h" + +#if KYTY_PLATFORM != KYTY_PLATFORM_LINUX +// #error "KYTY_PLATFORM != KYTY_PLATFORM_LINUX" +#else + +#include "common/assert.h" +#include "common/platform/sysVirtual.h" +#include "common/virtualMemory.h" + +#include +#include +#include + +// IWYU pragma: no_include +// IWYU pragma: no_include +// IWYU pragma: no_include +// IWYU pragma: no_include + +#if defined(MAP_FIXED_NOREPLACE) && KYTY_PLATFORM == KYTY_PLATFORM_LINUX +#define KYTY_FIXED_NOREPLACE +#endif + +namespace Common { + +static pthread_mutex_t g_virtual_mutex {}; +static std::map* g_allocs = nullptr; +static std::map* g_protects = nullptr; + +void SysVirtualInit() { + pthread_mutexattr_t attr {}; + + pthread_mutexattr_init(&attr); +#if KYTY_PLATFORM == KYTY_PLATFORM_LINUX + pthread_mutexattr_settype(&attr, PTHREAD_MUTEX_FAST_NP); +#else + pthread_mutexattr_settype(&attr, PTHREAD_MUTEX_NORMAL); +#endif + pthread_mutex_init(&g_virtual_mutex, &attr); + pthread_mutexattr_destroy(&attr); + + g_allocs = new std::map; + g_protects = new std::map; +} + +static int get_protection_flag(VirtualMemory::Mode mode) { + int protect = PROT_NONE; + switch (mode) { + case VirtualMemory::Mode::Read: protect = PROT_READ; break; + case VirtualMemory::Mode::Write: + case VirtualMemory::Mode::ReadWrite: protect = PROT_READ | PROT_WRITE; break; // NOLINT + case VirtualMemory::Mode::Execute: protect = PROT_EXEC; break; + case VirtualMemory::Mode::ExecuteRead: protect = PROT_EXEC | PROT_READ; break; // NOLINT + case VirtualMemory::Mode::ExecuteWrite: + case VirtualMemory::Mode::ExecuteReadWrite: + protect = PROT_EXEC | PROT_WRITE | PROT_READ; + break; // NOLINT + case VirtualMemory::Mode::NoAccess: + default: protect = PROT_NONE; break; + } + return protect; +} + +static VirtualMemory::Mode get_protection_flag(int mode) { + switch (mode) { + case PROT_NONE: return VirtualMemory::Mode::NoAccess; + case PROT_READ: return VirtualMemory::Mode::Read; + case PROT_WRITE: return VirtualMemory::Mode::Write; + case PROT_READ | PROT_WRITE: return VirtualMemory::Mode::ReadWrite; // NOLINT + case PROT_EXEC: return VirtualMemory::Mode::Execute; + case PROT_EXEC | PROT_WRITE: return VirtualMemory::Mode::ExecuteWrite; // NOLINT + case PROT_EXEC | PROT_READ: return VirtualMemory::Mode::ExecuteRead; // NOLINT + case PROT_EXEC | PROT_WRITE | PROT_READ: + return VirtualMemory::Mode::ExecuteReadWrite; // NOLINT + default: return VirtualMemory::Mode::NoAccess; + } +} + +uint64_t SysVirtualAlloc(uint64_t address, uint64_t size, VirtualMemory::Mode mode) { + EXIT_IF(g_allocs == nullptr); + + auto addr = static_cast(address); + + int protect = get_protection_flag(mode); + + void* ptr = + mmap(reinterpret_cast(addr), size, protect, MAP_PRIVATE | MAP_ANON, -1, 0); // NOLINT + + auto ret_addr = reinterpret_cast(ptr); + + if (ptr != MAP_FAILED) { + pthread_mutex_lock(&g_virtual_mutex); + (*g_allocs)[ret_addr] = size; + uintptr_t page_start = ret_addr >> 12u; + uintptr_t page_end = (ret_addr + size - 1) >> 12u; + for (uintptr_t page = page_start; page <= page_end; page++) { + (*g_protects)[page] = protect; + } + pthread_mutex_unlock(&g_virtual_mutex); + } + + return ret_addr; +} + +static uintptr_t align_up(uintptr_t addr, uint64_t alignment) { + return (addr + alignment - 1) & ~(alignment - 1); +} + +uint64_t SysVirtualAllocAligned(uint64_t address, uint64_t size, VirtualMemory::Mode mode, + uint64_t alignment) { + if (alignment == 0) { + return 0; + } + + EXIT_IF(g_allocs == nullptr); + + auto addr = static_cast(address); + int protect = get_protection_flag(mode); + + void* ptr = + mmap(reinterpret_cast(addr), size, protect, MAP_PRIVATE | MAP_ANON, -1, 0); // NOLINT + + auto ret_addr = reinterpret_cast(ptr); + + if (ptr != MAP_FAILED && ((ret_addr & (alignment - 1)) != 0)) { + munmap(ptr, size); + + ptr = mmap(reinterpret_cast(addr), size + alignment, protect, + MAP_PRIVATE | MAP_ANON | MAP_NORESERVE, -1, 0); // NOLINT + ret_addr = reinterpret_cast(ptr); + if (ptr != MAP_FAILED) { + munmap(ptr, size + alignment); + auto aligned_addr = align_up(ret_addr, alignment); +#ifdef KYTY_FIXED_NOREPLACE + // NOLINTNEXTLINE + ptr = mmap(reinterpret_cast(aligned_addr), size, protect, + MAP_FIXED_NOREPLACE | MAP_PRIVATE | MAP_ANON, -1, 0); +#else + // NOLINTNEXTLINE + ptr = mmap(reinterpret_cast(aligned_addr), size, protect, + MAP_FIXED | MAP_PRIVATE | MAP_ANON, -1, 0); +#endif + ret_addr = reinterpret_cast(ptr); + if (ptr != MAP_FAILED && ((ret_addr & (alignment - 1)) != 0)) { + munmap(ptr, size); + ret_addr = 0; + ptr = MAP_FAILED; + } + } + } + + if (ptr == MAP_FAILED) { + return SysVirtualAllocAligned(address, size, mode, alignment << 1u); + } + + pthread_mutex_lock(&g_virtual_mutex); + (*g_allocs)[ret_addr] = size; + uintptr_t page_start = ret_addr >> 12u; + uintptr_t page_end = (ret_addr + size - 1) >> 12u; + for (uintptr_t page = page_start; page <= page_end; page++) { + (*g_protects)[page] = protect; + } + pthread_mutex_unlock(&g_virtual_mutex); + + return ret_addr; +} + +static bool is_mapped(void* ptr, size_t length) { + FILE* file = fopen("/proc/self/maps", "r"); + char line[1024]; + bool ret = false; + auto addr = reinterpret_cast(ptr); + + while (feof(file) == 0) { + if (fgets(line, 1024, file) == nullptr) { + break; + } + uint64_t start = 0; + uint64_t end = 0; + // NOLINTNEXTLINE(cert-err34-c) + if (sscanf(line, "%" SCNx64 "-%" SCNx64, &start, &end) != 2) { + continue; + } + if (addr >= start && addr + length <= end) { + ret = true; + break; + } + } + fclose(file); + return ret; +} + +bool SysVirtualAllocFixed(uint64_t address, uint64_t size, VirtualMemory::Mode mode) { + EXIT_IF(g_allocs == nullptr); + + auto addr = static_cast(address); + int protect = get_protection_flag(mode); + +#ifdef KYTY_FIXED_NOREPLACE + // NOLINTNEXTLINE + void* ptr = mmap(reinterpret_cast(addr), size, protect, + MAP_FIXED_NOREPLACE | MAP_PRIVATE | MAP_ANON, -1, 0); +#else + // NOLINTNEXTLINE + void* ptr = (is_mapped(reinterpret_cast(addr), size) + ? MAP_FAILED + : mmap(reinterpret_cast(addr), size, protect, + MAP_FIXED | MAP_PRIVATE | MAP_ANON, -1, 0)); +#endif + + auto ret_addr = reinterpret_cast(ptr); + + if (ptr != MAP_FAILED && ret_addr != addr) { + munmap(ptr, size); + ret_addr = 0; + ptr = MAP_FAILED; + } + + if (ptr != MAP_FAILED) { + pthread_mutex_lock(&g_virtual_mutex); + (*g_allocs)[ret_addr] = size; + uintptr_t page_start = ret_addr >> 12u; + uintptr_t page_end = (ret_addr + size - 1) >> 12u; + for (uintptr_t page = page_start; page <= page_end; page++) { + (*g_protects)[page] = protect; + } + pthread_mutex_unlock(&g_virtual_mutex); + + return true; + } + + return false; +} + +bool SysVirtualCommit(uint64_t address, uint64_t size, VirtualMemory::Mode mode) { + return SysVirtualProtect(address, size, mode); +} + +uint64_t SysVirtualReserve(uint64_t address, uint64_t size) { + return SysVirtualReserveAligned(address, size, 1); +} + +uint64_t SysVirtualReserveAligned(uint64_t address, uint64_t size, uint64_t alignment) { + if (alignment == 0) { + return 0; + } + + EXIT_IF(g_allocs == nullptr); + + auto addr = static_cast(address); + + void* ptr = mmap(reinterpret_cast(addr), size, PROT_NONE, + MAP_PRIVATE | MAP_ANON | MAP_NORESERVE, -1, 0); // NOLINT + + auto ret_addr = reinterpret_cast(ptr); + + if (ptr != MAP_FAILED && ((ret_addr & (alignment - 1)) != 0)) { + munmap(ptr, size); + + ptr = mmap(reinterpret_cast(addr), size + alignment, PROT_NONE, + MAP_PRIVATE | MAP_ANON | MAP_NORESERVE, -1, 0); // NOLINT + ret_addr = reinterpret_cast(ptr); + if (ptr != MAP_FAILED) { + munmap(ptr, size + alignment); + auto aligned_addr = align_up(ret_addr, alignment); +#ifdef KYTY_FIXED_NOREPLACE + // NOLINTNEXTLINE + ptr = mmap(reinterpret_cast(aligned_addr), size, PROT_NONE, + MAP_FIXED_NOREPLACE | MAP_PRIVATE | MAP_ANON | MAP_NORESERVE, -1, 0); +#else + // NOLINTNEXTLINE + ptr = mmap(reinterpret_cast(aligned_addr), size, PROT_NONE, + MAP_FIXED | MAP_PRIVATE | MAP_ANON | MAP_NORESERVE, -1, 0); +#endif + ret_addr = reinterpret_cast(ptr); + if (ptr != MAP_FAILED && ((ret_addr & (alignment - 1)) != 0)) { + munmap(ptr, size); + ret_addr = 0; + ptr = MAP_FAILED; + } + } + } + + if (ptr == MAP_FAILED) { + return SysVirtualReserveAligned(address, size, alignment << 1u); + } + + pthread_mutex_lock(&g_virtual_mutex); + (*g_allocs)[ret_addr] = size; + uintptr_t page_start = ret_addr >> 12u; + uintptr_t page_end = (ret_addr + size - 1) >> 12u; + for (uintptr_t page = page_start; page <= page_end; page++) { + (*g_protects)[page] = PROT_NONE; + } + pthread_mutex_unlock(&g_virtual_mutex); + + return ret_addr; +} + +bool SysVirtualReserveFixed(uint64_t address, uint64_t size) { + EXIT_IF(g_allocs == nullptr); + + auto addr = static_cast(address); + +#ifdef KYTY_FIXED_NOREPLACE + // NOLINTNEXTLINE + void* ptr = mmap(reinterpret_cast(addr), size, PROT_NONE, + MAP_FIXED_NOREPLACE | MAP_PRIVATE | MAP_ANON | MAP_NORESERVE, -1, 0); +#else + // NOLINTNEXTLINE + void* ptr = (is_mapped(reinterpret_cast(addr), size) + ? MAP_FAILED + : mmap(reinterpret_cast(addr), size, PROT_NONE, + MAP_FIXED | MAP_PRIVATE | MAP_ANON | MAP_NORESERVE, -1, 0)); +#endif + + auto ret_addr = reinterpret_cast(ptr); + + if (ptr != MAP_FAILED && ret_addr != addr) { + munmap(ptr, size); + ret_addr = 0; + ptr = MAP_FAILED; + } + + if (ptr != MAP_FAILED) { + pthread_mutex_lock(&g_virtual_mutex); + (*g_allocs)[ret_addr] = size; + uintptr_t page_start = ret_addr >> 12u; + uintptr_t page_end = (ret_addr + size - 1) >> 12u; + for (uintptr_t page = page_start; page <= page_end; page++) { + (*g_protects)[page] = PROT_NONE; + } + pthread_mutex_unlock(&g_virtual_mutex); + + return true; + } + + return false; +} + +bool SysVirtualDecommit(uint64_t address, uint64_t size) { + return SysVirtualProtect(address, size, VirtualMemory::Mode::NoAccess); +} + +bool SysVirtualFree(uint64_t address) { + EXIT_IF(g_allocs == nullptr); + size_t size = 0; + + auto addr = static_cast(address & ~static_cast(0xfffu)); + + pthread_mutex_lock(&g_virtual_mutex); + if (auto s = g_allocs->find(addr); s != g_allocs->end()) { + size = s->second; + g_allocs->erase(s); + } + pthread_mutex_unlock(&g_virtual_mutex); + + if (size == 0) { + return false; + } + + if (munmap(reinterpret_cast(addr), size) == 0) { + uintptr_t page_start = addr >> 12u; + uintptr_t page_end = (addr + size - 1) >> 12u; + pthread_mutex_lock(&g_virtual_mutex); + for (uintptr_t page = page_start; page <= page_end; page++) { + g_protects->erase(page); + } + pthread_mutex_unlock(&g_virtual_mutex); + return true; + } + + return false; +} + +bool SysVirtualFreeRange(uint64_t address, uint64_t size) { + EXIT_IF(g_allocs == nullptr); + if (size == 0 || (address & 0xfffu) != 0 || (size & 0xfffu) != 0) { + return false; + } + + const auto addr = static_cast(address); + const auto end = addr + size; + if (end < addr) { + return false; + } + + pthread_mutex_lock(&g_virtual_mutex); + auto next = g_allocs->upper_bound(addr); + if (next == g_allocs->begin()) { + pthread_mutex_unlock(&g_virtual_mutex); + return false; + } + auto it = std::prev(next); + const auto alloc_addr = it->first; + const auto alloc_end = alloc_addr + it->second; + if (addr < alloc_addr || end > alloc_end || munmap(reinterpret_cast(addr), size) != 0) { + pthread_mutex_unlock(&g_virtual_mutex); + return false; + } + + g_allocs->erase(it); + if (alloc_addr < addr) { + (*g_allocs)[alloc_addr] = addr - alloc_addr; + } + if (end < alloc_end) { + (*g_allocs)[end] = alloc_end - end; + } + for (uintptr_t page = addr >> 12u; page <= (end - 1u) >> 12u; page++) { + g_protects->erase(page); + } + pthread_mutex_unlock(&g_virtual_mutex); + return true; +} + +bool SysVirtualProtect(uint64_t address, uint64_t size, VirtualMemory::Mode mode, + VirtualMemory::Mode* old_mode) { + auto addr = static_cast(address); + + pthread_mutex_lock(&g_virtual_mutex); + if (old_mode != nullptr) { + if (auto s = g_protects->find(addr >> 12u); s != g_protects->end()) { + *old_mode = get_protection_flag(s->second); + } else { + *old_mode = VirtualMemory::Mode::NoAccess; + } + } + pthread_mutex_unlock(&g_virtual_mutex); + + uintptr_t page_start = addr >> 12u; + uintptr_t page_end = (addr + size - 1) >> 12u; + if (mprotect(reinterpret_cast(page_start << 12u), (page_end - page_start + 1) << 12u, + get_protection_flag(mode)) == 0) { + pthread_mutex_lock(&g_virtual_mutex); + for (uintptr_t page = page_start; page <= page_end; page++) { + (*g_protects)[page] = get_protection_flag(mode); + } + pthread_mutex_unlock(&g_virtual_mutex); + return true; + } + + return false; +} + +bool SysVirtualFlushInstructionCache(uint64_t /*address*/, uint64_t /*size*/) { + return true; +} + +} // namespace Common + +#endif diff --git a/src/common/platform/sysSwapByteOrder.h b/src/common/platform/sysSwapByteOrder.h new file mode 100644 index 000000000..bdb1244d1 --- /dev/null +++ b/src/common/platform/sysSwapByteOrder.h @@ -0,0 +1,53 @@ +#ifndef KYTY_COMMON_PLATFORM_SYSSWAPBYTEORDER_H_ +#define KYTY_COMMON_PLATFORM_SYSSWAPBYTEORDER_H_ + +#include "common/common.h" + +#include + +inline uint16_t SwapByteOrder16(uint16_t value) { + uint16_t hi = value << 8u; + uint16_t lo = value >> 8u; + return hi | lo; +} + +inline uint32_t SwapByteOrder32(uint32_t value) { + return __builtin_bswap32(value); +} + +inline uint64_t SwapByteOrder64(uint64_t value) { + return __builtin_bswap64(value); +} + +template +inline void SwapByteOrder(T& x) { + if (sizeof(x) == 2) { + if (std::is_signed_v) { + x = static_cast>( + SwapByteOrder16(static_cast>(x))); + } else { + x = SwapByteOrder16(x); + } + } + if (sizeof(x) == 4) { + if (std::is_signed_v) { + x = static_cast>( + SwapByteOrder32(static_cast>(x))); + } else { + x = SwapByteOrder32(x); + } + } + if (sizeof(x) == 8) { + if (std::is_signed_v) { + x = static_cast>( + SwapByteOrder64(static_cast>(x))); + } else { + x = SwapByteOrder64(x); + } + } +} + +template +inline void NoSwapByteOrder(T& x) {} + +#endif /* KYTY_COMMON_PLATFORM_SYSSWAPBYTEORDER_H_ */ diff --git a/src/common/platform/sysTimer.h b/src/common/platform/sysTimer.h new file mode 100644 index 000000000..7c599ba05 --- /dev/null +++ b/src/common/platform/sysTimer.h @@ -0,0 +1,221 @@ +#ifndef KYTY_COMMON_PLATFORM_SYSTIMER_H_ +#define KYTY_COMMON_PLATFORM_SYSTIMER_H_ + +#include "common/common.h" + +#include + +#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS +#ifndef NOMINMAX +#define NOMINMAX +#endif +#include +#endif + +struct SysTimeStruct { + uint16_t Year; // NOLINT(readability-identifier-naming) + uint16_t Month; // NOLINT(readability-identifier-naming) + uint16_t Day; // NOLINT(readability-identifier-naming) + uint16_t Hour; // NOLINT(readability-identifier-naming) + uint16_t Minute; // NOLINT(readability-identifier-naming) + uint16_t Second; // NOLINT(readability-identifier-naming) + uint16_t Milliseconds; // NOLINT(readability-identifier-naming) + bool is_invalid; // NOLINT(readability-identifier-naming) +}; + +struct SysFileTimeStruct { +#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS + FILETIME time; +#elif KYTY_PLATFORM == KYTY_PLATFORM_LINUX + time_t time; +#endif + bool is_invalid; +}; + +#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS + +// NOLINTNEXTLINE(google-runtime-references) +inline void SysFileToSystemTimeUtc(const SysFileTimeStruct& f, SysTimeStruct& t) { + SYSTEMTIME s; + + if (f.is_invalid || (FileTimeToSystemTime(&f.time, &s) == 0)) { + t.is_invalid = true; + return; + } + + t.is_invalid = false; + t.Year = s.wYear; + t.Month = s.wMonth; + t.Day = s.wDay; + t.Hour = s.wHour; + t.Minute = s.wMinute; + t.Second = (s.wSecond == 60 ? 59 : s.wSecond); + t.Milliseconds = s.wMilliseconds; +} + +// NOLINTNEXTLINE(google-runtime-references) +inline void SysTimeTToSystem(time_t t, SysTimeStruct& s) { + SysFileTimeStruct ft {}; + LONGLONG ll = Int32x32To64(t, 10000000) + 116444736000000000; + ft.time.dwLowDateTime = static_cast(ll); + ft.time.dwHighDateTime = static_cast(static_cast(ll) >> 32u); + ft.is_invalid = false; + SysFileToSystemTimeUtc(ft, s); +} + +// NOLINTNEXTLINE(google-runtime-references) +inline void SysSystemToFileTimeUtc(const SysTimeStruct& f, SysFileTimeStruct& t) { + SYSTEMTIME s; + + s.wYear = f.Year; + s.wMonth = f.Month; + s.wDay = f.Day; + s.wHour = f.Hour; + s.wMinute = f.Minute; + s.wSecond = f.Second; + s.wMilliseconds = f.Milliseconds; + + t.is_invalid = (f.is_invalid || (SystemTimeToFileTime(&s, &t.time) == 0)); +} + +// Retrieves the current local date and time. +// NOLINTNEXTLINE(google-runtime-references) +inline void SysGetSystemTime(SysTimeStruct& t) { + SYSTEMTIME s; + GetLocalTime(&s); + + t.is_invalid = false; + t.Year = s.wYear; + t.Month = s.wMonth; + t.Day = s.wDay; + t.Hour = s.wHour; + t.Minute = s.wMinute; + t.Second = (s.wSecond == 60 ? 59 : s.wSecond); + t.Milliseconds = s.wMilliseconds; +} + +// Retrieves the current system date and time in Coordinated Universal Time (UTC). +// NOLINTNEXTLINE(google-runtime-references) +inline void SysGetSystemTimeUtc(SysTimeStruct& t) { + SYSTEMTIME s; + GetSystemTime(&s); + + t.is_invalid = false; + t.Year = s.wYear; + t.Month = s.wMonth; + t.Day = s.wDay; + t.Hour = s.wHour; + t.Minute = s.wMinute; + t.Second = (s.wSecond == 60 ? 59 : s.wSecond); + t.Milliseconds = s.wMilliseconds; +} + +inline void SysQueryPerformanceFrequency(uint64_t* freq) { + LARGE_INTEGER f; + QueryPerformanceFrequency(&f); + *freq = f.QuadPart; +} + +inline void SysQueryPerformanceCounter(uint64_t* counter) { + LARGE_INTEGER c; + QueryPerformanceCounter(&c); + *counter = c.QuadPart; +} + +#elif KYTY_PLATFORM == KYTY_PLATFORM_LINUX + +inline void SysFileToSystemTimeUtc(const SysFileTimeStruct& f, SysTimeStruct& t) { + struct tm i {}; + + if (f.is_invalid || gmtime_r(&f.time, &i) == nullptr) { + t.is_invalid = true; + return; + } + + t.is_invalid = false; + t.Year = i.tm_year + 1900; + t.Month = i.tm_mon + 1; + t.Day = i.tm_mday; + t.Hour = i.tm_hour; + t.Minute = i.tm_min; + t.Second = (i.tm_sec == 60 ? 59 : i.tm_sec); + t.Milliseconds = 0; +} + +inline void SysTimeTToSystem(time_t t, SysTimeStruct& s) { + SysFileTimeStruct ft {}; + ft.time = t; + ft.is_invalid = false; + SysFileToSystemTimeUtc(ft, s); +} + +inline time_t SysTimegm(struct tm* tm) { + return timegm(tm); +} + +inline void SysSystemToFileTimeUtc(const SysTimeStruct& f, SysFileTimeStruct& t) { + struct tm i {}; + + i.tm_year = f.Year - 1900; + i.tm_mon = f.Month - 1; + i.tm_mday = f.Day; + i.tm_hour = f.Hour; + i.tm_min = f.Minute; + i.tm_sec = f.Second; + + t.is_invalid = (f.is_invalid || (t.time = SysTimegm(&i)) == static_cast(-1)); +} + +// Retrieves the current local date and time. +inline void SysGetSystemTime(SysTimeStruct& t) { + time_t st {}; + struct tm i {}; + + if (time(&st) == static_cast(-1) || localtime_r(&st, &i) == nullptr) { + t.is_invalid = true; + return; + } + + t.is_invalid = false; + t.Year = i.tm_year + 1900; + t.Month = i.tm_mon + 1; + t.Day = i.tm_mday; + t.Hour = i.tm_hour; + t.Minute = i.tm_min; + t.Second = (i.tm_sec == 60 ? 59 : i.tm_sec); + t.Milliseconds = 0; +} + +// Retrieves the current system date and time in Coordinated Universal Time (UTC). +inline void SysGetSystemTimeUtc(SysTimeStruct& t) { + time_t st {}; + struct tm i {}; + + if (time(&st) == static_cast(-1) || gmtime_r(&st, &i) == nullptr) { + t.is_invalid = true; + return; + } + + t.is_invalid = false; + t.Year = i.tm_year + 1900; + t.Month = i.tm_mon + 1; + t.Day = i.tm_mday; + t.Hour = i.tm_hour; + t.Minute = i.tm_min; + t.Second = (i.tm_sec == 60 ? 59 : i.tm_sec); + t.Milliseconds = 0; +} + +inline void SysQueryPerformanceFrequency(uint64_t* freq) { + *freq = 1000000000LL; +} + +inline void SysQueryPerformanceCounter(uint64_t* counter) { + struct timespec now {}; + clock_gettime(CLOCK_MONOTONIC, &now); + *counter = now.tv_sec * 1000000000LL + now.tv_nsec; +} + +#endif + +#endif /* KYTY_COMMON_PLATFORM_SYSTIMER_H_ */ diff --git a/src/common/platform/sysVirtual.h b/src/common/platform/sysVirtual.h new file mode 100644 index 000000000..537fd122a --- /dev/null +++ b/src/common/platform/sysVirtual.h @@ -0,0 +1,28 @@ +#ifndef KYTY_COMMON_PLATFORM_SYSVIRTUAL_H_ +#define KYTY_COMMON_PLATFORM_SYSVIRTUAL_H_ + +#include "common/common.h" +#include "common/virtualMemory.h" + +namespace Common { + +void SysVirtualInit(); + +uint64_t SysVirtualAlloc(uint64_t address, uint64_t size, VirtualMemory::Mode mode); +uint64_t SysVirtualAllocAligned(uint64_t address, uint64_t size, VirtualMemory::Mode mode, + uint64_t alignment); +bool SysVirtualAllocFixed(uint64_t address, uint64_t size, VirtualMemory::Mode mode); +bool SysVirtualCommit(uint64_t address, uint64_t size, VirtualMemory::Mode mode); +uint64_t SysVirtualReserve(uint64_t address, uint64_t size); +uint64_t SysVirtualReserveAligned(uint64_t address, uint64_t size, uint64_t alignment); +bool SysVirtualReserveFixed(uint64_t address, uint64_t size); +bool SysVirtualDecommit(uint64_t address, uint64_t size); +bool SysVirtualFree(uint64_t address); +bool SysVirtualFreeRange(uint64_t address, uint64_t size); +bool SysVirtualProtect(uint64_t address, uint64_t size, VirtualMemory::Mode mode, + VirtualMemory::Mode* old_mode = nullptr); +bool SysVirtualFlushInstructionCache(uint64_t address, uint64_t size); + +} // namespace Common + +#endif /* KYTY_COMMON_PLATFORM_SYSVIRTUAL_H_ */ diff --git a/src/common/platform/sysWindowsDbg.cpp b/src/common/platform/sysWindowsDbg.cpp new file mode 100644 index 000000000..976d2506a --- /dev/null +++ b/src/common/platform/sysWindowsDbg.cpp @@ -0,0 +1,105 @@ +#include "common/common.h" + +// IWYU pragma: no_include +// IWYU pragma: no_include +// IWYU pragma: no_include +// IWYU pragma: no_include +// IWYU pragma: no_include + +#if KYTY_PLATFORM != KYTY_PLATFORM_WINDOWS +// #error "KYTY_PLATFORM != KYTY_PLATFORM_WINDOWS" +#else + +#include // IWYU pragma: keep + +#include "common/assert.h" +#include "common/platform/sysDbg.h" + +struct FrameS { + struct FrameS* next; + void* ret_addr; +}; + +int WalkStack(int z_stack_depth, void** z_stack_trace) { + CONTEXT context; + // KNONVOLATILE_CONTEXT_POINTERS NvContext; + PRUNTIME_FUNCTION runtime_function = nullptr; + PVOID handler_data = nullptr; + ULONG64 establisher_frame = 0; + ULONG64 image_base = 0; + + RtlCaptureContext(&context); + + int frame = 0; + + while (true) { + if (frame >= z_stack_depth) { + break; + } + + z_stack_trace[frame] = reinterpret_cast(context.Rip); + + frame++; + + runtime_function = RtlLookupFunctionEntry(context.Rip, &image_base, nullptr); + + if (runtime_function == nullptr) { + break; + } + + // RtlZeroMemory(&NvContext, sizeof(KNONVOLATILE_CONTEXT_POINTERS)); + RtlVirtualUnwind(0, image_base, context.Rip, runtime_function, &context, &handler_data, + &establisher_frame, nullptr /*&NvContext*/); + + if (context.Rip == 0u) { + break; + } + } + + return frame; +} + +void SysStackWalk(void** stack, int* depth) { + int n = WalkStack(*depth, stack); + *depth = n; + // unwind_info_t info = {stack, 0, *depth}; + // _Unwind_Backtrace(&trace_fcn, &info); + // *depth = info.depth; +} + +void SysStackUsagePrint(sys_dbg_stack_info_t& stack) { + printf("stack: (0x%" PRIx64 ", %" PRIu64 ") + (0x%" PRIx64 ", %" PRIu64 ") + (0x%" PRIx64 + ", %" PRIu64 ")\n", + static_cast(stack.reserved_addr), static_cast(stack.reserved_size), + static_cast(stack.guard_addr), static_cast(stack.guard_size), + static_cast(stack.commited_addr), static_cast(stack.commited_size)); +} + +void SysStackUsage(sys_dbg_stack_info_t& s) { + MEMORY_BASIC_INFORMATION mbi {}; + [[maybe_unused]] size_t ss = VirtualQuery(&mbi, &mbi, sizeof(mbi)); + EXIT_IF(ss == 0); + PVOID reserved = mbi.AllocationBase; + ss = VirtualQuery(reserved, &mbi, sizeof(mbi)); + EXIT_IF(ss == 0); + size_t reserved_size = mbi.RegionSize; + ss = VirtualQuery(static_cast(reserved) + reserved_size, &mbi, sizeof(mbi)); + EXIT_IF(ss == 0); + void* guard_page = mbi.BaseAddress; + size_t guard_page_size = mbi.RegionSize; + ss = VirtualQuery(static_cast(guard_page) + guard_page_size, &mbi, sizeof(mbi)); + EXIT_IF(ss == 0); + void* commited = mbi.BaseAddress; + size_t commited_size = mbi.RegionSize; + s.reserved_addr = reinterpret_cast(reserved); + s.reserved_size = reserved_size; + s.guard_addr = reinterpret_cast(guard_page); + s.guard_size = guard_page_size; + s.commited_addr = reinterpret_cast(commited); + s.commited_size = commited_size; + + s.addr = s.reserved_addr; + s.total_size = s.reserved_size + s.guard_size + s.commited_size; +} + +#endif diff --git a/src/common/platform/sysWindowsFileIO.cpp b/src/common/platform/sysWindowsFileIO.cpp new file mode 100644 index 000000000..2d6c67b54 --- /dev/null +++ b/src/common/platform/sysWindowsFileIO.cpp @@ -0,0 +1,576 @@ +#include "common/common.h" + +#if KYTY_PLATFORM != KYTY_PLATFORM_WINDOWS +// #error "KYTY_PLATFORM != KYTY_PLATFORM_WINDOWS" +#else + +#include "common/platform/sysFileIO.h" +#include "common/platform/sysTimer.h" +#include "common/stringUtils.h" + +#include +#include +#include // IWYU pragma: keep + +// NOLINTNEXTLINE(readability-identifier-naming) +enum sys_file_type_t { + SYS_FILE_ERROR, // NOLINT(readability-identifier-naming) + SYS_FILE_MEMORY_STAT, // NOLINT(readability-identifier-naming) + SYS_FILE_FILE, // NOLINT(readability-identifier-naming) + SYS_FILE_MEMORY_DYN // NOLINT(readability-identifier-naming) +}; + +// NOLINTNEXTLINE(readability-identifier-naming) +struct sys_file_mem_buf_t { + uint8_t* base; + uint8_t* ptr; + uint32_t size; +}; + +// NOLINTNEXTLINE(readability-identifier-naming) +struct sys_file_t { + sys_file_type_t type; + union { + HANDLE handle; + sys_file_mem_buf_t* buf; + }; +}; + +// IWYU pragma: no_include +// IWYU pragma: no_include +// IWYU pragma: no_include +// IWYU pragma: no_include +// IWYU pragma: no_include + +constexpr DWORD FILE_SHARE_POSIX = static_cast(FILE_SHARE_READ) | + static_cast(FILE_SHARE_WRITE) | + static_cast(FILE_SHARE_DELETE); + +static DWORD GetCacheAccessType(sys_file_cache_type_t t) { + if (t == SYS_FILE_CACHE_RANDOM_ACCESS) { + return FILE_FLAG_RANDOM_ACCESS; + } + + if (t == SYS_FILE_CACHE_SEQUENTIAL_SCAN) { + return SYS_FILE_CACHE_SEQUENTIAL_SCAN; + } + + return SYS_FILE_CACHE_AUTO; +} + +void SysFileRead(void* data, uint32_t size, sys_file_t& f, uint32_t* bytes_read) { + if (f.type == SYS_FILE_FILE) { + DWORD w = 0; + ReadFile(f.handle, data, size, &w, nullptr); + if (bytes_read != nullptr) { + *bytes_read = w; + } + } else if (f.type == SYS_FILE_MEMORY_STAT) { + uint32_t s = size; + if (f.buf->size != 0u) { + uint32_t l = f.buf->size - (f.buf->ptr - f.buf->base); + if (s > l) { + s = l; + } + } + std::memcpy(data, f.buf->ptr, s); + f.buf->ptr += s; + if (bytes_read != nullptr) { + *bytes_read = s; + } + } else if (f.type == SYS_FILE_MEMORY_DYN) { + uint32_t s = size; + if (f.buf->size != 0u) { + uint32_t l = f.buf->size - (f.buf->ptr - f.buf->base); + if (s > l) { + s = l; + } + } else { + s = 0; + } + std::memcpy(data, f.buf->ptr, s); + f.buf->ptr += s; + if (bytes_read != nullptr) { + *bytes_read = s; + } + } +} + +void SysFileWrite(const void* data, uint32_t size, sys_file_t& f, uint32_t* bytes_written) { + if (f.type == SYS_FILE_FILE) { + DWORD w = 0; + WriteFile(f.handle, data, size, &w, nullptr); + if (bytes_written != nullptr) { + *bytes_written = w; + } + } else if (f.type == SYS_FILE_MEMORY_STAT) { + uint32_t s = size; + if (f.buf->size != 0u) { + uint32_t l = f.buf->size - (f.buf->ptr - f.buf->base); + if (s > l) { + s = l; + } + } + std::memcpy(f.buf->ptr, data, s); + f.buf->ptr += s; + if (bytes_written != nullptr) { + *bytes_written = s; + } + } else if (f.type == SYS_FILE_MEMORY_DYN) { + uint32_t pos = f.buf->ptr - f.buf->base; + if (f.buf->size < pos + size) { + f.buf->base = static_cast(std::realloc(f.buf->base, pos + size)); + f.buf->ptr = f.buf->base + pos; + f.buf->size = pos + size; + } + std::memcpy(f.buf->ptr, data, size); + f.buf->ptr += size; + if (bytes_written != nullptr) { + *bytes_written = size; + } + } +} + +void SysFileWrite(uint32_t n, sys_file_t& f) { + SysFileWrite(&n, 4, f); +} + +sys_file_t* SysFileCreate(const std::filesystem::path& file_name) { + auto* ret = new sys_file_t; + + auto wide = file_name.wstring(); + HANDLE h_file = nullptr; + h_file = CreateFileW(wide.c_str(), + static_cast(GENERIC_READ) | static_cast(GENERIC_WRITE) | + static_cast(DELETE), + FILE_SHARE_POSIX, nullptr, CREATE_ALWAYS, FILE_ATTRIBUTE_NORMAL, nullptr); + + ret->handle = h_file; + ret->type = SYS_FILE_FILE; + + return ret; +} + +sys_file_t* SysFileOpenR(const std::filesystem::path& file_name, sys_file_cache_type_t cache_type) { + auto* ret = new sys_file_t; + + auto wide = file_name.wstring(); + HANDLE h_file = nullptr; + h_file = CreateFileW(wide.c_str(), GENERIC_READ, FILE_SHARE_POSIX, nullptr, OPEN_EXISTING, + GetCacheAccessType(cache_type), nullptr); + + if (h_file == INVALID_HANDLE_VALUE) { + ret->type = SYS_FILE_ERROR; + } else { + ret->type = SYS_FILE_FILE; + } + + ret->handle = h_file; + + return ret; +} + +sys_file_t* SysFileOpen(uint8_t* buf, uint32_t buf_size) { + auto* ret = new sys_file_t; + + ret->type = SYS_FILE_MEMORY_STAT; + ret->buf = new sys_file_mem_buf_t; + ret->buf->base = buf; + ret->buf->ptr = buf; + ret->buf->size = buf_size; + + return ret; +} + +sys_file_t* SysFileCreate() { + auto* ret = new sys_file_t; + + ret->type = SYS_FILE_MEMORY_DYN; + ret->buf = new sys_file_mem_buf_t; + ret->buf->base = nullptr; + ret->buf->ptr = nullptr; + ret->buf->size = 0; + + return ret; +} + +sys_file_t* SysFileOpenW(const std::filesystem::path& file_name, sys_file_cache_type_t cache_type) { + auto* ret = new sys_file_t; + + auto wide = file_name.wstring(); + HANDLE h_file = nullptr; + h_file = CreateFileW( + wide.c_str(), static_cast(GENERIC_WRITE) | static_cast(DELETE), + FILE_SHARE_POSIX, nullptr, OPEN_EXISTING, GetCacheAccessType(cache_type), nullptr); + + if (h_file == INVALID_HANDLE_VALUE) { + ret->type = SYS_FILE_ERROR; + } else { + ret->type = SYS_FILE_FILE; + } + + ret->handle = h_file; + + return ret; +} + +sys_file_t* SysFileOpenRw(const std::filesystem::path& file_name, + sys_file_cache_type_t cache_type) { + auto* ret = new sys_file_t; + + auto wide = file_name.wstring(); + HANDLE h_file = nullptr; + h_file = CreateFileW(wide.c_str(), + static_cast(GENERIC_READ) | static_cast(GENERIC_WRITE) | + static_cast(DELETE), + FILE_SHARE_POSIX, nullptr, OPEN_EXISTING, GetCacheAccessType(cache_type), + nullptr); + + if (h_file == INVALID_HANDLE_VALUE) { + ret->type = SYS_FILE_ERROR; + } else { + ret->type = SYS_FILE_FILE; + } + + ret->handle = h_file; + + return ret; +} + +void SysFileClose(sys_file_t* f) { + if (f->type == SYS_FILE_FILE) { + CloseHandle(f->handle); + } else if (f->type == SYS_FILE_MEMORY_STAT) { + delete f->buf; + } else if (f->type == SYS_FILE_MEMORY_DYN) { + std::free(f->buf->base); + delete f->buf; + } + + // f.type = SYS_FILE_ERROR; + delete f; +} + +uint64_t SysFileSize(sys_file_t& f) { + if (f.type == SYS_FILE_FILE) { + LARGE_INTEGER s; + GetFileSizeEx(f.handle, &s); + return s.QuadPart; + } + + if (f.type == SYS_FILE_MEMORY_STAT || f.type == SYS_FILE_MEMORY_DYN) { + return f.buf->size; + } + + return 0; +} + +uint64_t SysFileSize(const std::filesystem::path& file_name) { + LARGE_INTEGER s; + WIN32_FILE_ATTRIBUTE_DATA a; + + auto wide = file_name.wstring(); + if (GetFileAttributesExW(wide.c_str(), GetFileExInfoStandard, &a) == 0) { + return 0; + } + + s.HighPart = static_cast(a.nFileSizeHigh); + s.LowPart = a.nFileSizeLow; + + return s.QuadPart; +} + +bool SysFileTruncate(sys_file_t& f, uint64_t size) { + bool ok = false; + if (f.type == SYS_FILE_FILE) { + LARGE_INTEGER s {}; + LARGE_INTEGER r {}; + s.QuadPart = 0; + SetFilePointerEx(f.handle, s, &r, FILE_CURRENT); + s.QuadPart = static_cast(size); + ok = (SetFilePointerEx(f.handle, s, nullptr, FILE_BEGIN) != 0 && + SetEndOfFile(f.handle) != 0); + SetFilePointerEx(f.handle, r, nullptr, FILE_BEGIN); + } + + return ok; +} + +bool SysFileUnlink(sys_file_t& f, const std::filesystem::path& name) { + if (f.type == SYS_FILE_FILE) { + FILE_DISPOSITION_INFO info {}; + info.DeleteFile = TRUE; + if (SetFileInformationByHandle(f.handle, FileDispositionInfo, &info, sizeof(info)) != 0) { + return true; + } + } + + return SysFileDeleteFile(name); +} + +bool SysFileSeek(sys_file_t& f, uint64_t offset) { + bool ok = true; + if (f.type == SYS_FILE_FILE) { + LARGE_INTEGER s; + s.QuadPart = static_cast(offset); + ok = (SetFilePointerEx(f.handle, s, nullptr, FILE_BEGIN) != 0); + // printf("seek: %u\n", offset); + } else if (f.type == SYS_FILE_MEMORY_STAT || f.type == SYS_FILE_MEMORY_DYN) { + f.buf->ptr = f.buf->base + offset; + } + + return ok; +} + +uint64_t SysFileTell(sys_file_t& f) { + if (f.type == SYS_FILE_FILE) { + LARGE_INTEGER s {}; + LARGE_INTEGER r {}; + s.QuadPart = 0; + SetFilePointerEx(f.handle, s, &r, FILE_CURRENT); + // printf("tell: %u\n", r.QuadPart); + return r.QuadPart; + } + + if (f.type == SYS_FILE_MEMORY_STAT || f.type == SYS_FILE_MEMORY_DYN) { + return f.buf->ptr - f.buf->base; + } + + return 0; +} + +bool SysFileIsError(sys_file_t& f) { + return f.type == SYS_FILE_ERROR || + (f.type == SYS_FILE_FILE && f.handle == INVALID_HANDLE_VALUE); +} + +bool SysFileIsDirectoryExisting(const std::filesystem::path& path) { + auto wide = path.wstring(); + DWORD a = GetFileAttributesW(wide.c_str()); + return a != INVALID_FILE_ATTRIBUTES && + ((a & static_cast(FILE_ATTRIBUTE_DIRECTORY)) != 0u); +} + +bool SysFileIsFileExisting(const std::filesystem::path& name) { + auto wide = name.wstring(); + DWORD a = GetFileAttributesW(wide.c_str()); + return a != INVALID_FILE_ATTRIBUTES && + ((a & static_cast(FILE_ATTRIBUTE_DIRECTORY)) == 0u); +} + +bool SysFileCreateDirectory(const std::filesystem::path& path) { + auto wide = path.wstring(); + return CreateDirectoryW(wide.c_str(), nullptr) != 0; +} + +bool SysFileDeleteDirectory(const std::filesystem::path& path) { + auto wide = path.wstring(); + return RemoveDirectoryW(wide.c_str()) != 0; +} + +bool SysFileDeleteFile(const std::filesystem::path& name) { + auto wide = name.wstring(); + return DeleteFileW(wide.c_str()) != 0; +} + +bool SysFileFlush(sys_file_t& f) { + if (f.type == SYS_FILE_FILE && f.handle != INVALID_HANDLE_VALUE) { + return (FlushFileBuffers(f.handle) != 0); + } + + return false; +} + +SysFileTimeStruct SysFileGetLastAccessTimeUtc(const std::filesystem::path& name) { + SysFileTimeStruct r {}; + sys_file_t* f = SysFileOpenR(name); + r.is_invalid = + (f->type == SYS_FILE_ERROR || (GetFileTime(f->handle, nullptr, &r.time, nullptr) == 0)); + SysFileClose(f); + return r; +} + +SysFileTimeStruct SysFileGetLastWriteTimeUtc(const std::filesystem::path& name) { + SysFileTimeStruct r {}; + sys_file_t* f = SysFileOpenR(name); + r.is_invalid = + (f->type == SYS_FILE_ERROR || (GetFileTime(f->handle, nullptr, nullptr, &r.time) == 0)); + SysFileClose(f); + return r; +} + +void SysFileGetLastAccessAndWriteTimeUtc(const std::filesystem::path& name, SysFileTimeStruct& a, + SysFileTimeStruct& w) { + // TODO() open file with dwDesiredAccess = 0 + // TODO() open directory + sys_file_t* f = SysFileOpenR(name); + a.is_invalid = w.is_invalid = + (f->type == SYS_FILE_ERROR || (GetFileTime(f->handle, nullptr, &a.time, &w.time) == 0)); + SysFileClose(f); +} + +void SysFileGetLastAccessAndWriteTimeUtc(sys_file_t& f, SysFileTimeStruct& a, + SysFileTimeStruct& w) { + if (f.type == SYS_FILE_FILE) { + a.is_invalid = w.is_invalid = (GetFileTime(f.handle, nullptr, &a.time, &w.time) == 0); + } else if (f.type == SYS_FILE_MEMORY_STAT || f.type == SYS_FILE_MEMORY_DYN) { + SysTimeStruct t {}; + SysGetSystemTimeUtc(t); + SysSystemToFileTimeUtc(t, a); + SysSystemToFileTimeUtc(t, w); + } else { + a.is_invalid = w.is_invalid = true; + } +} + +bool SysFileSetLastAccessTimeUtc(const std::filesystem::path& name, SysFileTimeStruct& access) { + if (access.is_invalid) { + return false; + } + + bool ok = true; + + sys_file_t* f = SysFileOpenW(name); + + ok = !(f->type == SYS_FILE_ERROR || + (SetFileTime(f->handle, nullptr, &access.time, nullptr) == 0)); + + SysFileClose(f); + + return ok; +} + +bool SysFileSetLastWriteTimeUtc(const std::filesystem::path& name, SysFileTimeStruct& write) { + if (write.is_invalid) { + return false; + } + + bool ok = true; + + sys_file_t* f = SysFileOpenW(name); + + ok = !(f->type == SYS_FILE_ERROR || + (SetFileTime(f->handle, nullptr, nullptr, &write.time) == 0)); + + SysFileClose(f); + + return ok; +} + +bool SysFileSetLastAccessAndWriteTimeUtc(const std::filesystem::path& name, + SysFileTimeStruct& access, SysFileTimeStruct& write) { + if (access.is_invalid || write.is_invalid) { + return false; + } + + bool ok = true; + + sys_file_t* f = SysFileOpenW(name); + + ok = !(f->type == SYS_FILE_ERROR || + (SetFileTime(f->handle, nullptr, &access.time, &write.time) == 0)); + + SysFileClose(f); + + return ok; +} + +void SysFileFindFiles(const std::filesystem::path& path, std::vector& out) { + std::string real_path = Common::ReplaceChar(Common::PathToGenericString(path), '\\', '/'); + if (!Common::EndsWith(real_path, "/")) { + real_path += "/"; + } + + std::string pattern = real_path + "*"; + + HANDLE h = nullptr; + WIN32_FIND_DATAW data; + + auto wide_pattern = std::filesystem::path(pattern).wstring(); + h = FindFirstFileW(wide_pattern.c_str(), &data); + + if (h == INVALID_HANDLE_VALUE) { + return; + } + + do { + std::filesystem::path file_name(data.cFileName); + auto file_name_str = Common::PathToString(file_name); + + if (file_name_str == "." || file_name_str == "..") { + continue; + } + + if ((data.dwFileAttributes & static_cast(FILE_ATTRIBUTE_DIRECTORY)) != 0u) { + SysFileFindFiles(std::filesystem::path(real_path) / file_name, out); + } else { + sys_file_find_t r {}; + + r.path_with_name = std::filesystem::path(real_path) / file_name; + r.size = (static_cast(data.nFileSizeHigh) << 32u) + + static_cast(data.nFileSizeLow); + r.last_access_time.is_invalid = false; + r.last_access_time.time = data.ftLastAccessTime; + r.last_write_time.is_invalid = false; + r.last_write_time.time = data.ftLastWriteTime; + + out.push_back(r); + } + + } while (FindNextFileW(h, &data) != 0); + + FindClose(h); +} + +void SysFileGetDents(const std::filesystem::path& path, std::vector& out) { + std::string real_path = Common::ReplaceChar(Common::PathToGenericString(path), '\\', '/'); + if (!Common::EndsWith(real_path, "/")) { + real_path += "/"; + } + + std::string pattern = real_path + "*"; + + HANDLE h = nullptr; + WIN32_FIND_DATAW data; + + auto wide_pattern = std::filesystem::path(pattern).wstring(); + h = FindFirstFileW(wide_pattern.c_str(), &data); + + if (h == INVALID_HANDLE_VALUE) { + return; + } + + do { + std::filesystem::path file_name(data.cFileName); + + sys_dir_entry_t r {}; + + r.is_file = ((data.dwFileAttributes & static_cast(FILE_ATTRIBUTE_DIRECTORY)) == 0u); + r.name = Common::PathToString(file_name); + + out.push_back(r); + + } while (FindNextFileW(h, &data) != 0); + + FindClose(h); +} + +bool SysFileCopyFile(const std::filesystem::path& src, const std::filesystem::path& dst) { + auto src_wide = src.wstring(); + auto dst_wide = dst.wstring(); + return CopyFileW(src_wide.c_str(), dst_wide.c_str(), FALSE) != 0; +} + +bool SysFileMoveFile(const std::filesystem::path& src, const std::filesystem::path& dst) { + auto src_wide = src.wstring(); + auto dst_wide = dst.wstring(); + return MoveFileW(src_wide.c_str(), dst_wide.c_str()) != 0; +} + +void SysFileRemoveReadonly(const std::filesystem::path& name) { + auto wide = name.wstring(); + SetFileAttributesW(wide.c_str(), GetFileAttributesW(wide.c_str()) & + (~static_cast(FILE_ATTRIBUTE_READONLY))); +} + +#endif diff --git a/src/common/platform/sysWindowsVirtual.cpp b/src/common/platform/sysWindowsVirtual.cpp new file mode 100644 index 000000000..9370787fb --- /dev/null +++ b/src/common/platform/sysWindowsVirtual.cpp @@ -0,0 +1,355 @@ +#include "common/common.h" + +#if KYTY_PLATFORM != KYTY_PLATFORM_WINDOWS +// #error "KYTY_PLATFORM != KYTY_PLATFORM_WINDOWS" +#else + +#include "common/assert.h" +#include "common/platform/sysVirtual.h" +#include "common/virtualMemory.h" + +#include // IWYU pragma: keep + +// IWYU pragma: no_include +// IWYU pragma: no_include +// IWYU pragma: no_include +// IWYU pragma: no_include +// IWYU pragma: no_include +// IWYU pragma: no_include +// IWYU pragma: no_include +// IWYU pragma: no_include + +namespace Common { + +static DWORD GetProtectionFlag(VirtualMemory::Mode mode) { + DWORD protect = PAGE_NOACCESS; + switch (mode) { + case VirtualMemory::Mode::Read: protect = PAGE_READONLY; break; + + case VirtualMemory::Mode::Write: + case VirtualMemory::Mode::ReadWrite: protect = PAGE_READWRITE; break; + + case VirtualMemory::Mode::Execute: protect = PAGE_EXECUTE; break; + + case VirtualMemory::Mode::ExecuteRead: protect = PAGE_EXECUTE_READ; break; + + case VirtualMemory::Mode::ExecuteWrite: + case VirtualMemory::Mode::ExecuteReadWrite: protect = PAGE_EXECUTE_READWRITE; break; + + case VirtualMemory::Mode::NoAccess: + default: protect = PAGE_NOACCESS; break; + } + return protect; +} + +static VirtualMemory::Mode GetProtectionFlag(DWORD mode) { + switch (mode) { + case PAGE_NOACCESS: return VirtualMemory::Mode::NoAccess; + case PAGE_READONLY: return VirtualMemory::Mode::Read; + case PAGE_READWRITE: return VirtualMemory::Mode::ReadWrite; + case PAGE_EXECUTE: return VirtualMemory::Mode::Execute; + case PAGE_EXECUTE_READ: return VirtualMemory::Mode::ExecuteRead; + case PAGE_EXECUTE_READWRITE: return VirtualMemory::Mode::ExecuteReadWrite; + default: return VirtualMemory::Mode::NoAccess; + } +} + +void SysVirtualInit() {} + +uint64_t SysVirtualAlloc(uint64_t address, uint64_t size, VirtualMemory::Mode mode) { + auto ptr = (address == 0 ? SysVirtualAllocAligned(address, size, mode, 1) + : reinterpret_cast(VirtualAlloc( + reinterpret_cast(static_cast(address)), size, + static_cast(MEM_COMMIT) | static_cast(MEM_RESERVE), + GetProtectionFlag(mode)))); + if (ptr == 0) { + auto err = static_cast(GetLastError()); + + if (err != ERROR_INVALID_ADDRESS) { + printf("VirtualAlloc() failed: 0x%08" PRIx32 "\n", err); + } else { + return SysVirtualAllocAligned(address, size, mode, 1); + } + } + return ptr; +} + +using VirtualAlloc2_func_t = /*WINBASEAPI*/ PVOID WINAPI (*)(HANDLE, PVOID, SIZE_T, ULONG, ULONG, + MEM_EXTENDED_PARAMETER*, ULONG); + +static VirtualAlloc2_func_t ResolveVirtualAlloc2() { + HMODULE h = GetModuleHandle("KernelBase"); // @suppress("Invalid arguments") + if (h != nullptr) { + return reinterpret_cast(GetProcAddress(h, "VirtualAlloc2")); + } + return nullptr; +} + +static uint64_t AlignUp(uint64_t addr, uint64_t alignment) { + return (addr + alignment - 1) & ~(alignment - 1); +} + +uint64_t SysVirtualAllocAligned(uint64_t address, uint64_t size, VirtualMemory::Mode mode, + uint64_t alignment) { + if (alignment == 0) { + printf("VirtualAlloc2 failed: 0x%08" PRIx32 "\n", static_cast(GetLastError())); + return 0; + } + + static constexpr uint64_t SYSTEM_MANAGED_MIN = 0x0000040000u; + static constexpr uint64_t SYSTEM_MANAGED_MAX = 0x07FFFFBFFFu; + static constexpr uint64_t USER_MIN = 0x1000000000u; + static constexpr uint64_t USER_MAX = 0xFBFFFFFFFFu; + + MEM_ADDRESS_REQUIREMENTS req {}; + MEM_EXTENDED_PARAMETER param {}; + req.LowestStartingAddress = + (address == 0 ? reinterpret_cast(SYSTEM_MANAGED_MIN) + : reinterpret_cast(AlignUp(address, alignment))); + req.HighestEndingAddress = (address == 0 ? reinterpret_cast(SYSTEM_MANAGED_MAX) + : reinterpret_cast(USER_MAX)); + req.Alignment = alignment; + param.Type = MemExtendedParameterAddressRequirements; + param.Pointer = &req; + + MEM_ADDRESS_REQUIREMENTS req2 {}; + MEM_EXTENDED_PARAMETER param2 {}; + req2.LowestStartingAddress = + (address == 0 ? reinterpret_cast(USER_MIN) + : reinterpret_cast(AlignUp(address, alignment))); + req2.HighestEndingAddress = reinterpret_cast(USER_MAX); + req2.Alignment = alignment; + param2.Type = MemExtendedParameterAddressRequirements; + param2.Pointer = &req2; + + static auto virtual_alloc2 = ResolveVirtualAlloc2(); + + EXIT_NOT_IMPLEMENTED(virtual_alloc2 == nullptr); + + auto ptr = reinterpret_cast( + virtual_alloc2(GetCurrentProcess(), nullptr, size, + static_cast(MEM_COMMIT) | static_cast(MEM_RESERVE), + GetProtectionFlag(mode), ¶m, 1)); + + if (ptr == 0) { + ptr = reinterpret_cast( + virtual_alloc2(GetCurrentProcess(), nullptr, size, + static_cast(MEM_COMMIT) | static_cast(MEM_RESERVE), + GetProtectionFlag(mode), ¶m2, 1)); + } + + if (ptr == 0) { + auto err = static_cast(GetLastError()); + if (err != ERROR_INVALID_PARAMETER) { + printf("VirtualAlloc2(alignment = 0x%016" PRIx64 ") failed: 0x%08" PRIx32 "\n", + alignment, err); + } else { + return SysVirtualAllocAligned(address, size, mode, alignment << 1u); + } + } + return ptr; +} + +bool SysVirtualAllocFixed(uint64_t address, uint64_t size, VirtualMemory::Mode mode) { + auto ptr = reinterpret_cast(VirtualAlloc( + reinterpret_cast(static_cast(address)), size, + static_cast(MEM_COMMIT) | static_cast(MEM_RESERVE), GetProtectionFlag(mode))); + if (ptr == 0) { + auto err = static_cast(GetLastError()); + + printf("VirtualAlloc() failed: 0x%08" PRIx32 "\n", err); + return false; + } + + if (ptr != address) { + printf("VirtualAlloc() failed: wrong address\n"); + VirtualFree(reinterpret_cast(ptr), 0, MEM_RELEASE); + return false; + } + + return true; +} + +bool SysVirtualCommit(uint64_t address, uint64_t size, VirtualMemory::Mode mode) { + auto ptr = reinterpret_cast( + VirtualAlloc(reinterpret_cast(static_cast(address)), size, + static_cast(MEM_COMMIT), GetProtectionFlag(mode))); + if (ptr == 0) { + printf("VirtualAlloc(MEM_COMMIT) failed: 0x%08" PRIx32 "\n", + static_cast(GetLastError())); + return false; + } + + if (ptr != address) { + printf("VirtualAlloc(MEM_COMMIT) failed: wrong address\n"); + return false; + } + + return true; +} + +uint64_t SysVirtualReserve(uint64_t address, uint64_t size) { + auto ptr = (address == 0 ? SysVirtualReserveAligned(address, size, 1) + : reinterpret_cast(VirtualAlloc( + reinterpret_cast(static_cast(address)), size, + static_cast(MEM_RESERVE), PAGE_NOACCESS))); + if (ptr == 0) { + auto err = static_cast(GetLastError()); + + if (err != ERROR_INVALID_ADDRESS) { + printf("VirtualAlloc(MEM_RESERVE) failed: 0x%08" PRIx32 "\n", err); + } else { + return SysVirtualReserveAligned(address, size, 1); + } + } + return ptr; +} + +uint64_t SysVirtualReserveAligned(uint64_t address, uint64_t size, uint64_t alignment) { + if (alignment == 0) { + printf("VirtualAlloc2(MEM_RESERVE) failed: 0x%08" PRIx32 "\n", + static_cast(GetLastError())); + return 0; + } + + static constexpr uint64_t SYSTEM_MANAGED_MIN = 0x0000040000u; + static constexpr uint64_t SYSTEM_MANAGED_MAX = 0x07FFFFBFFFu; + static constexpr uint64_t USER_MIN = 0x1000000000u; + static constexpr uint64_t USER_MAX = 0xFBFFFFFFFFu; + + MEM_ADDRESS_REQUIREMENTS req {}; + MEM_EXTENDED_PARAMETER param {}; + req.LowestStartingAddress = + (address == 0 ? reinterpret_cast(SYSTEM_MANAGED_MIN) + : reinterpret_cast(AlignUp(address, alignment))); + req.HighestEndingAddress = (address == 0 ? reinterpret_cast(SYSTEM_MANAGED_MAX) + : reinterpret_cast(USER_MAX)); + req.Alignment = alignment; + param.Type = MemExtendedParameterAddressRequirements; + param.Pointer = &req; + + MEM_ADDRESS_REQUIREMENTS req2 {}; + MEM_EXTENDED_PARAMETER param2 {}; + req2.LowestStartingAddress = + (address == 0 ? reinterpret_cast(USER_MIN) + : reinterpret_cast(AlignUp(address, alignment))); + req2.HighestEndingAddress = reinterpret_cast(USER_MAX); + req2.Alignment = alignment; + param2.Type = MemExtendedParameterAddressRequirements; + param2.Pointer = &req2; + + static auto virtual_alloc2 = ResolveVirtualAlloc2(); + + EXIT_NOT_IMPLEMENTED(virtual_alloc2 == nullptr); + + auto ptr = reinterpret_cast(virtual_alloc2(GetCurrentProcess(), nullptr, size, + static_cast(MEM_RESERVE), + PAGE_NOACCESS, ¶m, 1)); + + if (ptr == 0) { + ptr = reinterpret_cast(virtual_alloc2(GetCurrentProcess(), nullptr, size, + static_cast(MEM_RESERVE), + PAGE_NOACCESS, ¶m2, 1)); + } + + if (ptr == 0) { + auto err = static_cast(GetLastError()); + if (err != ERROR_INVALID_PARAMETER) { + printf("VirtualAlloc2(MEM_RESERVE, alignment = 0x%016" PRIx64 ") failed: 0x%08" PRIx32 + "\n", + alignment, err); + } else { + return SysVirtualReserveAligned(address, size, alignment << 1u); + } + } + return ptr; +} + +bool SysVirtualReserveFixed(uint64_t address, uint64_t size) { + auto ptr = reinterpret_cast( + VirtualAlloc(reinterpret_cast(static_cast(address)), size, + static_cast(MEM_RESERVE), PAGE_NOACCESS)); + if (ptr == 0) { + printf("VirtualAlloc(MEM_RESERVE) failed: address=0x%016" PRIx64 ", size=0x%016" PRIx64 + ", err=0x%08" PRIx32 "\n", + address, size, static_cast(GetLastError())); + return false; + } + + if (ptr != address) { + printf("VirtualAlloc(MEM_RESERVE) failed: wrong address\n"); + VirtualFree(reinterpret_cast(ptr), 0, MEM_RELEASE); + return false; + } + + return true; +} + +bool SysVirtualDecommit(uint64_t address, uint64_t size) { + if (VirtualFree(reinterpret_cast(static_cast(address)), size, + MEM_DECOMMIT) == 0) { + printf("VirtualFree(MEM_DECOMMIT) failed: 0x%08" PRIx32 "\n", + static_cast(GetLastError())); + return false; + } + return true; +} + +bool SysVirtualFree(uint64_t address) { + if (VirtualFree(reinterpret_cast(static_cast(address)), 0, MEM_RELEASE) == + 0) { + printf("VirtualFree() failed: 0x%08" PRIx32 "\n", static_cast(GetLastError())); + return false; + } + return true; +} + +bool SysVirtualFreeRange(uint64_t address, uint64_t size) { + if (address == 0 || size == 0) { + return false; + } + + uint64_t current = address; + while (current - address < size) { + MEMORY_BASIC_INFORMATION info {}; + if (VirtualQuery(reinterpret_cast(current), &info, sizeof(info)) == 0 || + reinterpret_cast(info.AllocationBase) != address || info.State == MEM_FREE) { + return false; + } + const auto next = reinterpret_cast(info.BaseAddress) + info.RegionSize; + if (next <= current || next - address > size) { + return false; + } + current = next; + } + return current - address == size && SysVirtualFree(address); +} + +bool SysVirtualProtect(uint64_t address, uint64_t size, VirtualMemory::Mode mode, + VirtualMemory::Mode* old_mode) { + DWORD old_protect = 0; + if (VirtualProtect(reinterpret_cast(static_cast(address)), size, + GetProtectionFlag(mode), &old_protect) == 0) { + printf("VirtualProtect() failed: 0x%08" PRIx32 "\n", static_cast(GetLastError())); + return false; + } + if (old_mode != nullptr) { + *old_mode = GetProtectionFlag(old_protect); + } + return true; +} + +bool SysVirtualFlushInstructionCache(uint64_t address, uint64_t size) { + if (::FlushInstructionCache(GetCurrentProcess(), + reinterpret_cast(static_cast(address)), + size) == 0) { + printf("FlushInstructionCache() failed: 0x%08" PRIx32 "\n", + static_cast(GetLastError())); + return false; + } + return true; +} + +} // namespace Common + +#endif diff --git a/src/common/profiler.cpp b/src/common/profiler.cpp new file mode 100644 index 000000000..d061c4611 --- /dev/null +++ b/src/common/profiler.cpp @@ -0,0 +1,89 @@ +#include "common/profiler.h" + +#include "common/emulatorConfig.h" +#include "common/subsystems.h" + +#include +#include +#include +#include +#include +#include + +namespace { + +thread_local std::vector g_block_stack; + +void RemoveBlock(Profiler::ScopedBlock* block) { + auto block_it = std::find(g_block_stack.rbegin(), g_block_stack.rend(), block); + if (block_it != g_block_stack.rend()) { + g_block_stack.erase(std::next(block_it).base()); + } +} + +} // namespace + +namespace Profiler { + +ScopedBlock::ScopedBlock(const tracy::SourceLocationData* source_location) { + if (tracy::ProfilerAvailable()) { + m_zone.emplace(source_location, TRACY_CALLSTACK, true); + g_block_stack.push_back(this); + } +} + +ScopedBlock::~ScopedBlock() { + End(); +} + +void ScopedBlock::End() { + if (m_zone.has_value()) { + m_zone.reset(); + RemoveBlock(this); + } +} + +void EndBlock() { + if (!g_block_stack.empty()) { + g_block_stack.back()->End(); + } +} + +void SetThreadName(const char* name) { + if (tracy::ProfilerAvailable() && name != nullptr) { + tracy::SetThreadName(name); + } +} + +void Close() { + if (tracy::ProfilerAvailable()) { + tracy::ShutdownProfiler(); + } +} + +KYTY_SUBSYSTEM_INIT(Profiler) { + switch (Config::GetProfilerDirection()) { + case Config::ProfilerDirection::Network: + if (!tracy::ProfilerAvailable()) { + tracy::StartupProfiler(); + TracySetProgramName("KytyPS5"); + ::printf("Tracy profiler enabled: client %d.%d.%d, protocol %u, " + "broadcast %u, connect to 127.0.0.1:8086\n", + tracy::Version::Major, tracy::Version::Minor, tracy::Version::Patch, + tracy::ProtocolVersion, tracy::BroadcastVersion); + } + break; + case Config::ProfilerDirection::None: + default: break; + } +} + +KYTY_SUBSYSTEM_UNEXPECTED_SHUTDOWN(Profiler) { + Close(); +} + +KYTY_SUBSYSTEM_DESTROY(Profiler) { + Close(); +} + +} // namespace Profiler diff --git a/src/common/profiler.h b/src/common/profiler.h new file mode 100644 index 000000000..a84e308bd --- /dev/null +++ b/src/common/profiler.h @@ -0,0 +1,78 @@ +#ifndef KYTY_COMMON_PROFILER_H_ +#define KYTY_COMMON_PROFILER_H_ + +#include "common/common.h" +#include "common/subsystems.h" + +#include +#include +#include // IWYU pragma: export + +namespace profiler::colors { + +inline constexpr uint32_t RedA100 = 0xff8a80; +inline constexpr uint32_t Blue300 = 0x64b5f6; +inline constexpr uint32_t CyanA700 = 0x00b8d4; +inline constexpr uint32_t Green100 = 0xc8e6c9; +inline constexpr uint32_t Green200 = 0xa5d6a7; +inline constexpr uint32_t Green300 = 0x81c784; +inline constexpr uint32_t Green400 = 0x66bb6a; +inline constexpr uint32_t Amber300 = 0xffd54f; +inline constexpr uint32_t DeepOrangeA200 = 0xff6e40; + +} // namespace profiler::colors + +namespace Profiler { + +class ScopedBlock { +public: + explicit ScopedBlock(const tracy::SourceLocationData* source_location); + ScopedBlock(const ScopedBlock&) = delete; + ScopedBlock& operator=(const ScopedBlock&) = delete; + ScopedBlock(ScopedBlock&&) = delete; + ScopedBlock& operator=(ScopedBlock&&) = delete; + ~ScopedBlock(); + + void End(); + +private: + std::optional m_zone; +}; + +void EndBlock(); +void SetThreadName(const char* name); + +KYTY_SUBSYSTEM_DEFINE(Profiler); + +} // namespace Profiler + +#define KYTY_PROFILER_CONCAT_IMPL(a, b) a##b +#define KYTY_PROFILER_CONCAT(a, b) KYTY_PROFILER_CONCAT_IMPL(a, b) +#define KYTY_PROFILER_COLOR_OR_DEFAULT(default_color, ...) \ + KYTY_PROFILER_COLOR_OR_DEFAULT_IMPL(default_color __VA_OPT__(, ) __VA_ARGS__, default_color) +#define KYTY_PROFILER_COLOR_OR_DEFAULT_IMPL(default_color, color, ...) color + +#define KYTY_PROFILER_BLOCK(name, ...) \ + KYTY_PROFILER_BLOCK_IMPL(__LINE__, name __VA_OPT__(, ) __VA_ARGS__) +#define KYTY_PROFILER_BLOCK_IMPL(line, name, ...) \ + static constexpr tracy::SourceLocationData KYTY_PROFILER_CONCAT( \ + kyty_profiler_source_location_, line) {name, TracyFunction, TracyFile, \ + static_cast(line), \ + KYTY_PROFILER_COLOR_OR_DEFAULT(0, __VA_ARGS__)}; \ + Profiler::ScopedBlock KYTY_PROFILER_CONCAT(kyty_profiler_block_, line)( \ + &KYTY_PROFILER_CONCAT(kyty_profiler_source_location_, line)) + +#define KYTY_PROFILER_FUNCTION(...) KYTY_PROFILER_FUNCTION_IMPL(__LINE__ __VA_OPT__(, ) __VA_ARGS__) +#define KYTY_PROFILER_FUNCTION_IMPL(line, ...) \ + static constexpr tracy::SourceLocationData KYTY_PROFILER_CONCAT( \ + kyty_profiler_source_location_, line) {nullptr, TracyFunction, TracyFile, \ + static_cast(line), \ + KYTY_PROFILER_COLOR_OR_DEFAULT(0, __VA_ARGS__)}; \ + Profiler::ScopedBlock KYTY_PROFILER_CONCAT(kyty_profiler_block_, line)( \ + &KYTY_PROFILER_CONCAT(kyty_profiler_source_location_, line)) + +#define KYTY_PROFILER_END_BLOCK Profiler::EndBlock() + +#define KYTY_PROFILER_THREAD(name) Profiler::SetThreadName(name) + +#endif /* KYTY_COMMON_PROFILER_H_ */ diff --git a/src/common/singleton.h b/src/common/singleton.h new file mode 100644 index 000000000..504588bf2 --- /dev/null +++ b/src/common/singleton.h @@ -0,0 +1,36 @@ +#ifndef KYTY_COMMON_SINGLETON_H_ +#define KYTY_COMMON_SINGLETON_H_ + +#include +#include + +namespace Common { + +template +class Singleton { +public: + static T* Instance() { + if (!g_m_instance) { + // NOLINTNEXTLINE(cppcoreguidelines-no-malloc,hicpp-no-malloc) + g_m_instance = static_cast(std::malloc(sizeof(T))); + new (g_m_instance) T; + } + + return g_m_instance; + } + + KYTY_CLASS_NO_COPY(Singleton); + +protected: + Singleton(); + ~Singleton(); + +private: + static inline T* g_m_instance = nullptr; +}; + +// template T* Singleton::instance = 0; + +} // namespace Common + +#endif /* KYTY_COMMON_SINGLETON_H_ */ diff --git a/src/common/stringUtils.h b/src/common/stringUtils.h new file mode 100644 index 000000000..e460693ef --- /dev/null +++ b/src/common/stringUtils.h @@ -0,0 +1,344 @@ +#ifndef KYTY_COMMON_STRING_UTILS_H_ +#define KYTY_COMMON_STRING_UTILS_H_ + +#include "common/byteBuffer.h" +#include "common/common.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace Common { + +constexpr uint32_t FIND_INVALID_INDEX = static_cast(-1); + +inline std::string PathToString(const std::filesystem::path& path) { +#if defined(__cpp_char8_t) + auto u8 = path.u8string(); + return {u8.begin(), u8.end()}; +#else + return path.u8string(); +#endif +} + +inline std::string PathToGenericString(const std::filesystem::path& path) { +#if defined(__cpp_char8_t) + auto u8 = path.generic_u8string(); + return {u8.begin(), u8.end()}; +#else + return path.generic_u8string(); +#endif +} + +inline bool IsSpace(char c) { + return std::isspace(static_cast(c)) != 0; +} + +inline bool EqualNoCase(std::string_view lhs, std::string_view rhs) { + if (lhs.size() != rhs.size()) { + return false; + } + for (size_t i = 0; i < lhs.size(); i++) { + auto l = static_cast(lhs[i]); + auto r = static_cast(rhs[i]); + if (std::tolower(l) != std::tolower(r)) { + return false; + } + } + return true; +} + +inline std::string ToLower(std::string_view text) { + std::string ret(text); + std::transform(ret.begin(), ret.end(), ret.begin(), + [](unsigned char c) { return static_cast(std::tolower(c)); }); + return ret; +} + +inline bool ContainsStr(std::string_view text, std::string_view needle) { + return text.find(needle) != std::string_view::npos; +} + +inline bool StartsWith(std::string_view text, std::string_view prefix) { + return text.starts_with(prefix); +} + +inline bool StartsWith(std::string_view text, char prefix) { + return !text.empty() && text.front() == prefix; +} + +inline bool EndsWith(std::string_view text, std::string_view suffix) { + return text.ends_with(suffix); +} + +inline bool EndsWith(std::string_view text, char suffix) { + return !text.empty() && text.back() == suffix; +} + +inline bool ContainsChar(std::string_view text, char needle) { + return text.find(needle) != std::string_view::npos; +} + +inline bool IndexValid(std::string_view text, uint32_t index) { + return index < text.size(); +} + +inline uint32_t FindIndex(std::string_view text, std::string_view needle, size_t from = 0) { + const auto pos = text.find(needle, from); + return pos == std::string_view::npos ? FIND_INVALID_INDEX : static_cast(pos); +} + +inline uint32_t FindIndex(std::string_view text, char needle, size_t from = 0) { + const auto pos = text.find(needle, from); + return pos == std::string_view::npos ? FIND_INVALID_INDEX : static_cast(pos); +} + +inline uint32_t FindLastIndex(std::string_view text, std::string_view needle, + size_t from = FIND_INVALID_INDEX) { + if (needle.empty()) { + return 0; + } + if (from == FIND_INVALID_INDEX || from > text.size()) { + from = text.size(); + } + const auto pos = text.rfind(needle, from); + return pos == std::string_view::npos ? FIND_INVALID_INDEX : static_cast(pos); +} + +inline uint32_t FindLastIndex(std::string_view text, char needle, + size_t from = FIND_INVALID_INDEX) { + if (from == FIND_INVALID_INDEX || from > text.size()) { + from = text.size(); + } + const auto pos = text.rfind(needle, from); + return pos == std::string_view::npos ? FIND_INVALID_INDEX : static_cast(pos); +} + +inline std::string Mid(std::string_view text, size_t first, size_t count = std::string_view::npos) { + if (first >= text.size()) { + return {}; + } + return std::string(text.substr(first, count)); +} + +inline std::string Left(std::string_view text, size_t count) { + return Mid(text, 0, count); +} + +inline std::string Right(std::string_view text, size_t count) { + if (count >= text.size()) { + return std::string(text); + } + return Mid(text, text.size() - count, count); +} + +inline std::string RemoveLast(std::string_view text, size_t count) { + if (count >= text.size()) { + return {}; + } + return Left(text, text.size() - count); +} + +inline std::string RemoveFirst(std::string_view text, size_t count) { + if (count >= text.size()) { + return {}; + } + return Right(text, text.size() - count); +} + +inline std::string TrimRight(std::string_view text) { + size_t end = text.size(); + while (end > 0 && IsSpace(text[end - 1])) { + end--; + } + return std::string(text.substr(0, end)); +} + +inline std::string TrimLeft(std::string_view text) { + size_t first = 0; + while (first < text.size() && IsSpace(text[first])) { + first++; + } + return std::string(text.substr(first)); +} + +inline std::string Trim(std::string_view text) { + return TrimRight(TrimLeft(text)); +} + +inline std::string ReplaceChar(std::string_view text, char old_char, char new_char) { + std::string ret(text); + std::replace(ret.begin(), ret.end(), old_char, new_char); + return ret; +} + +inline std::string ReplaceStr(std::string text, std::string_view old_str, + std::string_view new_str) { + if (old_str.empty()) { + return text; + } + + size_t pos = 0; + while ((pos = text.find(old_str, pos)) != std::string::npos) { + text.replace(pos, old_str.size(), new_str); + pos += new_str.size(); + } + return text; +} + +inline std::string DirectoryWithoutFilename(std::string_view text) { + const auto pos = text.find_last_of('/'); + return pos == std::string_view::npos ? std::string() : std::string(text.substr(0, pos + 1)); +} + +inline std::string FilenameWithoutDirectory(std::string_view text) { + const auto pos = text.find_last_of('/'); + return pos == std::string_view::npos ? std::string(text) : std::string(text.substr(pos + 1)); +} + +inline std::string FilenameWithoutExtension(std::string_view text) { + const auto pos = text.find_last_of('.'); + return pos == std::string_view::npos ? std::string(text) : std::string(text.substr(0, pos)); +} + +inline std::string ExtensionWithoutFilename(std::string_view text) { + const auto pos = text.find_last_of('.'); + return pos == std::string_view::npos ? std::string() : std::string(text.substr(pos)); +} + +inline std::string FixFilenameSlash(std::string_view text) { + return ReplaceChar(text, '\\', '/'); +} + +inline std::string FixDirectorySlash(std::string_view text) { + auto ret = FixFilenameSlash(text); + if (!ret.ends_with('/')) { + ret += '/'; + } + return ret; +} + +inline std::vector Split(std::string_view text, std::string_view sep, + bool keep_empty = false) { + std::vector ret; + if (sep.empty()) { + for (char c: text) { + ret.emplace_back(1, c); + } + return ret; + } + + size_t start = 0; + for (;;) { + const auto pos = text.find(sep, start); + if (pos == std::string_view::npos) { + break; + } + if (keep_empty || pos != start) { + ret.emplace_back(text.substr(start, pos - start)); + } + start = pos + sep.size(); + } + if (keep_empty || start != text.size()) { + ret.emplace_back(text.substr(start)); + } + return ret; +} + +inline std::vector Split(std::string_view text, char sep, bool keep_empty = false) { + return Split(text, std::string_view(&sep, 1), keep_empty); +} + +inline std::string Concat(const std::vector& list, std::string_view sep) { + std::string ret; + for (size_t i = 0; i < list.size(); i++) { + if (i != 0) { + ret += sep; + } + ret += list[i]; + } + return ret; +} + +inline std::string Concat(const std::vector& list, char sep) { + return Concat(list, std::string_view(&sep, 1)); +} + +inline uint32_t ToUint32(std::string_view text, int base = 10) { + uint32_t value = 0; + std::from_chars(text.data(), text.data() + text.size(), value, base); + return value; +} + +inline uint64_t ToUint64(std::string_view text, int base = 10) { + uint64_t value = 0; + std::from_chars(text.data(), text.data() + text.size(), value, base); + return value; +} + +inline int32_t ToInt32(std::string_view text, int base = 10) { + int32_t value = 0; + std::from_chars(text.data(), text.data() + text.size(), value, base); + return value; +} + +inline int64_t ToInt64(std::string_view text, int base = 10) { + int64_t value = 0; + std::from_chars(text.data(), text.data() + text.size(), value, base); + return value; +} + +inline double ToDouble(std::string_view text) { + return std::stod(std::string(text)); +} + +inline float ToFloat(std::string_view text) { + return std::stof(std::string(text)); +} + +inline std::string SafeCsv(std::string_view text) { + const bool add_space = + !text.empty() && (text.front() == '+' || text.front() == '=' || text.front() == '-'); + if (text.find('"') != std::string_view::npos || text.find(';') != std::string_view::npos || + text.find('+') != std::string_view::npos || text.find('=') != std::string_view::npos || + text.find('-') != std::string_view::npos) { + return "\"" + std::string(add_space ? " " : "") + + ReplaceStr(std::string(text), "\"", "\"\"") + "\""; + } + return std::string(text); +} + +inline std::string Utf16ToUtf8(const char16_t* utf16) { + std::u16string input(utf16); + std::wstring_convert, char16_t> convert; + return convert.to_bytes(input); +} + +inline ByteBuffer HexToBin(std::string_view text) { + ByteBuffer ret; + for (size_t i = 0; i + 1 < text.size(); i += 2) { + ret.Add(static_cast((ToUint32(text.substr(i, 2), 16) & 0xffu))); + } + return ret; +} + +inline std::string HexFromBin(const ByteBuffer& bin) { + std::string ret; + ret.reserve(bin.Size() * 2); + for (uint32_t i = 0; i < bin.Size(); i++) { + ret += fmt::format("{:02X}", std::to_integer(bin.At(i))); + } + return ret; +} + +} // namespace Common + +#endif /* KYTY_COMMON_STRING_UTILS_H_ */ diff --git a/src/common/subsystems.cpp b/src/common/subsystems.cpp new file mode 100644 index 000000000..6b0d00915 --- /dev/null +++ b/src/common/subsystems.cpp @@ -0,0 +1,232 @@ +#include "common/subsystems.h" + +#include "common/assert.h" + +#include +#include +#include +#include +#include + +namespace Common { + +class SubsystemPrivate { +public: + SubsystemPrivate() = default; + virtual ~SubsystemPrivate() = default; + + KYTY_CLASS_NO_COPY(SubsystemPrivate); + + bool failed {false}; + std::string fail_msg; +}; + +class SubsystemsListPrivate { +public: + explicit SubsystemsListPrivate(SubsystemsList& p): parent(p) {} + + virtual ~SubsystemsListPrivate() = default; + + void SetArgs(int argc, char** argv) { + this->m_argc = argc; + this->m_argv = argv; + } + + void Add(Subsystem* s, std::initializer_list deps) { + EXIT_IF(!s); + + const char* name = s->Id(); + + EXIT_IF(FindByName(name) != nullptr); + + auto nl = std::make_unique(); + + nl->s = s; + nl->name = name; + nl->deps = nullptr; + nl->next = std::move(list); + nl->prev_init = nullptr; + + auto* node = nl.get(); + list = std::move(nl); + + for (auto* dep: deps) { + const char* str = dep->Id(); + + auto l = std::make_unique(); + l->dep_name = str; + l->next = std::move(node->deps); + + node->deps = std::move(l); + } + + node->initialized = false; + } + + bool InitAll(bool print_msg) { + while (SubsListStruct* n = FindNextToInitialize()) { + n->s->Init(&parent); + + if (n->s->m_p->failed) { + fail_msg = n->s->m_p->fail_msg.c_str(); + fail_name = n->name; + return false; + } + + if (print_msg) { + printf("Initialized: %s\n", n->name); + } + + n->initialized = true; + + SubsListStruct* last = last_init; + last_init = n; + n->prev_init = last; + } + + return true; + } + + void DestroyAll(bool print_msg) { + for (SubsListStruct* n = last_init; n != nullptr; n = n->prev_init) { + n->s->Destroy(&parent); + n->initialized = false; + + if (print_msg) { + printf("Destroyed: %s\n", n->name); + } + } + + last_init = nullptr; + } + + void ShutdownAll() { + for (SubsListStruct* n = last_init; n != nullptr; n = n->prev_init) { + n->s->UnexpectedShutdown(&parent); + n->initialized = false; + } + + last_init = nullptr; + } + + struct DepsListStruct { + const char* dep_name; + std::unique_ptr next; + }; + + struct SubsListStruct { + Subsystem* s; + const char* name; + std::unique_ptr deps; + std::unique_ptr next; + SubsListStruct* prev_init; + bool initialized; + }; + + [[nodiscard]] SubsListStruct* FindByName(const char* name) const { + for (SubsListStruct* n = list.get(); n != nullptr; n = n->next.get()) { + if (std::strcmp(n->name, name) == 0) { + return n; + } + } + + return nullptr; + } + + [[nodiscard]] SubsListStruct* FindNextToInitialize() const { + for (SubsListStruct* n = list.get(); n != nullptr; n = n->next.get()) { + if (n->initialized) { + continue; + } + + DepsListStruct* d = n->deps.get(); + for (; d != nullptr; d = d->next.get()) { + SubsListStruct* s = FindByName(d->dep_name); + if ((s == nullptr) || !s->initialized) { + break; + } + } + + if (d == nullptr) { + return n; + } + } + + return nullptr; + } + + KYTY_CLASS_NO_COPY(SubsystemsListPrivate); + + std::unique_ptr list; + SubsListStruct* last_init = nullptr; + int m_argc = 0; + char** m_argv = nullptr; + const char* fail_msg = nullptr; + const char* fail_name = nullptr; + SubsystemsList& parent; +}; + +SubsystemsList::SubsystemsList(): m_p(std::make_unique(*this)) {} + +SubsystemsList::~SubsystemsList() = default; + +void SubsystemsList::Add(Subsystem* s, std::initializer_list deps) { + m_p->Add(s, deps); +} + +bool SubsystemsList::InitAll(bool print_msg) { + return m_p->InitAll(print_msg); +} + +void SubsystemsList::DestroyAll(bool print_msg) { + m_p->DestroyAll(print_msg); +} + +int* SubsystemsList::GetArgc() { + return &m_p->m_argc; +} + +char** SubsystemsList::GetArgv() { + return m_p->m_argv; +} + +Subsystem::Subsystem(): m_p(std::make_unique()) {} + +Subsystem::~Subsystem() = default; + +void Subsystem::Fail(const char* format, ...) { + va_list args {}; + va_start(args, format); + + va_list args_copy {}; + va_copy(args_copy, args); + int len = std::vsnprintf(nullptr, 0, format, args_copy); + va_end(args_copy); + + if (len > 0) { + std::string msg(static_cast(len) + 1, '\0'); + std::vsnprintf(msg.data(), msg.size(), format, args); + m_p->fail_msg = msg.c_str(); + m_p->failed = true; + } + + va_end(args); +} + +const char* SubsystemsList::GetFailName() const { + return m_p->fail_name; +} + +const char* SubsystemsList::GetFailMsg() const { + return m_p->fail_msg; +} + +void SubsystemsList::SetArgs(int argc, char* argv[]) { + m_p->SetArgs(argc, argv); +} + +void SubsystemsList::ShutdownAll() { + m_p->ShutdownAll(); +} + +} // namespace Common diff --git a/src/common/subsystems.h b/src/common/subsystems.h new file mode 100644 index 000000000..d183dfcf2 --- /dev/null +++ b/src/common/subsystems.h @@ -0,0 +1,91 @@ +#ifndef KYTY_COMMON_SUBSYSTEMS_H_ +#define KYTY_COMMON_SUBSYSTEMS_H_ + +#include "common/common.h" +#include "common/singleton.h" + +#include +#include + +namespace Common { + +class Subsystem; +class SubsystemsListPrivate; +class SubsystemPrivate; + +class SubsystemsList { +public: + SubsystemsList(); + virtual ~SubsystemsList(); + void SetArgs(int argc, char* argv[]); + + // void Add(Subsystem* s, const char* name, ...); + void Add(Subsystem* s, std::initializer_list deps); + + bool InitAll(bool print_msg = false); + void DestroyAll(bool print_msg = false); + + int* GetArgc(); + char** GetArgv(); + + [[nodiscard]] const char* GetFailName() const; + [[nodiscard]] const char* GetFailMsg() const; + + void ShutdownAll(); + + static SubsystemsList* Instance() { return Common::Singleton::Instance(); } + + KYTY_CLASS_NO_COPY(SubsystemsList); + +private: + std::unique_ptr m_p; +}; + +using SubsystemsListSingleton = Common::Singleton; + +class Subsystem { +public: + Subsystem(); + virtual ~Subsystem(); + + virtual const char* Id() = 0; + virtual void Init(SubsystemsList* parent) = 0; + virtual void Destroy(SubsystemsList* parent) = 0; + virtual void UnexpectedShutdown(SubsystemsList* parent) = 0; + + friend class SubsystemsListPrivate; + + KYTY_CLASS_NO_COPY(Subsystem); + +protected: + void Fail(const char* format, ...) KYTY_FORMAT_PRINTF(2, 3); + +private: + std::unique_ptr m_p; +}; + +#define KYTY_SUBSYSTEM_DEFINE(s) \ + class s##Subsystem: public Common::Subsystem { \ + public: \ + static Subsystem* Instance() { return Common::Singleton::Instance(); } \ + const char* Id() { return #s; } \ + void Init(Common::SubsystemsList* parent); \ + void Destroy(Common::SubsystemsList* parent); \ + void UnexpectedShutdown(Common::SubsystemsList* parent); \ + }; \ + typedef Common::Singleton s##SubsystemSingleton; + +#define KYTY_SUBSYSTEM_INIT(s) \ + void s##Subsystem::Init([[maybe_unused]] Common::SubsystemsList* parent) +#define KYTY_SUBSYSTEM_DESTROY(s) \ + void s##Subsystem::Destroy([[maybe_unused]] Common::SubsystemsList* parent) +#define KYTY_SUBSYSTEM_UNEXPECTED_SHUTDOWN(s) \ + void s##Subsystem::UnexpectedShutdown([[maybe_unused]] Common::SubsystemsList* parent) + +// #define KYTY_SUBSYSTEM_ADD(list, s, ...) list.push_back(s##SubsystemSingleton::Instance(), #s, +// __VA_ARGS__); #define KYTY_SUBSYSTEM_ADD2(list, ...) list.Add2(s##SubsystemSingleton::Instance(), +// __VA_ARGS__); + +} // namespace Common + +#endif /* KYTY_COMMON_SUBSYSTEMS_H_ */ diff --git a/src/common/systemInfo.cpp b/src/common/systemInfo.cpp new file mode 100644 index 000000000..d7b42dad2 --- /dev/null +++ b/src/common/systemInfo.cpp @@ -0,0 +1,18 @@ +#include "common/systemInfo.h" + +#include "common/assert.h" +#include "cpuinfo.h" + +namespace Common { + +SystemInfo GetSystemInfo() { + static const bool initialized = cpuinfo_initialize(); + EXIT_IF(!initialized); + + const auto* package = cpuinfo_get_package(0); + EXIT_IF(package == nullptr); + + return {package->name}; +} + +} // namespace Common diff --git a/src/common/systemInfo.h b/src/common/systemInfo.h new file mode 100644 index 000000000..454491ea2 --- /dev/null +++ b/src/common/systemInfo.h @@ -0,0 +1,16 @@ +#ifndef KYTY_COMMON_SYSTEM_INFO_H_ +#define KYTY_COMMON_SYSTEM_INFO_H_ + +#include + +namespace Common { + +struct SystemInfo { + std::string ProcessorName; +}; + +[[nodiscard]] SystemInfo GetSystemInfo(); + +} // namespace Common + +#endif /* KYTY_COMMON_SYSTEM_INFO_H_ */ diff --git a/src/common/threads.cpp b/src/common/threads.cpp new file mode 100644 index 000000000..4446cb8a2 --- /dev/null +++ b/src/common/threads.cpp @@ -0,0 +1,419 @@ +#include "common/threads.h" + +#include "common/assert.h" +#include "common/debug.h" + +#include +#include +#include // IWYU pragma: keep +#include // IWYU pragma: keep +#include +#include + +#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS && KYTY_COMPILER == KYTY_COMPILER_CLANG +#define KYTY_WIN_CS +#endif + +#include +#include +#include + +#ifdef KYTY_WIN_CS +#include // IWYU pragma: keep +// IWYU pragma: no_include +constexpr DWORD KYTY_CS_SPIN_COUNT = 4000; + +#ifndef CREATE_WAITABLE_TIMER_HIGH_RESOLUTION +#define CREATE_WAITABLE_TIMER_HIGH_RESOLUTION 0x00000002 +#endif + +static void SleepHighResolution100ns(uint64_t units_100ns) { + if (units_100ns == 0) { + return; + } + + if (units_100ns <= 10000) { + LARGE_INTEGER frequency {}; + LARGE_INTEGER start {}; + if (QueryPerformanceFrequency(&frequency) != 0 && QueryPerformanceCounter(&start) != 0 && + frequency.QuadPart > 0) { + const auto wait_ticks = + static_cast((static_cast(units_100ns) * + static_cast(frequency.QuadPart)) / + 10000000.0L); + const auto deadline = start.QuadPart + std::max(wait_ticks, 1); + LARGE_INTEGER now {}; + do { + if (QueryPerformanceCounter(&now) == 0) { + break; + } + YieldProcessor(); + } while (now.QuadPart < deadline); + return; + } + } + + thread_local HANDLE timer = CreateWaitableTimerExW( + nullptr, nullptr, CREATE_WAITABLE_TIMER_HIGH_RESOLUTION, TIMER_ALL_ACCESS); + if (timer == nullptr) { + thread_local HANDLE fallback_timer = CreateWaitableTimerW(nullptr, TRUE, nullptr); + timer = fallback_timer; + } + + if (timer != nullptr) { + LARGE_INTEGER due_time {}; + due_time.QuadPart = -static_cast(units_100ns); + if (SetWaitableTimerEx(timer, &due_time, 0, nullptr, nullptr, nullptr, 0) != 0) { + WaitForSingleObject(timer, INFINITE); + return; + } + } + + std::this_thread::sleep_for(std::chrono::nanoseconds(units_100ns * 100)); +} + +// IWYU pragma: no_include +// IWYU pragma: no_include +// IWYU pragma: no_include +// IWYU pragma: no_include <__mutex_base> +// IWYU pragma: no_include <__threading_support> +// IWYU pragma: no_include +// IWYU pragma: no_include + +using InitializeConditionVariable_func_t = /*WINBASEAPI*/ VOID WINAPI (*)(PCONDITION_VARIABLE); +using WakeConditionVariable_func_t = /*WINBASEAPI*/ VOID WINAPI (*)(PCONDITION_VARIABLE); +using WakeAllConditionVariable_func_t = /*WINBASEAPI*/ VOID WINAPI (*)(PCONDITION_VARIABLE); +using SleepConditionVariableCS_func_t = /*WINBASEAPI*/ BOOL WINAPI (*)(PCONDITION_VARIABLE, + PCRITICAL_SECTION, DWORD); + +static InitializeConditionVariable_func_t ResolveInitializeConditionVariable() { + if (HMODULE h = GetModuleHandle("KernelBase"); h != nullptr) // @suppress("Invalid arguments") + { + return reinterpret_cast( + GetProcAddress(h, "InitializeConditionVariable")); + } + return nullptr; +} +static WakeConditionVariable_func_t ResolveWakeConditionVariable() { + if (HMODULE h = GetModuleHandle("KernelBase"); h != nullptr) // @suppress("Invalid arguments") + { + return reinterpret_cast( + GetProcAddress(h, "WakeConditionVariable")); + } + return nullptr; +} +static WakeAllConditionVariable_func_t ResolveWakeAllConditionVariable() { + if (HMODULE h = GetModuleHandle("KernelBase"); h != nullptr) // @suppress("Invalid arguments") + { + return reinterpret_cast( + GetProcAddress(h, "WakeAllConditionVariable")); + } + return nullptr; +} +static SleepConditionVariableCS_func_t ResolveSleepConditionVariableCS() { + if (HMODULE h = GetModuleHandle("KernelBase"); h != nullptr) // @suppress("Invalid arguments") + { + return reinterpret_cast( + GetProcAddress(h, "SleepConditionVariableCS")); + } + return nullptr; +} + +#endif + +namespace Common { + +using thread_id_t = std::thread::id; + +struct MutexPrivate { +#ifdef KYTY_WIN_CS + MutexPrivate() { InitializeCriticalSectionAndSpinCount(&m_cs, KYTY_CS_SPIN_COUNT); } + ~MutexPrivate() { DeleteCriticalSection(&m_cs); } + KYTY_CLASS_NO_COPY(MutexPrivate); + CRITICAL_SECTION m_cs {}; +#else + std::recursive_mutex m_mutex; +#endif +}; + +struct CondVarPrivate { +#ifdef KYTY_WIN_CS + CondVarPrivate() { + static auto func = ResolveInitializeConditionVariable(); + EXIT_NOT_IMPLEMENTED(func == nullptr); + func(&m_cv); + } + ~CondVarPrivate() = default; + KYTY_CLASS_NO_COPY(CondVarPrivate); + CONDITION_VARIABLE m_cv {}; +#else + std::condition_variable_any m_cv; +#endif +}; + +static std::recursive_mutex g_cond_waiters_mutex; +static std::vector> g_cond_waiters; +static wait_poll_func_t g_cond_wait_poll_callback = nullptr; + +static void WakeCondVar(CondVarPrivate* cond_var) { +#ifdef KYTY_WIN_CS + static auto func = ResolveWakeAllConditionVariable(); + EXIT_NOT_IMPLEMENTED(func == nullptr); + func(&cond_var->m_cv); +#else + cond_var->m_cv.notify_all(); +#endif +} + +static void RegisterCondWaiter(CondVarPrivate* cond_var) { + std::lock_guard lock(g_cond_waiters_mutex); + g_cond_waiters.emplace_back(Thread::GetThreadIdUnique(), cond_var); +} + +static void UnregisterCondWaiter(CondVarPrivate* cond_var) { + const auto thread_id = Thread::GetThreadIdUnique(); + std::lock_guard lock(g_cond_waiters_mutex); + + const auto it = std::find_if(g_cond_waiters.begin(), g_cond_waiters.end(), + [thread_id, cond_var](const auto& waiter) { + return waiter.first == thread_id && waiter.second == cond_var; + }); + if (it != g_cond_waiters.end()) { + g_cond_waiters.erase(it); + } +} + +struct ThreadPrivate { + ThreadPrivate(thread_func_t f, void* a): func(f), arg(a), m_thread(&Run, this) {} + + static void Run(ThreadPrivate* t) { + t->unique_id = Thread::GetThreadIdUnique(); + t->started = true; + t->func(t->arg); + } + + thread_func_t func; + void* arg; + std::atomic_bool finished = false; + std::atomic_bool auto_delete = false; + std::atomic_bool started = false; + int unique_id = 0; + std::thread m_thread; +}; + +static thread_id_t g_main_thread; +static int g_main_thread_int; +static std::atomic g_thread_counter = 0; + +KYTY_SUBSYSTEM_INIT(Threads) { + g_main_thread = std::this_thread::get_id(); + g_main_thread_int = Thread::GetThreadIdUnique(); +} + +KYTY_SUBSYSTEM_UNEXPECTED_SHUTDOWN(Threads) {} + +KYTY_SUBSYSTEM_DESTROY(Threads) {} + +Thread::Thread(thread_func_t func, void* arg) + : m_thread(std::make_unique(func, arg)) { + while (!m_thread->started) { + Common::Thread::SleepMicro(1000); + } +} + +Thread::~Thread() { + EXIT_IF(!m_thread->finished && !m_thread->auto_delete); + + m_thread.reset(); +} + +void Thread::Join() { + EXIT_IF(m_thread->finished || m_thread->auto_delete); + + m_thread->m_thread.join(); + + m_thread->finished = true; +} + +void Thread::Detach() { + EXIT_IF(m_thread->finished || m_thread->auto_delete); + + m_thread->auto_delete = true; + m_thread->m_thread.detach(); +} + +void Thread::Sleep(uint32_t millis) { + std::this_thread::sleep_for(std::chrono::milliseconds(millis)); +} + +void Thread::SleepMicro(uint32_t micros) { +#ifdef KYTY_WIN_CS + SleepHighResolution100ns(static_cast(micros) * 10); +#else + std::this_thread::sleep_for(std::chrono::microseconds(micros)); +#endif +} + +void Thread::SleepNano(uint64_t nanos) { +#ifdef KYTY_WIN_CS + SleepHighResolution100ns((nanos + 99) / 100); +#else + std::this_thread::sleep_for(std::chrono::nanoseconds(nanos)); +#endif +} + +bool Thread::IsMainThread() { + return g_main_thread == std::this_thread::get_id(); +} + +std::string Thread::GetId() const { + std::stringstream ss; + ss << m_thread->m_thread.get_id(); + return ss.str(); +} + +int Thread::GetUniqueId() const { + return m_thread->unique_id; +} + +std::string Thread::GetThreadId() { + std::stringstream ss; + ss << std::this_thread::get_id(); + return ss.str(); +} + +Mutex::Mutex(): m_mutex(std::make_unique()) {} + +Mutex::~Mutex() { + m_mutex.reset(); +} + +void Mutex::Lock() { +#ifdef KYTY_WIN_CS + EnterCriticalSection(&m_mutex->m_cs); +#else + m_mutex->m_mutex.lock(); +#endif +} + +void Mutex::Unlock() { +#ifdef KYTY_WIN_CS + LeaveCriticalSection(&m_mutex->m_cs); +#else + m_mutex->m_mutex.unlock(); +#endif +} + +bool Mutex::TryLock() { +#ifdef KYTY_WIN_CS + return (TryEnterCriticalSection(&m_mutex->m_cs) != 0); +#else + return m_mutex->m_mutex.try_lock(); +#endif +} + +CondVar::CondVar(): m_cond_var(std::make_unique()) {} + +CondVar::~CondVar() { + m_cond_var.reset(); +} + +void CondVar::Wait(Mutex* mutex) { + RegisterCondWaiter(m_cond_var.get()); +#ifndef KYTY_WIN_CS + std::unique_lock cpp_lock(mutex->m_mutex->m_mutex, std::adopt_lock_t()); +#endif + auto poll_callback = [&] { + auto* callback = g_cond_wait_poll_callback; + if (callback == nullptr) { + return; + } +#if defined(KYTY_WIN_CS) + LeaveCriticalSection(&mutex->m_mutex->m_cs); + callback(); + EnterCriticalSection(&mutex->m_mutex->m_cs); +#else + cpp_lock.unlock(); + callback(); + cpp_lock.lock(); +#endif + }; +#ifdef KYTY_WIN_CS + static auto func = ResolveSleepConditionVariableCS(); + EXIT_NOT_IMPLEMENTED(func == nullptr); + if (g_cond_wait_poll_callback == nullptr) { + func(&m_cond_var->m_cv, &mutex->m_mutex->m_cs, INFINITE); + } else { + if (func(&m_cond_var->m_cv, &mutex->m_mutex->m_cs, 10) == 0 && + GetLastError() == ERROR_TIMEOUT) { + poll_callback(); + } + } +#else + if (g_cond_wait_poll_callback == nullptr) { + m_cond_var->m_cv.wait(cpp_lock); + } else { + if (m_cond_var->m_cv.wait_for(cpp_lock, std::chrono::microseconds(10000)) == + std::cv_status::timeout) { + poll_callback(); + } + } + cpp_lock.release(); +#endif + UnregisterCondWaiter(m_cond_var.get()); +} + +void CondVar::SetWaitPollCallback(wait_poll_func_t callback) { + g_cond_wait_poll_callback = callback; +} + +bool CondVar::WaitFor(Mutex* mutex, uint32_t micros) { + bool ok = false; + RegisterCondWaiter(m_cond_var.get()); +#ifndef KYTY_WIN_CS + std::unique_lock cpp_lock(mutex->m_mutex->m_mutex, std::adopt_lock_t()); +#endif +#ifdef KYTY_WIN_CS + static auto func = ResolveSleepConditionVariableCS(); + EXIT_NOT_IMPLEMENTED(func == nullptr); + ok = !(func(&m_cond_var->m_cv, &mutex->m_mutex->m_cs, (micros < 1000 ? 1 : micros / 1000)) == + 0 && + GetLastError() == ERROR_TIMEOUT); +#else + ok = (m_cond_var->m_cv.wait_for(cpp_lock, std::chrono::microseconds(micros)) == + std::cv_status::no_timeout); + cpp_lock.release(); +#endif + UnregisterCondWaiter(m_cond_var.get()); + return ok; +} + +void CondVar::Signal() { +#ifdef KYTY_WIN_CS + static auto func = ResolveWakeConditionVariable(); + EXIT_NOT_IMPLEMENTED(func == nullptr); + func(&m_cond_var->m_cv); +#else + m_cond_var->m_cv.notify_one(); +#endif +} + +void CondVar::SignalAll() { + WakeCondVar(m_cond_var.get()); +} + +void CondVar::SignalThread(int thread_id) { + std::lock_guard lock(g_cond_waiters_mutex); + for (const auto& waiter: g_cond_waiters) { + if (waiter.first == thread_id) { + WakeCondVar(waiter.second); + } + } +} + +int Thread::GetThreadIdUnique() { + static thread_local int tid = ++g_thread_counter; + return tid; +} + +} // namespace Common diff --git a/src/common/threads.h b/src/common/threads.h new file mode 100644 index 000000000..5f0c222ee --- /dev/null +++ b/src/common/threads.h @@ -0,0 +1,107 @@ +#ifndef KYTY_COMMON_THREADS_H_ +#define KYTY_COMMON_THREADS_H_ + +#include "common/common.h" +#include "common/subsystems.h" + +#include +#include + +namespace Common { + +KYTY_SUBSYSTEM_DEFINE(Threads); + +using thread_func_t = void (*)(void*); +using wait_poll_func_t = void (*)(); + +struct ThreadPrivate; +struct MutexPrivate; +struct CondVarPrivate; + +class Thread { +public: + Thread(thread_func_t func, void* arg); + ~Thread(); + + void Join(); + void Detach(); + + // Once a thread has finished, the id may be reused by another thread. + [[nodiscard]] std::string GetId() const; + + // The id is unique and can't be reused by another thread. + [[nodiscard]] int GetUniqueId() const; + + static void Sleep(uint32_t millis); + static void SleepMicro(uint32_t micros); + static void SleepNano(uint64_t nanos); + static bool IsMainThread(); + + // Get current thread id + // Once a thread has finished, the id may be reused by another thread. + static std::string GetThreadId(); + + // Get current thread id + // The id is unique and can't be reused by another thread. + static int GetThreadIdUnique(); + + KYTY_CLASS_NO_COPY(Thread); + +private: + std::unique_ptr m_thread; +}; + +class Mutex { +public: + Mutex(); + ~Mutex(); + + void Lock(); + void Unlock(); + bool TryLock(); + + friend class CondVar; + + KYTY_CLASS_NO_COPY(Mutex); + +private: + std::unique_ptr m_mutex; +}; + +class CondVar { +public: + CondVar(); + ~CondVar(); + + void Wait(Mutex* mutex); + bool WaitFor(Mutex* mutex, uint32_t micros); + void Signal(); + void SignalAll(); + + static void SignalThread(int thread_id); + static void SetWaitPollCallback(wait_poll_func_t callback); + + KYTY_CLASS_NO_COPY(CondVar); + +private: + std::unique_ptr m_cond_var; +}; + +class LockGuard { +public: + using mutex_type = Mutex; + + // NOLINTNEXTLINE(google-runtime-references) + explicit LockGuard(mutex_type& m): m_mutex(m) { m_mutex.Lock(); } + + ~LockGuard() { m_mutex.Unlock(); } + + KYTY_CLASS_NO_COPY(LockGuard); + +private: + mutex_type& m_mutex; +}; + +} // namespace Common + +#endif /* KYTY_COMMON_THREADS_H_ */ diff --git a/src/common/timer.cpp b/src/common/timer.cpp new file mode 100644 index 000000000..683a1641c --- /dev/null +++ b/src/common/timer.cpp @@ -0,0 +1,94 @@ +#include "common/timer.h" + +#include "common/assert.h" +#include "common/platform/sysTimer.h" + +namespace Common { + +Timer::Timer() noexcept { + SysQueryPerformanceFrequency(&m_Frequency); +} + +void Timer::Start() { + SysQueryPerformanceCounter(&m_StartTime); + m_is_paused = false; +} + +void Timer::Pause() { + EXIT_IF(m_is_paused); + + SysQueryPerformanceCounter(&m_PauseTime); + m_is_paused = true; +} + +void Timer::Resume() { + EXIT_IF(!m_is_paused); + + uint64_t current_time = 0; + SysQueryPerformanceCounter(¤t_time); + + m_StartTime += current_time - m_PauseTime; + + m_is_paused = false; +} + +bool Timer::IsPaused() const { + return m_is_paused; +} + +// return time in milliseconds +double Timer::GetTimeMs() const { + if (m_is_paused) { + return 1000.0 * (static_cast(m_PauseTime - m_StartTime)) / + static_cast(m_Frequency); + } + + uint64_t current_time = 0; + SysQueryPerformanceCounter(¤t_time); + + return 1000.0 * (static_cast(current_time - m_StartTime)) / + static_cast(m_Frequency); +} + +// return time in seconds +double Timer::GetTimeS() const { + if (m_is_paused) { + return (static_cast(m_PauseTime - m_StartTime)) / static_cast(m_Frequency); + } + + uint64_t current_time = 0; + SysQueryPerformanceCounter(¤t_time); + + return (static_cast(current_time - m_StartTime)) / static_cast(m_Frequency); +} + +// return time in ticks +uint64_t Timer::GetTicks() const { + if (m_is_paused) { + return (m_PauseTime - m_StartTime); + } + + uint64_t current_time = 0; + SysQueryPerformanceCounter(¤t_time); + + return (current_time - m_StartTime); +} + +// return ticks frequency +uint64_t Timer::GetFrequency() const { + return m_Frequency; +} + +uint64_t Timer::QueryPerformanceFrequency() { + uint64_t ret = 0; + SysQueryPerformanceFrequency(&ret); + return ret; +} + +uint64_t Timer::QueryPerformanceCounter() { + uint64_t ret = 0; + SysQueryPerformanceCounter(&ret); + return ret; +} + +} // namespace Common diff --git a/src/common/timer.h b/src/common/timer.h new file mode 100644 index 000000000..ac192a69b --- /dev/null +++ b/src/common/timer.h @@ -0,0 +1,47 @@ +#ifndef KYTY_COMMON_TIMER_H_ +#define KYTY_COMMON_TIMER_H_ + +#include "common/common.h" + +namespace Common { + +class Timer final { +public: + Timer() noexcept; + ~Timer() = default; + + void Start(); + + void Pause(); + + void Resume(); + + [[nodiscard]] bool IsPaused() const; + + // return time in milliseconds + [[nodiscard]] double GetTimeMs() const; + + // return time in seconds + [[nodiscard]] double GetTimeS() const; + + // return time in ticks + [[nodiscard]] uint64_t GetTicks() const; + + // return ticks frequency + [[nodiscard]] uint64_t GetFrequency() const; + + KYTY_CLASS_NO_COPY(Timer); + + [[nodiscard]] static uint64_t QueryPerformanceFrequency(); + [[nodiscard]] static uint64_t QueryPerformanceCounter(); + +private: + bool m_is_paused = true; + uint64_t m_Frequency = 0; + uint64_t m_StartTime = 0; + uint64_t m_PauseTime = 0; +}; + +} // namespace Common + +#endif /* KYTY_COMMON_TIMER_H_ */ diff --git a/src/common/virtualMemory.cpp b/src/common/virtualMemory.cpp new file mode 100644 index 000000000..99af5c8e2 --- /dev/null +++ b/src/common/virtualMemory.cpp @@ -0,0 +1,82 @@ +#include "common/virtualMemory.h" + +#include "common/platform/sysVirtual.h" + +namespace Common { + +namespace VirtualMemory { + +void Init() { + SysVirtualInit(); +} + +uint64_t Alloc(uint64_t address, uint64_t size, Mode mode) { + return SysVirtualAlloc(address, size, mode); +} + +uint64_t AllocAligned(uint64_t address, uint64_t size, Mode mode, uint64_t alignment) { + return SysVirtualAllocAligned(address, size, mode, alignment); +} + +bool AllocFixed(uint64_t address, uint64_t size, Mode mode) { + return SysVirtualAllocFixed(address, size, mode); +} + +bool Commit(uint64_t address, uint64_t size, Mode mode) { + return SysVirtualCommit(address, size, mode); +} + +uint64_t Reserve(uint64_t address, uint64_t size) { + return SysVirtualReserve(address, size); +} + +uint64_t ReserveAligned(uint64_t address, uint64_t size, uint64_t alignment) { + return SysVirtualReserveAligned(address, size, alignment); +} + +bool ReserveFixed(uint64_t address, uint64_t size) { + return SysVirtualReserveFixed(address, size); +} + +bool Decommit(uint64_t address, uint64_t size) { + return SysVirtualDecommit(address, size); +} + +bool Free(uint64_t address) { + return SysVirtualFree(address); +} + +bool FreeRange(uint64_t address, uint64_t size) { + return SysVirtualFreeRange(address, size); +} + +bool Protect(uint64_t address, uint64_t size, Mode mode, Mode* old_mode) { + return SysVirtualProtect(address, size, mode, old_mode); +} + +bool FlushInstructionCache(uint64_t address, uint64_t size) { + return SysVirtualFlushInstructionCache(address, size); +} + +bool PatchReplace(uint64_t vaddr, uint64_t value) { + Mode old_mode {}; + Protect(vaddr, 8, Mode::ReadWrite, &old_mode); + + auto* ptr = reinterpret_cast(vaddr); + + bool ret = (*ptr != value); + + *ptr = value; + + Protect(vaddr, 8, old_mode); + + if (IsExecute(old_mode)) { + FlushInstructionCache(vaddr, 8); + } + + return ret; +} + +} // namespace VirtualMemory + +} // namespace Common diff --git a/src/common/virtualMemory.h b/src/common/virtualMemory.h new file mode 100644 index 000000000..a36d596ce --- /dev/null +++ b/src/common/virtualMemory.h @@ -0,0 +1,46 @@ +#ifndef KYTY_COMMON_VIRTUALMEMORY_H_ +#define KYTY_COMMON_VIRTUALMEMORY_H_ + +#include "common/common.h" + +namespace Common { + +namespace VirtualMemory { + +enum class Mode : uint32_t { + NoAccess = 0, + Read = 1, + Write = 2, + ReadWrite = Read | Write, + Execute = 4, + ExecuteRead = Execute | Read, + ExecuteWrite = Execute | Write, + ExecuteReadWrite = Execute | Read | Write, +}; + +inline bool IsExecute(Mode mode) { + return (mode == Mode::Execute || mode == Mode::ExecuteRead || mode == Mode::ExecuteWrite || + mode == Mode::ExecuteReadWrite); +} + +void Init(); + +uint64_t Alloc(uint64_t address, uint64_t size, Mode mode); +uint64_t AllocAligned(uint64_t address, uint64_t size, Mode mode, uint64_t alignment); +bool AllocFixed(uint64_t address, uint64_t size, Mode mode); +bool Commit(uint64_t address, uint64_t size, Mode mode); +uint64_t Reserve(uint64_t address, uint64_t size); +uint64_t ReserveAligned(uint64_t address, uint64_t size, uint64_t alignment); +bool ReserveFixed(uint64_t address, uint64_t size); +bool Decommit(uint64_t address, uint64_t size); +bool Free(uint64_t address); +bool FreeRange(uint64_t address, uint64_t size); +bool Protect(uint64_t address, uint64_t size, Mode mode, Mode* old_mode = nullptr); +bool FlushInstructionCache(uint64_t address, uint64_t size); +bool PatchReplace(uint64_t vaddr, uint64_t value); + +} // namespace VirtualMemory + +} // namespace Common + +#endif /* KYTY_COMMON_VIRTUALMEMORY_H_ */ diff --git a/src/emulator.cpp b/src/emulator.cpp new file mode 100644 index 000000000..5ef3a79ff --- /dev/null +++ b/src/emulator.cpp @@ -0,0 +1,223 @@ +#include "emulator.h" + +#include "common/abi.h" +#include "common/assert.h" +#include "common/commonSubsystem.h" +#include "common/emulatorConfig.h" +#include "common/file.h" +#include "common/logging/log.h" +#include "common/profiler.h" +#include "common/singleton.h" +#include "common/stringUtils.h" +#include "common/subsystems.h" +#include "common/systemInfo.h" +#include "common/threads.h" +#include "graphics/presentation/window.h" +#include "kernel/fileSystem.h" +#include "kernel/memory.h" +#include "kernel/pthread.h" +#include "libs/agc.h" +#include "libs/audio.h" +#include "libs/controller.h" +#include "libs/libs.h" +#include "libs/network.h" +#include "loader/runtimeLinker.h" +#include "loader/systemContent.h" +#include "loader/timer.h" + +#include +#include + +namespace Emulator { + +static void PrintSystemInfo() { + Common::SystemInfo info = Common::GetSystemInfo(); + + LOGF("ProcessorName = %s\n", info.ProcessorName.c_str()); +} + +static void KytyClose() { + auto* rt = Common::Singleton::Instance(); + + rt->Clear(); + + LOGF("done!\n"); + + Common::SubsystemsListSingleton::Instance()->ShutdownAll(); +} + +static void MountOrCreateDir(const std::filesystem::path& dir, const std::string& point) { + if (!Common::File::IsDirectoryExisting(dir)) { + Common::File::CreateDirectories(dir); + } + + EXIT_NOT_IMPLEMENTED(!Common::File::IsDirectoryExisting(dir)); + + Libs::LibKernel::FileSystem::Mount(dir, point); + auto dir_text = Common::PathToString(dir); + LOGF("Mounted %s -> %s\n", point.c_str(), dir_text.c_str()); +} + +static void MountSandboxDirs() { + std::string title_id; + if (!Loader::SystemContentParamSfoGetString("TITLE_ID", &title_id) || title_id.empty()) { + title_id = "UNKNOWN"; + } + + MountOrCreateDir("_DownloadData/" + title_id, "/download0"); + MountOrCreateDir("_TempData/" + title_id, "/temp0"); + MountOrCreateDir("_TempData/" + title_id, "/temp"); +} + +static bool ClearDirectoryContents(const std::filesystem::path& dir) { + bool ok = true; + + for (const auto& entry: Common::File::GetDirEntries(dir)) { + if (entry.name == "." || entry.name == "..") { + continue; + } + + auto path = dir / entry.name; + + if (entry.is_file) { + Common::File::RemoveReadonly(path); + ok = Common::File::DeleteFile(path) && ok; + } else { + ok = ClearDirectoryContents(path) && ok; + ok = Common::File::DeleteDirectory(path) && ok; + } + } + + return ok; +} + +static void ClearDebugTextureFolder() { + const std::string debug_texture_folder = "_Textures"; + + if (!Common::File::IsDirectoryExisting(debug_texture_folder)) { + Common::File::CreateDirectories(debug_texture_folder); + return; + } + + if (!ClearDirectoryContents(debug_texture_folder)) { + LOGF_COLOR(Log::Color::BrightYellow, "TextureDump: failed to completely clear %s\n", + debug_texture_folder.c_str()); + } +} + +static void Init(const Config::ConfigOptions& cfg) { + EXIT_IF(!Common::Thread::IsMainThread()); + + auto* slist = Common::SubsystemsList::Instance(); + + auto* audio = Libs::Audio::AudioSubsystem::Instance(); + auto* config = Config::ConfigSubsystem::Instance(); + auto* controller = Libs::Controller::ControllerSubsystem::Instance(); + auto* core = Common::CommonSubsystem::Instance(); + auto* file_system = Libs::LibKernel::FileSystem::FileSystemSubsystem::Instance(); + auto* graphics = Libs::Graphics::GraphicsSubsystem::Instance(); + auto* log = Log::LogSubsystem::Instance(); + auto* memory = Libs::LibKernel::Memory::MemorySubsystem::Instance(); + auto* network = Libs::Network::NetworkSubsystem::Instance(); + auto* profiler = Profiler::ProfilerSubsystem::Instance(); + auto* pthread = Libs::LibKernel::PthreadSubsystem::Instance(); + auto* timer = Loader::Timer::TimerSubsystem::Instance(); + + slist->Add(config, {core}); + slist->InitAll(true); + + Config::Load(cfg); + + slist->Add(audio, {core, log, pthread, memory}); + slist->Add(controller, {core, log, config}); + slist->Add(file_system, {core, log, pthread}); + slist->Add(graphics, {core, log, pthread, memory, config, profiler, controller}); + slist->Add(log, {core, config}); + slist->Add(memory, {core, log}); + slist->Add(network, {core, log, pthread}); + slist->Add(profiler, {core, config}); + slist->Add(pthread, {core, log, timer}); + slist->Add(timer, {core, log}); + + slist->InitAll(true); +} + +static void LoadElf(const std::filesystem::path& elf, bool dbg_print_reloc = false, + const std::filesystem::path& save_name = {}) { + auto* rt = Common::Singleton::Instance(); + + auto* program = rt->LoadProgram( + Libs::LibKernel::FileSystem::GetRealFilename(Common::PathToGenericString(elf))); + + if (dbg_print_reloc) { + program->dbg_print_reloc = true; + } + + if (!save_name.empty()) { + rt->SaveProgram(program, Libs::LibKernel::FileSystem::GetRealFilename( + Common::PathToGenericString(save_name))); + } +} + +static void Execute() { + int thread_model = 1; + + if (thread_model == 0) { + Common::Thread t([](void* /*unused*/) { Libs::Graphics::WindowRun(); }, nullptr); + t.Detach(); + auto* rt = Common::Singleton::Instance(); + rt->Execute(); + } else { + Common::Thread t( + [](void* /*unused*/) { + auto* rt = Common::Singleton::Instance(); + rt->Execute(); + }, + nullptr); + t.Detach(); + Libs::Graphics::WindowRun(); + t.Join(); + } +} + +void Run(const RunOptions& options) { + if (options.app0_dir.empty()) { + EXIT("app0 directory is required\n"); + } + + if (options.elf.empty()) { + EXIT("ELF is required\n"); + } + + Init(options.config); + + ClearDebugTextureFolder(); + + PrintSystemInfo(); + + int ok = atexit(KytyClose); + EXIT_NOT_IMPLEMENTED(ok != 0); + + Libs::LibKernel::FileSystem::Mount(options.app0_dir, "/app0"); + Libs::LibKernel::FileSystem::Mount(options.app0_dir, "/hostapp"); + + auto param_json = options.app0_dir / "sce_sys" / "param.json"; + if (Common::File::IsFileExisting(param_json)) { + Loader::SystemContentLoadParamSfo(param_json); + if (auto flexible_memory_size = Loader::SystemContentGetFlexibleMemorySize(); + flexible_memory_size != 0) { + Libs::LibKernel::Memory::SetFlexibleMemorySize(flexible_memory_size); + } + } + + MountSandboxDirs(); + + auto* rt = Common::Singleton::Instance(); + Libs::InitAll(rt->Symbols()); + + LoadElf(options.elf); + + Execute(); +} + +} // namespace Emulator diff --git a/src/emulator.h b/src/emulator.h new file mode 100644 index 000000000..fab32fe82 --- /dev/null +++ b/src/emulator.h @@ -0,0 +1,21 @@ +#ifndef EMULATOR_INCLUDE_EMULATOR_EMULATOR_H_ +#define EMULATOR_INCLUDE_EMULATOR_EMULATOR_H_ + +#include "common/emulatorConfig.h" +#include "common/stringUtils.h" + +#include + +namespace Emulator { + +struct RunOptions { + Config::ConfigOptions config; + std::filesystem::path app0_dir; + std::filesystem::path elf; +}; + +void Run(const RunOptions& options); + +} // namespace Emulator + +#endif /* EMULATOR_INCLUDE_EMULATOR_EMULATOR_H_ */ diff --git a/src/generate_version.cmake b/src/generate_version.cmake new file mode 100644 index 000000000..04d814cbc --- /dev/null +++ b/src/generate_version.cmake @@ -0,0 +1,8 @@ +#message("${GIT_EXECUTABLE}") +if(GIT_EXECUTABLE) + execute_process(COMMAND ${GIT_EXECUTABLE} describe --tags --always OUTPUT_VARIABLE KYTY_GIT_VERSION) +else() + set(KYTY_GIT_VERSION "unknown") +endif() +string(STRIP ${KYTY_GIT_VERSION} KYTY_GIT_VERSION) +configure_file(${INPUT_FILE} ${OUTPUT_FILE}) diff --git a/src/graphics/asyncJob.h b/src/graphics/asyncJob.h new file mode 100644 index 000000000..1ea110d14 --- /dev/null +++ b/src/graphics/asyncJob.h @@ -0,0 +1,96 @@ +#ifndef EMULATOR_INCLUDE_EMULATOR_GRAPHICS_ASYNCJOB_H_ +#define EMULATOR_INCLUDE_EMULATOR_GRAPHICS_ASYNCJOB_H_ + +#include "common/common.h" +#include "common/profiler.h" +#include "common/threads.h" + +#include +#include +#include + +namespace Libs::Graphics { + +class AsyncJob final { +public: + using Task = std::function; + + explicit AsyncJob(std::string thread_name = {}) + : m_thread_name(std::move(thread_name)), m_worker_thread(WorkerEntry, this) {} + + ~AsyncJob() { + { + Common::LockGuard lock(m_mutex); + m_stop_requested = true; + m_task_available_condition.Signal(); + } + + m_worker_thread.Join(); + } + + KYTY_CLASS_NO_COPY(AsyncJob); + + void Execute(Task task) { + Common::LockGuard lock(m_mutex); + while (m_is_busy) { + m_idle_condition.Wait(&m_mutex); + } + + m_task = std::move(task); + m_is_busy = true; + m_task_available_condition.Signal(); + } + + void Wait() { + Common::LockGuard lock(m_mutex); + while (m_is_busy) { + m_idle_condition.Wait(&m_mutex); + } + } + +private: + static void WorkerEntry(void* data) { static_cast(data)->WorkerLoop(); } + + void WorkerLoop() { + if (!m_thread_name.empty()) { + KYTY_PROFILER_THREAD(m_thread_name.c_str()); + } + + for (;;) { + Task task; + { + Common::LockGuard lock(m_mutex); + while (!m_is_busy && !m_stop_requested) { + m_task_available_condition.Wait(&m_mutex); + } + + if (!m_is_busy) { + return; + } + + task = std::move(m_task); + } + + task(); + + { + Common::LockGuard lock(m_mutex); + m_is_busy = false; + m_idle_condition.SignalAll(); + } + } + } + + std::string m_thread_name; + Common::Mutex m_mutex; + Common::CondVar m_task_available_condition; + Common::CondVar m_idle_condition; + Task m_task; + bool m_is_busy = false; + bool m_stop_requested = false; + Common::Thread m_worker_thread; +}; + +} // namespace Libs::Graphics + +#endif /* EMULATOR_INCLUDE_EMULATOR_GRAPHICS_ASYNCJOB_H_ */ diff --git a/src/graphics/guest_gpu/command_processor/commandProcessor.h b/src/graphics/guest_gpu/command_processor/commandProcessor.h new file mode 100644 index 000000000..a202f1484 --- /dev/null +++ b/src/graphics/guest_gpu/command_processor/commandProcessor.h @@ -0,0 +1,268 @@ +#ifndef GRAPHICS_GUEST_GPU_COMMAND_PROCESSOR_COMMAND_PROCESSOR_H +#define GRAPHICS_GUEST_GPU_COMMAND_PROCESSOR_COMMAND_PROCESSOR_H + +#include "common/assert.h" +#include "common/threads.h" +#include "graphics/guest_gpu/hardwareContext.h" +#include "graphics/host_gpu/renderer/render.h" +#include "graphics/host_gpu/renderer/renderContext.h" + +#include +#include + +namespace Libs::Graphics { + +class CommandProcessorGuestAccessScope; + +inline constexpr uint32_t AcquireGcrGl2Writeback = 1u << 15u; + +bool TestWaitRegMemValue(uint64_t value, uint64_t ref, uint64_t mask, uint32_t func); + +class CommandScheduler { +public: + static constexpr int BuffersNum = 8; + + void SetQueue(int queue) { m_queue = queue; } + int Queue() const { return m_queue; } + bool Active() const { return m_current >= 0 && m_current < BuffersNum; } + void CheckActive() const { EXIT_IF(!Active()); } + + CommandBuffer* Current() const { + CheckActive(); + EXIT_IF(m_buffers[m_current] == nullptr); + return m_buffers[m_current]; + } + + void Init() { + if (m_current >= 0) { + return; + } + for (auto& buf: m_buffers) { + EXIT_IF(buf != nullptr); + buf = new CommandBuffer(m_queue); + } + m_current = 0; + Current()->Begin(); + } + + void Flush() { + SubmitCurrent(); + BeginNext(); + } + + CommandBuffer* FlushAndGetSubmitted() { + auto* submitted = SubmitCurrent(); + BeginNext(); + return submitted; + } + + void CopyBuffers(std::array* out) const { + EXIT_IF(out == nullptr); + for (int i = 0; i < BuffersNum; i++) { + auto* buf = m_buffers[i]; + EXIT_IF(buf == nullptr); + (*out)[i] = buf; + } + } + + void WaitAll() { + for (auto* buf: m_buffers) { + EXIT_IF(buf == nullptr); + buf->WaitForFenceAndReset(); + } + } + +private: + CommandBuffer* SubmitCurrent() { + auto* submitted = Current(); + submitted->End(); + submitted->Execute(); + return submitted; + } + + void BeginNext() { + m_current = (m_current + 1) % BuffersNum; + Current()->WaitForFenceAndReset(); + Current()->Begin(); + } + + CommandBuffer* m_buffers[BuffersNum] = {}; + int m_current = -1; + int m_queue = -1; +}; + +class CommandProcessor { +public: + struct FlipInfo { + int handle = 0; + int index = 0; + int flip_mode = 0; + int64_t flip_arg = 0; + }; + + CommandProcessor() = default; + ~CommandProcessor() { KYTY_NOT_IMPLEMENTED; } + + KYTY_CLASS_NO_COPY(CommandProcessor); + + void Reset(); + + void BufferInit(); + void BufferFlush(); + void BufferFlushAndWait(); + void BufferWait(); + [[nodiscard]] int BeginReadbackTransaction(bool mutex_preowned) { + if (!mutex_preowned) { + m_mutex.Lock(); + } + m_scheduler.Flush(); + m_scheduler.WaitAll(); + return m_scheduler.Queue(); + } + void EndReadbackTransaction(bool mutex_preowned) { + if (!mutex_preowned) { + m_mutex.Unlock(); + } + } + + void RunLock() { m_run_mutex.Lock(); } + void RunUnlock() { m_run_mutex.Unlock(); } + + HW::Context* GetCtx() { return &m_ctx; } + HW::UserConfig* GetUcfg() { return &m_ucfg; } + HW::Shader* GetShCtx() { return &m_sh_ctx; } + + void SetIndexType(uint32_t index_type_and_size); + void SetIndexBaseAddress(uint64_t index_base_addr); + void SetIndexBufferSize(uint32_t index_buffer_size); + void SetDrawIndirectArgsBaseAddress(uint64_t draw_indirect_args_base_addr); + void SetDispatchIndirectArgsBaseAddress(uint64_t dispatch_indirect_args_base_addr); + void SetNumInstances(uint32_t num_instances); + void DrawIndex(uint32_t index_count, const void* index_addr, uint32_t flags, uint32_t type, + uint32_t instance_count = 0, const void* object_ids = nullptr, + uint32_t render_target_slice_offset = 0, int32_t vertex_offset_add = 0, + uint32_t first_instance = 0); + void DrawIndexOffset(uint32_t index_offset, uint32_t index_count, uint32_t flags); + void DrawIndexAuto(uint32_t index_count, uint32_t flags, + uint32_t render_target_slice_offset = 0, uint32_t instance_count = 1, + uint32_t first_vertex = 0, uint32_t first_instance = 0); + void DrawIndirect(uint32_t data_offset, uint32_t draw_initiator, bool indexed); + void DrawIndirectMulti(uint32_t data_offset, uint32_t max_count_or_count, + const volatile uint32_t* count_addr, uint32_t stride_in_bytes, + uint32_t draw_initiator, bool indexed); + void WriteAtEndOfPipe32(uint32_t cache_policy, uint32_t event_write_dest, + uint32_t eop_event_type, uint32_t cache_action, uint32_t event_index, + uint32_t event_write_source, void* dst_gpu_addr, uint32_t value, + uint32_t interrupt_selector, uint32_t interrupt_context_id = 0); + void WriteAtEndOfPipe64(uint32_t cache_policy, uint32_t event_write_dest, + uint32_t eop_event_type, uint32_t cache_action, uint32_t event_index, + uint32_t event_write_source, void* dst_gpu_addr, uint64_t value, + uint32_t interrupt_selector, uint32_t interrupt_context_id = 0); + void Flip(); + void Flip(void* dst_gpu_addr, uint32_t value); + void FlipWithInterrupt(uint32_t eop_event_type, uint32_t cache_action, void* dst_gpu_addr, + uint32_t value); + void SynchronizeGpu(); + void SynchronizeGpuRange(uint64_t vaddr, uint64_t size); + void MemoryBarrier(); + void TriggerEopEventAtEndOfPipe(uint32_t interrupt_context_id); + void RenderTextureBarrier(uint64_t vaddr, uint64_t size); + void DepthStencilBarrier(uint64_t vaddr, uint64_t size); + void DispatchDirect(uint32_t thread_group_x, uint32_t thread_group_y, uint32_t thread_group_z, + uint32_t mode); + void DispatchIndirect(uint32_t data_offset, uint32_t mode); + void WaitFlipDone(uint32_t video_out_handle, uint32_t display_buffer_index); + void TriggerEvent(uint32_t event_type, uint32_t event_index); + + void SetUserDataMarker(HW::UserSgprType type) { m_user_data_marker = type; } + [[nodiscard]] HW::UserSgprType GetUserDataMarker() const { return m_user_data_marker; } + void SetEmbeddedDataMarker(const uint32_t* buffer, uint32_t num_dw, uint32_t align) {} + void PushMarker(const char* str) {} + void PopMarker() {} + + void PrefetchL2(void* addr, uint32_t size) {} + void ClearGds(uint64_t dw_offset, uint32_t dw_num, uint32_t clear_value); + void ReadGds(uint32_t* dst, uint32_t dw_offset, uint32_t dw_size); + + void ResetDeCe(); + void WaitCe(); + void WaitDeDiff(uint32_t diff); + void IncremenetDe(); + void IncremenetCe(); + + void WriteConstRam(uint32_t offset, const uint32_t* src, uint32_t dw_num); + void DumpConstRam(uint32_t* dst, uint32_t offset, uint32_t dw_num); + + template + void WaitRegMem(uint32_t func, const T* addr, T ref, T mask, uint32_t poll, uint32_t wait_op); + void WriteData(uint32_t* dst, const uint32_t* src, uint32_t dw_num, uint32_t write_control); + void DmaData(uint8_t engine, uint8_t dst_sel, uint8_t dst_cache_policy, + uint64_t dst_address_or_offset, uint8_t src_sel, uint8_t src_cache_policy, + uint64_t src_address_or_offset_or_immediate, uint32_t num_bytes, + uint8_t wait_for_previous, uint8_t write_confirm, uint8_t block_engine); + void SetPredication(uint32_t condition, uint32_t op, uint32_t wait_op, + const volatile void* address, uint32_t count_in_dwords); + [[nodiscard]] bool ShouldSkipPredicatedPackets() const { return m_predicate_skip; } + + void Run(uint32_t* data, uint32_t num_dw); + + void SetQueue(int queue) { m_scheduler.SetQueue(queue); } + [[nodiscard]] int GetQueue() const { return m_scheduler.Queue(); } + + [[nodiscard]] const FlipInfo& GetFlip() const { return m_flip; } + void SetFlip(const FlipInfo& flip) { m_flip = flip; } + + [[nodiscard]] uint64_t GetSubmitId() const { return m_submit_id; } + void SetSubmitId(uint64_t submit_id) { m_submit_id = submit_id; } + +private: + friend class CommandProcessorGuestAccessScope; + + struct Counter { + Common::Mutex mutex; + Common::CondVar cond_var; + uint32_t value = 0; + }; + + template + void WriteAtEndOfPipe(uint32_t cache_policy, uint32_t event_write_dest, uint32_t eop_event_type, + uint32_t cache_action, uint32_t event_index, uint32_t event_write_source, + void* dst_gpu_addr, T value, uint32_t interrupt_selector, + uint32_t interrupt_context_id); + + CommandBuffer* CurrentBuffer() { return m_scheduler.Current(); } + void CheckBuffer() const { m_scheduler.CheckActive(); } + GpuResourceManager* GetGpuResources() const { + EXIT_IF(g_render_ctx == nullptr); + return g_render_ctx->GetGpuResources(); + } + + HW::Context m_ctx; + HW::UserConfig m_ucfg; + HW::Shader m_sh_ctx; + HW::UserSgprType m_user_data_marker = HW::UserSgprType::Unknown; + uint32_t m_index_type_and_size = 0; + uint32_t m_index_buffer_size = 0; + uint64_t m_index_base_addr = 0; + uint64_t m_draw_indirect_args_base_addr = 0; + uint64_t m_dispatch_indirect_args_base_addr = 0; + uint32_t m_num_instances = 1; + + Common::Mutex m_mutex; + Common::Mutex m_run_mutex; + + CommandScheduler m_scheduler; + + Counter m_de_counter; + Counter m_ce_counter; + + uint32_t m_const_ram[0x3000] = {0}; + + FlipInfo m_flip; + uint64_t m_submit_id = 0; + bool m_predicate_skip = false; +}; + +} // namespace Libs::Graphics + +#endif // GRAPHICS_GUEST_GPU_COMMAND_PROCESSOR_COMMAND_PROCESSOR_H diff --git a/src/graphics/guest_gpu/command_processor/pm4Dispatch.cpp b/src/graphics/guest_gpu/command_processor/pm4Dispatch.cpp new file mode 100644 index 000000000..8cb888452 --- /dev/null +++ b/src/graphics/guest_gpu/command_processor/pm4Dispatch.cpp @@ -0,0 +1,260 @@ +#include "graphics/guest_gpu/command_processor/pm4Dispatch.h" + +namespace Libs::Graphics { + +hw_ctx_parser_func_t g_hw_ctx_func[Pm4::CX_NUM] = {}; +hw_ctx_indirect_func_t g_hw_ctx_indirect_func[Pm4::CX_NUM] = {}; +hw_sh_parser_func_t g_hw_sh_func[Pm4::SH_NUM] = {}; +hw_sh_indirect_func_t g_hw_sh_indirect_func[Pm4::SH_NUM] = {}; +hw_uc_parser_func_t g_hw_uc_func[Pm4::UC_NUM] = {}; +hw_uc_indirect_func_t g_hw_uc_indirect_func[Pm4::UC_NUM] = {}; +cp_op_parser_func_t g_cp_op_func[256] = {}; +cp_op_parser_func_t g_cp_op_custom_func[Pm4::R_NUM] = {}; + +void GraphicsInitJmpTables() { + for (auto& func: g_hw_ctx_func) { + func = nullptr; + } + + g_hw_ctx_func[Pm4::DB_RENDER_CONTROL] = HwCtxSetRenderControl; + g_hw_ctx_func[Pm4::DB_COUNT_CONTROL] = HwCtxSetDepthMetadataRegisters; + g_hw_ctx_func[Pm4::DB_RENDER_OVERRIDE] = HwCtxSetDepthMetadataRegisters; + g_hw_ctx_func[Pm4::DB_RENDER_OVERRIDE2] = HwCtxSetDepthMetadataRegisters; + g_hw_ctx_func[Pm4::DB_DFSM_CONTROL] = HwCtxSetDepthMetadataRegisters; + g_hw_ctx_func[Pm4::DB_RMI_L2_CACHE_CONTROL] = HwCtxSetDepthMetadataRegisters; + g_hw_ctx_func[Pm4::CB_RMI_GL2_CACHE_CONTROL] = HwCtxSetDepthMetadataRegisters; + g_hw_ctx_func[Pm4::TA_BC_BASE_ADDR] = HwCtxSetBorderColorTableAddr; + g_hw_ctx_func[Pm4::TA_BC_BASE_ADDR_HI] = HwCtxSetBorderColorTableAddr; + g_hw_ctx_func[Pm4::DB_STENCIL_CLEAR] = HwCtxSetStencilClear; + g_hw_ctx_func[Pm4::DB_DEPTH_CLEAR] = HwCtxSetDepthClear; + g_hw_ctx_func[Pm4::PA_SC_SCREEN_SCISSOR_TL] = HwCtxSetScreenScissor; + g_hw_ctx_func[Pm4::DB_Z_INFO] = HwCtxSetDepthRenderTarget; + g_hw_ctx_func[Pm4::DB_STENCIL_INFO] = HwCtxSetStencilInfo; + g_hw_ctx_func[Pm4::DB_HTILE_SURFACE] = HwCtxSetDepthHtileSurface; + g_hw_ctx_func[Pm4::PA_SU_HARDWARE_SCREEN_OFFSET] = HwCtxSetHardwareScreenOffset; + g_hw_ctx_func[Pm4::PA_SC_WINDOW_OFFSET] = HwCtxSetWindowOffset; + g_hw_ctx_func[Pm4::PA_SC_WINDOW_SCISSOR_TL] = HwCtxSetWindowScissor; + for (auto cmd_offset = Pm4::PA_SC_CLIPRECT_RULE; cmd_offset < Pm4::PA_SC_CLIPRECT_0_TL + 8u; + cmd_offset++) { + g_hw_ctx_func[cmd_offset] = HwCtxSetClipRect; + } + g_hw_ctx_func[Pm4::CB_TARGET_MASK] = HwCtxSetRenderTargetMask; + g_hw_ctx_func[Pm4::PA_SC_GENERIC_SCISSOR_TL] = HwCtxSetGenericScissor; + g_hw_ctx_func[Pm4::CB_BLEND_RED] = HwCtxSetBlendColor; + g_hw_ctx_func[Pm4::CB_DCC_CONTROL] = HwCtxSetCbDccControl; + g_hw_ctx_func[Pm4::DB_STENCIL_CONTROL] = HwCtxSetStencilControl; + g_hw_ctx_func[Pm4::DB_STENCILREFMASK] = HwCtxSetStencilMask; + g_hw_ctx_func[Pm4::SPI_PS_INPUT_CNTL_0] = HwCtxSetPsInput; + g_hw_ctx_func[Pm4::SPI_TMPRING_SIZE] = HwCtxSetSpiTmpringSize; + g_hw_ctx_func[Pm4::DB_DEPTH_CONTROL] = HwCtxSetDepthControl; + g_hw_ctx_func[Pm4::DB_EQAA] = HwCtxSetEqaaControl; + g_hw_ctx_func[Pm4::CB_COLOR_CONTROL] = HwCtxSetColorControl; + g_hw_ctx_func[Pm4::DB_DEPTH_BOUNDS_MIN] = HwCtxSetDepthBounds; + g_hw_ctx_func[Pm4::DB_DEPTH_BOUNDS_MAX] = HwCtxSetDepthBounds; + g_hw_ctx_func[Pm4::PA_SU_POINT_SIZE] = HwCtxSetPointState; + g_hw_ctx_func[Pm4::PA_SU_POINT_MINMAX] = HwCtxSetPointState; + g_hw_ctx_func[Pm4::PA_CL_CLIP_CNTL] = HwCtxSetClipControl; + g_hw_ctx_func[Pm4::PA_SU_SC_MODE_CNTL] = HwCtxSetModeControl; + g_hw_ctx_func[Pm4::PA_SU_POLY_OFFSET_DB_FMT_CNTL] = HwCtxSetPolyOffsetRegisters; + g_hw_ctx_func[Pm4::PA_SU_POLY_OFFSET_CLAMP] = HwCtxSetPolyOffsetRegisters; + g_hw_ctx_func[Pm4::PA_SU_POLY_OFFSET_FRONT_SCALE] = HwCtxSetPolyOffsetRegisters; + g_hw_ctx_func[Pm4::PA_SU_POLY_OFFSET_FRONT_OFFSET] = HwCtxSetPolyOffsetRegisters; + g_hw_ctx_func[Pm4::PA_SU_POLY_OFFSET_BACK_SCALE] = HwCtxSetPolyOffsetRegisters; + g_hw_ctx_func[Pm4::PA_SU_POLY_OFFSET_BACK_OFFSET] = HwCtxSetPolyOffsetRegisters; + g_hw_ctx_func[Pm4::PA_CL_VTE_CNTL] = HwCtxSetViewportTransformControl; + g_hw_ctx_func[Pm4::PA_SU_LINE_CNTL] = HwCtxSetLineControl; + g_hw_ctx_func[Pm4::PA_SC_FOV_WINDOW_LR] = HwCtxSetFovWindow; + g_hw_ctx_func[Pm4::PA_SC_FOV_WINDOW_TB] = HwCtxSetFovWindow; + g_hw_ctx_func[Pm4::PA_SC_MODE_CNTL_0] = HwCtxSetScanModeControl; + g_hw_ctx_func[Pm4::PA_SC_MODE_CNTL_1] = HwCtxSetScanModeControl1; + g_hw_ctx_func[Pm4::PA_SC_AA_CONFIG] = HwCtxSetAaConfig; + for (auto cmd_offset = Pm4::PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_0; + cmd_offset < Pm4::PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_0 + 16; cmd_offset++) { + g_hw_ctx_func[cmd_offset] = HwCtxSetAaSampleControl; + } + g_hw_ctx_func[Pm4::PA_SC_CENTROID_PRIORITY_0] = HwCtxSetCentroidPriority; + g_hw_ctx_func[Pm4::PA_SC_CENTROID_PRIORITY_1] = HwCtxSetCentroidPriority; + g_hw_ctx_func[Pm4::PA_SC_AA_MASK_X0Y0_X1Y0] = HwCtxSetAaMask; + g_hw_ctx_func[Pm4::PA_SC_AA_MASK_X0Y1_X1Y1] = HwCtxSetAaMask; + g_hw_ctx_func[Pm4::PA_SU_VTX_CNTL] = HwCtxSetVtxControl; + g_hw_ctx_func[Pm4::PS_SHADER_SAMPLE_EXCLUSION_MASK] = HwCtxSetAaMask; + g_hw_ctx_func[Pm4::PA_SC_BINNER_CNTL_0] = HwCtxSetPaScExtendedControl; + g_hw_ctx_func[Pm4::PA_SC_BINNER_CNTL_1] = HwCtxSetPaScExtendedControl; + g_hw_ctx_func[Pm4::PA_SC_CONSERVATIVE_RASTERIZATION_CNTL] = HwCtxSetPaScExtendedControl; + g_hw_ctx_func[Pm4::PA_SC_NGG_MODE_CNTL] = HwCtxSetPaScExtendedControl; + g_hw_ctx_func[Pm4::DB_ALPHA_TO_MASK] = HwCtxSetAlphaToMask; + g_hw_ctx_func[Pm4::VGT_DRAW_PAYLOAD_CNTL] = HwCtxSetDrawPayloadControl; + g_hw_ctx_func[Pm4::VGT_PRIMITIVEID_RESET] = HwCtxSetPrimitiveIdReset; + g_hw_ctx_func[Pm4::PA_CL_OBJPRIM_ID_CNTL] = HwCtxSetObjprimIdControl; + g_hw_ctx_func[Pm4::VGT_SHADER_STAGES_EN] = HwCtxSetShaderStages; + g_hw_ctx_func[Pm4::PA_CL_GB_VERT_CLIP_ADJ] = HwCtxSetGuardBands; + + for (uint32_t slot = 0; slot < 8; slot++) { + g_hw_ctx_func[Pm4::CB_COLOR0_BASE + slot * 15] = HwCtxSetRenderTarget; + g_hw_ctx_func[Pm4::CB_COLOR0_INFO + slot * 15] = HwCtxSetColorInfo; + + g_hw_ctx_func[Pm4::CB_BLEND0_CONTROL + slot * 1] = HwCtxSetBlendControl; + } + + for (uint32_t viewport = 0; viewport < 16; viewport++) { + g_hw_ctx_func[Pm4::PA_SC_VPORT_SCISSOR_0_TL + viewport * 2] = HwCtxSetViewportScissor; + g_hw_ctx_func[Pm4::PA_SC_VPORT_SCISSOR_0_BR + viewport * 2] = HwCtxSetViewportScissor; + g_hw_ctx_func[Pm4::PA_SC_VPORT_ZMIN_0 + viewport * 2] = HwCtxSetViewportZ; + g_hw_ctx_func[Pm4::PA_SC_VPORT_ZMAX_0 + viewport * 2] = HwCtxSetViewportZ; + for (uint32_t field = 0; field < 6; field++) { + g_hw_ctx_func[Pm4::PA_CL_VPORT_XSCALE + viewport * 6 + field] = + HwCtxSetViewportScaleOffset; + } + } + + for (auto& func: g_hw_sh_func) { + func = nullptr; + } + + for (uint32_t slot = 0; slot < 32; slot++) { + g_hw_sh_func[Pm4::SPI_SHADER_USER_DATA_PS_0 + slot * 1] = HwShSetPsUserSgpr; + g_hw_sh_func[Pm4::SPI_SHADER_USER_DATA_GS_0 + slot * 1] = HwShSetGsUserSgpr; + g_hw_sh_func[Pm4::SPI_SHADER_USER_DATA_HS_0 + slot * 1] = HwShSetHsUserSgpr; + } + for (uint32_t slot = 0; slot < 16; slot++) { + g_hw_sh_func[Pm4::COMPUTE_USER_DATA_0 + slot * 1] = HwShSetCsUserSgpr; + } + + for (uint32_t slot = 0; slot < 4; slot++) { + g_hw_sh_func[Pm4::SPI_SHADER_USER_ACCUM_PS_0 + slot] = HwShSetPsUserAccumSgpr; + } + + for (uint32_t slot = 0; slot < 4; slot++) { + g_hw_sh_func[Pm4::SPI_SHADER_USER_ACCUM_ESGS_0 + slot] = HwShSetGsUserAccumSgpr; + } + for (uint32_t slot = 0; slot < 4; slot++) { + g_hw_sh_func[Pm4::SPI_SHADER_USER_ACCUM_LSHS_0 + slot] = HwShSetHsUserAccumSgpr; + } + + for (uint32_t slot = 0; Pm4::COMPUTE_USER_ACCUM_0 + slot < Pm4::COMPUTE_USER_DATA_0; slot++) { + g_hw_sh_func[Pm4::COMPUTE_USER_ACCUM_0 + slot] = HwShSetCsUserAccumSgpr; + } + g_hw_sh_func[Pm4::COMPUTE_PGM_LO] = HwShSetCsRegisters; + g_hw_sh_func[Pm4::COMPUTE_PGM_HI] = HwShSetCsRegisters; + g_hw_sh_func[Pm4::COMPUTE_TBA_LO] = HwShSetCsRegisters; + g_hw_sh_func[Pm4::COMPUTE_TBA_HI] = HwShSetCsRegisters; + g_hw_sh_func[Pm4::COMPUTE_TMA_LO] = HwShSetCsRegisters; + g_hw_sh_func[Pm4::COMPUTE_TMA_HI] = HwShSetCsRegisters; + g_hw_sh_func[Pm4::COMPUTE_PGM_RSRC1] = HwShSetCsRegisters; + g_hw_sh_func[Pm4::COMPUTE_PGM_RSRC2] = HwShSetCsRegisters; + g_hw_sh_func[Pm4::COMPUTE_START_X] = HwShSetCsRegisters; + g_hw_sh_func[Pm4::COMPUTE_START_Y] = HwShSetCsRegisters; + g_hw_sh_func[Pm4::COMPUTE_START_Z] = HwShSetCsRegisters; + g_hw_sh_func[Pm4::COMPUTE_NUM_THREAD_X] = HwShSetCsRegisters; + g_hw_sh_func[Pm4::COMPUTE_NUM_THREAD_Y] = HwShSetCsRegisters; + g_hw_sh_func[Pm4::COMPUTE_NUM_THREAD_Z] = HwShSetCsRegisters; + g_hw_sh_func[Pm4::COMPUTE_RESOURCE_LIMITS] = HwShSetCsRegisters; + g_hw_sh_func[Pm4::COMPUTE_DESTINATION_EN_SE0] = HwShSetCsRegisters; + g_hw_sh_func[Pm4::COMPUTE_DESTINATION_EN_SE1] = HwShSetCsRegisters; + g_hw_sh_func[Pm4::COMPUTE_TMPRING_SIZE] = HwShSetCsRegisters; + g_hw_sh_func[Pm4::COMPUTE_DESTINATION_EN_SE2] = HwShSetCsRegisters; + g_hw_sh_func[Pm4::COMPUTE_DESTINATION_EN_SE3] = HwShSetCsRegisters; + g_hw_sh_func[Pm4::COMPUTE_PGM_RSRC3] = HwShSetCsRegisters; + g_hw_sh_func[Pm4::COMPUTE_SHADER_CHKSUM] = HwShSetCsRegisters; + g_hw_sh_func[Pm4::COMPUTE_DISPATCH_TUNNEL] = HwShSetCsRegisters; + g_hw_sh_func[Pm4::SPI_SHADER_TBA_LO_PS] = HwShIgnoreRegisters; + g_hw_sh_func[Pm4::SPI_SHADER_PGM_RSRC3_PS] = HwShIgnoreRegisters; + g_hw_sh_func[Pm4::SPI_SHADER_PGM_RSRC4_PS] = HwShIgnoreRegisters; + g_hw_sh_func[Pm4::SPI_SHADER_TMA_LO_PS] = HwShIgnoreRegisters; + g_hw_sh_func[Pm4::SPI_SHADER_TMA_HI_PS] = HwShIgnoreRegisters; + g_hw_sh_func[Pm4::SPI_SHADER_REQ_CTRL_PS] = HwShIgnoreRegisters; + g_hw_sh_func[Pm4::SPI_SHADER_PGM_RSRC3_GS] = HwShIgnoreRegisters; + g_hw_sh_func[Pm4::SPI_SHADER_PGM_RSRC4_GS] = HwShIgnoreRegisters; + g_hw_sh_func[Pm4::SPI_SHADER_USER_DATA_ADDR_LO_GS] = HwShIgnoreRegisters; + g_hw_sh_func[Pm4::SPI_SHADER_USER_DATA_ADDR_HI_GS] = HwShIgnoreRegisters; + g_hw_sh_func[Pm4::SPI_SHADER_REQ_CTRL_ESGS] = HwShIgnoreRegisters; + g_hw_sh_func[Pm4::SPI_SHADER_PGM_RSRC3_HS] = HwShIgnoreRegisters; + g_hw_sh_func[Pm4::SPI_SHADER_PGM_RSRC4_HS] = HwShIgnoreRegisters; + g_hw_sh_func[Pm4::SPI_SHADER_USER_DATA_ADDR_LO_HS] = HwShIgnoreRegisters; + g_hw_sh_func[Pm4::SPI_SHADER_USER_DATA_ADDR_HI_HS] = HwShIgnoreRegisters; + g_hw_sh_func[Pm4::SPI_SHADER_REQ_CTRL_LSHS] = HwShIgnoreRegisters; + + for (auto& func: g_hw_uc_func) { + func = nullptr; + } + + g_hw_uc_func[Pm4::VGT_PRIMITIVE_TYPE] = HwUcSetPrimitiveType; + g_hw_uc_func[Pm4::VGT_INDEX_TYPE] = HwUcSetIndexType; + g_hw_uc_func[Pm4::VGT_OBJECT_ID] = HwUcSetObjectId; + g_hw_uc_func[Pm4::TEXTURE_GRADIENT_FACTORS] = HwUcSetTextureGradientFactors; + g_hw_uc_func[Pm4::IA_MULTI_VGT_PARAM] = HwUcSetIaMultiVgtParam; + g_hw_uc_func[Pm4::GE_MULTI_PRIM_IB_RESET_EN] = HwUcSetMultiPrimIbReset; + g_hw_uc_func[Pm4::TA_CS_BC_BASE_ADDR] = HwUcSetBorderColorTableAddr; + g_hw_uc_func[Pm4::TA_CS_BC_BASE_ADDR_HI] = HwUcSetBorderColorTableAddr; + g_hw_uc_func[Pm4::GE_INDX_OFFSET] = HwUcSetGeIndexOffset; + g_hw_uc_func[Pm4::GDS_OA_CNTL] = HwUcSetGdsOaRegisters; + g_hw_uc_func[Pm4::GDS_OA_COUNTER] = HwUcSetGdsOaRegisters; + g_hw_uc_func[Pm4::GDS_OA_ADDRESS] = HwUcSetGdsOaRegisters; + + for (auto& func: g_cp_op_func) { + func = nullptr; + } + + g_cp_op_func[Pm4::IT_NOP] = CpOpNop; + g_cp_op_func[Pm4::IT_SET_BASE] = CpOpSetBase; + g_cp_op_func[Pm4::IT_CLEAR_STATE] = CpOpClearState; + g_cp_op_func[Pm4::IT_INDEX_BASE] = CpOpIndexBase; + g_cp_op_func[Pm4::IT_INDEX_BUFFER_SIZE] = CpOpIndexBufferSize; + g_cp_op_func[Pm4::IT_DISPATCH_DIRECT] = CpOpDispatchDirect; + g_cp_op_func[Pm4::IT_DISPATCH_INDIRECT] = CpOpDispatchIndirect; + g_cp_op_func[Pm4::IT_DRAW_INDIRECT] = CpOpDrawIndirect; + g_cp_op_func[Pm4::IT_DRAW_INDEX_INDIRECT] = CpOpDrawIndirect; + g_cp_op_func[Pm4::IT_DRAW_INDIRECT_MULTI] = CpOpDrawIndirectMulti; + g_cp_op_func[Pm4::IT_DRAW_INDEX_INDIRECT_MULTI] = CpOpDrawIndirectMulti; + g_cp_op_func[Pm4::IT_DRAW_INDEX_2] = CpOpDrawIndex; + g_cp_op_func[Pm4::IT_DRAW_INDEX_OFFSET_2] = CpOpDrawIndexOffset; + g_cp_op_func[Pm4::IT_DISPATCH_DRAW_PREAMBLE] = CpOpDrawIndex; + g_cp_op_func[Pm4::IT_PFP_SYNC_ME] = CpOpPfpSyncMe; + g_cp_op_func[Pm4::IT_INDEX_TYPE] = CpOpIndexType; + g_cp_op_func[Pm4::IT_NUM_INSTANCES] = CpOpNumInstances; + g_cp_op_func[Pm4::IT_DRAW_INDEX_AUTO] = CpOpDrawIndexAuto; + g_cp_op_func[Pm4::IT_COND_EXEC] = CpOpCondExec; + g_cp_op_func[Pm4::IT_SET_PREDICATION] = CpOpSetPredication; + g_cp_op_func[Pm4::IT_WRITE_DATA] = CpOpWriteData; + g_cp_op_func[Pm4::IT_INDIRECT_BUFFER] = CpOpIndirectBuffer; + g_cp_op_func[Pm4::IT_COPY_DATA] = CpOpCopyData; + g_cp_op_func[Pm4::IT_EVENT_WRITE] = CpOpEventWrite; + g_cp_op_func[Pm4::IT_EVENT_WRITE_EOP] = CpOpEventWriteEop; + g_cp_op_func[Pm4::IT_EVENT_WRITE_EOS] = CpOpEventWriteEos; + g_cp_op_func[Pm4::IT_DMA_DATA] = CpOpDmaData; + g_cp_op_func[Pm4::IT_ACQUIRE_MEM] = CpOpAcquireMem; + g_cp_op_func[Pm4::IT_SET_CONTEXT_REG] = CpOpSetContextReg; + g_cp_op_func[Pm4::IT_SET_SH_REG] = CpOpSetShaderReg; + g_cp_op_func[Pm4::IT_SET_UCONFIG_REG] = CpOpSetUconfigReg; + g_cp_op_func[Pm4::IT_SET_UCONFIG_REG_INDEX] = CpOpSetUconfigReg; + g_cp_op_func[Pm4::IT_SET_CONTEXT_REG_INDIRECT] = CpOpIndirectCxRegs; + g_cp_op_func[Pm4::IT_SET_SH_REG_INDIRECT] = CpOpIndirectShRegs; + g_cp_op_func[Pm4::IT_SET_UCONFIG_REG_INDIRECT] = CpOpIndirectUcRegs; + g_cp_op_func[Pm4::IT_WRITE_CONST_RAM] = CpOpWriteConstRam; + g_cp_op_func[Pm4::IT_DUMP_CONST_RAM] = CpOpDumpConstRam; + g_cp_op_func[Pm4::IT_INCREMENT_CE_COUNTER] = CpOpIncrementCeCounter; + g_cp_op_func[Pm4::IT_INCREMENT_DE_COUNTER] = CpOpIncrementDeCounter; + g_cp_op_func[Pm4::IT_WAIT_ON_CE_COUNTER] = CpOpWaitOnCeCounter; + g_cp_op_func[Pm4::IT_WAIT_ON_DE_COUNTER_DIFF] = CpOpWaitOnDeCounterDiff; + g_cp_op_func[Pm4::IT_GET_LOD_STATS] = CpOpGetLodStats; + + for (auto& func: g_cp_op_custom_func) { + func = nullptr; + } + + g_cp_op_custom_func[Pm4::R_DISPATCH_RESET] = CpOpDispatchReset; + g_cp_op_custom_func[Pm4::R_WAIT_MEM_32] = CpOpWaitRegMem32; + g_cp_op_custom_func[Pm4::R_WAIT_FLIP_DONE] = CpOpWaitFlipDone; + g_cp_op_custom_func[Pm4::R_PUSH_MARKER] = CpOpPushMarker; + g_cp_op_custom_func[Pm4::R_POP_MARKER] = CpOpPopMarker; + g_cp_op_custom_func[Pm4::R_ACQUIRE_MEM] = CpOpAcquireMem; + g_cp_op_custom_func[Pm4::R_WAIT_MEM_64] = CpOpWaitRegMem64; + g_cp_op_custom_func[Pm4::R_FLIP] = CpOpFlip; + g_cp_op_custom_func[Pm4::R_RELEASE_MEM] = CpOpReleaseMem; + + GraphicsInitJmpTablesCxIndirect(); + GraphicsInitJmpTablesShIndirect(); + GraphicsInitJmpTablesUcIndirect(); +} + +} // namespace Libs::Graphics diff --git a/src/graphics/guest_gpu/command_processor/pm4Dispatch.h b/src/graphics/guest_gpu/command_processor/pm4Dispatch.h new file mode 100644 index 000000000..120edcd97 --- /dev/null +++ b/src/graphics/guest_gpu/command_processor/pm4Dispatch.h @@ -0,0 +1,171 @@ +#ifndef GRAPHICS_GUEST_GPU_COMMAND_PROCESSOR_PM4_DISPATCH_H +#define GRAPHICS_GUEST_GPU_COMMAND_PROCESSOR_PM4_DISPATCH_H + +#include "graphics/guest_gpu/pm4.h" + +#include + +namespace Libs::Graphics { + +class CommandProcessor; + +using hw_ctx_parser_func_t = uint32_t (*)(CommandProcessor*, uint32_t, uint32_t, const uint32_t*, + uint32_t); +using hw_uc_parser_func_t = uint32_t (*)(CommandProcessor*, uint32_t, uint32_t, const uint32_t*, + uint32_t); +using hw_sh_parser_func_t = uint32_t (*)(CommandProcessor*, uint32_t, uint32_t, const uint32_t*, + uint32_t); +using cp_op_parser_func_t = uint32_t (*)(CommandProcessor*, uint32_t, const uint32_t*, uint32_t, + uint32_t); +using hw_ctx_indirect_func_t = void (*)(CommandProcessor*, uint32_t, uint32_t); +using hw_uc_indirect_func_t = void (*)(CommandProcessor*, uint32_t, uint32_t); +using hw_sh_indirect_func_t = void (*)(CommandProcessor*, uint32_t, uint32_t); + +extern hw_ctx_parser_func_t g_hw_ctx_func[Pm4::CX_NUM]; +extern hw_ctx_indirect_func_t g_hw_ctx_indirect_func[Pm4::CX_NUM]; +extern hw_sh_parser_func_t g_hw_sh_func[Pm4::SH_NUM]; +extern hw_sh_indirect_func_t g_hw_sh_indirect_func[Pm4::SH_NUM]; +extern hw_uc_parser_func_t g_hw_uc_func[Pm4::UC_NUM]; +extern hw_uc_indirect_func_t g_hw_uc_indirect_func[Pm4::UC_NUM]; +extern cp_op_parser_func_t g_cp_op_func[256]; +extern cp_op_parser_func_t g_cp_op_custom_func[Pm4::R_NUM]; + +void GraphicsInitJmpTables(); +void GraphicsInitJmpTablesCxIndirect(); +void GraphicsInitJmpTablesShIndirect(); +void GraphicsInitJmpTablesUcIndirect(); + +uint32_t HwCtxSetRenderControl(CommandProcessor*, uint32_t, uint32_t, const uint32_t*, uint32_t); +uint32_t HwCtxSetDepthMetadataRegisters(CommandProcessor*, uint32_t, uint32_t, const uint32_t*, + uint32_t); +uint32_t HwCtxSetBorderColorTableAddr(CommandProcessor*, uint32_t, uint32_t, const uint32_t*, + uint32_t); +uint32_t HwCtxSetStencilClear(CommandProcessor*, uint32_t, uint32_t, const uint32_t*, uint32_t); +uint32_t HwCtxSetDepthClear(CommandProcessor*, uint32_t, uint32_t, const uint32_t*, uint32_t); +uint32_t HwCtxSetScreenScissor(CommandProcessor*, uint32_t, uint32_t, const uint32_t*, uint32_t); +uint32_t HwCtxSetDepthRenderTarget(CommandProcessor*, uint32_t, uint32_t, const uint32_t*, + uint32_t); +uint32_t HwCtxSetStencilInfo(CommandProcessor*, uint32_t, uint32_t, const uint32_t*, uint32_t); +uint32_t HwCtxSetDepthHtileSurface(CommandProcessor*, uint32_t, uint32_t, const uint32_t*, + uint32_t); +uint32_t HwCtxSetHardwareScreenOffset(CommandProcessor*, uint32_t, uint32_t, const uint32_t*, + uint32_t); +uint32_t HwCtxSetWindowOffset(CommandProcessor*, uint32_t, uint32_t, const uint32_t*, uint32_t); +uint32_t HwCtxSetWindowScissor(CommandProcessor*, uint32_t, uint32_t, const uint32_t*, uint32_t); +uint32_t HwCtxSetClipRect(CommandProcessor*, uint32_t, uint32_t, const uint32_t*, uint32_t); +uint32_t HwCtxSetRenderTargetMask(CommandProcessor*, uint32_t, uint32_t, const uint32_t*, uint32_t); +uint32_t HwCtxSetGenericScissor(CommandProcessor*, uint32_t, uint32_t, const uint32_t*, uint32_t); +uint32_t HwCtxSetBlendColor(CommandProcessor*, uint32_t, uint32_t, const uint32_t*, uint32_t); +uint32_t HwCtxSetCbDccControl(CommandProcessor*, uint32_t, uint32_t, const uint32_t*, uint32_t); +uint32_t HwCtxSetStencilControl(CommandProcessor*, uint32_t, uint32_t, const uint32_t*, uint32_t); +uint32_t HwCtxSetStencilMask(CommandProcessor*, uint32_t, uint32_t, const uint32_t*, uint32_t); +uint32_t HwCtxSetPsInput(CommandProcessor*, uint32_t, uint32_t, const uint32_t*, uint32_t); +uint32_t HwCtxSetSpiTmpringSize(CommandProcessor*, uint32_t, uint32_t, const uint32_t*, uint32_t); +uint32_t HwCtxSetDepthControl(CommandProcessor*, uint32_t, uint32_t, const uint32_t*, uint32_t); +uint32_t HwCtxSetEqaaControl(CommandProcessor*, uint32_t, uint32_t, const uint32_t*, uint32_t); +uint32_t HwCtxSetColorControl(CommandProcessor*, uint32_t, uint32_t, const uint32_t*, uint32_t); +uint32_t HwCtxSetDepthBounds(CommandProcessor*, uint32_t, uint32_t, const uint32_t*, uint32_t); +uint32_t HwCtxSetPointState(CommandProcessor*, uint32_t, uint32_t, const uint32_t*, uint32_t); +uint32_t HwCtxSetClipControl(CommandProcessor*, uint32_t, uint32_t, const uint32_t*, uint32_t); +uint32_t HwCtxSetModeControl(CommandProcessor*, uint32_t, uint32_t, const uint32_t*, uint32_t); +uint32_t HwCtxSetPolyOffsetRegisters(CommandProcessor*, uint32_t, uint32_t, const uint32_t*, + uint32_t); +uint32_t HwCtxSetViewportTransformControl(CommandProcessor*, uint32_t, uint32_t, const uint32_t*, + uint32_t); +uint32_t HwCtxSetLineControl(CommandProcessor*, uint32_t, uint32_t, const uint32_t*, uint32_t); +uint32_t HwCtxSetFovWindow(CommandProcessor*, uint32_t, uint32_t, const uint32_t*, uint32_t); +uint32_t HwCtxSetScanModeControl(CommandProcessor*, uint32_t, uint32_t, const uint32_t*, uint32_t); +uint32_t HwCtxSetScanModeControl1(CommandProcessor*, uint32_t, uint32_t, const uint32_t*, uint32_t); +uint32_t HwCtxSetAaConfig(CommandProcessor*, uint32_t, uint32_t, const uint32_t*, uint32_t); +uint32_t HwCtxSetAaSampleControl(CommandProcessor*, uint32_t, uint32_t, const uint32_t*, uint32_t); +uint32_t HwCtxSetCentroidPriority(CommandProcessor*, uint32_t, uint32_t, const uint32_t*, uint32_t); +uint32_t HwCtxSetAaMask(CommandProcessor*, uint32_t, uint32_t, const uint32_t*, uint32_t); +uint32_t HwCtxSetVtxControl(CommandProcessor*, uint32_t, uint32_t, const uint32_t*, uint32_t); +uint32_t HwCtxSetPaScExtendedControl(CommandProcessor*, uint32_t, uint32_t, const uint32_t*, + uint32_t); +uint32_t HwCtxSetAlphaToMask(CommandProcessor*, uint32_t, uint32_t, const uint32_t*, uint32_t); +uint32_t HwCtxSetDrawPayloadControl(CommandProcessor*, uint32_t, uint32_t, const uint32_t*, + uint32_t); +uint32_t HwCtxSetPrimitiveIdReset(CommandProcessor*, uint32_t, uint32_t, const uint32_t*, uint32_t); +uint32_t HwCtxSetObjprimIdControl(CommandProcessor*, uint32_t, uint32_t, const uint32_t*, uint32_t); +uint32_t HwCtxSetShaderStages(CommandProcessor*, uint32_t, uint32_t, const uint32_t*, uint32_t); +uint32_t HwCtxSetGuardBands(CommandProcessor*, uint32_t, uint32_t, const uint32_t*, uint32_t); +uint32_t HwCtxSetRenderTarget(CommandProcessor*, uint32_t, uint32_t, const uint32_t*, uint32_t); +uint32_t HwCtxSetColorInfo(CommandProcessor*, uint32_t, uint32_t, const uint32_t*, uint32_t); +uint32_t HwCtxSetBlendControl(CommandProcessor*, uint32_t, uint32_t, const uint32_t*, uint32_t); +uint32_t HwCtxSetViewportScissor(CommandProcessor*, uint32_t, uint32_t, const uint32_t*, uint32_t); +uint32_t HwCtxSetViewportZ(CommandProcessor*, uint32_t, uint32_t, const uint32_t*, uint32_t); +uint32_t HwCtxSetViewportScaleOffset(CommandProcessor*, uint32_t, uint32_t, const uint32_t*, + uint32_t); +uint32_t HwShSetPsUserSgpr(CommandProcessor*, uint32_t, uint32_t, const uint32_t*, uint32_t); +uint32_t HwShSetGsUserSgpr(CommandProcessor*, uint32_t, uint32_t, const uint32_t*, uint32_t); +uint32_t HwShSetHsUserSgpr(CommandProcessor*, uint32_t, uint32_t, const uint32_t*, uint32_t); +uint32_t HwShSetCsUserSgpr(CommandProcessor*, uint32_t, uint32_t, const uint32_t*, uint32_t); +uint32_t HwShSetPsUserAccumSgpr(CommandProcessor*, uint32_t, uint32_t, const uint32_t*, uint32_t); +uint32_t HwShSetGsUserAccumSgpr(CommandProcessor*, uint32_t, uint32_t, const uint32_t*, uint32_t); +uint32_t HwShSetHsUserAccumSgpr(CommandProcessor*, uint32_t, uint32_t, const uint32_t*, uint32_t); +uint32_t HwShSetCsUserAccumSgpr(CommandProcessor*, uint32_t, uint32_t, const uint32_t*, uint32_t); +uint32_t HwShSetCsRegisters(CommandProcessor*, uint32_t, uint32_t, const uint32_t*, uint32_t); +uint32_t HwShIgnoreRegisters(CommandProcessor*, uint32_t, uint32_t, const uint32_t*, uint32_t); +uint32_t HwUcSetPrimitiveType(CommandProcessor*, uint32_t, uint32_t, const uint32_t*, uint32_t); +uint32_t HwUcSetIndexType(CommandProcessor*, uint32_t, uint32_t, const uint32_t*, uint32_t); +uint32_t HwUcSetObjectId(CommandProcessor*, uint32_t, uint32_t, const uint32_t*, uint32_t); +uint32_t HwUcSetTextureGradientFactors(CommandProcessor*, uint32_t, uint32_t, const uint32_t*, + uint32_t); +uint32_t HwUcSetIaMultiVgtParam(CommandProcessor*, uint32_t, uint32_t, const uint32_t*, uint32_t); +uint32_t HwUcSetMultiPrimIbReset(CommandProcessor*, uint32_t, uint32_t, const uint32_t*, uint32_t); +uint32_t HwUcSetBorderColorTableAddr(CommandProcessor*, uint32_t, uint32_t, const uint32_t*, + uint32_t); +uint32_t HwUcSetGeIndexOffset(CommandProcessor*, uint32_t, uint32_t, const uint32_t*, uint32_t); +uint32_t HwUcSetGdsOaRegisters(CommandProcessor*, uint32_t, uint32_t, const uint32_t*, uint32_t); + +uint32_t CpOpNop(CommandProcessor*, uint32_t, const uint32_t*, uint32_t, uint32_t); +uint32_t CpOpSetBase(CommandProcessor*, uint32_t, const uint32_t*, uint32_t, uint32_t); +uint32_t CpOpClearState(CommandProcessor*, uint32_t, const uint32_t*, uint32_t, uint32_t); +uint32_t CpOpIndexBase(CommandProcessor*, uint32_t, const uint32_t*, uint32_t, uint32_t); +uint32_t CpOpIndexBufferSize(CommandProcessor*, uint32_t, const uint32_t*, uint32_t, uint32_t); +uint32_t CpOpDispatchDirect(CommandProcessor*, uint32_t, const uint32_t*, uint32_t, uint32_t); +uint32_t CpOpDispatchIndirect(CommandProcessor*, uint32_t, const uint32_t*, uint32_t, uint32_t); +uint32_t CpOpDrawIndirect(CommandProcessor*, uint32_t, const uint32_t*, uint32_t, uint32_t); +uint32_t CpOpDrawIndirectMulti(CommandProcessor*, uint32_t, const uint32_t*, uint32_t, uint32_t); +uint32_t CpOpDrawIndex(CommandProcessor*, uint32_t, const uint32_t*, uint32_t, uint32_t); +uint32_t CpOpDrawIndexOffset(CommandProcessor*, uint32_t, const uint32_t*, uint32_t, uint32_t); +uint32_t CpOpPfpSyncMe(CommandProcessor*, uint32_t, const uint32_t*, uint32_t, uint32_t); +uint32_t CpOpIndexType(CommandProcessor*, uint32_t, const uint32_t*, uint32_t, uint32_t); +uint32_t CpOpNumInstances(CommandProcessor*, uint32_t, const uint32_t*, uint32_t, uint32_t); +uint32_t CpOpDrawIndexAuto(CommandProcessor*, uint32_t, const uint32_t*, uint32_t, uint32_t); +uint32_t CpOpCondExec(CommandProcessor*, uint32_t, const uint32_t*, uint32_t, uint32_t); +uint32_t CpOpSetPredication(CommandProcessor*, uint32_t, const uint32_t*, uint32_t, uint32_t); +uint32_t CpOpWriteData(CommandProcessor*, uint32_t, const uint32_t*, uint32_t, uint32_t); +uint32_t CpOpIndirectBuffer(CommandProcessor*, uint32_t, const uint32_t*, uint32_t, uint32_t); +uint32_t CpOpCopyData(CommandProcessor*, uint32_t, const uint32_t*, uint32_t, uint32_t); +uint32_t CpOpEventWrite(CommandProcessor*, uint32_t, const uint32_t*, uint32_t, uint32_t); +uint32_t CpOpEventWriteEop(CommandProcessor*, uint32_t, const uint32_t*, uint32_t, uint32_t); +uint32_t CpOpEventWriteEos(CommandProcessor*, uint32_t, const uint32_t*, uint32_t, uint32_t); +uint32_t CpOpDmaData(CommandProcessor*, uint32_t, const uint32_t*, uint32_t, uint32_t); +uint32_t CpOpAcquireMem(CommandProcessor*, uint32_t, const uint32_t*, uint32_t, uint32_t); +uint32_t CpOpSetContextReg(CommandProcessor*, uint32_t, const uint32_t*, uint32_t, uint32_t); +uint32_t CpOpSetShaderReg(CommandProcessor*, uint32_t, const uint32_t*, uint32_t, uint32_t); +uint32_t CpOpSetUconfigReg(CommandProcessor*, uint32_t, const uint32_t*, uint32_t, uint32_t); +uint32_t CpOpIndirectCxRegs(CommandProcessor*, uint32_t, const uint32_t*, uint32_t, uint32_t); +uint32_t CpOpIndirectShRegs(CommandProcessor*, uint32_t, const uint32_t*, uint32_t, uint32_t); +uint32_t CpOpIndirectUcRegs(CommandProcessor*, uint32_t, const uint32_t*, uint32_t, uint32_t); +uint32_t CpOpWriteConstRam(CommandProcessor*, uint32_t, const uint32_t*, uint32_t, uint32_t); +uint32_t CpOpDumpConstRam(CommandProcessor*, uint32_t, const uint32_t*, uint32_t, uint32_t); +uint32_t CpOpIncrementCeCounter(CommandProcessor*, uint32_t, const uint32_t*, uint32_t, uint32_t); +uint32_t CpOpIncrementDeCounter(CommandProcessor*, uint32_t, const uint32_t*, uint32_t, uint32_t); +uint32_t CpOpWaitOnCeCounter(CommandProcessor*, uint32_t, const uint32_t*, uint32_t, uint32_t); +uint32_t CpOpWaitOnDeCounterDiff(CommandProcessor*, uint32_t, const uint32_t*, uint32_t, uint32_t); +uint32_t CpOpGetLodStats(CommandProcessor*, uint32_t, const uint32_t*, uint32_t, uint32_t); +uint32_t CpOpDispatchReset(CommandProcessor*, uint32_t, const uint32_t*, uint32_t, uint32_t); +uint32_t CpOpWaitRegMem32(CommandProcessor*, uint32_t, const uint32_t*, uint32_t, uint32_t); +uint32_t CpOpWaitFlipDone(CommandProcessor*, uint32_t, const uint32_t*, uint32_t, uint32_t); +uint32_t CpOpPushMarker(CommandProcessor*, uint32_t, const uint32_t*, uint32_t, uint32_t); +uint32_t CpOpPopMarker(CommandProcessor*, uint32_t, const uint32_t*, uint32_t, uint32_t); +uint32_t CpOpWaitRegMem64(CommandProcessor*, uint32_t, const uint32_t*, uint32_t, uint32_t); +uint32_t CpOpFlip(CommandProcessor*, uint32_t, const uint32_t*, uint32_t, uint32_t); +uint32_t CpOpReleaseMem(CommandProcessor*, uint32_t, const uint32_t*, uint32_t, uint32_t); + +} // namespace Libs::Graphics + +#endif // GRAPHICS_GUEST_GPU_COMMAND_PROCESSOR_PM4_DISPATCH_H diff --git a/src/graphics/guest_gpu/command_processor/pm4Handlers.cpp b/src/graphics/guest_gpu/command_processor/pm4Handlers.cpp new file mode 100644 index 000000000..af0e2ca19 --- /dev/null +++ b/src/graphics/guest_gpu/command_processor/pm4Handlers.cpp @@ -0,0 +1,4778 @@ +#include "common/assert.h" +#include "common/logging/log.h" +#include "common/profiler.h" +#include "common/stringUtils.h" +#include "graphics/guest_gpu/command_processor/commandProcessor.h" +#include "graphics/guest_gpu/command_processor/pm4Dispatch.h" +#include "graphics/guest_gpu/graphicsRun.h" +#include "graphics/host_gpu/graphicContext.h" +#include "graphics/host_gpu/objects/label.h" +#include "graphics/host_gpu/renderer/render.h" +#include "graphics/host_gpu/renderer/renderContext.h" +#include "graphics/presentation/videoOut.h" +#include "graphics/presentation/window.h" +#include "graphics/shader/shader.h" +#include "kernel/memory.h" +#include "libs/agc.h" +#include "libs/errno.h" + +#include +#include +#include +#include +#include + +#define KYTY_HW_CTX_PARSER_ARGS \ + [[maybe_unused]] CommandProcessor *cp, uint32_t cmd_id, [[maybe_unused]] uint32_t cmd_offset, \ + const uint32_t *buffer, [[maybe_unused]] uint32_t dw +#define KYTY_HW_UC_PARSER_ARGS \ + [[maybe_unused]] CommandProcessor *cp, uint32_t cmd_id, [[maybe_unused]] uint32_t cmd_offset, \ + const uint32_t *buffer, [[maybe_unused]] uint32_t dw +#define KYTY_HW_SH_PARSER_ARGS \ + [[maybe_unused]] CommandProcessor *cp, uint32_t cmd_id, [[maybe_unused]] uint32_t cmd_offset, \ + const uint32_t *buffer, [[maybe_unused]] uint32_t dw + +#define KYTY_HW_CTX_PARSER(f) uint32_t f(KYTY_HW_CTX_PARSER_ARGS) +#define KYTY_HW_UC_PARSER(f) uint32_t f(KYTY_HW_UC_PARSER_ARGS) +#define KYTY_HW_SH_PARSER(f) uint32_t f(KYTY_HW_SH_PARSER_ARGS) + +#define KYTY_HW_CTX_INDIRECT_ARGS \ + CommandProcessor *cp, [[maybe_unused]] uint32_t cmd_offset, uint32_t value +#define KYTY_HW_UC_INDIRECT_ARGS \ + CommandProcessor *cp, [[maybe_unused]] uint32_t cmd_offset, uint32_t value +#define KYTY_HW_SH_INDIRECT_ARGS \ + CommandProcessor *cp, [[maybe_unused]] uint32_t cmd_offset, uint32_t value + +#define KYTY_CP_OP_PARSER_ARGS \ + [[maybe_unused]] CommandProcessor *cp, [[maybe_unused]] uint32_t cmd_id, \ + [[maybe_unused]] const uint32_t *buffer, [[maybe_unused]] uint32_t dw, \ + [[maybe_unused]] uint32_t num_dw +#define KYTY_CP_OP_PARSER(f) uint32_t f(KYTY_CP_OP_PARSER_ARGS) + +namespace Libs::Graphics { + +namespace { + +constexpr uint32_t GcrGl2MetadataInvalidate = 1u << 1u; +constexpr uint32_t GcrGl0VectorInvalidate = 1u << 2u; +constexpr uint32_t GcrGl1Invalidate = 1u << 3u; +constexpr uint32_t GcrGl2Unshared = 1u << 4u; +constexpr uint32_t GcrGl2Invalidate = 1u << 8u; +constexpr uint32_t GcrGl2Writeback = 1u << 9u; +constexpr uint32_t GcrOrder012 = 1u << 10u; +constexpr uint32_t GcrOrder210 = 2u << 10u; +constexpr uint32_t GcrKnownMask = GcrGl2MetadataInvalidate | GcrGl0VectorInvalidate | + GcrGl1Invalidate | GcrGl2Unshared | GcrGl2Invalidate | + GcrGl2Writeback | GcrOrder012 | GcrOrder210; +constexpr uint32_t RegisterSelectorMask = 0x70000000u; + +uint32_t NormalizeRegisterOffset(uint32_t raw_offset) { + return (raw_offset & ~RegisterSelectorMask); +} + +bool ReleaseMemGcrNeedsBarrier(uint32_t eop_event_type, uint32_t gcr_cntl) { + return eop_event_type != 0x28u || + (gcr_cntl & (GcrGl2MetadataInvalidate | GcrGl0VectorInvalidate | GcrGl1Invalidate | + GcrGl2Invalidate | GcrGl2Writeback)) != 0; +} + +uint32_t ReleaseMemCacheActionFromGcr(uint32_t gcr_cntl) { + return ((gcr_cntl & GcrGl2Writeback) != 0 ? 0x38u : 0x00u); +} + +void LogUnknownReleaseMemGcr(uint32_t gcr_cntl) { + const uint32_t unknown = (gcr_cntl & ~GcrKnownMask); + if (unknown != 0) { + LOGF("\t warning: release_mem uses unknown GCR bits: 0x%04" PRIx32 "\n", unknown); + } +} + +} // namespace + +static HW::AaConfig ParseAaConfig(uint32_t value) { + HW::AaConfig r; + + r.msaa_num_samples = KYTY_PM4_GET(value, PA_SC_AA_CONFIG, MSAA_NUM_SAMPLES); + r.aa_mask_centroid_dtmn = KYTY_PM4_GET(value, PA_SC_AA_CONFIG, AA_MASK_CENTROID_DTMN) != 0; + r.max_sample_dist = KYTY_PM4_GET(value, PA_SC_AA_CONFIG, MAX_SAMPLE_DIST); + r.msaa_exposed_samples = KYTY_PM4_GET(value, PA_SC_AA_CONFIG, MSAA_EXPOSED_SAMPLES); + + return r; +} + +KYTY_HW_CTX_PARSER(HwCtxSetAaConfig) { + EXIT_NOT_IMPLEMENTED(cmd_id != 0xc0016900); + EXIT_NOT_IMPLEMENTED(cmd_offset != Pm4::PA_SC_AA_CONFIG); + + cp->GetCtx()->SetAaConfig(ParseAaConfig(buffer[0])); + + return 1; +} + +KYTY_HW_CTX_PARSER(HwCtxSetAaSampleControl) { + if (cmd_id == 0xc0016900 && cmd_offset >= Pm4::PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_0 && + cmd_offset < Pm4::PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_0 + 16) { + auto r = cp->GetCtx()->GetAaSampleControl(); + r.locations[cmd_offset - Pm4::PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_0] = buffer[0]; + cp->GetCtx()->SetAaSampleControl(r); + return 1; + } + + EXIT_NOT_IMPLEMENTED(cmd_id != 0xc0106900); + EXIT_NOT_IMPLEMENTED(cmd_offset != Pm4::PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_0); + + uint32_t count = 1; + + if (dw >= 20 && buffer[16] == 0xc0026900 && buffer[17] == Pm4::PA_SC_CENTROID_PRIORITY_0) { + count = 20; + + HW::AaSampleControl r; + + memcpy(r.locations, buffer, static_cast(16) * 4); + + r.centroid_priority = + static_cast(buffer[18]) | (static_cast(buffer[19]) << 32u); + + cp->GetCtx()->SetAaSampleControl(r); + } else { + KYTY_NOT_IMPLEMENTED; + } + + return count; +} + +KYTY_HW_CTX_PARSER(HwCtxSetCentroidPriority) { + auto num_values = KYTY_PM4_LEN(cmd_id) - 2u; + auto r = cp->GetCtx()->GetAaSampleControl(); + + for (uint32_t i = 0; i < num_values; i++) { + auto offset = cmd_offset + i; + switch (offset) { + case Pm4::PA_SC_CENTROID_PRIORITY_0: + r.centroid_priority = (r.centroid_priority & 0xffffffff00000000ull) | buffer[i]; + break; + case Pm4::PA_SC_CENTROID_PRIORITY_1: + r.centroid_priority = (r.centroid_priority & 0x00000000ffffffffull) | + (static_cast(buffer[i]) << 32u); + break; + default: break; + } + } + + cp->GetCtx()->SetAaSampleControl(r); + + return num_values; +} + +static void HwCtxIgnoreAaMaskRegister([[maybe_unused]] uint32_t cmd_offset, + [[maybe_unused]] uint32_t value) {} + +static void HwCtxIgnoreAlphaToMaskRegister([[maybe_unused]] uint32_t value) {} + +static void HwCtxIgnoreDrawPayloadControl([[maybe_unused]] uint32_t value) {} + +static void HwCtxIgnoreObjprimIdControl([[maybe_unused]] uint32_t value) {} + +static void HwCtxIgnorePrimitiveIdReset([[maybe_unused]] uint32_t value) {} + +static void HwCtxIgnoreVtxControl([[maybe_unused]] uint32_t value) {} + +static void HwCtxSetDepthBoundsRegister(HW::Context* ctx, uint32_t cmd_offset, uint32_t value) { + EXIT_IF(ctx == nullptr); + + auto fvalue = *reinterpret_cast(&value); + + switch (cmd_offset) { + case Pm4::DB_DEPTH_BOUNDS_MIN: ctx->SetDepthBoundsMin(fvalue); break; + case Pm4::DB_DEPTH_BOUNDS_MAX: ctx->SetDepthBoundsMax(fvalue); break; + default: EXIT("unknown depth bounds register: 0x%08" PRIx32 "\n", cmd_offset); + } +} + +static void HwCtxIgnoreDepthMetadataRegister([[maybe_unused]] uint32_t cmd_offset, + [[maybe_unused]] uint32_t value) {} + +static void HwCtxIgnoreFovWindow([[maybe_unused]] uint32_t value) {} + +static void HwCtxIgnoreFsrRegister([[maybe_unused]] uint32_t cmd_offset, + [[maybe_unused]] uint32_t value) {} + +static void HwCtxIgnoreScanModeControl1([[maybe_unused]] uint32_t value) {} + +static void HwCtxIgnorePaScExtendedControl([[maybe_unused]] uint32_t cmd_offset, + [[maybe_unused]] uint32_t value) {} + +static void HwCtxIgnoreCbDccControl([[maybe_unused]] uint32_t value) {} + +static void HwCtxIgnorePointState([[maybe_unused]] uint32_t cmd_offset, + [[maybe_unused]] uint32_t value) {} + +static void HwCtxIgnoreBorderColorTableAddr([[maybe_unused]] uint32_t cmd_offset, + [[maybe_unused]] uint32_t value) {} + +static void HwCtxIgnoreSpiTmpringSize(uint32_t value) { + static std::atomic log_count {0}; + if (log_count.fetch_add(1, std::memory_order_relaxed) < 4) { + LOGF("\t temporary: accepting SPI_TMPRING_SIZE = 0x%08" PRIx32 "\n", value); + } +} + +KYTY_HW_CTX_PARSER(HwCtxSetSpiTmpringSize) { + auto reg_num = (cmd_id >> 16u) & 0x3fffu; + + EXIT_NOT_IMPLEMENTED(reg_num != 1); + + HwCtxIgnoreSpiTmpringSize(buffer[0]); + + return 1; +} + +static bool HwCtxIsFakeRegister(uint32_t cmd_offset) { + switch (cmd_offset) { + case 0x18: + case 0x19: + case Pm4::CX_NOP: return true; + default: return false; + } +} + +static void HwCtxIgnoreFakeRegister([[maybe_unused]] uint32_t cmd_offset, + [[maybe_unused]] uint32_t value) {} + +static bool HwCtxTrySetFakeRegister(uint32_t cmd_offset, uint32_t value) { + if (!HwCtxIsFakeRegister(cmd_offset)) { + return false; + } + + if (cmd_offset != Pm4::CX_NOP) { + HwCtxIgnoreFakeRegister(cmd_offset, value); + } + + return true; +} + +KYTY_HW_CTX_PARSER(HwCtxSetDepthBounds) { + auto num_values = KYTY_PM4_LEN(cmd_id) - 2u; + + for (uint32_t i = 0; i < num_values; i++) { + HwCtxSetDepthBoundsRegister(cp->GetCtx(), cmd_offset + i, buffer[i]); + } + + return num_values; +} + +KYTY_HW_CTX_PARSER(HwCtxSetDepthMetadataRegisters) { + auto num_values = KYTY_PM4_LEN(cmd_id) - 2u; + + for (uint32_t i = 0; i < num_values; i++) { + HwCtxIgnoreDepthMetadataRegister(cmd_offset + i, buffer[i]); + } + + return num_values; +} + +KYTY_HW_CTX_PARSER(HwCtxSetBorderColorTableAddr) { + auto num_values = KYTY_PM4_LEN(cmd_id) - 2u; + + for (uint32_t i = 0; i < num_values; i++) { + HwCtxIgnoreBorderColorTableAddr(cmd_offset + i, buffer[i]); + } + + return num_values; +} + +KYTY_HW_CTX_PARSER(HwCtxSetPaScExtendedControl) { + auto num_values = KYTY_PM4_LEN(cmd_id) - 2u; + + for (uint32_t i = 0; i < num_values; i++) { + HwCtxIgnorePaScExtendedControl(cmd_offset + i, buffer[i]); + } + + return num_values; +} + +KYTY_HW_CTX_PARSER(HwCtxSetCbDccControl) { + auto num_values = KYTY_PM4_LEN(cmd_id) - 2u; + + for (uint32_t i = 0; i < num_values; i++) { + HwCtxIgnoreCbDccControl(buffer[i]); + } + + return num_values; +} + +KYTY_HW_CTX_PARSER(HwCtxSetPointState) { + auto num_values = KYTY_PM4_LEN(cmd_id) - 2u; + + for (uint32_t i = 0; i < num_values; i++) { + HwCtxIgnorePointState(cmd_offset + i, buffer[i]); + } + + return num_values; +} + +KYTY_HW_CTX_PARSER(HwCtxSetAlphaToMask) { + auto num_values = KYTY_PM4_LEN(cmd_id) - 2u; + + for (uint32_t i = 0; i < num_values; i++) { + HwCtxIgnoreAlphaToMaskRegister(buffer[i]); + } + + return num_values; +} + +KYTY_HW_CTX_PARSER(HwCtxSetDrawPayloadControl) { + auto num_values = KYTY_PM4_LEN(cmd_id) - 2u; + + for (uint32_t i = 0; i < num_values; i++) { + HwCtxIgnoreDrawPayloadControl(buffer[i]); + } + + return num_values; +} + +KYTY_HW_CTX_PARSER(HwCtxSetObjprimIdControl) { + auto num_values = KYTY_PM4_LEN(cmd_id) - 2u; + + for (uint32_t i = 0; i < num_values; i++) { + HwCtxIgnoreObjprimIdControl(buffer[i]); + } + + return num_values; +} + +KYTY_HW_CTX_PARSER(HwCtxSetPrimitiveIdReset) { + auto num_values = KYTY_PM4_LEN(cmd_id) - 2u; + + for (uint32_t i = 0; i < num_values; i++) { + HwCtxIgnorePrimitiveIdReset(buffer[i]); + } + + return num_values; +} + +KYTY_HW_CTX_PARSER(HwCtxSetVtxControl) { + auto num_values = KYTY_PM4_LEN(cmd_id) - 2u; + + for (uint32_t i = 0; i < num_values; i++) { + HwCtxIgnoreVtxControl(buffer[i]); + } + + return num_values; +} + +KYTY_HW_CTX_PARSER(HwCtxSetFovWindow) { + auto num_values = KYTY_PM4_LEN(cmd_id) - 2u; + + for (uint32_t i = 0; i < num_values; i++) { + HwCtxIgnoreFovWindow(buffer[i]); + } + + return num_values; +} + +KYTY_HW_CTX_PARSER(HwCtxSetScanModeControl1) { + auto num_values = KYTY_PM4_LEN(cmd_id) - 2u; + + for (uint32_t i = 0; i < num_values; i++) { + HwCtxIgnoreScanModeControl1(buffer[i]); + } + + return num_values; +} + +KYTY_HW_CTX_PARSER(HwCtxSetAaMask) { + auto num_values = KYTY_PM4_LEN(cmd_id) - 2u; + + for (uint32_t i = 0; i < num_values; i++) { + HwCtxIgnoreAaMaskRegister(cmd_offset + i, buffer[i]); + } + + return num_values; +} + +KYTY_HW_CTX_PARSER(HwCtxSetBlendColor) { + EXIT_NOT_IMPLEMENTED(cmd_id != 0xc0046900); + EXIT_NOT_IMPLEMENTED(cmd_offset != Pm4::CB_BLEND_RED); + + HW::BlendColor r; + + r.red = *reinterpret_cast(&buffer[0]); + r.green = *reinterpret_cast(&buffer[1]); + r.blue = *reinterpret_cast(&buffer[2]); + r.alpha = *reinterpret_cast(&buffer[3]); + + cp->GetCtx()->SetBlendColor(r); + + return 4; +} + +KYTY_HW_CTX_PARSER(HwCtxSetBlendControl) { + EXIT_NOT_IMPLEMENTED(cmd_id != 0xC0016900); + + uint32_t param = (cmd_offset - Pm4::CB_BLEND0_CONTROL) / 1; + + HW::BlendControl r; + + r.color_srcblend = (buffer[0] >> Pm4::CB_BLEND0_CONTROL_COLOR_SRCBLEND_SHIFT) & + Pm4::CB_BLEND0_CONTROL_COLOR_SRCBLEND_MASK; + r.color_comb_fcn = (buffer[0] >> Pm4::CB_BLEND0_CONTROL_COLOR_COMB_FCN_SHIFT) & + Pm4::CB_BLEND0_CONTROL_COLOR_COMB_FCN_MASK; + r.color_destblend = (buffer[0] >> Pm4::CB_BLEND0_CONTROL_COLOR_DESTBLEND_SHIFT) & + Pm4::CB_BLEND0_CONTROL_COLOR_DESTBLEND_MASK; + r.alpha_srcblend = (buffer[0] >> Pm4::CB_BLEND0_CONTROL_ALPHA_SRCBLEND_SHIFT) & + Pm4::CB_BLEND0_CONTROL_ALPHA_SRCBLEND_MASK; + r.alpha_comb_fcn = (buffer[0] >> Pm4::CB_BLEND0_CONTROL_ALPHA_COMB_FCN_SHIFT) & + Pm4::CB_BLEND0_CONTROL_ALPHA_COMB_FCN_MASK; + r.alpha_destblend = (buffer[0] >> Pm4::CB_BLEND0_CONTROL_ALPHA_DESTBLEND_SHIFT) & + Pm4::CB_BLEND0_CONTROL_ALPHA_DESTBLEND_MASK; + r.separate_alpha_blend = ((buffer[0] >> Pm4::CB_BLEND0_CONTROL_SEPARATE_ALPHA_BLEND_SHIFT) & + Pm4::CB_BLEND0_CONTROL_SEPARATE_ALPHA_BLEND_MASK) != 0; + r.enable = ((buffer[0] >> Pm4::CB_BLEND0_CONTROL_ENABLE_SHIFT) & + Pm4::CB_BLEND0_CONTROL_ENABLE_MASK) != 0; + + cp->GetCtx()->SetBlendControl(param, r); + + return 1; +} + +KYTY_HW_CTX_PARSER(HwCtxSetClipControl) { + EXIT_NOT_IMPLEMENTED(cmd_id != 0xC0016900); + EXIT_NOT_IMPLEMENTED(cmd_offset != Pm4::PA_CL_CLIP_CNTL); + + HW::ClipControl r; + + // r.user_clip_planes = buffer[0] & 0x3fu; + // r.user_clip_plane_mode = (buffer[0] >> 14u) & 0x3u; + // r.clip_space = (buffer[0] >> 19u) & 0x1u; + // r.vertex_kill_mode = (buffer[0] >> 21u) & 0x1u; + // r.min_z_clip_enable = (buffer[0] >> 26u) & 0x1u; + // r.max_z_clip_enable = (buffer[0] >> 27u) & 0x1u; + // r.user_clip_plane_negate_y = (buffer[0] & 0x00002000u) != 0; + // r.clip_enable = (buffer[0] & 0x00010000u) != 0; + // r.user_clip_plane_cull_only = (buffer[0] & 0x00020000u) != 0; + // r.cull_on_clipping_error_disable = (buffer[0] & 0x00100000u) != 0; + // r.linear_attribute_clip_enable = (buffer[0] & 0x01000000u) != 0; + // r.force_viewport_index_from_vs_enable = (buffer[0] & 0x02000000u) != 0; + + r.user_clip_planes = KYTY_PM4_GET(buffer[0], PA_CL_CLIP_CNTL, UCP_ENA); + r.user_clip_plane_mode = KYTY_PM4_GET(buffer[0], PA_CL_CLIP_CNTL, PS_UCP_MODE); + r.dx_clip_space = KYTY_PM4_GET(buffer[0], PA_CL_CLIP_CNTL, DX_CLIP_SPACE_DEF) != 0; + r.vertex_kill_any = KYTY_PM4_GET(buffer[0], PA_CL_CLIP_CNTL, VTX_KILL_OR) != 0; + r.min_z_clip_disable = KYTY_PM4_GET(buffer[0], PA_CL_CLIP_CNTL, ZCLIP_NEAR_DISABLE) != 0; + r.max_z_clip_disable = KYTY_PM4_GET(buffer[0], PA_CL_CLIP_CNTL, ZCLIP_FAR_DISABLE) != 0; + r.user_clip_plane_negate_y = KYTY_PM4_GET(buffer[0], PA_CL_CLIP_CNTL, PS_UCP_Y_SCALE_NEG) != 0; + r.clip_disable = KYTY_PM4_GET(buffer[0], PA_CL_CLIP_CNTL, CLIP_DISABLE) != 0; + r.user_clip_plane_cull_only = KYTY_PM4_GET(buffer[0], PA_CL_CLIP_CNTL, UCP_CULL_ONLY_ENA) != 0; + r.cull_on_clipping_error_disable = + KYTY_PM4_GET(buffer[0], PA_CL_CLIP_CNTL, DIS_CLIP_ERR_DETECT) != 0; + r.linear_attribute_clip_enable = + KYTY_PM4_GET(buffer[0], PA_CL_CLIP_CNTL, DX_LINEAR_ATTR_CLIP_ENA) != 0; + r.force_viewport_index_from_vs_enable = + KYTY_PM4_GET(buffer[0], PA_CL_CLIP_CNTL, VTE_VPORT_PROVOKE_DISABLE) != 0; + + cp->GetCtx()->SetClipControl(r); + + return 1; +} + +KYTY_HW_CTX_PARSER(HwCtxSetColorControl) { + EXIT_NOT_IMPLEMENTED(cmd_id != 0xc0016900); + EXIT_NOT_IMPLEMENTED(cmd_offset != Pm4::CB_COLOR_CONTROL); + + HW::ColorControl r; + + r.mode = KYTY_PM4_GET(buffer[0], CB_COLOR_CONTROL, MODE); + r.op = KYTY_PM4_GET(buffer[0], CB_COLOR_CONTROL, ROP3); + + cp->GetCtx()->SetColorControl(r); + + return 1; +} + +KYTY_HW_CTX_PARSER(HwCtxSetColorInfo) { + EXIT_NOT_IMPLEMENTED(cmd_id != 0xc0016900); + + uint32_t param = (cmd_offset - Pm4::CB_COLOR0_INFO) / 15; + + HW::ColorInfo r; + + // r.fmask_compression_enable = (buffer[4] & 0x4000u) != 0; + // // r.fmask_compression_mode = (buffer[4] >> 26u) & 0x3u; + // r.fmask_data_compression_disable = ((buffer[4] >> 26u) & 0x1u) != 0; + // r.fmask_one_frag_mode = ((buffer[4] >> 27u) & 0x1u) != 0; + // r.cmask_fast_clear_enable = (buffer[4] & 0x2000u) != 0; + // r.dcc_compression_enable = (buffer[4] & 0x10000000u) != 0; + // r.cmask_is_linear = (buffer[4] >> 19u) & 0x1u; + // r.cmask_addr_type = (buffer[4] >> 29u) & 0x3u; + // r.alt_tile_mode = ((buffer[4] >> 31u) & 0x1u) != 0; + // r.format = (buffer[4] >> 2u) & 0x1fu; + // r.channel_type = (buffer[4] >> 8u) & 0x7u; + // r.channel_order = (buffer[4] >> 11u) & 0x3u; + r.format = KYTY_PM4_GET(buffer[0], CB_COLOR0_INFO, FORMAT); + r.channel_type = KYTY_PM4_GET(buffer[0], CB_COLOR0_INFO, NUMBER_TYPE); + r.channel_order = KYTY_PM4_GET(buffer[0], CB_COLOR0_INFO, COMP_SWAP); + r.cmask_fast_clear_enable = KYTY_PM4_GET(buffer[0], CB_COLOR0_INFO, FAST_CLEAR) != 0; + r.fmask_compression_enable = KYTY_PM4_GET(buffer[0], CB_COLOR0_INFO, COMPRESSION) != 0; + r.blend_clamp = KYTY_PM4_GET(buffer[0], CB_COLOR0_INFO, BLEND_CLAMP) != 0; + r.blend_bypass = KYTY_PM4_GET(buffer[0], CB_COLOR0_INFO, BLEND_BYPASS) != 0; + r.round_mode = KYTY_PM4_GET(buffer[0], CB_COLOR0_INFO, ROUND_MODE) != 0; + r.cmask_is_linear = KYTY_PM4_GET(buffer[0], CB_COLOR0_INFO, CMASK_IS_LINEAR); + r.fmask_data_compression_disable = + KYTY_PM4_GET(buffer[0], CB_COLOR0_INFO, FMASK_COMPRESSION_DISABLE) != 0; + r.fmask_one_frag_mode = KYTY_PM4_GET(buffer[0], CB_COLOR0_INFO, FMASK_COMPRESS_1FRAG_ONLY) != 0; + r.dcc_compression_enable = KYTY_PM4_GET(buffer[0], CB_COLOR0_INFO, DCC_ENABLE) != 0; + r.cmask_addr_type = KYTY_PM4_GET(buffer[0], CB_COLOR0_INFO, CMASK_ADDR_TYPE); + r.alt_tile_mode = KYTY_PM4_GET(buffer[0], CB_COLOR0_INFO, ALT_TILE_MODE) != 0; + + cp->GetCtx()->SetColorInfo(param, r); + + return 1; +} + +KYTY_HW_CTX_PARSER(HwCtxSetDepthClear) { + EXIT_NOT_IMPLEMENTED(cmd_id != 0xC0016900); + EXIT_NOT_IMPLEMENTED(cmd_offset != Pm4::DB_DEPTH_CLEAR); + + cp->GetCtx()->SetDepthClearValue(*reinterpret_cast(buffer)); + + return 1; +} + +KYTY_HW_CTX_PARSER(HwCtxSetDepthControl) { + EXIT_NOT_IMPLEMENTED(cmd_id != 0xC0016900); + EXIT_NOT_IMPLEMENTED(cmd_offset != Pm4::DB_DEPTH_CONTROL); + + HW::DepthControl r; + + r.stencil_enable = KYTY_PM4_GET(buffer[0], DB_DEPTH_CONTROL, STENCIL_ENABLE) != 0; + r.z_enable = KYTY_PM4_GET(buffer[0], DB_DEPTH_CONTROL, Z_ENABLE) != 0; + r.z_write_enable = KYTY_PM4_GET(buffer[0], DB_DEPTH_CONTROL, Z_WRITE_ENABLE) != 0; + r.depth_bounds_enable = KYTY_PM4_GET(buffer[0], DB_DEPTH_CONTROL, DEPTH_BOUNDS_ENABLE) != 0; + r.zfunc = KYTY_PM4_GET(buffer[0], DB_DEPTH_CONTROL, ZFUNC); + r.backface_enable = KYTY_PM4_GET(buffer[0], DB_DEPTH_CONTROL, BACKFACE_ENABLE) != 0; + r.stencilfunc = KYTY_PM4_GET(buffer[0], DB_DEPTH_CONTROL, STENCILFUNC); + r.stencilfunc_bf = KYTY_PM4_GET(buffer[0], DB_DEPTH_CONTROL, STENCILFUNC_BF); + r.color_writes_on_depth_fail_enable = + KYTY_PM4_GET(buffer[0], DB_DEPTH_CONTROL, ENABLE_COLOR_WRITES_ON_DEPTH_FAIL) != 0; + r.color_writes_on_depth_pass_disable = + KYTY_PM4_GET(buffer[0], DB_DEPTH_CONTROL, DISABLE_COLOR_WRITES_ON_DEPTH_PASS) != 0; + + cp->GetCtx()->SetDepthControl(r); + + return 1; +} + +static HW::DepthRenderTargetHTileSurface ParseDepthHtileSurface(uint32_t value) { + HW::DepthRenderTargetHTileSurface r; + r.linear = KYTY_PM4_GET(value, DB_HTILE_SURFACE, LINEAR); + r.full_cache = KYTY_PM4_GET(value, DB_HTILE_SURFACE, FULL_CACHE); + r.htile_uses_preload_win = KYTY_PM4_GET(value, DB_HTILE_SURFACE, HTILE_USES_PRELOAD_WIN); + r.preload = KYTY_PM4_GET(value, DB_HTILE_SURFACE, PRELOAD); + r.prefetch_width = KYTY_PM4_GET(value, DB_HTILE_SURFACE, PREFETCH_WIDTH); + r.prefetch_height = KYTY_PM4_GET(value, DB_HTILE_SURFACE, PREFETCH_HEIGHT); + r.dst_outside_zero_to_one = KYTY_PM4_GET(value, DB_HTILE_SURFACE, DST_OUTSIDE_ZERO_TO_ONE); + return r; +} + +static void HwCtxSetDepthHtileSurfaceRegister(HW::Context* ctx, uint32_t value) { + EXIT_IF(ctx == nullptr); + + auto target = ctx->GetDepthRenderTarget(); + target.htile_surface = ParseDepthHtileSurface(value); + ctx->SetDepthRenderTarget(target); +} + +KYTY_HW_CTX_PARSER(HwCtxSetDepthHtileSurface) { + auto num_values = KYTY_PM4_LEN(cmd_id) - 2u; + + EXIT_NOT_IMPLEMENTED(num_values != 1); + EXIT_NOT_IMPLEMENTED(cmd_offset != Pm4::DB_HTILE_SURFACE); + + HwCtxSetDepthHtileSurfaceRegister(cp->GetCtx(), buffer[0]); + + return num_values; +} + +KYTY_HW_CTX_PARSER(HwCtxSetDepthRenderTarget) { + EXIT_NOT_IMPLEMENTED(cmd_offset != Pm4::DB_Z_INFO); + + uint32_t count = 1; + + if (cmd_id == 0xC0016900) { + HW::DepthZInfo r; + + // r.expclear_enabled = (buffer[0] & 0x08000000u) != 0; + // r.format = (buffer[0] >> Pm4::DB_Z_INFO_FORMAT_SHIFT) & + // Pm4::DB_Z_INFO_FORMAT_MASK; r.num_samples = (buffer[0] >> + // Pm4::DB_Z_INFO_NUM_SAMPLES_SHIFT) & Pm4::DB_Z_INFO_NUM_SAMPLES_MASK; r.tile_mode_index = + //(buffer[0] >> Pm4::DB_Z_INFO_TILE_MODE_INDEX_SHIFT) & Pm4::DB_Z_INFO_TILE_MODE_INDEX_MASK; + // r.tile_surface_enable = ((buffer[0] >> Pm4::DB_Z_INFO_TILE_SURFACE_ENABLE_SHIFT) & + // Pm4::DB_Z_INFO_TILE_SURFACE_ENABLE_MASK) !=0 + // r.zrange_precision = (buffer[0] >> Pm4::DB_Z_INFO_ZRANGE_PRECISION_SHIFT) & + // Pm4::DB_Z_INFO_ZRANGE_PRECISION_MASK; + + r.format = KYTY_PM4_GET(buffer[0], DB_Z_INFO, FORMAT); + r.num_samples = KYTY_PM4_GET(buffer[0], DB_Z_INFO, NUM_SAMPLES); + r.embedded_sample_locations = KYTY_PM4_GET(buffer[0], DB_Z_INFO, ITERATE_FLUSH) != 0; + r.partially_resident = KYTY_PM4_GET(buffer[0], DB_Z_INFO, PARTIALLY_RESIDENT) != 0; + r.num_mip_levels = KYTY_PM4_GET(buffer[0], DB_Z_INFO, MAXMIP); + r.tile_mode_index = KYTY_PM4_GET(buffer[0], DB_Z_INFO, TILE_MODE_INDEX); + r.plane_compression = KYTY_PM4_GET(buffer[0], DB_Z_INFO, DECOMPRESS_ON_N_ZPLANES); + r.expclear_enabled = KYTY_PM4_GET(buffer[0], DB_Z_INFO, ALLOW_EXPCLEAR) != 0; + r.tile_surface_enable = KYTY_PM4_GET(buffer[0], DB_Z_INFO, TILE_SURFACE_ENABLE) != 0; + r.zrange_precision = KYTY_PM4_GET(buffer[0], DB_Z_INFO, ZRANGE_PRECISION); + + cp->GetCtx()->SetDepthZInfo(r); + } else if (cmd_id == 0xC0086900) { + if (dw >= 22 && buffer[8] == 0xC0016900 && buffer[9] == Pm4::DB_DEPTH_INFO && + buffer[11] == 0xC0016900 && buffer[12] == Pm4::DB_DEPTH_VIEW && + buffer[14] == 0xC0016900 && buffer[15] == Pm4::DB_HTILE_DATA_BASE && + buffer[17] == 0xC0016900 && buffer[18] == Pm4::DB_HTILE_SURFACE && + buffer[20] == 0xC0001000) { + count = 22; + + HW::DepthRenderTarget z; + + // z.z_info.expclear_enabled = (buffer[0] & 0x08000000u) != 0; + // z.z_info.format = (buffer[0] >> Pm4::DB_Z_INFO_FORMAT_SHIFT) & + // Pm4::DB_Z_INFO_FORMAT_MASK; z.z_info.num_samples = (buffer[0] >> + // Pm4::DB_Z_INFO_NUM_SAMPLES_SHIFT) & Pm4::DB_Z_INFO_NUM_SAMPLES_MASK; + // z.z_info.tile_mode_index = (buffer[0] >> + // Pm4::DB_Z_INFO_TILE_MODE_INDEX_SHIFT) & + // Pm4::DB_Z_INFO_TILE_MODE_INDEX_MASK; z.z_info.tile_surface_enable = + // KYTY_PM4_GET(buffer[0], DB_Z_INFO, TILE_SURFACE_ENABLE) != 0; + // z.z_info.zrange_precision = (buffer[0] >> Pm4::DB_Z_INFO_ZRANGE_PRECISION_SHIFT) & + // Pm4::DB_Z_INFO_ZRANGE_PRECISION_MASK; + + z.z_info.format = KYTY_PM4_GET(buffer[0], DB_Z_INFO, FORMAT); + z.z_info.num_samples = KYTY_PM4_GET(buffer[0], DB_Z_INFO, NUM_SAMPLES); + z.z_info.embedded_sample_locations = + KYTY_PM4_GET(buffer[0], DB_Z_INFO, ITERATE_FLUSH) != 0; + z.z_info.partially_resident = + KYTY_PM4_GET(buffer[0], DB_Z_INFO, PARTIALLY_RESIDENT) != 0; + z.z_info.num_mip_levels = KYTY_PM4_GET(buffer[0], DB_Z_INFO, MAXMIP); + z.z_info.tile_mode_index = KYTY_PM4_GET(buffer[0], DB_Z_INFO, TILE_MODE_INDEX); + z.z_info.plane_compression = + KYTY_PM4_GET(buffer[0], DB_Z_INFO, DECOMPRESS_ON_N_ZPLANES); + z.z_info.expclear_enabled = KYTY_PM4_GET(buffer[0], DB_Z_INFO, ALLOW_EXPCLEAR) != 0; + z.z_info.tile_surface_enable = + KYTY_PM4_GET(buffer[0], DB_Z_INFO, TILE_SURFACE_ENABLE) != 0; + z.z_info.zrange_precision = KYTY_PM4_GET(buffer[0], DB_Z_INFO, ZRANGE_PRECISION); + + // z.stencil_info.expclear_enabled = (buffer[1] & 0x08000000u) != 0; + // z.stencil_info.tile_split = (buffer[1] >> 13u) & 0x7u; + // z.stencil_info.format = KYTY_PM4_GET(buffer[1], + // DB_STENCIL_INFO, FORMAT); z.stencil_info.tile_mode_index = + // KYTY_PM4_GET(buffer[1], DB_STENCIL_INFO, TILE_MODE_INDEX); + // z.stencil_info.tile_stencil_disable = KYTY_PM4_GET(buffer[1], + // DB_STENCIL_INFO, TILE_STENCIL_DISABLE); + z.stencil_info.format = KYTY_PM4_GET(buffer[1], DB_STENCIL_INFO, FORMAT); + z.stencil_info.texture_compatible_stencil = + KYTY_PM4_GET(buffer[1], DB_STENCIL_INFO, ITERATE_FLUSH) != 0; + z.stencil_info.partially_resident = + KYTY_PM4_GET(buffer[1], DB_STENCIL_INFO, PARTIALLY_RESIDENT) != 0; + z.stencil_info.tile_split = KYTY_PM4_GET(buffer[1], DB_STENCIL_INFO, RESERVED_FIELD_1); + z.stencil_info.tile_mode_index = + KYTY_PM4_GET(buffer[1], DB_STENCIL_INFO, TILE_MODE_INDEX); + z.stencil_info.expclear_enabled = + KYTY_PM4_GET(buffer[1], DB_STENCIL_INFO, ALLOW_EXPCLEAR) != 0; + z.stencil_info.tile_stencil_disable = + KYTY_PM4_GET(buffer[1], DB_STENCIL_INFO, TILE_STENCIL_DISABLE) != 0; + + z.z_read_base_addr = static_cast(buffer[2]) << 8u; + z.stencil_read_base_addr = static_cast(buffer[3]) << 8u; + z.z_write_base_addr = static_cast(buffer[4]) << 8u; + z.stencil_write_base_addr = static_cast(buffer[5]) << 8u; + + // DB_DEPTH_SIZE + z.pitch_div8_minus1 = (buffer[6] >> Pm4::DB_DEPTH_SIZE_PITCH_TILE_MAX_SHIFT) & + Pm4::DB_DEPTH_SIZE_PITCH_TILE_MAX_MASK; + z.height_div8_minus1 = (buffer[6] >> Pm4::DB_DEPTH_SIZE_HEIGHT_TILE_MAX_SHIFT) & + Pm4::DB_DEPTH_SIZE_HEIGHT_TILE_MAX_MASK; + z.pitch_height_valid = true; + + // DB_DEPTH_SLICE + z.slice_div64_minus1 = (buffer[7] >> Pm4::DB_DEPTH_SLICE_SLICE_TILE_MAX_SHIFT) & + Pm4::DB_DEPTH_SLICE_SLICE_TILE_MAX_MASK; + + z.depth_info.addr5_swizzle_mask = + KYTY_PM4_GET(buffer[10], DB_DEPTH_INFO, ADDR5_SWIZZLE_MASK); + z.depth_info.array_mode = (buffer[10] >> Pm4::DB_DEPTH_INFO_ARRAY_MODE_SHIFT) & + Pm4::DB_DEPTH_INFO_ARRAY_MODE_MASK; + z.depth_info.pipe_config = (buffer[10] >> Pm4::DB_DEPTH_INFO_PIPE_CONFIG_SHIFT) & + Pm4::DB_DEPTH_INFO_PIPE_CONFIG_MASK; + z.depth_info.bank_width = (buffer[10] >> Pm4::DB_DEPTH_INFO_BANK_WIDTH_SHIFT) & + Pm4::DB_DEPTH_INFO_BANK_WIDTH_MASK; + z.depth_info.bank_height = (buffer[10] >> Pm4::DB_DEPTH_INFO_BANK_HEIGHT_SHIFT) & + Pm4::DB_DEPTH_INFO_BANK_HEIGHT_MASK; + z.depth_info.macro_tile_aspect = + KYTY_PM4_GET(buffer[10], DB_DEPTH_INFO, MACRO_TILE_ASPECT); + z.depth_info.num_banks = (buffer[10] >> Pm4::DB_DEPTH_INFO_NUM_BANKS_SHIFT) & + Pm4::DB_DEPTH_INFO_NUM_BANKS_MASK; + + // z.depth_view.slice_start = (buffer[13] >> Pm4::DB_DEPTH_VIEW_SLICE_START_SHIFT) & + // Pm4::DB_DEPTH_VIEW_SLICE_START_MASK; z.depth_view.slice_max = (buffer[13] >> + // Pm4::DB_DEPTH_VIEW_SLICE_MAX_SHIFT) & Pm4::DB_DEPTH_VIEW_SLICE_MAX_MASK; + + z.depth_view.slice_start = + KYTY_PM4_GET(buffer[13], DB_DEPTH_VIEW, SLICE_START) + + (KYTY_PM4_GET(buffer[13], DB_DEPTH_VIEW, SLICE_START_HI) << 11u); + z.depth_view.slice_max = KYTY_PM4_GET(buffer[13], DB_DEPTH_VIEW, SLICE_MAX) + + (KYTY_PM4_GET(buffer[13], DB_DEPTH_VIEW, SLICE_MAX_HI) << 11u); + z.depth_view.depth_write_disable = + KYTY_PM4_GET(buffer[13], DB_DEPTH_VIEW, Z_READ_ONLY) != 0; + z.depth_view.stencil_write_disable = + KYTY_PM4_GET(buffer[13], DB_DEPTH_VIEW, STENCIL_READ_ONLY) != 0; + z.depth_view.current_mip_level = KYTY_PM4_GET(buffer[13], DB_DEPTH_VIEW, MIPID); + + z.htile_data_base_addr = static_cast(buffer[16]) << 8u; + + z.htile_surface = ParseDepthHtileSurface(buffer[19]); + + if (buffer[21] != 0) { + // CxDepthRenderTarget stores width/height as value - 1 in register 13. + z.width = ((buffer[21] >> 0u) & 0x3fffu) + 1u; + z.height = ((buffer[21] >> 16u) & 0x3fffu) + 1u; + z.width_height_valid = true; + } + + cp->GetCtx()->SetDepthRenderTarget(z); + } else { + KYTY_NOT_IMPLEMENTED; + } + } else { + KYTY_NOT_IMPLEMENTED; + } + + return count; +} + +static HW::EqaaControl ParseEqaaControl(uint32_t value) { + HW::EqaaControl r; + + r.max_anchor_samples = KYTY_PM4_GET(value, DB_EQAA, MAX_ANCHOR_SAMPLES); + r.ps_iter_samples = KYTY_PM4_GET(value, DB_EQAA, PS_ITER_SAMPLES); + r.mask_export_num_samples = KYTY_PM4_GET(value, DB_EQAA, MASK_EXPORT_NUM_SAMPLES); + r.alpha_to_mask_num_samples = KYTY_PM4_GET(value, DB_EQAA, ALPHA_TO_MASK_NUM_SAMPLES); + r.high_quality_intersections = KYTY_PM4_GET(value, DB_EQAA, HIGH_QUALITY_INTERSECTIONS) != 0; + r.incoherent_eqaa_reads = KYTY_PM4_GET(value, DB_EQAA, INCOHERENT_EQAA_READS) != 0; + r.interpolate_comp_z = KYTY_PM4_GET(value, DB_EQAA, INTERPOLATE_COMP_Z) != 0; + r.static_anchor_associations = KYTY_PM4_GET(value, DB_EQAA, STATIC_ANCHOR_ASSOCIATIONS) != 0; + + return r; +} + +KYTY_HW_CTX_PARSER(HwCtxSetEqaaControl) { + EXIT_NOT_IMPLEMENTED(cmd_id != 0xC0016900); + EXIT_NOT_IMPLEMENTED(cmd_offset != Pm4::DB_EQAA); + + cp->GetCtx()->SetEqaaControl(ParseEqaaControl(buffer[0])); + + return 1; +} + +KYTY_HW_CTX_PARSER(HwCtxSetGenericScissor) { + auto num_values = KYTY_PM4_LEN(cmd_id) - 2u; + EXIT_NOT_IMPLEMENTED(num_values == 0); + EXIT_NOT_IMPLEMENTED(cmd_offset < Pm4::PA_SC_GENERIC_SCISSOR_TL); + EXIT_NOT_IMPLEMENTED(cmd_offset + num_values - 1u > Pm4::PA_SC_GENERIC_SCISSOR_BR); + + for (uint32_t i = 0; i < num_values; i++) { + g_hw_ctx_indirect_func[(cmd_offset + i) & (Pm4::CX_NUM - 1)](cp, cmd_offset + i, buffer[i]); + } + + return num_values; +} + +static int ParseScissorCoord15(uint32_t value) { + return static_cast(static_cast(value & 0x7fffu)); +} + +KYTY_HW_CTX_PARSER(HwCtxSetViewportScissor) { + auto reg_num = (cmd_id >> 16u) & 0x3fffu; + EXIT_NOT_IMPLEMENTED(reg_num == 0); + EXIT_NOT_IMPLEMENTED(cmd_offset < Pm4::PA_SC_VPORT_SCISSOR_0_TL); + EXIT_NOT_IMPLEMENTED(cmd_offset + reg_num - 1u > Pm4::PA_SC_VPORT_SCISSOR_15_BR); + + for (uint32_t i = 0; i < reg_num; i++) { + const auto reg = cmd_offset + i; + const auto viewport = (reg - Pm4::PA_SC_VPORT_SCISSOR_0_TL) / 2u; + const auto value = buffer[i]; + + EXIT_NOT_IMPLEMENTED(viewport >= 16); + if (((reg - Pm4::PA_SC_VPORT_SCISSOR_0_TL) % 2u) == 0) { + const int left = ParseScissorCoord15(value); + const int top = ParseScissorCoord15(value >> 16u); + const bool window_offset_disable = (value & 0x80000000u) != 0; + cp->GetCtx()->SetViewportScissorTL(viewport, left, top, !window_offset_disable); + } else { + const int right = ParseScissorCoord15(value); + const int bottom = ParseScissorCoord15(value >> 16u); + cp->GetCtx()->SetViewportScissorBR(viewport, right, bottom); + } + } + + return reg_num; +} + +static void HwCtxSetClipRectRegister(HW::Context* ctx, uint32_t cmd_offset, uint32_t value) { + EXIT_IF(ctx == nullptr); + + if (cmd_offset == Pm4::PA_SC_CLIPRECT_RULE) { + ctx->SetClipRectRule(static_cast(value & 0xffffu)); + return; + } + + if (cmd_offset >= Pm4::PA_SC_CLIPRECT_0_TL && cmd_offset < Pm4::PA_SC_CLIPRECT_0_TL + 8u) { + const uint32_t rect_id = (cmd_offset - Pm4::PA_SC_CLIPRECT_0_TL) / 2u; + if (((cmd_offset - Pm4::PA_SC_CLIPRECT_0_TL) & 1u) == 0) { + const int left = ParseScissorCoord15(value); + const int top = ParseScissorCoord15(value >> 16u); + const bool window_offset_disable = (value & 0x80000000u) != 0; + ctx->SetClipRectTL(rect_id, left, top, !window_offset_disable); + } else { + const int right = ParseScissorCoord15(value); + const int bottom = ParseScissorCoord15(value >> 16u); + ctx->SetClipRectBR(rect_id, right, bottom); + } + return; + } + + EXIT("unknown clip-rect register: 0x%08" PRIx32 "\n", cmd_offset); +} + +KYTY_HW_CTX_PARSER(HwCtxSetClipRect) { + const auto value_count = (cmd_id >> 16u) & 0x3fffu; + EXIT_NOT_IMPLEMENTED(value_count == 0); + EXIT_NOT_IMPLEMENTED(cmd_offset < Pm4::PA_SC_CLIPRECT_RULE); + EXIT_NOT_IMPLEMENTED(cmd_offset + value_count > Pm4::PA_SC_CLIPRECT_0_TL + 8u); + + for (uint32_t i = 0; i < value_count; i++) { + HwCtxSetClipRectRegister(cp->GetCtx(), cmd_offset + i, buffer[i]); + } + + return value_count; +} + +KYTY_HW_CTX_PARSER(HwCtxSetGuardBands) { + EXIT_NOT_IMPLEMENTED(cmd_id != 0xC0046900); + EXIT_NOT_IMPLEMENTED(cmd_offset != Pm4::PA_CL_GB_VERT_CLIP_ADJ); + + auto vert_clip = *reinterpret_cast(&buffer[0]); // PA_CL_GB_VERT_CLIP_ADJ + auto vert_discard = *reinterpret_cast(&buffer[1]); // PA_CL_GB_VERT_DISC_ADJ + auto horz_clip = *reinterpret_cast(&buffer[2]); // PA_CL_GB_HORZ_CLIP_ADJ + auto horz_discard = *reinterpret_cast(&buffer[3]); // PA_CL_GB_HORZ_DISC_ADJ + + cp->GetCtx()->SetGuardBands(horz_clip, vert_clip, horz_discard, vert_discard); + + return 4; +} + +KYTY_HW_CTX_PARSER(HwCtxSetHardwareScreenOffset) { + EXIT_NOT_IMPLEMENTED(cmd_id != 0xC0016900); + EXIT_NOT_IMPLEMENTED(cmd_offset != Pm4::PA_SU_HARDWARE_SCREEN_OFFSET); + + // uint32_t x = static_cast(buffer[0] & 0xffffu); + // uint32_t y = static_cast((buffer[0] >> 16u) & 0xffffu); + + uint32_t x = KYTY_PM4_GET(buffer[0], PA_SU_HARDWARE_SCREEN_OFFSET, HW_SCREEN_OFFSET_X); + uint32_t y = KYTY_PM4_GET(buffer[0], PA_SU_HARDWARE_SCREEN_OFFSET, HW_SCREEN_OFFSET_Y); + + cp->GetCtx()->SetHardwareScreenOffset(x, y); + + return 1; +} + +KYTY_HW_CTX_PARSER(HwCtxSetWindowOffset) { + EXIT_NOT_IMPLEMENTED(cmd_id != 0xC0016900); + EXIT_NOT_IMPLEMENTED(cmd_offset != Pm4::PA_SC_WINDOW_OFFSET); + + int offset_x = static_cast( + static_cast(KYTY_PM4_GET(buffer[0], PA_SC_WINDOW_OFFSET, WINDOW_X))); + int offset_y = static_cast( + static_cast(KYTY_PM4_GET(buffer[0], PA_SC_WINDOW_OFFSET, WINDOW_Y))); + + cp->GetCtx()->SetWindowOffset(offset_x, offset_y); + + return 1; +} + +KYTY_HW_CTX_PARSER(HwCtxSetWindowScissor) { + auto num_values = KYTY_PM4_LEN(cmd_id) - 2u; + EXIT_NOT_IMPLEMENTED(num_values == 0); + EXIT_NOT_IMPLEMENTED(cmd_offset < Pm4::PA_SC_WINDOW_SCISSOR_TL); + EXIT_NOT_IMPLEMENTED(cmd_offset + num_values - 1u > Pm4::PA_SC_WINDOW_SCISSOR_BR); + + for (uint32_t i = 0; i < num_values; i++) { + g_hw_ctx_indirect_func[(cmd_offset + i) & (Pm4::CX_NUM - 1)](cp, cmd_offset + i, buffer[i]); + } + + return num_values; +} + +KYTY_HW_CTX_PARSER(HwCtxSetLineControl) { + EXIT_NOT_IMPLEMENTED(cmd_id != 0xC0016900); + EXIT_NOT_IMPLEMENTED(cmd_offset != Pm4::PA_SU_LINE_CNTL); + + auto line_width = KYTY_PM4_GET(buffer[0], PA_SU_LINE_CNTL, WIDTH); + + if (line_width == 8) { + cp->GetCtx()->SetLineWidth(1.0f); + } else { + cp->GetCtx()->SetLineWidth(static_cast(line_width) / 8.0f); + } + + return 1; +} + +KYTY_HW_CTX_PARSER(HwCtxSetModeControl) { + EXIT_NOT_IMPLEMENTED(cmd_id != 0xC0016900); + EXIT_NOT_IMPLEMENTED(cmd_offset != Pm4::PA_SU_SC_MODE_CNTL); + + HW::ModeControl r; + + r.cull_front = KYTY_PM4_GET(buffer[0], PA_SU_SC_MODE_CNTL, CULL_FRONT) != 0; + r.cull_back = KYTY_PM4_GET(buffer[0], PA_SU_SC_MODE_CNTL, CULL_BACK) != 0; + r.face = KYTY_PM4_GET(buffer[0], PA_SU_SC_MODE_CNTL, FACE) != 0; + r.poly_mode = KYTY_PM4_GET(buffer[0], PA_SU_SC_MODE_CNTL, POLY_MODE); + r.polymode_front_ptype = KYTY_PM4_GET(buffer[0], PA_SU_SC_MODE_CNTL, POLYMODE_FRONT_PTYPE); + r.polymode_back_ptype = KYTY_PM4_GET(buffer[0], PA_SU_SC_MODE_CNTL, POLYMODE_BACK_PTYPE); + r.poly_offset_front_enable = + KYTY_PM4_GET(buffer[0], PA_SU_SC_MODE_CNTL, POLY_OFFSET_FRONT_ENABLE) != 0; + r.poly_offset_back_enable = + KYTY_PM4_GET(buffer[0], PA_SU_SC_MODE_CNTL, POLY_OFFSET_BACK_ENABLE) != 0; + r.vtx_window_offset_enable = + KYTY_PM4_GET(buffer[0], PA_SU_SC_MODE_CNTL, VTX_WINDOW_OFFSET_ENABLE) != 0; + r.provoking_vtx_last = KYTY_PM4_GET(buffer[0], PA_SU_SC_MODE_CNTL, PROVOKING_VTX_LAST) != 0; + r.persp_corr_dis = KYTY_PM4_GET(buffer[0], PA_SU_SC_MODE_CNTL, PERSP_CORR_DIS) != 0; + + cp->GetCtx()->SetModeControl(r); + + return 1; +} + +static void HwCtxIgnorePolyOffsetRegister(uint32_t cmd_offset, uint32_t value) { + static std::atomic log_count = 0; + + auto count = log_count.fetch_add(1); + if (count < 8) { + LOGF_COLOR(Log::Color::Red, + "\t temporary: ignoring polygon offset context register 0x%03" PRIx32 + " = 0x%08" PRIx32 " (depth bias not implemented)\n", + cmd_offset, value); + } +} + +KYTY_HW_CTX_PARSER(HwCtxSetPolyOffsetRegisters) { + auto num_values = KYTY_PM4_LEN(cmd_id) - 2u; + + for (uint32_t i = 0; i < num_values; i++) { + HwCtxIgnorePolyOffsetRegister(cmd_offset + i, buffer[i]); + } + + return num_values; +} + +KYTY_HW_CTX_PARSER(HwCtxSetPsInput) { + EXIT_NOT_IMPLEMENTED(cmd_offset != Pm4::SPI_PS_INPUT_CNTL_0); + + uint32_t count = (cmd_id >> 16u) & 0x3fffu; + + EXIT_NOT_IMPLEMENTED(count == 0); + EXIT_NOT_IMPLEMENTED(count > 32); + + for (uint32_t i = 0; i < count; i++) { + cp->GetCtx()->SetPsInputSettings(i, buffer[i]); + } + + return count; +} +KYTY_HW_CTX_PARSER(HwCtxSetRenderControl) { + EXIT_NOT_IMPLEMENTED(cmd_id != 0xC0016900); + EXIT_NOT_IMPLEMENTED(cmd_offset != Pm4::DB_RENDER_CONTROL); + + HW::RenderControl r; + + r.depth_clear_enable = KYTY_PM4_GET(buffer[0], DB_RENDER_CONTROL, DEPTH_CLEAR_ENABLE) != 0; + r.stencil_clear_enable = KYTY_PM4_GET(buffer[0], DB_RENDER_CONTROL, STENCIL_CLEAR_ENABLE) != 0; + r.resummarize_enable = KYTY_PM4_GET(buffer[0], DB_RENDER_CONTROL, RESUMMARIZE_ENABLE) != 0; + r.stencil_compress_disable = + KYTY_PM4_GET(buffer[0], DB_RENDER_CONTROL, STENCIL_COMPRESS_DISABLE) != 0; + r.depth_compress_disable = + KYTY_PM4_GET(buffer[0], DB_RENDER_CONTROL, DEPTH_COMPRESS_DISABLE) != 0; + r.copy_centroid = KYTY_PM4_GET(buffer[0], DB_RENDER_CONTROL, COPY_CENTROID) != 0; + r.copy_sample = KYTY_PM4_GET(buffer[0], DB_RENDER_CONTROL, COPY_SAMPLE); + + cp->GetCtx()->SetRenderControl(r); + + return 1; +} + +KYTY_HW_CTX_PARSER(HwCtxSetRenderTarget) { + uint32_t count = (cmd_id >> 16u) & 0x3fffu; + if (count < 11 || count > 14) { + bool handled = (count > 0); + for (uint32_t i = 0; i < count; i++) { + if (cmd_offset + i >= Pm4::CX_NUM || + g_hw_ctx_indirect_func[(cmd_offset + i) & (Pm4::CX_NUM - 1)] == nullptr) { + handled = false; + break; + } + } + if (handled) { + for (uint32_t i = 0; i < count; i++) { + g_hw_ctx_indirect_func[(cmd_offset + i) & (Pm4::CX_NUM - 1)](cp, cmd_offset + i, + buffer[i]); + } + return count; + } + + EXIT_NOT_IMPLEMENTED(count < 11 || count > 14); + } + + uint32_t slot = (cmd_offset - Pm4::CB_COLOR0_BASE) / 15; + + auto* ctx = cp->GetCtx(); + + HW::ColorBase base; + HW::ColorPitch pitch; + HW::ColorSlice slice; + HW::ColorView view; + HW::ColorInfo info; + HW::ColorAttrib attrib; + HW::ColorDccControl dcc; + HW::ColorCmask cmask; + HW::ColorCmaskSlice cmask_slice; + HW::ColorFmask fmask; + HW::ColorFmaskSlice fmask_slice; + + base.addr = static_cast(buffer[0]) << 8u; + pitch.pitch_div8_minus1 = buffer[1] & 0x7ffu; + pitch.fmask_pitch_div8_minus1 = (buffer[1] >> 20u) & 0x7ffu; + slice.slice_div64_minus1 = buffer[2] & 0x3fffffu; + + // view.base_array_slice_index = buffer[3] & 0x7ffu; + // view.last_array_slice_index = (buffer[3] >> 13u) & 0x7ffu; + view.base_array_slice_index = KYTY_PM4_GET(buffer[3], CB_COLOR0_VIEW, SLICE_START); + view.last_array_slice_index = KYTY_PM4_GET(buffer[3], CB_COLOR0_VIEW, SLICE_MAX); + view.current_mip_level = KYTY_PM4_GET(buffer[3], CB_COLOR0_VIEW, MIP_LEVEL); + + // info.fmask_compression_enable = (buffer[4] & 0x4000u) != 0; + // + // // info.fmask_compression_mode = (buffer[4] >> 26u) & 0x3u; + // info.fmask_data_compression_disable = ((buffer[4] >> 26u) & 0x1u) != 0; + // info.fmask_one_frag_mode = ((buffer[4] >> 27u) & 0x1u) != 0; + // + // info.cmask_fast_clear_enable = (buffer[4] & 0x2000u) != 0; + // info.dcc_compression_enable = (buffer[4] & 0x10000000u) != 0; + // info.cmask_is_linear = (buffer[4] >> 19u) & 0x1u; + // info.cmask_addr_type = (buffer[4] >> 29u) & 0x3u; + // info.alt_tile_mode = ((buffer[4] >> 31u) & 0x1u) != 0; + // info.format = (buffer[4] >> 2u) & 0x1fu; + // info.channel_type = (buffer[4] >> 8u) & 0x7u; + // info.channel_order = (buffer[4] >> 11u) & 0x3u; + info.format = KYTY_PM4_GET(buffer[4], CB_COLOR0_INFO, FORMAT); + info.channel_type = KYTY_PM4_GET(buffer[4], CB_COLOR0_INFO, NUMBER_TYPE); + info.channel_order = KYTY_PM4_GET(buffer[4], CB_COLOR0_INFO, COMP_SWAP); + info.cmask_fast_clear_enable = KYTY_PM4_GET(buffer[4], CB_COLOR0_INFO, FAST_CLEAR) != 0; + info.fmask_compression_enable = KYTY_PM4_GET(buffer[4], CB_COLOR0_INFO, COMPRESSION) != 0; + info.blend_clamp = KYTY_PM4_GET(buffer[4], CB_COLOR0_INFO, BLEND_CLAMP) != 0; + info.blend_bypass = KYTY_PM4_GET(buffer[4], CB_COLOR0_INFO, BLEND_BYPASS) != 0; + info.round_mode = KYTY_PM4_GET(buffer[4], CB_COLOR0_INFO, ROUND_MODE) != 0; + info.cmask_is_linear = KYTY_PM4_GET(buffer[4], CB_COLOR0_INFO, CMASK_IS_LINEAR); + info.fmask_data_compression_disable = + KYTY_PM4_GET(buffer[4], CB_COLOR0_INFO, FMASK_COMPRESSION_DISABLE) != 0; + info.fmask_one_frag_mode = + KYTY_PM4_GET(buffer[4], CB_COLOR0_INFO, FMASK_COMPRESS_1FRAG_ONLY) != 0; + info.dcc_compression_enable = KYTY_PM4_GET(buffer[4], CB_COLOR0_INFO, DCC_ENABLE) != 0; + info.cmask_addr_type = KYTY_PM4_GET(buffer[4], CB_COLOR0_INFO, CMASK_ADDR_TYPE); + info.alt_tile_mode = KYTY_PM4_GET(buffer[4], CB_COLOR0_INFO, ALT_TILE_MODE) != 0; + + // attrib.force_dest_alpha_to_one = (buffer[5] & 0x20000u) != 0; + // attrib.tile_mode = buffer[5] & 0x1fu; + // attrib.fmask_tile_mode = (buffer[5] >> 5u) & 0x1fu; + // attrib.num_samples = (buffer[5] >> 12u) & 0x7u; + // attrib.num_fragments = (buffer[5] >> 15u) & 0x3u; + attrib.force_dest_alpha_to_one = + KYTY_PM4_GET(buffer[5], CB_COLOR0_ATTRIB, FORCE_DST_ALPHA_1) != 0; + attrib.tile_mode = KYTY_PM4_GET(buffer[5], CB_COLOR0_ATTRIB, TILE_MODE_INDEX); + attrib.fmask_tile_mode = KYTY_PM4_GET(buffer[5], CB_COLOR0_ATTRIB, FMASK_TILE_MODE_INDEX); + attrib.num_samples = KYTY_PM4_GET(buffer[5], CB_COLOR0_ATTRIB, NUM_SAMPLES); + attrib.num_fragments = KYTY_PM4_GET(buffer[5], CB_COLOR0_ATTRIB, NUM_FRAGMENTS); + + // dcc.max_uncompressed_block_size = (buffer[6] >> 2u) & 0x3u; + // dcc.max_compressed_block_size = (buffer[6] >> 5u) & 0x3u; + // dcc.min_compressed_block_size = (buffer[6] >> 4u) & 0x1u; + // dcc.color_transform = (buffer[6] >> 7u) & 0x3u; + // dcc.overwrite_combiner_disable = (buffer[6] & 0x1u) != 0; + // dcc.independent_64b_blocks = (buffer[6] & 0x200u) != 0; + dcc.overwrite_combiner_disable = + KYTY_PM4_GET(buffer[6], CB_COLOR0_DCC_CONTROL, OVERWRITE_COMBINER_DISABLE) != 0; + dcc.dcc_clear_key_enable = + KYTY_PM4_GET(buffer[6], CB_COLOR0_DCC_CONTROL, KEY_CLEAR_ENABLE) != 0; + dcc.max_uncompressed_block_size = + KYTY_PM4_GET(buffer[6], CB_COLOR0_DCC_CONTROL, MAX_UNCOMPRESSED_BLOCK_SIZE); + dcc.min_compressed_block_size = + KYTY_PM4_GET(buffer[6], CB_COLOR0_DCC_CONTROL, MIN_COMPRESSED_BLOCK_SIZE); + dcc.max_compressed_block_size = + KYTY_PM4_GET(buffer[6], CB_COLOR0_DCC_CONTROL, MAX_COMPRESSED_BLOCK_SIZE); + dcc.color_transform = KYTY_PM4_GET(buffer[6], CB_COLOR0_DCC_CONTROL, COLOR_TRANSFORM); + dcc.independent_64b_blocks = + KYTY_PM4_GET(buffer[6], CB_COLOR0_DCC_CONTROL, INDEPENDENT_64B_BLOCKS) != 0; + dcc.data_write_on_dcc_clear_to_reg = + KYTY_PM4_GET(buffer[6], CB_COLOR0_DCC_CONTROL, ENABLE_CONSTANT_ENCODE_REG_WRITE) != 0; + dcc.independent_128b_blocks = + KYTY_PM4_GET(buffer[6], CB_COLOR0_DCC_CONTROL, INDEPENDENT_128B_BLOCKS) != 0; + + cmask.addr = static_cast(buffer[7]) << 8u; + cmask_slice.slice_minus1 = buffer[8] & 0x3fffu; + + fmask.addr = static_cast(buffer[9]) << 8u; + fmask_slice.slice_minus1 = buffer[10] & 0x3fffffu; + + ctx->SetColorBase(slot, base); + ctx->SetColorPitch(slot, pitch); + ctx->SetColorSlice(slot, slice); + ctx->SetColorView(slot, view); + ctx->SetColorInfo(slot, info); + ctx->SetColorAttrib(slot, attrib); + ctx->SetColorDccControl(slot, dcc); + ctx->SetColorCmask(slot, cmask); + ctx->SetColorCmaskSlice(slot, cmask_slice); + ctx->SetColorFmask(slot, fmask); + ctx->SetColorFmaskSlice(slot, fmask_slice); + + if (count > 11) { + HW::ColorClearWord0 clear_word0; + + clear_word0.word0 = buffer[11]; + + ctx->SetColorClearWord0(slot, clear_word0); + } + if (count > 12) { + HW::ColorClearWord1 clear_word1; + + clear_word1.word1 = buffer[12]; + + ctx->SetColorClearWord1(slot, clear_word1); + } + if (count > 13) { + HW::ColorDccAddr dcc_addr; + + dcc_addr.addr = static_cast(buffer[13]) << 8u; + + ctx->SetColorDccAddr(slot, dcc_addr); + } + + if (dw >= count + 2 && buffer[count] == 0xC0001000) { + HW::ColorSize size; + + size.width = (buffer[count + 1] >> 0u) & 0xffffu; + size.height = (buffer[count + 1] >> 16u) & 0xffffu; + + ctx->SetColorSize(slot, size); + + count += 2; + } + + return count; +} + +KYTY_HW_CTX_PARSER(HwCtxSetRenderTargetMask) { + EXIT_NOT_IMPLEMENTED(cmd_id != 0xC0016900); + EXIT_NOT_IMPLEMENTED(cmd_offset != Pm4::CB_TARGET_MASK); + + cp->GetCtx()->SetRenderTargetMask(*buffer); + + return 1; +} + +KYTY_HW_CTX_PARSER(HwCtxSetScanModeControl) { + EXIT_NOT_IMPLEMENTED(cmd_id != 0xc0016900); + EXIT_NOT_IMPLEMENTED(cmd_offset != Pm4::PA_SC_MODE_CNTL_0); + + HW::ScanModeControl r; + + r.msaa_enable = KYTY_PM4_GET(buffer[0], PA_SC_MODE_CNTL_0, MSAA_ENABLE) != 0; + r.vport_scissor_enable = KYTY_PM4_GET(buffer[0], PA_SC_MODE_CNTL_0, VPORT_SCISSOR_ENABLE) != 0; + r.line_stipple_enable = KYTY_PM4_GET(buffer[0], PA_SC_MODE_CNTL_0, LINE_STIPPLE_ENABLE) != 0; + + cp->GetCtx()->SetScanModeControl(r); + + return 1; +} + +KYTY_HW_CTX_PARSER(HwCtxSetScreenScissor) { + auto num_values = KYTY_PM4_LEN(cmd_id) - 2u; + EXIT_NOT_IMPLEMENTED(num_values == 0); + EXIT_NOT_IMPLEMENTED(cmd_offset < Pm4::PA_SC_SCREEN_SCISSOR_TL); + EXIT_NOT_IMPLEMENTED(cmd_offset + num_values - 1u > Pm4::PA_SC_SCREEN_SCISSOR_BR); + + for (uint32_t i = 0; i < num_values; i++) { + g_hw_ctx_indirect_func[(cmd_offset + i) & (Pm4::CX_NUM - 1)](cp, cmd_offset + i, buffer[i]); + } + + return num_values; +} + +KYTY_HW_CTX_PARSER(HwCtxSetShaderStages) { + EXIT_NOT_IMPLEMENTED(cmd_id != 0xC0016900); + EXIT_NOT_IMPLEMENTED(cmd_offset != Pm4::VGT_SHADER_STAGES_EN); + + cp->GetCtx()->SetShaderStages(buffer[0]); + + return 1; +} + +KYTY_HW_CTX_PARSER(HwCtxSetStencilClear) { + EXIT_NOT_IMPLEMENTED(cmd_id != 0xC0016900); + EXIT_NOT_IMPLEMENTED(cmd_offset != Pm4::DB_STENCIL_CLEAR); + + cp->GetCtx()->SetStencilClearValue(KYTY_PM4_GET(buffer[0], DB_STENCIL_CLEAR, CLEAR)); + + return 1; +} + +KYTY_HW_CTX_PARSER(HwCtxSetStencilControl) { + EXIT_NOT_IMPLEMENTED(cmd_id != 0xC0016900); + EXIT_NOT_IMPLEMENTED(cmd_offset != Pm4::DB_STENCIL_CONTROL); + + HW::StencilControl r; + + r.stencil_fail = KYTY_PM4_GET(buffer[0], DB_STENCIL_CONTROL, STENCILFAIL); + r.stencil_zpass = KYTY_PM4_GET(buffer[0], DB_STENCIL_CONTROL, STENCILZPASS); + r.stencil_zfail = KYTY_PM4_GET(buffer[0], DB_STENCIL_CONTROL, STENCILZFAIL); + r.stencil_fail_bf = KYTY_PM4_GET(buffer[0], DB_STENCIL_CONTROL, STENCILFAIL_BF); + r.stencil_zpass_bf = KYTY_PM4_GET(buffer[0], DB_STENCIL_CONTROL, STENCILZPASS_BF); + r.stencil_zfail_bf = KYTY_PM4_GET(buffer[0], DB_STENCIL_CONTROL, STENCILZFAIL_BF); + + cp->GetCtx()->SetStencilControl(r); + + return 1; +} + +KYTY_HW_CTX_PARSER(HwCtxSetStencilInfo) { + EXIT_NOT_IMPLEMENTED(cmd_id != 0xC0016900); + EXIT_NOT_IMPLEMENTED(cmd_offset != Pm4::DB_STENCIL_INFO); + + HW::DepthStencilInfo r; + + // r.expclear_enabled = (buffer[0] & 0x08000000u) != 0; + // r.tile_split = (buffer[0] >> 13u) & 0x7u; + // r.format = (buffer[0] >> Pm4::DB_STENCIL_INFO_FORMAT_SHIFT) & + // Pm4::DB_STENCIL_INFO_FORMAT_MASK; r.tile_mode_index = (buffer[0] >> + // Pm4::DB_STENCIL_INFO_TILE_MODE_INDEX_SHIFT) & Pm4::DB_STENCIL_INFO_TILE_MODE_INDEX_MASK; + // r.tile_stencil_disable = + // ((buffer[0] >> Pm4::DB_STENCIL_INFO_TILE_STENCIL_DISABLE_SHIFT) & + // Pm4::DB_STENCIL_INFO_TILE_STENCIL_DISABLE_MASK) != 0; + + r.format = KYTY_PM4_GET(buffer[1], DB_STENCIL_INFO, FORMAT); + r.texture_compatible_stencil = KYTY_PM4_GET(buffer[1], DB_STENCIL_INFO, ITERATE_FLUSH) != 0; + r.partially_resident = KYTY_PM4_GET(buffer[1], DB_STENCIL_INFO, PARTIALLY_RESIDENT) != 0; + r.tile_split = KYTY_PM4_GET(buffer[1], DB_STENCIL_INFO, RESERVED_FIELD_1); + r.tile_mode_index = KYTY_PM4_GET(buffer[1], DB_STENCIL_INFO, TILE_MODE_INDEX); + r.expclear_enabled = KYTY_PM4_GET(buffer[1], DB_STENCIL_INFO, ALLOW_EXPCLEAR) != 0; + r.tile_stencil_disable = KYTY_PM4_GET(buffer[1], DB_STENCIL_INFO, TILE_STENCIL_DISABLE) != 0; + + cp->GetCtx()->SetDepthStencilInfo(r); + + return 1; +} + +KYTY_HW_CTX_PARSER(HwCtxSetStencilMask) { + EXIT_NOT_IMPLEMENTED(cmd_id != 0xc0026900); + EXIT_NOT_IMPLEMENTED(cmd_offset != Pm4::DB_STENCILREFMASK); + + HW::StencilMask r; + + r.stencil_testval = KYTY_PM4_GET(buffer[0], DB_STENCILREFMASK, STENCILTESTVAL); + r.stencil_mask = KYTY_PM4_GET(buffer[0], DB_STENCILREFMASK, STENCILMASK); + r.stencil_writemask = KYTY_PM4_GET(buffer[0], DB_STENCILREFMASK, STENCILWRITEMASK); + r.stencil_opval = KYTY_PM4_GET(buffer[0], DB_STENCILREFMASK, STENCILOPVAL); + r.stencil_testval_bf = KYTY_PM4_GET(buffer[1], DB_STENCILREFMASK_BF, STENCILTESTVAL_BF); + r.stencil_mask_bf = KYTY_PM4_GET(buffer[1], DB_STENCILREFMASK_BF, STENCILMASK_BF); + r.stencil_writemask_bf = KYTY_PM4_GET(buffer[1], DB_STENCILREFMASK_BF, STENCILWRITEMASK_BF); + r.stencil_opval_bf = KYTY_PM4_GET(buffer[1], DB_STENCILREFMASK_BF, STENCILOPVAL_BF); + + cp->GetCtx()->SetStencilMask(r); + + return 2; +} + +KYTY_HW_CTX_PARSER(HwCtxSetViewportScaleOffset) { + auto reg_num = (cmd_id >> 16u) & 0x3fffu; + EXIT_NOT_IMPLEMENTED(reg_num == 0); + EXIT_NOT_IMPLEMENTED(cmd_offset < Pm4::PA_CL_VPORT_XSCALE); + EXIT_NOT_IMPLEMENTED(cmd_offset + reg_num - 1u > Pm4::PA_CL_VPORT_ZOFFSET_15); + + for (uint32_t i = 0; i < reg_num; i++) { + const auto reg = cmd_offset + i; + const auto viewport = (reg - Pm4::PA_CL_VPORT_XSCALE) / 6u; + const auto field = (reg - Pm4::PA_CL_VPORT_XSCALE) % 6u; + const auto value = *reinterpret_cast(buffer + i); + + EXIT_NOT_IMPLEMENTED(viewport >= 16); + switch (field) { + case 0: cp->GetCtx()->SetViewportXScale(viewport, value); break; + case 1: cp->GetCtx()->SetViewportXOffset(viewport, value); break; + case 2: cp->GetCtx()->SetViewportYScale(viewport, value); break; + case 3: cp->GetCtx()->SetViewportYOffset(viewport, value); break; + case 4: cp->GetCtx()->SetViewportZScale(viewport, value); break; + case 5: cp->GetCtx()->SetViewportZOffset(viewport, value); break; + default: EXIT("invalid viewport scale/offset field\n"); + } + } + + return reg_num; +} + +KYTY_HW_CTX_PARSER(HwCtxSetViewportTransformControl) { + EXIT_NOT_IMPLEMENTED(cmd_id != 0xC0016900); + EXIT_NOT_IMPLEMENTED(cmd_offset != Pm4::PA_CL_VTE_CNTL); + + cp->GetCtx()->SetViewportTransformControl(*buffer); + + return 1; +} + +KYTY_HW_CTX_PARSER(HwCtxSetViewportZ) { + auto reg_num = (cmd_id >> 16u) & 0x3fffu; + EXIT_NOT_IMPLEMENTED(reg_num == 0); + EXIT_NOT_IMPLEMENTED(cmd_offset < Pm4::PA_SC_VPORT_ZMIN_0); + EXIT_NOT_IMPLEMENTED(cmd_offset + reg_num - 1u > Pm4::PA_SC_VPORT_ZMAX_15); + + for (uint32_t i = 0; i < reg_num; i++) { + const auto reg = cmd_offset + i; + const auto viewport = (reg - Pm4::PA_SC_VPORT_ZMIN_0) / 2u; + const auto value = *reinterpret_cast(buffer + i); + + EXIT_NOT_IMPLEMENTED(viewport >= 16); + if (((reg - Pm4::PA_SC_VPORT_ZMIN_0) % 2u) == 0) { + cp->GetCtx()->SetViewportZMin(viewport, value); + } else { + cp->GetCtx()->SetViewportZMax(viewport, value); + } + } + + return reg_num; +} + +static void HwShIgnoreComputeRegister([[maybe_unused]] uint32_t cmd_offset, + [[maybe_unused]] uint32_t value) {} +static void HwShIgnoreShaderRegister([[maybe_unused]] uint32_t cmd_offset, + [[maybe_unused]] uint32_t value) {} + +KYTY_HW_SH_PARSER(HwShIgnoreRegisters) { + return (cmd_id >> 16u) & 0x3fffu; +} + +static void HwShSetCsRegister(CommandProcessor* cp, uint32_t cmd_offset, uint32_t value) { + auto cs_regs = cp->GetShCtx()->GetCs().cs_regs; + + switch (cmd_offset) { + case Pm4::COMPUTE_PGM_LO: + cs_regs.data_addr &= 0xFFFFFF00000000FFull; + cs_regs.data_addr |= static_cast(value) << 8u; + break; + case Pm4::COMPUTE_PGM_HI: + cs_regs.data_addr &= 0xFFFF00FFFFFFFFFFull; + cs_regs.data_addr |= (static_cast(value) & 0xffu) << 40u; + break; + case Pm4::COMPUTE_TBA_LO: + case Pm4::COMPUTE_TBA_HI: + case Pm4::COMPUTE_TMA_LO: + case Pm4::COMPUTE_TMA_HI: HwShIgnoreComputeRegister(cmd_offset, value); break; + case Pm4::COMPUTE_PGM_RSRC1: + cs_regs.vgprs = + (value >> Pm4::COMPUTE_PGM_RSRC1_VGPRS_SHIFT) & Pm4::COMPUTE_PGM_RSRC1_VGPRS_MASK; + cs_regs.sgprs = + (value >> Pm4::COMPUTE_PGM_RSRC1_SGPRS_SHIFT) & Pm4::COMPUTE_PGM_RSRC1_SGPRS_MASK; + cs_regs.bulky = + (value >> Pm4::COMPUTE_PGM_RSRC1_BULKY_SHIFT) & Pm4::COMPUTE_PGM_RSRC1_BULKY_MASK; + cs_regs.wave_size = (((value >> Pm4::COMPUTE_PGM_RSRC1_W32_EN_SHIFT) & + Pm4::COMPUTE_PGM_RSRC1_W32_EN_MASK) != 0u) + ? 32u + : 64u; + break; + case Pm4::COMPUTE_PGM_RSRC2: + cs_regs.scratch_en = (value >> Pm4::COMPUTE_PGM_RSRC2_SCRATCH_EN_SHIFT) & + Pm4::COMPUTE_PGM_RSRC2_SCRATCH_EN_MASK; + cs_regs.user_sgpr = (value >> Pm4::COMPUTE_PGM_RSRC2_USER_SGPR_SHIFT) & + Pm4::COMPUTE_PGM_RSRC2_USER_SGPR_MASK; + cs_regs.tgid_x_en = (value >> Pm4::COMPUTE_PGM_RSRC2_TGID_X_EN_SHIFT) & + Pm4::COMPUTE_PGM_RSRC2_TGID_X_EN_MASK; + cs_regs.tgid_y_en = (value >> Pm4::COMPUTE_PGM_RSRC2_TGID_Y_EN_SHIFT) & + Pm4::COMPUTE_PGM_RSRC2_TGID_Y_EN_MASK; + cs_regs.tgid_z_en = (value >> Pm4::COMPUTE_PGM_RSRC2_TGID_Z_EN_SHIFT) & + Pm4::COMPUTE_PGM_RSRC2_TGID_Z_EN_MASK; + cs_regs.tg_size_en = (value >> Pm4::COMPUTE_PGM_RSRC2_TG_SIZE_EN_SHIFT) & + Pm4::COMPUTE_PGM_RSRC2_TG_SIZE_EN_MASK; + cs_regs.tidig_comp_cnt = (value >> Pm4::COMPUTE_PGM_RSRC2_TIDIG_COMP_CNT_SHIFT) & + Pm4::COMPUTE_PGM_RSRC2_TIDIG_COMP_CNT_MASK; + cs_regs.lds_size = (value >> Pm4::COMPUTE_PGM_RSRC2_LDS_SIZE_SHIFT) & + Pm4::COMPUTE_PGM_RSRC2_LDS_SIZE_MASK; + break; + case Pm4::COMPUTE_NUM_THREAD_X: cs_regs.num_thread_x = value; break; + case Pm4::COMPUTE_NUM_THREAD_Y: cs_regs.num_thread_y = value; break; + case Pm4::COMPUTE_NUM_THREAD_Z: cs_regs.num_thread_z = value; break; + case Pm4::COMPUTE_START_X: + case Pm4::COMPUTE_START_Y: + case Pm4::COMPUTE_START_Z: + case Pm4::COMPUTE_RESOURCE_LIMITS: + case Pm4::COMPUTE_DESTINATION_EN_SE0: + case Pm4::COMPUTE_DESTINATION_EN_SE1: + case Pm4::COMPUTE_TMPRING_SIZE: + case Pm4::COMPUTE_DESTINATION_EN_SE2: + case Pm4::COMPUTE_DESTINATION_EN_SE3: + case Pm4::COMPUTE_PGM_RSRC3: + case Pm4::COMPUTE_SHADER_CHKSUM: + case Pm4::COMPUTE_DISPATCH_TUNNEL: HwShIgnoreComputeRegister(cmd_offset, value); break; + default: + EXIT("unsupported compute SH register 0x%08" PRIx32 " = 0x%08" PRIx32 "\n", cmd_offset, + value); + } + + cp->GetShCtx()->SetCsShader(cs_regs); +} + +KYTY_HW_SH_PARSER(HwShSetCsRegisters) { + auto reg_num = (cmd_id >> 16u) & 0x3fffu; + + for (uint32_t i = 0; i < reg_num; i++) { + HwShSetCsRegister(cp, cmd_offset + i, buffer[i]); + } + + return reg_num; +} + +static uint32_t UserSgprWriteNum(uint32_t slot, uint32_t reg_num, const char* stage) { + EXIT_IF(stage == nullptr); + + if (slot >= HW::UserSgprInfo::SGPRS_MAX) { + LOGF("\t %s user sgpr write starts out of range: slot = %" PRIu32 ", reg_num = %" PRIu32 + "\n", + stage, slot, reg_num); + return 0; + } + + const auto available = static_cast(HW::UserSgprInfo::SGPRS_MAX) - slot; + if (reg_num > available) { + LOGF("\t %s user sgpr write truncated: slot = %" PRIu32 ", reg_num = %" PRIu32 + ", available = %" PRIu32 "\n", + stage, slot, reg_num, available); + return available; + } + + return reg_num; +} + +KYTY_HW_SH_PARSER(HwShSetCsUserSgpr) { + EXIT_NOT_IMPLEMENTED( + !(cmd_offset >= Pm4::COMPUTE_USER_DATA_0 && cmd_offset <= Pm4::COMPUTE_USER_DATA_15)); + + uint32_t slot = (cmd_offset - Pm4::COMPUTE_USER_DATA_0) / 1; + + auto reg_num = (cmd_id >> 16u) & 0x3fffu; + auto write_num = UserSgprWriteNum(slot, reg_num, "cs"); + + for (uint32_t i = 0; i < write_num; i++) { + cp->GetShCtx()->SetCsUserSgpr(slot + i, buffer[i], cp->GetUserDataMarker()); + } + cp->SetUserDataMarker(HW::UserSgprType::Unknown); + + return reg_num; +} + +KYTY_HW_SH_PARSER(HwShSetCsUserAccumSgpr) { + EXIT_NOT_IMPLEMENTED(cmd_offset < Pm4::COMPUTE_USER_ACCUM_0); + + uint32_t slot = 16u + (cmd_offset - Pm4::COMPUTE_USER_ACCUM_0); + + auto reg_num = (cmd_id >> 16u) & 0x3fffu; + auto write_num = UserSgprWriteNum(slot, reg_num, "cs accum"); + + for (uint32_t i = 0; i < write_num; i++) { + cp->GetShCtx()->SetCsUserSgpr(slot + i, buffer[i], cp->GetUserDataMarker()); + } + cp->SetUserDataMarker(HW::UserSgprType::Unknown); + + return reg_num; +} + +KYTY_HW_SH_PARSER(HwShSetPsUserSgpr) { + EXIT_NOT_IMPLEMENTED(!(cmd_offset >= Pm4::SPI_SHADER_USER_DATA_PS_0 && + cmd_offset <= Pm4::SPI_SHADER_USER_DATA_PS_31)); + + uint32_t slot = (cmd_offset - Pm4::SPI_SHADER_USER_DATA_PS_0) / 1; + + auto reg_num = (cmd_id >> 16u) & 0x3fffu; + auto write_num = UserSgprWriteNum(slot, reg_num, "ps"); + + for (uint32_t i = 0; i < write_num; i++) { + cp->GetShCtx()->SetPsUserSgpr(slot + i, buffer[i], cp->GetUserDataMarker()); + } + cp->SetUserDataMarker(HW::UserSgprType::Unknown); + + return reg_num; +} + +KYTY_HW_SH_PARSER(HwShSetPsUserAccumSgpr) { + EXIT_NOT_IMPLEMENTED(!(cmd_offset >= Pm4::SPI_SHADER_USER_ACCUM_PS_0 && + cmd_offset <= Pm4::SPI_SHADER_USER_ACCUM_PS_3)); + + uint32_t slot = 16u + (cmd_offset - Pm4::SPI_SHADER_USER_ACCUM_PS_0); + + auto reg_num = (cmd_id >> 16u) & 0x3fffu; + auto write_num = UserSgprWriteNum(slot, reg_num, "ps accum"); + + for (uint32_t i = 0; i < write_num; i++) { + cp->GetShCtx()->SetPsUserSgpr(slot + i, buffer[i], cp->GetUserDataMarker()); + } + cp->SetUserDataMarker(HW::UserSgprType::Unknown); + + return reg_num; +} + +KYTY_HW_SH_PARSER(HwShSetGsUserSgpr) { + EXIT_NOT_IMPLEMENTED(!(cmd_offset >= Pm4::SPI_SHADER_USER_DATA_GS_0 && + cmd_offset <= Pm4::SPI_SHADER_USER_DATA_GS_31)); + + uint32_t slot = (cmd_offset - Pm4::SPI_SHADER_USER_DATA_GS_0) / 1; + + auto reg_num = (cmd_id >> 16u) & 0x3fffu; + auto write_num = UserSgprWriteNum(slot, reg_num, "gs"); + + for (uint32_t i = 0; i < write_num; i++) { + cp->GetShCtx()->SetGsUserSgpr(slot + i, buffer[i], cp->GetUserDataMarker()); + } + cp->SetUserDataMarker(HW::UserSgprType::Unknown); + + return reg_num; +} + +KYTY_HW_SH_PARSER(HwShSetGsUserAccumSgpr) { + EXIT_NOT_IMPLEMENTED(!(cmd_offset >= Pm4::SPI_SHADER_USER_ACCUM_ESGS_0 && + cmd_offset <= Pm4::SPI_SHADER_USER_ACCUM_ESGS_3)); + + uint32_t slot = 16u + (cmd_offset - Pm4::SPI_SHADER_USER_ACCUM_ESGS_0); + + auto reg_num = (cmd_id >> 16u) & 0x3fffu; + auto write_num = UserSgprWriteNum(slot, reg_num, "gs accum"); + + for (uint32_t i = 0; i < write_num; i++) { + cp->GetShCtx()->SetGsUserSgpr(slot + i, buffer[i], cp->GetUserDataMarker()); + } + cp->SetUserDataMarker(HW::UserSgprType::Unknown); + + return reg_num; +} + +KYTY_HW_SH_PARSER(HwShSetHsUserSgpr) { + EXIT_NOT_IMPLEMENTED(!(cmd_offset >= Pm4::SPI_SHADER_USER_DATA_HS_0 && + cmd_offset <= Pm4::SPI_SHADER_USER_DATA_HS_31)); + + uint32_t slot = (cmd_offset - Pm4::SPI_SHADER_USER_DATA_HS_0) / 1; + + auto reg_num = (cmd_id >> 16u) & 0x3fffu; + auto write_num = UserSgprWriteNum(slot, reg_num, "hs"); + + for (uint32_t i = 0; i < write_num; i++) { + cp->GetShCtx()->SetHsUserSgpr(slot + i, buffer[i], cp->GetUserDataMarker()); + } + cp->SetUserDataMarker(HW::UserSgprType::Unknown); + + return reg_num; +} + +KYTY_HW_SH_PARSER(HwShSetHsUserAccumSgpr) { + EXIT_NOT_IMPLEMENTED(!(cmd_offset >= Pm4::SPI_SHADER_USER_ACCUM_LSHS_0 && + cmd_offset <= Pm4::SPI_SHADER_USER_ACCUM_LSHS_3)); + uint32_t slot = 16u + (cmd_offset - Pm4::SPI_SHADER_USER_ACCUM_LSHS_0); + + auto reg_num = (cmd_id >> 16u) & 0x3fffu; + auto write_num = UserSgprWriteNum(slot, reg_num, "hs accum"); + + for (uint32_t i = 0; i < write_num; i++) { + cp->GetShCtx()->SetHsUserSgpr(slot + i, buffer[i], cp->GetUserDataMarker()); + } + cp->SetUserDataMarker(HW::UserSgprType::Unknown); + + return reg_num; +} + +KYTY_HW_UC_PARSER(HwUcSetPrimitiveType) { + EXIT_NOT_IMPLEMENTED(cmd_id != 0xC0017900 && cmd_id != 0xC0017A00); + EXIT_NOT_IMPLEMENTED(cmd_offset != Pm4::VGT_PRIMITIVE_TYPE); + + uint32_t prim_type = KYTY_PM4_GET(buffer[0], VGT_PRIMITIVE_TYPE, PRIM_TYPE); + + cp->GetUcfg()->SetPrimitiveType(prim_type); + + return 1; +} + +KYTY_HW_UC_PARSER(HwUcSetIndexType) { + EXIT_NOT_IMPLEMENTED(cmd_offset != Pm4::VGT_INDEX_TYPE); + cp->SetIndexType(buffer[0] & 0x3u); + + return 1; +} + +KYTY_HW_UC_PARSER(HwUcSetObjectId) { + EXIT_NOT_IMPLEMENTED(cmd_offset != Pm4::VGT_OBJECT_ID); + cp->GetUcfg()->SetObjectId(buffer[0]); + + return 1; +} + +static void HwUcIgnoreTextureGradientFactors([[maybe_unused]] uint32_t value) {} + +KYTY_HW_UC_PARSER(HwUcSetTextureGradientFactors) { + auto num_values = KYTY_PM4_LEN(cmd_id) - 2u; + + for (uint32_t i = 0; i < num_values; i++) { + HwUcIgnoreTextureGradientFactors(buffer[i]); + } + + return num_values; +} + +static void HwUcIgnoreIaMultiVgtParam([[maybe_unused]] uint32_t value) {} + +KYTY_HW_UC_PARSER(HwUcSetIaMultiVgtParam) { + auto num_values = KYTY_PM4_LEN(cmd_id) - 2u; + + for (uint32_t i = 0; i < num_values; i++) { + HwUcIgnoreIaMultiVgtParam(buffer[i]); + } + + return num_values; +} + +static void HwIgnoreMultiPrimIbReset([[maybe_unused]] uint32_t value) {} + +KYTY_HW_UC_PARSER(HwUcSetMultiPrimIbReset) { + auto num_values = KYTY_PM4_LEN(cmd_id) - 2u; + + for (uint32_t i = 0; i < num_values; i++) { + HwIgnoreMultiPrimIbReset(buffer[i]); + } + + return num_values; +} + +static void HwUcIgnoreBorderColorTableAddr([[maybe_unused]] uint32_t cmd_offset, + [[maybe_unused]] uint32_t value) {} + +KYTY_HW_UC_PARSER(HwUcSetBorderColorTableAddr) { + auto num_values = KYTY_PM4_LEN(cmd_id) - 2u; + + for (uint32_t i = 0; i < num_values; i++) { + HwUcIgnoreBorderColorTableAddr(cmd_offset + i, buffer[i]); + } + + return num_values; +} + +KYTY_HW_UC_PARSER(HwUcSetGeIndexOffset) { + EXIT_NOT_IMPLEMENTED(cmd_offset != Pm4::GE_INDX_OFFSET); + cp->GetUcfg()->SetIndexOffset(buffer[0]); + + return 1; +} + +static void HwUcSetGdsOaRegister(CommandProcessor* cp, uint32_t cmd_offset, uint32_t value) { + EXIT_IF(cp == nullptr); + + auto* ucfg = cp->GetUcfg(); + + switch (cmd_offset) { + case Pm4::GDS_OA_CNTL: ucfg->SetGdsOaCntl(value); break; + case Pm4::GDS_OA_COUNTER: ucfg->SetGdsOaCounter(value); break; + case Pm4::GDS_OA_ADDRESS: ucfg->SetGdsOaAddress(value); break; + default: EXIT("unknown GDS OA register: 0x%08" PRIx32 "\n", cmd_offset); + } + + const auto index = ucfg->GetGdsOaState().GetIndex(); + const auto& oa = ucfg->GetGdsOaCounter(index); + static std::atomic log_count {0}; + if (oa.IsCounterEnabled() && log_count.fetch_add(1, std::memory_order_relaxed) < 128) { + LOGF("GDS_OA: index=%u address_bytes=0x%04" PRIx32 " space=0x%08" PRIx32 + " crawler=%u crawler_id=%u " + "alloc_crawler=%s raw_cntl=0x%08" PRIx32 " raw_counter=0x%08" PRIx32 + " raw_address=0x%08" PRIx32 "\n", + index, oa.GetAddressBytes(), oa.GetSpaceAvailable(), oa.GetCrawlerType(), + oa.GetCrawlerId(), oa.IsAllocationCrawlerDisabled() ? "disabled" : "enabled", + ucfg->GetGdsOaState().cntl, oa.counter, oa.address); + } +} + +KYTY_HW_UC_PARSER(HwUcSetGdsOaRegisters) { + auto num_values = KYTY_PM4_LEN(cmd_id) - 2u; + + for (uint32_t i = 0; i < num_values; i++) { + HwUcSetGdsOaRegister(cp, cmd_offset + i, buffer[i]); + } + + return num_values; +} + +static bool HwUcIsFakeRegister(uint32_t cmd_offset) { + switch (cmd_offset) { + case Pm4::FSR_EXTEND_SUBPIXEL_ROUNDING: + case Pm4::FSR_ALPHA_VALUE0: + case Pm4::FSR_ALPHA_VALUE1: + case Pm4::FSR_CONTROL_POINT0: + case Pm4::FSR_CONTROL_POINT1: + case Pm4::FSR_CONTROL_POINT2: + case Pm4::FSR_CONTROL_POINT3: + case Pm4::FSR_WINDOW0: + case Pm4::FSR_WINDOW1: + case Pm4::TEXTURE_GRADIENT_CONTROL: + case Pm4::MEMORY_MAPPING_MASK: + case Pm4::UC_NOP: return true; + default: return false; + } +} + +static void HwUcIgnoreFakeRegister([[maybe_unused]] uint32_t cmd_offset, + [[maybe_unused]] uint32_t value) {} + +static bool HwUcTrySetFakeRegister(uint32_t cmd_offset, uint32_t value) { + if (!HwUcIsFakeRegister(cmd_offset)) { + return false; + } + + if (cmd_offset != Pm4::UC_NOP) { + HwUcIgnoreFakeRegister(cmd_offset, value); + } + + return true; +} + +static bool HwUcTrySetFakeRegisterRange(uint32_t cmd_offset, const uint32_t* buffer, + uint32_t num_values) { + if (!HwUcIsFakeRegister(cmd_offset)) { + return false; + } + + for (uint32_t i = 0; i < num_values; i++) { + auto reg = cmd_offset + i; + if (!HwUcTrySetFakeRegister(reg, buffer[i])) { + EXIT("unsupported fake UC register range at 0x%08" PRIx32 " + %" PRIu32 + ", value = 0x%08" PRIx32 "\n", + cmd_offset, i, buffer[i]); + } + } + + return true; +} + +// NOLINTNEXTLINE(readability-function-cognitive-complexity) +KYTY_CP_OP_PARSER(CpOpAcquireMem) { + KYTY_PROFILER_FUNCTION(); + + EXIT_NOT_IMPLEMENTED(cmd_id != 0xC0055800 && cmd_id != 0xc0061050); + + bool custom = (cmd_id == 0xc0061050); + + uint32_t engine = buffer[0] >> 31u; + uint32_t stall_mode = (custom ? 1u : engine); + uint32_t cache_action = buffer[0] & 0x7fffffffu; + uint64_t size_lo = buffer[1]; + uint32_t size_hi = buffer[2]; + uint64_t base_lo = buffer[3]; + uint32_t base_hi = buffer[4]; + uint32_t poll = buffer[5]; + [[maybe_unused]] uint32_t gcr_cntl = (custom ? buffer[6] : 0); + + uint32_t target_mask = cache_action & 0x00007FC0u; + uint32_t extended_action = cache_action & 0x2E000000u; + uint32_t action = + ((cache_action & 0x00C00000u) >> 0x12u) | ((cache_action & 0x00058000u) >> 0xfu); + + if (custom && engine > 1) { + LOGF("\t warning: custom acquire_mem unsupported engine: %" PRIu32 "\n", engine); + } + + // EXIT_NOT_IMPLEMENTED(stall_mode != 1); + EXIT_NOT_IMPLEMENTED(size_hi != 0); + EXIT_NOT_IMPLEMENTED(base_hi != 0); + if (poll != 10) { + LOGF("\t warning: acquire_mem unexpected poll interval: %" PRIu32 "\n", poll); + } + + switch (cache_action) { + case 0x00000000: { + if (custom && gcr_cntl != 0) { + LOGF("\t custom acquire_mem GCR-only barrier, gcr_cntl = 0x%08" PRIx32 + ", base = 0x%016" PRIx64 ", size = 0x%016" PRIx64 "\n", + gcr_cntl, base_lo << 8u, size_lo << 8u); + + cp->MemoryBarrier(); + if ((gcr_cntl & AcquireGcrGl2Writeback) != 0) { + cp->SynchronizeGpu(); + } + } + } break; + case 0x00000040: + case 0x00003fc0: + case 0x00004000: + case 0x00007fc0: { + // target_mask set, no CB/DB action bits. Treat as an ordering barrier. + EXIT_IF(target_mask != cache_action); + EXIT_IF(extended_action != 0x00000000); + EXIT_IF(action != 0x00); + + LOGF("\t temporary: acquire_mem target-mask-only barrier, target_mask = 0x%08" PRIx32 + ", gcr_cntl = 0x%08" PRIx32 ", base = 0x%016" PRIx64 ", size = 0x%016" PRIx64 "\n", + target_mask, gcr_cntl, base_lo << 8u, size_lo << 8u); + + cp->MemoryBarrier(); + } break; + case 0x02000000: { + // target_mask: 0x00000000 (none) + // extended_action: 0x02000000 (FlushAndInvalidateCbCache) + // action: 0x00 (none) + EXIT_IF(target_mask != 0x00000000); + EXIT_IF(extended_action != 0x02000000); + EXIT_IF(action != 0x00); + + LOGF("\t temporary: acquire_mem CB-cache-only barrier, gcr_cntl = 0x%08" PRIx32 + ", base = 0x%016" PRIx64 ", size = 0x%016" PRIx64 "\n", + gcr_cntl, base_lo << 8u, size_lo << 8u); + + cp->MemoryBarrier(); + } break; + case 0x04000000: { + // target_mask: 0x00000000 (none) + // extended_action: 0x04000000 (FlushAndInvalidateDbCache) + // action: 0x00 (none) + EXIT_IF(target_mask != 0x00000000); + EXIT_IF(extended_action != 0x04000000); + EXIT_IF(action != 0x00); + + LOGF("\t temporary: acquire_mem DB-cache-only barrier, gcr_cntl = 0x%08" PRIx32 + ", base = 0x%016" PRIx64 ", size = 0x%016" PRIx64 "\n", + gcr_cntl, base_lo << 8u, size_lo << 8u); + + cp->MemoryBarrier(); + } break; + case 0x04004000: + case 0x04007fc0: { + // target_mask: 0x00004000 (Depth Target), 0x00007fc0 (all rt and depth) + // extended_action: 0x04000000 (FlushAndInvalidateDbCache) + // action: 0x00 (none) + EXIT_IF(target_mask != 0x00004000 && target_mask != 0x00007FC0); + EXIT_IF(extended_action != 0x04000000); + EXIT_IF(action != 0x00); + + LOGF("\t temporary: acquire_mem DB target barrier, target_mask = 0x%08" PRIx32 + ", gcr_cntl = 0x%08" PRIx32 ", base = 0x%016" PRIx64 ", size = 0x%016" PRIx64 "\n", + target_mask, gcr_cntl, base_lo << 8u, size_lo << 8u); + + if (size_lo != 0) { + cp->DepthStencilBarrier(base_lo << 8u, size_lo << 8u); + } else { + cp->MemoryBarrier(); + } + } break; + case 0x02c40040: + case 0x02c43fc0: + case 0x02c47fc0: { + // target_mask: 0x00000040 (rt0), 0x00003fc0 (all rt), 0x00007fc0 (all rt and depth) + // extended_action: 0x02000000 (FlushAndInvalidateCbCache) + // action: 0x38 (WriteBackAndInvalidateL1andL2) + EXIT_IF(target_mask != 0x00000040 && target_mask != 0x00003FC0 && + target_mask != 0x00007FC0); + EXIT_IF(extended_action != 0x02000000); + EXIT_IF(action != 0x38); + EXIT_NOT_IMPLEMENTED(size_lo == 0); + EXIT_NOT_IMPLEMENTED(base_lo == 0); + + cp->RenderTextureBarrier(base_lo << 8u, size_lo << 8u); + cp->SynchronizeGpu(); + } break; + case 0x02003fc0: + case 0x02007fc0: { + // target_mask: 0x00003FC0 (all rt), 0x00007fc0 (all rt and depth) + // extended_action: 0x02000000 (FlushAndInvalidateCbCache) + // action: 0x00 (none) + EXIT_IF(target_mask != 0x00003FC0 && target_mask != 0x00007fc0); + EXIT_IF(extended_action != 0x02000000); + EXIT_IF(action != 0x00); + + if (size_lo == 0) { + if (base_lo != 0) { + LOGF("\t warning: acquire_mem CB-cache barrier with non-zero base"); + } + + cp->MemoryBarrier(); + } else { + EXIT_NOT_IMPLEMENTED(base_lo == 0); + + cp->RenderTextureBarrier(base_lo << 8u, size_lo << 8u); + } + } break; + case 0x00C40000: { + // target_mask: 0x00000000 (none) + // extended_action: 0x00000000 (none) + // action: 0x38 (WriteBackAndInvalidateL1andL2) + EXIT_IF(target_mask != 0x00000000); + EXIT_IF(extended_action != 0x00000000); + EXIT_IF(action != 0x38); + EXIT_NOT_IMPLEMENTED(size_lo != 1); + EXIT_NOT_IMPLEMENTED(base_lo != 0); + + cp->MemoryBarrier(); + cp->SynchronizeGpu(); + } break; + case 0x00400000: { + // target_mask: 0x00000000 (none) + // extended_action: 0x00000000 (none) + // action: 0x10 (InvalidateL1) + EXIT_IF(target_mask != 0x00000000); + EXIT_IF(extended_action != 0x00000000); + EXIT_IF(action != 0x10); + EXIT_NOT_IMPLEMENTED(size_lo != 1); + EXIT_NOT_IMPLEMENTED(base_lo != 0); + + cp->MemoryBarrier(); + } break; + case 0x04c44000: { + // target_mask: 0x00004000 (Depth Target) + // extended_action: 0x04000000 (FlushAndInvalidateDbCache) + // action: 0x38 (WriteBackAndInvalidateL1andL2) + EXIT_IF(target_mask != 0x00004000); + EXIT_IF(extended_action != 0x04000000); + EXIT_IF(action != 0x38); + + cp->DepthStencilBarrier(base_lo << 8u, size_lo << 8u); + cp->SynchronizeGpu(); + } break; + + case 0x06000040: + case 0x06000080: + case 0x06003fc0: + case 0x06007fc0: { + // target_mask: 0x00000040 (rt0), 0x00000080 (rt1), 0x00003fc0 (all rt), 0x00007fc0 + // (all rt and depth) extended_action: 0x06000000 (Flush Cb & Db) action: 0x00 + // (none) + if (gcr_cntl != 0 && gcr_cntl != 0x280 && gcr_cntl != 0x300) { + LOGF("\t temporary: acquire_mem CB+DB barrier with unhandled GCR control " + "0x%08" PRIx32 "\n", + gcr_cntl); + } + + EXIT_IF(target_mask != 0x00000040 && target_mask != 0x00000080 && + target_mask != 0x00003fc0 && target_mask != 0x00007fc0); + EXIT_IF(extended_action != 0x06000000); + EXIT_IF(action != 0x00); + + if (size_lo != 0) { + if ((target_mask & 0x00003fc0) != 0) { + cp->RenderTextureBarrier(base_lo << 8u, size_lo << 8u); + } + if ((target_mask & 0x00004000) != 0) { + cp->DepthStencilBarrier(base_lo << 8u, size_lo << 8u); + } + } else { + cp->MemoryBarrier(); + } + } break; + + default: + EXIT("unknown barrier: 0x%08" PRIx32 ", 0x%08" PRIx32 ", 0x%08" PRIx32 ", 0x%08" PRIx32 + "\n", + cache_action, target_mask, extended_action, action); + } + + if (stall_mode == 0) { + cp->BufferFlush(); + cp->BufferWait(); + } + + return (custom ? 7 : 6); +} + +KYTY_CP_OP_PARSER(CpOpDispatchDirect) { + KYTY_PROFILER_FUNCTION(); + + EXIT_NOT_IMPLEMENTED(((cmd_id >> 8u) & 0xffu) != Pm4::IT_DISPATCH_DIRECT); + EXIT_NOT_IMPLEMENTED(KYTY_PM4_LEN(cmd_id) != 5u); + + uint32_t thread_group_x = buffer[0]; + uint32_t thread_group_y = buffer[1]; + uint32_t thread_group_z = buffer[2]; + uint32_t mode = buffer[3]; + + cp->DispatchDirect(thread_group_x, thread_group_y, thread_group_z, mode); + + return KYTY_PM4_LEN(cmd_id) - 1; +} + +KYTY_CP_OP_PARSER(CpOpDispatchIndirect) { + KYTY_PROFILER_FUNCTION(); + + EXIT_NOT_IMPLEMENTED(cmd_id != 0xc0011600 && cmd_id != 0xc0021600); + + if (cmd_id == 0xc0021600) { + struct DispatchIndirectArgs { + uint32_t thread_group_x; + uint32_t thread_group_y; + uint32_t thread_group_z; + }; + + auto* args = reinterpret_cast( + buffer[0] | (static_cast(buffer[1]) << 32u)); + uint32_t mode = buffer[2]; + + EXIT_NOT_IMPLEMENTED(args == nullptr); + cp->DispatchDirect(args->thread_group_x, args->thread_group_y, args->thread_group_z, mode); + + return 3; + } + + uint32_t data_offset = buffer[0]; + uint32_t mode = buffer[1]; + + cp->DispatchIndirect(data_offset, mode); + + return 2; +} + +KYTY_CP_OP_PARSER(CpOpGetLodStats) { + KYTY_PROFILER_FUNCTION(); + + EXIT_NOT_IMPLEMENTED(cmd_id != 0xc0038e00); + + const auto buffer_size = buffer[0]; + auto* dst = reinterpret_cast((buffer[1] & 0xffffffc0u) | + (static_cast(buffer[2]) << 32u)); + + if (dst != nullptr && buffer_size != 0) { + memset(dst, 0, buffer_size); + // Hack? + if (buffer_size >= sizeof(uint32_t)) { + auto* label = static_cast(dst); + *label = 1; + } + } + + return 4; +} + +KYTY_CP_OP_PARSER(CpOpDispatchReset) { + KYTY_PROFILER_FUNCTION(); + + EXIT_NOT_IMPLEMENTED(cmd_id != 0xC0001024); + + cp->Reset(); + + return 1; +} + +KYTY_CP_OP_PARSER(CpOpPfpSyncMe) { + KYTY_PROFILER_FUNCTION(); + + EXIT_NOT_IMPLEMENTED(cmd_id != 0xc0004200); + + return 1; +} + +KYTY_CP_OP_PARSER(CpOpSetPredication) { + KYTY_PROFILER_FUNCTION(); + + EXIT_NOT_IMPLEMENTED(((cmd_id >> 8u) & 0xffu) != Pm4::IT_SET_PREDICATION); + + auto payload_dw = KYTY_PM4_LEN(cmd_id) - 1u; + + EXIT_NOT_IMPLEMENTED(payload_dw < 2); + + uint32_t flags = 0; + uint64_t address_value = 0; + constexpr uint32_t predication_flags_mask = 0x00071100u; + if (payload_dw >= 3 && (buffer[0] & ~predication_flags_mask) == 0 && buffer[2] <= 0x0000ffffu) { + flags = buffer[0]; + address_value = (static_cast(buffer[2]) << 32u) | (buffer[1] & 0xfffffff0u); + } else { + flags = buffer[1]; + address_value = + (buffer[0] & 0xfffffff0u) | (static_cast(buffer[1] & 0xffu) << 32u); + } + + auto condition = (flags >> 8u) & 0x1u; + auto wait_op = (flags >> 12u) & 0x1u; + auto op = (flags >> 16u) & 0x7u; + auto* address = reinterpret_cast(address_value); + auto count_in_dwords = 0u; + + cp->SetPredication(condition, op, wait_op, address, count_in_dwords); + + return payload_dw; +} + +KYTY_CP_OP_PARSER(CpOpCondExec) { + KYTY_PROFILER_FUNCTION(); + + EXIT_NOT_IMPLEMENTED(((cmd_id >> 8u) & 0xffu) != Pm4::IT_COND_EXEC); + + auto payload_dw = KYTY_PM4_LEN(cmd_id) - 1u; + + EXIT_NOT_IMPLEMENTED(payload_dw < 4); + + auto addr = (static_cast(buffer[1]) << 32u) | (buffer[0] & 0xfffffffcu); + auto control = buffer[0] & 0x3u; + auto exec_count = buffer[3] & 0x3fffu; + + EXIT_NOT_IMPLEMENTED(control != 0); + EXIT_NOT_IMPLEMENTED(buffer[2] != 0); + EXIT_NOT_IMPLEMENTED(addr == 0); + EXIT_NOT_IMPLEMENTED(payload_dw + exec_count >= dw); + + if (*reinterpret_cast(addr) == 0) { + return payload_dw + exec_count; + } + + return payload_dw; +} + +KYTY_CP_OP_PARSER(CpOpBranch) { + KYTY_PROFILER_FUNCTION(); + + EXIT_NOT_IMPLEMENTED(((cmd_id >> 8u) & 0xffu) != Pm4::IT_INDIRECT_BUFFER); + + auto payload_dw = KYTY_PM4_LEN(cmd_id) - 1u; + + EXIT_NOT_IMPLEMENTED(payload_dw != 13); + + auto* compare_addr = reinterpret_cast( + (buffer[1] & 0xfffffff8u) | (static_cast(buffer[2]) << 32u)); + uint64_t mask = buffer[3] | (static_cast(buffer[4]) << 32u); + uint64_t reference = buffer[5] | (static_cast(buffer[6]) << 32u); + uint32_t mode = buffer[0] & 0x3u; + uint32_t function = (buffer[0] >> 8u) & 0x7u; + auto* then_buffer = reinterpret_cast((buffer[7] & 0xfffffffcu) | + (static_cast(buffer[8]) << 32u)); + uint32_t then_num_dw = buffer[9] & 0xfffffu; + auto* else_buffer = reinterpret_cast((buffer[10] & 0xfffffffcu) | + (static_cast(buffer[11]) << 32u)); + uint32_t else_num_dw = buffer[12] & 0xfffffu; + + EXIT_NOT_IMPLEMENTED(compare_addr == nullptr); + EXIT_NOT_IMPLEMENTED(mode != 1 && mode != 2); + EXIT_NOT_IMPLEMENTED(function > 6); + EXIT_NOT_IMPLEMENTED(then_buffer == nullptr || then_num_dw == 0); + + const bool take_then = TestWaitRegMemValue(*compare_addr, reference, mask, function); + LOGF("\t branch: take=%u then=0x%016" PRIx64 "/%" PRIu32 " else=0x%016" PRIx64 "/%" PRIu32 "\n", + take_then ? 1u : 0u, reinterpret_cast(then_buffer), then_num_dw, + reinterpret_cast(else_buffer), else_num_dw); + + if (take_then) { + cp->Run(then_buffer, then_num_dw); + } else if (mode == 2 && else_num_dw != 0) { + EXIT_NOT_IMPLEMENTED(else_buffer == nullptr); + cp->Run(else_buffer, else_num_dw); + } + + return payload_dw; +} + +static uint8_t CopyDataDstToDma(uint32_t dst) { + switch (dst) { + case 2: + case 4: + case 5: return 3; + case 3: + case 6: + case 7: return 1; + default: break; + } + + EXIT("unsupported copyData destination selector 0x%02" PRIx32 "\n", dst); +} + +static uint8_t CopyDataSrcToDma(uint32_t src) { + switch (src) { + case 2: + case 4: + case 5: return 3; + case 3: + case 6: + case 7: return 1; + case 5 << 1: + case (5 << 1) | 1: return 2; + default: break; + } + + EXIT("unsupported copyData source selector 0x%02" PRIx32 "\n", src); +} + +KYTY_CP_OP_PARSER(CpOpCopyData) { + KYTY_PROFILER_FUNCTION(); + + EXIT_NOT_IMPLEMENTED(cmd_id != KYTY_PM4(6, Pm4::IT_COPY_DATA, 0u)); + + const uint32_t control = buffer[0]; + const uint32_t src_sel = ((control & 0xfu) << 1u) | ((control >> 30u) & 0x1u); + const uint32_t dst_sel = ((control >> 8u) & 0xfu) << 1u; + const uint8_t src_cache = static_cast((control >> 13u) & 0x3u); + const uint8_t dst_cache = static_cast((control >> 25u) & 0x3u); + const uint8_t write_confirm = static_cast((control >> 20u) & 0x1u); + const uint32_t num_bytes = ((control >> 16u) & 0x1u) != 0 ? 8u : 4u; + const uint64_t src = buffer[1] | (static_cast(buffer[2]) << 32u); + const uint64_t dst = buffer[3] | (static_cast(buffer[4]) << 32u); + const auto dma_src = CopyDataSrcToDma(src_sel); + if (dma_src == 2 && num_bytes == 8) { + EXIT("unsupported 64-bit immediate copyData\n"); + } + + cp->DmaData(0, CopyDataDstToDma(dst_sel), dst_cache, dst, dma_src, src_cache, src, num_bytes, 1, + write_confirm, 1); + + return 5; +} + +KYTY_CP_OP_PARSER(CpOpDmaData) { + KYTY_PROFILER_FUNCTION(); + + EXIT_NOT_IMPLEMENTED(cmd_id != KYTY_PM4(7, Pm4::IT_DMA_DATA, 0u)); + + const uint32_t control = buffer[0]; + const uint32_t control2 = buffer[5]; + const uint64_t src = buffer[1] | (static_cast(buffer[2]) << 32u); + const uint64_t dst = buffer[3] | (static_cast(buffer[4]) << 32u); + + if (control == 0x60000000 && dst == 0x0003022c && (control2 >> 21u) == 0x141u) { + auto* addr = reinterpret_cast(src); + + cp->PrefetchL2(addr, control2 & 0x1fffffu); + return 6; + } + + const uint8_t engine = static_cast(control & 0x1u); + const uint8_t src_cache_policy = static_cast((control >> 13u) & 0x3u); + const uint8_t dst_sel = static_cast( + ((control >> 20u) & 0x3u) | ((control2 >> 25u) & 0x4u) | ((control2 >> 26u) & 0x8u)); + const uint8_t dst_cache_policy = static_cast((control >> 25u) & 0x3u); + const uint8_t src_sel = static_cast( + ((control >> 29u) & 0x3u) | ((control2 >> 24u) & 0x4u) | ((control2 >> 25u) & 0x8u)); + const uint8_t block_engine = static_cast((control >> 31u) & 0x1u); + const uint8_t wait_previous = static_cast((control2 >> 30u) & 0x1u); + const uint8_t write_confirm = static_cast((control2 >> 31u) & 0x1u); + const uint32_t num_bytes = control2 & 0x03ffffffu; + + cp->DmaData(engine, dst_sel, dst_cache_policy, dst, src_sel, src_cache_policy, src, num_bytes, + wait_previous, write_confirm, block_engine); + + return 6; +} + +KYTY_CP_OP_PARSER(CpOpDrawIndex) { + KYTY_PROFILER_FUNCTION(); + + EXIT_NOT_IMPLEMENTED(cmd_id != 0xc0073a00 && cmd_id != 0xc0042700); + + if (cmd_id == 0xc0073a00) { + uint32_t index_count = buffer[0]; + auto* index_addr = + reinterpret_cast(buffer[1] | (static_cast(buffer[2]) << 32u)); + uint32_t max_instance_count = buffer[3]; + auto* object_ids = + reinterpret_cast(buffer[4] | (static_cast(buffer[5]) << 32u)); + uint32_t instance_count = buffer[6]; + uint32_t flags = buffer[7]; + + EXIT_NOT_IMPLEMENTED(instance_count > max_instance_count); + EXIT_NOT_IMPLEMENTED((flags & ~0xa0u) != 0); + + cp->DrawIndex(index_count, index_addr, 0, 1, instance_count, object_ids); + + return 8; + } + + if (cmd_id == 0xc0042700) { + uint32_t max_index_count = buffer[0]; + auto* index_addr = + reinterpret_cast(buffer[1] | (static_cast(buffer[2]) << 32u)); + uint32_t index_count = buffer[3]; + uint32_t flags = buffer[4]; + + EXIT_NOT_IMPLEMENTED(index_count > max_index_count); + EXIT_NOT_IMPLEMENTED((flags & ~0x20u) != 0); + + cp->DrawIndex(index_count, index_addr, 0, 1); + + return 5; + } + + return 1; +} + +KYTY_CP_OP_PARSER(CpOpDrawIndirect) { + KYTY_PROFILER_FUNCTION(); + + EXIT_NOT_IMPLEMENTED(cmd_id != 0xc0032400 && cmd_id != 0xc0032500); + + const auto data_offset = buffer[0]; + const auto draw_initiator = buffer[3]; + const bool indexed = (cmd_id == 0xc0032500); + + cp->DrawIndirect(data_offset, draw_initiator, indexed); + + return 4; +} + +KYTY_CP_OP_PARSER(CpOpDrawIndirectMulti) { + KYTY_PROFILER_FUNCTION(); + + EXIT_NOT_IMPLEMENTED(cmd_id != 0xc0082c00 && cmd_id != 0xc0083800); + + const auto data_offset = buffer[0]; + const auto count_indirect = (buffer[3] >> 30u) & 0x1u; + const auto max_count_or_count = buffer[4]; + auto* count_addr = reinterpret_cast( + buffer[5] | (static_cast(buffer[6]) << 32u)); + const auto stride_in_bytes = buffer[7]; + const auto draw_initiator = buffer[8]; + const bool indexed = (cmd_id == 0xc0083800); + + if (count_indirect == 0) { + count_addr = nullptr; + } + + cp->DrawIndirectMulti(data_offset, max_count_or_count, count_addr, stride_in_bytes, + draw_initiator, indexed); + + return 9; +} + +KYTY_CP_OP_PARSER(CpOpDrawIndexOffset) { + KYTY_PROFILER_FUNCTION(); + + EXIT_NOT_IMPLEMENTED(cmd_id != 0xc0033500); + + uint32_t max_index_size = buffer[0]; + uint32_t index_offset = buffer[1]; + uint32_t index_count = buffer[2]; + uint32_t flags = buffer[3]; + + EXIT_NOT_IMPLEMENTED(index_count > max_index_size); + EXIT_NOT_IMPLEMENTED((flags & ~0x20u) != 0); + + cp->DrawIndexOffset(index_offset, index_count, 0); + + return 4; +} + +KYTY_CP_OP_PARSER(CpOpDrawIndexAuto) { + KYTY_PROFILER_FUNCTION(); + + EXIT_NOT_IMPLEMENTED(cmd_id != 0xc0012d00); + + uint32_t index_count = buffer[0]; + uint32_t flags = buffer[1]; + + EXIT_NOT_IMPLEMENTED((flags & ~0x22u) != 0); + + cp->DrawIndexAuto(index_count, 0); + + return 2; +} + +KYTY_CP_OP_PARSER(CpOpClearState) { + KYTY_PROFILER_FUNCTION(); + + EXIT_NOT_IMPLEMENTED(cmd_id != 0xc0001200); + EXIT_NOT_IMPLEMENTED((buffer[0] & ~0xfu) != 0); + + cp->Reset(); + + return 1; +} + +KYTY_CP_OP_PARSER(CpOpDumpConstRam) { + KYTY_PROFILER_FUNCTION(); + + EXIT_NOT_IMPLEMENTED(cmd_id != 0xC0038300); + + auto offset = buffer[0]; + auto dw_num = buffer[1]; + auto* dst = reinterpret_cast(buffer[2] | (static_cast(buffer[3]) << 32u)); + + EXIT_NOT_IMPLEMENTED(dw_num >= 0x3000); + EXIT_NOT_IMPLEMENTED(offset > 0xbffc); + EXIT_NOT_IMPLEMENTED((offset & 0x3u) != 0); + + cp->DumpConstRam(dst, offset, dw_num); + + return 4; +} + +KYTY_CP_OP_PARSER(CpOpEventWrite) { + KYTY_PROFILER_FUNCTION(); + + EXIT_NOT_IMPLEMENTED(((cmd_id >> 8u) & 0xffu) != Pm4::IT_EVENT_WRITE); + + uint32_t event_index = (buffer[0] >> 8u) & 0x7u; + uint32_t event_type = (buffer[0]) & 0x3fu; + + cp->TriggerEvent(event_type, event_index); + + return KYTY_PM4_LEN(cmd_id) - 1u; +} + +KYTY_CP_OP_PARSER(CpOpEventWriteEop) { + KYTY_PROFILER_FUNCTION(); + + EXIT_NOT_IMPLEMENTED(cmd_id != 0xC0044700); + + uint32_t cache_policy = (buffer[0] >> 25u) & 0x3u; + uint32_t event_write_dest = ((buffer[0] >> 23u) & 0x10u) | ((buffer[2] >> 16u) & 0x01u); + uint32_t eop_event_type = (buffer[0]) & 0x3fu; + uint32_t cache_action = (buffer[0] >> 12u) & 0x3fu; + uint32_t event_index = (buffer[0] >> 8u) & 0x7u; + uint32_t event_write_source = ((buffer[2] >> 29u) & 0x7u); + uint32_t interrupt_selector = (buffer[2] >> 24u) & 0x7u; + auto* dst_gpu_addr = + reinterpret_cast(buffer[1] | (static_cast(buffer[2] & 0xffffu) << 32u)); + uint64_t value = (buffer[3] | (static_cast(buffer[4]) << 32u)); + + cp->WriteAtEndOfPipe64(cache_policy, event_write_dest, eop_event_type, cache_action, + event_index, event_write_source, dst_gpu_addr, value, + interrupt_selector); + + return 5; +} + +KYTY_CP_OP_PARSER(CpOpEventWriteEos) { + KYTY_PROFILER_FUNCTION(); + + EXIT_NOT_IMPLEMENTED(cmd_id != 0xC0034802); + + uint32_t cache_policy = (buffer[0] >> 25u) & 0x3u; + uint32_t event_write_dest = 0; + uint32_t eop_event_type = (buffer[0]) & 0x3fu; + uint32_t cache_action = (buffer[0] >> 12u) & 0x3fu; + uint32_t event_index = (buffer[0] >> 8u) & 0x7u; + uint32_t event_write_source = ((buffer[2] >> 29u) & 0x7u); + uint32_t interrupt_selector = (buffer[2] >> 24u) & 0x7u; + + auto* dst_gpu_addr = + reinterpret_cast(buffer[1] | (static_cast(buffer[2] & 0xffffu) << 32u)); + uint32_t value = buffer[3]; + + cp->WriteAtEndOfPipe32(cache_policy, event_write_dest, eop_event_type, cache_action, + event_index, event_write_source, dst_gpu_addr, value, + interrupt_selector); + + return 4; +} + +KYTY_CP_OP_PARSER(CpOpFlip) { + KYTY_PROFILER_FUNCTION(); + + EXIT_NOT_IMPLEMENTED(cmd_id != 0xc004105c); + + CommandProcessor::FlipInfo f; + + f.handle = static_cast(buffer[0]); + f.index = static_cast(buffer[1]); + f.flip_mode = static_cast(buffer[2]); + f.flip_arg = static_cast(buffer[3] | (static_cast(buffer[4]) << 32u)); + + cp->SetFlip(f); + cp->Flip(); + cp->BufferFlush(); + return 5; +} + +KYTY_CP_OP_PARSER(CpOpIncrementCeCounter) { + KYTY_PROFILER_FUNCTION(); + + EXIT_NOT_IMPLEMENTED(cmd_id != 0xC0008400); + EXIT_NOT_IMPLEMENTED(buffer[0] != 1); + + cp->IncremenetCe(); + + return 1; +} + +KYTY_CP_OP_PARSER(CpOpIncrementDeCounter) { + KYTY_PROFILER_FUNCTION(); + + EXIT_NOT_IMPLEMENTED(cmd_id != 0xC0008500); + EXIT_NOT_IMPLEMENTED(buffer[0] != 0); + + cp->IncremenetDe(); + + return 1; +} + +KYTY_CP_OP_PARSER(CpOpIndexType) { + KYTY_PROFILER_FUNCTION(); + + EXIT_NOT_IMPLEMENTED(cmd_id != 0xC0002A00); + + cp->SetIndexType(buffer[0]); + + return 1; +} + +KYTY_CP_OP_PARSER(CpOpIndexBufferSize) { + KYTY_PROFILER_FUNCTION(); + + EXIT_NOT_IMPLEMENTED(cmd_id != 0xc0001300); + + cp->SetIndexBufferSize(buffer[0]); + + return 1; +} + +KYTY_CP_OP_PARSER(CpOpIndexBase) { + KYTY_PROFILER_FUNCTION(); + + EXIT_NOT_IMPLEMENTED(cmd_id != 0xc0012600); + + auto index_base_addr = buffer[0] | (static_cast(buffer[1]) << 32u); + + cp->SetIndexBaseAddress(index_base_addr); + + return 2; +} + +KYTY_CP_OP_PARSER(CpOpSetBase) { + KYTY_PROFILER_FUNCTION(); + + EXIT_NOT_IMPLEMENTED(((cmd_id >> 8u) & 0xffu) != Pm4::IT_SET_BASE); + EXIT_NOT_IMPLEMENTED(KYTY_PM4_LEN(cmd_id) != 4u); + + auto base_index = buffer[0] & 0xfu; + auto base_addr = (buffer[1] & ~0x7ull) | (static_cast(buffer[2] & 0xffffu) << 32u); + // libSceAgc::setBaseIndirectArgs encodes Gnmp::ShaderType in PM4 header bit 1. + auto shader_type = (cmd_id >> 1u) & 0x3u; + + EXIT_NOT_IMPLEMENTED(base_index != 1u); + EXIT_NOT_IMPLEMENTED(shader_type > 1u); + + if (shader_type == 0u) { + cp->SetDrawIndirectArgsBaseAddress(base_addr); + } else { + cp->SetDispatchIndirectArgsBaseAddress(base_addr); + } + + return 3; +} + +KYTY_CP_OP_PARSER(CpOpIndirectBuffer) { + KYTY_PROFILER_FUNCTION(); + + EXIT_NOT_IMPLEMENTED(((cmd_id >> 8u) & 0xffu) != Pm4::IT_INDIRECT_BUFFER); + + if (KYTY_PM4_LEN(cmd_id) == 14u) { + return CpOpBranch(cp, cmd_id, buffer, dw, num_dw); + } + + EXIT_NOT_IMPLEMENTED(KYTY_PM4_LEN(cmd_id) != 4u); + + const uint32_t control = buffer[2]; + + const uint32_t control_flags = control & 0x0fe00000u; + if (control_flags != 0x0f200000u) { + LOGF("\t temporary: accepting PS5 indirect buffer control flags = 0x%08" PRIx32 "\n", + control_flags); + } + + auto* indirect_buffer = + reinterpret_cast(buffer[0] | (static_cast(buffer[1]) << 32u)); + uint32_t indirect_num_dw = control & 0xfffffu; + static std::atomic indirect_log_count {0}; + if (indirect_log_count.fetch_add(1) < 128) { + LOGF("\t indirect buffer: addr=0x%016" PRIx64 ", num_dw=%" PRIu32 ", control=0x%08" PRIx32 + "\n", + reinterpret_cast(indirect_buffer), indirect_num_dw, control); + } + + if (indirect_num_dw == 0) { + return 3; + } + if (indirect_buffer == nullptr) { + EXIT("indirect command buffer has null address, num_dw = %" PRIu32 + ", control = 0x%08" PRIx32 "\n", + indirect_num_dw, buffer[2]); + } + + GraphicsDbgDumpDcb("ci", indirect_num_dw, indirect_buffer); + + cp->Run(indirect_buffer, indirect_num_dw); + + return 3; +} + +KYTY_CP_OP_PARSER(CpOpIndirectCxRegs) { + KYTY_PROFILER_FUNCTION(); + + EXIT_NOT_IMPLEMENTED(((cmd_id >> 8u) & 0xffu) != Pm4::IT_SET_CONTEXT_REG_INDIRECT); + EXIT_NOT_IMPLEMENTED(KYTY_PM4_LEN(cmd_id) != 5u); + + auto* indirect_buffer = + reinterpret_cast((static_cast(buffer[0]) & 0xfffffffcu) | + (static_cast(buffer[1]) << 32u)); + uint32_t indirect_num_dw = buffer[3] & 0x3fffu; + + if (indirect_num_dw == 0) { + return KYTY_PM4_LEN(cmd_id) - 1u; + } + if (indirect_buffer == nullptr) { + EXIT("indirect CX registers have null address, num_regs = %" PRIu32 "\n", indirect_num_dw); + } + + for (uint32_t i = 0; i < indirect_num_dw; i++, indirect_buffer += 2) { + auto cmd_offset = indirect_buffer[0]; + auto value = indirect_buffer[1]; + + if (HwCtxTrySetFakeRegister(cmd_offset, value)) { + continue; + } + + if (cmd_offset == 0xffffffffu) { + static bool logged = false; + if (!logged) { + LOGF("\t temporary: skipping indirect CX sentinel pair offset = 0xffffffff, value " + "= 0x%08" PRIx32 "\n", + value); + logged = true; + } + continue; + } + + if (cmd_offset >= Pm4::CX_NUM) { + static std::atomic log_count {0}; + if (log_count.fetch_add(1, std::memory_order_relaxed) < 16) { + LOGF("\t temporary: skipping unknown indirect CX extended offset = 0x%08" PRIx32 + ", value = 0x%08" PRIx32 "\n", + cmd_offset, value); + } + continue; + } + + auto pfunc = g_hw_ctx_indirect_func[cmd_offset & (Pm4::CX_NUM - 1)]; + + if (pfunc == nullptr) { + EXIT("unknown cx reg at %05" PRIx32 ": 0x%" PRIx32 "\n", num_dw - dw, cmd_offset); + } + + pfunc(cp, cmd_offset, value); + } + + return KYTY_PM4_LEN(cmd_id) - 1u; +} + +KYTY_CP_OP_PARSER(CpOpIndirectShRegs) { + KYTY_PROFILER_FUNCTION(); + + EXIT_NOT_IMPLEMENTED(((cmd_id >> 8u) & 0xffu) != Pm4::IT_SET_SH_REG_INDIRECT); + EXIT_NOT_IMPLEMENTED(KYTY_PM4_LEN(cmd_id) != 5u); + + auto* indirect_buffer = + reinterpret_cast((static_cast(buffer[0]) & 0xfffffffcu) | + (static_cast(buffer[1]) << 32u)); + uint32_t indirect_num_dw = buffer[3] & 0x3fffu; + + if (indirect_num_dw == 0) { + return KYTY_PM4_LEN(cmd_id) - 1u; + } + if (indirect_buffer == nullptr) { + EXIT("indirect SH registers have null address, num_regs = %" PRIu32 "\n", indirect_num_dw); + } + + for (uint32_t i = 0; i < indirect_num_dw; i++, indirect_buffer += 2) { + auto raw_cmd_offset = indirect_buffer[0]; + auto cmd_offset = NormalizeRegisterOffset(raw_cmd_offset); + auto value = indirect_buffer[1]; + + //Not sure if this is correct + if (raw_cmd_offset != cmd_offset) { + LOGF_COLOR(Log::Color::Red, + "\t temporary: normalized indirect SH register offset 0x%08" PRIx32 + " -> 0x%08" PRIx32 "\n", + raw_cmd_offset, cmd_offset); + } + + //Not sure if this is correct + if (cmd_offset == Pm4::SH_NOP || raw_cmd_offset == 0xffffffffu) { + continue; + } + + if (cmd_offset >= Pm4::SH_NUM) { + EXIT("unsupported indirect SH register offset 0x%08" PRIx32 " (raw 0x%08" PRIx32 + "), value = 0x%08" PRIx32 "\n", + cmd_offset, raw_cmd_offset, value); + } + + auto pfunc = g_hw_sh_indirect_func[cmd_offset]; + + if (pfunc == nullptr) { + LOGF("unknown indirect SH register: index=%" PRIu32 "/%" PRIu32 ", regs=0x%016" PRIx64 + ", offset=0x%08" PRIx32 ", value=0x%08" PRIx32 "\n", + i, indirect_num_dw, reinterpret_cast(indirect_buffer - i * 2), + cmd_offset, value); + auto* dump_regs = indirect_buffer - i * 2; + for (uint32_t j = 0; j < indirect_num_dw && j < 16; j++) { + LOGF("\t sh_indirect[%" PRIu32 "] offset=0x%08" PRIx32 ", value=0x%08" PRIx32 "\n", + j, dump_regs[j * 2], dump_regs[j * 2 + 1]); + } + EXIT("unknown sh reg at %05" PRIx32 ": 0x%" PRIx32 "\n", num_dw - dw, cmd_offset); + } + + pfunc(cp, cmd_offset, value); + } + + return KYTY_PM4_LEN(cmd_id) - 1u; +} + +KYTY_CP_OP_PARSER(CpOpIndirectUcRegs) { + KYTY_PROFILER_FUNCTION(); + + EXIT_NOT_IMPLEMENTED(((cmd_id >> 8u) & 0xffu) != Pm4::IT_SET_UCONFIG_REG_INDIRECT); + EXIT_NOT_IMPLEMENTED(KYTY_PM4_LEN(cmd_id) != 5u); + + auto* indirect_buffer = + reinterpret_cast((static_cast(buffer[0]) & 0xfffffffcu) | + (static_cast(buffer[1]) << 32u)); + uint32_t indirect_num_dw = buffer[3] & 0x3fffu; + + if (indirect_num_dw == 0) { + return KYTY_PM4_LEN(cmd_id) - 1u; + } + if (indirect_buffer == nullptr) { + EXIT("indirect UC registers have null address, num_regs = %" PRIu32 "\n", indirect_num_dw); + } + + for (uint32_t i = 0; i < indirect_num_dw; i++, indirect_buffer += 2) { + auto raw_cmd_offset = indirect_buffer[0]; + auto cmd_offset = NormalizeRegisterOffset(raw_cmd_offset); + auto value = indirect_buffer[1]; + + //Not sure if this is correct + if (raw_cmd_offset != cmd_offset) { + LOGF_COLOR(Log::Color::Red, + "\t temporary: normalized indirect UC register offset 0x%08" PRIx32 + " -> 0x%08" PRIx32 "\n", + raw_cmd_offset, cmd_offset); + } + if (HwUcTrySetFakeRegister(cmd_offset, value)) { + continue; + } + if (cmd_offset < Pm4::CX_NUM) { + auto ctx_func = g_hw_ctx_indirect_func[cmd_offset & (Pm4::CX_NUM - 1)]; + if (ctx_func != nullptr) { + ctx_func(cp, cmd_offset, value); + continue; + } + } + if (cmd_offset >= Pm4::UC_NUM) { + EXIT("unsupported indirect UC register offset 0x%08" PRIx32 " (raw 0x%08" PRIx32 + "), value = 0x%08" PRIx32 "\n", + cmd_offset, raw_cmd_offset, value); + } + + auto pfunc = g_hw_uc_indirect_func[cmd_offset & (Pm4::UC_NUM - 1)]; + + if (pfunc == nullptr) { + auto* dump_regs = indirect_buffer - i * 2; + for (uint32_t j = 0; j < indirect_num_dw && j < 16; j++) { + LOGF("\t uc_indirect[%" PRIu32 "] offset=0x%08" PRIx32 ", value=0x%08" PRIx32 "\n", + j, dump_regs[j * 2], dump_regs[j * 2 + 1]); + } + EXIT("unknown uc reg at %05" PRIx32 ": 0x%" PRIx32 "\n", num_dw - dw, cmd_offset); + } + pfunc(cp, cmd_offset, value); + } + + return KYTY_PM4_LEN(cmd_id) - 1u; +} + +KYTY_CP_OP_PARSER(CpOpMarker) { + KYTY_PROFILER_FUNCTION(); + + // EXIT_NOT_IMPLEMENTED(cmd_id != 0xC0001000); + + uint32_t id = buffer[0] & 0xfffu; + uint32_t align = (buffer[0] >> 12u) & 0xfu; + uint32_t len_dw = ((cmd_id >> 16u) & 0x3fffu); + + switch (id) { + case 0x0: cp->SetEmbeddedDataMarker(buffer + 1, len_dw, align); break; + case 0x4: cp->SetUserDataMarker(HW::UserSgprType::Vsharp); break; + case 0xd: cp->SetUserDataMarker(HW::UserSgprType::Region); break; + case 0x777: { + cp->Flip(); + break; + } + case 0x778: { + auto* addr = + reinterpret_cast(buffer[1] | (static_cast(buffer[2]) << 32u)); + uint32_t value = buffer[3]; + cp->Flip(addr, value); + break; + } + case 0x781: { + auto* addr = + reinterpret_cast(buffer[1] | (static_cast(buffer[2]) << 32u)); + uint32_t value = buffer[3]; + uint32_t eop_event_type = buffer[4]; + uint32_t cache_action = buffer[5]; + cp->FlipWithInterrupt(eop_event_type, cache_action, addr, value); + break; + } + default: EXIT("unknown marker at %05" PRIx32 ": 0x%" PRIx32 "\n", num_dw - dw, id); + } + + return len_dw + 1; +} + +KYTY_CP_OP_PARSER(CpOpNop) { + KYTY_PROFILER_FUNCTION(); + + auto r = KYTY_PM4_R(cmd_id); + + if (r == Pm4::R_ZERO) { + if ((buffer[0] & 0xffff0000u) == 0x68750000u) { + return CpOpMarker(cp, cmd_id, buffer, dw, num_dw); + } + return KYTY_PM4_LEN(cmd_id) - 1; + } + + auto cp_op = g_cp_op_custom_func[r]; + + if (cp_op != nullptr) { + return cp_op(cp, cmd_id, buffer, dw, num_dw); + } + + EXIT("unknown custom code at 0x%05" PRIx32 ": 0x%02" PRIx32 "\n", num_dw - dw, r); + + return 0; +} + +KYTY_CP_OP_PARSER(CpOpNumInstances) { + KYTY_PROFILER_FUNCTION(); + + EXIT_NOT_IMPLEMENTED(cmd_id != 0xc0002f00); + + cp->SetNumInstances(buffer[0]); + + return 1; +} + +KYTY_CP_OP_PARSER(CpOpPopMarker) { + KYTY_PROFILER_FUNCTION(); + + auto dw_num = (cmd_id >> 16u) & 0x3fffu; + + static std::atomic pop_marker_log_count {0}; + if (pop_marker_log_count.fetch_add(1) < 64) { + LOGF("Pop marker\n"); + } + + cp->PopMarker(); + + return dw_num + 1; +} + +KYTY_CP_OP_PARSER(CpOpPushMarker) { + KYTY_PROFILER_FUNCTION(); + + auto dw_num = (cmd_id >> 16u) & 0x3fffu; + + const char* str = reinterpret_cast(buffer); + + static std::atomic push_marker_log_count {0}; + if (push_marker_log_count.fetch_add(1) < 128) { + LOGF("Push marker: %s\n", str); + } + + cp->PushMarker(str); + + return dw_num + 1; +} + +KYTY_CP_OP_PARSER(CpOpReleaseMem) { + KYTY_PROFILER_FUNCTION(); + + EXIT_NOT_IMPLEMENTED(cmd_id != 0xc0061060); + + uint32_t cache_policy = (buffer[0] >> 25u) & 0x3u; + uint32_t eop_event_type = buffer[0] & 0x3fu; + uint32_t event_index = (buffer[0] >> 8u) & 0x7u; + uint32_t gcr_cntl = (buffer[0] >> 12u) & 0xfffu; + uint32_t release_dst = (buffer[1] >> 16u) & 0x3u; + uint32_t event_write_dest = 0; + uint32_t data_sel = (buffer[1] >> 29u) & 0x7u; + uint32_t interrupt_selector = (buffer[1] >> 24u) & 0x7u; + auto* dst_gpu_addr = + reinterpret_cast(buffer[2] | (static_cast(buffer[3]) << 32u)); + uint64_t value = buffer[4] | (static_cast(buffer[5]) << 32u); + uint32_t interrupt_context_id = buffer[6] & 0x07ffffffu; + + constexpr uint32_t ReleaseMemDstMemory = 0; + constexpr uint32_t ReleaseMemDstTcL2 = 1; + EXIT_NOT_IMPLEMENTED(release_dst != ReleaseMemDstMemory && release_dst != ReleaseMemDstTcL2); + EXIT_NOT_IMPLEMENTED(data_sel != 0 && data_sel != 1 && data_sel != 2 && data_sel != 3 && + data_sel != 5); + + LogUnknownReleaseMemGcr(gcr_cntl); + + const bool gl2_writeback = ((gcr_cntl & GcrGl2Writeback) != 0); + + auto trigger_interrupt = [&]() { + switch (interrupt_selector) { + case 0x00: + case 0x03: break; + case 0x01: + case 0x02: + case 0x04: cp->TriggerEopEventAtEndOfPipe(interrupt_context_id); break; + default: EXIT("unknown release_mem interrupt selector\n"); + } + }; + + if (data_sel == 0 || interrupt_selector == 4) { + if (eop_event_type != 0x28 || gcr_cntl != 0) { + cp->MemoryBarrier(); + } + + if (gl2_writeback) { + cp->SynchronizeGpu(); + } + + trigger_interrupt(); + + return 7; + } + + if (release_dst == ReleaseMemDstMemory && dst_gpu_addr == nullptr) { + if (eop_event_type != 0x28 || gcr_cntl != 0) { + cp->MemoryBarrier(); + } + + if (gl2_writeback) { + cp->SynchronizeGpu(); + } + + trigger_interrupt(); + + return 7; + } + + if (ReleaseMemGcrNeedsBarrier(eop_event_type, gcr_cntl)) { + cp->MemoryBarrier(); + } + + auto cache_action = ReleaseMemCacheActionFromGcr(gcr_cntl); + cache_policy = 0; + + if (data_sel == 1) { + static std::atomic log_count {0}; + const auto dst_addr = reinterpret_cast(dst_gpu_addr); + if ((dst_addr & 0xffffffffff000000ull) == 0x0000001021000000ull || log_count.load() < 64) { + if (log_count.fetch_add(1) < 256) { + LOGF( + "\t runtime release_mem32: dst=0x%016" PRIx64 ", value=0x%08" PRIx32 + ", action=0x%02" PRIx32 ", gcr=0x%04" PRIx32 ", cache_action=0x%08" PRIx32 "\n", + dst_addr, static_cast(value), eop_event_type, gcr_cntl, cache_action); + } + } + + eop_event_type = 0x2f; + event_index = 6; + auto event_source = 2u; + + cp->WriteAtEndOfPipe32( + cache_policy, event_write_dest, eop_event_type, cache_action, event_index, event_source, + dst_gpu_addr, static_cast(value), interrupt_selector, interrupt_context_id); + cp->BufferFlush(); + + return 7; + } + + if (data_sel == 5) { + if (gl2_writeback) { + cp->SynchronizeGpu(); + cache_action = 0; + } + + eop_event_type = 0x2f; + event_index = 6; + auto event_source = 1u; + + cp->WriteAtEndOfPipe32( + cache_policy, event_write_dest, eop_event_type, cache_action, event_index, event_source, + dst_gpu_addr, static_cast(value), interrupt_selector, interrupt_context_id); + cp->BufferFlush(); + + return 7; + } + + const bool keep_native_event_index = + (eop_event_type == 0x04 && cache_action == 0x00) || + (eop_event_type == 0x2f && event_index == 0x06 && cache_action == 0x38); + + if (data_sel == 2) { + if (!keep_native_event_index) { + event_index = 0; + } + } else { + EXIT_IF(data_sel != 3); + if (!keep_native_event_index) { + event_index = 0; + } + data_sel = 4; + } + + cp->WriteAtEndOfPipe64(cache_policy, event_write_dest, eop_event_type, cache_action, + event_index, data_sel, dst_gpu_addr, value, interrupt_selector, + interrupt_context_id); + + return 7; +} + +KYTY_CP_OP_PARSER(CpOpSetContextReg) { + KYTY_PROFILER_FUNCTION(); + + auto cmd_offset = NormalizeRegisterOffset(buffer[0]); + + if (HwCtxTrySetFakeRegister(cmd_offset, buffer[1])) { + return 2; + } + + if (cmd_offset >= Pm4::CX_NUM) { + EXIT("unknown extended context register\n\t%05" PRIx32 ":\n\tcmd_id = %08" PRIx32 + "\n\tcmd_offset = %08" PRIx32 "\n\tvalue = %08" PRIx32 "\n", + num_dw - dw, cmd_id, cmd_offset, buffer[1]); + } + + auto pfunc = g_hw_ctx_func[cmd_offset & (Pm4::CX_NUM - 1)]; + + if (pfunc == nullptr) { + auto num_values = KYTY_PM4_LEN(cmd_id) - 2u; + bool handled = true; + for (uint32_t i = 0; i < num_values; i++) { + if (cmd_offset + i >= Pm4::CX_NUM || + g_hw_ctx_indirect_func[(cmd_offset + i) & (Pm4::CX_NUM - 1)] == nullptr) { + handled = false; + break; + } + } + if (handled) { + for (uint32_t i = 0; i < num_values; i++) { + g_hw_ctx_indirect_func[(cmd_offset + i) & (Pm4::CX_NUM - 1)](cp, cmd_offset + i, + buffer[1 + i]); + } + return num_values + 1u; + } + EXIT("unknown context register\n\t%05" PRIx32 ":\n\tcmd_id = %08" PRIx32 + "\n\tcmd_offset = %08" PRIx32 "\n", + num_dw - dw, cmd_id, cmd_offset); + } + + auto s = pfunc(cp, cmd_id, cmd_offset, buffer + 1, dw); + + EXIT_IF(s == 0); + + return s + 1; +} + +KYTY_CP_OP_PARSER(CpOpSetShaderReg) { + KYTY_PROFILER_FUNCTION(); + + auto cmd_offset = buffer[0]; + if (cmd_offset == Pm4::SH_NOP) { + return 2; + } + + EXIT_NOT_IMPLEMENTED(cmd_offset >= Pm4::SH_NUM); + + auto pfunc = g_hw_sh_func[cmd_offset]; + + if (pfunc == nullptr) { + EXIT("unknown shader register\n\t%05" PRIx32 ":\n\tcmd_id = %08" PRIx32 + "\n\tcmd_offset = %08" PRIx32 "\n", + num_dw - dw, cmd_id, cmd_offset); + } + + auto s = pfunc(cp, cmd_id, cmd_offset, buffer + 1, dw); + EXIT_IF(s == 0); + return s + 1; +} + +KYTY_CP_OP_PARSER(CpOpSetUconfigReg) { + KYTY_PROFILER_FUNCTION(); + + auto raw_cmd_offset = buffer[0]; + auto cmd_offset = NormalizeRegisterOffset(raw_cmd_offset); + + if (((cmd_id >> 8u) & 0xffu) == Pm4::IT_SET_UCONFIG_REG_INDEX) { + cmd_offset = raw_cmd_offset & 0x0fffffffu; + } + + if (raw_cmd_offset != cmd_offset) { + LOGF_COLOR(Log::Color::Red, + "\t temporary: normalized UC register offset 0x%08" PRIx32 " -> 0x%08" PRIx32 + "\n", + raw_cmd_offset, cmd_offset); + } + if (HwUcTrySetFakeRegisterRange(cmd_offset, buffer + 1, KYTY_PM4_LEN(cmd_id) - 2u)) { + return KYTY_PM4_LEN(cmd_id) - 1u; + } + if (cmd_offset >= Pm4::UC_NUM) { + EXIT("unsupported UC register offset 0x%08" PRIx32 " (raw 0x%08" PRIx32 + "), cmd_id = 0x%08" PRIx32 "\n", + cmd_offset, raw_cmd_offset, cmd_id); + } + + auto pfunc = g_hw_uc_func[cmd_offset & (Pm4::UC_NUM - 1)]; + + if (pfunc == nullptr) { + auto num_values = KYTY_PM4_LEN(cmd_id) - 2u; + bool handled = true; + for (uint32_t i = 0; i < num_values; i++) { + if (cmd_offset + i >= Pm4::UC_NUM || + g_hw_uc_indirect_func[(cmd_offset + i) & (Pm4::UC_NUM - 1)] == nullptr) { + handled = false; + break; + } + } + if (handled) { + for (uint32_t i = 0; i < num_values; i++) { + g_hw_uc_indirect_func[(cmd_offset + i) & (Pm4::UC_NUM - 1)](cp, cmd_offset + i, + buffer[1 + i]); + } + return num_values + 1u; + } + EXIT("unknown user config register\n\t%05" PRIx32 ":\n\tcmd_id = %08" PRIx32 + "\n\tcmd_offset = %08" PRIx32 "\n", + num_dw - dw, cmd_id, cmd_offset); + } + + auto s = pfunc(cp, cmd_id, cmd_offset, buffer + 1, dw); + EXIT_IF(s == 0); + return s + 1; +} + +KYTY_CP_OP_PARSER(CpOpWaitFlipDone) { + KYTY_PROFILER_FUNCTION(); + + EXIT_NOT_IMPLEMENTED(cmd_id != 0xc0051018); + + auto video_out_handle = buffer[0]; + auto display_buffer_index = buffer[1]; + + cp->WaitFlipDone(video_out_handle, display_buffer_index); + + return 6; +} + +template +static T CpOpWaitRegMemReadValue(const uint32_t* buffer) { + static_assert(sizeof(T) == sizeof(uint32_t) || sizeof(T) == sizeof(uint64_t)); + + if constexpr (sizeof(T) == sizeof(uint32_t)) { + return buffer[0]; + } else { + return buffer[0] | (static_cast(buffer[1]) << 32u); + } +} + +template +static uint32_t CpOpWaitRegMemWaitOp(uint32_t control) { + static_assert(sizeof(T) == sizeof(uint32_t) || sizeof(T) == sizeof(uint64_t)); + + if constexpr (sizeof(T) == sizeof(uint32_t)) { + return ((control >> 8u) & 0x3u) | ((control >> 4u) & 0xcu); + } else { + return ((control >> 8u) & 0x1u) | ((control >> 5u) & 0x6u); + } +} + +template +static uint32_t CpOpWaitRegMemSized(CommandProcessor* cp, uint32_t cmd_id, const uint32_t* buffer) { + static_assert(sizeof(T) == sizeof(uint32_t) || sizeof(T) == sizeof(uint64_t)); + + constexpr auto value_dw = static_cast(sizeof(T) / sizeof(uint32_t)); + constexpr auto payload_dw = 4u + value_dw * 2u; + constexpr auto packet_dw = payload_dw + 1u; + constexpr auto register_id = + (sizeof(T) == sizeof(uint32_t) ? Pm4::R_WAIT_MEM_32 : Pm4::R_WAIT_MEM_64); + + EXIT_NOT_IMPLEMENTED(KYTY_PM4_R(cmd_id) != register_id); + EXIT_NOT_IMPLEMENTED(KYTY_PM4_LEN(cmd_id) != packet_dw); + + auto* addr = reinterpret_cast(buffer[0] | (static_cast(buffer[1]) << 32u)); + auto mask = CpOpWaitRegMemReadValue(buffer + 2u); + auto ref = CpOpWaitRegMemReadValue(buffer + 2u + value_dw); + auto ctrl = buffer[2u + value_dw * 2u]; + auto poll = buffer[3u + value_dw * 2u]; + + EXIT_NOT_IMPLEMENTED((ctrl & 0x10u) == 0); + + cp->WaitRegMem(ctrl & 0x7u, addr, ref, mask, poll, CpOpWaitRegMemWaitOp(ctrl)); + + return payload_dw; +} + +KYTY_CP_OP_PARSER(CpOpWaitRegMem32) { + KYTY_PROFILER_FUNCTION(); + + return CpOpWaitRegMemSized(cp, cmd_id, buffer); +} + +KYTY_CP_OP_PARSER(CpOpWaitRegMem64) { + KYTY_PROFILER_FUNCTION(); + + return CpOpWaitRegMemSized(cp, cmd_id, buffer); +} + +KYTY_CP_OP_PARSER(CpOpWaitOnCeCounter) { + KYTY_PROFILER_FUNCTION(); + + EXIT_NOT_IMPLEMENTED(cmd_id != 0xC0008600); + EXIT_NOT_IMPLEMENTED(buffer[0] != 1); + + cp->WaitCe(); + + return 1; +} + +KYTY_CP_OP_PARSER(CpOpWaitOnDeCounterDiff) { + KYTY_PROFILER_FUNCTION(); + + EXIT_NOT_IMPLEMENTED(cmd_id != 0xC0008800); + + cp->WaitDeDiff(buffer[0]); + + return 1; +} + +KYTY_CP_OP_PARSER(CpOpWriteConstRam) { + KYTY_PROFILER_FUNCTION(); + + auto dw_num = (cmd_id >> 16u) & 0x3fffu; + auto offset = buffer[0]; + + EXIT_NOT_IMPLEMENTED(dw_num >= 0x3000); + EXIT_NOT_IMPLEMENTED(offset > 0xbffc); + EXIT_NOT_IMPLEMENTED((offset & 0x3u) != 0); + + cp->WriteConstRam(offset, buffer + 1, dw_num); + + return 1 + dw_num; +} + +KYTY_CP_OP_PARSER(CpOpWriteData) { + KYTY_PROFILER_FUNCTION(); + + auto op = (cmd_id >> 8u) & 0xffu; + + EXIT_NOT_IMPLEMENTED(op != Pm4::IT_WRITE_DATA); + + auto dw_num = (cmd_id >> 16u) & 0x3fffu; + + auto write_control = buffer[0]; + auto* dst = reinterpret_cast(buffer[1] | (static_cast(buffer[2]) << 32u)); + + cp->WriteData(dst, buffer + 3, dw_num - 2, write_control); + + return 1 + dw_num; +} + +void GraphicsInitJmpTablesCxIndirect() { + for (auto& func: g_hw_ctx_indirect_func) { + func = nullptr; + } + + for (auto cmd_offset = Pm4::SPI_PS_INPUT_CNTL_0; cmd_offset <= Pm4::SPI_PS_INPUT_CNTL_31; + cmd_offset++) { + g_hw_ctx_indirect_func[cmd_offset] = [](KYTY_HW_CTX_INDIRECT_ARGS) { + uint32_t slot = cmd_offset - Pm4::SPI_PS_INPUT_CNTL_0; + cp->GetCtx()->SetPsInputSettings(slot, value); + }; + } + + for (auto cmd_offset = Pm4::CB_COLOR0_BASE; cmd_offset <= Pm4::CB_COLOR7_BASE; + cmd_offset += 15) { + g_hw_ctx_indirect_func[cmd_offset] = [](KYTY_HW_CTX_INDIRECT_ARGS) { + uint32_t slot = (cmd_offset - Pm4::CB_COLOR0_BASE) / 15; + auto base = cp->GetCtx()->GetRenderTarget(slot).base; + base.addr &= 0xFFFFFF00000000FFull; + base.addr |= static_cast(value) << 8u; + cp->GetCtx()->SetColorBase(slot, base); + }; + } + + for (auto cmd_offset = Pm4::CB_COLOR0_BASE + 1; cmd_offset <= Pm4::CB_COLOR7_BASE + 1; + cmd_offset += 15) { + g_hw_ctx_indirect_func[cmd_offset] = [](KYTY_HW_CTX_INDIRECT_ARGS) { + uint32_t slot = (cmd_offset - (Pm4::CB_COLOR0_BASE + 1)) / 15; + HW::ColorPitch pitch; + pitch.pitch_div8_minus1 = value & 0x7ffu; + pitch.fmask_pitch_div8_minus1 = (value >> 20u) & 0x7ffu; + cp->GetCtx()->SetColorPitch(slot, pitch); + }; + } + + for (auto cmd_offset = Pm4::CB_COLOR0_BASE + 2; cmd_offset <= Pm4::CB_COLOR7_BASE + 2; + cmd_offset += 15) { + g_hw_ctx_indirect_func[cmd_offset] = [](KYTY_HW_CTX_INDIRECT_ARGS) { + uint32_t slot = (cmd_offset - (Pm4::CB_COLOR0_BASE + 2)) / 15; + HW::ColorSlice slice; + slice.slice_div64_minus1 = value & 0x3fffffu; + cp->GetCtx()->SetColorSlice(slot, slice); + }; + } + + for (auto cmd_offset = Pm4::CB_COLOR0_BASE_EXT; cmd_offset <= Pm4::CB_COLOR7_BASE_EXT; + cmd_offset++) { + g_hw_ctx_indirect_func[cmd_offset] = [](KYTY_HW_CTX_INDIRECT_ARGS) { + uint32_t slot = (cmd_offset - Pm4::CB_COLOR0_BASE_EXT); + auto base = cp->GetCtx()->GetRenderTarget(slot).base; + base.addr &= 0xFFFF00FFFFFFFFFFull; + base.addr |= (static_cast(value) & 0xffu) << 40u; + cp->GetCtx()->SetColorBase(slot, base); + }; + } + + for (auto cmd_offset = Pm4::CB_COLOR0_VIEW; cmd_offset <= Pm4::CB_COLOR7_VIEW; + cmd_offset += 15) { + g_hw_ctx_indirect_func[cmd_offset] = [](KYTY_HW_CTX_INDIRECT_ARGS) { + uint32_t slot = (cmd_offset - Pm4::CB_COLOR0_VIEW) / 15; + HW::ColorView view; + view.base_array_slice_index = KYTY_PM4_GET(value, CB_COLOR0_VIEW, SLICE_START); + view.last_array_slice_index = KYTY_PM4_GET(value, CB_COLOR0_VIEW, SLICE_MAX); + view.current_mip_level = KYTY_PM4_GET(value, CB_COLOR0_VIEW, MIP_LEVEL); + cp->GetCtx()->SetColorView(slot, view); + }; + } + + for (auto cmd_offset = Pm4::CB_COLOR0_INFO; cmd_offset <= Pm4::CB_COLOR7_INFO; + cmd_offset += 15) { + g_hw_ctx_indirect_func[cmd_offset] = [](KYTY_HW_CTX_INDIRECT_ARGS) { + uint32_t slot = (cmd_offset - Pm4::CB_COLOR0_INFO) / 15; + HW::ColorInfo info; + info.format = KYTY_PM4_GET(value, CB_COLOR0_INFO, FORMAT); + info.channel_type = KYTY_PM4_GET(value, CB_COLOR0_INFO, NUMBER_TYPE); + info.channel_order = KYTY_PM4_GET(value, CB_COLOR0_INFO, COMP_SWAP); + info.cmask_fast_clear_enable = KYTY_PM4_GET(value, CB_COLOR0_INFO, FAST_CLEAR) != 0; + info.fmask_compression_enable = KYTY_PM4_GET(value, CB_COLOR0_INFO, COMPRESSION) != 0; + info.blend_clamp = KYTY_PM4_GET(value, CB_COLOR0_INFO, BLEND_CLAMP) != 0; + info.blend_bypass = KYTY_PM4_GET(value, CB_COLOR0_INFO, BLEND_BYPASS) != 0; + info.round_mode = KYTY_PM4_GET(value, CB_COLOR0_INFO, ROUND_MODE) != 0; + info.cmask_is_linear = KYTY_PM4_GET(value, CB_COLOR0_INFO, CMASK_IS_LINEAR); + info.fmask_data_compression_disable = + KYTY_PM4_GET(value, CB_COLOR0_INFO, FMASK_COMPRESSION_DISABLE) != 0; + info.fmask_one_frag_mode = + KYTY_PM4_GET(value, CB_COLOR0_INFO, FMASK_COMPRESS_1FRAG_ONLY) != 0; + info.dcc_compression_enable = KYTY_PM4_GET(value, CB_COLOR0_INFO, DCC_ENABLE) != 0; + info.cmask_addr_type = KYTY_PM4_GET(value, CB_COLOR0_INFO, CMASK_ADDR_TYPE); + info.alt_tile_mode = KYTY_PM4_GET(value, CB_COLOR0_INFO, ALT_TILE_MODE) != 0; + cp->GetCtx()->SetColorInfo(slot, info); + }; + } + + for (auto cmd_offset = Pm4::CB_COLOR0_ATTRIB; cmd_offset <= Pm4::CB_COLOR7_ATTRIB; + cmd_offset += 15) { + g_hw_ctx_indirect_func[cmd_offset] = [](KYTY_HW_CTX_INDIRECT_ARGS) { + uint32_t slot = (cmd_offset - Pm4::CB_COLOR0_ATTRIB) / 15; + HW::ColorAttrib attrib; + attrib.force_dest_alpha_to_one = + KYTY_PM4_GET(value, CB_COLOR0_ATTRIB, FORCE_DST_ALPHA_1) != 0; + attrib.tile_mode = KYTY_PM4_GET(value, CB_COLOR0_ATTRIB, TILE_MODE_INDEX); + attrib.fmask_tile_mode = KYTY_PM4_GET(value, CB_COLOR0_ATTRIB, FMASK_TILE_MODE_INDEX); + attrib.num_samples = KYTY_PM4_GET(value, CB_COLOR0_ATTRIB, NUM_SAMPLES); + attrib.num_fragments = KYTY_PM4_GET(value, CB_COLOR0_ATTRIB, NUM_FRAGMENTS); + cp->GetCtx()->SetColorAttrib(slot, attrib); + }; + } + + for (auto cmd_offset = Pm4::CB_COLOR0_DCC_CONTROL; cmd_offset <= Pm4::CB_COLOR7_DCC_CONTROL; + cmd_offset += 15) { + g_hw_ctx_indirect_func[cmd_offset] = [](KYTY_HW_CTX_INDIRECT_ARGS) { + uint32_t slot = (cmd_offset - Pm4::CB_COLOR0_DCC_CONTROL) / 15; + HW::ColorDccControl dcc; + dcc.overwrite_combiner_disable = + KYTY_PM4_GET(value, CB_COLOR0_DCC_CONTROL, OVERWRITE_COMBINER_DISABLE) != 0; + dcc.dcc_clear_key_enable = + KYTY_PM4_GET(value, CB_COLOR0_DCC_CONTROL, KEY_CLEAR_ENABLE) != 0; + dcc.max_uncompressed_block_size = + KYTY_PM4_GET(value, CB_COLOR0_DCC_CONTROL, MAX_UNCOMPRESSED_BLOCK_SIZE); + dcc.min_compressed_block_size = + KYTY_PM4_GET(value, CB_COLOR0_DCC_CONTROL, MIN_COMPRESSED_BLOCK_SIZE); + dcc.max_compressed_block_size = + KYTY_PM4_GET(value, CB_COLOR0_DCC_CONTROL, MAX_COMPRESSED_BLOCK_SIZE); + dcc.color_transform = KYTY_PM4_GET(value, CB_COLOR0_DCC_CONTROL, COLOR_TRANSFORM); + dcc.independent_64b_blocks = + KYTY_PM4_GET(value, CB_COLOR0_DCC_CONTROL, INDEPENDENT_64B_BLOCKS) != 0; + dcc.data_write_on_dcc_clear_to_reg = + KYTY_PM4_GET(value, CB_COLOR0_DCC_CONTROL, ENABLE_CONSTANT_ENCODE_REG_WRITE) != 0; + dcc.independent_128b_blocks = + KYTY_PM4_GET(value, CB_COLOR0_DCC_CONTROL, INDEPENDENT_128B_BLOCKS) != 0; + cp->GetCtx()->SetColorDccControl(slot, dcc); + }; + } + + for (auto cmd_offset = Pm4::CB_COLOR0_CMASK; cmd_offset <= Pm4::CB_COLOR7_CMASK; + cmd_offset += 15) { + g_hw_ctx_indirect_func[cmd_offset] = [](KYTY_HW_CTX_INDIRECT_ARGS) { + uint32_t slot = (cmd_offset - Pm4::CB_COLOR0_CMASK) / 15; + auto base = cp->GetCtx()->GetRenderTarget(slot).cmask; + base.addr &= 0xFFFFFF00000000FFull; + base.addr |= static_cast(value) << 8u; + cp->GetCtx()->SetColorCmask(slot, base); + }; + } + + for (auto cmd_offset = Pm4::CB_COLOR0_CMASK + 1; cmd_offset <= Pm4::CB_COLOR7_BASE + 8; + cmd_offset += 15) { + g_hw_ctx_indirect_func[cmd_offset] = [](KYTY_HW_CTX_INDIRECT_ARGS) { + uint32_t slot = (cmd_offset - (Pm4::CB_COLOR0_CMASK + 1)) / 15; + HW::ColorCmaskSlice cmask_slice; + cmask_slice.slice_minus1 = value & 0x3fffu; + cp->GetCtx()->SetColorCmaskSlice(slot, cmask_slice); + }; + } + + for (auto cmd_offset = Pm4::CB_COLOR0_CMASK_BASE_EXT; + cmd_offset <= Pm4::CB_COLOR7_CMASK_BASE_EXT; cmd_offset++) { + g_hw_ctx_indirect_func[cmd_offset] = [](KYTY_HW_CTX_INDIRECT_ARGS) { + uint32_t slot = (cmd_offset - Pm4::CB_COLOR0_CMASK_BASE_EXT); + auto base = cp->GetCtx()->GetRenderTarget(slot).cmask; + base.addr &= 0xFFFF00FFFFFFFFFFull; + base.addr |= (static_cast(value) & 0xffu) << 40u; + cp->GetCtx()->SetColorCmask(slot, base); + }; + } + + for (auto cmd_offset = Pm4::CB_COLOR0_FMASK; cmd_offset <= Pm4::CB_COLOR7_FMASK; + cmd_offset += 15) { + g_hw_ctx_indirect_func[cmd_offset] = [](KYTY_HW_CTX_INDIRECT_ARGS) { + uint32_t slot = (cmd_offset - Pm4::CB_COLOR0_FMASK) / 15; + auto base = cp->GetCtx()->GetRenderTarget(slot).fmask; + base.addr &= 0xFFFFFF00000000FFull; + base.addr |= static_cast(value) << 8u; + cp->GetCtx()->SetColorFmask(slot, base); + }; + } + + for (auto cmd_offset = Pm4::CB_COLOR0_FMASK + 1; cmd_offset <= Pm4::CB_COLOR7_BASE + 10; + cmd_offset += 15) { + g_hw_ctx_indirect_func[cmd_offset] = [](KYTY_HW_CTX_INDIRECT_ARGS) { + uint32_t slot = (cmd_offset - (Pm4::CB_COLOR0_FMASK + 1)) / 15; + HW::ColorFmaskSlice fmask_slice; + fmask_slice.slice_minus1 = value & 0x3fffffu; + cp->GetCtx()->SetColorFmaskSlice(slot, fmask_slice); + }; + } + + for (auto cmd_offset = Pm4::CB_COLOR0_FMASK_BASE_EXT; + cmd_offset <= Pm4::CB_COLOR7_FMASK_BASE_EXT; cmd_offset++) { + g_hw_ctx_indirect_func[cmd_offset] = [](KYTY_HW_CTX_INDIRECT_ARGS) { + uint32_t slot = (cmd_offset - Pm4::CB_COLOR0_FMASK_BASE_EXT); + auto base = cp->GetCtx()->GetRenderTarget(slot).fmask; + base.addr &= 0xFFFF00FFFFFFFFFFull; + base.addr |= (static_cast(value) & 0xffu) << 40u; + cp->GetCtx()->SetColorFmask(slot, base); + }; + } + + for (auto cmd_offset = Pm4::CB_COLOR0_CLEAR_WORD0; cmd_offset <= Pm4::CB_COLOR7_CLEAR_WORD0; + cmd_offset += 15) { + g_hw_ctx_indirect_func[cmd_offset] = [](KYTY_HW_CTX_INDIRECT_ARGS) { + HW::ColorClearWord0 clear_word0; + clear_word0.word0 = value; + uint32_t slot = (cmd_offset - Pm4::CB_COLOR0_CLEAR_WORD0) / 15; + cp->GetCtx()->SetColorClearWord0(slot, clear_word0); + }; + } + + for (auto cmd_offset = Pm4::CB_COLOR0_CLEAR_WORD1; cmd_offset <= Pm4::CB_COLOR7_CLEAR_WORD1; + cmd_offset += 15) { + g_hw_ctx_indirect_func[cmd_offset] = [](KYTY_HW_CTX_INDIRECT_ARGS) { + HW::ColorClearWord1 clear_word1; + clear_word1.word1 = value; + uint32_t slot = (cmd_offset - Pm4::CB_COLOR0_CLEAR_WORD1) / 15; + cp->GetCtx()->SetColorClearWord1(slot, clear_word1); + }; + } + + for (auto cmd_offset = Pm4::CB_COLOR0_DCC_BASE; cmd_offset <= Pm4::CB_COLOR7_DCC_BASE; + cmd_offset += 15) { + g_hw_ctx_indirect_func[cmd_offset] = [](KYTY_HW_CTX_INDIRECT_ARGS) { + uint32_t slot = (cmd_offset - Pm4::CB_COLOR0_DCC_BASE) / 15; + auto base = cp->GetCtx()->GetRenderTarget(slot).dcc_addr; + base.addr &= 0xFFFFFF00000000FFull; + base.addr |= static_cast(value) << 8u; + cp->GetCtx()->SetColorDccAddr(slot, base); + }; + } + + for (auto cmd_offset = Pm4::CB_COLOR0_DCC_BASE_EXT; cmd_offset <= Pm4::CB_COLOR7_DCC_BASE_EXT; + cmd_offset++) { + g_hw_ctx_indirect_func[cmd_offset] = [](KYTY_HW_CTX_INDIRECT_ARGS) { + uint32_t slot = (cmd_offset - Pm4::CB_COLOR0_DCC_BASE_EXT); + auto base = cp->GetCtx()->GetRenderTarget(slot).dcc_addr; + base.addr &= 0xFFFF00FFFFFFFFFFull; + base.addr |= (static_cast(value) & 0xffu) << 40u; + cp->GetCtx()->SetColorDccAddr(slot, base); + }; + } + + for (auto cmd_offset = Pm4::CB_COLOR0_ATTRIB2; cmd_offset <= Pm4::CB_COLOR7_ATTRIB2; + cmd_offset++) { + g_hw_ctx_indirect_func[cmd_offset] = [](KYTY_HW_CTX_INDIRECT_ARGS) { + uint32_t slot = (cmd_offset - Pm4::CB_COLOR0_ATTRIB2); + HW::ColorAttrib2 attrib2; + attrib2.height = KYTY_PM4_GET(value, CB_COLOR0_ATTRIB2, MIP0_HEIGHT); + attrib2.width = KYTY_PM4_GET(value, CB_COLOR0_ATTRIB2, MIP0_WIDTH); + attrib2.num_mip_levels = KYTY_PM4_GET(value, CB_COLOR0_ATTRIB2, MAX_MIP); + cp->GetCtx()->SetColorAttrib2(slot, attrib2); + }; + } + + for (auto cmd_offset = Pm4::CB_COLOR0_ATTRIB3; cmd_offset <= Pm4::CB_COLOR7_ATTRIB3; + cmd_offset++) { + g_hw_ctx_indirect_func[cmd_offset] = [](KYTY_HW_CTX_INDIRECT_ARGS) { + uint32_t slot = (cmd_offset - Pm4::CB_COLOR0_ATTRIB3); + HW::ColorAttrib3 attrib3; + attrib3.depth = KYTY_PM4_GET(value, CB_COLOR0_ATTRIB3, MIP0_DEPTH); + attrib3.tile_mode = KYTY_PM4_GET(value, CB_COLOR0_ATTRIB3, COLOR_SW_MODE); + attrib3.dimension = KYTY_PM4_GET(value, CB_COLOR0_ATTRIB3, RESOURCE_TYPE); + attrib3.cmask_pipe_aligned = KYTY_PM4_GET(value, CB_COLOR0_ATTRIB3, CMASK_PIPE_ALIGNED); + attrib3.dcc_pipe_aligned = KYTY_PM4_GET(value, CB_COLOR0_ATTRIB3, DCC_PIPE_ALIGNED); + cp->GetCtx()->SetColorAttrib3(slot, attrib3); + }; + } + + for (auto cmd_offset = Pm4::PA_CL_VPORT_XSCALE; cmd_offset <= Pm4::PA_CL_VPORT_XSCALE_15; + cmd_offset += 6) { + g_hw_ctx_indirect_func[cmd_offset] = [](KYTY_HW_CTX_INDIRECT_ARGS) { + cp->GetCtx()->SetViewportXScale((cmd_offset - Pm4::PA_CL_VPORT_XSCALE) / 6, + *reinterpret_cast(&value)); + }; + } + + for (auto cmd_offset = Pm4::PA_CL_VPORT_XOFFSET; cmd_offset <= Pm4::PA_CL_VPORT_XOFFSET_15; + cmd_offset += 6) { + g_hw_ctx_indirect_func[cmd_offset] = [](KYTY_HW_CTX_INDIRECT_ARGS) { + cp->GetCtx()->SetViewportXOffset((cmd_offset - Pm4::PA_CL_VPORT_XOFFSET) / 6, + *reinterpret_cast(&value)); + }; + } + + for (auto cmd_offset = Pm4::PA_CL_VPORT_YSCALE; cmd_offset <= Pm4::PA_CL_VPORT_YSCALE_15; + cmd_offset += 6) { + g_hw_ctx_indirect_func[cmd_offset] = [](KYTY_HW_CTX_INDIRECT_ARGS) { + cp->GetCtx()->SetViewportYScale((cmd_offset - Pm4::PA_CL_VPORT_YSCALE) / 6, + *reinterpret_cast(&value)); + }; + } + + for (auto cmd_offset = Pm4::PA_CL_VPORT_YOFFSET; cmd_offset <= Pm4::PA_CL_VPORT_YOFFSET_15; + cmd_offset += 6) { + g_hw_ctx_indirect_func[cmd_offset] = [](KYTY_HW_CTX_INDIRECT_ARGS) { + cp->GetCtx()->SetViewportYOffset((cmd_offset - Pm4::PA_CL_VPORT_YOFFSET) / 6, + *reinterpret_cast(&value)); + }; + } + + for (auto cmd_offset = Pm4::PA_CL_VPORT_ZSCALE; cmd_offset <= Pm4::PA_CL_VPORT_ZSCALE_15; + cmd_offset += 6) { + g_hw_ctx_indirect_func[cmd_offset] = [](KYTY_HW_CTX_INDIRECT_ARGS) { + cp->GetCtx()->SetViewportZScale((cmd_offset - Pm4::PA_CL_VPORT_ZSCALE) / 6, + *reinterpret_cast(&value)); + }; + } + + for (auto cmd_offset = Pm4::PA_CL_VPORT_ZOFFSET; cmd_offset <= Pm4::PA_CL_VPORT_ZOFFSET_15; + cmd_offset += 6) { + g_hw_ctx_indirect_func[cmd_offset] = [](KYTY_HW_CTX_INDIRECT_ARGS) { + cp->GetCtx()->SetViewportZOffset((cmd_offset - Pm4::PA_CL_VPORT_ZOFFSET) / 6, + *reinterpret_cast(&value)); + }; + } + + for (auto cmd_offset = Pm4::PA_SC_VPORT_SCISSOR_0_TL; + cmd_offset <= Pm4::PA_SC_VPORT_SCISSOR_15_TL; cmd_offset += 2) { + g_hw_ctx_indirect_func[cmd_offset] = [](KYTY_HW_CTX_INDIRECT_ARGS) { + int left = static_cast( + static_cast(KYTY_PM4_GET(value, PA_SC_VPORT_SCISSOR_0_TL, TL_X))); + int top = static_cast( + static_cast(KYTY_PM4_GET(value, PA_SC_VPORT_SCISSOR_0_TL, TL_Y))); + bool window_offset_disable = + KYTY_PM4_GET(value, PA_SC_VPORT_SCISSOR_0_TL, WINDOW_OFFSET_DISABLE) != 0; + cp->GetCtx()->SetViewportScissorTL((cmd_offset - Pm4::PA_SC_VPORT_SCISSOR_0_TL) / 2, + left, top, !window_offset_disable); + }; + } + + for (auto cmd_offset = Pm4::PA_SC_VPORT_SCISSOR_0_BR; + cmd_offset <= Pm4::PA_SC_VPORT_SCISSOR_15_BR; cmd_offset += 2) { + g_hw_ctx_indirect_func[cmd_offset] = [](KYTY_HW_CTX_INDIRECT_ARGS) { + int right = static_cast( + static_cast(KYTY_PM4_GET(value, PA_SC_VPORT_SCISSOR_0_BR, BR_X))); + int bottom = static_cast( + static_cast(KYTY_PM4_GET(value, PA_SC_VPORT_SCISSOR_0_BR, BR_Y))); + cp->GetCtx()->SetViewportScissorBR((cmd_offset - Pm4::PA_SC_VPORT_SCISSOR_0_BR) / 2, + right, bottom); + }; + } + + for (auto cmd_offset = Pm4::PA_SC_VPORT_ZMIN_0; cmd_offset <= Pm4::PA_SC_VPORT_ZMIN_15; + cmd_offset += 2) { + g_hw_ctx_indirect_func[cmd_offset] = [](KYTY_HW_CTX_INDIRECT_ARGS) { + cp->GetCtx()->SetViewportZMin((cmd_offset - Pm4::PA_SC_VPORT_ZMIN_0) / 2, + *reinterpret_cast(&value)); + }; + } + + for (auto cmd_offset = Pm4::PA_SC_VPORT_ZMAX_0; cmd_offset <= Pm4::PA_SC_VPORT_ZMAX_15; + cmd_offset += 2) { + g_hw_ctx_indirect_func[cmd_offset] = [](KYTY_HW_CTX_INDIRECT_ARGS) { + cp->GetCtx()->SetViewportZMax((cmd_offset - Pm4::PA_SC_VPORT_ZMAX_0) / 2, + *reinterpret_cast(&value)); + }; + } + + g_hw_ctx_indirect_func[Pm4::PA_CL_GB_VERT_CLIP_ADJ] = [](KYTY_HW_CTX_INDIRECT_ARGS) { + auto vp = cp->GetCtx()->GetScreenViewport(); + cp->GetCtx()->SetGuardBands(vp.guard_band_horz_clip, + *reinterpret_cast(&value), + vp.guard_band_horz_discard, vp.guard_band_vert_discard); + }; + + g_hw_ctx_indirect_func[Pm4::PA_CL_GB_VERT_DISC_ADJ] = [](KYTY_HW_CTX_INDIRECT_ARGS) { + auto vp = cp->GetCtx()->GetScreenViewport(); + cp->GetCtx()->SetGuardBands(vp.guard_band_horz_clip, vp.guard_band_vert_clip, + vp.guard_band_horz_discard, + *reinterpret_cast(&value)); + }; + + g_hw_ctx_indirect_func[Pm4::PA_CL_GB_HORZ_CLIP_ADJ] = [](KYTY_HW_CTX_INDIRECT_ARGS) { + auto vp = cp->GetCtx()->GetScreenViewport(); + cp->GetCtx()->SetGuardBands(*reinterpret_cast(&value), + vp.guard_band_vert_clip, vp.guard_band_horz_discard, + vp.guard_band_vert_discard); + }; + + g_hw_ctx_indirect_func[Pm4::PA_CL_GB_HORZ_DISC_ADJ] = [](KYTY_HW_CTX_INDIRECT_ARGS) { + auto vp = cp->GetCtx()->GetScreenViewport(); + cp->GetCtx()->SetGuardBands(vp.guard_band_horz_clip, vp.guard_band_vert_clip, + *reinterpret_cast(&value), + vp.guard_band_vert_discard); + }; + + for (auto cmd_offset = Pm4::CB_BLEND_RED; cmd_offset <= Pm4::CB_BLEND_ALPHA; cmd_offset++) { + g_hw_ctx_indirect_func[cmd_offset] = [](KYTY_HW_CTX_INDIRECT_ARGS) { + auto color = cp->GetCtx()->GetBlendColor(); + auto f = *reinterpret_cast(&value); + switch (cmd_offset) { + case Pm4::CB_BLEND_RED: color.red = f; break; + case Pm4::CB_BLEND_GREEN: color.green = f; break; + case Pm4::CB_BLEND_BLUE: color.blue = f; break; + case Pm4::CB_BLEND_ALPHA: color.alpha = f; break; + default: break; + } + cp->GetCtx()->SetBlendColor(color); + }; + } + g_hw_ctx_indirect_func[Pm4::CB_DCC_CONTROL] = [](KYTY_HW_CTX_INDIRECT_ARGS) { + HwCtxIgnoreCbDccControl(value); + }; + g_hw_ctx_indirect_func[Pm4::DB_COUNT_CONTROL] = [](KYTY_HW_CTX_INDIRECT_ARGS) { + HwCtxIgnoreDepthMetadataRegister(cmd_offset, value); + }; + g_hw_ctx_indirect_func[Pm4::DB_RENDER_OVERRIDE] = [](KYTY_HW_CTX_INDIRECT_ARGS) { + HwCtxIgnoreDepthMetadataRegister(cmd_offset, value); + }; + g_hw_ctx_indirect_func[Pm4::DB_RENDER_OVERRIDE2] = [](KYTY_HW_CTX_INDIRECT_ARGS) { + HwCtxIgnoreDepthMetadataRegister(cmd_offset, value); + }; + g_hw_ctx_indirect_func[Pm4::DB_DFSM_CONTROL] = [](KYTY_HW_CTX_INDIRECT_ARGS) { + HwCtxIgnoreDepthMetadataRegister(cmd_offset, value); + }; + g_hw_ctx_indirect_func[Pm4::DB_RMI_L2_CACHE_CONTROL] = [](KYTY_HW_CTX_INDIRECT_ARGS) { + HwCtxIgnoreDepthMetadataRegister(cmd_offset, value); + }; + g_hw_ctx_indirect_func[Pm4::CB_RMI_GL2_CACHE_CONTROL] = [](KYTY_HW_CTX_INDIRECT_ARGS) { + HwCtxIgnoreDepthMetadataRegister(cmd_offset, value); + }; + g_hw_ctx_indirect_func[Pm4::TA_BC_BASE_ADDR] = [](KYTY_HW_CTX_INDIRECT_ARGS) { + HwCtxIgnoreBorderColorTableAddr(cmd_offset, value); + }; + g_hw_ctx_indirect_func[Pm4::TA_BC_BASE_ADDR_HI] = [](KYTY_HW_CTX_INDIRECT_ARGS) { + HwCtxIgnoreBorderColorTableAddr(cmd_offset, value); + }; + g_hw_ctx_indirect_func[Pm4::PA_SU_POINT_SIZE] = [](KYTY_HW_CTX_INDIRECT_ARGS) { + HwCtxIgnorePointState(cmd_offset, value); + }; + g_hw_ctx_indirect_func[Pm4::PA_SU_POINT_MINMAX] = [](KYTY_HW_CTX_INDIRECT_ARGS) { + HwCtxIgnorePointState(cmd_offset, value); + }; + + g_hw_ctx_indirect_func[Pm4::SPI_VS_OUT_CONFIG] = [](KYTY_HW_CTX_INDIRECT_ARGS) { + cp->GetCtx()->SetVsOutConfig(value); + }; + + g_hw_ctx_indirect_func[Pm4::SPI_SHADER_POS_FORMAT] = [](KYTY_HW_CTX_INDIRECT_ARGS) { + cp->GetCtx()->SetShaderPosFormat(value); + }; + g_hw_ctx_indirect_func[Pm4::SPI_SHADER_IDX_FORMAT] = [](KYTY_HW_CTX_INDIRECT_ARGS) { + cp->GetCtx()->SetShaderIdxFormat(value); + }; + g_hw_ctx_indirect_func[Pm4::SPI_TMPRING_SIZE] = [](KYTY_HW_CTX_INDIRECT_ARGS) { + HwCtxIgnoreSpiTmpringSize(value); + }; + g_hw_ctx_indirect_func[Pm4::PA_CL_VS_OUT_CNTL] = [](KYTY_HW_CTX_INDIRECT_ARGS) { + cp->GetCtx()->SetClVsOutCntl(value); + }; + g_hw_ctx_indirect_func[Pm4::GE_NGG_SUBGRP_CNTL] = [](KYTY_HW_CTX_INDIRECT_ARGS) { + cp->GetCtx()->SetNggSubgrpCntl(value); + }; + g_hw_ctx_indirect_func[Pm4::VGT_GS_INSTANCE_CNT] = [](KYTY_HW_CTX_INDIRECT_ARGS) { + cp->GetCtx()->SetGsInstanceCnt(value); + }; + g_hw_ctx_indirect_func[Pm4::VGT_GS_ONCHIP_CNTL] = [](KYTY_HW_CTX_INDIRECT_ARGS) { + cp->GetCtx()->SetGsOnchipCntl(value); + }; + g_hw_ctx_indirect_func[Pm4::VGT_HOS_MAX_TESS_LEVEL] = [](KYTY_HW_CTX_INDIRECT_ARGS) { + cp->GetCtx()->SetHosMaxTessLevel(value); + }; + g_hw_ctx_indirect_func[Pm4::VGT_HOS_MIN_TESS_LEVEL] = [](KYTY_HW_CTX_INDIRECT_ARGS) { + cp->GetCtx()->SetHosMinTessLevel(value); + }; + + g_hw_ctx_indirect_func[Pm4::GE_MAX_OUTPUT_PER_SUBGROUP] = [](KYTY_HW_CTX_INDIRECT_ARGS) { + cp->GetCtx()->SetMaxOutputPerSubgroup(value); + }; + + g_hw_ctx_indirect_func[Pm4::VGT_ESGS_RING_ITEMSIZE] = [](KYTY_HW_CTX_INDIRECT_ARGS) { + cp->GetCtx()->SetEsgsRingItemsize(value); + }; + g_hw_ctx_indirect_func[Pm4::VGT_GS_MAX_VERT_OUT] = [](KYTY_HW_CTX_INDIRECT_ARGS) { + cp->GetCtx()->SetGsMaxVertOut(value); + }; + g_hw_ctx_indirect_func[Pm4::VGT_PRIMITIVEID_EN] = [](KYTY_HW_CTX_INDIRECT_ARGS) { + cp->GetCtx()->SetPrimitiveIdEn(value); + }; + g_hw_ctx_indirect_func[Pm4::VGT_REUSE_OFF] = [](KYTY_HW_CTX_INDIRECT_ARGS) { + cp->GetCtx()->SetReuseOff(value); + }; + g_hw_ctx_indirect_func[Pm4::VGT_MULTI_PRIM_IB_RESET_INDX] = [](KYTY_HW_CTX_INDIRECT_ARGS) { + HwIgnoreMultiPrimIbReset(value); + }; + g_hw_ctx_indirect_func[Pm4::VGT_TESS_DISTRIBUTION] = [](KYTY_HW_CTX_INDIRECT_ARGS) { + cp->GetCtx()->SetTessDistribution(value); + }; + g_hw_ctx_indirect_func[Pm4::VGT_SHADER_STAGES_EN] = [](KYTY_HW_CTX_INDIRECT_ARGS) { + cp->GetCtx()->SetShaderStages(value); + }; + g_hw_ctx_indirect_func[Pm4::VGT_LS_HS_CONFIG] = [](KYTY_HW_CTX_INDIRECT_ARGS) { + cp->GetCtx()->SetLsHsConfig(value); + }; + g_hw_ctx_indirect_func[Pm4::VGT_GS_OUT_PRIM_TYPE] = [](KYTY_HW_CTX_INDIRECT_ARGS) { + cp->GetCtx()->SetGsOutPrimType(value); + }; + g_hw_ctx_indirect_func[Pm4::VGT_TF_PARAM] = [](KYTY_HW_CTX_INDIRECT_ARGS) { + cp->GetCtx()->SetTfParam(value); + }; + g_hw_ctx_indirect_func[Pm4::VGT_DRAW_PAYLOAD_CNTL] = [](KYTY_HW_CTX_INDIRECT_ARGS) { + HwCtxIgnoreDrawPayloadControl(value); + }; + g_hw_ctx_indirect_func[Pm4::VGT_PRIMITIVEID_RESET] = [](KYTY_HW_CTX_INDIRECT_ARGS) { + HwCtxIgnorePrimitiveIdReset(value); + }; + g_hw_ctx_indirect_func[Pm4::SPI_SHADER_Z_FORMAT] = [](KYTY_HW_CTX_INDIRECT_ARGS) { + cp->GetCtx()->SetShaderZFormat(value); + }; + g_hw_ctx_indirect_func[Pm4::PA_CL_OBJPRIM_ID_CNTL] = [](KYTY_HW_CTX_INDIRECT_ARGS) { + HwCtxIgnoreObjprimIdControl(value); + }; + + for (auto cmd_offset = Pm4::PA_SC_CLIPRECT_RULE; cmd_offset < Pm4::PA_SC_CLIPRECT_0_TL + 8u; + cmd_offset++) { + g_hw_ctx_indirect_func[cmd_offset] = [](KYTY_HW_CTX_INDIRECT_ARGS) { + HwCtxSetClipRectRegister(cp->GetCtx(), cmd_offset, value); + }; + } + + g_hw_ctx_indirect_func[Pm4::SPI_SHADER_COL_FORMAT] = [](KYTY_HW_CTX_INDIRECT_ARGS) { + for (uint32_t i = 0; i < 8; i++) { + cp->GetCtx()->SetTargetOutputMode(i, (value >> (i * 4)) & 0xFu); + } + }; + + g_hw_ctx_indirect_func[Pm4::SPI_PS_INPUT_ENA] = [](KYTY_HW_CTX_INDIRECT_ARGS) { + cp->GetCtx()->SetPsInputEna(value); + }; + g_hw_ctx_indirect_func[Pm4::SPI_PS_INPUT_ADDR] = [](KYTY_HW_CTX_INDIRECT_ARGS) { + cp->GetCtx()->SetPsInputAddr(value); + }; + g_hw_ctx_indirect_func[Pm4::SPI_PS_IN_CONTROL] = [](KYTY_HW_CTX_INDIRECT_ARGS) { + cp->GetCtx()->SetPsInControl(value); + }; + g_hw_ctx_indirect_func[Pm4::SPI_BARYC_CNTL] = [](KYTY_HW_CTX_INDIRECT_ARGS) { + cp->GetCtx()->SetBarycCntl(value); + }; + + g_hw_ctx_indirect_func[Pm4::DB_SHADER_CONTROL] = [](KYTY_HW_CTX_INDIRECT_ARGS) { + HW::DepthShaderControl db_shader_control {}; + db_shader_control.other_bits = value & 0xFFFF908Eu; + db_shader_control.conservative_z_export_value = + KYTY_PM4_GET(value, DB_SHADER_CONTROL, CONSERVATIVE_Z_EXPORT); + db_shader_control.shader_z_behavior = KYTY_PM4_GET(value, DB_SHADER_CONTROL, Z_ORDER); + db_shader_control.shader_kill_enable = + KYTY_PM4_GET(value, DB_SHADER_CONTROL, KILL_ENABLE) != 0; + db_shader_control.shader_z_export_enable = + KYTY_PM4_GET(value, DB_SHADER_CONTROL, Z_EXPORT_ENABLE) != 0; + db_shader_control.shader_mask_export_enable = + KYTY_PM4_GET(value, DB_SHADER_CONTROL, MASK_EXPORT_ENABLE) != 0; + db_shader_control.shader_dual_export_enable = + KYTY_PM4_GET(value, DB_SHADER_CONTROL, DUAL_EXPORT_ENABLE) != 0; + db_shader_control.shader_execute_on_noop = + KYTY_PM4_GET(value, DB_SHADER_CONTROL, EXEC_ON_NOOP) != 0; + db_shader_control.alpha_to_mask_disable = + KYTY_PM4_GET(value, DB_SHADER_CONTROL, ALPHA_TO_MASK_DISABLE) != 0; + cp->GetCtx()->SetDepthShaderControl(db_shader_control); + }; + + g_hw_ctx_indirect_func[Pm4::CB_SHADER_MASK] = [](KYTY_HW_CTX_INDIRECT_ARGS) { + cp->GetCtx()->SetShaderMask(value); + }; + g_hw_ctx_indirect_func[Pm4::PA_SC_SHADER_CONTROL] = [](KYTY_HW_CTX_INDIRECT_ARGS) { + cp->GetCtx()->SetScShaderControl(value); + }; + g_hw_ctx_indirect_func[Pm4::CB_TARGET_MASK] = [](KYTY_HW_CTX_INDIRECT_ARGS) { + cp->GetCtx()->SetRenderTargetMask(value); + }; + + g_hw_ctx_indirect_func[Pm4::PA_SC_SCREEN_SCISSOR_TL] = [](KYTY_HW_CTX_INDIRECT_ARGS) { + const auto& vp = cp->GetCtx()->GetScreenViewport(); + int left = static_cast( + static_cast(KYTY_PM4_GET(value, PA_SC_SCREEN_SCISSOR_TL, TL_X))); + int top = static_cast( + static_cast(KYTY_PM4_GET(value, PA_SC_SCREEN_SCISSOR_TL, TL_Y))); + cp->GetCtx()->SetScreenScissor(left, top, vp.screen_scissor_right, + vp.screen_scissor_bottom); + }; + + g_hw_ctx_indirect_func[Pm4::PA_SC_SCREEN_SCISSOR_BR] = [](KYTY_HW_CTX_INDIRECT_ARGS) { + const auto& vp = cp->GetCtx()->GetScreenViewport(); + int right = static_cast( + static_cast(KYTY_PM4_GET(value, PA_SC_SCREEN_SCISSOR_BR, BR_X))); + int bottom = static_cast( + static_cast(KYTY_PM4_GET(value, PA_SC_SCREEN_SCISSOR_BR, BR_Y))); + cp->GetCtx()->SetScreenScissor(vp.screen_scissor_left, vp.screen_scissor_top, right, + bottom); + }; + + g_hw_ctx_indirect_func[Pm4::PA_SC_GENERIC_SCISSOR_TL] = [](KYTY_HW_CTX_INDIRECT_ARGS) { + const auto& vp = cp->GetCtx()->GetScreenViewport(); + int left = static_cast( + static_cast(KYTY_PM4_GET(value, PA_SC_GENERIC_SCISSOR_TL, TL_X))); + int top = static_cast( + static_cast(KYTY_PM4_GET(value, PA_SC_GENERIC_SCISSOR_TL, TL_Y))); + bool window_offset_disable = + KYTY_PM4_GET(value, PA_SC_GENERIC_SCISSOR_TL, WINDOW_OFFSET_DISABLE) != 0; + cp->GetCtx()->SetGenericScissor(left, top, vp.generic_scissor_right, + vp.generic_scissor_bottom, !window_offset_disable); + }; + + g_hw_ctx_indirect_func[Pm4::PA_SC_GENERIC_SCISSOR_BR] = [](KYTY_HW_CTX_INDIRECT_ARGS) { + const auto& vp = cp->GetCtx()->GetScreenViewport(); + int right = static_cast( + static_cast(KYTY_PM4_GET(value, PA_SC_GENERIC_SCISSOR_BR, BR_X))); + int bottom = static_cast( + static_cast(KYTY_PM4_GET(value, PA_SC_GENERIC_SCISSOR_BR, BR_Y))); + cp->GetCtx()->SetGenericScissor(vp.generic_scissor_left, vp.generic_scissor_top, right, + bottom, vp.generic_scissor_window_offset_enable); + }; + + g_hw_ctx_indirect_func[Pm4::PA_SU_HARDWARE_SCREEN_OFFSET] = [](KYTY_HW_CTX_INDIRECT_ARGS) { + uint32_t x = KYTY_PM4_GET(value, PA_SU_HARDWARE_SCREEN_OFFSET, HW_SCREEN_OFFSET_X); + uint32_t y = KYTY_PM4_GET(value, PA_SU_HARDWARE_SCREEN_OFFSET, HW_SCREEN_OFFSET_Y); + cp->GetCtx()->SetHardwareScreenOffset(x, y); + }; + + g_hw_ctx_indirect_func[Pm4::PA_SC_WINDOW_OFFSET] = [](KYTY_HW_CTX_INDIRECT_ARGS) { + int offset_x = static_cast( + static_cast(KYTY_PM4_GET(value, PA_SC_WINDOW_OFFSET, WINDOW_X))); + int offset_y = static_cast( + static_cast(KYTY_PM4_GET(value, PA_SC_WINDOW_OFFSET, WINDOW_Y))); + + cp->GetCtx()->SetWindowOffset(offset_x, offset_y); + }; + + g_hw_ctx_indirect_func[Pm4::PA_SC_WINDOW_SCISSOR_TL] = [](KYTY_HW_CTX_INDIRECT_ARGS) { + const auto& vp = cp->GetCtx()->GetScreenViewport(); + int left = static_cast( + static_cast(KYTY_PM4_GET(value, PA_SC_WINDOW_SCISSOR_TL, TL_X))); + int top = static_cast( + static_cast(KYTY_PM4_GET(value, PA_SC_WINDOW_SCISSOR_TL, TL_Y))); + bool window_offset_disable = + KYTY_PM4_GET(value, PA_SC_WINDOW_SCISSOR_TL, WINDOW_OFFSET_DISABLE) != 0; + cp->GetCtx()->SetWindowScissor(left, top, vp.window_scissor_right, vp.window_scissor_bottom, + !window_offset_disable); + }; + + g_hw_ctx_indirect_func[Pm4::PA_SC_WINDOW_SCISSOR_BR] = [](KYTY_HW_CTX_INDIRECT_ARGS) { + const auto& vp = cp->GetCtx()->GetScreenViewport(); + int right = static_cast( + static_cast(KYTY_PM4_GET(value, PA_SC_WINDOW_SCISSOR_BR, BR_X))); + int bottom = static_cast( + static_cast(KYTY_PM4_GET(value, PA_SC_WINDOW_SCISSOR_BR, BR_Y))); + cp->GetCtx()->SetWindowScissor(vp.window_scissor_left, vp.window_scissor_top, right, bottom, + vp.window_scissor_window_offset_enable); + }; + + for (uint32_t slot = 0; slot < 8; slot++) { + g_hw_ctx_indirect_func[Pm4::CB_BLEND0_CONTROL + slot] = [](KYTY_HW_CTX_INDIRECT_ARGS) { + uint32_t param = cmd_offset - Pm4::CB_BLEND0_CONTROL; + + HW::BlendControl r; + r.color_srcblend = KYTY_PM4_GET(value, CB_BLEND0_CONTROL, COLOR_SRCBLEND); + r.color_comb_fcn = KYTY_PM4_GET(value, CB_BLEND0_CONTROL, COLOR_COMB_FCN); + r.color_destblend = KYTY_PM4_GET(value, CB_BLEND0_CONTROL, COLOR_DESTBLEND); + r.alpha_srcblend = KYTY_PM4_GET(value, CB_BLEND0_CONTROL, ALPHA_SRCBLEND); + r.alpha_comb_fcn = KYTY_PM4_GET(value, CB_BLEND0_CONTROL, ALPHA_COMB_FCN); + r.alpha_destblend = KYTY_PM4_GET(value, CB_BLEND0_CONTROL, ALPHA_DESTBLEND); + r.separate_alpha_blend = + KYTY_PM4_GET(value, CB_BLEND0_CONTROL, SEPARATE_ALPHA_BLEND) != 0; + r.enable = KYTY_PM4_GET(value, CB_BLEND0_CONTROL, ENABLE) != 0; + + cp->GetCtx()->SetBlendControl(param, r); + }; + } + + g_hw_ctx_indirect_func[Pm4::PA_SU_SC_MODE_CNTL] = [](KYTY_HW_CTX_INDIRECT_ARGS) { + HW::ModeControl r; + r.cull_front = KYTY_PM4_GET(value, PA_SU_SC_MODE_CNTL, CULL_FRONT) != 0; + r.cull_back = KYTY_PM4_GET(value, PA_SU_SC_MODE_CNTL, CULL_BACK) != 0; + r.face = KYTY_PM4_GET(value, PA_SU_SC_MODE_CNTL, FACE) != 0; + r.poly_mode = KYTY_PM4_GET(value, PA_SU_SC_MODE_CNTL, POLY_MODE); + r.polymode_front_ptype = KYTY_PM4_GET(value, PA_SU_SC_MODE_CNTL, POLYMODE_FRONT_PTYPE); + r.polymode_back_ptype = KYTY_PM4_GET(value, PA_SU_SC_MODE_CNTL, POLYMODE_BACK_PTYPE); + r.poly_offset_front_enable = + KYTY_PM4_GET(value, PA_SU_SC_MODE_CNTL, POLY_OFFSET_FRONT_ENABLE) != 0; + r.poly_offset_back_enable = + KYTY_PM4_GET(value, PA_SU_SC_MODE_CNTL, POLY_OFFSET_BACK_ENABLE) != 0; + r.vtx_window_offset_enable = + KYTY_PM4_GET(value, PA_SU_SC_MODE_CNTL, VTX_WINDOW_OFFSET_ENABLE) != 0; + r.provoking_vtx_last = KYTY_PM4_GET(value, PA_SU_SC_MODE_CNTL, PROVOKING_VTX_LAST) != 0; + r.persp_corr_dis = KYTY_PM4_GET(value, PA_SU_SC_MODE_CNTL, PERSP_CORR_DIS) != 0; + cp->GetCtx()->SetModeControl(r); + }; + + g_hw_ctx_indirect_func[Pm4::PA_SU_POLY_OFFSET_DB_FMT_CNTL] = [](KYTY_HW_CTX_INDIRECT_ARGS) { + HwCtxIgnorePolyOffsetRegister(cmd_offset, value); + }; + g_hw_ctx_indirect_func[Pm4::PA_SU_POLY_OFFSET_CLAMP] = [](KYTY_HW_CTX_INDIRECT_ARGS) { + HwCtxIgnorePolyOffsetRegister(cmd_offset, value); + }; + g_hw_ctx_indirect_func[Pm4::PA_SU_POLY_OFFSET_FRONT_SCALE] = [](KYTY_HW_CTX_INDIRECT_ARGS) { + HwCtxIgnorePolyOffsetRegister(cmd_offset, value); + }; + g_hw_ctx_indirect_func[Pm4::PA_SU_POLY_OFFSET_FRONT_OFFSET] = [](KYTY_HW_CTX_INDIRECT_ARGS) { + HwCtxIgnorePolyOffsetRegister(cmd_offset, value); + }; + g_hw_ctx_indirect_func[Pm4::PA_SU_POLY_OFFSET_BACK_SCALE] = [](KYTY_HW_CTX_INDIRECT_ARGS) { + HwCtxIgnorePolyOffsetRegister(cmd_offset, value); + }; + g_hw_ctx_indirect_func[Pm4::PA_SU_POLY_OFFSET_BACK_OFFSET] = [](KYTY_HW_CTX_INDIRECT_ARGS) { + HwCtxIgnorePolyOffsetRegister(cmd_offset, value); + }; + + g_hw_ctx_indirect_func[Pm4::DB_Z_INFO] = [](KYTY_HW_CTX_INDIRECT_ARGS) { + HW::DepthZInfo r; + r.format = KYTY_PM4_GET(value, DB_Z_INFO, FORMAT); + r.num_samples = KYTY_PM4_GET(value, DB_Z_INFO, NUM_SAMPLES); + r.embedded_sample_locations = KYTY_PM4_GET(value, DB_Z_INFO, ITERATE_FLUSH) != 0; + r.partially_resident = KYTY_PM4_GET(value, DB_Z_INFO, PARTIALLY_RESIDENT) != 0; + r.num_mip_levels = KYTY_PM4_GET(value, DB_Z_INFO, MAXMIP); + r.tile_mode_index = KYTY_PM4_GET(value, DB_Z_INFO, TILE_MODE_INDEX); + r.plane_compression = KYTY_PM4_GET(value, DB_Z_INFO, DECOMPRESS_ON_N_ZPLANES); + r.expclear_enabled = KYTY_PM4_GET(value, DB_Z_INFO, ALLOW_EXPCLEAR) != 0; + r.tile_surface_enable = KYTY_PM4_GET(value, DB_Z_INFO, TILE_SURFACE_ENABLE) != 0; + r.zrange_precision = KYTY_PM4_GET(value, DB_Z_INFO, ZRANGE_PRECISION); + cp->GetCtx()->SetDepthZInfo(r); + }; + + g_hw_ctx_indirect_func[Pm4::DB_DEPTH_INFO] = [](KYTY_HW_CTX_INDIRECT_ARGS) { + auto z = cp->GetCtx()->GetDepthRenderTarget(); + z.depth_info.addr5_swizzle_mask = KYTY_PM4_GET(value, DB_DEPTH_INFO, ADDR5_SWIZZLE_MASK); + z.depth_info.array_mode = + (value >> Pm4::DB_DEPTH_INFO_ARRAY_MODE_SHIFT) & Pm4::DB_DEPTH_INFO_ARRAY_MODE_MASK; + z.depth_info.pipe_config = + (value >> Pm4::DB_DEPTH_INFO_PIPE_CONFIG_SHIFT) & Pm4::DB_DEPTH_INFO_PIPE_CONFIG_MASK; + z.depth_info.bank_width = + (value >> Pm4::DB_DEPTH_INFO_BANK_WIDTH_SHIFT) & Pm4::DB_DEPTH_INFO_BANK_WIDTH_MASK; + z.depth_info.bank_height = + (value >> Pm4::DB_DEPTH_INFO_BANK_HEIGHT_SHIFT) & Pm4::DB_DEPTH_INFO_BANK_HEIGHT_MASK; + z.depth_info.macro_tile_aspect = KYTY_PM4_GET(value, DB_DEPTH_INFO, MACRO_TILE_ASPECT); + z.depth_info.num_banks = + (value >> Pm4::DB_DEPTH_INFO_NUM_BANKS_SHIFT) & Pm4::DB_DEPTH_INFO_NUM_BANKS_MASK; + cp->GetCtx()->SetDepthRenderTarget(z); + }; + + g_hw_ctx_indirect_func[Pm4::DB_DEPTH_BOUNDS_MIN] = [](KYTY_HW_CTX_INDIRECT_ARGS) { + HwCtxSetDepthBoundsRegister(cp->GetCtx(), cmd_offset, value); + }; + g_hw_ctx_indirect_func[Pm4::DB_DEPTH_BOUNDS_MAX] = [](KYTY_HW_CTX_INDIRECT_ARGS) { + HwCtxSetDepthBoundsRegister(cp->GetCtx(), cmd_offset, value); + }; + + g_hw_ctx_indirect_func[Pm4::DB_STENCIL_INFO] = [](KYTY_HW_CTX_INDIRECT_ARGS) { + HW::DepthStencilInfo r; + r.format = KYTY_PM4_GET(value, DB_STENCIL_INFO, FORMAT); + r.texture_compatible_stencil = KYTY_PM4_GET(value, DB_STENCIL_INFO, ITERATE_FLUSH) != 0; + r.partially_resident = KYTY_PM4_GET(value, DB_STENCIL_INFO, PARTIALLY_RESIDENT) != 0; + r.tile_split = KYTY_PM4_GET(value, DB_STENCIL_INFO, RESERVED_FIELD_1); + r.tile_mode_index = KYTY_PM4_GET(value, DB_STENCIL_INFO, TILE_MODE_INDEX); + r.expclear_enabled = KYTY_PM4_GET(value, DB_STENCIL_INFO, ALLOW_EXPCLEAR) != 0; + r.tile_stencil_disable = KYTY_PM4_GET(value, DB_STENCIL_INFO, TILE_STENCIL_DISABLE) != 0; + cp->GetCtx()->SetDepthStencilInfo(r); + }; + + g_hw_ctx_indirect_func[Pm4::DB_Z_READ_BASE] = [](KYTY_HW_CTX_INDIRECT_ARGS) { + auto base = cp->GetCtx()->GetDepthRenderTarget().z_read_base_addr; + base &= 0xFFFFFF00000000FFull; + base |= static_cast(value) << 8u; + cp->GetCtx()->SetDepthZReadBase(base); + }; + + g_hw_ctx_indirect_func[Pm4::DB_Z_READ_BASE_HI] = [](KYTY_HW_CTX_INDIRECT_ARGS) { + auto base = cp->GetCtx()->GetDepthRenderTarget().z_read_base_addr; + base &= 0xFFFF00FFFFFFFFFFull; + base |= (static_cast(value) & 0xffu) << 40u; + cp->GetCtx()->SetDepthZReadBase(base); + }; + + g_hw_ctx_indirect_func[Pm4::DB_STENCIL_READ_BASE] = [](KYTY_HW_CTX_INDIRECT_ARGS) { + auto base = cp->GetCtx()->GetDepthRenderTarget().stencil_read_base_addr; + base &= 0xFFFFFF00000000FFull; + base |= static_cast(value) << 8u; + cp->GetCtx()->SetDepthStencilReadBase(base); + }; + + g_hw_ctx_indirect_func[Pm4::DB_STENCIL_READ_BASE_HI] = [](KYTY_HW_CTX_INDIRECT_ARGS) { + auto base = cp->GetCtx()->GetDepthRenderTarget().stencil_read_base_addr; + base &= 0xFFFF00FFFFFFFFFFull; + base |= (static_cast(value) & 0xffu) << 40u; + cp->GetCtx()->SetDepthStencilReadBase(base); + }; + + g_hw_ctx_indirect_func[Pm4::DB_Z_WRITE_BASE] = [](KYTY_HW_CTX_INDIRECT_ARGS) { + auto base = cp->GetCtx()->GetDepthRenderTarget().z_write_base_addr; + base &= 0xFFFFFF00000000FFull; + base |= static_cast(value) << 8u; + cp->GetCtx()->SetDepthZWriteBase(base); + }; + + g_hw_ctx_indirect_func[Pm4::DB_Z_WRITE_BASE_HI] = [](KYTY_HW_CTX_INDIRECT_ARGS) { + auto base = cp->GetCtx()->GetDepthRenderTarget().z_write_base_addr; + base &= 0xFFFF00FFFFFFFFFFull; + base |= (static_cast(value) & 0xffu) << 40u; + cp->GetCtx()->SetDepthZWriteBase(base); + }; + + g_hw_ctx_indirect_func[Pm4::DB_STENCIL_WRITE_BASE] = [](KYTY_HW_CTX_INDIRECT_ARGS) { + auto base = cp->GetCtx()->GetDepthRenderTarget().stencil_write_base_addr; + base &= 0xFFFFFF00000000FFull; + base |= static_cast(value) << 8u; + cp->GetCtx()->SetDepthStencilWriteBase(base); + }; + + g_hw_ctx_indirect_func[Pm4::DB_STENCIL_WRITE_BASE_HI] = [](KYTY_HW_CTX_INDIRECT_ARGS) { + auto base = cp->GetCtx()->GetDepthRenderTarget().stencil_write_base_addr; + base &= 0xFFFF00FFFFFFFFFFull; + base |= (static_cast(value) & 0xffu) << 40u; + cp->GetCtx()->SetDepthStencilWriteBase(base); + }; + + g_hw_ctx_indirect_func[Pm4::DB_HTILE_DATA_BASE] = [](KYTY_HW_CTX_INDIRECT_ARGS) { + auto base = cp->GetCtx()->GetDepthRenderTarget().htile_data_base_addr; + base &= 0xFFFFFF00000000FFull; + base |= static_cast(value) << 8u; + cp->GetCtx()->SetDepthHTileDataBase(base); + }; + + g_hw_ctx_indirect_func[Pm4::DB_HTILE_DATA_BASE_HI] = [](KYTY_HW_CTX_INDIRECT_ARGS) { + auto base = cp->GetCtx()->GetDepthRenderTarget().htile_data_base_addr; + base &= 0xFFFF00FFFFFFFFFFull; + base |= (static_cast(value) & 0xffu) << 40u; + cp->GetCtx()->SetDepthHTileDataBase(base); + }; + + g_hw_ctx_indirect_func[Pm4::DB_HTILE_SURFACE] = [](KYTY_HW_CTX_INDIRECT_ARGS) { + HwCtxSetDepthHtileSurfaceRegister(cp->GetCtx(), value); + }; + + g_hw_ctx_indirect_func[Pm4::DB_DEPTH_VIEW] = [](KYTY_HW_CTX_INDIRECT_ARGS) { + HW::DepthDepthView r; + r.slice_start = KYTY_PM4_GET(value, DB_DEPTH_VIEW, SLICE_START) + + (KYTY_PM4_GET(value, DB_DEPTH_VIEW, SLICE_START_HI) << 11u); + r.slice_max = KYTY_PM4_GET(value, DB_DEPTH_VIEW, SLICE_MAX) + + (KYTY_PM4_GET(value, DB_DEPTH_VIEW, SLICE_MAX_HI) << 11u); + r.depth_write_disable = KYTY_PM4_GET(value, DB_DEPTH_VIEW, Z_READ_ONLY) != 0; + r.stencil_write_disable = KYTY_PM4_GET(value, DB_DEPTH_VIEW, STENCIL_READ_ONLY) != 0; + r.current_mip_level = KYTY_PM4_GET(value, DB_DEPTH_VIEW, MIPID); + cp->GetCtx()->SetDepthDepthView(r); + }; + + g_hw_ctx_indirect_func[Pm4::DB_DEPTH_SIZE_XY] = [](KYTY_HW_CTX_INDIRECT_ARGS) { + HW::DepthDepthSizeXY r; + r.x_max = KYTY_PM4_GET(value, DB_DEPTH_SIZE_XY, X_MAX); + r.y_max = KYTY_PM4_GET(value, DB_DEPTH_SIZE_XY, Y_MAX); + r.valid = (r.x_max != 0 || r.y_max != 0); + cp->GetCtx()->SetDepthDepthSizeXY(r); + }; + g_hw_ctx_indirect_func[Pm4::DB_DEPTH_SIZE] = [](KYTY_HW_CTX_INDIRECT_ARGS) { + auto target = cp->GetCtx()->GetDepthRenderTarget(); + target.pitch_div8_minus1 = KYTY_PM4_GET(value, DB_DEPTH_SIZE, PITCH_TILE_MAX); + target.height_div8_minus1 = KYTY_PM4_GET(value, DB_DEPTH_SIZE, HEIGHT_TILE_MAX); + target.pitch_height_valid = true; + cp->GetCtx()->SetDepthRenderTarget(target); + }; + g_hw_ctx_indirect_func[Pm4::DB_DEPTH_SLICE] = [](KYTY_HW_CTX_INDIRECT_ARGS) { + auto target = cp->GetCtx()->GetDepthRenderTarget(); + target.slice_div64_minus1 = KYTY_PM4_GET(value, DB_DEPTH_SLICE, SLICE_TILE_MAX); + cp->GetCtx()->SetDepthRenderTarget(target); + }; + + g_hw_ctx_indirect_func[Pm4::DB_DEPTH_CLEAR] = [](KYTY_HW_CTX_INDIRECT_ARGS) { + cp->GetCtx()->SetDepthClearValue(*reinterpret_cast(&value)); + }; + g_hw_ctx_indirect_func[Pm4::DB_STENCIL_CLEAR] = [](KYTY_HW_CTX_INDIRECT_ARGS) { + cp->GetCtx()->SetStencilClearValue(KYTY_PM4_GET(value, DB_STENCIL_CLEAR, CLEAR)); + }; + + g_hw_ctx_indirect_func[Pm4::DB_STENCIL_CONTROL] = [](KYTY_HW_CTX_INDIRECT_ARGS) { + HW::StencilControl r; + r.stencil_fail = KYTY_PM4_GET(value, DB_STENCIL_CONTROL, STENCILFAIL); + r.stencil_zpass = KYTY_PM4_GET(value, DB_STENCIL_CONTROL, STENCILZPASS); + r.stencil_zfail = KYTY_PM4_GET(value, DB_STENCIL_CONTROL, STENCILZFAIL); + r.stencil_fail_bf = KYTY_PM4_GET(value, DB_STENCIL_CONTROL, STENCILFAIL_BF); + r.stencil_zpass_bf = KYTY_PM4_GET(value, DB_STENCIL_CONTROL, STENCILZPASS_BF); + r.stencil_zfail_bf = KYTY_PM4_GET(value, DB_STENCIL_CONTROL, STENCILZFAIL_BF); + cp->GetCtx()->SetStencilControl(r); + }; + + g_hw_ctx_indirect_func[Pm4::DB_RENDER_CONTROL] = [](KYTY_HW_CTX_INDIRECT_ARGS) { + HW::RenderControl r; + r.depth_clear_enable = KYTY_PM4_GET(value, DB_RENDER_CONTROL, DEPTH_CLEAR_ENABLE) != 0; + r.stencil_clear_enable = KYTY_PM4_GET(value, DB_RENDER_CONTROL, STENCIL_CLEAR_ENABLE) != 0; + r.resummarize_enable = KYTY_PM4_GET(value, DB_RENDER_CONTROL, RESUMMARIZE_ENABLE) != 0; + r.stencil_compress_disable = + KYTY_PM4_GET(value, DB_RENDER_CONTROL, STENCIL_COMPRESS_DISABLE) != 0; + r.depth_compress_disable = + KYTY_PM4_GET(value, DB_RENDER_CONTROL, DEPTH_COMPRESS_DISABLE) != 0; + r.copy_centroid = KYTY_PM4_GET(value, DB_RENDER_CONTROL, COPY_CENTROID) != 0; + r.copy_sample = KYTY_PM4_GET(value, DB_RENDER_CONTROL, COPY_SAMPLE); + cp->GetCtx()->SetRenderControl(r); + }; + + g_hw_ctx_indirect_func[Pm4::DB_STENCILREFMASK] = [](KYTY_HW_CTX_INDIRECT_ARGS) { + auto r = cp->GetCtx()->GetStencilMask(); + r.stencil_testval = KYTY_PM4_GET(value, DB_STENCILREFMASK, STENCILTESTVAL); + r.stencil_mask = KYTY_PM4_GET(value, DB_STENCILREFMASK, STENCILMASK); + r.stencil_writemask = KYTY_PM4_GET(value, DB_STENCILREFMASK, STENCILWRITEMASK); + r.stencil_opval = KYTY_PM4_GET(value, DB_STENCILREFMASK, STENCILOPVAL); + cp->GetCtx()->SetStencilMask(r); + }; + + g_hw_ctx_indirect_func[Pm4::DB_STENCILREFMASK_BF] = [](KYTY_HW_CTX_INDIRECT_ARGS) { + auto r = cp->GetCtx()->GetStencilMask(); + r.stencil_testval_bf = KYTY_PM4_GET(value, DB_STENCILREFMASK_BF, STENCILTESTVAL_BF); + r.stencil_mask_bf = KYTY_PM4_GET(value, DB_STENCILREFMASK_BF, STENCILMASK_BF); + r.stencil_writemask_bf = KYTY_PM4_GET(value, DB_STENCILREFMASK_BF, STENCILWRITEMASK_BF); + r.stencil_opval_bf = KYTY_PM4_GET(value, DB_STENCILREFMASK_BF, STENCILOPVAL_BF); + cp->GetCtx()->SetStencilMask(r); + }; + + g_hw_ctx_indirect_func[Pm4::PA_CL_CLIP_CNTL] = [](KYTY_HW_CTX_INDIRECT_ARGS) { + HW::ClipControl r; + r.user_clip_planes = KYTY_PM4_GET(value, PA_CL_CLIP_CNTL, UCP_ENA); + r.user_clip_plane_mode = KYTY_PM4_GET(value, PA_CL_CLIP_CNTL, PS_UCP_MODE); + r.dx_clip_space = KYTY_PM4_GET(value, PA_CL_CLIP_CNTL, DX_CLIP_SPACE_DEF) != 0; + r.vertex_kill_any = KYTY_PM4_GET(value, PA_CL_CLIP_CNTL, VTX_KILL_OR) != 0; + r.min_z_clip_disable = KYTY_PM4_GET(value, PA_CL_CLIP_CNTL, ZCLIP_NEAR_DISABLE) != 0; + r.max_z_clip_disable = KYTY_PM4_GET(value, PA_CL_CLIP_CNTL, ZCLIP_FAR_DISABLE) != 0; + r.user_clip_plane_negate_y = KYTY_PM4_GET(value, PA_CL_CLIP_CNTL, PS_UCP_Y_SCALE_NEG) != 0; + r.clip_disable = KYTY_PM4_GET(value, PA_CL_CLIP_CNTL, CLIP_DISABLE) != 0; + r.user_clip_plane_cull_only = KYTY_PM4_GET(value, PA_CL_CLIP_CNTL, UCP_CULL_ONLY_ENA) != 0; + r.cull_on_clipping_error_disable = + KYTY_PM4_GET(value, PA_CL_CLIP_CNTL, DIS_CLIP_ERR_DETECT) != 0; + r.linear_attribute_clip_enable = + KYTY_PM4_GET(value, PA_CL_CLIP_CNTL, DX_LINEAR_ATTR_CLIP_ENA) != 0; + r.force_viewport_index_from_vs_enable = + KYTY_PM4_GET(value, PA_CL_CLIP_CNTL, VTE_VPORT_PROVOKE_DISABLE) != 0; + cp->GetCtx()->SetClipControl(r); + }; + + g_hw_ctx_indirect_func[Pm4::PA_SU_LINE_CNTL] = [](KYTY_HW_CTX_INDIRECT_ARGS) { + auto line_width = KYTY_PM4_GET(value, PA_SU_LINE_CNTL, WIDTH); + cp->GetCtx()->SetLineWidth(line_width == 8 ? 1.0f : static_cast(line_width) / 8.0f); + }; + g_hw_ctx_indirect_func[Pm4::PA_SC_FOV_WINDOW_LR] = [](KYTY_HW_CTX_INDIRECT_ARGS) { + HwCtxIgnoreFovWindow(value); + }; + g_hw_ctx_indirect_func[Pm4::PA_SC_FOV_WINDOW_TB] = [](KYTY_HW_CTX_INDIRECT_ARGS) { + HwCtxIgnoreFovWindow(value); + }; + g_hw_ctx_indirect_func[Pm4::PA_SC_FSR_ENABLE] = [](KYTY_HW_CTX_INDIRECT_ARGS) { + HwCtxIgnoreFsrRegister(cmd_offset, value); + }; + g_hw_ctx_indirect_func[Pm4::FSR_RECURSIONS0] = [](KYTY_HW_CTX_INDIRECT_ARGS) { + HwCtxIgnoreFsrRegister(cmd_offset, value); + }; + g_hw_ctx_indirect_func[Pm4::FSR_RECURSIONS1] = [](KYTY_HW_CTX_INDIRECT_ARGS) { + HwCtxIgnoreFsrRegister(cmd_offset, value); + }; + + g_hw_ctx_indirect_func[Pm4::PA_CL_VTE_CNTL] = [](KYTY_HW_CTX_INDIRECT_ARGS) { + cp->GetCtx()->SetViewportTransformControl(value); + }; + + g_hw_ctx_indirect_func[Pm4::PA_SC_MODE_CNTL_0] = [](KYTY_HW_CTX_INDIRECT_ARGS) { + HW::ScanModeControl r; + r.msaa_enable = KYTY_PM4_GET(value, PA_SC_MODE_CNTL_0, MSAA_ENABLE) != 0; + r.vport_scissor_enable = KYTY_PM4_GET(value, PA_SC_MODE_CNTL_0, VPORT_SCISSOR_ENABLE) != 0; + r.line_stipple_enable = KYTY_PM4_GET(value, PA_SC_MODE_CNTL_0, LINE_STIPPLE_ENABLE) != 0; + cp->GetCtx()->SetScanModeControl(r); + }; + g_hw_ctx_indirect_func[Pm4::PA_SC_MODE_CNTL_1] = [](KYTY_HW_CTX_INDIRECT_ARGS) { + HwCtxIgnoreScanModeControl1(value); + }; + + g_hw_ctx_indirect_func[Pm4::PA_SC_AA_CONFIG] = [](KYTY_HW_CTX_INDIRECT_ARGS) { + cp->GetCtx()->SetAaConfig(ParseAaConfig(value)); + }; + + for (auto cmd_offset = Pm4::PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_0; + cmd_offset < Pm4::PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_0 + 16; cmd_offset++) { + g_hw_ctx_indirect_func[cmd_offset] = [](KYTY_HW_CTX_INDIRECT_ARGS) { + auto r = cp->GetCtx()->GetAaSampleControl(); + r.locations[cmd_offset - Pm4::PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_0] = value; + cp->GetCtx()->SetAaSampleControl(r); + }; + } + + g_hw_ctx_indirect_func[Pm4::PA_SC_CENTROID_PRIORITY_0] = [](KYTY_HW_CTX_INDIRECT_ARGS) { + auto r = cp->GetCtx()->GetAaSampleControl(); + r.centroid_priority = (r.centroid_priority & 0xffffffff00000000ull) | value; + cp->GetCtx()->SetAaSampleControl(r); + }; + + g_hw_ctx_indirect_func[Pm4::PA_SC_CENTROID_PRIORITY_1] = [](KYTY_HW_CTX_INDIRECT_ARGS) { + auto r = cp->GetCtx()->GetAaSampleControl(); + r.centroid_priority = + (r.centroid_priority & 0x00000000ffffffffull) | (static_cast(value) << 32u); + cp->GetCtx()->SetAaSampleControl(r); + }; + + g_hw_ctx_indirect_func[Pm4::PA_SC_AA_MASK_X0Y0_X1Y0] = [](KYTY_HW_CTX_INDIRECT_ARGS) { + HwCtxIgnoreAaMaskRegister(cmd_offset, value); + }; + g_hw_ctx_indirect_func[Pm4::PA_SC_AA_MASK_X0Y1_X1Y1] = [](KYTY_HW_CTX_INDIRECT_ARGS) { + HwCtxIgnoreAaMaskRegister(cmd_offset, value); + }; + g_hw_ctx_indirect_func[Pm4::PA_SU_VTX_CNTL] = [](KYTY_HW_CTX_INDIRECT_ARGS) { + HwCtxIgnoreVtxControl(value); + }; + g_hw_ctx_indirect_func[Pm4::PS_SHADER_SAMPLE_EXCLUSION_MASK] = [](KYTY_HW_CTX_INDIRECT_ARGS) { + HwCtxIgnoreAaMaskRegister(cmd_offset, value); + }; + g_hw_ctx_indirect_func[Pm4::PA_SC_BINNER_CNTL_0] = [](KYTY_HW_CTX_INDIRECT_ARGS) { + HwCtxIgnorePaScExtendedControl(cmd_offset, value); + }; + g_hw_ctx_indirect_func[Pm4::PA_SC_BINNER_CNTL_1] = [](KYTY_HW_CTX_INDIRECT_ARGS) { + HwCtxIgnorePaScExtendedControl(cmd_offset, value); + }; + g_hw_ctx_indirect_func[Pm4::PA_SC_CONSERVATIVE_RASTERIZATION_CNTL] = + [](KYTY_HW_CTX_INDIRECT_ARGS) { HwCtxIgnorePaScExtendedControl(cmd_offset, value); }; + g_hw_ctx_indirect_func[Pm4::PA_SC_NGG_MODE_CNTL] = [](KYTY_HW_CTX_INDIRECT_ARGS) { + HwCtxIgnorePaScExtendedControl(cmd_offset, value); + }; + g_hw_ctx_indirect_func[Pm4::DB_ALPHA_TO_MASK] = [](KYTY_HW_CTX_INDIRECT_ARGS) { + HwCtxIgnoreAlphaToMaskRegister(value); + }; + g_hw_ctx_indirect_func[Pm4::CB_COLOR_CONTROL] = [](KYTY_HW_CTX_INDIRECT_ARGS) { + HW::ColorControl r; + r.mode = KYTY_PM4_GET(value, CB_COLOR_CONTROL, MODE); + r.op = KYTY_PM4_GET(value, CB_COLOR_CONTROL, ROP3); + cp->GetCtx()->SetColorControl(r); + }; + + g_hw_ctx_indirect_func[Pm4::DB_DEPTH_CONTROL] = [](KYTY_HW_CTX_INDIRECT_ARGS) { + HW::DepthControl r; + r.stencil_enable = KYTY_PM4_GET(value, DB_DEPTH_CONTROL, STENCIL_ENABLE) != 0; + r.z_enable = KYTY_PM4_GET(value, DB_DEPTH_CONTROL, Z_ENABLE) != 0; + r.z_write_enable = KYTY_PM4_GET(value, DB_DEPTH_CONTROL, Z_WRITE_ENABLE) != 0; + r.depth_bounds_enable = KYTY_PM4_GET(value, DB_DEPTH_CONTROL, DEPTH_BOUNDS_ENABLE) != 0; + r.zfunc = KYTY_PM4_GET(value, DB_DEPTH_CONTROL, ZFUNC); + r.backface_enable = KYTY_PM4_GET(value, DB_DEPTH_CONTROL, BACKFACE_ENABLE) != 0; + r.stencilfunc = KYTY_PM4_GET(value, DB_DEPTH_CONTROL, STENCILFUNC); + r.stencilfunc_bf = KYTY_PM4_GET(value, DB_DEPTH_CONTROL, STENCILFUNC_BF); + r.color_writes_on_depth_fail_enable = + KYTY_PM4_GET(value, DB_DEPTH_CONTROL, ENABLE_COLOR_WRITES_ON_DEPTH_FAIL) != 0; + r.color_writes_on_depth_pass_disable = + KYTY_PM4_GET(value, DB_DEPTH_CONTROL, DISABLE_COLOR_WRITES_ON_DEPTH_PASS) != 0; + cp->GetCtx()->SetDepthControl(r); + }; + + g_hw_ctx_indirect_func[Pm4::DB_EQAA] = [](KYTY_HW_CTX_INDIRECT_ARGS) { + cp->GetCtx()->SetEqaaControl(ParseEqaaControl(value)); + }; +} + +void GraphicsInitJmpTablesShIndirect() { + for (auto& func: g_hw_sh_indirect_func) { + func = nullptr; + } + + g_hw_sh_indirect_func[Pm4::COMPUTE_PGM_LO] = [](KYTY_HW_SH_INDIRECT_ARGS) { + HwShSetCsRegister(cp, cmd_offset, value); + }; + g_hw_sh_indirect_func[Pm4::COMPUTE_PGM_HI] = [](KYTY_HW_SH_INDIRECT_ARGS) { + HwShSetCsRegister(cp, cmd_offset, value); + }; + g_hw_sh_indirect_func[Pm4::COMPUTE_TBA_LO] = [](KYTY_HW_SH_INDIRECT_ARGS) { + HwShSetCsRegister(cp, cmd_offset, value); + }; + g_hw_sh_indirect_func[Pm4::COMPUTE_TBA_HI] = [](KYTY_HW_SH_INDIRECT_ARGS) { + HwShSetCsRegister(cp, cmd_offset, value); + }; + g_hw_sh_indirect_func[Pm4::COMPUTE_TMA_LO] = [](KYTY_HW_SH_INDIRECT_ARGS) { + HwShSetCsRegister(cp, cmd_offset, value); + }; + g_hw_sh_indirect_func[Pm4::COMPUTE_TMA_HI] = [](KYTY_HW_SH_INDIRECT_ARGS) { + HwShSetCsRegister(cp, cmd_offset, value); + }; + g_hw_sh_indirect_func[Pm4::COMPUTE_PGM_RSRC1] = [](KYTY_HW_SH_INDIRECT_ARGS) { + HwShSetCsRegister(cp, cmd_offset, value); + }; + g_hw_sh_indirect_func[Pm4::COMPUTE_PGM_RSRC2] = [](KYTY_HW_SH_INDIRECT_ARGS) { + HwShSetCsRegister(cp, cmd_offset, value); + }; + g_hw_sh_indirect_func[Pm4::COMPUTE_START_X] = [](KYTY_HW_SH_INDIRECT_ARGS) { + HwShSetCsRegister(cp, cmd_offset, value); + }; + g_hw_sh_indirect_func[Pm4::COMPUTE_START_Y] = [](KYTY_HW_SH_INDIRECT_ARGS) { + HwShSetCsRegister(cp, cmd_offset, value); + }; + g_hw_sh_indirect_func[Pm4::COMPUTE_START_Z] = [](KYTY_HW_SH_INDIRECT_ARGS) { + HwShSetCsRegister(cp, cmd_offset, value); + }; + g_hw_sh_indirect_func[Pm4::COMPUTE_NUM_THREAD_X] = [](KYTY_HW_SH_INDIRECT_ARGS) { + HwShSetCsRegister(cp, cmd_offset, value); + }; + g_hw_sh_indirect_func[Pm4::COMPUTE_NUM_THREAD_Y] = [](KYTY_HW_SH_INDIRECT_ARGS) { + HwShSetCsRegister(cp, cmd_offset, value); + }; + g_hw_sh_indirect_func[Pm4::COMPUTE_NUM_THREAD_Z] = [](KYTY_HW_SH_INDIRECT_ARGS) { + HwShSetCsRegister(cp, cmd_offset, value); + }; + g_hw_sh_indirect_func[Pm4::COMPUTE_RESOURCE_LIMITS] = [](KYTY_HW_SH_INDIRECT_ARGS) { + HwShSetCsRegister(cp, cmd_offset, value); + }; + g_hw_sh_indirect_func[Pm4::COMPUTE_DESTINATION_EN_SE0] = [](KYTY_HW_SH_INDIRECT_ARGS) { + HwShSetCsRegister(cp, cmd_offset, value); + }; + g_hw_sh_indirect_func[Pm4::COMPUTE_DESTINATION_EN_SE1] = [](KYTY_HW_SH_INDIRECT_ARGS) { + HwShSetCsRegister(cp, cmd_offset, value); + }; + g_hw_sh_indirect_func[Pm4::COMPUTE_TMPRING_SIZE] = [](KYTY_HW_SH_INDIRECT_ARGS) { + HwShSetCsRegister(cp, cmd_offset, value); + }; + g_hw_sh_indirect_func[Pm4::COMPUTE_DESTINATION_EN_SE2] = [](KYTY_HW_SH_INDIRECT_ARGS) { + HwShSetCsRegister(cp, cmd_offset, value); + }; + g_hw_sh_indirect_func[Pm4::COMPUTE_DESTINATION_EN_SE3] = [](KYTY_HW_SH_INDIRECT_ARGS) { + HwShSetCsRegister(cp, cmd_offset, value); + }; + g_hw_sh_indirect_func[Pm4::COMPUTE_PGM_RSRC3] = [](KYTY_HW_SH_INDIRECT_ARGS) { + HwShSetCsRegister(cp, cmd_offset, value); + }; + g_hw_sh_indirect_func[Pm4::COMPUTE_SHADER_CHKSUM] = [](KYTY_HW_SH_INDIRECT_ARGS) { + HwShSetCsRegister(cp, cmd_offset, value); + }; + g_hw_sh_indirect_func[Pm4::COMPUTE_DISPATCH_TUNNEL] = [](KYTY_HW_SH_INDIRECT_ARGS) { + HwShSetCsRegister(cp, cmd_offset, value); + }; + + for (uint32_t slot = 0; slot < 16; slot++) { + g_hw_sh_indirect_func[Pm4::COMPUTE_USER_DATA_0 + slot] = [](KYTY_HW_SH_INDIRECT_ARGS) { + auto sgpr = cmd_offset - Pm4::COMPUTE_USER_DATA_0; + cp->GetShCtx()->SetCsUserSgpr(sgpr, value, cp->GetUserDataMarker()); + cp->SetUserDataMarker(HW::UserSgprType::Unknown); + }; + } + for (uint32_t slot = 0; Pm4::COMPUTE_USER_ACCUM_0 + slot < Pm4::COMPUTE_USER_DATA_0; slot++) { + g_hw_sh_indirect_func[Pm4::COMPUTE_USER_ACCUM_0 + slot] = [](KYTY_HW_SH_INDIRECT_ARGS) { + auto sgpr = 16u + (cmd_offset - Pm4::COMPUTE_USER_ACCUM_0); + cp->GetShCtx()->SetCsUserSgpr(sgpr, value, cp->GetUserDataMarker()); + cp->SetUserDataMarker(HW::UserSgprType::Unknown); + }; + } + for (uint32_t slot = 0; slot < 32; slot++) { + g_hw_sh_indirect_func[Pm4::SPI_SHADER_USER_DATA_PS_0 + slot] = + [](KYTY_HW_SH_INDIRECT_ARGS) { + auto sgpr = cmd_offset - Pm4::SPI_SHADER_USER_DATA_PS_0; + cp->GetShCtx()->SetPsUserSgpr(sgpr, value, cp->GetUserDataMarker()); + cp->SetUserDataMarker(HW::UserSgprType::Unknown); + }; + g_hw_sh_indirect_func[Pm4::SPI_SHADER_USER_DATA_GS_0 + slot] = + [](KYTY_HW_SH_INDIRECT_ARGS) { + auto sgpr = cmd_offset - Pm4::SPI_SHADER_USER_DATA_GS_0; + cp->GetShCtx()->SetGsUserSgpr(sgpr, value, cp->GetUserDataMarker()); + cp->SetUserDataMarker(HW::UserSgprType::Unknown); + }; + g_hw_sh_indirect_func[Pm4::SPI_SHADER_USER_DATA_HS_0 + slot] = + [](KYTY_HW_SH_INDIRECT_ARGS) { + auto sgpr = cmd_offset - Pm4::SPI_SHADER_USER_DATA_HS_0; + cp->GetShCtx()->SetHsUserSgpr(sgpr, value, cp->GetUserDataMarker()); + cp->SetUserDataMarker(HW::UserSgprType::Unknown); + }; + } + for (uint32_t slot = 0; slot < 4; slot++) { + g_hw_sh_indirect_func[Pm4::SPI_SHADER_USER_ACCUM_PS_0 + slot] = + [](KYTY_HW_SH_INDIRECT_ARGS) { + auto sgpr = 16u + (cmd_offset - Pm4::SPI_SHADER_USER_ACCUM_PS_0); + cp->GetShCtx()->SetPsUserSgpr(sgpr, value, cp->GetUserDataMarker()); + cp->SetUserDataMarker(HW::UserSgprType::Unknown); + }; + g_hw_sh_indirect_func[Pm4::SPI_SHADER_USER_ACCUM_ESGS_0 + slot] = + [](KYTY_HW_SH_INDIRECT_ARGS) { + auto sgpr = 16u + (cmd_offset - Pm4::SPI_SHADER_USER_ACCUM_ESGS_0); + cp->GetShCtx()->SetGsUserSgpr(sgpr, value, cp->GetUserDataMarker()); + cp->SetUserDataMarker(HW::UserSgprType::Unknown); + }; + g_hw_sh_indirect_func[Pm4::SPI_SHADER_USER_ACCUM_LSHS_0 + slot] = + [](KYTY_HW_SH_INDIRECT_ARGS) { + auto sgpr = 16u + (cmd_offset - Pm4::SPI_SHADER_USER_ACCUM_LSHS_0); + cp->GetShCtx()->SetHsUserSgpr(sgpr, value, cp->GetUserDataMarker()); + cp->SetUserDataMarker(HW::UserSgprType::Unknown); + }; + } + g_hw_sh_indirect_func[Pm4::SPI_SHADER_TBA_LO_PS] = [](KYTY_HW_SH_INDIRECT_ARGS) { + HwShIgnoreShaderRegister(cmd_offset, value); + }; + g_hw_sh_indirect_func[Pm4::SPI_SHADER_PGM_RSRC3_PS] = [](KYTY_HW_SH_INDIRECT_ARGS) { + HwShIgnoreShaderRegister(cmd_offset, value); + }; + g_hw_sh_indirect_func[Pm4::SPI_SHADER_PGM_RSRC4_PS] = [](KYTY_HW_SH_INDIRECT_ARGS) { + HwShIgnoreShaderRegister(cmd_offset, value); + }; + g_hw_sh_indirect_func[Pm4::SPI_SHADER_TMA_LO_PS] = [](KYTY_HW_SH_INDIRECT_ARGS) { + HwShIgnoreShaderRegister(cmd_offset, value); + }; + g_hw_sh_indirect_func[Pm4::SPI_SHADER_TMA_HI_PS] = [](KYTY_HW_SH_INDIRECT_ARGS) { + HwShIgnoreShaderRegister(cmd_offset, value); + }; + g_hw_sh_indirect_func[Pm4::SPI_SHADER_REQ_CTRL_PS] = [](KYTY_HW_SH_INDIRECT_ARGS) { + HwShIgnoreShaderRegister(cmd_offset, value); + }; + g_hw_sh_indirect_func[Pm4::SPI_SHADER_PGM_RSRC4_GS] = [](KYTY_HW_SH_INDIRECT_ARGS) { + HwShIgnoreShaderRegister(cmd_offset, value); + }; + g_hw_sh_indirect_func[Pm4::SPI_SHADER_PGM_RSRC3_GS] = [](KYTY_HW_SH_INDIRECT_ARGS) { + HwShIgnoreShaderRegister(cmd_offset, value); + }; + g_hw_sh_indirect_func[Pm4::SPI_SHADER_USER_DATA_ADDR_LO_GS] = [](KYTY_HW_SH_INDIRECT_ARGS) { + HwShIgnoreShaderRegister(cmd_offset, value); + }; + g_hw_sh_indirect_func[Pm4::SPI_SHADER_USER_DATA_ADDR_HI_GS] = [](KYTY_HW_SH_INDIRECT_ARGS) { + HwShIgnoreShaderRegister(cmd_offset, value); + }; + g_hw_sh_indirect_func[Pm4::SPI_SHADER_REQ_CTRL_ESGS] = [](KYTY_HW_SH_INDIRECT_ARGS) { + HwShIgnoreShaderRegister(cmd_offset, value); + }; + g_hw_sh_indirect_func[Pm4::SPI_SHADER_PGM_RSRC4_HS] = [](KYTY_HW_SH_INDIRECT_ARGS) { + HwShIgnoreShaderRegister(cmd_offset, value); + }; + g_hw_sh_indirect_func[Pm4::SPI_SHADER_PGM_RSRC3_HS] = [](KYTY_HW_SH_INDIRECT_ARGS) { + HwShIgnoreShaderRegister(cmd_offset, value); + }; + g_hw_sh_indirect_func[Pm4::SPI_SHADER_USER_DATA_ADDR_LO_HS] = [](KYTY_HW_SH_INDIRECT_ARGS) { + HwShIgnoreShaderRegister(cmd_offset, value); + }; + g_hw_sh_indirect_func[Pm4::SPI_SHADER_USER_DATA_ADDR_HI_HS] = [](KYTY_HW_SH_INDIRECT_ARGS) { + HwShIgnoreShaderRegister(cmd_offset, value); + }; + g_hw_sh_indirect_func[Pm4::SPI_SHADER_REQ_CTRL_LSHS] = [](KYTY_HW_SH_INDIRECT_ARGS) { + HwShIgnoreShaderRegister(cmd_offset, value); + }; + g_hw_sh_indirect_func[Pm4::SPI_SHADER_PGM_LO_HS] = [](KYTY_HW_SH_INDIRECT_ARGS) { + auto base = cp->GetShCtx()->GetVs().hs_regs.data_addr; + base &= 0xFFFFFF00000000FFull; + base |= static_cast(value) << 8u; + cp->GetShCtx()->SetHsShaderBase(base); + }; + + g_hw_sh_indirect_func[Pm4::SPI_SHADER_PGM_HI_HS] = [](KYTY_HW_SH_INDIRECT_ARGS) { + auto base = cp->GetShCtx()->GetVs().hs_regs.data_addr; + base &= 0xFFFF00FFFFFFFFFFull; + base |= (static_cast(value) & 0xffu) << 40u; + cp->GetShCtx()->SetHsShaderBase(base); + }; + + g_hw_sh_indirect_func[Pm4::SPI_SHADER_PGM_CHKSUM_HS] = [](KYTY_HW_SH_INDIRECT_ARGS) { + cp->GetShCtx()->SetHsShaderChksum(value); + }; + + g_hw_sh_indirect_func[Pm4::SPI_SHADER_PGM_RSRC1_HS] = [](KYTY_HW_SH_INDIRECT_ARGS) { + HW::VsShaderResource1 r1; + r1.vgprs = KYTY_PM4_GET(value, SPI_SHADER_PGM_RSRC1_VS, VGPRS); + r1.sgprs = KYTY_PM4_GET(value, SPI_SHADER_PGM_RSRC1_VS, SGPRS); + r1.priority = KYTY_PM4_GET(value, SPI_SHADER_PGM_RSRC1_VS, PRIORITY); + r1.float_mode = KYTY_PM4_GET(value, SPI_SHADER_PGM_RSRC1_VS, FLOAT_MODE); + r1.dx10_clamp = KYTY_PM4_GET(value, SPI_SHADER_PGM_RSRC1_VS, DX10_CLAMP) != 0; + r1.ieee_mode = KYTY_PM4_GET(value, SPI_SHADER_PGM_RSRC1_VS, IEEE_MODE) != 0; + r1.vgpr_component_count = KYTY_PM4_GET(value, SPI_SHADER_PGM_RSRC1_VS, VGPR_COMP_CNT); + r1.cu_group_enable = KYTY_PM4_GET(value, SPI_SHADER_PGM_RSRC1_VS, CU_GROUP_ENABLE) != 0; + r1.require_forward_progress = + KYTY_PM4_GET(value, SPI_SHADER_PGM_RSRC1_VS, FWD_PROGRESS) != 0; + r1.fp16_overflow = KYTY_PM4_GET(value, SPI_SHADER_PGM_RSRC1_VS, FP16_OVFL) != 0; + cp->GetShCtx()->SetHsShaderResource1(r1); + }; + + g_hw_sh_indirect_func[Pm4::SPI_SHADER_PGM_RSRC2_HS] = [](KYTY_HW_SH_INDIRECT_ARGS) { + HW::VsShaderResource2 r2; + r2.scratch_en = KYTY_PM4_GET(value, SPI_SHADER_PGM_RSRC2_VS, SCRATCH_EN) != 0; + r2.user_sgpr = KYTY_PM4_GET(value, SPI_SHADER_PGM_RSRC2_VS, USER_SGPR) + + (KYTY_PM4_GET(value, SPI_SHADER_PGM_RSRC2_VS, USER_SGPR_MSB) << 5u); + r2.offchip_lds = KYTY_PM4_GET(value, SPI_SHADER_PGM_RSRC2_VS, OC_LDS_EN) != 0; + r2.streamout_enabled = KYTY_PM4_GET(value, SPI_SHADER_PGM_RSRC2_VS, SO_EN) != 0; + r2.shared_vgprs = KYTY_PM4_GET(value, SPI_SHADER_PGM_RSRC2_VS, SHARED_VGPR_CNT); + cp->GetShCtx()->SetHsShaderResource2(r2); + }; + + g_hw_sh_indirect_func[Pm4::SPI_SHADER_PGM_LO_LS] = [](KYTY_HW_SH_INDIRECT_ARGS) { + auto base = cp->GetShCtx()->GetVs().ls_regs.data_addr; + base &= 0xFFFFFF00000000FFull; + base |= static_cast(value) << 8u; + cp->GetShCtx()->SetLsShaderBase(base); + }; + + g_hw_sh_indirect_func[Pm4::SPI_SHADER_PGM_HI_LS] = [](KYTY_HW_SH_INDIRECT_ARGS) { + auto base = cp->GetShCtx()->GetVs().ls_regs.data_addr; + base &= 0xFFFF00FFFFFFFFFFull; + base |= (static_cast(value) & 0xffu) << 40u; + cp->GetShCtx()->SetLsShaderBase(base); + }; + + g_hw_sh_indirect_func[Pm4::SPI_SHADER_PGM_RSRC1_LS] = [](KYTY_HW_SH_INDIRECT_ARGS) { + HW::VsShaderResource1 r1; + r1.vgprs = KYTY_PM4_GET(value, SPI_SHADER_PGM_RSRC1_VS, VGPRS); + r1.sgprs = KYTY_PM4_GET(value, SPI_SHADER_PGM_RSRC1_VS, SGPRS); + r1.priority = KYTY_PM4_GET(value, SPI_SHADER_PGM_RSRC1_VS, PRIORITY); + r1.float_mode = KYTY_PM4_GET(value, SPI_SHADER_PGM_RSRC1_VS, FLOAT_MODE); + r1.dx10_clamp = KYTY_PM4_GET(value, SPI_SHADER_PGM_RSRC1_VS, DX10_CLAMP) != 0; + r1.ieee_mode = KYTY_PM4_GET(value, SPI_SHADER_PGM_RSRC1_VS, IEEE_MODE) != 0; + r1.vgpr_component_count = KYTY_PM4_GET(value, SPI_SHADER_PGM_RSRC1_VS, VGPR_COMP_CNT); + r1.cu_group_enable = KYTY_PM4_GET(value, SPI_SHADER_PGM_RSRC1_VS, CU_GROUP_ENABLE) != 0; + r1.require_forward_progress = + KYTY_PM4_GET(value, SPI_SHADER_PGM_RSRC1_VS, FWD_PROGRESS) != 0; + r1.fp16_overflow = KYTY_PM4_GET(value, SPI_SHADER_PGM_RSRC1_VS, FP16_OVFL) != 0; + cp->GetShCtx()->SetLsShaderResource1(r1); + }; + + g_hw_sh_indirect_func[Pm4::SPI_SHADER_PGM_RSRC2_LS] = [](KYTY_HW_SH_INDIRECT_ARGS) { + HW::VsShaderResource2 r2; + r2.scratch_en = KYTY_PM4_GET(value, SPI_SHADER_PGM_RSRC2_VS, SCRATCH_EN) != 0; + r2.user_sgpr = KYTY_PM4_GET(value, SPI_SHADER_PGM_RSRC2_VS, USER_SGPR) + + (KYTY_PM4_GET(value, SPI_SHADER_PGM_RSRC2_VS, USER_SGPR_MSB) << 5u); + r2.offchip_lds = KYTY_PM4_GET(value, SPI_SHADER_PGM_RSRC2_VS, OC_LDS_EN) != 0; + r2.streamout_enabled = KYTY_PM4_GET(value, SPI_SHADER_PGM_RSRC2_VS, SO_EN) != 0; + r2.shared_vgprs = KYTY_PM4_GET(value, SPI_SHADER_PGM_RSRC2_VS, SHARED_VGPR_CNT); + cp->GetShCtx()->SetLsShaderResource2(r2); + }; + g_hw_sh_indirect_func[Pm4::SPI_SHADER_PGM_LO_ES] = [](KYTY_HW_SH_INDIRECT_ARGS) { + auto base = cp->GetShCtx()->GetVs().es_regs.data_addr; + base &= 0xFFFFFF00000000FFull; + base |= static_cast(value) << 8u; + cp->GetShCtx()->SetEsShaderBase(base); + }; + + g_hw_sh_indirect_func[Pm4::SPI_SHADER_PGM_HI_ES] = [](KYTY_HW_SH_INDIRECT_ARGS) { + auto base = cp->GetShCtx()->GetVs().es_regs.data_addr; + base &= 0xFFFF00FFFFFFFFFFull; + base |= (static_cast(value) & 0xffu) << 40u; + cp->GetShCtx()->SetEsShaderBase(base); + }; + + g_hw_sh_indirect_func[Pm4::SPI_SHADER_PGM_LO_GS] = [](KYTY_HW_SH_INDIRECT_ARGS) { + auto base = cp->GetShCtx()->GetVs().gs_regs.data_addr; + base &= 0xFFFFFF00000000FFull; + base |= static_cast(value) << 8u; + cp->GetShCtx()->SetGsShaderBase(base); + }; + + g_hw_sh_indirect_func[Pm4::SPI_SHADER_PGM_HI_GS] = [](KYTY_HW_SH_INDIRECT_ARGS) { + auto base = cp->GetShCtx()->GetVs().gs_regs.data_addr; + base &= 0xFFFF00FFFFFFFFFFull; + base |= (static_cast(value) & 0xffu) << 40u; + cp->GetShCtx()->SetGsShaderBase(base); + }; + + g_hw_sh_indirect_func[Pm4::SPI_SHADER_PGM_CHKSUM_GS] = [](KYTY_HW_SH_INDIRECT_ARGS) { + cp->GetShCtx()->SetGsShaderChksum(value); + }; + + g_hw_sh_indirect_func[Pm4::SPI_SHADER_PGM_RSRC1_GS] = [](KYTY_HW_SH_INDIRECT_ARGS) { + HW::GsShaderResource1 r1; + r1.vgprs = KYTY_PM4_GET(value, SPI_SHADER_PGM_RSRC1_GS, VGPRS); + r1.sgprs = KYTY_PM4_GET(value, SPI_SHADER_PGM_RSRC1_GS, SGPRS); + r1.priority = KYTY_PM4_GET(value, SPI_SHADER_PGM_RSRC1_GS, PRIORITY); + r1.float_mode = KYTY_PM4_GET(value, SPI_SHADER_PGM_RSRC1_GS, FLOAT_MODE); + r1.dx10_clamp = KYTY_PM4_GET(value, SPI_SHADER_PGM_RSRC1_GS, DX10_CLAMP) != 0; + r1.debug_mode = KYTY_PM4_GET(value, SPI_SHADER_PGM_RSRC1_GS, DEBUG_MODE) != 0; + r1.ieee_mode = KYTY_PM4_GET(value, SPI_SHADER_PGM_RSRC1_GS, IEEE_MODE) != 0; + r1.cu_group_enable = KYTY_PM4_GET(value, SPI_SHADER_PGM_RSRC1_GS, CU_GROUP_ENABLE) != 0; + r1.require_forward_progress = + KYTY_PM4_GET(value, SPI_SHADER_PGM_RSRC1_GS, FWD_PROGRESS) != 0; + r1.lds_configuration = KYTY_PM4_GET(value, SPI_SHADER_PGM_RSRC1_GS, WGP_MODE) != 0; + r1.gs_vgpr_component_count = KYTY_PM4_GET(value, SPI_SHADER_PGM_RSRC1_GS, GS_VGPR_COMP_CNT); + r1.fp16_overflow = KYTY_PM4_GET(value, SPI_SHADER_PGM_RSRC1_GS, FP16_OVFL) != 0; + cp->GetShCtx()->SetGsShaderResource1(r1); + }; + + g_hw_sh_indirect_func[Pm4::SPI_SHADER_PGM_RSRC2_GS] = [](KYTY_HW_SH_INDIRECT_ARGS) { + HW::GsShaderResource2 r2; + r2.scratch_en = KYTY_PM4_GET(value, SPI_SHADER_PGM_RSRC2_GS, SCRATCH_EN) != 0; + r2.user_sgpr = KYTY_PM4_GET(value, SPI_SHADER_PGM_RSRC2_GS, USER_SGPR) + + (KYTY_PM4_GET(value, SPI_SHADER_PGM_RSRC2_GS, USER_SGPR_MSB) << 5u); + r2.es_vgpr_component_count = KYTY_PM4_GET(value, SPI_SHADER_PGM_RSRC2_GS, ES_VGPR_COMP_CNT); + r2.offchip_lds = KYTY_PM4_GET(value, SPI_SHADER_PGM_RSRC2_GS, OC_LDS_EN) != 0; + r2.lds_size = KYTY_PM4_GET(value, SPI_SHADER_PGM_RSRC2_GS, LDS_SIZE); + r2.shared_vgprs = KYTY_PM4_GET(value, SPI_SHADER_PGM_RSRC2_GS, SHARED_VGPR_CNT); + cp->GetShCtx()->SetGsShaderResource2(r2); + }; + + g_hw_sh_indirect_func[Pm4::SPI_SHADER_PGM_LO_PS] = [](KYTY_HW_SH_INDIRECT_ARGS) { + auto base = cp->GetShCtx()->GetPs().ps_regs.data_addr; + base &= 0xFFFFFF00000000FFull; + base |= static_cast(value) << 8u; + cp->GetShCtx()->SetPsShaderBase(base); + }; + + g_hw_sh_indirect_func[Pm4::SPI_SHADER_PGM_HI_PS] = [](KYTY_HW_SH_INDIRECT_ARGS) { + auto base = cp->GetShCtx()->GetPs().ps_regs.data_addr; + base &= 0xFFFF00FFFFFFFFFFull; + base |= (static_cast(value) & 0xffu) << 40u; + cp->GetShCtx()->SetPsShaderBase(base); + }; + + g_hw_sh_indirect_func[Pm4::SPI_SHADER_PGM_CHKSUM_PS] = [](KYTY_HW_SH_INDIRECT_ARGS) { + cp->GetShCtx()->SetPsShaderChksum(value); + }; + + g_hw_sh_indirect_func[Pm4::SPI_SHADER_PGM_RSRC1_PS] = [](KYTY_HW_SH_INDIRECT_ARGS) { + HW::PsShaderResource1 r1; + r1.vgprs = KYTY_PM4_GET(value, SPI_SHADER_PGM_RSRC1_PS, VGPRS); + r1.sgprs = KYTY_PM4_GET(value, SPI_SHADER_PGM_RSRC1_PS, SGPRS); + r1.priority = KYTY_PM4_GET(value, SPI_SHADER_PGM_RSRC1_PS, PRIORITY); + r1.float_mode = KYTY_PM4_GET(value, SPI_SHADER_PGM_RSRC1_PS, FLOAT_MODE); + r1.dx10_clamp = KYTY_PM4_GET(value, SPI_SHADER_PGM_RSRC1_PS, DX10_CLAMP) != 0; + r1.debug_mode = KYTY_PM4_GET(value, SPI_SHADER_PGM_RSRC1_PS, DEBUG_MODE) != 0; + r1.ieee_mode = KYTY_PM4_GET(value, SPI_SHADER_PGM_RSRC1_PS, IEEE_MODE) != 0; + r1.cu_group_disable = KYTY_PM4_GET(value, SPI_SHADER_PGM_RSRC1_PS, CU_GROUP_DISABLE) != 0; + r1.require_forward_progress = + KYTY_PM4_GET(value, SPI_SHADER_PGM_RSRC1_PS, FWD_PROGRESS) != 0; + r1.fp16_overflow = KYTY_PM4_GET(value, SPI_SHADER_PGM_RSRC1_PS, FP16_OVFL) != 0; + cp->GetShCtx()->SetPsShaderResource1(r1); + }; + + g_hw_sh_indirect_func[Pm4::SPI_SHADER_PGM_RSRC2_PS] = [](KYTY_HW_SH_INDIRECT_ARGS) { + HW::PsShaderResource2 r2; + r2.scratch_en = KYTY_PM4_GET(value, SPI_SHADER_PGM_RSRC2_PS, SCRATCH_EN); + r2.user_sgpr = KYTY_PM4_GET(value, SPI_SHADER_PGM_RSRC2_PS, USER_SGPR) + + (KYTY_PM4_GET(value, SPI_SHADER_PGM_RSRC2_PS, USER_SGPR_MSB) << 5u); + r2.wave_cnt_en = KYTY_PM4_GET(value, SPI_SHADER_PGM_RSRC2_PS, WAVE_CNT_EN); + r2.extra_lds_size = KYTY_PM4_GET(value, SPI_SHADER_PGM_RSRC2_PS, EXTRA_LDS_SIZE); + r2.raster_ordered_shading = + KYTY_PM4_GET(value, SPI_SHADER_PGM_RSRC2_PS, LOAD_INTRAWAVE_COLLISION); + r2.shared_vgprs = KYTY_PM4_GET(value, SPI_SHADER_PGM_RSRC2_PS, SHARED_VGPR_CNT); + cp->GetShCtx()->SetPsShaderResource2(r2); + }; +} + +void GraphicsInitJmpTablesUcIndirect() { + for (auto& func: g_hw_uc_indirect_func) { + func = nullptr; + } + + g_hw_uc_indirect_func[Pm4::GE_CNTL] = [](KYTY_HW_UC_INDIRECT_ARGS) { + HW::GeControl r; + r.primitive_group_size = KYTY_PM4_GET(value, GE_CNTL, PRIM_GRP_SIZE); + r.vertex_group_size = KYTY_PM4_GET(value, GE_CNTL, VERT_GRP_SIZE); + cp->GetUcfg()->SetGeControl(r); + }; + + g_hw_uc_indirect_func[Pm4::GE_USER_VGPR_EN] = [](KYTY_HW_UC_INDIRECT_ARGS) { + HW::GeUserVgprEn r; + r.vgpr1 = KYTY_PM4_GET(value, GE_USER_VGPR_EN, EN_USER_VGPR1) != 0; + r.vgpr2 = KYTY_PM4_GET(value, GE_USER_VGPR_EN, EN_USER_VGPR2) != 0; + r.vgpr3 = KYTY_PM4_GET(value, GE_USER_VGPR_EN, EN_USER_VGPR3) != 0; + cp->GetUcfg()->SetGeUserVgprEn(r); + }; + + g_hw_uc_indirect_func[Pm4::VGT_PRIMITIVE_TYPE] = [](KYTY_HW_UC_INDIRECT_ARGS) { + uint32_t prim_type = KYTY_PM4_GET(value, VGT_PRIMITIVE_TYPE, PRIM_TYPE); + cp->GetUcfg()->SetPrimitiveType(prim_type); + }; + g_hw_uc_indirect_func[Pm4::VGT_INDEX_TYPE] = [](KYTY_HW_UC_INDIRECT_ARGS) { + cp->SetIndexType(value & 0x3u); + }; + g_hw_uc_indirect_func[Pm4::VGT_OBJECT_ID] = [](KYTY_HW_UC_INDIRECT_ARGS) { + cp->GetUcfg()->SetObjectId(value); + }; + g_hw_uc_indirect_func[Pm4::TEXTURE_GRADIENT_FACTORS] = [](KYTY_HW_UC_INDIRECT_ARGS) { + HwUcIgnoreTextureGradientFactors(value); + }; + g_hw_uc_indirect_func[Pm4::IA_MULTI_VGT_PARAM] = [](KYTY_HW_UC_INDIRECT_ARGS) { + HwUcIgnoreIaMultiVgtParam(value); + }; + g_hw_uc_indirect_func[Pm4::GE_MULTI_PRIM_IB_RESET_EN] = [](KYTY_HW_UC_INDIRECT_ARGS) { + HwIgnoreMultiPrimIbReset(value); + }; + g_hw_uc_indirect_func[Pm4::TA_CS_BC_BASE_ADDR] = [](KYTY_HW_UC_INDIRECT_ARGS) { + HwUcIgnoreBorderColorTableAddr(cmd_offset, value); + }; + g_hw_uc_indirect_func[Pm4::TA_CS_BC_BASE_ADDR_HI] = [](KYTY_HW_UC_INDIRECT_ARGS) { + HwUcIgnoreBorderColorTableAddr(cmd_offset, value); + }; + + g_hw_uc_indirect_func[Pm4::GE_INDX_OFFSET] = [](KYTY_HW_UC_INDIRECT_ARGS) { + cp->GetUcfg()->SetIndexOffset(value); + }; + g_hw_uc_indirect_func[Pm4::GDS_OA_CNTL] = [](KYTY_HW_UC_INDIRECT_ARGS) { + HwUcSetGdsOaRegister(cp, cmd_offset, value); + }; + g_hw_uc_indirect_func[Pm4::GDS_OA_COUNTER] = [](KYTY_HW_UC_INDIRECT_ARGS) { + HwUcSetGdsOaRegister(cp, cmd_offset, value); + }; + g_hw_uc_indirect_func[Pm4::GDS_OA_ADDRESS] = [](KYTY_HW_UC_INDIRECT_ARGS) { + HwUcSetGdsOaRegister(cp, cmd_offset, value); + }; + + g_hw_uc_indirect_func[Pm4::GE_STEREO_CNTL] = [](KYTY_HW_UC_INDIRECT_ARGS) { + static std::atomic log_count {0}; + if (log_count.fetch_add(1) < 16) { + LOGF("warning: ignoring indirect uc reg GE_STEREO_CNTL at 0x%" PRIx32 + ", value = 0x%08" PRIx32 "\n", + cmd_offset, value); + } + }; +} + +} // namespace Libs::Graphics diff --git a/src/graphics/guest_gpu/gpu_defs.h b/src/graphics/guest_gpu/gpu_defs.h new file mode 100644 index 000000000..0c02d1f46 --- /dev/null +++ b/src/graphics/guest_gpu/gpu_defs.h @@ -0,0 +1,437 @@ +#ifndef EMULATOR_INCLUDE_EMULATOR_GRAPHICS_GUEST_GPU_GPU_DEFS_H_ +#define EMULATOR_INCLUDE_EMULATOR_GRAPHICS_GUEST_GPU_GPU_DEFS_H_ + +#include "common/common.h" + +#include + +namespace Libs::Graphics::Prospero { + +template +constexpr auto GpuEnumValue(T value) { + return static_cast>(value); +} + +enum class PrimitiveType : uint32_t { + kNone = 0, + kPointList = 1, + kLineList = 2, + kLineStrip = 3, + kTriList = 4, + kTriFan = 5, + kTriStrip = 6, + kRectList = 7, + kPatch = 9, + kLineListAdjacency = 10, + kLineStripAdjacency = 11, + kTriListAdjacency = 12, + kTriStripAdjacency = 13, + kRectListLegacy = 17, + kLineLoop = 18, + kQuadListLegacy = 19, + kQuadStripLegacy = 20, + kPolygon = 21, +}; + +enum class IndexType : uint32_t { + kIndex16 = 0, + kIndex32 = 1, + kIndex8 = 2, +}; + +enum class ShaderBinaryType : uint8_t { + kCs = 0, + kPs = 1, + kGs = 2, + kHs = 3, + kGsFront = 4, + kHsFront = 5, + kGsBack = 6, + kHsBack = 7, + kFs = 8, +}; + +enum class GsOutputPrimitiveType : uint32_t { + kPoints = 0, + kLines = 1, + kTriangles = 2, + k2dRectangle = 3, + kRectList = 4, +}; + +enum class ChannelLayout : uint32_t { + kInvalid = 0, + k8 = 1, + k16 = 2, + k8_8 = 3, + k32 = 4, + k16_16 = 5, + k11_11_10 = 6, + k10_10_10_2 = 9, + k8_8_8_8 = 10, + k32_32 = 11, + k16_16_16_16 = 12, + k32_32_32_32 = 14, + k5_5_5_1 = 17, + k4_4_4_4 = 19, + kBc1 = 35, + kBc2 = 36, + kBc3 = 37, +}; + +enum class ChannelType : uint32_t { + kUNorm = 0, + kSNorm = 1, + kUInt = 4, + kSInt = 5, + kSrgb = 6, + kFloat = 7, +}; + +enum class NumberFormat : uint32_t { + kUNorm = 0, + kSNorm = 1, + kUInt = 4, + kSInt = 5, + kFloat = 7, + kSrgb = 9, +}; + +enum class ChannelOrder : uint32_t { + kStandard = 0, + kAlt = 1, + kReversed = 2, + kAltReversed = 3, +}; + +enum class DepthFormat : uint32_t { + kInvalid = 0, + kZ16 = 1, + kZ32F = 3, +}; + +enum class StencilFormat : uint32_t { + kInvalid = 0, + k8UInt = 1, +}; + +enum class TileMode : uint32_t { + kLinear = 0x00, + kStandard256B = 0x01, + kStandard4KB = 0x05, + kStandard64KB = 0x09, + kPrt = 0x11, + kDepth = 0x18, + kRenderTarget = 0x1b, +}; + +enum class StencilOp : uint32_t { + kKeep = 0x00, + kZero = 0x01, + kOnes = 0x02, + kReplaceTest = 0x03, + kReplaceOp = 0x04, + kAddClamp = 0x05, + kSubClamp = 0x06, + kInvert = 0x07, + kAddWrap = 0x08, + kSubWrap = 0x09, + kAnd = 0x0a, + kOr = 0x0b, + kXor = 0x0c, + kNand = 0x0d, + kNor = 0x0e, + kXnor = 0x0f, +}; + +enum class BlendFactor : uint32_t { + kZero = 0x00, + kOne = 0x01, + kSrcColor = 0x02, + kOneMinusSrcColor = 0x03, + kSrcAlpha = 0x04, + kOneMinusSrcAlpha = 0x05, + kDstAlpha = 0x06, + kOneMinusDstAlpha = 0x07, + kDstColor = 0x08, + kOneMinusDstColor = 0x09, + kSrcAlphaSaturate = 0x0a, + kConstantColor = 0x0d, + kOneMinusConstantColor = 0x0e, + kSrc1Color = 0x0f, + kOneMinusSrc1Color = 0x10, + kSrc1Alpha = 0x11, + kOneMinusSrc1Alpha = 0x12, + kConstantAlpha = 0x13, + kOneMinusConstantAlpha = 0x14, +}; + +enum class BlendOp : uint32_t { + kAdd = 0x00, + kSubtract = 0x01, + kMin = 0x02, + kMax = 0x03, + kReverseSubtract = 0x04, +}; + +enum class CompSwizzle : uint32_t { + kZero = 0, + kOne = 1, + kRed = 4, + kGreen = 5, + kBlue = 6, + kAlpha = 7, +}; + +enum class SamplerClampMode : uint32_t { + kWrap = 0, + kMirror = 1, + kClampLastTexel = 2, + kMirrorOnceLastTexel = 3, + kClampHalfBorder = 4, + kMirrorOnceHalfBorder = 5, + kClampBorder = 6, + kMirrorOnceBorder = 7, +}; + +enum class SamplerAnisoRatio : uint32_t { + kOne = 0, + kTwo = 1, + kFour = 2, + kEight = 3, + kSixteen = 4, +}; + +enum class SamplerBorderColor : uint32_t { + kTransBlack = 0, + kOpaqueBlack = 1, + kOpaqueWhite = 2, + kFromTable = 3, +}; + +enum class SamplerFilter : uint32_t { + kPoint = 0, + kBilinear = 1, + kAnisoPoint = 2, + kAnisoLinear = 3, +}; + +enum class SamplerMipFilter : uint32_t { + kNone = 0, + kPoint = 1, + kLinear = 2, +}; + +enum class ImageType : uint32_t { + kColor1D = 8, + kColor2D = 9, + kColor3D = 10, + kCube = 11, + kColor1DArray = 12, + kColor2DArray = 13, + kColor2DMsaa = 14, + kColor2DMsaaArray = 15, +}; + +enum class DescriptorKind : uint32_t { + kBuffer = 1, + kSampler = 2, + kUnused = 3, +}; + +enum class ShaderUsageType : uint32_t { + kReadOnlyResource = 0x00, + kSampler = 0x01, + kConstantBuffer = 0x02, + kReadWriteResource = 0x04, + kGdsMemoryRange = 0x07, + kFetchShader = 0x12, + kVertexBufferTable = 0x17, + kExtendedUserData = 0x1b, +}; + +enum class BufferFormat : uint32_t { + kInvalid = 0, + k8UNorm = 1, + k8SNorm = 2, + k8UScaled = 3, + k8SScaled = 4, + k8UInt = 5, + k8SInt = 6, + k16UNorm = 7, + k16SNorm = 8, + k16UScaled = 9, + k16SScaled = 10, + k16UInt = 11, + k16SInt = 12, + k16Float = 13, + k8_8UNorm = 14, + k8_8SNorm = 15, + k8_8UScaled = 16, + k8_8SScaled = 17, + k8_8UInt = 18, + k8_8SInt = 19, + k32UInt = 20, + k32SInt = 21, + k32Float = 22, + k16_16UNorm = 23, + k16_16SNorm = 24, + k16_16UScaled = 25, + k16_16SScaled = 26, + k16_16UInt = 27, + k16_16SInt = 28, + k16_16Float = 29, + k11_11_10UNorm = 30, + k11_11_10SNorm = 31, + k11_11_10UScaled = 32, + k11_11_10SScaled = 33, + k11_11_10UInt = 34, + k11_11_10SInt = 35, + k11_11_10Float = 36, + k10_11_11UNorm = 37, + k10_11_11SNorm = 38, + k10_11_11UScaled = 39, + k10_11_11SScaled = 40, + k10_11_11UInt = 41, + k10_11_11SInt = 42, + k10_11_11Float = 43, + k2_10_10_10UNorm = 44, + k2_10_10_10SNorm = 45, + k2_10_10_10UScaled = 46, + k2_10_10_10SScaled = 47, + k2_10_10_10UInt = 48, + k2_10_10_10SInt = 49, + k10_10_10_2UNorm = 50, + k10_10_10_2SNorm = 51, + k10_10_10_2UScaled = 52, + k10_10_10_2SScaled = 53, + k10_10_10_2UInt = 54, + k10_10_10_2SInt = 55, + k8_8_8_8UNorm = 56, + k8_8_8_8SNorm = 57, + k8_8_8_8UScaled = 58, + k8_8_8_8SScaled = 59, + k8_8_8_8UInt = 60, + k8_8_8_8SInt = 61, + k32_32UInt = 62, + k32_32SInt = 63, + k32_32Float = 64, + k16_16_16_16UNorm = 65, + k16_16_16_16SNorm = 66, + k16_16_16_16UScaled = 67, + k16_16_16_16SScaled = 68, + k16_16_16_16UInt = 69, + k16_16_16_16SInt = 70, + k16_16_16_16Float = 71, + k32_32_32UInt = 72, + k32_32_32SInt = 73, + k32_32_32Float = 74, + k32_32_32_32UInt = 75, + k32_32_32_32SInt = 76, + k32_32_32_32Float = 77, + k8_8_8_8Srgb = 130, + k9_9_9_5Float = 132, + k5_6_5UNorm = 133, + k5_5_5_1UNorm = 134, + k4_4_4_4UNorm = 136, + kFmask8_S4_F4 = 161, + kBc1UNorm = 169, + kBc1Srgb = 170, + kBc2UNorm = 171, + kBc2Srgb = 172, + kBc3UNorm = 173, + kBc3Srgb = 174, + kBc4UNorm = 175, + kBc4SNorm = 176, + kBc5UNorm = 177, + kBc5SNorm = 178, + kBc6UFloat = 179, + kBc6SFloat = 180, + kBc7UNorm = 181, + kBc7Srgb = 182, +}; + +enum class VertexAttribFormat : uint32_t { + kInvalid = 0, + k8UNorm = 4, + k8SNorm = 8, + k8UScaled = 12, + k8SScaled = 16, + k8UInt = 20, + k8SInt = 24, + k16UNorm = 28, + k16SNorm = 32, + k16UScaled = 36, + k16SScaled = 40, + k16UInt = 44, + k16SInt = 48, + k16Float = 52, + k8_8UNorm = 57, + k8_8SNorm = 61, + k8_8UScaled = 65, + k8_8SScaled = 69, + k8_8UInt = 73, + k8_8SInt = 77, + k32UInt = 80, + k32SInt = 84, + k32Float = 88, + k16_16UNorm = 93, + k16_16SNorm = 97, + k16_16UScaled = 101, + k16_16SScaled = 105, + k16_16UInt = 109, + k16_16SInt = 113, + k16_16Float = 117, + k11_11_10UNorm = 122, + k11_11_10SNorm = 126, + k11_11_10UScaled = 130, + k11_11_10SScaled = 134, + k11_11_10UInt = 138, + k11_11_10SInt = 142, + k11_11_10Float = 146, + k10_11_11UNorm = 150, + k10_11_11SNorm = 154, + k10_11_11UScaled = 158, + k10_11_11SScaled = 162, + k10_11_11UInt = 166, + k10_11_11SInt = 170, + k10_11_11Float = 174, + k2_10_10_10UNorm = 179, + k2_10_10_10SNorm = 183, + k2_10_10_10UScaled = 187, + k2_10_10_10SScaled = 191, + k2_10_10_10UInt = 195, + k2_10_10_10SInt = 199, + k10_10_10_2UNorm = 203, + k10_10_10_2SNorm = 207, + k10_10_10_2UScaled = 211, + k10_10_10_2SScaled = 215, + k10_10_10_2UInt = 219, + k10_10_10_2SInt = 223, + k8_8_8_8UNorm = 227, + k8_8_8_8SNorm = 231, + k8_8_8_8UScaled = 235, + k8_8_8_8SScaled = 239, + k8_8_8_8UInt = 243, + k8_8_8_8SInt = 247, + k32_32UInt = 249, + k32_32SInt = 253, + k32_32Float = 257, + k16_16_16_16UNorm = 263, + k16_16_16_16SNorm = 267, + k16_16_16_16UScaled = 271, + k16_16_16_16SScaled = 275, + k16_16_16_16UInt = 279, + k16_16_16_16SInt = 283, + k16_16_16_16Float = 287, + k32_32_32UInt = 290, + k32_32_32SInt = 294, + k32_32_32Float = 298, + k32_32_32_32UInt = 303, + k32_32_32_32SInt = 307, + k32_32_32_32Float = 311, +}; + +} // namespace Libs::Graphics::Prospero + +#endif /* EMULATOR_INCLUDE_EMULATOR_GRAPHICS_GUEST_GPU_GPU_DEFS_H_ */ diff --git a/src/graphics/guest_gpu/gpu_format.cpp b/src/graphics/guest_gpu/gpu_format.cpp new file mode 100644 index 000000000..77fdb5f5f --- /dev/null +++ b/src/graphics/guest_gpu/gpu_format.cpp @@ -0,0 +1,145 @@ +#include "graphics/guest_gpu/gpu_format.h" + +#include "graphics/guest_gpu/gpu_defs.h" + +#include + +namespace Libs::Graphics::Prospero { +namespace { + +constexpr FormatInfo kFormatInfo[] = { + {GpuEnumValue(BufferFormat::k8UNorm), VK_FORMAT_R8_UNORM, 1, 0, 1, true, false}, + {GpuEnumValue(BufferFormat::k8SNorm), VK_FORMAT_UNDEFINED, 0, 0, 1, false, false}, + {GpuEnumValue(BufferFormat::k8UInt), VK_FORMAT_R8_UINT, 1, 0, 1, true, true}, + {GpuEnumValue(BufferFormat::k16UNorm), VK_FORMAT_R16_UNORM, 2, 0, 2, true, false}, + {GpuEnumValue(BufferFormat::k16SNorm), VK_FORMAT_R16_SNORM, 2, 0, 2, true, false}, + {GpuEnumValue(BufferFormat::k16UInt), VK_FORMAT_R16_UINT, 2, 0, 2, true, true}, + {GpuEnumValue(BufferFormat::k16SInt), VK_FORMAT_R16_SINT, 2, 0, 2, false, false}, + {GpuEnumValue(BufferFormat::k16Float), VK_FORMAT_R16_SFLOAT, 2, 0, 2, true, false}, + {GpuEnumValue(BufferFormat::k8_8UNorm), VK_FORMAT_R8G8_UNORM, 2, 0, 2, true, false}, + {GpuEnumValue(BufferFormat::k8_8SNorm), VK_FORMAT_R8G8_SNORM, 2, 0, 2, true, false}, + {GpuEnumValue(BufferFormat::k8_8UInt), VK_FORMAT_R8G8_UINT, 2, 0, 2, true, true}, + {GpuEnumValue(BufferFormat::k8_8SInt), VK_FORMAT_R8G8_SINT, 2, 0, 2, false, false}, + {GpuEnumValue(BufferFormat::k32UInt), VK_FORMAT_R32_UINT, 4, 0, 4, true, true}, + {GpuEnumValue(BufferFormat::k32SInt), VK_FORMAT_R32_SINT, 4, 0, 4, false, false}, + {GpuEnumValue(BufferFormat::k32Float), VK_FORMAT_R32_SFLOAT, 4, 0, 4, true, false}, + {GpuEnumValue(BufferFormat::k16_16UNorm), VK_FORMAT_R16G16_UNORM, 4, 0, 4, true, false}, + {GpuEnumValue(BufferFormat::k16_16SNorm), VK_FORMAT_R16G16_SNORM, 4, 0, 4, true, false}, + {GpuEnumValue(BufferFormat::k16_16UInt), VK_FORMAT_R16G16_UINT, 4, 0, 4, true, true}, + {GpuEnumValue(BufferFormat::k16_16SInt), VK_FORMAT_R16G16_SINT, 4, 0, 4, false, false}, + {GpuEnumValue(BufferFormat::k16_16Float), VK_FORMAT_R16G16_SFLOAT, 4, 0, 4, true, false}, + {GpuEnumValue(BufferFormat::k11_11_10Float), VK_FORMAT_B10G11R11_UFLOAT_PACK32, 4, 0, 4, + true, false}, + {GpuEnumValue(BufferFormat::k10_10_10_2UNorm), VK_FORMAT_A2B10G10R10_UNORM_PACK32, 4, 0, 4, + true, false}, + {GpuEnumValue(BufferFormat::k8_8_8_8UNorm), VK_FORMAT_R8G8B8A8_UNORM, 4, 0, 4, true, false}, + {GpuEnumValue(BufferFormat::k8_8_8_8SNorm), VK_FORMAT_R8G8B8A8_SNORM, 4, 0, 4, true, false}, + {GpuEnumValue(BufferFormat::k8_8_8_8UInt), VK_FORMAT_R8G8B8A8_UINT, 4, 0, 4, true, true}, + {GpuEnumValue(BufferFormat::k8_8_8_8SInt), VK_FORMAT_R8G8B8A8_SINT, 4, 0, 4, false, false}, + {GpuEnumValue(BufferFormat::k32_32UInt), VK_FORMAT_R32G32_UINT, 8, 0, 8, true, true}, + {GpuEnumValue(BufferFormat::k32_32SInt), VK_FORMAT_R32G32_SINT, 8, 0, 8, false, false}, + {GpuEnumValue(BufferFormat::k32_32Float), VK_FORMAT_R32G32_SFLOAT, 8, 0, 8, true, false}, + {GpuEnumValue(BufferFormat::k16_16_16_16UNorm), VK_FORMAT_R16G16B16A16_UNORM, 8, 0, 8, true, + false}, + {GpuEnumValue(BufferFormat::k16_16_16_16SNorm), VK_FORMAT_R16G16B16A16_SNORM, 8, 0, 8, true, + false}, + {GpuEnumValue(BufferFormat::k16_16_16_16UInt), VK_FORMAT_R16G16B16A16_UINT, 8, 0, 8, true, + true}, + {GpuEnumValue(BufferFormat::k16_16_16_16SInt), VK_FORMAT_R16G16B16A16_SINT, 8, 0, 8, false, + false}, + {GpuEnumValue(BufferFormat::k16_16_16_16Float), VK_FORMAT_R16G16B16A16_SFLOAT, 8, 0, 8, true, + false}, + {GpuEnumValue(BufferFormat::k32_32_32UInt), VK_FORMAT_R32G32B32_UINT, 12, 0, 12, true, true}, + {GpuEnumValue(BufferFormat::k32_32_32SInt), VK_FORMAT_R32G32B32_SINT, 12, 0, 12, false, + false}, + {GpuEnumValue(BufferFormat::k32_32_32Float), VK_FORMAT_R32G32B32_SFLOAT, 12, 0, 12, true, + false}, + {GpuEnumValue(BufferFormat::k32_32_32_32UInt), VK_FORMAT_R32G32B32A32_UINT, 16, 0, 16, true, + true}, + {GpuEnumValue(BufferFormat::k32_32_32_32SInt), VK_FORMAT_R32G32B32A32_SINT, 16, 0, 16, false, + false}, + {GpuEnumValue(BufferFormat::k32_32_32_32Float), VK_FORMAT_R32G32B32A32_SFLOAT, 16, 0, 16, + true, false}, + {GpuEnumValue(BufferFormat::k8_8_8_8Srgb), VK_FORMAT_R8G8B8A8_SRGB, 4, 0, 4, true, false}, + {GpuEnumValue(BufferFormat::k9_9_9_5Float), VK_FORMAT_E5B9G9R9_UFLOAT_PACK32, 4, 0, 0, true, + false}, + {GpuEnumValue(BufferFormat::k5_6_5UNorm), VK_FORMAT_B5G6R5_UNORM_PACK16, 2, 0, 2, true, + false}, + {GpuEnumValue(BufferFormat::k5_5_5_1UNorm), VK_FORMAT_R5G5B5A1_UNORM_PACK16, 2, 0, 2, true, + false}, + {GpuEnumValue(BufferFormat::k4_4_4_4UNorm), VK_FORMAT_R4G4B4A4_UNORM_PACK16, 2, 0, 2, true, + false}, + {GpuEnumValue(BufferFormat::kFmask8_S4_F4), VK_FORMAT_R32_SFLOAT, 1, 0, 1, true, false}, + {GpuEnumValue(BufferFormat::kBc1UNorm), VK_FORMAT_BC1_RGBA_UNORM_BLOCK, 0, 8, 0, true, + false}, + {GpuEnumValue(BufferFormat::kBc1Srgb), VK_FORMAT_BC1_RGBA_SRGB_BLOCK, 0, 8, 0, true, false}, + {GpuEnumValue(BufferFormat::kBc2UNorm), VK_FORMAT_BC2_UNORM_BLOCK, 0, 16, 0, true, false}, + {GpuEnumValue(BufferFormat::kBc2Srgb), VK_FORMAT_BC2_SRGB_BLOCK, 0, 16, 0, true, false}, + {GpuEnumValue(BufferFormat::kBc3UNorm), VK_FORMAT_BC3_UNORM_BLOCK, 0, 16, 0, true, false}, + {GpuEnumValue(BufferFormat::kBc3Srgb), VK_FORMAT_BC3_SRGB_BLOCK, 0, 16, 0, true, false}, + {GpuEnumValue(BufferFormat::kBc4UNorm), VK_FORMAT_BC4_UNORM_BLOCK, 0, 8, 0, true, false}, + {GpuEnumValue(BufferFormat::kBc4SNorm), VK_FORMAT_BC4_SNORM_BLOCK, 0, 8, 0, true, false}, + {GpuEnumValue(BufferFormat::kBc5UNorm), VK_FORMAT_BC5_UNORM_BLOCK, 0, 16, 0, true, false}, + {GpuEnumValue(BufferFormat::kBc5SNorm), VK_FORMAT_BC5_SNORM_BLOCK, 0, 16, 0, true, false}, + {GpuEnumValue(BufferFormat::kBc6UFloat), VK_FORMAT_BC6H_UFLOAT_BLOCK, 0, 16, 0, true, false}, + {GpuEnumValue(BufferFormat::kBc6SFloat), VK_FORMAT_BC6H_SFLOAT_BLOCK, 0, 16, 0, true, false}, + {GpuEnumValue(BufferFormat::kBc7UNorm), VK_FORMAT_BC7_UNORM_BLOCK, 0, 16, 0, true, false}, + {GpuEnumValue(BufferFormat::kBc7Srgb), VK_FORMAT_BC7_SRGB_BLOCK, 0, 16, 0, true, false}, +}; + +constexpr auto MakeFormatInfoLookup() { + constexpr uint32_t kMaxFormat = GpuEnumValue(BufferFormat::kBc7Srgb); + std::array lookup {}; + for (const auto& info: kFormatInfo) { + lookup[info.format] = &info; + } + return lookup; +} + +constexpr auto kFormatInfoLookup = MakeFormatInfoLookup(); + +} // namespace + +std::span Formats() { + return kFormatInfo; +} + +const FormatInfo* FindFormatInfo(uint32_t format) { + return format < kFormatInfoLookup.size() ? kFormatInfoLookup[format] : nullptr; +} + +VkFormat SurfaceFormat(uint32_t format) { + const auto* info = FindFormatInfo(format); + return info != nullptr ? info->vk_format : VK_FORMAT_UNDEFINED; +} + +uint32_t NumBytesPerElement(uint32_t format) { + const auto* info = FindFormatInfo(format); + return info != nullptr ? info->bytes_per_element : 0; +} + +uint32_t BlockCompressedBytesPerBlock(uint32_t format) { + const auto* info = FindFormatInfo(format); + return info != nullptr ? info->block_compressed_bytes_per_block : 0; +} + +uint32_t RenderTargetBytesPerElement(uint32_t format) { + const auto* info = FindFormatInfo(format); + return info != nullptr ? info->render_target_bytes_per_element : 0; +} + +bool IsSupportedTextureFormat(uint32_t format) { + const auto* info = FindFormatInfo(format); + return info != nullptr && info->supported_sampled_texture_format; +} + +bool IsUintTextureFormat(uint32_t format) { + const auto* info = FindFormatInfo(format); + return info != nullptr && info->unsigned_integer_texture_format; +} + +bool IsFmaskTextureFormat(uint32_t format) { + return format == GpuEnumValue(BufferFormat::kFmask8_S4_F4); +} + +} // namespace Libs::Graphics::Prospero diff --git a/src/graphics/guest_gpu/gpu_format.h b/src/graphics/guest_gpu/gpu_format.h new file mode 100644 index 000000000..b90f5ccd1 --- /dev/null +++ b/src/graphics/guest_gpu/gpu_format.h @@ -0,0 +1,33 @@ +#ifndef EMULATOR_INCLUDE_EMULATOR_GRAPHICS_GUEST_GPU_GPU_FORMAT_H_ +#define EMULATOR_INCLUDE_EMULATOR_GRAPHICS_GUEST_GPU_GPU_FORMAT_H_ + +#include "common/common.h" + +#include +#include + +namespace Libs::Graphics::Prospero { + +struct FormatInfo { + uint32_t format = 0; + VkFormat vk_format = VK_FORMAT_UNDEFINED; + uint32_t bytes_per_element = 0; + uint32_t block_compressed_bytes_per_block = 0; + uint32_t render_target_bytes_per_element = 0; + bool supported_sampled_texture_format = false; + bool unsigned_integer_texture_format = false; +}; + +std::span Formats(); +const FormatInfo* FindFormatInfo(uint32_t format); +VkFormat SurfaceFormat(uint32_t format); +uint32_t NumBytesPerElement(uint32_t format); +uint32_t BlockCompressedBytesPerBlock(uint32_t format); +uint32_t RenderTargetBytesPerElement(uint32_t format); +bool IsSupportedTextureFormat(uint32_t format); +bool IsUintTextureFormat(uint32_t format); +bool IsFmaskTextureFormat(uint32_t format); + +} // namespace Libs::Graphics::Prospero + +#endif /* EMULATOR_INCLUDE_EMULATOR_GRAPHICS_GUEST_GPU_GPU_FORMAT_H_ */ diff --git a/src/graphics/guest_gpu/graphicsRun.cpp b/src/graphics/guest_gpu/graphicsRun.cpp new file mode 100644 index 000000000..581439792 --- /dev/null +++ b/src/graphics/guest_gpu/graphicsRun.cpp @@ -0,0 +1,1968 @@ +#include "graphics/guest_gpu/graphicsRun.h" + +#include "common/assert.h" +#include "common/emulatorConfig.h" +#include "common/logging/log.h" +#include "common/profiler.h" +#include "common/stringUtils.h" +#include "common/threads.h" +#include "graphics/asyncJob.h" +#include "graphics/guest_gpu/command_processor/commandProcessor.h" +#include "graphics/guest_gpu/command_processor/pm4Dispatch.h" +#include "graphics/guest_gpu/hardwareContext.h" +#include "graphics/guest_gpu/pm4.h" +#include "graphics/host_gpu/graphicContext.h" +#include "graphics/host_gpu/objects/label.h" +#include "graphics/host_gpu/renderer/render.h" +#include "graphics/host_gpu/renderer/renderContext.h" +#include "graphics/presentation/videoOut.h" +#include "graphics/presentation/window.h" +#include "graphics/shader/shader.h" +#include "kernel/memory.h" +#include "libs/agc.h" +#include "libs/errno.h" + +#include +#include +#include +#include +#include +#include +#include + +namespace Libs::Graphics { + +enum class CommandProcessorGuestAccess : uint32_t { + None, + WaitPoll, + DirectFence, +}; + +static thread_local CommandProcessor* g_current_run_cp = nullptr; +static thread_local uint32_t g_submission_pause_depth = 0; +static thread_local bool g_gpu_mutex_owned = false; +static thread_local bool g_cp_readback_owned = false; +static thread_local bool g_cp_readback_preowned = false; +static thread_local CommandProcessorGuestAccess g_guest_memory_access = + CommandProcessorGuestAccess::None; + +class GpuMutexLock final { +public: + explicit GpuMutexLock(Common::Mutex& mutex): m_mutex(mutex) { + if (g_gpu_mutex_owned) { + EXIT("recursive GPU mutex acquisition\n"); + } + g_gpu_mutex_owned = true; + m_mutex.Lock(); + } + ~GpuMutexLock() { + if (!g_gpu_mutex_owned) { + EXIT("invalid GPU mutex release\n"); + } + m_mutex.Unlock(); + g_gpu_mutex_owned = false; + } + +private: + Common::Mutex& m_mutex; +}; + +class CommandProcessorGuestAccessScope final { +public: + CommandProcessorGuestAccessScope(CommandProcessorGuestAccess access, uint64_t vaddr, + uint64_t size) { + if (access == CommandProcessorGuestAccess::None || + g_guest_memory_access != CommandProcessorGuestAccess::None || + g_current_run_cp == nullptr) { + EXIT("invalid nested command-processor guest-memory access\n"); + } + if (g_render_ctx == nullptr) { + EXIT("command-processor guest-memory access has no render context\n"); + } + m_cp = g_current_run_cp; + if (access == CommandProcessorGuestAccess::WaitPoll) { + // Preserve serialized scheduler semantics: a poll owns the command processor before it + // enters the shared buffer/texture transaction, and releases both before yielding. + m_cp->m_mutex.Lock(); + m_cp_mutex_owned = true; + } + m_resource_lock = + g_render_ctx->GetGpuResources()->LockCommandProcessorGuestAccess(vaddr, size); + g_guest_memory_access = access; + } + ~CommandProcessorGuestAccessScope() { + if (g_guest_memory_access == CommandProcessorGuestAccess::None || + !m_resource_lock.owns_lock() || m_cp == nullptr) { + EXIT("command-processor guest-memory access scope is not active\n"); + } + g_guest_memory_access = CommandProcessorGuestAccess::None; + m_resource_lock.unlock(); + if (m_cp_mutex_owned) { + m_cp->m_mutex.Unlock(); + } + } + +private: + CommandProcessor* m_cp = nullptr; + bool m_cp_mutex_owned = false; + std::unique_lock m_resource_lock; + KYTY_CLASS_NO_COPY(CommandProcessorGuestAccessScope); +}; + +struct OwnedCmdBuffer { + std::vector storage; + uint32_t* data = nullptr; + uint32_t num_dw = 0; + + OwnedCmdBuffer() = default; + + OwnedCmdBuffer(const OwnedCmdBuffer& other): storage(other.storage), num_dw(other.num_dw) { + data = (storage.empty() ? other.data : storage.data()); + } + + OwnedCmdBuffer& operator=(const OwnedCmdBuffer& other) { + if (this != &other) { + storage = other.storage; + num_dw = other.num_dw; + data = (storage.empty() ? other.data : storage.data()); + } + return *this; + } + + OwnedCmdBuffer(OwnedCmdBuffer&& other) noexcept + : storage(std::move(other.storage)), data(other.data), num_dw(other.num_dw) { + data = (storage.empty() ? other.data : storage.data()); + } + + OwnedCmdBuffer& operator=(OwnedCmdBuffer&& other) noexcept { + if (this != &other) { + storage = std::move(other.storage); + num_dw = other.num_dw; + data = (storage.empty() ? other.data : storage.data()); + } + return *this; + } + + void Assign(const uint32_t* src, uint32_t count) { + num_dw = count; + if (src != nullptr && count != 0) { + storage.assign(src, src + count); + data = storage.data(); + } else { + storage.clear(); + data = nullptr; + } + } +}; + +class GraphicsRing { +public: + GraphicsRing(): m_draw_job("Thread_Gfx_Draw"), m_constant_job("Thread_Gfx_Const") { + EXIT_NOT_IMPLEMENTED(!Common::Thread::IsMainThread()); + } + ~GraphicsRing() { KYTY_NOT_IMPLEMENTED; } + + KYTY_CLASS_NO_COPY(GraphicsRing); + + void Submit(uint32_t* cmd_draw_buffer, uint32_t num_draw_dw, uint32_t* cmd_const_buffer, + uint32_t num_const_dw, int handle, int index, int flip_mode, int64_t flip_arg, + bool trigger_agc_interrupt_on_done); + void Done(); + void WaitForIdle(); + bool IsIdle(); + + void SetCp(CommandProcessor* cp) { + m_cp = cp; + Start(); + } + +private: + void Start() { + Common::Thread t(ThreadBatchRun, this); + t.Detach(); + } + + struct CmdBatch { + OwnedCmdBuffer draw_buffer; + OwnedCmdBuffer const_buffer; + + CommandProcessor::FlipInfo flip; + bool trigger_agc_interrupt_on_done = false; + }; + + static void ThreadBatchRun(void* data); + + CmdBatch GetCmdBatch(); + + Common::Mutex m_mutex; + Common::CondVar m_cond_var; + Common::CondVar m_idle_cond_var; + std::list m_cmd_batches; + bool m_done = true; + bool m_idle = true; + + AsyncJob m_draw_job; + AsyncJob m_constant_job; + + CommandProcessor* m_cp = nullptr; +}; + +class ComputeRing { +public: + ComputeRing() = default; + ~ComputeRing() { KYTY_NOT_IMPLEMENTED; } + + KYTY_CLASS_NO_COPY(ComputeRing); + + void Submit(uint32_t* cmd_buffer, uint32_t num_dw, bool trigger_agc_interrupt_on_done); + void Done(); + void WaitForIdle(); + bool IsIdle(); + + void SetCp(CommandProcessor* cp) { + m_cp = cp; + Start(); + } + + [[nodiscard]] int GetQueueId() const { return m_queue_id; } + void SetQueueId(int id) { m_queue_id = id; } + +private: + void Start() { + Common::Thread t(ThreadRun, this); + t.Detach(); + } + + static void ThreadRun(void* data); + + Common::Mutex m_mutex; + Common::CondVar m_cond_var; + Common::CondVar m_idle_cond_var; + bool m_done = true; + bool m_idle = true; + + CommandProcessor* m_cp = nullptr; + int m_queue_id = -1; + + struct DirectBatch { + OwnedCmdBuffer buffer; + bool trigger_agc_interrupt_on_done = false; + }; + + std::list m_direct_batches; +}; + +class Gpu { +public: + Gpu() { + EXIT_NOT_IMPLEMENTED(!Common::Thread::IsMainThread()); + Init(); + } + ~Gpu() { KYTY_NOT_IMPLEMENTED; } + + KYTY_CLASS_NO_COPY(Gpu); + + void Submit(uint32_t* cmd_draw_buffer, uint32_t num_draw_dw, uint32_t* cmd_const_buffer, + uint32_t num_const_dw, bool trigger_agc_interrupt_on_done); + void SubmitCompute(uint32_t queue, uint32_t* cmd_buffer, uint32_t num_dw, + bool trigger_agc_interrupt_on_done); + void Done(); + void PauseSubmissions(); + void ResumeSubmissions(); + int GetFrameNum(); + +private: + void Init(); + void WaitLocked(); + + ComputeRing* GetRing(uint32_t ring_id); + + Common::Mutex m_mutex; + + CommandProcessor* m_gfx_cp = nullptr; + GraphicsRing* m_gfx_ring = nullptr; + + CommandProcessor* m_compute_cp[8] = {}; + ComputeRing* m_compute_ring[64] = {}; + + std::atomic_int m_done_num = 0; +}; + +static Gpu* g_gpu = nullptr; + +static bool GraphicsRunDebugDumpEnabled() { + return Config::GraphicsDebugDumpEnabled() && + Config::GetPrintfDirection() != Config::OutputDirection::Silent; +} + +void GraphicsRunInit() { + EXIT_NOT_IMPLEMENTED(!Common::Thread::IsMainThread()); + + EXIT_IF(g_gpu != nullptr); + + GraphicsInitJmpTables(); + + g_gpu = new Gpu; +} + +void Gpu::Submit(uint32_t* cmd_draw_buffer, uint32_t num_draw_dw, uint32_t* cmd_const_buffer, + uint32_t num_const_dw, bool trigger_agc_interrupt_on_done) { + GpuMutexLock lock(m_mutex); + + m_gfx_ring->Submit(cmd_draw_buffer, num_draw_dw, cmd_const_buffer, num_const_dw, 0, 0, 0, 0, + trigger_agc_interrupt_on_done); +} + +void Gpu::SubmitCompute(uint32_t queue, uint32_t* cmd_buffer, uint32_t num_dw, + bool trigger_agc_interrupt_on_done) { + GpuMutexLock lock(m_mutex); + + constexpr uint32_t compute_queue_base = 0x20u; + constexpr uint32_t compute_queue_num = 7u * 8u; + EXIT_NOT_IMPLEMENTED(queue < compute_queue_base || + queue >= compute_queue_base + compute_queue_num); + + uint32_t compute_queue = queue - compute_queue_base; + uint32_t pipe_id = (compute_queue >> 3u) & 0x7u; + uint32_t queue_id = compute_queue & 0x7u; + EXIT_NOT_IMPLEMENTED(pipe_id >= 7u); + EXIT_NOT_IMPLEMENTED(queue_id >= 8u); + + uint32_t ring_id = compute_queue + 1u; + + auto* ring = GetRing(ring_id); + + ring->Submit(cmd_buffer, num_dw, trigger_agc_interrupt_on_done); +} + +void Gpu::Done() { + GraphicsRing* gfx_ring = nullptr; + CommandProcessor* gfx_cp = nullptr; + ComputeRing* compute_rings[64] {}; + CommandProcessor* compute_cps[8] {}; + + { + GpuMutexLock lock(m_mutex); + + gfx_ring = m_gfx_ring; + gfx_cp = m_gfx_cp; + + std::copy(std::begin(m_compute_ring), std::end(m_compute_ring), std::begin(compute_rings)); + std::copy(std::begin(m_compute_cp), std::end(m_compute_cp), std::begin(compute_cps)); + + m_done_num++; + } + + if (gfx_ring != nullptr) { + gfx_ring->Done(); + } + for (auto& cr: compute_rings) { + if (cr != nullptr) { + cr->Done(); + } + } + if (gfx_ring != nullptr) { + gfx_ring->WaitForIdle(); + } + if (gfx_cp != nullptr) { + gfx_cp->BufferWait(); + } + for (auto& cr: compute_rings) { + if (cr != nullptr) { + cr->WaitForIdle(); + } + } + for (auto& cp: compute_cps) { + if (cp != nullptr) { + cp->BufferWait(); + } + } +} + +int Gpu::GetFrameNum() { + // Common::LockGuard lock(m_mutex); + + return m_done_num; +} + +void Gpu::WaitLocked() { + GraphicsRing* gfx_ring = nullptr; + CommandProcessor* gfx_cp = nullptr; + ComputeRing* compute_rings[64] {}; + CommandProcessor* compute_cps[8] {}; + + gfx_ring = m_gfx_ring; + gfx_cp = m_gfx_cp; + std::copy(std::begin(m_compute_ring), std::end(m_compute_ring), std::begin(compute_rings)); + std::copy(std::begin(m_compute_cp), std::end(m_compute_cp), std::begin(compute_cps)); + + if (gfx_ring != nullptr) { + gfx_ring->WaitForIdle(); + } + if (gfx_cp != nullptr) { + gfx_cp->BufferWait(); + } + for (auto& cr: compute_rings) { + if (cr != nullptr) { + cr->WaitForIdle(); + } + } + for (auto& cp: compute_cps) { + if (cp != nullptr) { + cp->BufferWait(); + } + } +} + +void Gpu::Init() { + EXIT_IF(m_gfx_cp != nullptr); + EXIT_IF(m_gfx_ring != nullptr); + + m_gfx_cp = new CommandProcessor; + m_gfx_ring = new GraphicsRing; + m_gfx_cp->SetQueue(GraphicContext::QUEUE_GFX); + m_gfx_ring->SetCp(m_gfx_cp); + + EXIT_IF(GraphicContext::QUEUE_COMPUTE_NUM < 8); +} + +ComputeRing* Gpu::GetRing(uint32_t ring_id) { + int v = static_cast(ring_id - 1); + int pipe_id = v / 8; + int queue_id = v % 8; + + if (m_compute_cp[pipe_id] == nullptr) { + m_compute_cp[pipe_id] = new CommandProcessor; + m_compute_cp[pipe_id]->SetQueue(GraphicContext::QUEUE_COMPUTE_START + pipe_id); + } + + if (m_compute_ring[v] == nullptr) { + m_compute_ring[v] = new ComputeRing; + m_compute_ring[v]->SetQueueId(queue_id); + m_compute_ring[v]->SetCp(m_compute_cp[pipe_id]); + } + + return m_compute_ring[v]; +} + +void CommandProcessor::Reset() { + BufferWait(); + + Common::LockGuard lock(m_mutex); + + GraphicsRenderDeleteBuffers(); + + m_sh_ctx.Reset(); + m_ucfg.Reset(); + m_ctx.Reset(); + m_index_type_and_size = 0; + m_index_buffer_size = 0; + m_user_data_marker = HW::UserSgprType::Unknown; + m_draw_indirect_args_base_addr = 0; + m_dispatch_indirect_args_base_addr = 0; + + std::memset(m_const_ram, 0, sizeof(m_const_ram)); +} + +void CommandProcessor::BufferInit() { + Common::LockGuard lock(m_mutex); + m_scheduler.Init(); +} + +void CommandProcessor::BufferFlush() { + Common::LockGuard lock(m_mutex); + m_scheduler.Flush(); +} + +void CommandProcessor::BufferFlushAndWait() { + CommandBuffer* submitted = nullptr; + + { + Common::LockGuard lock(m_mutex); + submitted = m_scheduler.FlushAndGetSubmitted(); + } + + submitted->WaitForFence(); +} + +void CommandProcessor::BufferWait() { + if (g_current_run_cp != this) { + m_run_mutex.Lock(); + BufferInit(); + + std::array buffers {}; + { + Common::LockGuard lock(m_mutex); + m_scheduler.CopyBuffers(&buffers); + } + + for (auto* buf: buffers) { + buf->WaitForFenceAndReset(); + } + + m_run_mutex.Unlock(); + return; + } + + BufferInit(); + + Common::LockGuard lock(m_mutex); + m_scheduler.WaitAll(); +} + +void CommandProcessor::ResetDeCe() { + m_de_counter.mutex.Lock(); + m_de_counter.value = 0; + m_de_counter.cond_var.Signal(); + m_de_counter.mutex.Unlock(); + m_ce_counter.mutex.Lock(); + m_ce_counter.value = 0; + m_ce_counter.cond_var.Signal(); + m_ce_counter.mutex.Unlock(); +} + +void CommandProcessor::WaitCe() { + m_de_counter.mutex.Lock(); + auto de_value = m_de_counter.value; + m_de_counter.mutex.Unlock(); + + m_ce_counter.mutex.Lock(); + while (!(m_ce_counter.value > de_value)) { + m_ce_counter.cond_var.Wait(&m_ce_counter.mutex); + } + m_ce_counter.mutex.Unlock(); +} + +void CommandProcessor::WaitDeDiff(uint32_t diff) { + m_ce_counter.mutex.Lock(); + auto ce_value = m_ce_counter.value; + m_ce_counter.mutex.Unlock(); + + m_de_counter.mutex.Lock(); + while (!(ce_value - m_de_counter.value < diff)) { + m_de_counter.cond_var.Wait(&m_de_counter.mutex); + } + m_de_counter.mutex.Unlock(); +} + +void CommandProcessor::IncremenetDe() { + BufferFlush(); + BufferWait(); + + m_de_counter.mutex.Lock(); + m_de_counter.value++; + m_de_counter.cond_var.Signal(); + m_de_counter.mutex.Unlock(); +} + +void CommandProcessor::IncremenetCe() { + m_ce_counter.mutex.Lock(); + m_ce_counter.value++; + m_ce_counter.cond_var.Signal(); + m_ce_counter.mutex.Unlock(); +} + +void CommandProcessor::WriteConstRam(uint32_t offset, const uint32_t* src, uint32_t dw_num) { + Common::LockGuard lock(m_mutex); + + memcpy(m_const_ram + offset / 4, src, static_cast(dw_num) * 4); +} + +void CommandProcessor::DumpConstRam(uint32_t* dst, uint32_t offset, uint32_t dw_num) { + Common::LockGuard lock(m_mutex); + + memcpy(dst, m_const_ram + offset / 4, static_cast(dw_num) * 4); +} + +bool TestWaitRegMemValue(uint64_t value, uint64_t ref, uint64_t mask, uint32_t func) { + switch (func) { + case 0: return true; + case 1: return (value & mask) < ref; + case 2: return (value & mask) <= ref; + case 3: return (value & mask) == ref; + case 4: return (value & mask) != ref; + case 5: return (value & mask) >= ref; + case 6: return (value & mask) > ref; + default: EXIT("unknown wait compare function: %" PRIu32 "\n", func); + } + + return false; +} + +template +static T LoadGuestWaitValue(const T* addr) { + CommandProcessorGuestAccessScope guest_access(CommandProcessorGuestAccess::WaitPoll, + reinterpret_cast(addr), sizeof(*addr)); + return *addr; +} + +static void YieldCommandProcessorWait(uint32_t poll_interval_cycles) noexcept { + // PS5 specifies GPU poll cycles, not host microseconds. Yield at the scheduler boundary now; + // a future resumable guest GPU scheduler can replace this one function without touching polling + // or BufferCache coherence. + (void)poll_interval_cycles; + std::this_thread::yield(); +} + +template +void CommandProcessor::WaitRegMem(uint32_t func, const T* addr, T ref, T mask, uint32_t poll, + uint32_t wait_op) { + EXIT_IF(addr == nullptr); + if ((wait_op & ~1u) != 0) { + EXIT("unsupported wait_reg_mem operation: 0x%08" PRIx32 "\n", wait_op); + } + + const auto addr_value = reinterpret_cast(addr); + const auto log_width = static_cast(sizeof(T) * 2u); + const auto bits = static_cast(sizeof(T) * 8u); + + if (!TestWaitRegMemValue(LoadGuestWaitValue(addr), ref, mask, func)) { + SynchronizeGpuRange(addr_value, sizeof(*addr)); + } + uint64_t spin_count = 0; + for (;;) { + const auto value = LoadGuestWaitValue(addr); + if (TestWaitRegMemValue(value, ref, mask, func)) { + break; + } + if ((++spin_count % 100000u) == 0) { + LOGF("\t wait_reg_mem%u still waiting: addr = 0x%016" PRIx64 ", value = 0x%0*" PRIx64 + ", ref = 0x%0*" PRIx64 ", mask = 0x%0*" PRIx64 ", func = %" PRIu32 "\n", + bits, addr_value, log_width, static_cast(value), log_width, + static_cast(ref), log_width, static_cast(mask), func); + } + YieldCommandProcessorWait(poll); + } +} + +template void CommandProcessor::WaitRegMem(uint32_t, const uint32_t*, uint32_t, uint32_t, + uint32_t, uint32_t); +template void CommandProcessor::WaitRegMem(uint32_t, const uint64_t*, uint64_t, uint64_t, + uint32_t, uint32_t); + +void CommandProcessor::WriteData(uint32_t* dst, const uint32_t* src, uint32_t dw_num, + uint32_t write_control) { + Common::LockGuard lock(m_mutex); + + const uint32_t dst_sel = ((write_control >> 30u) & 0x1u) | ((write_control >> 7u) & 0x1eu); + const uint32_t cache_policy = (write_control >> 25u) & 0x3u; + const uint32_t increment = (write_control >> 16u) & 0x1u; + const uint32_t write_confirm = (write_control >> 20u) & 0x1u; + + if (dst_sel != 0 && dst_sel != 2 && dst_sel != 4 && dst_sel != 5) { + LOGF("\t warning: unexpected write_data dst_sel %" PRIu32 "\n", dst_sel); + } + EXIT_NOT_IMPLEMENTED(increment != 0); + + if (cache_policy > 3 || write_confirm > 1) { + LOGF("\t warning: unexpected write_data control 0x%08" PRIx32 "\n", write_control); + } + + memcpy(dst, src, static_cast(dw_num) * 4); +} + +void CommandProcessor::DmaData(uint8_t engine, uint8_t dst_sel, uint8_t dst_cache_policy, + uint64_t dst_address_or_offset, uint8_t src_sel, + uint8_t src_cache_policy, + uint64_t src_address_or_offset_or_immediate, uint32_t num_bytes, + uint8_t wait_for_previous, uint8_t write_confirm, + uint8_t block_engine) { + Common::LockGuard lock(m_mutex); + + EXIT_NOT_IMPLEMENTED(engine > 1); + if (num_bytes == 0) { + return; + } + EXIT_NOT_IMPLEMENTED((num_bytes & 3u) != 0); + EXIT_NOT_IMPLEMENTED(dst_cache_policy > 3); + EXIT_NOT_IMPLEMENTED(src_cache_policy > 3); + EXIT_NOT_IMPLEMENTED(wait_for_previous > 1); + EXIT_NOT_IMPLEMENTED(write_confirm > 1); + EXIT_NOT_IMPLEMENTED(block_engine > 1); + const bool dst_memory = dst_sel == 0 || dst_sel == 3; + const bool src_memory = src_sel == 0 || src_sel == 3; + if (!dst_memory) { + EXIT("unsupported dmaData destination selector 0x%02" PRIx8 "\n", dst_sel); + } + if (src_sel == 2) { + GetGpuResources()->FillBuffer( + CurrentBuffer(), dst_address_or_offset, num_bytes, + static_cast(src_address_or_offset_or_immediate & 0xffffffffu)); + return; + } + if (src_memory) { + GetGpuResources()->CopyBuffer(CurrentBuffer(), dst_address_or_offset, + src_address_or_offset_or_immediate, num_bytes); + return; + } + EXIT("unsupported dmaData source selector 0x%02" PRIx8 "\n", src_sel); +} + +void GraphicsRing::Submit(uint32_t* cmd_draw_buffer, uint32_t num_draw_dw, + uint32_t* cmd_const_buffer, uint32_t num_const_dw, int handle, int index, + int flip_mode, int64_t flip_arg, bool trigger_agc_interrupt_on_done) { + EXIT_IF(m_cp == nullptr); + + Common::LockGuard lock(m_mutex); + + WindowWaitForGraphicInitialized(); + GraphicsRenderCreateContext(); + + if (m_done) { + while (!m_idle) { + m_idle_cond_var.Wait(&m_mutex); + } + m_done = false; + + m_cp->Reset(); + } + + auto& buf = m_cmd_batches.emplace_back(); + buf.draw_buffer.Assign(cmd_draw_buffer, num_draw_dw); + buf.const_buffer.Assign(cmd_const_buffer, num_const_dw); + buf.flip.handle = handle; + buf.flip.index = index; + buf.flip.flip_mode = flip_mode; + buf.flip.flip_arg = flip_arg; + buf.trigger_agc_interrupt_on_done = trigger_agc_interrupt_on_done; + + m_idle = false; + m_cond_var.Signal(); +} + +void GraphicsRing::Done() { + Common::LockGuard lock(m_mutex); + if (m_done) { + while (!m_idle) { + m_idle_cond_var.Wait(&m_mutex); + } + } + m_done = true; +} + +void GraphicsRing::WaitForIdle() { + Common::LockGuard lock(m_mutex); + while (!m_idle) { + m_idle_cond_var.Wait(&m_mutex); + } +} + +bool GraphicsRing::IsIdle() { + Common::LockGuard lock(m_mutex); + return m_idle; +} + +GraphicsRing::CmdBatch GraphicsRing::GetCmdBatch() { + Common::LockGuard lock(m_mutex); + + while (m_cmd_batches.empty()) { + m_idle = true; + m_idle_cond_var.Signal(); + + m_cond_var.Wait(&m_mutex); + } + + m_idle = false; + + CmdBatch buf = std::move(m_cmd_batches.front()); + m_cmd_batches.pop_front(); + + return buf; +} + +void GraphicsRing::ThreadBatchRun(void* data) { + EXIT_IF(data == nullptr); + + static std::atomic_uint64_t seq = 0; + + auto* ring = static_cast(data); + auto* cp = ring->m_cp; + + EXIT_IF(ring == nullptr); + EXIT_IF(cp == nullptr); + + for (;;) { + CmdBatch buf = ring->GetCmdBatch(); + + cp->RunLock(); + { + cp->BufferInit(); + cp->ResetDeCe(); + cp->SetFlip(buf.flip); + cp->SetSubmitId(++seq); + + ring->m_draw_job.Execute( + [cp, buf] { cp->Run(buf.draw_buffer.data, buf.draw_buffer.num_dw); }); + ring->m_constant_job.Execute( + [cp, buf] { cp->Run(buf.const_buffer.data, buf.const_buffer.num_dw); }); + ring->m_draw_job.Wait(); + ring->m_constant_job.Wait(); + + cp->BufferFlush(); + if (buf.trigger_agc_interrupt_on_done) { + GraphicsRenderTriggerEopEvent(0); + } + } + cp->RunUnlock(); + } +} + +void Gpu::PauseSubmissions() { + if (g_gpu_mutex_owned) { + EXIT("GPU submissions are already paused by this thread\n"); + } + g_gpu_mutex_owned = true; + m_mutex.Lock(); + WaitLocked(); + LabelDrain(); +} + +void Gpu::ResumeSubmissions() { + if (!g_gpu_mutex_owned) { + EXIT("GPU submissions resumed without an active pause\n"); + } + m_mutex.Unlock(); + g_gpu_mutex_owned = false; +} + +void ComputeRing::ThreadRun(void* data) { + EXIT_IF(data == nullptr); + + auto* ring = static_cast(data); + auto* cp = ring->m_cp; + + EXIT_IF(ring == nullptr); + EXIT_IF(cp == nullptr); + + KYTY_PROFILER_THREAD("Thread_Compute"); + + ring->m_mutex.Lock(); + + for (;;) { + while (ring->m_direct_batches.empty()) { + ring->m_idle = true; + ring->m_idle_cond_var.Signal(); + ring->m_cond_var.Wait(&ring->m_mutex); + } + + ring->m_idle = false; + + uint32_t* buffer = nullptr; + uint32_t num_dw = 0; + bool trigger_agc_interrupt_on_done = false; + DirectBatch direct_batch; + + direct_batch = std::move(ring->m_direct_batches.front()); + ring->m_direct_batches.pop_front(); + buffer = direct_batch.buffer.data; + num_dw = direct_batch.buffer.num_dw; + trigger_agc_interrupt_on_done = direct_batch.trigger_agc_interrupt_on_done; + + static std::atomic compute_batch_log_count {0}; + if (num_dw <= 128 && buffer != nullptr && compute_batch_log_count.fetch_add(1) < 32) { + LOGF("compute direct batch: queue=%d, data=0x%016" PRIx64 ", num_dw=%" PRIu32 "\n", + ring->m_queue_id, reinterpret_cast(buffer), num_dw); + for (uint32_t i = 0; i < std::min(num_dw, 16); i++) { + LOGF("\t compute[%02" PRIu32 "] = 0x%08" PRIx32 "\n", i, buffer[i]); + } + } + + ring->m_mutex.Unlock(); + + cp->RunLock(); + { + cp->BufferInit(); + cp->ResetDeCe(); + + GraphicsDbgDumpDcb("cc", num_dw, buffer); + + cp->Run(buffer, num_dw); + + cp->BufferFlush(); + if (trigger_agc_interrupt_on_done) { + GraphicsRenderTriggerAgcUserInterrupt(); + } + } + cp->RunUnlock(); + + ring->m_mutex.Lock(); + } +} + +void ComputeRing::Submit(uint32_t* cmd_buffer, uint32_t num_dw, + bool trigger_agc_interrupt_on_done) { + Common::LockGuard lock(m_mutex); + + EXIT_IF(cmd_buffer == nullptr); + EXIT_IF(num_dw == 0); + + WindowWaitForGraphicInitialized(); + GraphicsRenderCreateContext(); + + if (m_done) { + m_done = false; + } + + DirectBatch batch {}; + batch.buffer.Assign(cmd_buffer, num_dw); + batch.trigger_agc_interrupt_on_done = trigger_agc_interrupt_on_done; + m_direct_batches.push_back(std::move(batch)); + + m_idle = false; + m_cond_var.Signal(); +} + +void ComputeRing::Done() { + Common::LockGuard lock(m_mutex); + m_done = true; +} + +void ComputeRing::WaitForIdle() { + Common::LockGuard lock(m_mutex); + while (!m_idle) { + m_idle_cond_var.Wait(&m_mutex); + } +} + +bool ComputeRing::IsIdle() { + Common::LockGuard lock(m_mutex); + return m_idle; +} + +void CommandProcessor::Run(uint32_t* data, uint32_t num_dw) { + KYTY_PROFILER_BLOCK("CommandProcessor::Run"); + + struct RunScope { + explicit RunScope(CommandProcessor* cp): prev(g_current_run_cp) { g_current_run_cp = cp; } + ~RunScope() { g_current_run_cp = prev; } + + CommandProcessor* prev; + } run_scope(this); + + auto* cmd = data; + auto dw = num_dw; + for (;;) { + if (dw == 0) { + break; + } + + EXIT_NOT_IMPLEMENTED(dw > num_dw); + + auto cmd_id = *cmd++; + + if (cmd_id == 0x80000000u) { + dw--; + continue; + } + + EXIT_NOT_IMPLEMENTED(dw < 2); + + auto op = (cmd_id >> 8u) & 0xffu; + if (GraphicsRunDebugDumpEnabled()) { + LOGF("CP packet: offset=0x%05" PRIx32 " cmd_id=0x%08" PRIx32 " op=0x%02" PRIx32 + " len=%" PRIu32 "\n", + num_dw - dw, cmd_id, op, KYTY_PM4_LEN(cmd_id)); + } + + if ((cmd_id & 1u) != 0 && ShouldSkipPredicatedPackets()) { + auto packet_len = KYTY_PM4_LEN(cmd_id); + EXIT_NOT_IMPLEMENTED(packet_len == 0 || packet_len > dw); + static std::atomic skip_log_count {0}; + if (skip_log_count.fetch_add(1) < 2048) { + LOGF("\t predicated skip: op=0x%02" PRIx32 ", r=0x%02" PRIx32 ", len=%" PRIu32 + ", packet=0x%016" PRIx64 ", cmd_id=0x%08" PRIx32 "\n", + op, KYTY_PM4_R(cmd_id), packet_len, reinterpret_cast(cmd - 1), + cmd_id); + } + if (op == Pm4::IT_NOP && KYTY_PM4_R(cmd_id) == Pm4::R_RELEASE_MEM && packet_len >= 7) { + static std::atomic log_count {0}; + if (log_count.fetch_add(1) < 128) { + const auto dst = cmd[2] | (static_cast(cmd[3]) << 32u); + const auto val = cmd[4] | (static_cast(cmd[5]) << 32u); + LOGF("\t predicated skip: R_RELEASE_MEM dst=0x%016" PRIx64 + ", value=0x%016" PRIx64 ", action=0x%08" PRIx32 + ", gcr/data/int=0x%08" PRIx32 "\n", + dst, val, cmd[0], cmd[1]); + } + } + cmd += packet_len - 1u; + dw -= packet_len; + continue; + } + + auto pfunc = g_cp_op_func[op]; + + if (pfunc == nullptr) { + const auto offset = num_dw - dw; + LOGF("unknown PM4 packet: data=0x%016" PRIx64 ", num_dw=%" PRIu32 + ", offset=0x%05" PRIx32 ", current=0x%016" PRIx64 "\n", + reinterpret_cast(data), num_dw, offset, + reinterpret_cast(cmd - 1)); + const auto dump_begin = (offset > 8 ? offset - 8 : 0); + const auto dump_end = std::min(num_dw, offset + 16); + for (uint32_t i = dump_begin; i < dump_end; i++) { + LOGF("\t%05" PRIx32 "%s %08" PRIx32 "\n", i, (i == offset ? ":" : " "), data[i]); + } + EXIT("unknown op\n\t%05" PRIx32 ":\n\tcmd_id = %08" PRIx32 "\n", num_dw - dw, cmd_id); + } + + auto s = pfunc(this, cmd_id & ~1u, cmd, dw, num_dw); + + // LOGF("\t %05" PRIx32 ": %u\n", num_dw - dw, s); + + cmd += s; + dw -= s + 1; + } +} + +void CommandProcessor::SetIndexType(uint32_t index_type_and_size) { + Common::LockGuard lock(m_mutex); + + m_index_type_and_size = index_type_and_size & 0x3u; +} + +void CommandProcessor::SetIndexBaseAddress(uint64_t index_base_addr) { + Common::LockGuard lock(m_mutex); + + m_index_base_addr = index_base_addr; +} + +void CommandProcessor::SetIndexBufferSize(uint32_t index_buffer_size) { + Common::LockGuard lock(m_mutex); + + m_index_buffer_size = index_buffer_size; +} + +void CommandProcessor::SetDrawIndirectArgsBaseAddress(uint64_t draw_indirect_args_base_addr) { + Common::LockGuard lock(m_mutex); + + m_draw_indirect_args_base_addr = draw_indirect_args_base_addr; +} + +void CommandProcessor::SetDispatchIndirectArgsBaseAddress( + uint64_t dispatch_indirect_args_base_addr) { + Common::LockGuard lock(m_mutex); + + m_dispatch_indirect_args_base_addr = dispatch_indirect_args_base_addr; +} + +void CommandProcessor::SetNumInstances(uint32_t num_instances) { + Common::LockGuard lock(m_mutex); + + if (num_instances == 0) { + num_instances = 1; + } + + m_num_instances = num_instances; +} + +void CommandProcessor::SetPredication(uint32_t condition, uint32_t op, uint32_t wait_op, + const volatile void* address, uint32_t count_in_dwords) { + if (wait_op != 0) { + BufferFlushAndWait(); + } + + Common::LockGuard lock(m_mutex); + + (void)count_in_dwords; + + switch (op) { + case 0x00: { + m_predicate_skip = false; + } break; + case 0x03: { + EXIT_NOT_IMPLEMENTED(address == nullptr); + + auto value = *reinterpret_cast(address); + + switch (condition) { + case 0x00: m_predicate_skip = (value != 0); break; + case 0x01: m_predicate_skip = (value == 0); break; + default: EXIT("unknown predication condition: 0x%08" PRIx32 "\n", condition); + } + static std::atomic log_count {0}; + if (log_count.fetch_add(1) < 128) { + LOGF("\t bool predication: addr=0x%016" PRIx64 ", value=0x%016" PRIx64 + ", condition=%" PRIu32 ", skip=%u, wait_op=%" PRIu32 "\n", + reinterpret_cast(address), value, condition, + m_predicate_skip ? 1u : 0u, wait_op); + } + } break; + default: EXIT("unknown predication op: 0x%08" PRIx32 "\n", op); + } +} + +void CommandProcessor::DrawIndex(uint32_t index_count, const void* index_addr, uint32_t flags, + uint32_t type, uint32_t instance_count, const void* object_ids, + uint32_t render_target_slice_offset, int32_t vertex_offset_add, + uint32_t first_instance) { + Common::LockGuard lock(m_mutex); + + CheckBuffer(); + + if (instance_count == 0) { + instance_count = m_num_instances; + } + if (object_ids != nullptr) { + LOGF("\t draw indexed multi-instanced objectIds = 0x%016" PRIx64 "\n", + reinterpret_cast(object_ids)); + } + if (render_target_slice_offset != 0) { + LOGF("\t draw render target slice offset = %" PRIu32 "\n", render_target_slice_offset); + } + if (vertex_offset_add != 0 || first_instance != 0) { + LOGF("\t draw indexed offsets: vertex_offset_add = %" PRId32 ", first_instance = %" PRIu32 + "\n", + vertex_offset_add, first_instance); + } + const auto frame_num = GraphicsRunGetFrameNum(); + if (frame_num >= 480) { + static std::atomic log_count {0}; + if (log_count.fetch_add(1, std::memory_order_relaxed) < 512) { + const auto& oa = m_ucfg.GetGdsOaCounter(m_ucfg.GetGdsOaState().GetIndex()); + LOGF("QueuePoint DrawIndex: frame=%u submit=%" PRIu64 + " queue=%d index_count=%u instances=%u prim=%u " + "es=0x%016" PRIx64 " ps=0x%016" PRIx64 + " oa_index=%u oa_enabled=%s oa_addr=0x%04" PRIx32 " oa_space=0x%08" PRIx32 "\n", + frame_num, m_submit_id, m_scheduler.Queue(), index_count, instance_count, + m_ucfg.GetPrimType(), m_sh_ctx.GetVs().es_regs.data_addr, + m_sh_ctx.GetPs().ps_regs.data_addr, m_ucfg.GetGdsOaState().GetIndex(), + oa.IsCounterEnabled() ? "true" : "false", oa.GetAddressBytes(), + oa.GetSpaceAvailable()); + } + } + + RenderDrawIndex(m_submit_id, CurrentBuffer(), &m_ctx, &m_ucfg, &m_sh_ctx, m_index_type_and_size, + index_count, index_addr, flags, type, instance_count, + render_target_slice_offset, vertex_offset_add, first_instance); +} + +void CommandProcessor::DrawIndexOffset(uint32_t index_offset, uint32_t index_count, + uint32_t flags) { + Common::LockGuard lock(m_mutex); + + CheckBuffer(); + + uint64_t index_size = 0; + switch (m_index_type_and_size) { + case 0: index_size = 2; break; + case 1: index_size = 4; break; + case 2: index_size = 1; break; + default: EXIT("unknown index_type_and_size: %u\n", m_index_type_and_size); + } + + auto* index_addr = reinterpret_cast( + m_index_base_addr + static_cast(index_offset) * index_size); + + RenderDrawIndex(m_submit_id, CurrentBuffer(), &m_ctx, &m_ucfg, &m_sh_ctx, m_index_type_and_size, + index_count, index_addr, flags, 1, m_num_instances); +} + +void CommandProcessor::DrawIndirect(uint32_t data_offset, uint32_t draw_initiator, bool indexed) { + struct DrawIndirectArgs { + uint32_t vertex_count_per_instance; + uint32_t instance_count; + uint32_t start_vertex_location; + uint32_t start_instance_location; + }; + struct DrawIndexedIndirectArgs { + uint32_t index_count_per_instance; + uint32_t instance_count; + uint32_t start_index_location; + uint32_t base_vertex_location; + uint32_t start_instance_location; + }; + + EXIT_NOT_IMPLEMENTED((draw_initiator & ~0x20u) != 2u); + EXIT_NOT_IMPLEMENTED(m_draw_indirect_args_base_addr == 0); + + const auto args_addr = m_draw_indirect_args_base_addr + data_offset; + + if (!indexed) { + auto* args = reinterpret_cast(args_addr); + if (args->instance_count != 1u || args->start_vertex_location != 0u || + args->start_instance_location != 0u) { + static std::atomic log_count {0}; + if (log_count.fetch_add(1) < 64) { + LOGF("\t warning: partial DrawIndirect args: vertex_count=%" PRIu32 + ", instance_count=%" PRIu32 ", start_vertex=%" PRIu32 + ", start_instance=%" PRIu32 "\n", + args->vertex_count_per_instance, args->instance_count, + args->start_vertex_location, args->start_instance_location); + } + } + DrawIndexAuto(args->vertex_count_per_instance, 0, 0, args->instance_count, + args->start_vertex_location, args->start_instance_location); + return; + } + + auto* args = reinterpret_cast(args_addr); + if (args->base_vertex_location != 0u || args->start_instance_location != 0u) { + static std::atomic log_count {0}; + if (log_count.fetch_add(1) < 64) { + LOGF("\t warning: partial DrawIndexIndirect args: index_count=%" PRIu32 + ", instance_count=%" PRIu32 ", start_index=%" PRIu32 ", base_vertex=%" PRIu32 + ", start_instance=%" PRIu32 "\n", + args->index_count_per_instance, args->instance_count, args->start_index_location, + args->base_vertex_location, args->start_instance_location); + } + } + + uint64_t index_size = 0; + switch (m_index_type_and_size) { + case 0: index_size = 2; break; + case 1: index_size = 4; break; + case 2: index_size = 1; break; + default: EXIT("unknown index_type_and_size: %u\n", m_index_type_and_size); + } + + auto* index_addr = reinterpret_cast( + m_index_base_addr + static_cast(args->start_index_location) * index_size); + + const uint32_t index_count = + (m_index_buffer_size != 0 ? std::min(args->index_count_per_instance, m_index_buffer_size) + : args->index_count_per_instance); + if (GraphicsRunDebugDumpEnabled() && index_count != args->index_count_per_instance) { + static std::atomic log_count {0}; + if (log_count.fetch_add(1, std::memory_order_relaxed) < 64) { + LOGF("\t DrawIndexIndirect: clamped index_count from %" PRIu32 " to %" PRIu32 + " using INDEX_BUFFER_SIZE\n", + args->index_count_per_instance, index_count); + } + } + + DrawIndex(index_count, index_addr, 0, 1, args->instance_count, nullptr, 0, + static_cast(args->base_vertex_location), args->start_instance_location); +} + +void CommandProcessor::DrawIndirectMulti(uint32_t data_offset, uint32_t max_count_or_count, + const volatile uint32_t* count_addr, + uint32_t stride_in_bytes, uint32_t draw_initiator, + bool indexed) { + struct DrawIndirectArgs { + uint32_t vertex_count_per_instance; + uint32_t instance_count; + uint32_t start_vertex_location; + uint32_t start_instance_location; + }; + struct DrawIndexedIndirectArgs { + uint32_t index_count_per_instance; + uint32_t instance_count; + uint32_t start_index_location; + uint32_t base_vertex_location; + uint32_t start_instance_location; + }; + + EXIT_NOT_IMPLEMENTED((draw_initiator & ~0x20u) != 2u); + EXIT_NOT_IMPLEMENTED(m_draw_indirect_args_base_addr == 0); + + uint32_t draw_count = max_count_or_count; + if (count_addr != nullptr) { + draw_count = *count_addr; + if (draw_count > max_count_or_count) { + draw_count = max_count_or_count; + } + } + + if (draw_count == 0) { + return; + } + + const auto args_size = indexed ? sizeof(DrawIndexedIndirectArgs) : sizeof(DrawIndirectArgs); + EXIT_NOT_IMPLEMENTED(stride_in_bytes < args_size); + + for (uint32_t i = 0; i < draw_count; i++) { + const auto args_addr = m_draw_indirect_args_base_addr + data_offset + + static_cast(i) * stride_in_bytes; + + if (!indexed) { + auto* args = reinterpret_cast(args_addr); + if (args->instance_count != 1u || args->start_vertex_location != 0u || + args->start_instance_location != 0u) { + static std::atomic log_count {0}; + if (log_count.fetch_add(1) < 64) { + LOGF("\t warning: partial DrawIndirectMulti args[%u]: vertex_count=%" PRIu32 + ", instance_count=%" PRIu32 ", start_vertex=%" PRIu32 + ", start_instance=%" PRIu32 "\n", + i, args->vertex_count_per_instance, args->instance_count, + args->start_vertex_location, args->start_instance_location); + } + } + DrawIndexAuto(args->vertex_count_per_instance, 0, 0, args->instance_count, + args->start_vertex_location, args->start_instance_location); + continue; + } + + auto* args = reinterpret_cast(args_addr); + if (args->base_vertex_location != 0u || args->start_instance_location != 0u) { + static std::atomic log_count {0}; + if (log_count.fetch_add(1) < 64) { + LOGF("\t warning: partial DrawIndexIndirectMulti args[%u]: index_count=%" PRIu32 + ", instance_count=%" PRIu32 ", start_index=%" PRIu32 ", base_vertex=%" PRIu32 + ", start_instance=%" PRIu32 "\n", + i, args->index_count_per_instance, args->instance_count, + args->start_index_location, args->base_vertex_location, + args->start_instance_location); + } + } + + uint64_t index_size = 0; + switch (m_index_type_and_size) { + case 0: index_size = 2; break; + case 1: index_size = 4; break; + case 2: index_size = 1; break; + default: EXIT("unknown index_type_and_size: %u\n", m_index_type_and_size); + } + + auto* index_addr = reinterpret_cast( + m_index_base_addr + static_cast(args->start_index_location) * index_size); + + const uint32_t index_count = + (m_index_buffer_size != 0 + ? std::min(args->index_count_per_instance, m_index_buffer_size) + : args->index_count_per_instance); + if (GraphicsRunDebugDumpEnabled() && index_count != args->index_count_per_instance) { + static std::atomic log_count {0}; + if (log_count.fetch_add(1, std::memory_order_relaxed) < 64) { + LOGF("\t DrawIndexIndirectMulti: clamped index_count from %" PRIu32 " to %" PRIu32 + " using INDEX_BUFFER_SIZE\n", + args->index_count_per_instance, index_count); + } + } + + DrawIndex(index_count, index_addr, 0, 1, args->instance_count, nullptr, 0, + static_cast(args->base_vertex_location), args->start_instance_location); + } +} + +void CommandProcessor::DispatchDirect(uint32_t thread_group_x, uint32_t thread_group_y, + uint32_t thread_group_z, uint32_t mode) { + uint32_t frame_num = 0; + uint32_t local_x = 1; + uint32_t local_y = 1; + uint32_t local_z = 1; + + { + Common::LockGuard lock(m_mutex); + + CheckBuffer(); + frame_num = GraphicsRunGetFrameNum(); + if (GraphicsRunDebugDumpEnabled()) { + static std::atomic log_count {0}; + if (log_count.fetch_add(1, std::memory_order_relaxed) < 1024) { + const auto& cs = m_sh_ctx.GetCs().cs_regs; + const auto& oa = m_ucfg.GetGdsOaCounter(m_ucfg.GetGdsOaState().GetIndex()); + LOGF("QueuePoint DispatchDirect: frame=%u submit=%" PRIu64 + " queue=%d groups=%ux%ux%u local=%ux%ux%u mode=0x%08" PRIx32 + " wave=%u cs=0x%016" PRIx64 " oa_index=%u oa_enabled=%s oa_addr=0x%04" PRIx32 + " oa_space=0x%08" PRIx32 "\n", + frame_num, m_submit_id, m_scheduler.Queue(), thread_group_x, thread_group_y, + thread_group_z, std::max(cs.num_thread_x, 1u), std::max(cs.num_thread_y, 1u), + std::max(cs.num_thread_z, 1u), mode, static_cast(cs.wave_size), + cs.data_addr, m_ucfg.GetGdsOaState().GetIndex(), + oa.IsCounterEnabled() ? "true" : "false", oa.GetAddressBytes(), + oa.GetSpaceAvailable()); + } + } + + const auto& cs = m_sh_ctx.GetCs().cs_regs; + local_x = std::max(cs.num_thread_x, 1u); + local_y = std::max(cs.num_thread_y, 1u); + local_z = std::max(cs.num_thread_z, 1u); + if (cs.wave_size == 64u) { + static std::atomic_bool logged_wave64_shader {false}; + if (!logged_wave64_shader.exchange(true, std::memory_order_relaxed)) { + LOGF("warning: executing wave64 compute shader cs=0x%016" PRIx64 "\n", + cs.data_addr); + std::printf("warning: executing wave64 compute shader cs=0x%016" PRIx64 "\n", + cs.data_addr); + std::fflush(stdout); + } + } + + RenderDispatchDirect(m_submit_id, CurrentBuffer(), &m_ctx, &m_sh_ctx, thread_group_x, + thread_group_y, thread_group_z, mode); + } + + constexpr uint32_t DispatchInitiatorUseThreadDimensions = 1u << 5u; + auto group_count = [](uint32_t threads, uint32_t group_size) { + return (threads == 0 + ? 0u + : (threads + std::max(group_size, 1u) - 1u) / std::max(group_size, 1u)); + }; + + auto groups_x = thread_group_x; + auto groups_y = thread_group_y; + auto groups_z = thread_group_z; + if ((mode & DispatchInitiatorUseThreadDimensions) != 0) { + groups_x = group_count(thread_group_x, local_x); + groups_y = group_count(thread_group_y, local_y); + groups_z = group_count(thread_group_z, local_z); + } + + const uint64_t invocations = + static_cast(groups_x) * groups_y * groups_z * local_x * local_y * local_z; + if (invocations != 0) { + BufferFlushAndWait(); + } +} + +void CommandProcessor::DispatchIndirect(uint32_t data_offset, uint32_t mode) { + struct DispatchIndirectArgs { + uint32_t thread_group_x; + uint32_t thread_group_y; + uint32_t thread_group_z; + }; + + EXIT_NOT_IMPLEMENTED(m_dispatch_indirect_args_base_addr == 0); + + const auto args_addr = m_dispatch_indirect_args_base_addr + data_offset; + auto* args = reinterpret_cast(args_addr); + + DispatchDirect(args->thread_group_x, args->thread_group_y, args->thread_group_z, mode); +} + +void CommandProcessor::DrawIndexAuto(uint32_t index_count, uint32_t flags, + uint32_t render_target_slice_offset, uint32_t instance_count, + uint32_t first_vertex, uint32_t first_instance) { + Common::LockGuard lock(m_mutex); + + CheckBuffer(); + const auto frame_num = GraphicsRunGetFrameNum(); + if (frame_num >= 480) { + static std::atomic log_count {0}; + if (log_count.fetch_add(1, std::memory_order_relaxed) < 512) { + const auto& oa = m_ucfg.GetGdsOaCounter(m_ucfg.GetGdsOaState().GetIndex()); + LOGF("QueuePoint DrawIndexAuto: frame=%u submit=%" PRIu64 + " queue=%d index_count=%u instances=%u prim=%u " + "first_vertex=%u first_instance=%u es=0x%016" PRIx64 " ps=0x%016" PRIx64 + " oa_index=%u oa_enabled=%s oa_addr=0x%04" PRIx32 " oa_space=0x%08" PRIx32 "\n", + frame_num, m_submit_id, m_scheduler.Queue(), index_count, instance_count, + m_ucfg.GetPrimType(), first_vertex, first_instance, + m_sh_ctx.GetVs().es_regs.data_addr, m_sh_ctx.GetPs().ps_regs.data_addr, + m_ucfg.GetGdsOaState().GetIndex(), oa.IsCounterEnabled() ? "true" : "false", + oa.GetAddressBytes(), oa.GetSpaceAvailable()); + } + } + + RenderDrawIndexAuto(m_submit_id, CurrentBuffer(), &m_ctx, &m_ucfg, &m_sh_ctx, index_count, + flags, render_target_slice_offset, instance_count, first_vertex, + first_instance); +} + +void CommandProcessor::ClearGds(uint64_t dw_offset, uint32_t dw_num, uint32_t clear_value) { + Common::LockGuard lock(m_mutex); + + GraphicsRenderClearGds(dw_offset, dw_num, clear_value); +} + +void CommandProcessor::ReadGds(uint32_t* dst, uint32_t dw_offset, uint32_t dw_size) { + Common::LockGuard lock(m_mutex); + + GraphicsRenderReadGds(dst, dw_offset, dw_size); +} + +void CommandProcessor::WaitFlipDone(uint32_t video_out_handle, uint32_t display_buffer_index) { + BufferFlush(); + + VideoOut::VideoOutWaitFlipDone(static_cast(video_out_handle), + static_cast(display_buffer_index)); +} + +template +void CommandProcessor::WriteAtEndOfPipe(uint32_t cache_policy, uint32_t event_write_dest, + uint32_t eop_event_type, uint32_t cache_action, + uint32_t event_index, uint32_t event_write_source, + void* dst_gpu_addr, T value, uint32_t interrupt_selector, + uint32_t interrupt_context_id) { + static_assert(sizeof(T) == sizeof(uint32_t) || sizeof(T) == sizeof(uint64_t)); + + Common::LockGuard lock(m_mutex); + + CheckBuffer(); + + if (GraphicsRunDebugDumpEnabled()) { + const auto bits = static_cast(sizeof(T) * 8u); + const auto log_width = static_cast(sizeof(T) * 2u); + + LOGF("CommandProcessor::WriteAtEndOfPipe%u()\n" + "\t cache_policy = 0x%08" PRIx32 "\n" + "\t event_write_dest = 0x%08" PRIx32 "\n" + "\t eop_event_type = 0x%08" PRIx32 "\n" + "\t cache_action = 0x%08" PRIx32 "\n" + "\t event_index = 0x%08" PRIx32 "\n" + "\t event_write_source = 0x%08" PRIx32 "\n" + "\t interrupt_selector = 0x%08" PRIx32 "\n" + "\t interrupt_context = 0x%08" PRIx32 "\n" + "\t dst_gpu_addr = 0x%016" PRIx64 "\n" + "\t value = 0x%0*" PRIx64 "\n", + bits, cache_policy, event_write_dest, eop_event_type, cache_action, event_index, + event_write_source, interrupt_selector, interrupt_context_id, + reinterpret_cast(dst_gpu_addr), log_width, static_cast(value)); + } + + EXIT_NOT_IMPLEMENTED(cache_policy != 0x00000000); + EXIT_NOT_IMPLEMENTED(event_write_dest != 0x00000000); + + bool with_interrupt = false; + switch (interrupt_selector) { + case 0x00: + case 0x03: with_interrupt = false; break; + case 0x01: + GraphicsRenderTriggerEopEventAtEndOfPipe(CurrentBuffer(), interrupt_context_id); + return; + case 0x02: with_interrupt = true; break; + default: EXIT("unknown interrupt_selector\n"); + } + + auto write32 = [&](bool with_writeback) { + auto* dst = static_cast(dst_gpu_addr); + auto data = static_cast(value); + CommandProcessorGuestAccessScope guest_access(CommandProcessorGuestAccess::DirectFence, + reinterpret_cast(dst), + sizeof(*dst)); + + if (with_interrupt) { + if (with_writeback) { + GraphicsRenderWriteAtEndOfPipeWithInterruptWriteBack32( + m_submit_id, CurrentBuffer(), dst, data, interrupt_context_id); + } else { + GraphicsRenderWriteAtEndOfPipeWithInterrupt32(m_submit_id, CurrentBuffer(), dst, + data, interrupt_context_id); + } + } else if (with_writeback) { + GraphicsRenderWriteAtEndOfPipeWithWriteBack32(m_submit_id, CurrentBuffer(), dst, data); + } else { + GraphicsRenderWriteAtEndOfPipe32(m_submit_id, CurrentBuffer(), dst, data); + } + }; + + switch (event_write_source) { + case 0x01: + if constexpr (sizeof(T) == sizeof(uint32_t)) { + if (eop_event_type == 0x2f && cache_action == 0x00 && event_index == 0x06) { + GraphicsRenderWriteAtEndOfPipeGds32(m_submit_id, CurrentBuffer(), + static_cast(dst_gpu_addr), + value & 0xffffu, value >> 16u); + return; + } + } else if (eop_event_type == 0x04 && cache_action == 0x00 && event_index == 0x05) { + write32(false); + return; + } + break; + case 0x02: + if constexpr (sizeof(T) == sizeof(uint32_t)) { + if (eop_event_type == 0x2f && event_index == 0x06) { + switch (cache_action) { + case 0x00: write32(false); return; + case 0x38: write32(true); return; + default: break; + } + } + } else { + auto write64 = [&](bool with_writeback) { + auto* dst = static_cast(dst_gpu_addr); + CommandProcessorGuestAccessScope guest_access( + CommandProcessorGuestAccess::DirectFence, reinterpret_cast(dst), + sizeof(*dst)); + + if (with_interrupt) { + if (with_writeback) { + GraphicsRenderWriteAtEndOfPipeWithInterruptWriteBack64( + m_submit_id, CurrentBuffer(), dst, value, interrupt_context_id); + } else { + GraphicsRenderWriteAtEndOfPipeWithInterrupt64( + m_submit_id, CurrentBuffer(), dst, value, interrupt_context_id); + } + } else if (with_writeback) { + GraphicsRenderWriteAtEndOfPipeWithWriteBack64(m_submit_id, CurrentBuffer(), + dst, value); + } else { + GraphicsRenderWriteAtEndOfPipe64(m_submit_id, CurrentBuffer(), dst, value); + } + }; + + switch (cache_action) { + case 0x00: + switch (eop_event_type) { + case 0x04: + case 0x28: + if ((eop_event_type == 0x04 && event_index == 0x05) || + (eop_event_type == 0x28 && event_index == 0x00)) { + write64(false); + return; + } + break; + case 0x2b: + case 0x2d: + case 0x2f: + case 0x30: + if (event_index == 0x00 && !with_interrupt) { + write64(false); + return; + } + break; + default: break; + } + break; + case 0x38: + switch (eop_event_type) { + case 0x04: + case 0x14: + case 0x28: + if (((eop_event_type == 0x04 || eop_event_type == 0x28) && + event_index == 0x05 && !with_interrupt) || + (event_index == 0x00)) { + write64(true); + return; + } + break; + case 0x2b: + case 0x2d: + if (event_index == 0x00 && !with_interrupt) { + write64(true); + return; + } + break; + case 0x2f: + if (event_index == 0x06 && !with_interrupt) { + write64(true); + return; + } + break; + default: break; + } + break; + case 0x3b: + if (eop_event_type == 0x04 && event_index == 0x05 && with_interrupt) { + write64(true); + return; + } + break; + default: break; + } + } + break; + case 0x04: + if constexpr (sizeof(T) == sizeof(uint64_t)) { + CommandProcessorGuestAccessScope guest_access( + CommandProcessorGuestAccess::DirectFence, + reinterpret_cast(dst_gpu_addr), sizeof(uint64_t)); + switch (cache_action) { + case 0x00: + if (((eop_event_type == 0x04 && event_index == 0x05) || + (eop_event_type == 0x28 && event_index == 0x00)) && + !with_interrupt) { + GraphicsRenderWriteAtEndOfPipeClockCounter( + m_submit_id, CurrentBuffer(), static_cast(dst_gpu_addr)); + return; + } + break; + case 0x38: + if (((eop_event_type == 0x04 && + (event_index == 0x00 || event_index == 0x05)) || + (eop_event_type == 0x28 && event_index == 0x00)) && + !with_interrupt) { + GraphicsRenderWriteAtEndOfPipeClockCounterWithWriteBack( + m_submit_id, CurrentBuffer(), static_cast(dst_gpu_addr)); + return; + } + break; + default: break; + } + } + break; + default: break; + } + + EXIT("unknown event type\n"); +} + +void CommandProcessor::WriteAtEndOfPipe32(uint32_t cache_policy, uint32_t event_write_dest, + uint32_t eop_event_type, uint32_t cache_action, + uint32_t event_index, uint32_t event_write_source, + void* dst_gpu_addr, uint32_t value, + uint32_t interrupt_selector, + uint32_t interrupt_context_id) { + WriteAtEndOfPipe(cache_policy, event_write_dest, eop_event_type, cache_action, event_index, + event_write_source, dst_gpu_addr, value, interrupt_selector, + interrupt_context_id); +} + +void CommandProcessor::WriteAtEndOfPipe64(uint32_t cache_policy, uint32_t event_write_dest, + uint32_t eop_event_type, uint32_t cache_action, + uint32_t event_index, uint32_t event_write_source, + void* dst_gpu_addr, uint64_t value, + uint32_t interrupt_selector, + uint32_t interrupt_context_id) { + WriteAtEndOfPipe(cache_policy, event_write_dest, eop_event_type, cache_action, event_index, + event_write_source, dst_gpu_addr, value, interrupt_selector, + interrupt_context_id); +} + +void CommandProcessor::MemoryBarrier() { + Common::LockGuard lock(m_mutex); + + CheckBuffer(); + + GraphicsRenderMemoryBarrier(CurrentBuffer()); +} + +void CommandProcessor::TriggerEopEventAtEndOfPipe(uint32_t interrupt_context_id) { + Common::LockGuard lock(m_mutex); + + CheckBuffer(); + + GraphicsRenderTriggerEopEventAtEndOfPipe(CurrentBuffer(), interrupt_context_id); +} + +void CommandProcessor::RenderTextureBarrier(uint64_t vaddr, uint64_t size) { + Common::LockGuard lock(m_mutex); + + CheckBuffer(); + + GraphicsRenderTextureBarrier(CurrentBuffer(), vaddr, size); +} + +void CommandProcessor::DepthStencilBarrier(uint64_t vaddr, uint64_t size) { + Common::LockGuard lock(m_mutex); + + CheckBuffer(); + + GraphicsRenderDepthStencilBarrier(CurrentBuffer(), vaddr, size); +} + +void CommandProcessor::TriggerEvent(uint32_t event_type, uint32_t event_index) { + if (GraphicsRunDebugDumpEnabled()) { + LOGF("CommandProcessor::TriggerEvent()\n" + "\t event_type = 0x%08" PRIx32 "\n" + "\t event_index = 0x%08" PRIx32 "\n", + event_type, event_index); + } + + const auto valid_cache_event_index = event_index == 0x00000000 || event_index == 0x00000007; + switch (event_type) { + // CsPartialFlush, GsPartialFlush, PsPartialFlush. + case 0x00000007: + case 0x0000000f: + case 0x00000010: MemoryBarrier(); break; + // CbDbDataWritebackInvalidate, CbDataWritebackInvalidate. + case 0x00000016: + case 0x00000031: + if (!valid_cache_event_index) { + EXIT("unknown event type: 0x%08" PRIx32 ", 0x%08" PRIx32 "\n", event_type, + event_index); + } + MemoryBarrier(); + SynchronizeGpu(); + break; + // DbDataWritebackInvalidate, DbMetadataWritebackInvalidate, CbMetadataWritebackInvalidate. + case 0x0000002a: + case 0x0000002c: + case 0x0000002e: + if (!valid_cache_event_index) { + EXIT("unknown event type: 0x%08" PRIx32 ", 0x%08" PRIx32 "\n", event_type, + event_index); + } + MemoryBarrier(); + break; + case 0x0000000d: + case 0x0000000e: + case 0x00000012: + case 0x00000017: + case 0x00000018: + case 0x00000019: + case 0x0000001a: + case 0x0000001b: + case 0x00000038: + case 0x00000039: + case 0x0000003a: + LOGF("\t temporary: ignoring unsupported event_write type 0x%08" PRIx32 + ", index 0x%08" PRIx32 "\n", + event_type, event_index); + break; + default: + EXIT("unknown event type: 0x%08" PRIx32 ", 0x%08" PRIx32 "\n", event_type, event_index); + } +} + +void CommandProcessor::Flip() { + Common::LockGuard lock(m_mutex); + + CheckBuffer(); + + if (GraphicsRunDebugDumpEnabled()) { + LOGF("CommandProcessor::Flip()\n"); + } + + GraphicsRenderWriteAtEndOfPipeOnlyFlip(m_submit_id, CurrentBuffer(), m_flip.handle, + m_flip.index, m_flip.flip_mode, m_flip.flip_arg); +} + +void CommandProcessor::Flip(void* dst_gpu_addr, uint32_t value) { + Common::LockGuard lock(m_mutex); + + CheckBuffer(); + + if (GraphicsRunDebugDumpEnabled()) { + LOGF("CommandProcessor::Flip()\n" + "\t dst_gpu_addr = 0x%016" PRIx64 "\n" + "\t value = 0x%08" PRIx32 "\n", + reinterpret_cast(dst_gpu_addr), value); + } + + CommandProcessorGuestAccessScope guest_access(CommandProcessorGuestAccess::DirectFence, + reinterpret_cast(dst_gpu_addr), + sizeof(uint32_t)); + GraphicsRenderWriteAtEndOfPipeWithFlip32( + m_submit_id, CurrentBuffer(), static_cast(dst_gpu_addr), value, m_flip.handle, + m_flip.index, m_flip.flip_mode, m_flip.flip_arg); +} + +void CommandProcessor::FlipWithInterrupt(uint32_t eop_event_type, uint32_t cache_action, + void* dst_gpu_addr, uint32_t value) { + Common::LockGuard lock(m_mutex); + + CheckBuffer(); + + if (GraphicsRunDebugDumpEnabled()) { + LOGF("CommandProcessor::FlipWithInterrupt()\n" + "\t eop_event_type = 0x%08" PRIx32 "\n" + "\t cache_action = 0x%08" PRIx32 "\n" + "\t dst_gpu_addr = 0x%016" PRIx64 "\n" + "\t value = 0x%08" PRIx32 "\n", + eop_event_type, cache_action, reinterpret_cast(dst_gpu_addr), value); + } + + if (eop_event_type == 0x00000004 && cache_action == 0x00000038) { + CommandProcessorGuestAccessScope guest_access(CommandProcessorGuestAccess::DirectFence, + reinterpret_cast(dst_gpu_addr), + sizeof(uint32_t)); + GraphicsRenderWriteAtEndOfPipeWithInterruptWriteBackFlip32( + m_submit_id, CurrentBuffer(), static_cast(dst_gpu_addr), value, + m_flip.handle, m_flip.index, m_flip.flip_mode, m_flip.flip_arg); + } else { + EXIT("unknown event type\n"); + } +} + +void CommandProcessor::SynchronizeGpu() { + BufferFlush(); + BufferWait(); +} + +void CommandProcessor::SynchronizeGpuRange(uint64_t vaddr, uint64_t size) { + if (vaddr == 0 || size == 0 || size > UINT64_MAX - vaddr) { + EXIT("invalid GPU synchronization range\n"); + } + SynchronizeGpu(); +} + +void GraphicsRunSubmit(uint32_t* cmd_draw_buffer, uint32_t num_draw_dw, uint32_t* cmd_const_buffer, + uint32_t num_const_dw, bool trigger_agc_interrupt_on_done) { + EXIT_IF(cmd_draw_buffer == nullptr); + EXIT_IF(num_draw_dw == 0); + EXIT_IF(g_gpu == nullptr); + + g_gpu->Submit(cmd_draw_buffer, num_draw_dw, cmd_const_buffer, num_const_dw, + trigger_agc_interrupt_on_done); +} + +void GraphicsRunSubmitCompute(uint32_t queue, uint32_t* cmd_buffer, uint32_t num_dw, + bool trigger_agc_interrupt_on_done) { + EXIT_IF(cmd_buffer == nullptr); + EXIT_IF(num_dw == 0); + EXIT_IF(g_gpu == nullptr); + + g_gpu->SubmitCompute(queue, cmd_buffer, num_dw, trigger_agc_interrupt_on_done); +} + +void GraphicsRunWait() { + GraphicsRunSubmissionLock lock; +} + +GraphicsRunSubmissionLock::GraphicsRunSubmissionLock() { + if (g_gpu == nullptr || g_current_run_cp != nullptr || g_submission_pause_depth == UINT32_MAX) { + EXIT("cannot acquire GPU submission lock in the current state\n"); + } + if (g_submission_pause_depth++ == 0) { + g_gpu->PauseSubmissions(); + } +} + +GraphicsRunSubmissionLock::~GraphicsRunSubmissionLock() { + if (g_gpu == nullptr || g_submission_pause_depth == 0) { + EXIT("GPU submission lock released without ownership\n"); + } + if (--g_submission_pause_depth == 0) { + g_gpu->ResumeSubmissions(); + } +} + +void GraphicsRunDone() { + EXIT_IF(g_gpu == nullptr); + + g_gpu->Done(); +} + +int GraphicsRunGetFrameNum() { + EXIT_IF(g_gpu == nullptr); + + return g_gpu->GetFrameNum(); +} + +bool GraphicsRunIsCommandProcessorThread() noexcept { + return g_current_run_cp != nullptr; +} + +bool GraphicsRunIsGuestMemoryPoll() noexcept { + return g_current_run_cp != nullptr && + g_guest_memory_access == CommandProcessorGuestAccess::WaitPoll; +} + +bool GraphicsRunIsCommandProcessorGuestAccess() noexcept { + return g_current_run_cp != nullptr && + g_guest_memory_access != CommandProcessorGuestAccess::None; +} + +bool GraphicsRunSubmissionLockHeld() noexcept { + return g_submission_pause_depth != 0; +} + +bool GraphicsRunGpuLockHeld() noexcept { + return g_gpu_mutex_owned; +} + +int GraphicsRunBeginCommandProcessorReadback() { + if (g_current_run_cp == nullptr || g_submission_pause_depth != 0 || g_gpu_mutex_owned || + g_cp_readback_owned || g_guest_memory_access == CommandProcessorGuestAccess::None) { + EXIT("GPU command-thread readback requires an unpaused command processor\n"); + } + const auto queue = g_current_run_cp->GetQueue(); + if (queue < 0 || queue >= GraphicContext::QUEUES_NUM) { + EXIT("GPU command-thread readback has invalid queue %d\n", queue); + } + g_cp_readback_owned = true; + // Direct fences enter from an already serialized command handler. Wait polls acquire the + // same processor mutex in CommandProcessorGuestAccessScope before the resource transaction. + g_cp_readback_preowned = true; + const auto synchronized_queue = + g_current_run_cp->BeginReadbackTransaction(g_cp_readback_preowned); + if (synchronized_queue != queue) { + EXIT("GPU command-thread queue changed during readback\n"); + } + return synchronized_queue; +} + +void GraphicsRunEndCommandProcessorReadback() { + if (g_current_run_cp == nullptr || !g_cp_readback_owned) { + EXIT("GPU command-thread readback ended without ownership\n"); + } + const auto preowned = g_cp_readback_preowned; + g_cp_readback_owned = false; + g_cp_readback_preowned = false; + g_current_run_cp->EndReadbackTransaction(preowned); +} + +} // namespace Libs::Graphics diff --git a/src/graphics/guest_gpu/graphicsRun.h b/src/graphics/guest_gpu/graphicsRun.h new file mode 100644 index 000000000..99682deda --- /dev/null +++ b/src/graphics/guest_gpu/graphicsRun.h @@ -0,0 +1,38 @@ +#ifndef EMULATOR_INCLUDE_EMULATOR_GRAPHICS_GRAPHICSRUN_H_ +#define EMULATOR_INCLUDE_EMULATOR_GRAPHICS_GRAPHICSRUN_H_ + +#include "common/abi.h" +#include "common/common.h" + +namespace Libs::Graphics { + +class GraphicsRunSubmissionLock final { +public: + GraphicsRunSubmissionLock(); + ~GraphicsRunSubmissionLock(); + KYTY_CLASS_NO_COPY(GraphicsRunSubmissionLock); +}; + +void GraphicsRunInit(); + +void GraphicsRunSubmit(uint32_t* cmd_draw_buffer, uint32_t num_draw_dw, uint32_t* cmd_const_buffer, + uint32_t num_const_dw, bool trigger_agc_interrupt_on_done = false); +void GraphicsRunSubmitCompute(uint32_t queue, uint32_t* cmd_buffer, uint32_t num_dw, + bool trigger_agc_interrupt_on_done = false); +void GraphicsRunWait(); +void GraphicsRunDone(); +int GraphicsRunGetFrameNum(); +[[nodiscard]] bool GraphicsRunIsCommandProcessorThread() noexcept; +[[nodiscard]] bool GraphicsRunIsGuestMemoryPoll() noexcept; +[[nodiscard]] bool GraphicsRunIsCommandProcessorGuestAccess() noexcept; +[[nodiscard]] bool GraphicsRunSubmissionLockHeld() noexcept; +[[nodiscard]] bool GraphicsRunGpuLockHeld() noexcept; +// Begin/end an exclusive same-command-processor readback transaction. The begin call finishes +// work already recorded on that queue and retains the processor lock so its peer draw/constant +// job cannot record new work until BufferCache publishes the faulted page. +[[nodiscard]] int GraphicsRunBeginCommandProcessorReadback(); +void GraphicsRunEndCommandProcessorReadback(); + +} // namespace Libs::Graphics + +#endif /* EMULATOR_INCLUDE_EMULATOR_GRAPHICS_GRAPHICSRUN_H_ */ diff --git a/src/graphics/guest_gpu/hardwareContext.h b/src/graphics/guest_gpu/hardwareContext.h new file mode 100644 index 000000000..6d8d1f0bb --- /dev/null +++ b/src/graphics/guest_gpu/hardwareContext.h @@ -0,0 +1,1047 @@ +#ifndef EMULATOR_INCLUDE_EMULATOR_GRAPHICS_HARDWARECONTEXT_H_ +#define EMULATOR_INCLUDE_EMULATOR_GRAPHICS_HARDWARECONTEXT_H_ + +#include "common/abi.h" +#include "common/common.h" + +namespace Libs::Graphics::HW { + +struct ColorBase { + uint64_t addr = 0; +}; + +struct ColorPitch { + uint32_t pitch_div8_minus1 = 0; + uint32_t fmask_pitch_div8_minus1 = 0; +}; + +struct ColorSlice { + uint32_t slice_div64_minus1 = 0; +}; + +struct ColorView { + uint32_t base_array_slice_index = 0; + uint32_t last_array_slice_index = 0; + uint32_t current_mip_level = 0; +}; + +struct ColorInfo { + bool fmask_compression_enable = false; + // uint32_t fmask_compression_mode = 0; + bool fmask_data_compression_disable = false; + bool fmask_one_frag_mode = false; + bool cmask_fast_clear_enable = false; + bool dcc_compression_enable = false; + bool blend_clamp = false; + bool blend_bypass = false; + bool round_mode = false; + uint32_t cmask_is_linear = 0; + uint32_t cmask_addr_type = 0; + bool alt_tile_mode = false; + uint32_t format = 0; + uint32_t channel_type = 0; + uint32_t channel_order = 0; +}; + +struct ColorAttrib { + bool force_dest_alpha_to_one = false; + uint32_t tile_mode = 0; + uint32_t fmask_tile_mode = 0; + uint32_t num_samples = 0; + uint32_t num_fragments = 0; +}; + +struct ColorAttrib2 { + uint32_t height = 0; + uint32_t width = 0; + uint32_t num_mip_levels = 0; +}; + +struct ColorAttrib3 { + uint32_t depth = 0; + uint32_t tile_mode = 0; + uint32_t dimension = 0; + bool cmask_pipe_aligned = false; + bool dcc_pipe_aligned = false; +}; + +struct ColorDccControl { + uint32_t max_uncompressed_block_size = 0; + uint32_t max_compressed_block_size = 0; + uint32_t min_compressed_block_size = 0; + uint32_t color_transform = 0; + bool dcc_clear_key_enable = false; + bool overwrite_combiner_disable = false; + bool independent_64b_blocks = false; + bool independent_128b_blocks = false; + bool data_write_on_dcc_clear_to_reg = false; +}; + +struct ColorCmask { + uint64_t addr = 0; +}; + +struct ColorCmaskSlice { + uint32_t slice_minus1 = 0; +}; + +struct ColorFmask { + uint64_t addr = 0; +}; + +struct ColorFmaskSlice { + uint32_t slice_minus1 = 0; +}; + +struct ColorClearWord0 { + uint32_t word0 = 0; +}; + +struct ColorClearWord1 { + uint32_t word1 = 0; +}; + +struct ColorDccAddr { + uint64_t addr = 0; +}; + +struct ColorSize { + uint32_t width = 0; + uint32_t height = 0; +}; + +struct RenderTarget { + ColorBase base; + ColorPitch pitch; + ColorSlice slice; + ColorView view; + ColorInfo info; + ColorAttrib attrib; + ColorAttrib2 attrib2; + ColorAttrib3 attrib3; + ColorDccControl dcc; + ColorCmask cmask; + ColorCmaskSlice cmask_slice; + ColorFmask fmask; + ColorFmaskSlice fmask_slice; + ColorClearWord0 clear_word0; + ColorClearWord1 clear_word1; + ColorDccAddr dcc_addr; + ColorSize size; +}; + +struct DepthZInfo { + uint32_t format = 0; + uint32_t tile_mode_index = 0; + uint32_t num_samples = 0; + uint32_t zrange_precision = 0; + bool tile_surface_enable = false; + bool expclear_enabled = false; + bool embedded_sample_locations = false; + bool partially_resident = false; + uint8_t num_mip_levels = 0; + uint8_t plane_compression = 0; +}; + +struct DepthStencilInfo { + uint32_t format = 0; + uint32_t tile_mode_index = 0; + uint32_t tile_split = 0; + bool expclear_enabled = false; + bool tile_stencil_disable = false; + bool texture_compatible_stencil = false; + bool partially_resident = false; +}; + +struct DepthRenderTargetDepthInfo { + uint32_t addr5_swizzle_mask = 0; + uint32_t array_mode = 0; + uint32_t pipe_config = 0; + uint32_t bank_width = 0; + uint32_t bank_height = 0; + uint32_t macro_tile_aspect = 0; + uint32_t num_banks = 0; +}; + +struct DepthDepthView { + uint32_t slice_start = 0; + uint32_t slice_max = 0; + uint8_t current_mip_level = 0; + bool depth_write_disable = false; + bool stencil_write_disable = false; +}; + +struct DepthDepthSizeXY { + uint16_t x_max = 0; + uint16_t y_max = 0; + bool valid = false; +}; + +struct DepthRenderTargetHTileSurface { + uint32_t linear = 0; + uint32_t full_cache = 0; + uint32_t htile_uses_preload_win = 0; + uint32_t preload = 0; + uint32_t prefetch_width = 0; + uint32_t prefetch_height = 0; + uint32_t dst_outside_zero_to_one = 0; +}; + +struct DepthRenderTarget { + DepthZInfo z_info; + DepthStencilInfo stencil_info; + DepthDepthView depth_view; + DepthDepthSizeXY size; + DepthRenderTargetDepthInfo depth_info; + DepthRenderTargetHTileSurface htile_surface; + + uint64_t z_read_base_addr = 0; + uint64_t stencil_read_base_addr = 0; + uint64_t z_write_base_addr = 0; + uint64_t stencil_write_base_addr = 0; + uint32_t pitch_div8_minus1 = 0; + uint32_t height_div8_minus1 = 0; + bool pitch_height_valid = false; + uint32_t slice_div64_minus1 = 0; + uint64_t htile_data_base_addr = 0; + uint32_t width = 0; + uint32_t height = 0; + bool width_height_valid = false; +}; + +struct RenderControl { + bool depth_clear_enable = false; + bool stencil_clear_enable = false; + bool resummarize_enable = false; + bool stencil_compress_disable = false; + bool depth_compress_disable = false; + bool copy_centroid = false; + uint8_t copy_sample = 0; +}; + +struct GdsOaCounter { + uint32_t counter = 0; + uint32_t address = 0; + + [[nodiscard]] uint32_t GetSpaceAvailable() const { return counter; } + [[nodiscard]] uint32_t GetAddressBytes() const { return address & 0x0000ffffu; } + [[nodiscard]] uint32_t GetCrawlerType() const { return (address >> 16u) & 0x3u; } + [[nodiscard]] uint32_t GetCrawlerId() const { return (address >> 20u) & 0xfu; } + [[nodiscard]] bool IsCounterEnabled() const { return (address & 0x80000000u) != 0; } + [[nodiscard]] bool IsAllocationCrawlerDisabled() const { return (address & 0x40000000u) != 0; } +}; + +struct GdsOaState { + static constexpr uint32_t COUNTERS_NUM = 8; + + uint32_t cntl = 0; + GdsOaCounter counters[COUNTERS_NUM]; + + [[nodiscard]] uint32_t GetIndex() const { return cntl & (COUNTERS_NUM - 1u); } +}; + +struct ClipControl { + uint8_t user_clip_planes = 0; + uint8_t user_clip_plane_mode = 0; + bool dx_clip_space = false; + bool vertex_kill_any = false; + bool min_z_clip_disable = false; + bool max_z_clip_disable = false; + bool user_clip_plane_negate_y = false; + bool clip_disable = false; + bool user_clip_plane_cull_only = false; + bool cull_on_clipping_error_disable = false; + bool linear_attribute_clip_enable = false; + bool force_viewport_index_from_vs_enable = false; +}; + +struct DepthControl { + bool stencil_enable = false; + bool z_enable = false; + bool z_write_enable = false; + bool depth_bounds_enable = false; + uint8_t zfunc = 0; + bool backface_enable = false; + uint8_t stencilfunc = 0; + uint8_t stencilfunc_bf = 0; + bool color_writes_on_depth_fail_enable = false; + bool color_writes_on_depth_pass_disable = false; +}; + +struct StencilControl { + uint8_t stencil_fail = 0; + uint8_t stencil_zpass = 0; + uint8_t stencil_zfail = 0; + uint8_t stencil_fail_bf = 0; + uint8_t stencil_zpass_bf = 0; + uint8_t stencil_zfail_bf = 0; +}; + +struct StencilMask { + uint8_t stencil_testval = 0; + uint8_t stencil_mask = 0; + uint8_t stencil_writemask = 0; + uint8_t stencil_opval = 0; + uint8_t stencil_testval_bf = 0; + uint8_t stencil_mask_bf = 0; + uint8_t stencil_writemask_bf = 0; + uint8_t stencil_opval_bf = 0; +}; + +struct ModeControl { + bool cull_front = false; + bool cull_back = false; + bool face = false; + uint8_t poly_mode = 0; + uint8_t polymode_front_ptype = 0; + uint8_t polymode_back_ptype = 0; + bool poly_offset_front_enable = false; + bool poly_offset_back_enable = false; + bool vtx_window_offset_enable = false; + bool provoking_vtx_last = false; + bool persp_corr_dis = false; +}; + +struct BlendControl { + uint8_t color_srcblend = 1; + uint8_t color_comb_fcn = 0; + uint8_t color_destblend = 0; + uint8_t alpha_srcblend = 1; + uint8_t alpha_comb_fcn = 0; + uint8_t alpha_destblend = 0; + bool separate_alpha_blend = true; + bool enable = false; +}; + +struct BlendColor { + float red = 0.0f; + float green = 0.0f; + float blue = 0.0f; + float alpha = 0.0f; +}; + +struct EqaaControl { + uint8_t max_anchor_samples = 0; + uint8_t ps_iter_samples = 0; + uint8_t mask_export_num_samples = 0; + uint8_t alpha_to_mask_num_samples = 0; + bool high_quality_intersections = false; + bool incoherent_eqaa_reads = false; + bool interpolate_comp_z = false; + bool static_anchor_associations = false; +}; + +struct ColorControl { + uint8_t mode = 1; + uint8_t op = 0xCC; +}; + +struct ScanModeControl { + bool msaa_enable = false; + bool vport_scissor_enable = true; + bool line_stipple_enable = false; +}; + +struct AaSampleControl { + uint64_t centroid_priority = 0; + uint32_t locations[16] = {}; +}; + +struct DepthShaderControl { + uint32_t other_bits = 0; + uint8_t conservative_z_export_value = 0; + uint8_t shader_z_behavior = 0; + bool shader_kill_enable = false; + bool shader_z_export_enable = false; + bool shader_mask_export_enable = false; + bool shader_dual_export_enable = false; + bool shader_execute_on_noop = false; + bool alpha_to_mask_disable = false; +}; + +struct AaConfig { + uint8_t msaa_num_samples = 0; + bool aa_mask_centroid_dtmn = false; + uint8_t max_sample_dist = 0; + uint8_t msaa_exposed_samples = 0; +}; + +struct Viewport { + float zmin = 0.0f; + float zmax = 0.0f; + float xscale = 0.0f; + float xoffset = 0.0f; + float yscale = 0.0f; + float yoffset = 0.0f; + float zscale = 0.0f; + float zoffset = 0.0f; + int viewport_scissor_left = 0; + int viewport_scissor_top = 0; + int viewport_scissor_right = 0; + int viewport_scissor_bottom = 0; + bool viewport_scissor_window_offset_enable = false; +}; + +struct ScreenViewport { + Viewport viewports[16]; + uint32_t transform_control = 1087; + int screen_scissor_left = 0; + int screen_scissor_top = 0; + int screen_scissor_right = 0; + int screen_scissor_bottom = 0; + int window_scissor_left = 0; + int window_scissor_top = 0; + int window_scissor_right = 0; + int window_scissor_bottom = 0; + bool window_scissor_window_offset_enable = false; + int generic_scissor_left = 0; + int generic_scissor_top = 0; + int generic_scissor_right = 0; + int generic_scissor_bottom = 0; + bool generic_scissor_window_offset_enable = false; + int window_offset_x = 0; + int window_offset_y = 0; + uint32_t hw_offset_x = 0; + uint32_t hw_offset_y = 0; + float guard_band_horz_clip = 0.0f; + float guard_band_vert_clip = 0.0f; + float guard_band_horz_discard = 0.0f; + float guard_band_vert_discard = 0.0f; + uint16_t clip_rect_rule = 0xffffu; + int clip_rect_left[4] = {}; + int clip_rect_top[4] = {}; + int clip_rect_right[4] = {}; + int clip_rect_bottom[4] = {}; + bool clip_rect_window_offset_enable[4] = {}; +}; + +struct VsShaderResource1 { + uint8_t vgprs = 0; + uint8_t sgprs = 0; + uint8_t priority = 0; + uint8_t float_mode = 0; + bool dx10_clamp = false; + bool ieee_mode = false; + uint8_t vgpr_component_count = 0; + bool cu_group_enable = false; + bool require_forward_progress = false; + bool fp16_overflow = false; +}; + +struct VsShaderResource2 { + bool scratch_en = false; + uint8_t user_sgpr = 0; + bool offchip_lds = false; + bool streamout_enabled = false; + uint8_t shared_vgprs = 0; +}; + +struct PsShaderResource1 { + uint8_t vgprs = 0; + uint8_t sgprs = 0; + uint8_t priority = 0; + uint8_t float_mode = 0; + bool dx10_clamp = false; + bool debug_mode = false; + bool ieee_mode = false; + bool cu_group_disable = false; + bool require_forward_progress = false; + bool fp16_overflow = false; +}; + +struct PsShaderResource2 { + bool scratch_en = false; + uint8_t user_sgpr = 0; + bool wave_cnt_en = false; + uint8_t extra_lds_size = 0; + bool raster_ordered_shading = false; + uint8_t shared_vgprs = 0; +}; + +struct PsStageRegisters { + uint64_t data_addr = 0; + PsShaderResource1 rsrc1; + PsShaderResource2 rsrc2; + uint64_t chksum = 0; +}; + +struct CsStageRegisters { + + uint64_t data_addr = 0; + uint32_t num_thread_x = 0; + uint32_t num_thread_y = 0; + uint32_t num_thread_z = 0; + uint8_t vgprs = 0; + uint8_t sgprs = 0; + uint8_t bulky = 0; + uint8_t wave_size = 64; + uint8_t scratch_en = 0; + uint8_t user_sgpr = 0; + uint8_t tgid_x_en = 0; + uint8_t tgid_y_en = 0; + uint8_t tgid_z_en = 0; + uint8_t tg_size_en = 0; + uint8_t tidig_comp_cnt = 0; + uint8_t lds_size = 0; +}; + +struct EsStageRegisters { + uint64_t data_addr = 0; +}; + +struct LsStageRegisters { + uint64_t data_addr = 0; + VsShaderResource1 rsrc1; + VsShaderResource2 rsrc2; +}; + +struct HsStageRegisters { + uint64_t data_addr = 0; + VsShaderResource1 rsrc1; + VsShaderResource2 rsrc2; + uint64_t chksum = 0; +}; + +struct GsShaderResource1 { + uint8_t vgprs = 0; + uint8_t sgprs = 0; + uint8_t priority = 0; + uint8_t float_mode = 0; + bool dx10_clamp = false; + bool debug_mode = false; + bool ieee_mode = false; + bool cu_group_enable = false; + bool require_forward_progress = false; + bool lds_configuration = false; + uint8_t gs_vgpr_component_count = 0; + bool fp16_overflow = false; +}; + +struct GsShaderResource2 { + bool scratch_en = false; + uint8_t user_sgpr = 0; + uint8_t es_vgpr_component_count = 0; + bool offchip_lds = false; + uint8_t lds_size = 0; + uint8_t shared_vgprs = 0; +}; + +struct GsStageRegisters { + uint64_t data_addr = 0; + GsShaderResource1 rsrc1; + GsShaderResource2 rsrc2; + uint64_t chksum = 0; +}; + +struct ShaderRegisters { + uint32_t m_spiVsOutConfig = 0; + uint32_t m_spiShaderPosFormat = 0; + uint32_t m_paClVsOutCntl = 0; + + uint32_t ps_interpolator_settings[32] = {0}; + // uint32_t ps_input_num = 0; + + uint32_t m_spiShaderIdxFormat = 0; + uint32_t m_geNggSubgrpCntl = 0; + uint32_t m_vgtGsInstanceCnt = 0; + uint32_t m_vgtGsOnchipCntl = 0; + uint32_t m_vgtHosMaxTessLevel = 0; + uint32_t m_vgtHosMinTessLevel = 0; + uint32_t m_geMaxOutputPerSubgroup = 0; + uint32_t m_vgtEsgsRingItemsize = 0; + uint32_t m_vgtGsMaxVertOut = 0; + uint32_t m_vgtGsOutPrimType = 0; + uint32_t m_vgtPrimitiveIdEn = 0; + uint32_t m_vgtReuseOff = 0; + uint32_t m_vgtTessDistribution = 0; + uint32_t m_vgtLsHsConfig = 0; + uint32_t m_vgtTfParam = 0; + + uint32_t shader_z_format = 0; + uint8_t target_output_mode[8] = {}; + uint32_t ps_input_ena = 0; + uint32_t ps_input_addr = 0; + uint32_t ps_in_control = 0; + uint32_t baryc_cntl = 0; + uint32_t m_cbShaderMask = 0; + + uint32_t m_paScShaderControl = 0; + + DepthShaderControl db_shader_control; + + [[nodiscard]] uint32_t GetExportCount() const { + return 1u + ((m_spiVsOutConfig >> 1u) & 0x1Fu); + } + [[nodiscard]] uint32_t GetEsVertsPerSubgrp() const { + return (m_vgtGsOnchipCntl >> 0u) & 0x7FFu; + } + [[nodiscard]] uint32_t GetGsPrimsPerSubgrp() const { + return (m_vgtGsOnchipCntl >> 11u) & 0x7FFu; + } + [[nodiscard]] uint32_t GetGsInstPrimsInSubgrp() const { + return (m_vgtGsOnchipCntl >> 22u) & 0x3FFu; + } +}; + +enum class UserSgprType { Unknown, Region, Vsharp }; + +struct UserSgprInfo { + // Changed from 16 to 64 + static constexpr int SGPRS_MAX = 64; + + uint32_t value[SGPRS_MAX] = {0}; + UserSgprType type[SGPRS_MAX] = {}; + uint32_t count = 0; +}; + +struct VertexShaderInfo { + EsStageRegisters es_regs; + LsStageRegisters ls_regs; + HsStageRegisters hs_regs; + GsStageRegisters gs_regs; + UserSgprInfo hs_user_sgpr; + UserSgprInfo gs_user_sgpr; +}; + +struct PixelShaderInfo { + PsStageRegisters ps_regs; + UserSgprInfo ps_user_sgpr; +}; + +struct ComputeShaderInfo { + CsStageRegisters cs_regs; + UserSgprInfo cs_user_sgpr; +}; + +struct GeControl { + uint16_t primitive_group_size = 0; + uint16_t vertex_group_size = 0; +}; + +struct GeUserVgprEn { + bool vgpr1 = false; + bool vgpr2 = false; + bool vgpr3 = false; +}; + +class Context { +public: + Context() = default; + virtual ~Context() = default; + + KYTY_CLASS_DEFAULT_COPY(Context); + + void Reset() { *this = Context(); } + + void SetColorBase(uint32_t slot, const ColorBase& base) { m_render_targets[slot].base = base; } + void SetColorPitch(uint32_t slot, const ColorPitch& pitch) { + m_render_targets[slot].pitch = pitch; + } + void SetColorSlice(uint32_t slot, const ColorSlice& slice) { + m_render_targets[slot].slice = slice; + } + void SetColorView(uint32_t slot, const ColorView& view) { m_render_targets[slot].view = view; } + void SetColorInfo(uint32_t slot, const ColorInfo& info) { m_render_targets[slot].info = info; } + void SetColorAttrib(uint32_t slot, const ColorAttrib& attrib) { + m_render_targets[slot].attrib = attrib; + } + void SetColorAttrib2(uint32_t slot, const ColorAttrib2& attrib2) { + m_render_targets[slot].attrib2 = attrib2; + } + void SetColorAttrib3(uint32_t slot, const ColorAttrib3& attrib3) { + m_render_targets[slot].attrib3 = attrib3; + } + void SetColorDccControl(uint32_t slot, const ColorDccControl& dcc) { + m_render_targets[slot].dcc = dcc; + } + void SetColorCmask(uint32_t slot, const ColorCmask& cmask) { + m_render_targets[slot].cmask = cmask; + } + void SetColorCmaskSlice(uint32_t slot, const ColorCmaskSlice& cmask_slice) { + m_render_targets[slot].cmask_slice = cmask_slice; + } + void SetColorFmask(uint32_t slot, const ColorFmask& fmask) { + m_render_targets[slot].fmask = fmask; + } + void SetColorFmaskSlice(uint32_t slot, const ColorFmaskSlice& fmask_slice) { + m_render_targets[slot].fmask_slice = fmask_slice; + } + void SetColorClearWord0(uint32_t slot, const ColorClearWord0& clear_word0) { + m_render_targets[slot].clear_word0 = clear_word0; + } + void SetColorClearWord1(uint32_t slot, const ColorClearWord1& clear_word1) { + m_render_targets[slot].clear_word1 = clear_word1; + } + void SetColorDccAddr(uint32_t slot, const ColorDccAddr& dcc_addr) { + m_render_targets[slot].dcc_addr = dcc_addr; + } + void SetColorSize(uint32_t slot, const ColorSize& size) { m_render_targets[slot].size = size; } + [[nodiscard]] const RenderTarget& GetRenderTarget(uint32_t slot) const { + return m_render_targets[slot]; + } + + void SetBlendControl(uint32_t slot, const BlendControl& control) { + m_blend_control[slot] = control; + } + [[nodiscard]] const BlendControl& GetBlendControl(uint32_t slot) const { + return m_blend_control[slot]; + } + + void SetRenderTargetMask(uint32_t mask) { m_render_target_mask = mask; } + [[nodiscard]] uint32_t GetRenderTargetMask() const { return m_render_target_mask; } + + void SetShaderStages(uint32_t flags) { m_shader_stages = flags; } + [[nodiscard]] uint32_t GetShaderStages() const { return m_shader_stages; } + + void SetDepthRenderTarget(const DepthRenderTarget& target) { m_depth_render_target = target; } + [[nodiscard]] const DepthRenderTarget& GetDepthRenderTarget() const { + return m_depth_render_target; + } + void SetDepthZInfo(const DepthZInfo& info) { m_depth_render_target.z_info = info; } + [[nodiscard]] const DepthZInfo& GetDepthZInfo() const { return m_depth_render_target.z_info; } + void SetDepthStencilInfo(const DepthStencilInfo& info) { + m_depth_render_target.stencil_info = info; + } + [[nodiscard]] const DepthStencilInfo& GetDepthStencilInfo() const { + return m_depth_render_target.stencil_info; + } + void SetDepthZReadBase(uint64_t addr) { m_depth_render_target.z_read_base_addr = addr; } + void SetDepthStencilReadBase(uint64_t addr) { + m_depth_render_target.stencil_read_base_addr = addr; + } + void SetDepthZWriteBase(uint64_t addr) { m_depth_render_target.z_write_base_addr = addr; } + void SetDepthStencilWriteBase(uint64_t addr) { + m_depth_render_target.stencil_write_base_addr = addr; + } + void SetDepthHTileDataBase(uint64_t addr) { m_depth_render_target.htile_data_base_addr = addr; } + void SetDepthDepthView(const DepthDepthView& view) { m_depth_render_target.depth_view = view; } + [[nodiscard]] const DepthDepthView& GetDepthDepthView() const { + return m_depth_render_target.depth_view; + } + void SetDepthDepthSizeXY(const DepthDepthSizeXY& size) { m_depth_render_target.size = size; } + [[nodiscard]] const DepthDepthSizeXY& GetDepthDepthSizeXY() const { + return m_depth_render_target.size; + } + + void SetViewportZ(uint32_t viewport_id, float zmin, float zmax) { + m_screen_viewport.viewports[viewport_id].zmin = zmin; + m_screen_viewport.viewports[viewport_id].zmax = zmax; + } + void SetViewportZMin(uint32_t viewport_id, float zmin) { + m_screen_viewport.viewports[viewport_id].zmin = zmin; + } + void SetViewportZMax(uint32_t viewport_id, float zmax) { + m_screen_viewport.viewports[viewport_id].zmax = zmax; + } + void SetViewportScaleOffset(uint32_t viewport_id, float xscale, float xoffset, float yscale, + float yoffset, float zscale, float zoffset) { + m_screen_viewport.viewports[viewport_id].xscale = xscale; + m_screen_viewport.viewports[viewport_id].xoffset = xoffset; + m_screen_viewport.viewports[viewport_id].yscale = yscale; + m_screen_viewport.viewports[viewport_id].yoffset = yoffset; + m_screen_viewport.viewports[viewport_id].zscale = zscale; + m_screen_viewport.viewports[viewport_id].zoffset = zoffset; + } + void SetViewportXScale(uint32_t viewport_id, float xscale) { + m_screen_viewport.viewports[viewport_id].xscale = xscale; + } + void SetViewportXOffset(uint32_t viewport_id, float xoffset) { + m_screen_viewport.viewports[viewport_id].xoffset = xoffset; + } + void SetViewportYScale(uint32_t viewport_id, float yscale) { + m_screen_viewport.viewports[viewport_id].yscale = yscale; + } + void SetViewportYOffset(uint32_t viewport_id, float yoffset) { + m_screen_viewport.viewports[viewport_id].yoffset = yoffset; + } + void SetViewportZScale(uint32_t viewport_id, float zscale) { + m_screen_viewport.viewports[viewport_id].zscale = zscale; + } + void SetViewportZOffset(uint32_t viewport_id, float zoffset) { + m_screen_viewport.viewports[viewport_id].zoffset = zoffset; + } + void SetViewportScissor(uint32_t viewport_id, int left, int top, int right, int bottom, + bool window_offset_enable) { + m_screen_viewport.viewports[viewport_id].viewport_scissor_left = left; + m_screen_viewport.viewports[viewport_id].viewport_scissor_top = top; + m_screen_viewport.viewports[viewport_id].viewport_scissor_right = right; + m_screen_viewport.viewports[viewport_id].viewport_scissor_bottom = bottom; + m_screen_viewport.viewports[viewport_id].viewport_scissor_window_offset_enable = + window_offset_enable; + } + void SetViewportScissorTL(uint32_t viewport_id, int left, int top, bool window_offset_enable) { + m_screen_viewport.viewports[viewport_id].viewport_scissor_left = left; + m_screen_viewport.viewports[viewport_id].viewport_scissor_top = top; + m_screen_viewport.viewports[viewport_id].viewport_scissor_window_offset_enable = + window_offset_enable; + } + void SetViewportScissorBR(uint32_t viewport_id, int right, int bottom) { + m_screen_viewport.viewports[viewport_id].viewport_scissor_right = right; + m_screen_viewport.viewports[viewport_id].viewport_scissor_bottom = bottom; + } + void SetViewportTransformControl(uint32_t control) { + m_screen_viewport.transform_control = control; + } + void SetScreenScissor(int left, int top, int right, int bottom) { + m_screen_viewport.screen_scissor_left = left; + m_screen_viewport.screen_scissor_top = top; + m_screen_viewport.screen_scissor_right = right; + m_screen_viewport.screen_scissor_bottom = bottom; + } + void SetWindowScissor(int left, int top, int right, int bottom, bool window_offset_enable) { + m_screen_viewport.window_scissor_left = left; + m_screen_viewport.window_scissor_top = top; + m_screen_viewport.window_scissor_right = right; + m_screen_viewport.window_scissor_bottom = bottom; + m_screen_viewport.window_scissor_window_offset_enable = window_offset_enable; + } + void SetGenericScissor(int left, int top, int right, int bottom, bool window_offset_enable) { + m_screen_viewport.generic_scissor_left = left; + m_screen_viewport.generic_scissor_top = top; + m_screen_viewport.generic_scissor_right = right; + m_screen_viewport.generic_scissor_bottom = bottom; + m_screen_viewport.generic_scissor_window_offset_enable = window_offset_enable; + } + void SetWindowOffset(int offset_x, int offset_y) { + m_screen_viewport.window_offset_x = offset_x; + m_screen_viewport.window_offset_y = offset_y; + } + void SetHardwareScreenOffset(uint32_t offset_x, uint32_t offset_y) { + m_screen_viewport.hw_offset_x = offset_x; + m_screen_viewport.hw_offset_y = offset_y; + } + void SetGuardBands(float horz_clip, float vert_clip, float horz_discard, float vert_discard) { + m_screen_viewport.guard_band_horz_clip = horz_clip; + m_screen_viewport.guard_band_vert_clip = vert_clip; + m_screen_viewport.guard_band_horz_discard = horz_discard; + m_screen_viewport.guard_band_vert_discard = vert_discard; + } + void SetClipRectRule(uint16_t rule) { m_screen_viewport.clip_rect_rule = rule; } + void SetClipRectTL(uint32_t rect_id, int left, int top, bool window_offset_enable) { + m_screen_viewport.clip_rect_left[rect_id] = left; + m_screen_viewport.clip_rect_top[rect_id] = top; + m_screen_viewport.clip_rect_window_offset_enable[rect_id] = window_offset_enable; + } + void SetClipRectBR(uint32_t rect_id, int right, int bottom) { + m_screen_viewport.clip_rect_right[rect_id] = right; + m_screen_viewport.clip_rect_bottom[rect_id] = bottom; + } + [[nodiscard]] const ScreenViewport& GetScreenViewport() const { return m_screen_viewport; } + + [[nodiscard]] const BlendColor& GetBlendColor() const { return m_blend_color; } + void SetBlendColor(const BlendColor& color) { m_blend_color = color; } + [[nodiscard]] const ClipControl& GetClipControl() const { return m_clip_control; } + void SetClipControl(const ClipControl& control) { m_clip_control = control; } + [[nodiscard]] const RenderControl& GetRenderControl() const { return m_render_control; } + void SetRenderControl(const RenderControl& control) { m_render_control = control; } + [[nodiscard]] const DepthControl& GetDepthControl() const { return m_depth_control; } + void SetDepthControl(const DepthControl& control) { m_depth_control = control; } + [[nodiscard]] const ModeControl& GetModeControl() const { return m_mode_control; } + void SetModeControl(const ModeControl& control) { m_mode_control = control; } + [[nodiscard]] const EqaaControl& GetEqaaControl() const { return m_eqaa_control; } + void SetEqaaControl(const EqaaControl& control) { m_eqaa_control = control; } + [[nodiscard]] const StencilControl& GetStencilControl() const { return m_stencil_control; } + void SetStencilControl(const StencilControl& control) { m_stencil_control = control; } + [[nodiscard]] const StencilMask& GetStencilMask() const { return m_stencil_mask; } + void SetStencilMask(const StencilMask& mask) { m_stencil_mask = mask; } + [[nodiscard]] const ColorControl& GetColorControl() const { return m_color_control; } + void SetColorControl(const ColorControl& control) { m_color_control = control; } + [[nodiscard]] const ScanModeControl& GetScanModeControl() const { return m_scan_mode_control; } + void SetScanModeControl(const ScanModeControl& control) { m_scan_mode_control = control; } + [[nodiscard]] const AaSampleControl& GetAaSampleControl() const { return m_aa_sample_control; } + void SetAaSampleControl(const AaSampleControl& control) { m_aa_sample_control = control; } + [[nodiscard]] const AaConfig& GetAaConfig() const { return m_aa_config; } + void SetAaConfig(const AaConfig& config) { m_aa_config = config; } + + [[nodiscard]] float GetDepthClearValue() const { return m_depth_clear_value; } + void SetDepthClearValue(float clear_value) { m_depth_clear_value = clear_value; } + [[nodiscard]] float GetDepthBoundsMin() const { return m_depth_bounds_min; } + [[nodiscard]] float GetDepthBoundsMax() const { return m_depth_bounds_max; } + void SetDepthBoundsMin(float value) { m_depth_bounds_min = value; } + void SetDepthBoundsMax(float value) { m_depth_bounds_max = value; } + [[nodiscard]] uint8_t GetStencilClearValue() const { return m_stencil_clear_value; } + void SetStencilClearValue(uint8_t clear_value) { m_stencil_clear_value = clear_value; } + + [[nodiscard]] float GetLineWidth() const { return m_line_width; } + void SetLineWidth(float width) { m_line_width = width; } + + [[nodiscard]] const ShaderRegisters& GetShaderRegisters() const { return m_sh_regs; } + + void SetVsOutConfig(uint32_t value) { m_sh_regs.m_spiVsOutConfig = value; } + void SetShaderPosFormat(uint32_t value) { m_sh_regs.m_spiShaderPosFormat = value; } + void SetClVsOutCntl(uint32_t value) { m_sh_regs.m_paClVsOutCntl = value; } + void SetShaderIdxFormat(uint32_t value) { m_sh_regs.m_spiShaderIdxFormat = value; } + void SetNggSubgrpCntl(uint32_t value) { m_sh_regs.m_geNggSubgrpCntl = value; } + void SetGsInstanceCnt(uint32_t value) { m_sh_regs.m_vgtGsInstanceCnt = value; } + void SetGsOnchipCntl(uint32_t value) { m_sh_regs.m_vgtGsOnchipCntl = value; } + void SetHosMaxTessLevel(uint32_t value) { m_sh_regs.m_vgtHosMaxTessLevel = value; } + void SetHosMinTessLevel(uint32_t value) { m_sh_regs.m_vgtHosMinTessLevel = value; } + void SetMaxOutputPerSubgroup(uint32_t value) { m_sh_regs.m_geMaxOutputPerSubgroup = value; } + void SetEsgsRingItemsize(uint32_t value) { m_sh_regs.m_vgtEsgsRingItemsize = value; } + void SetGsMaxVertOut(uint32_t value) { m_sh_regs.m_vgtGsMaxVertOut = value; } + void SetGsOutPrimType(uint32_t value) { m_sh_regs.m_vgtGsOutPrimType = value; } + void SetPrimitiveIdEn(uint32_t value) { m_sh_regs.m_vgtPrimitiveIdEn = value; } + void SetReuseOff(uint32_t value) { m_sh_regs.m_vgtReuseOff = value; } + void SetTessDistribution(uint32_t value) { m_sh_regs.m_vgtTessDistribution = value; } + void SetLsHsConfig(uint32_t value) { m_sh_regs.m_vgtLsHsConfig = value; } + void SetTfParam(uint32_t value) { m_sh_regs.m_vgtTfParam = value; } + + void SetPsInputSettings(uint32_t id, uint32_t value) { + m_sh_regs.ps_interpolator_settings[id] = value; + // m_sh_regs.ps_input_num = ((id + 1) > m_sh_regs.ps_input_num ? (id + 1) : + // m_sh_regs.ps_input_num); + } + + void SetShaderZFormat(uint32_t value) { m_sh_regs.shader_z_format = value; } + void SetTargetOutputMode(uint32_t slot, uint8_t value) { + m_sh_regs.target_output_mode[slot] = value; + } + void SetPsInputEna(uint32_t value) { m_sh_regs.ps_input_ena = value; } + void SetPsInputAddr(uint32_t value) { m_sh_regs.ps_input_addr = value; } + void SetPsInControl(uint32_t value) { m_sh_regs.ps_in_control = value; } + void SetBarycCntl(uint32_t value) { m_sh_regs.baryc_cntl = value; } + void SetShaderMask(uint32_t value) { m_sh_regs.m_cbShaderMask = value; } + void SetDepthShaderControl(const DepthShaderControl& value) { + m_sh_regs.db_shader_control = value; + } + + void SetScShaderControl(uint32_t value) { m_sh_regs.m_paScShaderControl = value; } + +private: + float m_line_width = 1.0f; + + BlendControl m_blend_control[8]; + BlendColor m_blend_color; + RenderTarget m_render_targets[8]; + uint32_t m_render_target_mask = 0; + ScreenViewport m_screen_viewport; + ClipControl m_clip_control; + ColorControl m_color_control; + ScanModeControl m_scan_mode_control; + + AaSampleControl m_aa_sample_control; + AaConfig m_aa_config; + + uint32_t m_shader_stages = 0; + + DepthRenderTarget m_depth_render_target; + RenderControl m_render_control; + DepthControl m_depth_control; + StencilControl m_stencil_control; + StencilMask m_stencil_mask; + float m_depth_clear_value = 0.0f; + float m_depth_bounds_min = 0.0f; + float m_depth_bounds_max = 1.0f; + uint8_t m_stencil_clear_value = 0; + + ModeControl m_mode_control; + EqaaControl m_eqaa_control; + + ShaderRegisters m_sh_regs; +}; + +class UserConfig { +public: + UserConfig() = default; + virtual ~UserConfig() = default; + + KYTY_CLASS_DEFAULT_COPY(UserConfig); + + void Reset() { *this = UserConfig(); } + + void SetPrimitiveType(uint32_t prim_type) { m_prim_type = prim_type; } + [[nodiscard]] uint32_t GetPrimType() const { return m_prim_type; } + void SetIndexOffset(uint32_t index_offset) { m_index_offset = index_offset; } + [[nodiscard]] uint32_t GetIndexOffset() const { return m_index_offset; } + void SetObjectId(uint32_t object_id) { m_object_id = object_id; } + [[nodiscard]] uint32_t GetObjectId() const { return m_object_id; } + + [[nodiscard]] const GeControl& GetGeControl() const { return m_ge_cntl; } + void SetGeControl(const GeControl& control) { m_ge_cntl = control; } + [[nodiscard]] const GeUserVgprEn& GetGeUserVgprEn() const { return m_ge_user_vgpr_en; } + void SetGeUserVgprEn(const GeUserVgprEn& en) { m_ge_user_vgpr_en = en; } + [[nodiscard]] const GdsOaState& GetGdsOaState() const { return m_gds_oa; } + [[nodiscard]] const GdsOaCounter& GetGdsOaCounter(uint32_t index) const { + return m_gds_oa.counters[index & 0x0fu]; + } + void SetGdsOaCntl(uint32_t value) { m_gds_oa.cntl = value; } + void SetGdsOaCounter(uint32_t value) { m_gds_oa.counters[m_gds_oa.GetIndex()].counter = value; } + void SetGdsOaAddress(uint32_t value) { m_gds_oa.counters[m_gds_oa.GetIndex()].address = value; } + +private: + uint32_t m_prim_type = 0; + uint32_t m_index_offset = 0; + uint32_t m_object_id = 0; + + GeControl m_ge_cntl; + GeUserVgprEn m_ge_user_vgpr_en; + GdsOaState m_gds_oa; +}; + +class Shader { +public: + Shader() = default; + virtual ~Shader() = default; + + KYTY_CLASS_DEFAULT_COPY(Shader); + + void Reset() { *this = Shader(); } + + void SetEsShaderBase(uint64_t addr) { m_vs.es_regs.data_addr = addr; } + void SetLsShaderBase(uint64_t addr) { m_vs.ls_regs.data_addr = addr; } + void SetLsShaderResource1(const VsShaderResource1& rsrc1) { m_vs.ls_regs.rsrc1 = rsrc1; } + void SetLsShaderResource2(const VsShaderResource2& rsrc2) { m_vs.ls_regs.rsrc2 = rsrc2; } + void SetHsShaderBase(uint64_t addr) { m_vs.hs_regs.data_addr = addr; } + void SetHsShaderResource1(const VsShaderResource1& rsrc1) { m_vs.hs_regs.rsrc1 = rsrc1; } + void SetHsShaderResource2(const VsShaderResource2& rsrc2) { m_vs.hs_regs.rsrc2 = rsrc2; } + void SetHsShaderChksum(uint32_t value) { m_vs.hs_regs.chksum = value; } + void SetGsShaderBase(uint64_t addr) { m_vs.gs_regs.data_addr = addr; } + void SetGsShaderResource1(const GsShaderResource1& rsrc1) { m_vs.gs_regs.rsrc1 = rsrc1; } + void SetGsShaderResource2(const GsShaderResource2& rsrc2) { m_vs.gs_regs.rsrc2 = rsrc2; } + void SetGsShaderChksum(uint32_t value) { m_vs.gs_regs.chksum = value; } + + void SetPsShaderBase(uint64_t addr) { m_ps.ps_regs.data_addr = addr; } + void SetPsShaderResource1(const PsShaderResource1& rsrc1) { m_ps.ps_regs.rsrc1 = rsrc1; } + void SetPsShaderResource2(const PsShaderResource2& rsrc2) { m_ps.ps_regs.rsrc2 = rsrc2; } + void SetPsShaderChksum(uint32_t value) { m_ps.ps_regs.chksum = value; } + + void SetCsShader(const CsStageRegisters& cs_regs) { m_cs.cs_regs = cs_regs; } + + void SetPsUserSgpr(uint32_t id, uint32_t value, UserSgprType type) { + m_ps.ps_user_sgpr.value[id] = value; + m_ps.ps_user_sgpr.type[id] = type; + m_ps.ps_user_sgpr.count = + ((id + 1) > m_ps.ps_user_sgpr.count ? (id + 1) : m_ps.ps_user_sgpr.count); + } + void SetCsUserSgpr(uint32_t id, uint32_t value, UserSgprType type) { + m_cs.cs_user_sgpr.value[id] = value; + m_cs.cs_user_sgpr.type[id] = type; + m_cs.cs_user_sgpr.count = + ((id + 1) > m_cs.cs_user_sgpr.count ? (id + 1) : m_cs.cs_user_sgpr.count); + } + void SetHsUserSgpr(uint32_t id, uint32_t value, UserSgprType type) { + m_vs.hs_user_sgpr.value[id] = value; + m_vs.hs_user_sgpr.type[id] = type; + m_vs.hs_user_sgpr.count = + ((id + 1) > m_vs.hs_user_sgpr.count ? (id + 1) : m_vs.hs_user_sgpr.count); + } + void SetGsUserSgpr(uint32_t id, uint32_t value, UserSgprType type) { + m_vs.gs_user_sgpr.value[id] = value; + m_vs.gs_user_sgpr.type[id] = type; + m_vs.gs_user_sgpr.count = + ((id + 1) > m_vs.gs_user_sgpr.count ? (id + 1) : m_vs.gs_user_sgpr.count); + } + + [[nodiscard]] const PixelShaderInfo& GetPs() const { return m_ps; } + [[nodiscard]] const VertexShaderInfo& GetVs() const { return m_vs; } + [[nodiscard]] const ComputeShaderInfo& GetCs() const { return m_cs; } + +private: + VertexShaderInfo m_vs; + PixelShaderInfo m_ps; + ComputeShaderInfo m_cs; +}; + +} // namespace Libs::Graphics::HW + +#endif /* EMULATOR_INCLUDE_EMULATOR_GRAPHICS_HARDWARECONTEXT_H_ */ diff --git a/src/graphics/guest_gpu/pm4.cpp b/src/graphics/guest_gpu/pm4.cpp new file mode 100644 index 000000000..592ac409c --- /dev/null +++ b/src/graphics/guest_gpu/pm4.cpp @@ -0,0 +1,153 @@ +#include "graphics/guest_gpu/pm4.h" + +#include "common/assert.h" +#include "common/file.h" + +namespace Libs::Graphics::Pm4 { + +static const char* g_names[256] = {}; +static const char* g_r_names[Pm4::R_NUM] = {}; +static bool g_names_initialized = false; + +static void InitNames() { + if (!g_names_initialized) { + for (auto& n: g_names) { + n = ""; + } + + for (auto& n: g_r_names) { + n = ""; + } + + g_r_names[R_ZERO] = "R_ZERO"; + g_r_names[R_DRAW_RESET] = "R_DRAW_RESET"; + g_r_names[R_WAIT_FLIP_DONE] = "R_WAIT_FLIP_DONE"; + g_r_names[R_DISPATCH_RESET] = "R_DISPATCH_RESET"; + g_r_names[R_WAIT_MEM_32] = "R_WAIT_MEM_32"; + g_r_names[R_PUSH_MARKER] = "R_PUSH_MARKER"; + g_r_names[R_POP_MARKER] = "R_POP_MARKER"; + g_r_names[R_ACQUIRE_MEM] = "R_ACQUIRE_MEM"; + g_r_names[R_WRITE_DATA] = "R_WRITE_DATA"; + g_r_names[R_WAIT_MEM_64] = "R_WAIT_MEM_64"; + g_r_names[R_FLIP] = "R_FLIP"; + g_r_names[R_RELEASE_MEM] = "R_RELEASE_MEM"; + g_r_names[R_DMA_DATA] = "R_DMA_DATA"; + + g_names[IT_NOP] = "IT_NOP"; + g_names[IT_SET_BASE] = "IT_SET_BASE"; + g_names[IT_CLEAR_STATE] = "IT_CLEAR_STATE"; + g_names[IT_INDEX_BUFFER_SIZE] = "IT_INDEX_BUFFER_SIZE"; + g_names[IT_DISPATCH_DIRECT] = "IT_DISPATCH_DIRECT"; + g_names[IT_DISPATCH_INDIRECT] = "IT_DISPATCH_INDIRECT"; + g_names[IT_SET_PREDICATION] = "IT_SET_PREDICATION"; + g_names[IT_COND_EXEC] = "IT_COND_EXEC"; + g_names[IT_DRAW_INDIRECT] = "IT_DRAW_INDIRECT"; + g_names[IT_DRAW_INDEX_INDIRECT] = "IT_DRAW_INDEX_INDIRECT"; + g_names[IT_INDEX_BASE] = "IT_INDEX_BASE"; + g_names[IT_DRAW_INDEX_2] = "IT_DRAW_INDEX_2"; + g_names[IT_CONTEXT_CONTROL] = "IT_CONTEXT_CONTROL"; + g_names[IT_INDEX_TYPE] = "IT_INDEX_TYPE"; + g_names[IT_DRAW_INDIRECT_MULTI] = "IT_DRAW_INDIRECT_MULTI"; + g_names[IT_DRAW_INDEX_AUTO] = "IT_DRAW_INDEX_AUTO"; + g_names[IT_NUM_INSTANCES] = "IT_NUM_INSTANCES"; + g_names[IT_INDIRECT_BUFFER_CNST] = "IT_INDIRECT_BUFFER_CNST"; + g_names[IT_DRAW_INDEX_OFFSET_2] = "IT_DRAW_INDEX_OFFSET_2"; + g_names[IT_WRITE_DATA] = "IT_WRITE_DATA"; + g_names[IT_MEM_SEMAPHORE] = "IT_MEM_SEMAPHORE"; + g_names[IT_DRAW_INDEX_INDIRECT_MULTI] = "IT_DRAW_INDEX_INDIRECT_MULTI"; + g_names[IT_INDIRECT_BUFFER] = "IT_INDIRECT_BUFFER"; + g_names[IT_COPY_DATA] = "IT_COPY_DATA"; + g_names[IT_CP_DMA] = "IT_CP_DMA"; + g_names[IT_PFP_SYNC_ME] = "IT_PFP_SYNC_ME"; + g_names[IT_SURFACE_SYNC] = "IT_SURFACE_SYNC"; + g_names[IT_EVENT_WRITE] = "IT_EVENT_WRITE"; + g_names[IT_EVENT_WRITE_EOP] = "IT_EVENT_WRITE_EOP"; + g_names[IT_EVENT_WRITE_EOS] = "IT_EVENT_WRITE_EOS"; + g_names[IT_RELEASE_MEM] = "IT_RELEASE_MEM"; + g_names[IT_DMA_DATA] = "IT_DMA_DATA"; + g_names[IT_ACQUIRE_MEM] = "IT_ACQUIRE_MEM"; + g_names[IT_REWIND] = "IT_REWIND"; + g_names[IT_SET_SH_REG_INDIRECT] = "IT_SET_SH_REG_INDIRECT"; + g_names[IT_SET_UCONFIG_REG_INDIRECT] = "IT_SET_UCONFIG_REG_INDIRECT"; + g_names[IT_SET_CONFIG_REG] = "IT_SET_CONFIG_REG"; + g_names[IT_SET_CONTEXT_REG] = "IT_SET_CONTEXT_REG"; + g_names[IT_SET_SH_REG] = "IT_SET_SH_REG"; + g_names[IT_SET_QUEUE_REG] = "IT_SET_QUEUE_REG"; + g_names[IT_SET_UCONFIG_REG] = "IT_SET_UCONFIG_REG"; + g_names[IT_SET_UCONFIG_REG_INDEX] = "IT_SET_UCONFIG_REG_INDEX"; + g_names[IT_WRITE_CONST_RAM] = "IT_WRITE_CONST_RAM"; + g_names[IT_DUMP_CONST_RAM] = "IT_DUMP_CONST_RAM"; + g_names[IT_INCREMENT_CE_COUNTER] = "IT_INCREMENT_CE_COUNTER"; + g_names[IT_INCREMENT_DE_COUNTER] = "IT_INCREMENT_DE_COUNTER"; + g_names[IT_WAIT_ON_CE_COUNTER] = "IT_WAIT_ON_CE_COUNTER"; + g_names[IT_WAIT_ON_DE_COUNTER_DIFF] = "IT_WAIT_ON_DE_COUNTER_DIFF"; + g_names[IT_DISPATCH_DRAW_PREAMBLE] = "IT_DISPATCH_DRAW_PREAMBLE"; + g_names[IT_DISPATCH_DRAW] = "IT_DISPATCH_DRAW"; + g_names[IT_GET_LOD_STATS] = "IT_GET_LOD_STATS"; + g_names[IT_SET_CONTEXT_REG_INDIRECT] = "IT_SET_CONTEXT_REG_INDIRECT"; + + g_names_initialized = true; + } +} + +void DumpPm4PacketStream(Common::File* file, uint32_t* cmd_buffer, uint32_t start_dw, + uint32_t num_dw) { + InitNames(); + + // db_dump(); + + file->Printf("----- Buffer --- dwords: 0x%05" PRIx32 ", offset : %u, addr: %016" PRIx64 + " ----- \n", + num_dw, start_dw, reinterpret_cast(cmd_buffer)); + + auto* cmd = cmd_buffer + start_dw; + auto dw = num_dw; + while (dw != 0) { + EXIT_NOT_IMPLEMENTED(dw < 2); + EXIT_NOT_IMPLEMENTED(dw > num_dw); + + auto cmd_id = *cmd++; + + file->Printf("%05" PRIx32 " | 0x%08" PRIx32 " | ", start_dw, cmd_id); + + uint32_t len = 0; + + if ((cmd_id & 0xC0000000u) == 0xC0000000u) { + const bool sh_gx = (cmd_id & 0x2u) == 0; + const uint8_t op = ((cmd_id >> 8u) & 0xffu); + const auto r = KYTY_PM4_R(cmd_id); + len = ((cmd_id >> 16u) & 0x3fffu) + 1; + + file->Printf("%s %s(OP:0x%02" PRIx8 ") SH:%s CNT:%u\n", g_names[op], + (op == IT_NOP ? g_r_names[r] : ""), op, sh_gx ? "GX" : "CX", len); + + for (uint32_t i = 0; i < len; i++) { + file->Printf(" | 0x%08" PRIx32 " | \n", cmd[i]); + } + + if ((op == IT_SET_CONTEXT_REG_INDIRECT || op == IT_SET_SH_REG_INDIRECT || + op == IT_SET_UCONFIG_REG_INDIRECT) && + len == 4) { + auto* indirect_buffer = + reinterpret_cast((static_cast(cmd[0]) & 0xfffffffcu) | + (static_cast(cmd[1]) << 32u)); + const uint32_t indirect_num_regs = cmd[3] & 0x3fffu; + + for (uint32_t i = 0; indirect_buffer != nullptr && i < indirect_num_regs; + i++, indirect_buffer += 2) { + file->Printf(" | | offset = 0x%08" PRIx32 + ", value = 0x%08" PRIx32 "\n", + indirect_buffer[0], indirect_buffer[1]); + } + } + } else { + file->Printf("?????\n"); + } + + cmd += len; + dw -= len + 1; + start_dw += len + 1; + } +} + +} // namespace Libs::Graphics::Pm4 diff --git a/src/graphics/guest_gpu/pm4.h b/src/graphics/guest_gpu/pm4.h new file mode 100644 index 000000000..d21636199 --- /dev/null +++ b/src/graphics/guest_gpu/pm4.h @@ -0,0 +1,1041 @@ +#ifndef EMULATOR_INCLUDE_EMULATOR_GRAPHICS_PM4_H_ +#define EMULATOR_INCLUDE_EMULATOR_GRAPHICS_PM4_H_ + +#include "common/abi.h" +#include "common/common.h" + +namespace Common { +class File; +} // namespace Common + +#define KYTY_PM4_GET(u, r, f) (((u) >> Pm4::r##_##f##_SHIFT) & Pm4::r##_##f##_MASK) + +#define KYTY_PM4(len, op, r) \ + (0xC0000000u | (((static_cast(len) - 2u) & 0x3fffu) << 16u) | \ + (((op) & 0xffu) << 8u) | (((r) & (Pm4::R_NUM - 1u)) << 2u)) + +#define KYTY_PM4_R(cmd_id) (((cmd_id) >> 2u) & (Pm4::R_NUM - 1u)) +#define KYTY_PM4_LEN(cmd_id) ((((cmd_id) >> 16u) & 0x3fffu) + 2u) + +namespace Libs::Graphics::Pm4 { + +constexpr uint32_t IT_NOP = 0x10; +constexpr uint32_t IT_SET_BASE = 0x11; +constexpr uint32_t IT_CLEAR_STATE = 0x12; +constexpr uint32_t IT_INDEX_BUFFER_SIZE = 0x13; +constexpr uint32_t IT_DISPATCH_DIRECT = 0x15; +constexpr uint32_t IT_DISPATCH_INDIRECT = 0x16; +constexpr uint32_t IT_SET_PREDICATION = 0x20; +constexpr uint32_t IT_COND_EXEC = 0x22; +constexpr uint32_t IT_DRAW_INDIRECT = 0x24; +constexpr uint32_t IT_DRAW_INDEX_INDIRECT = 0x25; +constexpr uint32_t IT_INDEX_BASE = 0x26; +constexpr uint32_t IT_DRAW_INDEX_2 = 0x27; +constexpr uint32_t IT_CONTEXT_CONTROL = 0x28; +constexpr uint32_t IT_INDEX_TYPE = 0x2A; +constexpr uint32_t IT_DRAW_INDIRECT_MULTI = 0x2C; +constexpr uint32_t IT_DRAW_INDEX_AUTO = 0x2D; +constexpr uint32_t IT_NUM_INSTANCES = 0x2F; +constexpr uint32_t IT_INDIRECT_BUFFER_CNST = 0x33; +constexpr uint32_t IT_DRAW_INDEX_OFFSET_2 = 0x35; +constexpr uint32_t IT_WRITE_DATA = 0x37; +constexpr uint32_t IT_MEM_SEMAPHORE = 0x39; +constexpr uint32_t IT_DRAW_INDEX_INDIRECT_MULTI = 0x38; +constexpr uint32_t IT_DISPATCH_DRAW_PREAMBLE = 0x3A; +constexpr uint32_t IT_INDIRECT_BUFFER = 0x3F; +constexpr uint32_t IT_COPY_DATA = 0x40; +constexpr uint32_t IT_CP_DMA = 0x41; +constexpr uint32_t IT_PFP_SYNC_ME = 0x42; +constexpr uint32_t IT_SURFACE_SYNC = 0x43; +constexpr uint32_t IT_EVENT_WRITE = 0x46; +constexpr uint32_t IT_EVENT_WRITE_EOP = 0x47; +constexpr uint32_t IT_EVENT_WRITE_EOS = 0x48; +constexpr uint32_t IT_RELEASE_MEM = 0x49; +constexpr uint32_t IT_DMA_DATA = 0x50; +constexpr uint32_t IT_ACQUIRE_MEM = 0x58; +constexpr uint32_t IT_REWIND = 0x59; +constexpr uint32_t IT_SET_SH_REG_INDIRECT = 0x63; +constexpr uint32_t IT_SET_UCONFIG_REG_INDIRECT = 0x64; +constexpr uint32_t IT_SET_CONFIG_REG = 0x68; +constexpr uint32_t IT_SET_CONTEXT_REG = 0x69; +constexpr uint32_t IT_SET_SH_REG = 0x76; +constexpr uint32_t IT_SET_QUEUE_REG = 0x78; +constexpr uint32_t IT_SET_UCONFIG_REG = 0x79; +constexpr uint32_t IT_SET_UCONFIG_REG_INDEX = 0x7A; +constexpr uint32_t IT_WRITE_CONST_RAM = 0x81; +constexpr uint32_t IT_DUMP_CONST_RAM = 0x83; +constexpr uint32_t IT_INCREMENT_CE_COUNTER = 0x84; +constexpr uint32_t IT_INCREMENT_DE_COUNTER = 0x85; +constexpr uint32_t IT_WAIT_ON_CE_COUNTER = 0x86; +constexpr uint32_t IT_WAIT_ON_DE_COUNTER_DIFF = 0x88; +constexpr uint32_t IT_DISPATCH_DRAW = 0x8D; +constexpr uint32_t IT_GET_LOD_STATS = 0x8E; +constexpr uint32_t IT_SET_CONTEXT_REG_INDIRECT = 0x9F; + +/* Custom codes. Implemented via IT_NOP */ + +constexpr uint32_t R_ZERO = 0x00; +constexpr uint32_t R_DRAW_RESET = 0x05; +constexpr uint32_t R_WAIT_FLIP_DONE = 0x06; +constexpr uint32_t R_DISPATCH_RESET = 0x09; +constexpr uint32_t R_WAIT_MEM_32 = 0x0A; +constexpr uint32_t R_PUSH_MARKER = 0x0B; +constexpr uint32_t R_POP_MARKER = 0x0C; +constexpr uint32_t R_ACQUIRE_MEM = 0x14; +constexpr uint32_t R_WRITE_DATA = 0x15; +constexpr uint32_t R_WAIT_MEM_64 = 0x16; +constexpr uint32_t R_FLIP = 0x17; +constexpr uint32_t R_RELEASE_MEM = 0x18; +constexpr uint32_t R_DMA_DATA = 0x19; + +constexpr uint32_t R_NUM = 0x3F + 1; + +/* Context registers */ + +constexpr uint32_t DB_RENDER_CONTROL = 0x0; +constexpr uint32_t DB_RENDER_CONTROL_DEPTH_CLEAR_ENABLE_SHIFT = 0; +constexpr uint32_t DB_RENDER_CONTROL_DEPTH_CLEAR_ENABLE_MASK = 0x1; +constexpr uint32_t DB_RENDER_CONTROL_STENCIL_CLEAR_ENABLE_SHIFT = 1; +constexpr uint32_t DB_RENDER_CONTROL_STENCIL_CLEAR_ENABLE_MASK = 0x1; +constexpr uint32_t DB_RENDER_CONTROL_RESUMMARIZE_ENABLE_SHIFT = 4; +constexpr uint32_t DB_RENDER_CONTROL_RESUMMARIZE_ENABLE_MASK = 0x1; +constexpr uint32_t DB_RENDER_CONTROL_STENCIL_COMPRESS_DISABLE_SHIFT = 5; +constexpr uint32_t DB_RENDER_CONTROL_STENCIL_COMPRESS_DISABLE_MASK = 0x1; +constexpr uint32_t DB_RENDER_CONTROL_DEPTH_COMPRESS_DISABLE_SHIFT = 6; +constexpr uint32_t DB_RENDER_CONTROL_DEPTH_COMPRESS_DISABLE_MASK = 0x1; +constexpr uint32_t DB_RENDER_CONTROL_COPY_CENTROID_SHIFT = 7; +constexpr uint32_t DB_RENDER_CONTROL_COPY_CENTROID_MASK = 0x1; +constexpr uint32_t DB_RENDER_CONTROL_COPY_SAMPLE_SHIFT = 8; +constexpr uint32_t DB_RENDER_CONTROL_COPY_SAMPLE_MASK = 0xF; + +constexpr uint32_t DB_COUNT_CONTROL = 0x1; + +constexpr uint32_t DB_DEPTH_VIEW = 0x2; +constexpr uint32_t DB_DEPTH_VIEW_SLICE_START_SHIFT = 0; +constexpr uint32_t DB_DEPTH_VIEW_SLICE_START_MASK = 0x7FF; +constexpr uint32_t DB_DEPTH_VIEW_SLICE_START_HI_SHIFT = 11; +constexpr uint32_t DB_DEPTH_VIEW_SLICE_START_HI_MASK = 0x3; +constexpr uint32_t DB_DEPTH_VIEW_SLICE_MAX_SHIFT = 13; +constexpr uint32_t DB_DEPTH_VIEW_SLICE_MAX_MASK = 0x7FF; +constexpr uint32_t DB_DEPTH_VIEW_Z_READ_ONLY_SHIFT = 24; +constexpr uint32_t DB_DEPTH_VIEW_Z_READ_ONLY_MASK = 0x1; +constexpr uint32_t DB_DEPTH_VIEW_STENCIL_READ_ONLY_SHIFT = 25; +constexpr uint32_t DB_DEPTH_VIEW_STENCIL_READ_ONLY_MASK = 0x1; +constexpr uint32_t DB_DEPTH_VIEW_MIPID_SHIFT = 26; +constexpr uint32_t DB_DEPTH_VIEW_MIPID_MASK = 0xF; +constexpr uint32_t DB_DEPTH_VIEW_SLICE_MAX_HI_SHIFT = 30; +constexpr uint32_t DB_DEPTH_VIEW_SLICE_MAX_HI_MASK = 0x3; + +constexpr uint32_t DB_RENDER_OVERRIDE = 0x3; +constexpr uint32_t DB_RENDER_OVERRIDE2 = 0x4; +constexpr uint32_t DB_HTILE_DATA_BASE = 0x5; +constexpr uint32_t PS_SHADER_SAMPLE_EXCLUSION_MASK = 0x6; + +constexpr uint32_t DB_DEPTH_SIZE_XY = 0x7; +constexpr uint32_t DB_DEPTH_SIZE_XY_X_MAX_SHIFT = 0; +constexpr uint32_t DB_DEPTH_SIZE_XY_X_MAX_MASK = 0x3FFF; +constexpr uint32_t DB_DEPTH_SIZE_XY_Y_MAX_SHIFT = 16; +constexpr uint32_t DB_DEPTH_SIZE_XY_Y_MAX_MASK = 0x3FFF; + +constexpr uint32_t DB_DEPTH_BOUNDS_MIN = 0x8; +constexpr uint32_t DB_DEPTH_BOUNDS_MAX = 0x9; + +constexpr uint32_t DB_STENCIL_CLEAR = 0xA; +constexpr uint32_t DB_STENCIL_CLEAR_CLEAR_SHIFT = 0; +constexpr uint32_t DB_STENCIL_CLEAR_CLEAR_MASK = 0xFF; + +constexpr uint32_t DB_DEPTH_CLEAR = 0xB; +constexpr uint32_t DB_DEPTH_CLEAR_DEPTH_CLEAR_SHIFT = 0; +constexpr uint32_t DB_DEPTH_CLEAR_DEPTH_CLEAR_MASK = 0xFFFFFFFF; + +constexpr uint32_t PA_SC_SCREEN_SCISSOR_TL = 0xC; +constexpr uint32_t PA_SC_SCREEN_SCISSOR_TL_TL_X_SHIFT = 0; +constexpr uint32_t PA_SC_SCREEN_SCISSOR_TL_TL_X_MASK = 0xFFFF; +constexpr uint32_t PA_SC_SCREEN_SCISSOR_TL_TL_Y_SHIFT = 16; +constexpr uint32_t PA_SC_SCREEN_SCISSOR_TL_TL_Y_MASK = 0xFFFF; + +constexpr uint32_t PA_SC_SCREEN_SCISSOR_BR = 0xD; +constexpr uint32_t PA_SC_SCREEN_SCISSOR_BR_BR_X_SHIFT = 0; +constexpr uint32_t PA_SC_SCREEN_SCISSOR_BR_BR_X_MASK = 0xFFFF; +constexpr uint32_t PA_SC_SCREEN_SCISSOR_BR_BR_Y_SHIFT = 16; +constexpr uint32_t PA_SC_SCREEN_SCISSOR_BR_BR_Y_MASK = 0xFFFF; + +constexpr uint32_t DB_DFSM_CONTROL = 0xE; + +constexpr uint32_t DB_DEPTH_INFO = 0xF; +constexpr uint32_t DB_DEPTH_INFO_ADDR5_SWIZZLE_MASK_SHIFT = 0; +constexpr uint32_t DB_DEPTH_INFO_ADDR5_SWIZZLE_MASK_MASK = 0xF; +constexpr uint32_t DB_DEPTH_INFO_ARRAY_MODE_SHIFT = 4; +constexpr uint32_t DB_DEPTH_INFO_ARRAY_MODE_MASK = 0xF; +constexpr uint32_t DB_DEPTH_INFO_PIPE_CONFIG_SHIFT = 8; +constexpr uint32_t DB_DEPTH_INFO_PIPE_CONFIG_MASK = 0x1F; +constexpr uint32_t DB_DEPTH_INFO_BANK_WIDTH_SHIFT = 13; +constexpr uint32_t DB_DEPTH_INFO_BANK_WIDTH_MASK = 0x3; +constexpr uint32_t DB_DEPTH_INFO_BANK_HEIGHT_SHIFT = 15; +constexpr uint32_t DB_DEPTH_INFO_BANK_HEIGHT_MASK = 0x3; +constexpr uint32_t DB_DEPTH_INFO_MACRO_TILE_ASPECT_SHIFT = 17; +constexpr uint32_t DB_DEPTH_INFO_MACRO_TILE_ASPECT_MASK = 0x3; +constexpr uint32_t DB_DEPTH_INFO_NUM_BANKS_SHIFT = 19; +constexpr uint32_t DB_DEPTH_INFO_NUM_BANKS_MASK = 0x3; + +constexpr uint32_t DB_Z_INFO = 0x10; +constexpr uint32_t DB_Z_INFO_FORMAT_SHIFT = 0; +constexpr uint32_t DB_Z_INFO_FORMAT_MASK = 0x3; +constexpr uint32_t DB_Z_INFO_NUM_SAMPLES_SHIFT = 2; +constexpr uint32_t DB_Z_INFO_NUM_SAMPLES_MASK = 0x3; +constexpr uint32_t DB_Z_INFO_ITERATE_FLUSH_SHIFT = 11; +constexpr uint32_t DB_Z_INFO_ITERATE_FLUSH_MASK = 0x1; +constexpr uint32_t DB_Z_INFO_PARTIALLY_RESIDENT_SHIFT = 12; +constexpr uint32_t DB_Z_INFO_PARTIALLY_RESIDENT_MASK = 0x1; +constexpr uint32_t DB_Z_INFO_MAXMIP_SHIFT = 16; +constexpr uint32_t DB_Z_INFO_MAXMIP_MASK = 0xF; +constexpr uint32_t DB_Z_INFO_TILE_MODE_INDEX_SHIFT = 20; +constexpr uint32_t DB_Z_INFO_TILE_MODE_INDEX_MASK = 0x7; +constexpr uint32_t DB_Z_INFO_DECOMPRESS_ON_N_ZPLANES_SHIFT = 23; +constexpr uint32_t DB_Z_INFO_DECOMPRESS_ON_N_ZPLANES_MASK = 0xF; +constexpr uint32_t DB_Z_INFO_ALLOW_EXPCLEAR_SHIFT = 27; +constexpr uint32_t DB_Z_INFO_ALLOW_EXPCLEAR_MASK = 0x1; +constexpr uint32_t DB_Z_INFO_TILE_SURFACE_ENABLE_SHIFT = 29; +constexpr uint32_t DB_Z_INFO_TILE_SURFACE_ENABLE_MASK = 0x1; +constexpr uint32_t DB_Z_INFO_ZRANGE_PRECISION_SHIFT = 31; +constexpr uint32_t DB_Z_INFO_ZRANGE_PRECISION_MASK = 0x1; + +constexpr uint32_t DB_STENCIL_INFO = 0x11; +constexpr uint32_t DB_STENCIL_INFO_FORMAT_SHIFT = 0; +constexpr uint32_t DB_STENCIL_INFO_FORMAT_MASK = 0x1; +constexpr uint32_t DB_STENCIL_INFO_ITERATE_FLUSH_SHIFT = 11; +constexpr uint32_t DB_STENCIL_INFO_ITERATE_FLUSH_MASK = 0x1; +constexpr uint32_t DB_STENCIL_INFO_PARTIALLY_RESIDENT_SHIFT = 12; +constexpr uint32_t DB_STENCIL_INFO_PARTIALLY_RESIDENT_MASK = 0x1; +constexpr uint32_t DB_STENCIL_INFO_RESERVED_FIELD_1_SHIFT = 13; +constexpr uint32_t DB_STENCIL_INFO_RESERVED_FIELD_1_MASK = 0x7; +constexpr uint32_t DB_STENCIL_INFO_TILE_MODE_INDEX_SHIFT = 20; +constexpr uint32_t DB_STENCIL_INFO_TILE_MODE_INDEX_MASK = 0x7; +constexpr uint32_t DB_STENCIL_INFO_ALLOW_EXPCLEAR_SHIFT = 27; +constexpr uint32_t DB_STENCIL_INFO_ALLOW_EXPCLEAR_MASK = 0x1; +constexpr uint32_t DB_STENCIL_INFO_TILE_STENCIL_DISABLE_SHIFT = 29; +constexpr uint32_t DB_STENCIL_INFO_TILE_STENCIL_DISABLE_MASK = 0x1; + +constexpr uint32_t DB_Z_READ_BASE = 0x12; +constexpr uint32_t DB_STENCIL_READ_BASE = 0x13; +constexpr uint32_t DB_Z_WRITE_BASE = 0x14; +constexpr uint32_t DB_STENCIL_WRITE_BASE = 0x15; + +constexpr uint32_t DB_DEPTH_SIZE = 0x16; +constexpr uint32_t DB_DEPTH_SIZE_PITCH_TILE_MAX_SHIFT = 0; +constexpr uint32_t DB_DEPTH_SIZE_PITCH_TILE_MAX_MASK = 0x7FF; +constexpr uint32_t DB_DEPTH_SIZE_HEIGHT_TILE_MAX_SHIFT = 11; +constexpr uint32_t DB_DEPTH_SIZE_HEIGHT_TILE_MAX_MASK = 0x7FF; + +constexpr uint32_t DB_DEPTH_SLICE = 0x17; +constexpr uint32_t DB_DEPTH_SLICE_SLICE_TILE_MAX_SHIFT = 0; +constexpr uint32_t DB_DEPTH_SLICE_SLICE_TILE_MAX_MASK = 0x3FFFFF; + +constexpr uint32_t DB_Z_READ_BASE_HI = 0x1A; +constexpr uint32_t DB_STENCIL_READ_BASE_HI = 0x1B; +constexpr uint32_t DB_Z_WRITE_BASE_HI = 0x1C; +constexpr uint32_t DB_STENCIL_WRITE_BASE_HI = 0x1D; +constexpr uint32_t DB_HTILE_DATA_BASE_HI = 0x1E; +constexpr uint32_t DB_RMI_L2_CACHE_CONTROL = 0x1F; +constexpr uint32_t TA_BC_BASE_ADDR = 0x20; +constexpr uint32_t TA_BC_BASE_ADDR_HI = 0x21; +constexpr uint32_t PA_SC_WINDOW_OFFSET = 0x80; +constexpr uint32_t PA_SC_WINDOW_OFFSET_WINDOW_X_SHIFT = 0; +constexpr uint32_t PA_SC_WINDOW_OFFSET_WINDOW_X_MASK = 0xFFFF; +constexpr uint32_t PA_SC_WINDOW_OFFSET_WINDOW_Y_SHIFT = 16; +constexpr uint32_t PA_SC_WINDOW_OFFSET_WINDOW_Y_MASK = 0xFFFF; + +constexpr uint32_t PA_SC_WINDOW_SCISSOR_TL = 0x81; +constexpr uint32_t PA_SC_WINDOW_SCISSOR_TL_TL_X_SHIFT = 0; +constexpr uint32_t PA_SC_WINDOW_SCISSOR_TL_TL_X_MASK = 0x7FFF; +constexpr uint32_t PA_SC_WINDOW_SCISSOR_TL_TL_Y_SHIFT = 16; +constexpr uint32_t PA_SC_WINDOW_SCISSOR_TL_TL_Y_MASK = 0x7FFF; +constexpr uint32_t PA_SC_WINDOW_SCISSOR_TL_WINDOW_OFFSET_DISABLE_SHIFT = 31; +constexpr uint32_t PA_SC_WINDOW_SCISSOR_TL_WINDOW_OFFSET_DISABLE_MASK = 0x1; + +constexpr uint32_t PA_SC_WINDOW_SCISSOR_BR = 0x82; +constexpr uint32_t PA_SC_WINDOW_SCISSOR_BR_BR_X_SHIFT = 0; +constexpr uint32_t PA_SC_WINDOW_SCISSOR_BR_BR_X_MASK = 0x7FFF; +constexpr uint32_t PA_SC_WINDOW_SCISSOR_BR_BR_Y_SHIFT = 16; +constexpr uint32_t PA_SC_WINDOW_SCISSOR_BR_BR_Y_MASK = 0x7FFF; +constexpr uint32_t PA_SC_CLIPRECT_RULE = 0x83; +constexpr uint32_t PA_SC_CLIPRECT_0_TL = 0x84; +constexpr uint32_t PA_SC_CLIPRECT_0_BR = 0x85; + +constexpr uint32_t PA_SU_HARDWARE_SCREEN_OFFSET = 0x8D; +constexpr uint32_t PA_SU_HARDWARE_SCREEN_OFFSET_HW_SCREEN_OFFSET_X_SHIFT = 0; +constexpr uint32_t PA_SU_HARDWARE_SCREEN_OFFSET_HW_SCREEN_OFFSET_X_MASK = 0x1FF; +constexpr uint32_t PA_SU_HARDWARE_SCREEN_OFFSET_HW_SCREEN_OFFSET_Y_SHIFT = 16; +constexpr uint32_t PA_SU_HARDWARE_SCREEN_OFFSET_HW_SCREEN_OFFSET_Y_MASK = 0x1FF; + +constexpr uint32_t CB_TARGET_MASK = 0x8E; +constexpr uint32_t CB_SHADER_MASK = 0x8F; + +constexpr uint32_t PA_SC_GENERIC_SCISSOR_TL = 0x90; +constexpr uint32_t PA_SC_GENERIC_SCISSOR_TL_TL_X_SHIFT = 0; +constexpr uint32_t PA_SC_GENERIC_SCISSOR_TL_TL_X_MASK = 0x7FFF; +constexpr uint32_t PA_SC_GENERIC_SCISSOR_TL_TL_Y_SHIFT = 16; +constexpr uint32_t PA_SC_GENERIC_SCISSOR_TL_TL_Y_MASK = 0x7FFF; +constexpr uint32_t PA_SC_GENERIC_SCISSOR_TL_WINDOW_OFFSET_DISABLE_SHIFT = 31; +constexpr uint32_t PA_SC_GENERIC_SCISSOR_TL_WINDOW_OFFSET_DISABLE_MASK = 0x1; + +constexpr uint32_t PA_SC_GENERIC_SCISSOR_BR = 0x91; +constexpr uint32_t PA_SC_GENERIC_SCISSOR_BR_BR_X_SHIFT = 0; +constexpr uint32_t PA_SC_GENERIC_SCISSOR_BR_BR_X_MASK = 0x7FFF; +constexpr uint32_t PA_SC_GENERIC_SCISSOR_BR_BR_Y_SHIFT = 16; +constexpr uint32_t PA_SC_GENERIC_SCISSOR_BR_BR_Y_MASK = 0x7FFF; + +constexpr uint32_t PA_SC_VPORT_SCISSOR_0_TL = 0x94; +constexpr uint32_t PA_SC_VPORT_SCISSOR_0_TL_TL_X_SHIFT = 0; +constexpr uint32_t PA_SC_VPORT_SCISSOR_0_TL_TL_X_MASK = 0x7FFF; +constexpr uint32_t PA_SC_VPORT_SCISSOR_0_TL_TL_Y_SHIFT = 16; +constexpr uint32_t PA_SC_VPORT_SCISSOR_0_TL_TL_Y_MASK = 0x7FFF; +constexpr uint32_t PA_SC_VPORT_SCISSOR_0_TL_WINDOW_OFFSET_DISABLE_SHIFT = 31; +constexpr uint32_t PA_SC_VPORT_SCISSOR_0_TL_WINDOW_OFFSET_DISABLE_MASK = 0x1; + +constexpr uint32_t PA_SC_VPORT_SCISSOR_0_BR = 0x95; +constexpr uint32_t PA_SC_VPORT_SCISSOR_0_BR_BR_X_SHIFT = 0; +constexpr uint32_t PA_SC_VPORT_SCISSOR_0_BR_BR_X_MASK = 0x7FFF; +constexpr uint32_t PA_SC_VPORT_SCISSOR_0_BR_BR_Y_SHIFT = 16; +constexpr uint32_t PA_SC_VPORT_SCISSOR_0_BR_BR_Y_MASK = 0x7FFF; + +constexpr uint32_t PA_SC_VPORT_SCISSOR_15_TL = 0xB2; +constexpr uint32_t PA_SC_VPORT_SCISSOR_15_BR = 0xB3; +constexpr uint32_t PA_SC_VPORT_ZMIN_0 = 0xB4; +constexpr uint32_t PA_SC_VPORT_ZMAX_0 = 0xB5; +constexpr uint32_t PA_SC_VPORT_ZMIN_15 = 0xD2; +constexpr uint32_t PA_SC_VPORT_ZMAX_15 = 0xD3; +constexpr uint32_t PA_SC_RIGHT_VERT_GRID = 0xE8; +constexpr uint32_t PA_SC_LEFT_VERT_GRID = 0xE9; +constexpr uint32_t PA_SC_HORIZ_GRID = 0xEA; +constexpr uint32_t PA_SC_FOV_WINDOW_LR = 0xEB; +constexpr uint32_t PA_SC_FOV_WINDOW_TB = 0xEC; +constexpr uint32_t PA_SC_FSR_ENABLE = 0xF0; +constexpr uint32_t FSR_RECURSIONS0 = 0xF1; +constexpr uint32_t FSR_RECURSIONS1 = 0xF2; +constexpr uint32_t VGT_MULTI_PRIM_IB_RESET_INDX = 0x103; +constexpr uint32_t CB_RMI_GL2_CACHE_CONTROL = 0x104; +constexpr uint32_t CB_BLEND_RED = 0x105; +constexpr uint32_t CB_BLEND_GREEN = 0x106; +constexpr uint32_t CB_BLEND_BLUE = 0x107; +constexpr uint32_t CB_BLEND_ALPHA = 0x108; +constexpr uint32_t CB_DCC_CONTROL = 0x109; + +constexpr uint32_t DB_STENCIL_CONTROL = 0x10B; +constexpr uint32_t DB_STENCIL_CONTROL_STENCILFAIL_SHIFT = 0; +constexpr uint32_t DB_STENCIL_CONTROL_STENCILFAIL_MASK = 0xF; +constexpr uint32_t DB_STENCIL_CONTROL_STENCILZPASS_SHIFT = 4; +constexpr uint32_t DB_STENCIL_CONTROL_STENCILZPASS_MASK = 0xF; +constexpr uint32_t DB_STENCIL_CONTROL_STENCILZFAIL_SHIFT = 8; +constexpr uint32_t DB_STENCIL_CONTROL_STENCILZFAIL_MASK = 0xF; +constexpr uint32_t DB_STENCIL_CONTROL_STENCILFAIL_BF_SHIFT = 12; +constexpr uint32_t DB_STENCIL_CONTROL_STENCILFAIL_BF_MASK = 0xF; +constexpr uint32_t DB_STENCIL_CONTROL_STENCILZPASS_BF_SHIFT = 16; +constexpr uint32_t DB_STENCIL_CONTROL_STENCILZPASS_BF_MASK = 0xF; +constexpr uint32_t DB_STENCIL_CONTROL_STENCILZFAIL_BF_SHIFT = 20; +constexpr uint32_t DB_STENCIL_CONTROL_STENCILZFAIL_BF_MASK = 0xF; + +constexpr uint32_t DB_STENCILREFMASK = 0x10C; +constexpr uint32_t DB_STENCILREFMASK_STENCILTESTVAL_SHIFT = 0; +constexpr uint32_t DB_STENCILREFMASK_STENCILTESTVAL_MASK = 0xFF; +constexpr uint32_t DB_STENCILREFMASK_STENCILMASK_SHIFT = 8; +constexpr uint32_t DB_STENCILREFMASK_STENCILMASK_MASK = 0xFF; +constexpr uint32_t DB_STENCILREFMASK_STENCILWRITEMASK_SHIFT = 16; +constexpr uint32_t DB_STENCILREFMASK_STENCILWRITEMASK_MASK = 0xFF; +constexpr uint32_t DB_STENCILREFMASK_STENCILOPVAL_SHIFT = 24; +constexpr uint32_t DB_STENCILREFMASK_STENCILOPVAL_MASK = 0xFF; + +constexpr uint32_t DB_STENCILREFMASK_BF = 0x10D; +constexpr uint32_t DB_STENCILREFMASK_BF_STENCILTESTVAL_BF_SHIFT = 0; +constexpr uint32_t DB_STENCILREFMASK_BF_STENCILTESTVAL_BF_MASK = 0xFF; +constexpr uint32_t DB_STENCILREFMASK_BF_STENCILMASK_BF_SHIFT = 8; +constexpr uint32_t DB_STENCILREFMASK_BF_STENCILMASK_BF_MASK = 0xFF; +constexpr uint32_t DB_STENCILREFMASK_BF_STENCILWRITEMASK_BF_SHIFT = 16; +constexpr uint32_t DB_STENCILREFMASK_BF_STENCILWRITEMASK_BF_MASK = 0xFF; +constexpr uint32_t DB_STENCILREFMASK_BF_STENCILOPVAL_BF_SHIFT = 24; +constexpr uint32_t DB_STENCILREFMASK_BF_STENCILOPVAL_BF_MASK = 0xFF; + +constexpr uint32_t PA_CL_VPORT_XSCALE = 0x10F; +constexpr uint32_t PA_CL_VPORT_XOFFSET = 0x110; +constexpr uint32_t PA_CL_VPORT_YSCALE = 0x111; +constexpr uint32_t PA_CL_VPORT_YOFFSET = 0x112; +constexpr uint32_t PA_CL_VPORT_ZSCALE = 0x113; +constexpr uint32_t PA_CL_VPORT_ZOFFSET = 0x114; +constexpr uint32_t PA_CL_VPORT_XSCALE_15 = 0x169; +constexpr uint32_t PA_CL_VPORT_XOFFSET_15 = 0x16A; +constexpr uint32_t PA_CL_VPORT_YSCALE_15 = 0x16B; +constexpr uint32_t PA_CL_VPORT_YOFFSET_15 = 0x16C; +constexpr uint32_t PA_CL_VPORT_ZSCALE_15 = 0x16D; +constexpr uint32_t PA_CL_VPORT_ZOFFSET_15 = 0x16E; +constexpr uint32_t PA_CL_UCP_0_X = 0x16F; +constexpr uint32_t PA_CL_UCP_0_Y = 0x170; +constexpr uint32_t PA_CL_UCP_0_Z = 0x171; +constexpr uint32_t PA_CL_UCP_0_W = 0x172; +constexpr uint32_t SPI_PS_INPUT_CNTL_0 = 0x191; +constexpr uint32_t SPI_PS_INPUT_CNTL_31 = 0x1B0; +constexpr uint32_t SPI_VS_OUT_CONFIG = 0x1B1; +constexpr uint32_t SPI_PS_INPUT_ENA = 0x1B3; +constexpr uint32_t SPI_PS_INPUT_ADDR = 0x1B4; +constexpr uint32_t SPI_INTERP_CONTROL_0 = 0x1B5; +constexpr uint32_t SPI_PS_IN_CONTROL = 0x1B6; +constexpr uint32_t SPI_BARYC_CNTL = 0x1B8; +constexpr uint32_t SPI_TMPRING_SIZE = 0x1BA; +constexpr uint32_t SPI_SHADER_IDX_FORMAT = 0x1C2; +constexpr uint32_t SPI_SHADER_POS_FORMAT = 0x1C3; +constexpr uint32_t SPI_SHADER_Z_FORMAT = 0x1C4; +constexpr uint32_t SPI_SHADER_COL_FORMAT = 0x1C5; + +constexpr uint32_t CB_BLEND0_CONTROL = 0x1E0; +constexpr uint32_t CB_BLEND0_CONTROL_COLOR_SRCBLEND_SHIFT = 0; +constexpr uint32_t CB_BLEND0_CONTROL_COLOR_SRCBLEND_MASK = 0x1F; +constexpr uint32_t CB_BLEND0_CONTROL_COLOR_COMB_FCN_SHIFT = 5; +constexpr uint32_t CB_BLEND0_CONTROL_COLOR_COMB_FCN_MASK = 0x7; +constexpr uint32_t CB_BLEND0_CONTROL_COLOR_DESTBLEND_SHIFT = 8; +constexpr uint32_t CB_BLEND0_CONTROL_COLOR_DESTBLEND_MASK = 0x1F; +constexpr uint32_t CB_BLEND0_CONTROL_ALPHA_SRCBLEND_SHIFT = 16; +constexpr uint32_t CB_BLEND0_CONTROL_ALPHA_SRCBLEND_MASK = 0x1F; +constexpr uint32_t CB_BLEND0_CONTROL_ALPHA_COMB_FCN_SHIFT = 21; +constexpr uint32_t CB_BLEND0_CONTROL_ALPHA_COMB_FCN_MASK = 0x7; +constexpr uint32_t CB_BLEND0_CONTROL_ALPHA_DESTBLEND_SHIFT = 24; +constexpr uint32_t CB_BLEND0_CONTROL_ALPHA_DESTBLEND_MASK = 0x1F; +constexpr uint32_t CB_BLEND0_CONTROL_SEPARATE_ALPHA_BLEND_SHIFT = 29; +constexpr uint32_t CB_BLEND0_CONTROL_SEPARATE_ALPHA_BLEND_MASK = 0x1; +constexpr uint32_t CB_BLEND0_CONTROL_ENABLE_SHIFT = 30; +constexpr uint32_t CB_BLEND0_CONTROL_ENABLE_MASK = 0x1; + +constexpr uint32_t GE_MAX_OUTPUT_PER_SUBGROUP = 0x1FF; + +constexpr uint32_t DB_DEPTH_CONTROL = 0x200; +constexpr uint32_t DB_DEPTH_CONTROL_STENCIL_ENABLE_SHIFT = 0; +constexpr uint32_t DB_DEPTH_CONTROL_STENCIL_ENABLE_MASK = 0x1; +constexpr uint32_t DB_DEPTH_CONTROL_Z_ENABLE_SHIFT = 1; +constexpr uint32_t DB_DEPTH_CONTROL_Z_ENABLE_MASK = 0x1; +constexpr uint32_t DB_DEPTH_CONTROL_Z_WRITE_ENABLE_SHIFT = 2; +constexpr uint32_t DB_DEPTH_CONTROL_Z_WRITE_ENABLE_MASK = 0x1; +constexpr uint32_t DB_DEPTH_CONTROL_DEPTH_BOUNDS_ENABLE_SHIFT = 3; +constexpr uint32_t DB_DEPTH_CONTROL_DEPTH_BOUNDS_ENABLE_MASK = 0x1; +constexpr uint32_t DB_DEPTH_CONTROL_ZFUNC_SHIFT = 4; +constexpr uint32_t DB_DEPTH_CONTROL_ZFUNC_MASK = 0x7; +constexpr uint32_t DB_DEPTH_CONTROL_BACKFACE_ENABLE_SHIFT = 7; +constexpr uint32_t DB_DEPTH_CONTROL_BACKFACE_ENABLE_MASK = 0x1; +constexpr uint32_t DB_DEPTH_CONTROL_STENCILFUNC_SHIFT = 8; +constexpr uint32_t DB_DEPTH_CONTROL_STENCILFUNC_MASK = 0x7; +constexpr uint32_t DB_DEPTH_CONTROL_STENCILFUNC_BF_SHIFT = 20; +constexpr uint32_t DB_DEPTH_CONTROL_STENCILFUNC_BF_MASK = 0x7; +constexpr uint32_t DB_DEPTH_CONTROL_ENABLE_COLOR_WRITES_ON_DEPTH_FAIL_SHIFT = 30; +constexpr uint32_t DB_DEPTH_CONTROL_ENABLE_COLOR_WRITES_ON_DEPTH_FAIL_MASK = 0x1; +constexpr uint32_t DB_DEPTH_CONTROL_DISABLE_COLOR_WRITES_ON_DEPTH_PASS_SHIFT = 31; +constexpr uint32_t DB_DEPTH_CONTROL_DISABLE_COLOR_WRITES_ON_DEPTH_PASS_MASK = 0x1; + +constexpr uint32_t DB_EQAA = 0x201; +constexpr uint32_t DB_EQAA_MAX_ANCHOR_SAMPLES_SHIFT = 0; +constexpr uint32_t DB_EQAA_MAX_ANCHOR_SAMPLES_MASK = 0x7; +constexpr uint32_t DB_EQAA_PS_ITER_SAMPLES_SHIFT = 4; +constexpr uint32_t DB_EQAA_PS_ITER_SAMPLES_MASK = 0x7; +constexpr uint32_t DB_EQAA_MASK_EXPORT_NUM_SAMPLES_SHIFT = 8; +constexpr uint32_t DB_EQAA_MASK_EXPORT_NUM_SAMPLES_MASK = 0x7; +constexpr uint32_t DB_EQAA_ALPHA_TO_MASK_NUM_SAMPLES_SHIFT = 12; +constexpr uint32_t DB_EQAA_ALPHA_TO_MASK_NUM_SAMPLES_MASK = 0x7; +constexpr uint32_t DB_EQAA_HIGH_QUALITY_INTERSECTIONS_SHIFT = 16; +constexpr uint32_t DB_EQAA_HIGH_QUALITY_INTERSECTIONS_MASK = 0x1; +constexpr uint32_t DB_EQAA_INCOHERENT_EQAA_READS_SHIFT = 17; +constexpr uint32_t DB_EQAA_INCOHERENT_EQAA_READS_MASK = 0x1; +constexpr uint32_t DB_EQAA_INTERPOLATE_COMP_Z_SHIFT = 18; +constexpr uint32_t DB_EQAA_INTERPOLATE_COMP_Z_MASK = 0x1; +constexpr uint32_t DB_EQAA_STATIC_ANCHOR_ASSOCIATIONS_SHIFT = 20; +constexpr uint32_t DB_EQAA_STATIC_ANCHOR_ASSOCIATIONS_MASK = 0x1; + +constexpr uint32_t DB_SHADER_CONTROL = 0x203; +constexpr uint32_t DB_SHADER_CONTROL_Z_EXPORT_ENABLE_SHIFT = 0; +constexpr uint32_t DB_SHADER_CONTROL_Z_EXPORT_ENABLE_MASK = 0x1; +constexpr uint32_t DB_SHADER_CONTROL_Z_ORDER_SHIFT = 4; +constexpr uint32_t DB_SHADER_CONTROL_Z_ORDER_MASK = 0x3; +constexpr uint32_t DB_SHADER_CONTROL_KILL_ENABLE_SHIFT = 6; +constexpr uint32_t DB_SHADER_CONTROL_KILL_ENABLE_MASK = 0x1; +constexpr uint32_t DB_SHADER_CONTROL_MASK_EXPORT_ENABLE_SHIFT = 8; +constexpr uint32_t DB_SHADER_CONTROL_MASK_EXPORT_ENABLE_MASK = 0x1; +constexpr uint32_t DB_SHADER_CONTROL_DUAL_EXPORT_ENABLE_SHIFT = 9; +constexpr uint32_t DB_SHADER_CONTROL_DUAL_EXPORT_ENABLE_MASK = 0x1; +constexpr uint32_t DB_SHADER_CONTROL_EXEC_ON_NOOP_SHIFT = 10; +constexpr uint32_t DB_SHADER_CONTROL_EXEC_ON_NOOP_MASK = 0x1; +constexpr uint32_t DB_SHADER_CONTROL_ALPHA_TO_MASK_DISABLE_SHIFT = 11; +constexpr uint32_t DB_SHADER_CONTROL_ALPHA_TO_MASK_DISABLE_MASK = 0x1; +constexpr uint32_t DB_SHADER_CONTROL_CONSERVATIVE_Z_EXPORT_SHIFT = 13; +constexpr uint32_t DB_SHADER_CONTROL_CONSERVATIVE_Z_EXPORT_MASK = 0x3; + +constexpr uint32_t CB_COLOR_CONTROL = 0x202; +constexpr uint32_t CB_COLOR_CONTROL_MODE_SHIFT = 4; +constexpr uint32_t CB_COLOR_CONTROL_MODE_MASK = 0x7; +constexpr uint32_t CB_COLOR_CONTROL_ROP3_SHIFT = 16; +constexpr uint32_t CB_COLOR_CONTROL_ROP3_MASK = 0xFF; + +constexpr uint32_t PA_CL_CLIP_CNTL = 0x204; +constexpr uint32_t PA_CL_CLIP_CNTL_UCP_ENA_SHIFT = 0; +constexpr uint32_t PA_CL_CLIP_CNTL_UCP_ENA_MASK = 0x3F; +constexpr uint32_t PA_CL_CLIP_CNTL_PS_UCP_Y_SCALE_NEG_SHIFT = 13; +constexpr uint32_t PA_CL_CLIP_CNTL_PS_UCP_Y_SCALE_NEG_MASK = 0x1; +constexpr uint32_t PA_CL_CLIP_CNTL_PS_UCP_MODE_SHIFT = 14; +constexpr uint32_t PA_CL_CLIP_CNTL_PS_UCP_MODE_MASK = 0x3; +constexpr uint32_t PA_CL_CLIP_CNTL_CLIP_DISABLE_SHIFT = 16; +constexpr uint32_t PA_CL_CLIP_CNTL_CLIP_DISABLE_MASK = 0x1; +constexpr uint32_t PA_CL_CLIP_CNTL_UCP_CULL_ONLY_ENA_SHIFT = 17; +constexpr uint32_t PA_CL_CLIP_CNTL_UCP_CULL_ONLY_ENA_MASK = 0x1; +constexpr uint32_t PA_CL_CLIP_CNTL_BOUNDARY_EDGE_FLAG_ENA_SHIFT = 18; +constexpr uint32_t PA_CL_CLIP_CNTL_BOUNDARY_EDGE_FLAG_ENA_MASK = 0x1; +constexpr uint32_t PA_CL_CLIP_CNTL_DX_CLIP_SPACE_DEF_SHIFT = 19; +constexpr uint32_t PA_CL_CLIP_CNTL_DX_CLIP_SPACE_DEF_MASK = 0x1; +constexpr uint32_t PA_CL_CLIP_CNTL_DIS_CLIP_ERR_DETECT_SHIFT = 20; +constexpr uint32_t PA_CL_CLIP_CNTL_DIS_CLIP_ERR_DETECT_MASK = 0x1; +constexpr uint32_t PA_CL_CLIP_CNTL_VTX_KILL_OR_SHIFT = 21; +constexpr uint32_t PA_CL_CLIP_CNTL_VTX_KILL_OR_MASK = 0x1; +constexpr uint32_t PA_CL_CLIP_CNTL_DX_RASTERIZATION_KILL_SHIFT = 22; +constexpr uint32_t PA_CL_CLIP_CNTL_DX_RASTERIZATION_KILL_MASK = 0x1; +constexpr uint32_t PA_CL_CLIP_CNTL_DX_LINEAR_ATTR_CLIP_ENA_SHIFT = 24; +constexpr uint32_t PA_CL_CLIP_CNTL_DX_LINEAR_ATTR_CLIP_ENA_MASK = 0x1; +constexpr uint32_t PA_CL_CLIP_CNTL_VTE_VPORT_PROVOKE_DISABLE_SHIFT = 25; +constexpr uint32_t PA_CL_CLIP_CNTL_VTE_VPORT_PROVOKE_DISABLE_MASK = 0x1; +constexpr uint32_t PA_CL_CLIP_CNTL_ZCLIP_NEAR_DISABLE_SHIFT = 26; +constexpr uint32_t PA_CL_CLIP_CNTL_ZCLIP_NEAR_DISABLE_MASK = 0x1; +constexpr uint32_t PA_CL_CLIP_CNTL_ZCLIP_FAR_DISABLE_SHIFT = 27; +constexpr uint32_t PA_CL_CLIP_CNTL_ZCLIP_FAR_DISABLE_MASK = 0x1; +constexpr uint32_t PA_CL_CLIP_CNTL_ZCLIP_PROG_NEAR_ENA_SHIFT = 28; +constexpr uint32_t PA_CL_CLIP_CNTL_ZCLIP_PROG_NEAR_ENA_MASK = 0x1; + +constexpr uint32_t PA_SU_SC_MODE_CNTL = 0x205; +constexpr uint32_t PA_SU_SC_MODE_CNTL_CULL_FRONT_SHIFT = 0; +constexpr uint32_t PA_SU_SC_MODE_CNTL_CULL_FRONT_MASK = 0x1; +constexpr uint32_t PA_SU_SC_MODE_CNTL_CULL_BACK_SHIFT = 1; +constexpr uint32_t PA_SU_SC_MODE_CNTL_CULL_BACK_MASK = 0x1; +constexpr uint32_t PA_SU_SC_MODE_CNTL_FACE_SHIFT = 2; +constexpr uint32_t PA_SU_SC_MODE_CNTL_FACE_MASK = 0x1; +constexpr uint32_t PA_SU_SC_MODE_CNTL_POLY_MODE_SHIFT = 3; +constexpr uint32_t PA_SU_SC_MODE_CNTL_POLY_MODE_MASK = 0x3; +constexpr uint32_t PA_SU_SC_MODE_CNTL_POLYMODE_FRONT_PTYPE_SHIFT = 5; +constexpr uint32_t PA_SU_SC_MODE_CNTL_POLYMODE_FRONT_PTYPE_MASK = 0x7; +constexpr uint32_t PA_SU_SC_MODE_CNTL_POLYMODE_BACK_PTYPE_SHIFT = 8; +constexpr uint32_t PA_SU_SC_MODE_CNTL_POLYMODE_BACK_PTYPE_MASK = 0x7; +constexpr uint32_t PA_SU_SC_MODE_CNTL_POLY_OFFSET_FRONT_ENABLE_SHIFT = 11; +constexpr uint32_t PA_SU_SC_MODE_CNTL_POLY_OFFSET_FRONT_ENABLE_MASK = 0x1; +constexpr uint32_t PA_SU_SC_MODE_CNTL_POLY_OFFSET_BACK_ENABLE_SHIFT = 12; +constexpr uint32_t PA_SU_SC_MODE_CNTL_POLY_OFFSET_BACK_ENABLE_MASK = 0x1; +constexpr uint32_t PA_SU_SC_MODE_CNTL_VTX_WINDOW_OFFSET_ENABLE_SHIFT = 16; +constexpr uint32_t PA_SU_SC_MODE_CNTL_VTX_WINDOW_OFFSET_ENABLE_MASK = 0x1; +constexpr uint32_t PA_SU_SC_MODE_CNTL_PROVOKING_VTX_LAST_SHIFT = 19; +constexpr uint32_t PA_SU_SC_MODE_CNTL_PROVOKING_VTX_LAST_MASK = 0x1; +constexpr uint32_t PA_SU_SC_MODE_CNTL_PERSP_CORR_DIS_SHIFT = 20; +constexpr uint32_t PA_SU_SC_MODE_CNTL_PERSP_CORR_DIS_MASK = 0x1; + +constexpr uint32_t PA_CL_VTE_CNTL = 0x206; +constexpr uint32_t PA_CL_VS_OUT_CNTL = 0x207; +constexpr uint32_t PA_SU_SMALL_PRIM_FILTER_CNTL = 0x20C; +constexpr uint32_t PA_CL_OBJPRIM_ID_CNTL = 0x20D; +constexpr uint32_t PA_STEREO_CNTL = 0x210; +constexpr uint32_t PA_STATE_STEREO_X = 0x211; +constexpr uint32_t PA_SU_POINT_SIZE = 0x280; +constexpr uint32_t PA_SU_POINT_MINMAX = 0x281; + +constexpr uint32_t PA_SU_LINE_CNTL = 0x282; +constexpr uint32_t PA_SU_LINE_CNTL_WIDTH_SHIFT = 0; +constexpr uint32_t PA_SU_LINE_CNTL_WIDTH_MASK = 0xFFFF; + +constexpr uint32_t VGT_HOS_MAX_TESS_LEVEL = 0x286; +constexpr uint32_t VGT_HOS_MIN_TESS_LEVEL = 0x287; +constexpr uint32_t VGT_GS_ONCHIP_CNTL = 0x291; + +constexpr uint32_t PA_SC_MODE_CNTL_0 = 0x292; +constexpr uint32_t PA_SC_MODE_CNTL_0_MSAA_ENABLE_SHIFT = 0; +constexpr uint32_t PA_SC_MODE_CNTL_0_MSAA_ENABLE_MASK = 0x1; +constexpr uint32_t PA_SC_MODE_CNTL_0_VPORT_SCISSOR_ENABLE_SHIFT = 1; +constexpr uint32_t PA_SC_MODE_CNTL_0_VPORT_SCISSOR_ENABLE_MASK = 0x1; +constexpr uint32_t PA_SC_MODE_CNTL_0_LINE_STIPPLE_ENABLE_SHIFT = 2; +constexpr uint32_t PA_SC_MODE_CNTL_0_LINE_STIPPLE_ENABLE_MASK = 0x1; + +constexpr uint32_t PA_SC_MODE_CNTL_1 = 0x293; +constexpr uint32_t VGT_GS_OUT_PRIM_TYPE = 0x29B; +constexpr uint32_t VGT_PRIMITIVEID_EN = 0x2A1; +constexpr uint32_t VGT_PRIMITIVEID_RESET = 0x2A3; +constexpr uint32_t VGT_DRAW_PAYLOAD_CNTL = 0x2A6; +constexpr uint32_t VGT_ESGS_RING_ITEMSIZE = 0x2AB; +constexpr uint32_t VGT_REUSE_OFF = 0x2AD; + +constexpr uint32_t DB_HTILE_SURFACE = 0x2AF; +constexpr uint32_t DB_HTILE_SURFACE_LINEAR_SHIFT = 0; +constexpr uint32_t DB_HTILE_SURFACE_LINEAR_MASK = 0x1; +constexpr uint32_t DB_HTILE_SURFACE_FULL_CACHE_SHIFT = 1; +constexpr uint32_t DB_HTILE_SURFACE_FULL_CACHE_MASK = 0x1; +constexpr uint32_t DB_HTILE_SURFACE_HTILE_USES_PRELOAD_WIN_SHIFT = 2; +constexpr uint32_t DB_HTILE_SURFACE_HTILE_USES_PRELOAD_WIN_MASK = 0x1; +constexpr uint32_t DB_HTILE_SURFACE_PRELOAD_SHIFT = 3; +constexpr uint32_t DB_HTILE_SURFACE_PRELOAD_MASK = 0x1; +constexpr uint32_t DB_HTILE_SURFACE_PREFETCH_WIDTH_SHIFT = 4; +constexpr uint32_t DB_HTILE_SURFACE_PREFETCH_WIDTH_MASK = 0x3F; +constexpr uint32_t DB_HTILE_SURFACE_PREFETCH_HEIGHT_SHIFT = 10; +constexpr uint32_t DB_HTILE_SURFACE_PREFETCH_HEIGHT_MASK = 0x3F; +constexpr uint32_t DB_HTILE_SURFACE_DST_OUTSIDE_ZERO_TO_ONE_SHIFT = 16; +constexpr uint32_t DB_HTILE_SURFACE_DST_OUTSIDE_ZERO_TO_ONE_MASK = 0x1; + +constexpr uint32_t DB_SRESULTS_COMPARE_STATE0 = 0x2B0; +constexpr uint32_t DB_SRESULTS_COMPARE_STATE1 = 0x2B1; +constexpr uint32_t VGT_GS_MAX_VERT_OUT = 0x2CE; +constexpr uint32_t GE_NGG_SUBGRP_CNTL = 0x2D3; +constexpr uint32_t VGT_TESS_DISTRIBUTION = 0x2D4; +constexpr uint32_t VGT_SHADER_STAGES_EN = 0x2D5; +constexpr uint32_t VGT_LS_HS_CONFIG = 0x2D6; +constexpr uint32_t VGT_TF_PARAM = 0x2DB; +constexpr uint32_t DB_ALPHA_TO_MASK = 0x2DC; +constexpr uint32_t PA_SU_POLY_OFFSET_DB_FMT_CNTL = 0x2DE; +constexpr uint32_t PA_SU_POLY_OFFSET_CLAMP = 0x2DF; +constexpr uint32_t PA_SU_POLY_OFFSET_FRONT_SCALE = 0x2E0; +constexpr uint32_t PA_SU_POLY_OFFSET_FRONT_OFFSET = 0x2E1; +constexpr uint32_t PA_SU_POLY_OFFSET_BACK_SCALE = 0x2E2; +constexpr uint32_t PA_SU_POLY_OFFSET_BACK_OFFSET = 0x2E3; +constexpr uint32_t VGT_GS_INSTANCE_CNT = 0x2E4; +constexpr uint32_t PA_SC_CENTROID_PRIORITY_0 = 0x2F5; +constexpr uint32_t PA_SC_CENTROID_PRIORITY_1 = 0x2F6; + +constexpr uint32_t PA_SC_AA_CONFIG = 0x2F8; +constexpr uint32_t PA_SC_AA_CONFIG_MSAA_NUM_SAMPLES_SHIFT = 0; +constexpr uint32_t PA_SC_AA_CONFIG_MSAA_NUM_SAMPLES_MASK = 0x7; +constexpr uint32_t PA_SC_AA_CONFIG_AA_MASK_CENTROID_DTMN_SHIFT = 4; +constexpr uint32_t PA_SC_AA_CONFIG_AA_MASK_CENTROID_DTMN_MASK = 0x1; +constexpr uint32_t PA_SC_AA_CONFIG_MAX_SAMPLE_DIST_SHIFT = 13; +constexpr uint32_t PA_SC_AA_CONFIG_MAX_SAMPLE_DIST_MASK = 0xF; +constexpr uint32_t PA_SC_AA_CONFIG_MSAA_EXPOSED_SAMPLES_SHIFT = 20; +constexpr uint32_t PA_SC_AA_CONFIG_MSAA_EXPOSED_SAMPLES_MASK = 0x7; + +constexpr uint32_t PA_SU_VTX_CNTL = 0x2F9; +constexpr uint32_t PA_CL_GB_VERT_CLIP_ADJ = 0x2FA; +constexpr uint32_t PA_CL_GB_VERT_DISC_ADJ = 0x2FB; +constexpr uint32_t PA_CL_GB_HORZ_CLIP_ADJ = 0x2FC; +constexpr uint32_t PA_CL_GB_HORZ_DISC_ADJ = 0x2FD; +constexpr uint32_t PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_0 = 0x2FE; +constexpr uint32_t PA_SC_AA_MASK_X0Y0_X1Y0 = 0x30E; +constexpr uint32_t PA_SC_AA_MASK_X0Y1_X1Y1 = 0x30F; +constexpr uint32_t PA_SC_SHADER_CONTROL = 0x310; +constexpr uint32_t PA_SC_BINNER_CNTL_0 = 0x311; +constexpr uint32_t PA_SC_BINNER_CNTL_1 = 0x312; +constexpr uint32_t PA_SC_CONSERVATIVE_RASTERIZATION_CNTL = 0x313; +constexpr uint32_t PA_SC_NGG_MODE_CNTL = 0x314; +constexpr uint32_t CB_COLOR0_BASE = 0x318; + +constexpr uint32_t CB_COLOR0_VIEW = 0x31B; +constexpr uint32_t CB_COLOR0_VIEW_SLICE_START_SHIFT = 0; +constexpr uint32_t CB_COLOR0_VIEW_SLICE_START_MASK = 0x1FFF; +constexpr uint32_t CB_COLOR0_VIEW_SLICE_MAX_SHIFT = 13; +constexpr uint32_t CB_COLOR0_VIEW_SLICE_MAX_MASK = 0x1FFF; +constexpr uint32_t CB_COLOR0_VIEW_MIP_LEVEL_SHIFT = 26; +constexpr uint32_t CB_COLOR0_VIEW_MIP_LEVEL_MASK = 0xF; + +constexpr uint32_t CB_COLOR0_INFO = 0x31C; +constexpr uint32_t CB_COLOR0_INFO_FORMAT_SHIFT = 2; +constexpr uint32_t CB_COLOR0_INFO_FORMAT_MASK = 0x1F; +constexpr uint32_t CB_COLOR0_INFO_NUMBER_TYPE_SHIFT = 8; +constexpr uint32_t CB_COLOR0_INFO_NUMBER_TYPE_MASK = 0x7; +constexpr uint32_t CB_COLOR0_INFO_COMP_SWAP_SHIFT = 11; +constexpr uint32_t CB_COLOR0_INFO_COMP_SWAP_MASK = 0x3; +constexpr uint32_t CB_COLOR0_INFO_FAST_CLEAR_SHIFT = 13; +constexpr uint32_t CB_COLOR0_INFO_FAST_CLEAR_MASK = 0x1; +constexpr uint32_t CB_COLOR0_INFO_COMPRESSION_SHIFT = 14; +constexpr uint32_t CB_COLOR0_INFO_COMPRESSION_MASK = 0x1; +constexpr uint32_t CB_COLOR0_INFO_BLEND_CLAMP_SHIFT = 15; +constexpr uint32_t CB_COLOR0_INFO_BLEND_CLAMP_MASK = 0x1; +constexpr uint32_t CB_COLOR0_INFO_BLEND_BYPASS_SHIFT = 16; +constexpr uint32_t CB_COLOR0_INFO_BLEND_BYPASS_MASK = 0x1; +constexpr uint32_t CB_COLOR0_INFO_ROUND_MODE_SHIFT = 18; +constexpr uint32_t CB_COLOR0_INFO_ROUND_MODE_MASK = 0x1; +constexpr uint32_t CB_COLOR0_INFO_CMASK_IS_LINEAR_SHIFT = 19; +constexpr uint32_t CB_COLOR0_INFO_CMASK_IS_LINEAR_MASK = 0x1; +constexpr uint32_t CB_COLOR0_INFO_FMASK_COMPRESSION_DISABLE_SHIFT = 26; +constexpr uint32_t CB_COLOR0_INFO_FMASK_COMPRESSION_DISABLE_MASK = 0x1; +constexpr uint32_t CB_COLOR0_INFO_FMASK_COMPRESS_1FRAG_ONLY_SHIFT = 27; +constexpr uint32_t CB_COLOR0_INFO_FMASK_COMPRESS_1FRAG_ONLY_MASK = 0x1; +constexpr uint32_t CB_COLOR0_INFO_DCC_ENABLE_SHIFT = 28; +constexpr uint32_t CB_COLOR0_INFO_DCC_ENABLE_MASK = 0x1; +constexpr uint32_t CB_COLOR0_INFO_CMASK_ADDR_TYPE_SHIFT = 29; +constexpr uint32_t CB_COLOR0_INFO_CMASK_ADDR_TYPE_MASK = 0x3; +constexpr uint32_t CB_COLOR0_INFO_ALT_TILE_MODE_SHIFT = 31; +constexpr uint32_t CB_COLOR0_INFO_ALT_TILE_MODE_MASK = 0x1; + +constexpr uint32_t CB_COLOR0_ATTRIB = 0x31D; +constexpr uint32_t CB_COLOR0_ATTRIB_TILE_MODE_INDEX_SHIFT = 0; +constexpr uint32_t CB_COLOR0_ATTRIB_TILE_MODE_INDEX_MASK = 0x1F; +constexpr uint32_t CB_COLOR0_ATTRIB_FMASK_TILE_MODE_INDEX_SHIFT = 5; +constexpr uint32_t CB_COLOR0_ATTRIB_FMASK_TILE_MODE_INDEX_MASK = 0x1F; +constexpr uint32_t CB_COLOR0_ATTRIB_NUM_SAMPLES_SHIFT = 12; +constexpr uint32_t CB_COLOR0_ATTRIB_NUM_SAMPLES_MASK = 0x7; +constexpr uint32_t CB_COLOR0_ATTRIB_NUM_FRAGMENTS_SHIFT = 15; +constexpr uint32_t CB_COLOR0_ATTRIB_NUM_FRAGMENTS_MASK = 0x3; +constexpr uint32_t CB_COLOR0_ATTRIB_FORCE_DST_ALPHA_1_SHIFT = 17; +constexpr uint32_t CB_COLOR0_ATTRIB_FORCE_DST_ALPHA_1_MASK = 0x1; + +constexpr uint32_t CB_COLOR0_DCC_CONTROL = 0x31E; +constexpr uint32_t CB_COLOR0_DCC_CONTROL_OVERWRITE_COMBINER_DISABLE_SHIFT = 0; +constexpr uint32_t CB_COLOR0_DCC_CONTROL_OVERWRITE_COMBINER_DISABLE_MASK = 0x1; +constexpr uint32_t CB_COLOR0_DCC_CONTROL_KEY_CLEAR_ENABLE_SHIFT = 1; +constexpr uint32_t CB_COLOR0_DCC_CONTROL_KEY_CLEAR_ENABLE_MASK = 0x1; +constexpr uint32_t CB_COLOR0_DCC_CONTROL_MAX_UNCOMPRESSED_BLOCK_SIZE_SHIFT = 2; +constexpr uint32_t CB_COLOR0_DCC_CONTROL_MAX_UNCOMPRESSED_BLOCK_SIZE_MASK = 0x3; +constexpr uint32_t CB_COLOR0_DCC_CONTROL_MIN_COMPRESSED_BLOCK_SIZE_SHIFT = 4; +constexpr uint32_t CB_COLOR0_DCC_CONTROL_MIN_COMPRESSED_BLOCK_SIZE_MASK = 0x1; +constexpr uint32_t CB_COLOR0_DCC_CONTROL_MAX_COMPRESSED_BLOCK_SIZE_SHIFT = 5; +constexpr uint32_t CB_COLOR0_DCC_CONTROL_MAX_COMPRESSED_BLOCK_SIZE_MASK = 0x3; +constexpr uint32_t CB_COLOR0_DCC_CONTROL_COLOR_TRANSFORM_SHIFT = 7; +constexpr uint32_t CB_COLOR0_DCC_CONTROL_COLOR_TRANSFORM_MASK = 0x3; +constexpr uint32_t CB_COLOR0_DCC_CONTROL_INDEPENDENT_64B_BLOCKS_SHIFT = 9; +constexpr uint32_t CB_COLOR0_DCC_CONTROL_INDEPENDENT_64B_BLOCKS_MASK = 0x1; +constexpr uint32_t CB_COLOR0_DCC_CONTROL_ENABLE_CONSTANT_ENCODE_REG_WRITE_SHIFT = 19; +constexpr uint32_t CB_COLOR0_DCC_CONTROL_ENABLE_CONSTANT_ENCODE_REG_WRITE_MASK = 0x1; +constexpr uint32_t CB_COLOR0_DCC_CONTROL_INDEPENDENT_128B_BLOCKS_SHIFT = 20; +constexpr uint32_t CB_COLOR0_DCC_CONTROL_INDEPENDENT_128B_BLOCKS_MASK = 0x1; + +constexpr uint32_t CB_COLOR0_CMASK = 0x31F; +constexpr uint32_t CB_COLOR0_FMASK = 0x321; +constexpr uint32_t CB_COLOR0_CLEAR_WORD0 = 0x323; +constexpr uint32_t CB_COLOR0_CLEAR_WORD1 = 0x324; +constexpr uint32_t CB_COLOR0_DCC_BASE = 0x325; +constexpr uint32_t CB_COLOR7_BASE = 0x381; +constexpr uint32_t CB_COLOR7_VIEW = 0x384; +constexpr uint32_t CB_COLOR7_INFO = 0x385; +constexpr uint32_t CB_COLOR7_ATTRIB = 0x386; +constexpr uint32_t CB_COLOR7_DCC_CONTROL = 0x387; +constexpr uint32_t CB_COLOR7_CMASK = 0x388; +constexpr uint32_t CB_COLOR7_FMASK = 0x38A; +constexpr uint32_t CB_COLOR7_CLEAR_WORD0 = 0x38C; +constexpr uint32_t CB_COLOR7_CLEAR_WORD1 = 0x38D; +constexpr uint32_t CB_COLOR7_DCC_BASE = 0x38E; +constexpr uint32_t CB_COLOR0_BASE_EXT = 0x390; +constexpr uint32_t CB_COLOR7_BASE_EXT = 0x397; +constexpr uint32_t CB_COLOR0_CMASK_BASE_EXT = 0x398; +constexpr uint32_t CB_COLOR7_CMASK_BASE_EXT = 0x39F; +constexpr uint32_t CB_COLOR0_FMASK_BASE_EXT = 0x3A0; +constexpr uint32_t CB_COLOR7_FMASK_BASE_EXT = 0x3A7; +constexpr uint32_t CB_COLOR0_DCC_BASE_EXT = 0x3A8; +constexpr uint32_t CB_COLOR7_DCC_BASE_EXT = 0x3AF; + +constexpr uint32_t CB_COLOR0_ATTRIB2 = 0x3B0; +constexpr uint32_t CB_COLOR0_ATTRIB2_MIP0_HEIGHT_SHIFT = 0; +constexpr uint32_t CB_COLOR0_ATTRIB2_MIP0_HEIGHT_MASK = 0x3FFF; +constexpr uint32_t CB_COLOR0_ATTRIB2_MIP0_WIDTH_SHIFT = 14; +constexpr uint32_t CB_COLOR0_ATTRIB2_MIP0_WIDTH_MASK = 0x3FFF; +constexpr uint32_t CB_COLOR0_ATTRIB2_MAX_MIP_SHIFT = 28; +constexpr uint32_t CB_COLOR0_ATTRIB2_MAX_MIP_MASK = 0xF; + +constexpr uint32_t CB_COLOR7_ATTRIB2 = 0x3B7; + +constexpr uint32_t CB_COLOR0_ATTRIB3 = 0x3B8; +constexpr uint32_t CB_COLOR0_ATTRIB3_MIP0_DEPTH_SHIFT = 0; +constexpr uint32_t CB_COLOR0_ATTRIB3_MIP0_DEPTH_MASK = 0x1FFF; +constexpr uint32_t CB_COLOR0_ATTRIB3_COLOR_SW_MODE_SHIFT = 14; +constexpr uint32_t CB_COLOR0_ATTRIB3_COLOR_SW_MODE_MASK = 0x1F; +constexpr uint32_t CB_COLOR0_ATTRIB3_RESOURCE_TYPE_SHIFT = 24; +constexpr uint32_t CB_COLOR0_ATTRIB3_RESOURCE_TYPE_MASK = 0x3; +constexpr uint32_t CB_COLOR0_ATTRIB3_CMASK_PIPE_ALIGNED_SHIFT = 26; +constexpr uint32_t CB_COLOR0_ATTRIB3_CMASK_PIPE_ALIGNED_MASK = 0x1; +constexpr uint32_t CB_COLOR0_ATTRIB3_DCC_PIPE_ALIGNED_SHIFT = 30; +constexpr uint32_t CB_COLOR0_ATTRIB3_DCC_PIPE_ALIGNED_MASK = 0x1; + +constexpr uint32_t CB_COLOR7_ATTRIB3 = 0x3BF; + +/* Fake codes. Don't exist on real HW */ + +constexpr uint32_t CX_NOP = 0x800003FF; + +constexpr uint32_t CX_NUM = 0x3FF + 1; + +/* Shader registers */ + +constexpr uint32_t SPI_SHADER_TBA_LO_PS = 0x0; +constexpr uint32_t SPI_SHADER_PGM_RSRC4_PS = 0x1; +constexpr uint32_t SPI_SHADER_TMA_LO_PS = 0x2; +constexpr uint32_t SPI_SHADER_TMA_HI_PS = 0x3; +constexpr uint32_t SPI_SHADER_PGM_CHKSUM_PS = 0x6; +constexpr uint32_t SPI_SHADER_PGM_RSRC3_PS = 0x7; +constexpr uint32_t SPI_SHADER_PGM_LO_PS = 0x8; +constexpr uint32_t SPI_SHADER_PGM_HI_PS = 0x9; + +constexpr uint32_t SPI_SHADER_PGM_RSRC1_PS = 0xA; +constexpr uint32_t SPI_SHADER_PGM_RSRC1_PS_VGPRS_SHIFT = 0; +constexpr uint32_t SPI_SHADER_PGM_RSRC1_PS_VGPRS_MASK = 0x3F; +constexpr uint32_t SPI_SHADER_PGM_RSRC1_PS_SGPRS_SHIFT = 6; +constexpr uint32_t SPI_SHADER_PGM_RSRC1_PS_SGPRS_MASK = 0xF; +constexpr uint32_t SPI_SHADER_PGM_RSRC1_PS_PRIORITY_SHIFT = 10; +constexpr uint32_t SPI_SHADER_PGM_RSRC1_PS_PRIORITY_MASK = 0x3; +constexpr uint32_t SPI_SHADER_PGM_RSRC1_PS_FLOAT_MODE_SHIFT = 12; +constexpr uint32_t SPI_SHADER_PGM_RSRC1_PS_FLOAT_MODE_MASK = 0xFF; +constexpr uint32_t SPI_SHADER_PGM_RSRC1_PS_DX10_CLAMP_SHIFT = 21; +constexpr uint32_t SPI_SHADER_PGM_RSRC1_PS_DX10_CLAMP_MASK = 0x1; +constexpr uint32_t SPI_SHADER_PGM_RSRC1_PS_DEBUG_MODE_SHIFT = 22; +constexpr uint32_t SPI_SHADER_PGM_RSRC1_PS_DEBUG_MODE_MASK = 0x1; +constexpr uint32_t SPI_SHADER_PGM_RSRC1_PS_IEEE_MODE_SHIFT = 23; +constexpr uint32_t SPI_SHADER_PGM_RSRC1_PS_IEEE_MODE_MASK = 0x1; +constexpr uint32_t SPI_SHADER_PGM_RSRC1_PS_CU_GROUP_DISABLE_SHIFT = 24; +constexpr uint32_t SPI_SHADER_PGM_RSRC1_PS_CU_GROUP_DISABLE_MASK = 0x1; +constexpr uint32_t SPI_SHADER_PGM_RSRC1_PS_FWD_PROGRESS_SHIFT = 26; +constexpr uint32_t SPI_SHADER_PGM_RSRC1_PS_FWD_PROGRESS_MASK = 0x1; +constexpr uint32_t SPI_SHADER_PGM_RSRC1_PS_FP16_OVFL_SHIFT = 29; +constexpr uint32_t SPI_SHADER_PGM_RSRC1_PS_FP16_OVFL_MASK = 0x1; + +constexpr uint32_t SPI_SHADER_PGM_RSRC2_PS = 0xB; +constexpr uint32_t SPI_SHADER_PGM_RSRC2_PS_SCRATCH_EN_SHIFT = 0; +constexpr uint32_t SPI_SHADER_PGM_RSRC2_PS_SCRATCH_EN_MASK = 0x1; +constexpr uint32_t SPI_SHADER_PGM_RSRC2_PS_USER_SGPR_SHIFT = 1; +constexpr uint32_t SPI_SHADER_PGM_RSRC2_PS_USER_SGPR_MASK = 0x1F; +constexpr uint32_t SPI_SHADER_PGM_RSRC2_PS_WAVE_CNT_EN_SHIFT = 7; +constexpr uint32_t SPI_SHADER_PGM_RSRC2_PS_WAVE_CNT_EN_MASK = 0x1; +constexpr uint32_t SPI_SHADER_PGM_RSRC2_PS_EXTRA_LDS_SIZE_SHIFT = 8; +constexpr uint32_t SPI_SHADER_PGM_RSRC2_PS_EXTRA_LDS_SIZE_MASK = 0xFF; +constexpr uint32_t SPI_SHADER_PGM_RSRC2_PS_LOAD_INTRAWAVE_COLLISION_SHIFT = 26; +constexpr uint32_t SPI_SHADER_PGM_RSRC2_PS_LOAD_INTRAWAVE_COLLISION_MASK = 0x1; +constexpr uint32_t SPI_SHADER_PGM_RSRC2_PS_USER_SGPR_MSB_SHIFT = 27; +constexpr uint32_t SPI_SHADER_PGM_RSRC2_PS_USER_SGPR_MSB_MASK = 0x1; +constexpr uint32_t SPI_SHADER_PGM_RSRC2_PS_SHARED_VGPR_CNT_SHIFT = 28; +constexpr uint32_t SPI_SHADER_PGM_RSRC2_PS_SHARED_VGPR_CNT_MASK = 0xF; + +constexpr uint32_t SPI_SHADER_USER_DATA_PS_0 = 0xC; +constexpr uint32_t SPI_SHADER_USER_DATA_PS_15 = 0x1B; +constexpr uint32_t SPI_SHADER_USER_DATA_PS_31 = 0x2B; +constexpr uint32_t SPI_SHADER_REQ_CTRL_PS = 0x30; +constexpr uint32_t SPI_SHADER_USER_ACCUM_PS_0 = 0x32; +constexpr uint32_t SPI_SHADER_USER_ACCUM_PS_3 = 0x35; + +constexpr uint32_t SPI_SHADER_PGM_RSRC1_VS_VGPRS_SHIFT = 0; +constexpr uint32_t SPI_SHADER_PGM_RSRC1_VS_VGPRS_MASK = 0x3F; +constexpr uint32_t SPI_SHADER_PGM_RSRC1_VS_SGPRS_SHIFT = 6; +constexpr uint32_t SPI_SHADER_PGM_RSRC1_VS_SGPRS_MASK = 0xF; +constexpr uint32_t SPI_SHADER_PGM_RSRC1_VS_PRIORITY_SHIFT = 10; +constexpr uint32_t SPI_SHADER_PGM_RSRC1_VS_PRIORITY_MASK = 0x3; +constexpr uint32_t SPI_SHADER_PGM_RSRC1_VS_FLOAT_MODE_SHIFT = 12; +constexpr uint32_t SPI_SHADER_PGM_RSRC1_VS_FLOAT_MODE_MASK = 0xFF; +constexpr uint32_t SPI_SHADER_PGM_RSRC1_VS_DX10_CLAMP_SHIFT = 21; +constexpr uint32_t SPI_SHADER_PGM_RSRC1_VS_DX10_CLAMP_MASK = 0x1; +constexpr uint32_t SPI_SHADER_PGM_RSRC1_VS_IEEE_MODE_SHIFT = 23; +constexpr uint32_t SPI_SHADER_PGM_RSRC1_VS_IEEE_MODE_MASK = 0x1; +constexpr uint32_t SPI_SHADER_PGM_RSRC1_VS_VGPR_COMP_CNT_SHIFT = 24; +constexpr uint32_t SPI_SHADER_PGM_RSRC1_VS_VGPR_COMP_CNT_MASK = 0x3; +constexpr uint32_t SPI_SHADER_PGM_RSRC1_VS_CU_GROUP_ENABLE_SHIFT = 26; +constexpr uint32_t SPI_SHADER_PGM_RSRC1_VS_CU_GROUP_ENABLE_MASK = 0x1; +constexpr uint32_t SPI_SHADER_PGM_RSRC1_VS_FWD_PROGRESS_SHIFT = 28; +constexpr uint32_t SPI_SHADER_PGM_RSRC1_VS_FWD_PROGRESS_MASK = 0x1; +constexpr uint32_t SPI_SHADER_PGM_RSRC1_VS_FP16_OVFL_SHIFT = 31; +constexpr uint32_t SPI_SHADER_PGM_RSRC1_VS_FP16_OVFL_MASK = 0x1; + +constexpr uint32_t SPI_SHADER_PGM_RSRC2_VS_SCRATCH_EN_SHIFT = 0; +constexpr uint32_t SPI_SHADER_PGM_RSRC2_VS_SCRATCH_EN_MASK = 0x1; +constexpr uint32_t SPI_SHADER_PGM_RSRC2_VS_USER_SGPR_SHIFT = 1; +constexpr uint32_t SPI_SHADER_PGM_RSRC2_VS_USER_SGPR_MASK = 0x1F; +constexpr uint32_t SPI_SHADER_PGM_RSRC2_VS_OC_LDS_EN_SHIFT = 7; +constexpr uint32_t SPI_SHADER_PGM_RSRC2_VS_OC_LDS_EN_MASK = 0x1; +constexpr uint32_t SPI_SHADER_PGM_RSRC2_VS_SO_EN_SHIFT = 12; +constexpr uint32_t SPI_SHADER_PGM_RSRC2_VS_SO_EN_MASK = 0x1; +constexpr uint32_t SPI_SHADER_PGM_RSRC2_VS_USER_SGPR_MSB_SHIFT = 27; +constexpr uint32_t SPI_SHADER_PGM_RSRC2_VS_USER_SGPR_MSB_MASK = 0x1; +constexpr uint32_t SPI_SHADER_PGM_RSRC2_VS_SHARED_VGPR_CNT_SHIFT = 28; +constexpr uint32_t SPI_SHADER_PGM_RSRC2_VS_SHARED_VGPR_CNT_MASK = 0xF; + +constexpr uint32_t SPI_SHADER_PGM_CHKSUM_GS = 0x80; +constexpr uint32_t SPI_SHADER_PGM_RSRC4_GS = 0x81; +constexpr uint32_t SPI_SHADER_USER_DATA_ADDR_LO_GS = 0x82; +constexpr uint32_t SPI_SHADER_USER_DATA_ADDR_HI_GS = 0x83; +constexpr uint32_t SPI_SHADER_PGM_RSRC3_GS = 0x87; +constexpr uint32_t SPI_SHADER_PGM_LO_GS = 0x88; +constexpr uint32_t SPI_SHADER_PGM_HI_GS = 0x89; + +constexpr uint32_t SPI_SHADER_PGM_RSRC1_GS = 0x8A; +constexpr uint32_t SPI_SHADER_PGM_RSRC1_GS_VGPRS_SHIFT = 0; +constexpr uint32_t SPI_SHADER_PGM_RSRC1_GS_VGPRS_MASK = 0x3F; +constexpr uint32_t SPI_SHADER_PGM_RSRC1_GS_SGPRS_SHIFT = 6; +constexpr uint32_t SPI_SHADER_PGM_RSRC1_GS_SGPRS_MASK = 0xF; +constexpr uint32_t SPI_SHADER_PGM_RSRC1_GS_PRIORITY_SHIFT = 10; +constexpr uint32_t SPI_SHADER_PGM_RSRC1_GS_PRIORITY_MASK = 0x3; +constexpr uint32_t SPI_SHADER_PGM_RSRC1_GS_FLOAT_MODE_SHIFT = 12; +constexpr uint32_t SPI_SHADER_PGM_RSRC1_GS_FLOAT_MODE_MASK = 0xFF; +constexpr uint32_t SPI_SHADER_PGM_RSRC1_GS_DX10_CLAMP_SHIFT = 21; +constexpr uint32_t SPI_SHADER_PGM_RSRC1_GS_DX10_CLAMP_MASK = 0x1; +constexpr uint32_t SPI_SHADER_PGM_RSRC1_GS_DEBUG_MODE_SHIFT = 22; +constexpr uint32_t SPI_SHADER_PGM_RSRC1_GS_DEBUG_MODE_MASK = 0x1; +constexpr uint32_t SPI_SHADER_PGM_RSRC1_GS_IEEE_MODE_SHIFT = 23; +constexpr uint32_t SPI_SHADER_PGM_RSRC1_GS_IEEE_MODE_MASK = 0x1; +constexpr uint32_t SPI_SHADER_PGM_RSRC1_GS_CU_GROUP_ENABLE_SHIFT = 24; +constexpr uint32_t SPI_SHADER_PGM_RSRC1_GS_CU_GROUP_ENABLE_MASK = 0x1; +constexpr uint32_t SPI_SHADER_PGM_RSRC1_GS_FWD_PROGRESS_SHIFT = 26; +constexpr uint32_t SPI_SHADER_PGM_RSRC1_GS_FWD_PROGRESS_MASK = 0x1; +constexpr uint32_t SPI_SHADER_PGM_RSRC1_GS_WGP_MODE_SHIFT = 27; +constexpr uint32_t SPI_SHADER_PGM_RSRC1_GS_WGP_MODE_MASK = 0x1; +constexpr uint32_t SPI_SHADER_PGM_RSRC1_GS_GS_VGPR_COMP_CNT_SHIFT = 29; +constexpr uint32_t SPI_SHADER_PGM_RSRC1_GS_GS_VGPR_COMP_CNT_MASK = 0x3; +constexpr uint32_t SPI_SHADER_PGM_RSRC1_GS_FP16_OVFL_SHIFT = 31; +constexpr uint32_t SPI_SHADER_PGM_RSRC1_GS_FP16_OVFL_MASK = 0x1; + +constexpr uint32_t SPI_SHADER_PGM_RSRC2_GS = 0x8B; +constexpr uint32_t SPI_SHADER_PGM_RSRC2_GS_SCRATCH_EN_SHIFT = 0; +constexpr uint32_t SPI_SHADER_PGM_RSRC2_GS_SCRATCH_EN_MASK = 0x1; +constexpr uint32_t SPI_SHADER_PGM_RSRC2_GS_USER_SGPR_SHIFT = 1; +constexpr uint32_t SPI_SHADER_PGM_RSRC2_GS_USER_SGPR_MASK = 0x1F; +constexpr uint32_t SPI_SHADER_PGM_RSRC2_GS_ES_VGPR_COMP_CNT_SHIFT = 16; +constexpr uint32_t SPI_SHADER_PGM_RSRC2_GS_ES_VGPR_COMP_CNT_MASK = 0x3; +constexpr uint32_t SPI_SHADER_PGM_RSRC2_GS_OC_LDS_EN_SHIFT = 18; +constexpr uint32_t SPI_SHADER_PGM_RSRC2_GS_OC_LDS_EN_MASK = 0x1; +constexpr uint32_t SPI_SHADER_PGM_RSRC2_GS_LDS_SIZE_SHIFT = 19; +constexpr uint32_t SPI_SHADER_PGM_RSRC2_GS_LDS_SIZE_MASK = 0xFF; +constexpr uint32_t SPI_SHADER_PGM_RSRC2_GS_USER_SGPR_MSB_SHIFT = 27; +constexpr uint32_t SPI_SHADER_PGM_RSRC2_GS_USER_SGPR_MSB_MASK = 0x1; +constexpr uint32_t SPI_SHADER_PGM_RSRC2_GS_SHARED_VGPR_CNT_SHIFT = 28; +constexpr uint32_t SPI_SHADER_PGM_RSRC2_GS_SHARED_VGPR_CNT_MASK = 0xF; + +constexpr uint32_t SPI_SHADER_USER_DATA_GS_0 = 0x8C; +constexpr uint32_t SPI_SHADER_USER_DATA_GS_15 = 0x9B; +constexpr uint32_t SPI_SHADER_USER_DATA_GS_31 = 0xAB; +constexpr uint32_t SPI_SHADER_REQ_CTRL_ESGS = 0xB0; +constexpr uint32_t SPI_SHADER_USER_ACCUM_ESGS_0 = 0xB2; +constexpr uint32_t SPI_SHADER_USER_ACCUM_ESGS_3 = 0xB5; +constexpr uint32_t SPI_SHADER_PGM_RSRC2_ES_GS = 0xBC; +constexpr uint32_t SPI_SHADER_PGM_LO_ES = 0xC8; +constexpr uint32_t SPI_SHADER_PGM_HI_ES = 0xC9; +constexpr uint32_t SPI_SHADER_PGM_RSRC1_ES = 0xCA; +constexpr uint32_t SPI_SHADER_PGM_RSRC2_ES = 0xCB; +constexpr uint32_t SPI_SHADER_USER_DATA_ES_0 = 0xCC; +constexpr uint32_t SPI_SHADER_USER_DATA_ES_15 = 0xDB; +constexpr uint32_t SPI_SHADER_USER_DATA_ES_31 = 0xEB; +constexpr uint32_t SPI_SHADER_PGM_CHKSUM_HS = 0x100; +constexpr uint32_t SPI_SHADER_PGM_RSRC4_HS = 0x101; +constexpr uint32_t SPI_SHADER_USER_DATA_ADDR_LO_HS = 0x102; +constexpr uint32_t SPI_SHADER_USER_DATA_ADDR_HI_HS = 0x103; +constexpr uint32_t SPI_SHADER_PGM_RSRC3_HS = 0x107; +constexpr uint32_t SPI_SHADER_PGM_LO_HS = 0x108; +constexpr uint32_t SPI_SHADER_PGM_HI_HS = 0x109; +constexpr uint32_t SPI_SHADER_PGM_RSRC1_HS = 0x10A; +constexpr uint32_t SPI_SHADER_PGM_RSRC2_HS = 0x10B; +constexpr uint32_t SPI_SHADER_USER_DATA_HS_0 = 0x10C; +constexpr uint32_t SPI_SHADER_USER_DATA_HS_15 = 0x11B; +constexpr uint32_t SPI_SHADER_USER_DATA_HS_31 = 0x12B; +constexpr uint32_t SPI_SHADER_REQ_CTRL_LSHS = 0x130; +constexpr uint32_t SPI_SHADER_USER_ACCUM_LSHS_0 = 0x132; +constexpr uint32_t SPI_SHADER_USER_ACCUM_LSHS_3 = 0x135; +constexpr uint32_t SPI_SHADER_PGM_LO_LS = 0x148; +constexpr uint32_t SPI_SHADER_PGM_HI_LS = 0x149; +constexpr uint32_t SPI_SHADER_PGM_RSRC1_LS = 0x14A; +constexpr uint32_t SPI_SHADER_PGM_RSRC2_LS = 0x14B; +constexpr uint32_t COMPUTE_START_X = 0x204; +constexpr uint32_t COMPUTE_START_Y = 0x205; +constexpr uint32_t COMPUTE_START_Z = 0x206; +constexpr uint32_t COMPUTE_NUM_THREAD_X = 0x207; +constexpr uint32_t COMPUTE_NUM_THREAD_Y = 0x208; +constexpr uint32_t COMPUTE_NUM_THREAD_Z = 0x209; +constexpr uint32_t COMPUTE_PGM_LO = 0x20C; +constexpr uint32_t COMPUTE_PGM_HI = 0x20D; +constexpr uint32_t COMPUTE_TBA_LO = 0x20E; +constexpr uint32_t COMPUTE_TBA_HI = 0x20F; +constexpr uint32_t COMPUTE_TMA_LO = 0x210; +constexpr uint32_t COMPUTE_TMA_HI = 0x211; + +constexpr uint32_t COMPUTE_PGM_RSRC1 = 0x212; +constexpr uint32_t COMPUTE_PGM_RSRC1_VGPRS_SHIFT = 0; +constexpr uint32_t COMPUTE_PGM_RSRC1_VGPRS_MASK = 0x3F; +constexpr uint32_t COMPUTE_PGM_RSRC1_SGPRS_SHIFT = 6; +constexpr uint32_t COMPUTE_PGM_RSRC1_SGPRS_MASK = 0xF; +constexpr uint32_t COMPUTE_PGM_RSRC1_BULKY_SHIFT = 24; +constexpr uint32_t COMPUTE_PGM_RSRC1_BULKY_MASK = 0x1; +constexpr uint32_t COMPUTE_PGM_RSRC1_W32_EN_SHIFT = 30; +constexpr uint32_t COMPUTE_PGM_RSRC1_W32_EN_MASK = 0x1; + +constexpr uint32_t COMPUTE_PGM_RSRC2 = 0x213; +constexpr uint32_t COMPUTE_PGM_RSRC2_SCRATCH_EN_SHIFT = 0; +constexpr uint32_t COMPUTE_PGM_RSRC2_SCRATCH_EN_MASK = 0x1; +constexpr uint32_t COMPUTE_PGM_RSRC2_USER_SGPR_SHIFT = 1; +constexpr uint32_t COMPUTE_PGM_RSRC2_USER_SGPR_MASK = 0x1F; +constexpr uint32_t COMPUTE_PGM_RSRC2_TGID_X_EN_SHIFT = 7; +constexpr uint32_t COMPUTE_PGM_RSRC2_TGID_X_EN_MASK = 0x1; +constexpr uint32_t COMPUTE_PGM_RSRC2_TGID_Y_EN_SHIFT = 8; +constexpr uint32_t COMPUTE_PGM_RSRC2_TGID_Y_EN_MASK = 0x1; +constexpr uint32_t COMPUTE_PGM_RSRC2_TGID_Z_EN_SHIFT = 9; +constexpr uint32_t COMPUTE_PGM_RSRC2_TGID_Z_EN_MASK = 0x1; +constexpr uint32_t COMPUTE_PGM_RSRC2_TG_SIZE_EN_SHIFT = 10; +constexpr uint32_t COMPUTE_PGM_RSRC2_TG_SIZE_EN_MASK = 0x1; +constexpr uint32_t COMPUTE_PGM_RSRC2_TIDIG_COMP_CNT_SHIFT = 11; +constexpr uint32_t COMPUTE_PGM_RSRC2_TIDIG_COMP_CNT_MASK = 0x3; +constexpr uint32_t COMPUTE_PGM_RSRC2_LDS_SIZE_SHIFT = 15; +constexpr uint32_t COMPUTE_PGM_RSRC2_LDS_SIZE_MASK = 0x1FF; + +constexpr uint32_t COMPUTE_RESOURCE_LIMITS = 0x215; +constexpr uint32_t COMPUTE_DESTINATION_EN_SE0 = 0x216; +constexpr uint32_t COMPUTE_DESTINATION_EN_SE1 = 0x217; +constexpr uint32_t COMPUTE_TMPRING_SIZE = 0x218; +constexpr uint32_t COMPUTE_DESTINATION_EN_SE2 = 0x219; +constexpr uint32_t COMPUTE_DESTINATION_EN_SE3 = 0x21A; +constexpr uint32_t COMPUTE_USER_ACCUM_0 = 0x224; +constexpr uint32_t COMPUTE_PGM_RSRC3 = 0x228; +constexpr uint32_t COMPUTE_SHADER_CHKSUM = 0x22A; +constexpr uint32_t COMPUTE_USER_DATA_0 = 0x240; +constexpr uint32_t COMPUTE_USER_DATA_15 = 0x24F; +constexpr uint32_t COMPUTE_DISPATCH_TUNNEL = 0x27D; + +/* Fake codes. Don't exist on real HW */ + +constexpr uint32_t SH_NOP = 0x800002FF; + +constexpr uint32_t SH_NUM = 0x2FF + 1; + +/* User config registers */ + +constexpr uint32_t VGT_PRIMITIVE_TYPE = 0x242; +constexpr uint32_t VGT_PRIMITIVE_TYPE_PRIM_TYPE_SHIFT = 0; +constexpr uint32_t VGT_PRIMITIVE_TYPE_PRIM_TYPE_MASK = 0x3F; + +constexpr uint32_t VGT_INDEX_TYPE = 0x243; + +constexpr uint32_t VGT_OBJECT_ID = 0x248; +constexpr uint32_t GE_INDX_OFFSET = 0x24A; +constexpr uint32_t GE_MULTI_PRIM_IB_RESET_EN = 0x24B; +constexpr uint32_t VGT_HS_OFFCHIP_PARAM = 0x24F; +constexpr uint32_t VGT_TF_MEMORY_BASE = 0x250; + +constexpr uint32_t GE_CNTL = 0x25B; +constexpr uint32_t GE_CNTL_PRIM_GRP_SIZE_SHIFT = 0; +constexpr uint32_t GE_CNTL_PRIM_GRP_SIZE_MASK = 0x1FF; +constexpr uint32_t GE_CNTL_VERT_GRP_SIZE_SHIFT = 9; +constexpr uint32_t GE_CNTL_VERT_GRP_SIZE_MASK = 0x1FF; + +constexpr uint32_t GE_USER_VGPR1 = 0x25C; +constexpr uint32_t GE_USER_VGPR2 = 0x25D; +constexpr uint32_t GE_USER_VGPR3 = 0x25E; +constexpr uint32_t GE_STEREO_CNTL = 0x25F; + +constexpr uint32_t GE_USER_VGPR_EN = 0x262; +constexpr uint32_t GE_USER_VGPR_EN_EN_USER_VGPR1_SHIFT = 0; +constexpr uint32_t GE_USER_VGPR_EN_EN_USER_VGPR1_MASK = 0x1; +constexpr uint32_t GE_USER_VGPR_EN_EN_USER_VGPR2_SHIFT = 1; +constexpr uint32_t GE_USER_VGPR_EN_EN_USER_VGPR2_MASK = 0x1; +constexpr uint32_t GE_USER_VGPR_EN_EN_USER_VGPR3_SHIFT = 2; +constexpr uint32_t GE_USER_VGPR_EN_EN_USER_VGPR3_MASK = 0x1; + +constexpr uint32_t IA_MULTI_VGT_PARAM = 0x258; +constexpr uint32_t TA_CS_BC_BASE_ADDR = 0x380; +constexpr uint32_t TA_CS_BC_BASE_ADDR_HI = 0x381; +constexpr uint32_t TEXTURE_GRADIENT_FACTORS = 0x382; +constexpr uint32_t GDS_OA_CNTL = 0x41D; +constexpr uint32_t GDS_OA_COUNTER = 0x41E; +constexpr uint32_t GDS_OA_ADDRESS = 0x41F; + +/* Fake codes. Don't exist on real HW */ + +constexpr uint32_t FSR_EXTEND_SUBPIXEL_ROUNDING = 0x80003FF4; +constexpr uint32_t FSR_ALPHA_VALUE0 = 0x80003FF5; +constexpr uint32_t FSR_ALPHA_VALUE1 = 0x80003FF6; +constexpr uint32_t FSR_CONTROL_POINT0 = 0x80003FF7; +constexpr uint32_t FSR_CONTROL_POINT1 = 0x80003FF8; +constexpr uint32_t FSR_CONTROL_POINT2 = 0x80003FF9; +constexpr uint32_t FSR_CONTROL_POINT3 = 0x80003FFA; +constexpr uint32_t FSR_WINDOW0 = 0x80003FFB; +constexpr uint32_t FSR_WINDOW1 = 0x80003FFC; +constexpr uint32_t TEXTURE_GRADIENT_CONTROL = 0x80003FFD; +constexpr uint32_t MEMORY_MAPPING_MASK = 0x80003FFE; +constexpr uint32_t UC_NOP = 0x80003FFF; + +constexpr uint32_t UC_NUM = 0x3FFF + 1; + +void DumpPm4PacketStream(Common::File* file, uint32_t* cmd_buffer, uint32_t start_dw, + uint32_t num_dw); + +} // namespace Libs::Graphics::Pm4 + +#endif /* EMULATOR_INCLUDE_EMULATOR_GRAPHICS_PM4_H_ */ diff --git a/src/graphics/guest_gpu/tile.cpp b/src/graphics/guest_gpu/tile.cpp new file mode 100644 index 000000000..f28502581 --- /dev/null +++ b/src/graphics/guest_gpu/tile.cpp @@ -0,0 +1,3166 @@ +#include "graphics/guest_gpu/tile.h" + +#include "common/assert.h" +#include "common/logging/log.h" +#include "common/profiler.h" +#include "common/stringUtils.h" +#include "common/threads.h" +#include "graphics/asyncJob.h" +#include "graphics/guest_gpu/gpu_defs.h" +#include "graphics/guest_gpu/gpu_format.h" + +#include +#include +#include +#include +#include + +namespace Libs::Graphics { + +static uint32_t IntLog2(uint32_t i) { + return 31 - __builtin_clz(i | 1u); +} + +static uint32_t AlignUp(uint32_t value, uint32_t alignment) { + return (value + alignment - 1u) & ~(alignment - 1u); +} + +static uint32_t ShiftCeil(uint32_t value, uint32_t shift) { + return static_cast((static_cast(value) + (1ull << shift) - 1ull) >> shift); +} + +static uint32_t CalcLinearBlockWidth(uint32_t bytes_per_element) { + return std::max(1u, 256u / bytes_per_element); +} + +static uint32_t CalcLinearAlignedLevelPitch(uint32_t base_width, uint32_t base_height, + uint32_t level, uint32_t bytes_per_element, + uint32_t* padded_height, uint32_t* level_size) { + const uint32_t level_width = ShiftCeil(base_width, level); + const uint32_t level_height = ShiftCeil(base_height, level); + const uint32_t padded_width = + AlignUp(std::max(level_width, 1u), CalcLinearBlockWidth(bytes_per_element)); + const uint64_t size = + static_cast(padded_width) * std::max(level_height, 1u) * bytes_per_element; + EXIT_NOT_IMPLEMENTED(size > 0xffffffffull); + *padded_height = std::max(level_height, 1u); + *level_size = static_cast(size); + return padded_width; +} + +static uint32_t SetLinearMipChainLayout(uint32_t levels, const uint32_t* mip_pitch, + const uint32_t* mip_height, const uint32_t* mip_size, + TileSizeOffset* level_sizes, TilePaddedSize* padded_size) { + uint32_t offset = 0; + // AGC linear surfaces store smaller mip records first; mip 0 is last in the block slice. + for (int32_t l = static_cast(levels) - 1; l >= 0; l--) { + const auto level = static_cast(l); + if (level_sizes != nullptr) { + level_sizes[level].size = mip_size[level]; + level_sizes[level].offset = offset; + } + if (padded_size != nullptr) { + padded_size[level].width = mip_pitch[level]; + padded_size[level].height = mip_height[level]; + } + + offset += mip_size[level]; + } + + return AlignUp(offset, 256u); +} + +static bool Gen5Standard4KBLayout(uint32_t format, uint32_t* bytes_per_element, + uint32_t* texels_per_element_wide, + uint32_t* texels_per_element_tall, uint32_t* block_width_log2, + uint32_t* block_height_log2) { + const auto bytes = Prospero::NumBytesPerElement(format); + switch (bytes) { + case 1: + *bytes_per_element = 1; + *texels_per_element_wide = 1; + *texels_per_element_tall = 1; + *block_width_log2 = 6; + *block_height_log2 = 6; + return true; + case 2: + *bytes_per_element = 2; + *texels_per_element_wide = 1; + *texels_per_element_tall = 1; + *block_width_log2 = 6; + *block_height_log2 = 5; + return true; + case 4: + *bytes_per_element = 4; + *texels_per_element_wide = 1; + *texels_per_element_tall = 1; + *block_width_log2 = 5; + *block_height_log2 = 5; + return true; + case 8: + *bytes_per_element = 8; + *texels_per_element_wide = 1; + *texels_per_element_tall = 1; + *block_width_log2 = 5; + *block_height_log2 = 4; + return true; + case 16: + *bytes_per_element = 16; + *texels_per_element_wide = 1; + *texels_per_element_tall = 1; + *block_width_log2 = 4; + *block_height_log2 = 4; + return true; + default: break; + } + + switch (Prospero::BlockCompressedBytesPerBlock(format)) { + case 8: + *bytes_per_element = 8; + *texels_per_element_wide = 4; + *texels_per_element_tall = 4; + *block_width_log2 = 5; + *block_height_log2 = 4; + return true; + case 16: + *bytes_per_element = 16; + *texels_per_element_wide = 4; + *texels_per_element_tall = 4; + *block_width_log2 = 4; + *block_height_log2 = 4; + return true; + default: return false; + } +} + +bool TileGetStandard4KBVolumeLayout(uint32_t format, uint32_t* bytes_per_element, + uint32_t* texels_per_element_wide, + uint32_t* texels_per_element_tall, uint32_t* block_width_log2, + uint32_t* block_height_log2, uint32_t* block_depth_log2) { + if (!Gen5Standard4KBLayout(format, bytes_per_element, texels_per_element_wide, + texels_per_element_tall, block_width_log2, block_height_log2)) { + return false; + } + + switch (*bytes_per_element) { + case 1: + *block_width_log2 = 4; + *block_height_log2 = 4; + *block_depth_log2 = 4; + return true; + case 2: + *block_width_log2 = 3; + *block_height_log2 = 4; + *block_depth_log2 = 4; + return true; + case 4: + *block_width_log2 = 3; + *block_height_log2 = 4; + *block_depth_log2 = 3; + return true; + case 8: + *block_width_log2 = 3; + *block_height_log2 = 3; + *block_depth_log2 = 3; + return true; + case 16: + *block_width_log2 = 2; + *block_height_log2 = 3; + *block_depth_log2 = 3; + return true; + default: break; + } + + return false; +} + +static bool Gen5Standard256BLayout(uint32_t format, uint32_t* bytes_per_element, + uint32_t* texels_per_element_wide, + uint32_t* texels_per_element_tall, uint32_t* block_width_log2, + uint32_t* block_height_log2) { + const auto bytes = Prospero::NumBytesPerElement(format); + switch (bytes) { + case 1: + *bytes_per_element = 1; + *texels_per_element_wide = 1; + *texels_per_element_tall = 1; + *block_width_log2 = 4; + *block_height_log2 = 4; + return true; + case 2: + *bytes_per_element = 2; + *texels_per_element_wide = 1; + *texels_per_element_tall = 1; + *block_width_log2 = 4; + *block_height_log2 = 3; + return true; + case 4: + *bytes_per_element = 4; + *texels_per_element_wide = 1; + *texels_per_element_tall = 1; + *block_width_log2 = 3; + *block_height_log2 = 3; + return true; + case 8: + *bytes_per_element = 8; + *texels_per_element_wide = 1; + *texels_per_element_tall = 1; + *block_width_log2 = 3; + *block_height_log2 = 2; + return true; + case 16: + *bytes_per_element = 16; + *texels_per_element_wide = 1; + *texels_per_element_tall = 1; + *block_width_log2 = 2; + *block_height_log2 = 2; + return true; + default: break; + } + + switch (Prospero::BlockCompressedBytesPerBlock(format)) { + case 8: + *bytes_per_element = 8; + *texels_per_element_wide = 4; + *texels_per_element_tall = 4; + *block_width_log2 = 3; + *block_height_log2 = 2; + return true; + case 16: + *bytes_per_element = 16; + *texels_per_element_wide = 4; + *texels_per_element_tall = 4; + *block_width_log2 = 2; + *block_height_log2 = 2; + return true; + default: return false; + } +} + +static bool Gen5Standard64KBLayout(uint32_t format, uint32_t* bytes_per_element, + uint32_t* texels_per_element_wide, + uint32_t* texels_per_element_tall, uint32_t* block_width_log2, + uint32_t* block_height_log2) { + switch (Prospero::NumBytesPerElement(format)) { + case 1: + *bytes_per_element = 1; + *texels_per_element_wide = 1; + *texels_per_element_tall = 1; + *block_width_log2 = 8; + *block_height_log2 = 8; + return true; + case 2: + *bytes_per_element = 2; + *texels_per_element_wide = 1; + *texels_per_element_tall = 1; + *block_width_log2 = 8; + *block_height_log2 = 7; + return true; + case 4: + *bytes_per_element = 4; + *texels_per_element_wide = 1; + *texels_per_element_tall = 1; + *block_width_log2 = 7; + *block_height_log2 = 7; + return true; + case 8: + *bytes_per_element = 8; + *texels_per_element_wide = 1; + *texels_per_element_tall = 1; + *block_width_log2 = 7; + *block_height_log2 = 6; + return true; + case 16: + *bytes_per_element = 16; + *texels_per_element_wide = 1; + *texels_per_element_tall = 1; + *block_width_log2 = 6; + *block_height_log2 = 6; + return true; + default: break; + } + + switch (Prospero::BlockCompressedBytesPerBlock(format)) { + case 8: + *bytes_per_element = 8; + *texels_per_element_wide = 4; + *texels_per_element_tall = 4; + *block_width_log2 = 7; + *block_height_log2 = 6; + return true; + case 16: + *bytes_per_element = 16; + *texels_per_element_wide = 4; + *texels_per_element_tall = 4; + *block_width_log2 = 6; + *block_height_log2 = 6; + return true; + default: return false; + } +} + +static bool Gen5RenderTargetBlockSizeFromElementBytes(uint32_t bytes_per_element, + uint32_t* block_width, + uint32_t* block_height) { + // AGC render-target + switch (bytes_per_element) { + case 1: + *block_width = 256; + *block_height = 256; + return true; + case 2: + *block_width = 256; + *block_height = 128; + return true; + case 4: + *block_width = 128; + *block_height = 128; + return true; + case 8: + *block_width = 128; + *block_height = 64; + return true; + case 16: + *block_width = 64; + *block_height = 64; + return true; + default: break; + } + + return false; +} + +struct Gen5MipTailLocation { + uint32_t x; + uint32_t y; +}; + +static constexpr Gen5MipTailLocation GEN5_MIP_TAIL_LOCATIONS_THIN_4KB[5][8] = { + {{32, 0}, {16, 32}, {0, 48}, {0, 32}, {16, 16}, {16, 0}, {0, 16}, {0, 0}}, + {{32, 0}, {16, 16}, {0, 24}, {0, 16}, {16, 8}, {16, 0}, {0, 8}, {0, 0}}, + {{16, 0}, {8, 16}, {0, 24}, {0, 16}, {8, 8}, {8, 0}, {0, 8}, {0, 0}}, + {{16, 0}, {8, 8}, {0, 12}, {0, 8}, {8, 4}, {8, 0}, {0, 4}, {0, 0}}, + {{8, 0}, {4, 8}, {0, 12}, {0, 8}, {4, 4}, {4, 0}, {0, 4}, {0, 0}}, +}; + +static constexpr Gen5MipTailLocation GEN5_MIP_TAIL_LOCATIONS_THIN_64KB[5][12] = { + {{128, 0}, + {0, 128}, + {64, 0}, + {0, 64}, + {32, 0}, + {16, 32}, + {0, 48}, + {0, 32}, + {16, 16}, + {16, 0}, + {0, 16}, + {0, 0}}, + {{128, 0}, + {0, 64}, + {64, 0}, + {0, 32}, + {32, 0}, + {16, 16}, + {0, 24}, + {0, 16}, + {16, 8}, + {16, 0}, + {0, 8}, + {0, 0}}, + {{64, 0}, + {0, 64}, + {32, 0}, + {0, 32}, + {16, 0}, + {8, 16}, + {0, 24}, + {0, 16}, + {8, 8}, + {8, 0}, + {0, 8}, + {0, 0}}, + {{64, 0}, + {0, 32}, + {32, 0}, + {0, 16}, + {16, 0}, + {8, 8}, + {0, 12}, + {0, 8}, + {8, 4}, + {8, 0}, + {0, 4}, + {0, 0}}, + {{32, 0}, + {0, 32}, + {16, 0}, + {0, 16}, + {8, 0}, + {4, 8}, + {0, 12}, + {0, 8}, + {4, 4}, + {4, 0}, + {0, 4}, + {0, 0}}, +}; + +static uint32_t Gen5Standard4KBOffsetInBlock(uint32_t x, uint32_t y, uint32_t bytes_per_element) { + uint32_t offset = 0; + + switch (bytes_per_element) { + case 1: + offset ^= (y << 4u) & 0x1f0u; + offset ^= (y << 5u) & 0x400u; + offset ^= x & 0x00fu; + offset ^= (x << 5u) & 0x200u; + offset ^= (x << 6u) & 0x800u; + return offset; + case 2: + offset ^= (y << 4u) & 0x070u; + offset ^= (y << 5u) & 0x100u; + offset ^= (y << 6u) & 0x400u; + offset ^= (x << 1u) & 0x00eu; + offset ^= (x << 4u) & 0x080u; + offset ^= (x << 5u) & 0x200u; + offset ^= (x << 6u) & 0x800u; + return offset; + case 4: + offset ^= (y << 4u) & 0x070u; + offset ^= (y << 5u) & 0x100u; + offset ^= (y << 6u) & 0x400u; + offset ^= (x << 2u) & 0x00cu; + offset ^= (x << 5u) & 0x080u; + offset ^= (x << 6u) & 0x200u; + offset ^= (x << 7u) & 0x800u; + return offset; + case 8: + offset ^= (y << 4u) & 0x030u; + offset ^= (y << 6u) & 0x100u; + offset ^= (y << 7u) & 0x400u; + offset ^= (x << 3u) & 0x008u; + offset ^= (x << 5u) & 0x0c0u; + offset ^= (x << 6u) & 0x200u; + offset ^= (x << 7u) & 0x800u; + return offset; + case 16: + offset ^= (y << 4u) & 0x030u; + offset ^= (y << 6u) & 0x100u; + offset ^= (y << 7u) & 0x400u; + offset ^= (x << 6u) & 0x0c0u; + offset ^= (x << 7u) & 0x200u; + offset ^= (x << 8u) & 0x800u; + return offset; + default: EXIT("unsupported Standard4KB element size: %u\n", bytes_per_element); + } + return 0; +} + +static uint32_t Gen5Standard4KBVolumeOffsetInBlock(uint32_t x, uint32_t y, uint32_t z, + uint32_t bytes_per_element) { + uint32_t offset = 0; + + switch (bytes_per_element) { + case 1: + offset ^= x & 0x3u; + offset ^= (x << 4u) & 0x40u; + offset ^= (x << 6u) & 0x200u; + offset ^= (y << 3u) & 0x8u; + offset ^= (y << 4u) & 0x20u; + offset ^= (y << 6u) & 0x100u; + offset ^= (y << 8u) & 0x800u; + offset ^= (z << 2u) & 0x4u; + offset ^= (z << 3u) & 0x10u; + offset ^= (z << 5u) & 0x80u; + offset ^= (z << 7u) & 0x400u; + return offset; + case 2: + offset ^= (x << 1u) & 0x2u; + offset ^= (x << 5u) & 0x40u; + offset ^= (x << 7u) & 0x200u; + offset ^= (y << 3u) & 0x8u; + offset ^= (y << 4u) & 0x20u; + offset ^= (y << 6u) & 0x100u; + offset ^= (y << 8u) & 0x800u; + offset ^= (z << 2u) & 0x4u; + offset ^= (z << 3u) & 0x10u; + offset ^= (z << 5u) & 0x80u; + offset ^= (z << 7u) & 0x400u; + return offset; + case 4: + offset ^= (x << 2u) & 0x4u; + offset ^= (x << 5u) & 0x40u; + offset ^= (x << 7u) & 0x200u; + offset ^= (y << 3u) & 0x8u; + offset ^= (y << 4u) & 0x20u; + offset ^= (y << 6u) & 0x100u; + offset ^= (y << 8u) & 0x800u; + offset ^= (z << 4u) & 0x10u; + offset ^= (z << 6u) & 0x80u; + offset ^= (z << 8u) & 0x400u; + return offset; + case 8: + offset ^= (x << 3u) & 0x8u; + offset ^= (x << 5u) & 0x40u; + offset ^= (x << 7u) & 0x200u; + offset ^= (y << 5u) & 0x20u; + offset ^= (y << 7u) & 0x100u; + offset ^= (y << 9u) & 0x800u; + offset ^= (z << 4u) & 0x10u; + offset ^= (z << 6u) & 0x80u; + offset ^= (z << 8u) & 0x400u; + return offset; + case 16: + offset ^= (x << 6u) & 0x40u; + offset ^= (x << 8u) & 0x200u; + offset ^= (y << 5u) & 0x20u; + offset ^= (y << 7u) & 0x100u; + offset ^= (y << 9u) & 0x800u; + offset ^= (z << 4u) & 0x10u; + offset ^= (z << 6u) & 0x80u; + offset ^= (z << 8u) & 0x400u; + return offset; + default: EXIT("unsupported Standard4KB volume element size: %u\n", bytes_per_element); + } + return 0; +} + +bool TileIsStandard4KBTextureSupported(uint32_t format) { + uint32_t bytes_per_element = 0; + uint32_t texels_per_element_wide = 0; + uint32_t texels_per_element_tall = 0; + uint32_t block_width_log2 = 0; + uint32_t block_height_log2 = 0; + + return Gen5Standard4KBLayout(format, &bytes_per_element, &texels_per_element_wide, + &texels_per_element_tall, &block_width_log2, &block_height_log2); +} + +bool TileIsStandard256BTextureSupported(uint32_t format) { + uint32_t bytes_per_element = 0; + uint32_t texels_per_element_wide = 0; + uint32_t texels_per_element_tall = 0; + uint32_t block_width_log2 = 0; + uint32_t block_height_log2 = 0; + + return Gen5Standard256BLayout(format, &bytes_per_element, &texels_per_element_wide, + &texels_per_element_tall, &block_width_log2, &block_height_log2); +} + +bool TileIsStandard64KBTextureSupported(uint32_t format) { + uint32_t bytes_per_element = 0; + uint32_t texels_per_element_wide = 0; + uint32_t texels_per_element_tall = 0; + uint32_t block_width_log2 = 0; + uint32_t block_height_log2 = 0; + + return Gen5Standard64KBLayout(format, &bytes_per_element, &texels_per_element_wide, + &texels_per_element_tall, &block_width_log2, &block_height_log2); +} + +struct Uint128 { + uint64_t n[2]; +}; + +struct Uint256 { + Uint128 n[2]; +}; + +template +static void DetileStandard4KBTyped(T* dst, const T* src, uint32_t row_elements, + uint32_t width_elements, uint32_t height_elements, + uint32_t padded_width, uint32_t block_width_log2, + uint32_t block_height_log2, uint64_t dst_size, uint64_t src_size, + uint32_t src_x, uint32_t src_y) { + constexpr auto bytes_per_element = static_cast(sizeof(T)); + + std::array x_elements {}; + std::array y_elements {}; + + const uint32_t block_width = 1u << block_width_log2; + const uint32_t block_height = 1u << block_height_log2; + const uint32_t elements_per_block = 4096u / bytes_per_element; + const uint32_t blocks_per_row = padded_width >> block_width_log2; + + for (uint32_t x = 0; x < block_width; x++) { + x_elements[x] = Gen5Standard4KBOffsetInBlock(x, 0, bytes_per_element) / bytes_per_element; + } + + for (uint32_t y = 0; y < block_height; y++) { + y_elements[y] = Gen5Standard4KBOffsetInBlock(0, y, bytes_per_element) / bytes_per_element; + } + + const uint64_t dst_count = dst_size / bytes_per_element; + const uint64_t src_count = src_size / bytes_per_element; + + for (uint32_t block_y = 0; block_y < height_elements; block_y += block_height) { + const uint32_t copy_height = std::min(block_height, height_elements - block_y); + + for (uint32_t block_x = 0; block_x < width_elements; block_x += block_width) { + const uint32_t copy_width = std::min(block_width, width_elements - block_x); + const uint64_t block_index = + (static_cast(block_y >> block_height_log2) * blocks_per_row) + + (block_x >> block_width_log2); + const uint64_t block_base = block_index * elements_per_block; + + for (uint32_t local_y = 0; local_y < copy_height; local_y++) { + const uint64_t dst_row = + (static_cast(block_y + local_y) * row_elements) + block_x; + const uint64_t src_row = block_base + y_elements[src_y + local_y]; + + for (uint32_t local_x = 0; local_x < copy_width; local_x++) { + const uint64_t dst_index = dst_row + local_x; + const uint64_t src_index = src_row + x_elements[src_x + local_x]; + + if (src_index < src_count && dst_index < dst_count) { + dst[dst_index] = src[src_index]; + } + } + } + } + } +} + +template +static void DetileStandard4KBVolumeTyped(T* dst, const T* src, uint32_t row_elements, + uint32_t width_elements, uint32_t height_elements, + uint32_t depth_elements, uint32_t padded_width, + uint32_t padded_height, uint32_t block_width_log2, + uint32_t block_height_log2, uint32_t block_depth_log2, + uint64_t dst_slice_stride, uint64_t dst_size, + uint64_t src_size) { + constexpr auto bytes_per_element = static_cast(sizeof(T)); + + const uint32_t block_width = 1u << block_width_log2; + const uint32_t block_height = 1u << block_height_log2; + const uint32_t block_depth = 1u << block_depth_log2; + const uint32_t elements_per_block = 4096u / bytes_per_element; + const uint32_t blocks_per_row = padded_width >> block_width_log2; + const uint32_t blocks_per_column = padded_height >> block_height_log2; + const uint64_t blocks_per_slice = + static_cast(blocks_per_row) * static_cast(blocks_per_column); + const uint64_t dst_count = dst_size / bytes_per_element; + const uint64_t src_count = src_size / bytes_per_element; + const uint64_t dst_slice_elements = dst_slice_stride / bytes_per_element; + + for (uint32_t block_z = 0; block_z < depth_elements; block_z += block_depth) { + const uint32_t copy_depth = std::min(block_depth, depth_elements - block_z); + + for (uint32_t block_y = 0; block_y < height_elements; block_y += block_height) { + const uint32_t copy_height = std::min(block_height, height_elements - block_y); + + for (uint32_t block_x = 0; block_x < width_elements; block_x += block_width) { + const uint32_t copy_width = std::min(block_width, width_elements - block_x); + const uint64_t block_index = + ((static_cast(block_z >> block_depth_log2) * blocks_per_slice) + + (static_cast(block_y >> block_height_log2) * blocks_per_row) + + (block_x >> block_width_log2)); + const uint64_t block_base = block_index * elements_per_block; + + for (uint32_t local_z = 0; local_z < copy_depth; local_z++) { + for (uint32_t local_y = 0; local_y < copy_height; local_y++) { + const uint64_t dst_row = + (static_cast(block_z + local_z) * dst_slice_elements) + + (static_cast(block_y + local_y) * row_elements) + block_x; + + for (uint32_t local_x = 0; local_x < copy_width; local_x++) { + const uint64_t dst_index = dst_row + local_x; + const uint64_t src_index = + block_base + (Gen5Standard4KBVolumeOffsetInBlock( + local_x, local_y, local_z, bytes_per_element) / + bytes_per_element); + + if (src_index < src_count && dst_index < dst_count) { + dst[dst_index] = src[src_index]; + } + } + } + } + } + } + } +} + +static uint32_t Gen5Standard256BOffsetInBlock(uint32_t x, uint32_t y, uint32_t bytes_per_element) { + return Gen5Standard4KBOffsetInBlock(x, y, bytes_per_element) & 0xffu; +} + +template +static void +DetileStandard256BTyped(T* dst, const T* src, uint32_t row_elements, uint32_t width_elements, + uint32_t height_elements, uint32_t padded_width, uint32_t block_width_log2, + uint32_t block_height_log2, uint64_t dst_size, uint64_t src_size) { + constexpr auto bytes_per_element = static_cast(sizeof(T)); + + std::array x_elements {}; + std::array y_elements {}; + + const uint32_t block_width = 1u << block_width_log2; + const uint32_t block_height = 1u << block_height_log2; + const uint32_t elements_per_block = 256u / bytes_per_element; + const uint32_t blocks_per_row = padded_width >> block_width_log2; + + for (uint32_t x = 0; x < block_width; x++) { + x_elements[x] = Gen5Standard256BOffsetInBlock(x, 0, bytes_per_element) / bytes_per_element; + } + + for (uint32_t y = 0; y < block_height; y++) { + y_elements[y] = Gen5Standard256BOffsetInBlock(0, y, bytes_per_element) / bytes_per_element; + } + + const uint64_t dst_count = dst_size / bytes_per_element; + const uint64_t src_count = src_size / bytes_per_element; + + for (uint32_t block_y = 0; block_y < height_elements; block_y += block_height) { + const uint32_t copy_height = std::min(block_height, height_elements - block_y); + + for (uint32_t block_x = 0; block_x < width_elements; block_x += block_width) { + const uint32_t copy_width = std::min(block_width, width_elements - block_x); + const uint64_t block_index = + (static_cast(block_y >> block_height_log2) * blocks_per_row) + + (block_x >> block_width_log2); + const uint64_t block_base = block_index * elements_per_block; + + for (uint32_t local_y = 0; local_y < copy_height; local_y++) { + const uint64_t dst_row = + (static_cast(block_y + local_y) * row_elements) + block_x; + const uint64_t src_row = block_base + y_elements[local_y]; + + for (uint32_t local_x = 0; local_x < copy_width; local_x++) { + const uint64_t dst_index = dst_row + local_x; + const uint64_t src_index = src_row + x_elements[local_x]; + + if (src_index < src_count && dst_index < dst_count) { + dst[dst_index] = src[src_index]; + } + } + } + } + } +} + +class Tiler { +public: + Tiler() { EXIT_NOT_IMPLEMENTED(!Common::Thread::IsMainThread()); } + ~Tiler() { KYTY_NOT_IMPLEMENTED; } + + KYTY_CLASS_NO_COPY(Tiler); + + Common::Mutex m_mutex; + + AsyncJob m_job1; +}; + +class Tiler32 { +public: + uint32_t m_macro_tile_height = 0; + uint32_t m_bank_height = 0; + uint32_t m_num_banks = 0; + uint32_t m_num_pipes = 0; + uint32_t m_padded_width = 0; + uint32_t m_padded_height = 0; + uint32_t m_pipe_bits = 0; + uint32_t m_bank_bits = 0; + + void Init(uint32_t width, uint32_t height) { + m_macro_tile_height = 128; + m_bank_height = 2; + m_num_banks = 8; + m_num_pipes = 16; + m_padded_width = AlignUp(width, 128); + m_padded_height = AlignUp(height, m_macro_tile_height); + m_pipe_bits = 4; + m_bank_bits = 3; + } + + static uint32_t GetElementIndex(uint32_t x, uint32_t y) { + uint32_t elem = 0; + elem |= ((x >> 0u) & 0x1u) << 0u; + elem |= ((x >> 1u) & 0x1u) << 1u; + elem |= ((y >> 0u) & 0x1u) << 2u; + elem |= ((x >> 2u) & 0x1u) << 3u; + elem |= ((y >> 1u) & 0x1u) << 4u; + elem |= ((y >> 2u) & 0x1u) << 5u; + + return elem; + } + + static uint32_t GetPipeIndex(uint32_t x, uint32_t y) { + uint32_t pipe = 0; + + pipe |= (((x >> 3u) ^ (y >> 3u) ^ (x >> 4u)) & 0x1u) << 0u; + pipe |= (((x >> 4u) ^ (y >> 4u)) & 0x1u) << 1u; + pipe |= (((x >> 5u) ^ (y >> 5u)) & 0x1u) << 2u; + pipe |= (((x >> 6u) ^ (y >> 5u)) & 0x1u) << 3u; + + return pipe; + } + + static uint32_t GetBankIndex(uint32_t x, uint32_t y, uint32_t bank_width, uint32_t bank_height, + uint32_t num_banks, uint32_t num_pipes) { + const uint32_t x_shift_offset = IntLog2(bank_width * num_pipes); + const uint32_t y_shift_offset = IntLog2(bank_height); + const uint32_t xs = x >> x_shift_offset; + const uint32_t ys = y >> y_shift_offset; + uint32_t bank = 0; + switch (num_banks) { + case 8: + bank |= (((xs >> 3u) ^ (ys >> 5u)) & 0x1u) << 0u; + bank |= (((xs >> 4u) ^ (ys >> 4u) ^ (ys >> 5u)) & 0x1u) << 1u; + bank |= (((xs >> 5u) ^ (ys >> 3u)) & 0x1u) << 2u; + break; + case 16: + bank |= (((xs >> 3u) ^ (ys >> 6u)) & 0x1u) << 0u; + bank |= (((xs >> 4u) ^ (ys >> 5u) ^ (ys >> 6u)) & 0x1u) << 1u; + bank |= (((xs >> 5u) ^ (ys >> 4u)) & 0x1u) << 2u; + bank |= (((xs >> 6u) ^ (ys >> 3u)) & 0x1u) << 3u; + break; + default:; + } + + return bank; + } + + [[nodiscard]] uint64_t GetTiledOffset(uint32_t x, uint32_t y) const { + return GetTiledOffset(x, y, 32); + } + + [[nodiscard]] uint64_t GetTiledOffset(uint32_t x, uint32_t y, uint32_t bits_per_element) const { + uint64_t element_index = GetElementIndex(x, y); + + uint32_t xh = x; + uint32_t yh = y; + uint64_t pipe = GetPipeIndex(xh, yh); + uint64_t bank = GetBankIndex(xh, yh, 1, m_bank_height, m_num_banks, m_num_pipes); + uint32_t tile_bytes = (8 * 8 * bits_per_element + 7) / 8; + uint64_t element_offset = (element_index * bits_per_element); + uint64_t tile_split_slice = 0; + + if (tile_bytes > 512) { + tile_split_slice = element_offset / (static_cast(512) * 8); + element_offset %= (static_cast(512) * 8); + tile_bytes = 512; + } + + uint64_t macro_tile_bytes = + (128 / 8) * (m_macro_tile_height / 8) * tile_bytes / (m_num_pipes * m_num_banks); + uint64_t macro_tiles_per_row = m_padded_width / 128; + uint64_t macro_tile_row_index = y / m_macro_tile_height; + uint64_t macro_tile_column_index = x / 128; + uint64_t macro_tile_index = + (macro_tile_row_index * macro_tiles_per_row) + macro_tile_column_index; + uint64_t macro_tile_offset = macro_tile_index * macro_tile_bytes; + uint64_t macro_tiles_per_slice = + macro_tiles_per_row * (m_padded_height / m_macro_tile_height); + uint64_t slice_bytes = macro_tiles_per_slice * macro_tile_bytes; + uint64_t slice_offset = tile_split_slice * slice_bytes; + uint64_t tile_row_index = (y / 8) % m_bank_height; + uint64_t tile_index = tile_row_index; + uint64_t tile_offset = tile_index * tile_bytes; + + uint64_t tile_split_slice_rotation = ((m_num_banks / 2) + 1) * tile_split_slice; + bank ^= tile_split_slice_rotation; + bank &= (m_num_banks - 1); + + uint64_t total_offset = + (slice_offset + macro_tile_offset + tile_offset) * 8 + element_offset; + uint64_t bit_offset = total_offset & 0x7u; + total_offset /= 8; + + uint64_t pipe_interleave_offset = total_offset & 0xffu; + uint64_t offset = total_offset >> 8u; + uint64_t byte_offset = pipe_interleave_offset | (pipe << (8u)) | + (bank << (8u + m_pipe_bits)) | + (offset << (8u + m_pipe_bits + m_bank_bits)); + + return ((byte_offset << 3u) | bit_offset) / 8; + } +}; + +struct FastTiler32XInfo { + uint64_t macro_offset = 0; + uint16_t element_bytes = 0; + uint8_t pipe = 0; + uint8_t bank = 0; +}; + +struct FastTiler32YInfo { + uint64_t macro_offset = 0; + uint16_t tile_offset = 0; + uint16_t element_bytes = 0; + uint8_t pipe = 0; + uint8_t bank = 0; +}; + +class FastTiler32 { +public: + void Init(const Tiler32& t, uint32_t width, uint32_t height) { + constexpr uint32_t bytes_per_element = 4; + const uint32_t tile_bytes = 8u * 8u * bytes_per_element; + const uint64_t macro_tile_bytes = (128u / 8u) * (t.m_macro_tile_height / 8u) * tile_bytes / + (t.m_num_pipes * t.m_num_banks); + const uint64_t macro_tiles_per_row = t.m_padded_width / 128u; + + m_x.resize(width); + m_y.resize(height); + m_pipe_bits = t.m_pipe_bits; + m_bank_bits = t.m_bank_bits; + + for (uint32_t x = 0; x < width; x++) { + auto& info = m_x[x]; + + info.macro_offset = (x / 128u) * macro_tile_bytes; + info.element_bytes = + static_cast(Tiler32::GetElementIndex(x, 0) * bytes_per_element); + info.pipe = static_cast(Tiler32::GetPipeIndex(x, 0)); + info.bank = static_cast( + Tiler32::GetBankIndex(x, 0, 1, t.m_bank_height, t.m_num_banks, t.m_num_pipes)); + } + + for (uint32_t y = 0; y < height; y++) { + auto& info = m_y[y]; + + info.macro_offset = (static_cast(y) / t.m_macro_tile_height) * + macro_tiles_per_row * macro_tile_bytes; + info.tile_offset = static_cast(((y / 8u) % t.m_bank_height) * tile_bytes); + info.element_bytes = + static_cast(Tiler32::GetElementIndex(0, y) * bytes_per_element); + info.pipe = static_cast(Tiler32::GetPipeIndex(0, y)); + info.bank = static_cast( + Tiler32::GetBankIndex(0, y, 1, t.m_bank_height, t.m_num_banks, t.m_num_pipes)); + } + } + + [[nodiscard]] uint64_t GetTiledOffset(uint32_t x, uint32_t y) const { + const auto& x_info = m_x[x]; + const auto& y_info = m_y[y]; + + const uint64_t base = y_info.macro_offset + x_info.macro_offset + y_info.tile_offset + + y_info.element_bytes + x_info.element_bytes; + const uint64_t pipe = x_info.pipe ^ y_info.pipe; + const uint64_t bank = x_info.bank ^ y_info.bank; + + return (base & 0xffu) | (pipe << 8u) | (bank << (8u + m_pipe_bits)) | + ((base >> 8u) << (8u + m_pipe_bits + m_bank_bits)); + } + +private: + std::vector m_x; + std::vector m_y; + uint32_t m_pipe_bits = 0; + uint32_t m_bank_bits = 0; +}; + +static Tiler* g_tiler = nullptr; + +void TileInit() { + EXIT_IF(g_tiler != nullptr); + + g_tiler = new Tiler; +} + +// NOLINTNEXTLINE(readability-non-const-parameter) +static void Detile32(const Tiler32& t, uint32_t width, uint32_t height, uint32_t dst_pitch, + uint8_t* dst, const uint8_t* src) { + EXIT_IF(g_tiler == nullptr); + + Common::LockGuard lock(g_tiler->m_mutex); + + FastTiler32 fast_tiler; + fast_tiler.Init(t, width, height); + + struct DetileParams { + const FastTiler32* t; + uint32_t start_y; + uint32_t width; + uint32_t height; + uint32_t dst_pitch; + uint8_t* dst; + const uint8_t* src; + }; + + auto func = [](void* args) { + auto* p = static_cast(args); + + auto* dst = p->dst; + const auto* src = p->src; + const FastTiler32* t = p->t; + uint32_t start_y = p->start_y; + uint32_t width = p->width; + uint32_t height = p->height; + uint64_t dst_pitch = p->dst_pitch; + + for (uint32_t y = start_y; y < height; y++) { + uint32_t x = 0; + uint64_t linear_offset = y * dst_pitch * 4; + + for (; x + 1 < width; x += 2) { + auto tiled_offset = t->GetTiledOffset(x, y); + + *reinterpret_cast(dst + linear_offset) = + *reinterpret_cast(src + tiled_offset); + linear_offset += 8; + } + if (x < width) { + auto tiled_offset = t->GetTiledOffset(x, y); + + *reinterpret_cast(dst + linear_offset) = + *reinterpret_cast(src + tiled_offset); + } + } + }; + + const uint32_t split = height / 2; + DetileParams p1 {&fast_tiler, 0, width, split, dst_pitch, dst, src}; + DetileParams p2 {&fast_tiler, split, width, height, dst_pitch, dst, src}; + + g_tiler->m_job1.Execute([func, &p2] { func(&p2); }); + func(&p1); + g_tiler->m_job1.Wait(); +} + +static constexpr uint32_t GetStandard64KB32XPart(uint32_t x) { + // FIXME: Temporary PS5 AGC TileMode::kStandard64KB 32bpp detiler. + // AGC Standard64KB is block-linear: 32bpp surfaces use 128x128-element + // 64KB blocks, with fixed x/y bit interleaving inside each block. + uint32_t element_offset = 0; + element_offset ^= (x << 2u) & 0x0cu; + element_offset ^= (x << 5u) & 0x80u; + element_offset ^= (x << 6u) & 0x200u; + element_offset ^= (x << 7u) & 0x800u; + element_offset ^= (x << 8u) & 0x2000u; + // X6 selects the upper 32KB of each 128x128x32bpp block. + element_offset ^= (x << 9u) & 0x8000u; + + return element_offset; +} + +static constexpr uint32_t GetStandard64KB32YPart(uint32_t y) { + uint32_t element_offset = 0; + element_offset ^= (y << 4u) & 0x70u; + element_offset ^= (y << 5u) & 0x100u; + element_offset ^= (y << 6u) & 0x400u; + element_offset ^= (y << 7u) & 0x1000u; + element_offset ^= (y << 8u) & 0x4000u; + + return element_offset; +} + +struct Standard64KB32Tables { + std::array x_words {}; + std::array y_words {}; + + constexpr Standard64KB32Tables() { + for (uint32_t i = 0; i < 128; i++) { + uint32_t x_part = 0; + x_part ^= (i << 2u) & 0x0cu; + x_part ^= (i << 5u) & 0x80u; + x_part ^= (i << 6u) & 0x200u; + x_part ^= (i << 7u) & 0x800u; + x_part ^= (i << 8u) & 0x2000u; + x_part ^= (i << 9u) & 0x8000u; + + uint32_t y_part = 0; + y_part ^= (i << 4u) & 0x70u; + y_part ^= (i << 5u) & 0x100u; + y_part ^= (i << 6u) & 0x400u; + y_part ^= (i << 7u) & 0x1000u; + y_part ^= (i << 8u) & 0x4000u; + + x_words[i] = x_part >> 2u; + y_words[i] = y_part >> 2u; + } + } +}; + +static_assert(GetStandard64KB32XPart(127) < (64u * 1024u)); +static_assert(GetStandard64KB32YPart(127) < (64u * 1024u)); + +static const Standard64KB32Tables& GetStandard64KB32Tables() { + static constexpr Standard64KB32Tables tables; + return tables; +} + +struct Standard64KB16Tables { + std::array x_words {}; + std::array y_words {}; + + constexpr Standard64KB16Tables() { + for (uint32_t i = 0; i < 256; i++) { + uint32_t x_part = 0; + x_part ^= (i << 1u) & 0x0006u; + x_part ^= (i << 4u) & 0x0040u; + x_part ^= (i << 5u) & 0x0100u; + x_part ^= (i << 6u) & 0x0400u; + x_part ^= (i << 7u) & 0x1000u; + x_part ^= (i << 8u) & 0x4000u; + x_part ^= (i << 8u) & 0x8000u; + + x_words[i] = x_part >> 1u; + } + + for (uint32_t i = 0; i < 128; i++) { + uint32_t y_part = 0; + y_part ^= (i << 3u) & 0x0038u; + y_part ^= (i << 4u) & 0x0080u; + y_part ^= (i << 5u) & 0x0200u; + y_part ^= (i << 6u) & 0x0800u; + y_part ^= (i << 7u) & 0x2000u; + + y_words[i] = y_part >> 1u; + } + } +}; + +template +static uint32_t Gen5RenderTargetOffsetInBlock(uint32_t x, uint32_t y) { + uint32_t offset = 0; + + if constexpr (sizeof(T) == 1) { + offset ^= (y << 2u) & 0x0008u; + offset ^= (y << 4u) & 0x0010u; + offset ^= (y << 3u) & 0x00a0u; + offset ^= (y << 5u) & 0x0f00u; + offset ^= (y << 6u) & 0x1000u; + offset ^= (y << 7u) & 0x4000u; + + offset ^= x & 0x0007u; + offset ^= (x << 3u) & 0x0040u; + offset ^= (x << 5u) & 0x0300u; + offset ^= (x << 4u) & 0x0400u; + offset ^= (x << 6u) & 0x0800u; + offset ^= (x << 7u) & 0x2000u; + offset ^= (x << 8u) & 0x8000u; + } else if constexpr (sizeof(T) == 2) { + offset ^= (y << 4u) & 0x0070u; + offset ^= (y << 5u) & 0x0f00u; + offset ^= (y << 8u) & 0x5000u; + + offset ^= (x << 1u) & 0x000eu; + offset ^= (x << 4u) & 0x0480u; + offset ^= (x << 5u) & 0x0300u; + offset ^= (x << 6u) & 0x0800u; + offset ^= (x << 7u) & 0x2000u; + offset ^= (x << 8u) & 0x8000u; + } else if constexpr (sizeof(T) == 4) { + offset ^= (y << 3u) & 0x0008u; + offset ^= (y << 4u) & 0x0020u; + offset ^= (y << 5u) & 0x0f80u; + offset ^= (y << 9u) & 0x1000u; + offset ^= (y << 8u) & 0x4000u; + + offset ^= (x << 2u) & 0x0004u; + offset ^= (x << 3u) & 0x0010u; + offset ^= (x << 4u) & 0x0440u; + offset ^= (x << 5u) & 0x0300u; + offset ^= (x << 6u) & 0x0800u; + offset ^= (x << 9u) & 0xa000u; + } else if constexpr (sizeof(T) == 8) { + offset ^= (y << 4u) & 0x0010u; + offset ^= (y << 6u) & 0x0080u; + offset ^= (y << 5u) & 0x0f00u; + offset ^= (y << 10u) & 0x5000u; + + offset ^= (x << 3u) & 0x0008u; + offset ^= (x << 4u) & 0x0460u; + offset ^= (x << 5u) & 0x0300u; + offset ^= (x << 6u) & 0x0800u; + offset ^= (x << 10u) & 0x2000u; + offset ^= (x << 9u) & 0x8000u; + } else { + EXIT("unsupported render-target element size: %u\n", static_cast(sizeof(T))); + } + + return offset; +} + +template +static void ConvertRenderTargetTyped(uint32_t width, uint32_t height, uint32_t pitch, T* dst, + const T* src, uint64_t size) { + EXIT_IF(g_tiler == nullptr); + + uint32_t block_width = 0; + uint32_t block_height = 0; + EXIT_NOT_IMPLEMENTED(!Gen5RenderTargetBlockSizeFromElementBytes( + static_cast(sizeof(T)), &block_width, &block_height)); + + const uint64_t blocks_per_row = (static_cast(pitch) + block_width - 1u) / block_width; + const uint32_t block_columns = (width == 0 ? 0 : 1u + (width - 1u) / block_width); + const uint64_t elements_per_block = 65536u / sizeof(T); + const uint64_t count = size / sizeof(T); + + std::array x_elements {}; + std::array y_elements {}; + for (uint32_t x = 0; x < block_width; x++) { + x_elements[x] = Gen5RenderTargetOffsetInBlock(x, 0) / sizeof(T); + } + for (uint32_t y = 0; y < block_height; y++) { + y_elements[y] = Gen5RenderTargetOffsetInBlock(0, y) / sizeof(T); + } + + struct ConvertParams { + uint32_t start_block_row; + uint32_t end_block_row; + uint32_t width; + uint32_t height; + uint32_t pitch; + uint32_t block_width; + uint32_t block_height; + uint32_t block_columns; + uint64_t blocks_per_row; + uint64_t elements_per_block; + const uint32_t* x_elements; + const uint32_t* y_elements; + T* dst; + const T* src; + uint64_t count; + }; + + auto func = [](void* args) { + auto* p = static_cast(args); + + for (uint32_t block_row = p->start_block_row; block_row < p->end_block_row; block_row++) { + const uint32_t block_y = block_row * p->block_height; + const uint32_t copy_height = std::min(p->block_height, p->height - block_y); + const uint32_t block_y_element = + Gen5RenderTargetOffsetInBlock(0, block_y) / sizeof(T); + + for (uint32_t block_column = 0; block_column < p->block_columns; block_column++) { + const uint32_t block_x = + static_cast(static_cast(block_column) * p->block_width); + const uint32_t copy_width = std::min(p->block_width, p->width - block_x); + const uint64_t block_base = + (static_cast(block_row) * p->blocks_per_row + block_column) * + p->elements_per_block; + + for (uint32_t y = 0; y < copy_height; y++) { + const uint64_t linear_row = + static_cast(block_y + y) * p->pitch + block_x; + for (uint32_t x = 0; x < copy_width; x++) { + const uint64_t linear_index = linear_row + x; + const uint64_t tiled_index = + block_base + (p->x_elements[x] ^ p->y_elements[y] ^ block_y_element); + if (linear_index < p->count && tiled_index < p->count) { + if constexpr (tiled_to_linear) { + p->dst[linear_index] = p->src[tiled_index]; + } else { + p->dst[tiled_index] = p->src[linear_index]; + } + } + } + } + } + } + }; + + const uint32_t block_rows = (height == 0 ? 0 : 1u + (height - 1u) / block_height); + const uint32_t split = block_rows / 2; + ConvertParams p1 {0, + split, + width, + height, + pitch, + block_width, + block_height, + block_columns, + blocks_per_row, + elements_per_block, + x_elements.data(), + y_elements.data(), + dst, + src, + count}; + ConvertParams p2 {split, + block_rows, + width, + height, + pitch, + block_width, + block_height, + block_columns, + blocks_per_row, + elements_per_block, + x_elements.data(), + y_elements.data(), + dst, + src, + count}; + + Common::LockGuard lock(g_tiler->m_mutex); + g_tiler->m_job1.Execute([func, &p2] { func(&p2); }); + func(&p1); + g_tiler->m_job1.Wait(); +} + +static const Standard64KB16Tables& GetStandard64KB16Tables() { + static constexpr Standard64KB16Tables tables; + return tables; +} + +struct Standard64KB8Tables { + std::array x_words {}; + std::array y_words {}; + + constexpr Standard64KB8Tables() { + for (uint32_t i = 0; i < 256; i++) { + uint32_t x_part = 0; + x_part ^= i & 0x000fu; + x_part ^= (i << 5u) & 0x0200u; + x_part ^= (i << 6u) & 0x0800u; + x_part ^= (i << 7u) & 0x2000u; + x_part ^= (i << 8u) & 0x8000u; + + uint32_t y_part = 0; + y_part ^= (i << 4u) & 0x01f0u; + y_part ^= (i << 5u) & 0x0400u; + y_part ^= (i << 6u) & 0x1000u; + y_part ^= (i << 7u) & 0x4000u; + + x_words[i] = x_part; + y_words[i] = y_part; + } + } +}; + +static const Standard64KB8Tables& GetStandard64KB8Tables() { + static constexpr Standard64KB8Tables tables; + return tables; +} + +struct Standard64KB64Tables { + std::array x_words {}; + std::array y_words {}; + + constexpr Standard64KB64Tables() { + for (uint32_t i = 0; i < 128; i++) { + uint32_t x_part = 0; + x_part ^= (i << 3u) & 0x0008u; + x_part ^= (i << 5u) & 0x00c0u; + x_part ^= (i << 6u) & 0x0200u; + x_part ^= (i << 7u) & 0x0800u; + x_part ^= (i << 8u) & 0x2000u; + x_part ^= (i << 9u) & 0x8000u; + + x_words[i] = x_part >> 3u; + } + + for (uint32_t i = 0; i < 64; i++) { + uint32_t y_part = 0; + y_part ^= (i << 4u) & 0x0030u; + y_part ^= (i << 6u) & 0x0100u; + y_part ^= (i << 7u) & 0x0400u; + y_part ^= (i << 8u) & 0x1000u; + y_part ^= (i << 9u) & 0x4000u; + + y_words[i] = y_part >> 3u; + } + } +}; + +static const Standard64KB64Tables& GetStandard64KB64Tables() { + static constexpr Standard64KB64Tables tables; + return tables; +} + +struct Standard64KB128Tables { + std::array x_words {}; + std::array y_words {}; + + constexpr Standard64KB128Tables() { + for (uint32_t i = 0; i < 64; i++) { + uint32_t x_part = 0; + x_part ^= (i << 6u) & 0x00c0u; + x_part ^= (i << 7u) & 0x0200u; + x_part ^= (i << 8u) & 0x0800u; + x_part ^= (i << 9u) & 0x2000u; + x_part ^= (i << 10u) & 0x8000u; + + uint32_t y_part = 0; + y_part ^= (i << 4u) & 0x0030u; + y_part ^= (i << 6u) & 0x0100u; + y_part ^= (i << 7u) & 0x0400u; + y_part ^= (i << 8u) & 0x1000u; + y_part ^= (i << 9u) & 0x4000u; + + x_words[i] = x_part >> 4u; + y_words[i] = y_part >> 4u; + } + } +}; + +static const Standard64KB128Tables& GetStandard64KB128Tables() { + static constexpr Standard64KB128Tables tables; + return tables; +} + +void TileConvertTiledToLinearStandard64KB32(void* dst, const void* src, uint32_t width, + uint32_t height, uint32_t pitch, uint64_t size, + uint64_t src_size, uint32_t src_x, uint32_t src_y) { + KYTY_PROFILER_FUNCTION(); + + EXIT_IF(dst == nullptr || src == nullptr); + EXIT_NOT_IMPLEMENTED(pitch == 0); + const bool tail_block = (src_size != 0 && (src_x != 0 || src_y != 0 || size < src_size)); + EXIT_NOT_IMPLEMENTED(width > pitch); + EXIT_NOT_IMPLEMENTED(tail_block && (src_x + width > 128u || src_y + height > 128u)); + + auto* dst32 = static_cast(dst); + const auto* src32 = static_cast(src); + + const auto& tables = GetStandard64KB32Tables(); + const auto size_words = size >> 2u; + const auto src_size_words = (src_size != 0 ? src_size : size) >> 2u; + const auto blocks_per_row = + (tail_block ? 1u : static_cast(AlignUp(pitch, 128u) >> 7u)); + const auto dst_words = static_cast(pitch) * height; + const bool exact_surface = + (!tail_block && width == pitch && (size & 3u) == 0 && size_words == dst_words); + const auto* x_words = tables.x_words.data(); + const auto* y_words = tables.y_words.data(); + + if (!exact_surface) { + std::memset(dst32, 0, static_cast(size)); + } + + for (uint32_t block_y = 0; block_y < height; block_y += 128u) { + const uint32_t block_height = std::min(128u, height - block_y); + + for (uint32_t block_x = 0; block_x < width; block_x += 128u) { + const uint32_t block_width = std::min(128u, width - block_x); + const uint64_t block_base = + (tail_block + ? 0u + : (((static_cast(block_y) >> 7u) * blocks_per_row) + (block_x >> 7u)) + << 14u); + const bool full_block = + (!tail_block && block_width == 128u && block_height == 128u && + block_base + 16384u <= src_size_words && + (static_cast(block_y + 127u) * pitch) + block_x + 127u < size_words); + + if (full_block) { + for (uint32_t local_y = 0; local_y < 128u; local_y++) { + const uint64_t dst_row = + (static_cast(block_y + local_y) * pitch) + block_x; + const uint64_t src_row = block_base + y_words[src_y + local_y]; + + for (uint32_t local_x = 0; local_x < 128u; local_x += 4u) { + dst32[dst_row + local_x + 0u] = + src32[src_row + x_words[src_x + local_x + 0u]]; + dst32[dst_row + local_x + 1u] = + src32[src_row + x_words[src_x + local_x + 1u]]; + dst32[dst_row + local_x + 2u] = + src32[src_row + x_words[src_x + local_x + 2u]]; + dst32[dst_row + local_x + 3u] = + src32[src_row + x_words[src_x + local_x + 3u]]; + } + } + continue; + } + + for (uint32_t local_y = 0; local_y < block_height; local_y++) { + const uint64_t dst_row = + (static_cast(block_y + local_y) * pitch) + block_x; + const uint64_t src_row = block_base + y_words[src_y + local_y]; + + for (uint32_t local_x = 0; local_x < block_width; local_x++) { + const uint64_t dst_index = dst_row + local_x; + const uint64_t src_index = src_row + x_words[src_x + local_x]; + + if (src_index < src_size_words && dst_index < size_words) { + dst32[dst_index] = src32[src_index]; + } + } + } + } + } +} + +void TileConvertTiledToLinearStandard64KB16(void* dst, const void* src, uint32_t width, + uint32_t height, uint32_t pitch, uint64_t size, + uint64_t src_size, uint32_t src_x, uint32_t src_y) { + KYTY_PROFILER_FUNCTION(); + + EXIT_IF(dst == nullptr || src == nullptr); + EXIT_NOT_IMPLEMENTED(pitch == 0); + const bool tail_block = (src_size != 0 && (src_x != 0 || src_y != 0 || size < src_size)); + EXIT_NOT_IMPLEMENTED(width > pitch); + EXIT_NOT_IMPLEMENTED(tail_block && (src_x + width > 256u || src_y + height > 128u)); + + auto* dst16 = static_cast(dst); + const auto* src16 = static_cast(src); + + const auto& tables = GetStandard64KB16Tables(); + const auto size_words = size >> 1u; + const auto src_size_words = (src_size != 0 ? src_size : size) >> 1u; + const auto blocks_per_row = + (tail_block ? 1u : static_cast(AlignUp(pitch, 256u) >> 8u)); + const auto dst_words = static_cast(pitch) * height; + const bool exact_surface = + (!tail_block && width == pitch && (size & 1u) == 0 && size_words == dst_words); + const auto* x_words = tables.x_words.data(); + const auto* y_words = tables.y_words.data(); + + if (!exact_surface) { + std::memset(dst16, 0, static_cast(size)); + } + + for (uint32_t block_y = 0; block_y < height; block_y += 128u) { + const uint32_t block_height = std::min(128u, height - block_y); + + for (uint32_t block_x = 0; block_x < width; block_x += 256u) { + const uint32_t block_width = std::min(256u, width - block_x); + const uint64_t block_base = + (tail_block + ? 0u + : (((static_cast(block_y) >> 7u) * blocks_per_row) + (block_x >> 8u)) + << 15u); + const bool full_block = + (!tail_block && block_width == 256u && block_height == 128u && + block_base + 32768u <= src_size_words && + (static_cast(block_y + 127u) * pitch) + block_x + 255u < size_words); + + if (full_block) { + for (uint32_t local_y = 0; local_y < 128u; local_y++) { + const uint64_t dst_row = + (static_cast(block_y + local_y) * pitch) + block_x; + const uint64_t src_row = block_base + y_words[src_y + local_y]; + + for (uint32_t local_x = 0; local_x < 256u; local_x += 4u) { + dst16[dst_row + local_x + 0u] = + src16[src_row + x_words[src_x + local_x + 0u]]; + dst16[dst_row + local_x + 1u] = + src16[src_row + x_words[src_x + local_x + 1u]]; + dst16[dst_row + local_x + 2u] = + src16[src_row + x_words[src_x + local_x + 2u]]; + dst16[dst_row + local_x + 3u] = + src16[src_row + x_words[src_x + local_x + 3u]]; + } + } + continue; + } + + for (uint32_t local_y = 0; local_y < block_height; local_y++) { + const uint64_t dst_row = + (static_cast(block_y + local_y) * pitch) + block_x; + const uint64_t src_row = block_base + y_words[src_y + local_y]; + + for (uint32_t local_x = 0; local_x < block_width; local_x++) { + const uint64_t dst_index = dst_row + local_x; + const uint64_t src_index = src_row + x_words[src_x + local_x]; + + if (src_index < src_size_words && dst_index < size_words) { + dst16[dst_index] = src16[src_index]; + } + } + } + } + } +} + +static void TileConvertTiledToLinearStandard64KB8Elements( + void* dst, const void* src, uint32_t width_elements, uint32_t height_elements, + uint32_t pitch_elements, uint64_t size, uint64_t src_size, uint32_t src_x, uint32_t src_y) { + KYTY_PROFILER_FUNCTION(); + + EXIT_IF(dst == nullptr || src == nullptr); + EXIT_NOT_IMPLEMENTED(pitch_elements == 0); + const bool tail_block = (src_size != 0 && (src_x != 0 || src_y != 0 || size < src_size)); + EXIT_NOT_IMPLEMENTED(width_elements > pitch_elements); + EXIT_NOT_IMPLEMENTED(tail_block && + (src_x + width_elements > 256u || src_y + height_elements > 256u)); + + auto* dst8 = static_cast(dst); + const auto* src8 = static_cast(src); + + const auto& tables = GetStandard64KB8Tables(); + const auto size_words = size; + const auto src_size_words = (src_size != 0 ? src_size : size); + const auto blocks_per_row = + (tail_block ? 1u : static_cast(AlignUp(pitch_elements, 256u) >> 8u)); + const auto dst_words = static_cast(pitch_elements) * height_elements; + const bool exact_surface = (!tail_block && size_words == dst_words); + const auto* x_words = tables.x_words.data(); + const auto* y_words = tables.y_words.data(); + + if (!exact_surface) { + std::memset(dst8, 0, static_cast(size)); + } + + for (uint32_t block_y = 0; block_y < height_elements; block_y += 256u) { + const uint32_t block_height = std::min(256u, height_elements - block_y); + + for (uint32_t block_x = 0; block_x < width_elements; block_x += 256u) { + const uint32_t block_width = std::min(256u, width_elements - block_x); + const uint64_t block_base = + (tail_block + ? 0u + : (((static_cast(block_y) >> 8u) * blocks_per_row) + (block_x >> 8u)) + << 16u); + const bool full_block = + (!tail_block && block_width == 256u && block_height == 256u && + block_base + 65536u <= src_size_words && + (static_cast(block_y + 255u) * pitch_elements) + block_x + 255u < + size_words); + + if (full_block) { + for (uint32_t local_y = 0; local_y < 256u; local_y++) { + const uint64_t dst_row = + (static_cast(block_y + local_y) * pitch_elements) + block_x; + const uint64_t src_row = block_base + y_words[src_y + local_y]; + + for (uint32_t local_x = 0; local_x < 256u; local_x += 4u) { + dst8[dst_row + local_x + 0u] = + src8[src_row + x_words[src_x + local_x + 0u]]; + dst8[dst_row + local_x + 1u] = + src8[src_row + x_words[src_x + local_x + 1u]]; + dst8[dst_row + local_x + 2u] = + src8[src_row + x_words[src_x + local_x + 2u]]; + dst8[dst_row + local_x + 3u] = + src8[src_row + x_words[src_x + local_x + 3u]]; + } + } + continue; + } + + for (uint32_t local_y = 0; local_y < block_height; local_y++) { + const uint64_t dst_row = + (static_cast(block_y + local_y) * pitch_elements) + block_x; + const uint64_t src_row = block_base + y_words[src_y + local_y]; + + for (uint32_t local_x = 0; local_x < block_width; local_x++) { + const uint64_t dst_index = dst_row + local_x; + const uint64_t src_index = src_row + x_words[src_x + local_x]; + + if (src_index < src_size_words && dst_index < size_words) { + dst8[dst_index] = src8[src_index]; + } + } + } + } + } +} + +static void TileConvertTiledToLinearStandard64KB64Elements( + void* dst, const void* src, uint32_t width_elements, uint32_t height_elements, + uint32_t pitch_elements, uint64_t size, uint64_t src_size, uint32_t src_x, uint32_t src_y) { + KYTY_PROFILER_FUNCTION(); + + EXIT_IF(dst == nullptr || src == nullptr); + EXIT_NOT_IMPLEMENTED(pitch_elements == 0); + const bool tail_block = (src_size != 0 && (src_x != 0 || src_y != 0 || size < src_size)); + EXIT_NOT_IMPLEMENTED(width_elements > pitch_elements); + EXIT_NOT_IMPLEMENTED(tail_block && + (src_x + width_elements > 128u || src_y + height_elements > 64u)); + + auto* dst64 = static_cast(dst); + const auto* src64 = static_cast(src); + + const auto& tables = GetStandard64KB64Tables(); + const auto size_words = size >> 3u; + const auto src_size_words = (src_size != 0 ? src_size : size) >> 3u; + const auto blocks_per_row = + (tail_block ? 1u : static_cast(AlignUp(pitch_elements, 128u) >> 7u)); + const auto dst_words = static_cast(pitch_elements) * height_elements; + const bool exact_surface = (!tail_block && (size & 7u) == 0 && size_words == dst_words); + const auto* x_words = tables.x_words.data(); + const auto* y_words = tables.y_words.data(); + + if (!exact_surface) { + std::memset(dst64, 0, static_cast(size)); + } + + for (uint32_t block_y = 0; block_y < height_elements; block_y += 64u) { + const uint32_t block_height = std::min(64u, height_elements - block_y); + + for (uint32_t block_x = 0; block_x < width_elements; block_x += 128u) { + const uint32_t block_width = std::min(128u, width_elements - block_x); + const uint64_t block_base = + (tail_block + ? 0u + : (((static_cast(block_y) >> 6u) * blocks_per_row) + (block_x >> 7u)) + << 13u); + const bool full_block = + (!tail_block && block_width == 128u && block_height == 64u && + block_base + 8192u <= src_size_words && + (static_cast(block_y + 63u) * pitch_elements) + block_x + 127u < + size_words); + + if (full_block) { + for (uint32_t local_y = 0; local_y < 64u; local_y++) { + const uint64_t dst_row = + (static_cast(block_y + local_y) * pitch_elements) + block_x; + const uint64_t src_row = block_base + y_words[src_y + local_y]; + + for (uint32_t local_x = 0; local_x < 128u; local_x += 8u) { + dst64[dst_row + local_x + 0u] = + src64[src_row + x_words[src_x + local_x + 0u]]; + dst64[dst_row + local_x + 1u] = + src64[src_row + x_words[src_x + local_x + 1u]]; + dst64[dst_row + local_x + 2u] = + src64[src_row + x_words[src_x + local_x + 2u]]; + dst64[dst_row + local_x + 3u] = + src64[src_row + x_words[src_x + local_x + 3u]]; + dst64[dst_row + local_x + 4u] = + src64[src_row + x_words[src_x + local_x + 4u]]; + dst64[dst_row + local_x + 5u] = + src64[src_row + x_words[src_x + local_x + 5u]]; + dst64[dst_row + local_x + 6u] = + src64[src_row + x_words[src_x + local_x + 6u]]; + dst64[dst_row + local_x + 7u] = + src64[src_row + x_words[src_x + local_x + 7u]]; + } + } + continue; + } + + for (uint32_t local_y = 0; local_y < block_height; local_y++) { + const uint64_t dst_row = + (static_cast(block_y + local_y) * pitch_elements) + block_x; + const uint64_t src_row = block_base + y_words[src_y + local_y]; + + for (uint32_t local_x = 0; local_x < block_width; local_x++) { + const uint64_t dst_index = dst_row + local_x; + const uint64_t src_index = src_row + x_words[src_x + local_x]; + + if (src_index < src_size_words && dst_index < size_words) { + dst64[dst_index] = src64[src_index]; + } + } + } + } + } +} + +static void TileConvertTiledToLinearStandard64KB128Elements( + void* dst, const void* src, uint32_t width_elements, uint32_t height_elements, + uint32_t pitch_elements, uint64_t size, uint64_t src_size, uint32_t src_x, uint32_t src_y) { + KYTY_PROFILER_FUNCTION(); + + EXIT_IF(dst == nullptr || src == nullptr); + EXIT_NOT_IMPLEMENTED(pitch_elements == 0); + const bool tail_block = (src_size != 0 && (src_x != 0 || src_y != 0 || size < src_size)); + EXIT_NOT_IMPLEMENTED(width_elements > pitch_elements); + EXIT_NOT_IMPLEMENTED(tail_block && + (src_x + width_elements > 64u || src_y + height_elements > 64u)); + + auto* dst128 = static_cast(dst); + const auto* src128 = static_cast(src); + + const auto& tables = GetStandard64KB128Tables(); + const auto size_words = size >> 4u; + const auto src_size_words = (src_size != 0 ? src_size : size) >> 4u; + const auto blocks_per_row = + (tail_block ? 1u : static_cast(AlignUp(pitch_elements, 64u) >> 6u)); + const auto dst_words = static_cast(pitch_elements) * height_elements; + const bool exact_surface = (!tail_block && (size & 15u) == 0 && size_words == dst_words); + const auto* x_words = tables.x_words.data(); + const auto* y_words = tables.y_words.data(); + + if (!exact_surface) { + std::memset(dst128, 0, static_cast(size)); + } + + for (uint32_t block_y = 0; block_y < height_elements; block_y += 64u) { + const uint32_t block_height = std::min(64u, height_elements - block_y); + + for (uint32_t block_x = 0; block_x < width_elements; block_x += 64u) { + const uint32_t block_width = std::min(64u, width_elements - block_x); + const uint64_t block_base = + (tail_block + ? 0u + : (((static_cast(block_y) >> 6u) * blocks_per_row) + (block_x >> 6u)) + << 12u); + const bool full_block = + (!tail_block && block_width == 64u && block_height == 64u && + block_base + 4096u <= src_size_words && + (static_cast(block_y + 63u) * pitch_elements) + block_x + 63u < + size_words); + + if (full_block) { + for (uint32_t local_y = 0; local_y < 64u; local_y++) { + const uint64_t dst_row = + (static_cast(block_y + local_y) * pitch_elements) + block_x; + const uint64_t src_row = block_base + y_words[src_y + local_y]; + + for (uint32_t local_x = 0; local_x < 64u; local_x += 4u) { + dst128[dst_row + local_x + 0u] = + src128[src_row + x_words[src_x + local_x + 0u]]; + dst128[dst_row + local_x + 1u] = + src128[src_row + x_words[src_x + local_x + 1u]]; + dst128[dst_row + local_x + 2u] = + src128[src_row + x_words[src_x + local_x + 2u]]; + dst128[dst_row + local_x + 3u] = + src128[src_row + x_words[src_x + local_x + 3u]]; + } + } + continue; + } + + for (uint32_t local_y = 0; local_y < block_height; local_y++) { + const uint64_t dst_row = + (static_cast(block_y + local_y) * pitch_elements) + block_x; + const uint64_t src_row = block_base + y_words[src_y + local_y]; + + for (uint32_t local_x = 0; local_x < block_width; local_x++) { + const uint64_t dst_index = dst_row + local_x; + const uint64_t src_index = src_row + x_words[src_x + local_x]; + + if (src_index < src_size_words && dst_index < size_words) { + dst128[dst_index] = src128[src_index]; + } + } + } + } + } +} + +void TileConvertTiledToLinearStandard64KB(void* dst, const void* src, uint32_t format, + uint32_t width, uint32_t height, uint32_t pitch, + uint64_t size, uint64_t src_size, uint32_t src_x, + uint32_t src_y) { + uint32_t bytes_per_element = 0; + uint32_t texels_per_element_wide = 0; + uint32_t texels_per_element_tall = 0; + uint32_t block_width_log2 = 0; + uint32_t block_height_log2 = 0; + + EXIT_NOT_IMPLEMENTED(!Gen5Standard64KBLayout(format, &bytes_per_element, + &texels_per_element_wide, &texels_per_element_tall, + &block_width_log2, &block_height_log2)); + + const uint32_t width_elements = + std::max((width + texels_per_element_wide - 1u) / texels_per_element_wide, 1u); + const uint32_t height_elements = + std::max((height + texels_per_element_tall - 1u) / texels_per_element_tall, 1u); + const uint32_t pitch_elements = + std::max((pitch + texels_per_element_wide - 1u) / texels_per_element_wide, 1u); + + switch (bytes_per_element) { + case 1: + TileConvertTiledToLinearStandard64KB8Elements(dst, src, width_elements, height_elements, + pitch_elements, size, src_size, src_x, + src_y); + break; + case 2: + TileConvertTiledToLinearStandard64KB16(dst, src, width_elements, height_elements, + pitch_elements, size, src_size, src_x, src_y); + break; + case 4: + TileConvertTiledToLinearStandard64KB32(dst, src, width_elements, height_elements, + pitch_elements, size, src_size, src_x, src_y); + break; + case 8: + TileConvertTiledToLinearStandard64KB64Elements(dst, src, width_elements, + height_elements, pitch_elements, size, + src_size, src_x, src_y); + break; + case 16: + TileConvertTiledToLinearStandard64KB128Elements(dst, src, width_elements, + height_elements, pitch_elements, size, + src_size, src_x, src_y); + break; + default: EXIT("unsupported Standard64KB element size: %u\n", bytes_per_element); + } +} + +void TileConvertTiledToLinearStandard256B(void* dst, const void* src, uint32_t format, + uint32_t width, uint32_t height, uint32_t pitch, + uint64_t dst_size, uint64_t src_size) { + uint32_t bytes_per_element = 0; + uint32_t texels_per_element_wide = 0; + uint32_t texels_per_element_tall = 0; + uint32_t block_width_log2 = 0; + uint32_t block_height_log2 = 0; + + EXIT_NOT_IMPLEMENTED(!Gen5Standard256BLayout(format, &bytes_per_element, + &texels_per_element_wide, &texels_per_element_tall, + &block_width_log2, &block_height_log2)); + + const uint32_t row_elements = + std::max((pitch + texels_per_element_wide - 1u) / texels_per_element_wide, 1u); + const uint32_t width_elements = + std::max((width + texels_per_element_wide - 1u) / texels_per_element_wide, 1u); + const uint32_t height_elements = + std::max((height + texels_per_element_tall - 1u) / texels_per_element_tall, 1u); + const uint32_t padded_width = AlignUp(row_elements, 1u << block_width_log2); + + if (src_size == 0) { + src_size = dst_size; + } + + switch (bytes_per_element) { + case 1: + DetileStandard256BTyped(static_cast(dst), static_cast(src), + row_elements, width_elements, height_elements, padded_width, + block_width_log2, block_height_log2, dst_size, src_size); + break; + case 2: + DetileStandard256BTyped(static_cast(dst), static_cast(src), + row_elements, width_elements, height_elements, padded_width, + block_width_log2, block_height_log2, dst_size, src_size); + break; + case 4: + DetileStandard256BTyped(static_cast(dst), static_cast(src), + row_elements, width_elements, height_elements, padded_width, + block_width_log2, block_height_log2, dst_size, src_size); + break; + case 8: + DetileStandard256BTyped(static_cast(dst), static_cast(src), + row_elements, width_elements, height_elements, padded_width, + block_width_log2, block_height_log2, dst_size, src_size); + break; + case 16: + DetileStandard256BTyped(static_cast(dst), static_cast(src), + row_elements, width_elements, height_elements, padded_width, + block_width_log2, block_height_log2, dst_size, src_size); + break; + default: EXIT("unsupported Standard256B element size: %u\n", bytes_per_element); + } +} + +// Prospero depth-tile address table for 2D, 1xAA surfaces by element size. +static constexpr uint32_t Depth64KB16XOffsetBytes(uint32_t x) { + uint32_t offset = 0; + offset ^= (x << 1u) & 0x0002u; + offset ^= (x << 2u) & 0x0008u; + offset ^= (x << 3u) & 0x0020u; + offset ^= (x << 4u) & 0x0480u; + offset ^= (x << 5u) & 0x0300u; + offset ^= (x << 6u) & 0x0800u; + offset ^= (x << 7u) & 0x2000u; + offset ^= (x << 8u) & 0x8000u; + return offset; +} + +static constexpr uint32_t Depth64KB16YOffsetBytes(uint32_t y) { + uint32_t offset = 0; + offset ^= (y << 2u) & 0x0004u; + offset ^= (y << 3u) & 0x0010u; + offset ^= (y << 4u) & 0x0040u; + offset ^= (y << 5u) & 0x0f00u; + offset ^= (y << 8u) & 0x5000u; + return offset; +} + +static constexpr uint32_t Depth64KB32XOffsetBytes(uint32_t x) { + uint32_t offset = 0; + offset ^= (x << 2u) & 0x0004u; + offset ^= (x << 3u) & 0x0010u; + offset ^= (x << 4u) & 0x0440u; + offset ^= (x << 5u) & 0x0300u; + offset ^= (x << 6u) & 0x0800u; + offset ^= (x << 9u) & 0xa000u; + return offset; +} + +static constexpr uint32_t Depth64KB32YOffsetBytes(uint32_t y) { + uint32_t offset = 0; + offset ^= (y << 3u) & 0x0008u; + offset ^= (y << 4u) & 0x0020u; + offset ^= (y << 5u) & 0x0f80u; + offset ^= (y << 9u) & 0x1000u; + offset ^= (y << 8u) & 0x4000u; + return offset; +} + +static_assert(Depth64KB16XOffsetBytes(2) == 0x0008u); +static_assert(Depth64KB16YOffsetBytes(1) == 0x0004u); +static_assert((Depth64KB16XOffsetBytes(3) ^ Depth64KB16YOffsetBytes(5)) == 0x004eu); +static_assert(Depth64KB32XOffsetBytes(2) == 0x0010u); +static_assert(Depth64KB32YOffsetBytes(1) == 0x0008u); +static_assert((Depth64KB32XOffsetBytes(3) ^ Depth64KB32YOffsetBytes(5)) == 0x009cu); + +struct Depth64KB16Tables { + std::array x_words {}; + std::array y_words {}; + + constexpr Depth64KB16Tables() { + for (uint32_t x = 0; x < x_words.size(); x++) { + x_words[x] = Depth64KB16XOffsetBytes(x) / sizeof(uint16_t); + } + for (uint32_t y = 0; y < y_words.size(); y++) { + y_words[y] = Depth64KB16YOffsetBytes(y) / sizeof(uint16_t); + } + } +}; + +struct Depth64KB32Tables { + std::array x_words {}; + std::array y_words {}; + + constexpr Depth64KB32Tables() { + for (uint32_t x = 0; x < x_words.size(); x++) { + x_words[x] = Depth64KB32XOffsetBytes(x) / sizeof(uint32_t); + } + for (uint32_t y = 0; y < y_words.size(); y++) { + y_words[y] = Depth64KB32YOffsetBytes(y) / sizeof(uint32_t); + } + } +}; + +template +static void TileConvertDepthTyped(void* dst, const void* src, uint32_t width, uint32_t height, + uint32_t pitch, uint64_t size, + const std::array& x_words, + const std::array& y_words) { + constexpr uint32_t block_width = static_cast(x_count); + constexpr uint32_t block_height = static_cast(y_count); + const uint32_t block_columns = 1u + (width - 1u) / block_width; + const uint64_t blocks_per_row = pitch / block_width; + const uint32_t block_rows = 1u + (height - 1u) / block_height; + + struct ConvertParams { + uint32_t start_block_row; + uint32_t end_block_row; + uint32_t width; + uint32_t height; + uint32_t pitch; + uint32_t block_columns; + uint64_t blocks_per_row; + const uint32_t* x_words; + const uint32_t* y_words; + T* dst; + const T* src; + }; + + auto convert = [](void* args) { + auto* p = static_cast(args); + for (uint32_t block_row = p->start_block_row; block_row < p->end_block_row; block_row++) { + const uint32_t block_y = block_row * static_cast(y_count); + const uint32_t copy_height = + std::min(static_cast(y_count), p->height - block_y); + for (uint32_t block_column = 0; block_column < p->block_columns; block_column++) { + const uint32_t block_x = block_column * static_cast(x_count); + const uint32_t copy_width = + std::min(static_cast(x_count), p->width - block_x); + const uint64_t block_base = + (static_cast(block_row) * p->blocks_per_row + block_column) * + (65536u / sizeof(T)); + for (uint32_t y = 0; y < copy_height; y++) { + const uint64_t linear_row = + static_cast(block_y + y) * p->pitch + block_x; + const uint64_t tiled_row = block_base + p->y_words[y]; + for (uint32_t x = 0; x < copy_width; x++) { + const uint64_t linear_index = linear_row + x; + const uint64_t tiled_index = tiled_row ^ p->x_words[x]; + if constexpr (tiled_to_linear) { + p->dst[linear_index] = p->src[tiled_index]; + } else { + p->dst[tiled_index] = p->src[linear_index]; + } + } + } + } + } + }; + + auto* dst_typed = static_cast(dst); + const auto* src_typed = static_cast(src); + std::memset(dst_typed, 0, static_cast(size)); + const uint32_t split = block_rows / 2u; + ConvertParams first {0, + split, + width, + height, + pitch, + block_columns, + blocks_per_row, + x_words.data(), + y_words.data(), + dst_typed, + src_typed}; + ConvertParams second {split, block_rows, width, height, + pitch, block_columns, blocks_per_row, x_words.data(), + y_words.data(), dst_typed, src_typed}; + if (split == 0) { + convert(&second); + return; + } + EXIT_IF(g_tiler == nullptr); + Common::LockGuard lock(g_tiler->m_mutex); + g_tiler->m_job1.Execute([convert, &second] { convert(&second); }); + convert(&first); + g_tiler->m_job1.Wait(); +} + +template +static void TileConvertDepth(void* dst, const void* src, uint32_t format, uint32_t width, + uint32_t height, uint32_t pitch, uint64_t size) { + KYTY_PROFILER_FUNCTION(); + const uint32_t bytes_per_element = Prospero::NumBytesPerElement(format); + const uint32_t block_width = bytes_per_element == 2 ? 256u : 128u; + if (dst == nullptr || src == nullptr || width == 0 || height == 0 || pitch < width || + (bytes_per_element != 2 && bytes_per_element != 4) || pitch % block_width != 0 || + size == 0 || size % 65536u != 0) { + EXIT("unsupported depth conversion, dst=%p src=%p format=%u " + "extent=%ux%u pitch=%u size=0x%016" PRIx64 "\n", + dst, src, format, width, height, pitch, size); + } + const uint64_t block_rows = (static_cast(height) + 127u) / 128u; + const uint64_t blocks_per_row = pitch / block_width; + if (blocks_per_row > UINT64_MAX / block_rows || + blocks_per_row * block_rows > UINT64_MAX / 65536u || + size != blocks_per_row * block_rows * 65536u) { + EXIT("depth storage disagrees with 64 KiB block layout, " + "extent=%ux%u pitch=%u bpe=%u size=0x%016" PRIx64 "\n", + width, height, pitch, bytes_per_element, size); + } + if (bytes_per_element == 2) { + static constexpr Depth64KB16Tables tables; + TileConvertDepthTyped(dst, src, width, height, pitch, size, + tables.x_words, tables.y_words); + } else { + static constexpr Depth64KB32Tables tables; + TileConvertDepthTyped(dst, src, width, height, pitch, size, + tables.x_words, tables.y_words); + } +} + +void TileConvertTiledToLinearDepth(void* dst, const void* src, uint32_t format, uint32_t width, + uint32_t height, uint32_t pitch, uint64_t size) { + TileConvertDepth(dst, src, format, width, height, pitch, size); +} + +void TileConvertLinearToTiledDepth(void* dst, const void* src, uint32_t format, uint32_t width, + uint32_t height, uint32_t pitch, uint64_t size) { + TileConvertDepth(dst, src, format, width, height, pitch, size); +} + +void TileConvertTiledToLinearStandard4KB(void* dst, const void* src, uint32_t format, + uint32_t width, uint32_t height, uint32_t pitch, + uint64_t dst_size, uint64_t src_size, uint32_t src_x, + uint32_t src_y) { + KYTY_PROFILER_FUNCTION(); + + EXIT_IF(dst == nullptr); + EXIT_IF(src == nullptr); + EXIT_NOT_IMPLEMENTED(pitch == 0); + EXIT_NOT_IMPLEMENTED(width > pitch); + + uint32_t bytes_per_element = 0; + uint32_t texels_per_element_wide = 0; + uint32_t texels_per_element_tall = 0; + uint32_t block_width_log2 = 0; + uint32_t block_height_log2 = 0; + + EXIT_NOT_IMPLEMENTED(!Gen5Standard4KBLayout(format, &bytes_per_element, + &texels_per_element_wide, &texels_per_element_tall, + &block_width_log2, &block_height_log2)); + + const uint32_t row_elements = + std::max((pitch + texels_per_element_wide - 1u) / texels_per_element_wide, 1u); + const uint32_t width_elements = + std::max((width + texels_per_element_wide - 1u) / texels_per_element_wide, 1u); + const uint32_t height_elements = + std::max((height + texels_per_element_tall - 1u) / texels_per_element_tall, 1u); + const uint32_t block_width = 1u << block_width_log2; + const uint32_t block_height = 1u << block_height_log2; + const uint32_t padded_width = (row_elements + block_width - 1u) & ~(block_width - 1u); + const uint32_t padded_height = (height_elements + block_height - 1u) & ~(block_height - 1u); + + EXIT_IF((padded_width & (block_width - 1u)) != 0); + EXIT_IF((padded_height & (block_height - 1u)) != 0); + EXIT_NOT_IMPLEMENTED(src_x >= block_width || src_y >= block_height); + EXIT_NOT_IMPLEMENTED(src_x != 0 && src_x + width_elements > block_width); + EXIT_NOT_IMPLEMENTED(src_y != 0 && src_y + height_elements > block_height); + + auto* dst8 = static_cast(dst); + + std::memset(dst8, 0, static_cast(dst_size)); + + switch (bytes_per_element) { + case 1: + DetileStandard4KBTyped(static_cast(dst), static_cast(src), + row_elements, width_elements, height_elements, padded_width, + block_width_log2, block_height_log2, dst_size, src_size, src_x, + src_y); + break; + case 2: + DetileStandard4KBTyped(static_cast(dst), static_cast(src), + row_elements, width_elements, height_elements, padded_width, + block_width_log2, block_height_log2, dst_size, src_size, src_x, + src_y); + break; + case 4: + DetileStandard4KBTyped(static_cast(dst), static_cast(src), + row_elements, width_elements, height_elements, padded_width, + block_width_log2, block_height_log2, dst_size, src_size, src_x, + src_y); + break; + case 8: + DetileStandard4KBTyped(static_cast(dst), static_cast(src), + row_elements, width_elements, height_elements, padded_width, + block_width_log2, block_height_log2, dst_size, src_size, src_x, + src_y); + break; + case 16: + DetileStandard4KBTyped(static_cast(dst), static_cast(src), + row_elements, width_elements, height_elements, padded_width, + block_width_log2, block_height_log2, dst_size, src_size, src_x, + src_y); + break; + default: EXIT("unsupported Standard4KB element size: %u\n", bytes_per_element); + } +} + +void TileConvertTiledToLinearStandard4KB3D(void* dst, const void* src, uint32_t format, + uint32_t width, uint32_t height, uint32_t depth, + uint32_t pitch, uint64_t dst_slice_stride, + uint64_t dst_size, uint64_t src_size, bool clear_dst) { + KYTY_PROFILER_FUNCTION(); + + EXIT_IF(dst == nullptr); + EXIT_IF(src == nullptr); + EXIT_NOT_IMPLEMENTED(pitch == 0); + EXIT_NOT_IMPLEMENTED(width > pitch); + EXIT_NOT_IMPLEMENTED(depth == 0); + + uint32_t bytes_per_element = 0; + uint32_t texels_per_element_wide = 0; + uint32_t texels_per_element_tall = 0; + uint32_t block_width_log2 = 0; + uint32_t block_height_log2 = 0; + uint32_t block_depth_log2 = 0; + + EXIT_NOT_IMPLEMENTED(!TileGetStandard4KBVolumeLayout( + format, &bytes_per_element, &texels_per_element_wide, &texels_per_element_tall, + &block_width_log2, &block_height_log2, &block_depth_log2)); + EXIT_NOT_IMPLEMENTED(dst_slice_stride == 0); + EXIT_NOT_IMPLEMENTED((dst_slice_stride % bytes_per_element) != 0); + + const uint32_t row_elements = + std::max((pitch + texels_per_element_wide - 1u) / texels_per_element_wide, 1u); + const uint32_t width_elements = + std::max((width + texels_per_element_wide - 1u) / texels_per_element_wide, 1u); + const uint32_t height_elements = + std::max((height + texels_per_element_tall - 1u) / texels_per_element_tall, 1u); + const uint32_t block_width = 1u << block_width_log2; + const uint32_t block_height = 1u << block_height_log2; + const uint32_t padded_width = AlignUp(row_elements, block_width); + const uint32_t padded_height = AlignUp(height_elements, block_height); + + src_size = (src_size != 0 ? src_size : dst_size); + if (clear_dst) { + std::memset(dst, 0, static_cast(dst_size)); + } + + switch (bytes_per_element) { + case 1: + DetileStandard4KBVolumeTyped(static_cast(dst), + static_cast(src), row_elements, + width_elements, height_elements, depth, padded_width, + padded_height, block_width_log2, block_height_log2, + block_depth_log2, dst_slice_stride, dst_size, src_size); + break; + case 2: + DetileStandard4KBVolumeTyped(static_cast(dst), + static_cast(src), row_elements, + width_elements, height_elements, depth, padded_width, + padded_height, block_width_log2, block_height_log2, + block_depth_log2, dst_slice_stride, dst_size, src_size); + break; + case 4: + DetileStandard4KBVolumeTyped(static_cast(dst), + static_cast(src), row_elements, + width_elements, height_elements, depth, padded_width, + padded_height, block_width_log2, block_height_log2, + block_depth_log2, dst_slice_stride, dst_size, src_size); + break; + case 8: + DetileStandard4KBVolumeTyped(static_cast(dst), + static_cast(src), row_elements, + width_elements, height_elements, depth, padded_width, + padded_height, block_width_log2, block_height_log2, + block_depth_log2, dst_slice_stride, dst_size, src_size); + break; + case 16: + DetileStandard4KBVolumeTyped(static_cast(dst), + static_cast(src), row_elements, + width_elements, height_elements, depth, padded_width, + padded_height, block_width_log2, block_height_log2, + block_depth_log2, dst_slice_stride, dst_size, src_size); + break; + default: EXIT("unsupported Standard4KB volume element size: %u\n", bytes_per_element); + } +} + +void TileConvertTiledToLinear(void* dst, const void* src, TileMode mode, uint32_t width, + uint32_t height) { + KYTY_PROFILER_FUNCTION(); + + EXIT_NOT_IMPLEMENTED(mode != TileMode::VideoOutTiled); + + Tiler32 t; + t.Init(width, height); + + Detile32(t, width, height, width, static_cast(dst), static_cast(src)); +} + +void TileConvertTiledToLinearRenderTarget(void* dst, const void* src, uint32_t width, + uint32_t height, uint32_t pitch, + uint32_t bytes_per_element, uint64_t size, + uint64_t src_size, uint32_t src_x, uint32_t src_y) { + KYTY_PROFILER_FUNCTION(); + + EXIT_IF(dst == nullptr || src == nullptr); + EXIT_NOT_IMPLEMENTED(width > pitch); + EXIT_NOT_IMPLEMENTED(pitch == 0); + + src_size = (src_size != 0 ? src_size : size); + const bool tail_block = (src_size != 0 && (src_x != 0 || src_y != 0 || size < src_size)); + if (!tail_block && bytes_per_element <= 8) { + switch (bytes_per_element) { + case 1: + ConvertRenderTargetTyped(width, height, pitch, + static_cast(dst), + static_cast(src), size); + break; + case 2: + ConvertRenderTargetTyped(width, height, pitch, + static_cast(dst), + static_cast(src), size); + break; + case 4: + ConvertRenderTargetTyped(width, height, pitch, + static_cast(dst), + static_cast(src), size); + break; + case 8: + ConvertRenderTargetTyped(width, height, pitch, + static_cast(dst), + static_cast(src), size); + break; + default: EXIT("unsupported render-target element size: %u\n", bytes_per_element); + } + return; + } + + switch (bytes_per_element) { + case 1: + TileConvertTiledToLinearStandard64KB8Elements(dst, src, width, height, pitch, size, + src_size, src_x, src_y); + break; + case 2: + TileConvertTiledToLinearStandard64KB16(dst, src, width, height, pitch, size, src_size, + src_x, src_y); + break; + case 4: + TileConvertTiledToLinearStandard64KB32(dst, src, width, height, pitch, size, src_size, + src_x, src_y); + break; + case 8: + TileConvertTiledToLinearStandard64KB64Elements(dst, src, width, height, pitch, size, + src_size, src_x, src_y); + break; + case 16: + TileConvertTiledToLinearStandard64KB128Elements(dst, src, width, height, pitch, size, + src_size, src_x, src_y); + break; + default: EXIT("unsupported render-target element size: %u\n", bytes_per_element); + } +} + +void TileConvertLinearToTiledRenderTarget(void* dst, const void* src, uint32_t width, + uint32_t height, uint32_t pitch, + uint32_t bytes_per_element, uint64_t size) { + KYTY_PROFILER_FUNCTION(); + + if (dst == nullptr || src == nullptr || width == 0 || height == 0 || pitch == 0 || + width > pitch || bytes_per_element == 0 || size == 0 || size % bytes_per_element != 0) { + EXIT("invalid linear-to-tiled render-target conversion, dst=%p src=%p extent=%ux%u " + "pitch=%u bpe=%u size=0x%016" PRIx64 "\n", + dst, src, width, height, pitch, bytes_per_element, size); + } + const auto rows = static_cast(height - 1); + if (rows > (UINT64_MAX - width) / pitch || + (rows * pitch + width) > UINT64_MAX / bytes_per_element || + size < (rows * pitch + width) * bytes_per_element) { + EXIT("linear-to-tiled render-target storage is too small, extent=%ux%u pitch=%u " + "bpe=%u size=0x%016" PRIx64 "\n", + width, height, pitch, bytes_per_element, size); + } + + std::memset(dst, 0, size); + switch (bytes_per_element) { + case 1: + ConvertRenderTargetTyped(width, height, pitch, + static_cast(dst), + static_cast(src), size); + break; + case 2: + ConvertRenderTargetTyped(width, height, pitch, + static_cast(dst), + static_cast(src), size); + break; + case 4: + ConvertRenderTargetTyped(width, height, pitch, + static_cast(dst), + static_cast(src), size); + break; + case 8: + ConvertRenderTargetTyped(width, height, pitch, + static_cast(dst), + static_cast(src), size); + break; + default: EXIT("unsupported render-target element size: %u\n", bytes_per_element); + } +} + +bool TileGetDepthSize(uint32_t width, uint32_t height, uint32_t pitch, uint32_t z_format, + uint32_t stencil_format, bool htile, TileSizeAlign* stencil_size, + TileSizeAlign* htile_size, TileSizeAlign* depth_size) { + EXIT_IF(pitch != 0); + // Prospero derives uncompressed depth/stencil as independent 64 KiB block surfaces and HTile + // as 32 KiB metadata blocks covering 1024x512 pixels for a single-mip, single-slice target. + if (width > 0 && width <= 16384 && height > 0 && height <= 16384 && + (z_format == 1 || z_format == 3) && stencil_format <= 1) { + const uint32_t depth_bytes = z_format == 1 ? 2u : 4u; + const uint32_t depth_block_width = z_format == 1 ? 256u : 128u; + const uint64_t depth_bytes_total = + static_cast(AlignUp(width, depth_block_width)) * AlignUp(height, 128u) * + depth_bytes; + const uint64_t stencil_bytes_total = + stencil_format == 1 + ? static_cast(AlignUp(width, 256u)) * AlignUp(height, 256u) + : 0; + const uint64_t htile_bytes_total = + htile ? static_cast(AlignUp(width, 1024u) / 1024u) * + (AlignUp(height, 512u) / 512u) * 32768u + : 0; + if (depth_bytes_total <= UINT32_MAX && stencil_bytes_total <= UINT32_MAX && + htile_bytes_total <= UINT32_MAX) { + *depth_size = {static_cast(depth_bytes_total), 65536}; + *stencil_size = stencil_format == 1 + ? TileSizeAlign {static_cast(stencil_bytes_total), 65536} + : TileSizeAlign {}; + *htile_size = htile ? TileSizeAlign {static_cast(htile_bytes_total), 32768} + : TileSizeAlign {}; + return true; + } + } + *depth_size = TileSizeAlign(); + *htile_size = TileSizeAlign(); + *stencil_size = TileSizeAlign(); + return false; +} + +uint32_t TileGetRenderTargetPitch(uint32_t width, uint32_t bytes_per_element) { + uint32_t block_width = 0; + uint32_t block_height = 0; + if (width == 0 || !Gen5RenderTargetBlockSizeFromElementBytes(bytes_per_element, &block_width, + &block_height)) { + return 0; + } + const uint64_t pitch = (static_cast(width) + block_width - 1u) & + ~static_cast(block_width - 1u); + return pitch <= UINT32_MAX ? static_cast(pitch) : 0; +} + +bool TileGetRenderTargetSize(uint32_t width, uint32_t height, uint32_t pitch, + uint32_t bytes_per_element, TileSizeAlign* total_size) { + *total_size = {}; + uint32_t block_width = 0; + uint32_t block_height = 0; + if (height == 0 || pitch == 0 || + !Gen5RenderTargetBlockSizeFromElementBytes(bytes_per_element, &block_width, + &block_height) || + pitch != TileGetRenderTargetPitch(width, bytes_per_element)) { + return false; + } + const uint64_t padded_height = (static_cast(height) + block_height - 1u) & + ~static_cast(block_height - 1u); + const uint64_t size = static_cast(pitch) * padded_height * bytes_per_element; + if (size == 0 || size > UINT32_MAX) { + return false; + } + total_size->size = static_cast(size); + total_size->align = 65536; + return true; +} + +bool TileGetRenderTargetMipLayout(uint32_t width, uint32_t height, uint32_t pitch, + uint32_t bytes_per_element, uint32_t levels, + TileSizeAlign* total_size, TileSizeOffset* level_sizes, + TilePaddedSize* padded_size) { + *total_size = {}; + if (width == 0 || height == 0 || levels == 0 || levels > 16 || + pitch != TileGetRenderTargetPitch(width, bytes_per_element)) { + return false; + } + uint32_t max_levels = 1; + uint32_t max_dimension = std::max(width, height); + while (max_dimension > 1) { + max_dimension >>= 1u; + max_levels++; + } + if (levels > max_levels) { + return false; + } + uint32_t format = 0; + switch (bytes_per_element) { + case 1: format = Prospero::GpuEnumValue(Prospero::BufferFormat::k8UNorm); break; + case 2: format = Prospero::GpuEnumValue(Prospero::BufferFormat::k16UNorm); break; + case 4: format = Prospero::GpuEnumValue(Prospero::BufferFormat::k32Float); break; + case 8: format = Prospero::GpuEnumValue(Prospero::BufferFormat::k16_16_16_16Float); break; + case 16: format = Prospero::GpuEnumValue(Prospero::BufferFormat::k32_32_32_32Float); break; + default: return false; + } + TileGetTextureSize(format, width, height, pitch, levels, + Prospero::GpuEnumValue(Prospero::TileMode::kRenderTarget), total_size, level_sizes, + padded_size); + return total_size->size != 0 && total_size->align == 65536; +} + +void TileGetTextureSize(uint32_t format, uint32_t width, uint32_t height, uint32_t pitch, + uint32_t levels, uint32_t tile, TileSizeAlign* total_size, + TileSizeOffset* level_sizes, TilePaddedSize* padded_size) { + KYTY_PROFILER_FUNCTION(); + + EXIT_IF(levels == 0 || levels > 16); + + if (const uint32_t bytes_per_element = Prospero::NumBytesPerElement(format); + bytes_per_element != 0 && tile == 0) { + uint32_t mip_pitch[16] {}; + uint32_t mip_height[16] {}; + uint32_t mip_size[16] {}; + + for (uint32_t l = 0; l < levels; l++) { + mip_pitch[l] = CalcLinearAlignedLevelPitch(width, height, l, bytes_per_element, + &mip_height[l], &mip_size[l]); + } + + const uint32_t total = SetLinearMipChainLayout(levels, mip_pitch, mip_height, mip_size, + level_sizes, padded_size); + + if (total_size != nullptr) { + total_size->size = total; + total_size->align = 256; + } + + return; + } + + uint32_t std256_bytes_per_element = 0; + uint32_t std256_texels_per_element_wide = 0; + uint32_t std256_texels_per_element_tall = 0; + uint32_t std256_block_width_log2 = 0; + uint32_t std256_block_height_log2 = 0; + if (tile == 1 && + Gen5Standard256BLayout(format, &std256_bytes_per_element, &std256_texels_per_element_wide, + &std256_texels_per_element_tall, &std256_block_width_log2, + &std256_block_height_log2)) { + uint32_t offset = 0; + uint32_t mip_pitch = pitch; + uint32_t mip_height = height; + for (uint32_t l = 0; l < levels; l++) { + const uint32_t row_elements = std::max( + (mip_pitch + std256_texels_per_element_wide - 1u) / std256_texels_per_element_wide, + 1u); + const uint32_t height_elements = std::max( + (mip_height + std256_texels_per_element_tall - 1u) / std256_texels_per_element_tall, + 1u); + const uint32_t padded_width = AlignUp(row_elements, 1u << std256_block_width_log2); + const uint32_t padded_height = AlignUp(height_elements, 1u << std256_block_height_log2); + const uint32_t size = padded_width * padded_height * std256_bytes_per_element; + + if (level_sizes != nullptr) { + level_sizes[l].size = size; + level_sizes[l].offset = offset; + } + if (padded_size != nullptr) { + padded_size[l].width = padded_width * std256_texels_per_element_wide; + padded_size[l].height = padded_height * std256_texels_per_element_tall; + } + + offset += size; + mip_pitch = std::max(mip_pitch / 2u, 1u); + mip_height = std::max(mip_height / 2u, 1u); + } + + if (total_size != nullptr) { + total_size->size = AlignUp(offset, 256u); + total_size->align = 256; + } + + return; + } + + if (const uint32_t bytes_per_element = Prospero::NumBytesPerElement(format); + bytes_per_element != 0 && tile == 27 && levels == 1) { + uint32_t block_width = 0; + uint32_t block_height = 0; + EXIT_NOT_IMPLEMENTED(!Gen5RenderTargetBlockSizeFromElementBytes( + bytes_per_element, &block_width, &block_height)); + + const uint32_t padded_width = AlignUp(pitch, block_width); + const uint32_t padded_height = AlignUp(height, block_height); + const uint32_t size = padded_width * padded_height * bytes_per_element; + + if (total_size != nullptr) { + total_size->size = size; + total_size->align = 65536; + } + + if (level_sizes != nullptr) { + level_sizes[0].size = size; + level_sizes[0].offset = 0; + } + + if (padded_size != nullptr) { + padded_size[0].width = padded_width; + padded_size[0].height = padded_height; + } + + return; + } + + if (const uint32_t bytes_per_element = Prospero::NumBytesPerElement(format); + bytes_per_element != 0 && tile == 27 && levels > 1) { + uint32_t block_width = 0; + uint32_t block_height = 0; + EXIT_NOT_IMPLEMENTED(!Gen5RenderTargetBlockSizeFromElementBytes( + bytes_per_element, &block_width, &block_height)); + + const uint32_t bytes_log2 = IntLog2(bytes_per_element); + const uint32_t tail_width_limit = block_width >> 1u; + const uint32_t tail_height_limit = block_height; + constexpr uint32_t max_tail_levels = 12u; + + uint32_t first_tail_level = levels; + for (uint32_t l = 0; l < levels; l++) { + const uint32_t mip_pitch = std::max(ShiftCeil(pitch, l), 1u); + const uint32_t mip_height = std::max(ShiftCeil(height, l), 1u); + if (mip_pitch <= tail_width_limit && mip_height <= tail_height_limit && + levels - l <= max_tail_levels) { + first_tail_level = l; + break; + } + } + + uint32_t offset = (first_tail_level < levels ? 65536u : 0u); + + for (int32_t l = static_cast(first_tail_level) - 1; l >= 0; l--) { + const uint32_t level = static_cast(l); + const uint32_t mip_pitch = std::max(ShiftCeil(pitch, level), 1u); + const uint32_t mip_height = std::max(ShiftCeil(height, level), 1u); + const uint32_t padded_width = AlignUp(mip_pitch, block_width); + const uint32_t padded_height = AlignUp(mip_height, block_height); + const uint32_t size = padded_width * padded_height * bytes_per_element; + + if (level_sizes != nullptr) { + level_sizes[level].size = size; + level_sizes[level].offset = offset; + level_sizes[level].src_size = size; + level_sizes[level].src_offset = offset; + level_sizes[level].x = 0; + level_sizes[level].y = 0; + } + if (padded_size != nullptr) { + padded_size[level].width = padded_width; + padded_size[level].height = padded_height; + } + + offset += size; + } + + uint32_t tail_linear_offset = 0; + for (uint32_t l = first_tail_level; l < levels; l++) { + const uint32_t mip_pitch = std::max(ShiftCeil(pitch, l), 1u); + const uint32_t mip_height = std::max(ShiftCeil(height, l), 1u); + const uint32_t linear_size = mip_pitch * mip_height * bytes_per_element; + const auto& tail_location = + GEN5_MIP_TAIL_LOCATIONS_THIN_64KB[bytes_log2][l - first_tail_level]; + if (level_sizes != nullptr) { + level_sizes[l].size = linear_size; + level_sizes[l].offset = tail_linear_offset; + level_sizes[l].src_size = 65536u; + level_sizes[l].src_offset = 0; + level_sizes[l].x = tail_location.x; + level_sizes[l].y = tail_location.y; + } + if (padded_size != nullptr) { + padded_size[l].width = block_width; + padded_size[l].height = block_height; + } + tail_linear_offset += AlignUp(linear_size, bytes_per_element); + } + EXIT_NOT_IMPLEMENTED(first_tail_level < levels && tail_linear_offset > 65536u); + + if (total_size != nullptr) { + total_size->size = AlignUp(offset, 65536u); + total_size->align = 65536; + } + + return; + } + + if (auto bytes_per_block = Prospero::BlockCompressedBytesPerBlock(format); + bytes_per_block != 0 && tile == 0) { + uint32_t mip_pitch[16] {}; + uint32_t mip_height[16] {}; + uint32_t mip_size[16] {}; + + const uint32_t blocks_w0 = std::max((width + 3u) / 4u, 1u); + const uint32_t blocks_h0 = std::max((height + 3u) / 4u, 1u); + for (uint32_t l = 0; l < levels; l++) { + uint32_t padded_blocks_h = 0; + const uint32_t aligned_blocks_w = CalcLinearAlignedLevelPitch( + blocks_w0, blocks_h0, l, bytes_per_block, &padded_blocks_h, &mip_size[l]); + + mip_pitch[l] = std::max(aligned_blocks_w * 4u, 32u); + mip_height[l] = std::max(padded_blocks_h * 4u, 32u); + } + + const uint32_t total = SetLinearMipChainLayout(levels, mip_pitch, mip_height, mip_size, + level_sizes, padded_size); + + if (total_size != nullptr) { + total_size->size = total; + total_size->align = 256; + } + + return; + } + uint32_t bytes_per_element = 0; + uint32_t texels_per_element_wide = 0; + uint32_t texels_per_element_tall = 0; + uint32_t block_width_log2 = 0; + uint32_t block_height_log2 = 0; + if (tile == 5 && levels == 1 && + Gen5Standard4KBLayout(format, &bytes_per_element, &texels_per_element_wide, + &texels_per_element_tall, &block_width_log2, &block_height_log2)) { + const uint32_t row_elements = + std::max((pitch + texels_per_element_wide - 1u) / texels_per_element_wide, 1u); + const uint32_t height_elements = + std::max((height + texels_per_element_tall - 1u) / texels_per_element_tall, 1u); + const uint32_t block_width = 1u << block_width_log2; + const uint32_t block_height = 1u << block_height_log2; + const uint32_t padded_width = (row_elements + block_width - 1u) & ~(block_width - 1u); + const uint32_t padded_height = (height_elements + block_height - 1u) & ~(block_height - 1u); + const uint32_t size = padded_width * padded_height * bytes_per_element; + + if (total_size != nullptr) { + total_size->size = size; + total_size->align = 4096; + } + + if (level_sizes != nullptr) { + level_sizes[0].size = size; + level_sizes[0].offset = 0; + } + + if (padded_size != nullptr) { + padded_size[0].width = padded_width * texels_per_element_wide; + padded_size[0].height = padded_height * texels_per_element_tall; + } + + return; + } + + if (tile == 5 && levels > 1 && + Gen5Standard4KBLayout(format, &bytes_per_element, &texels_per_element_wide, + &texels_per_element_tall, &block_width_log2, &block_height_log2)) { + const uint32_t row_elements0 = + std::max((pitch + texels_per_element_wide - 1u) / texels_per_element_wide, 1u); + const uint32_t height_elements0 = + std::max((height + texels_per_element_tall - 1u) / texels_per_element_tall, 1u); + const uint32_t block_width = 1u << block_width_log2; + const uint32_t block_height = 1u << block_height_log2; + const uint32_t bytes_log2 = IntLog2(bytes_per_element); + const uint32_t tail_width_limit = block_width >> 1u; + const uint32_t tail_height_limit = block_height; + + uint32_t first_tail_level = levels; + for (uint32_t l = 0; l < levels; l++) { + const uint32_t row_elements = std::max(ShiftCeil(row_elements0, l), 1u); + const uint32_t height_elements = std::max(ShiftCeil(height_elements0, l), 1u); + if (row_elements <= tail_width_limit && height_elements <= tail_height_limit && + levels - l <= 8) { + first_tail_level = l; + break; + } + } + + uint32_t offset = (first_tail_level < levels ? 4096u : 0u); + + for (int32_t l = static_cast(first_tail_level) - 1; l >= 0; l--) { + const uint32_t level = static_cast(l); + const uint32_t row_elements = std::max(ShiftCeil(row_elements0, level), 1u); + const uint32_t height_elements = std::max(ShiftCeil(height_elements0, level), 1u); + const uint32_t padded_width = AlignUp(row_elements, block_width); + const uint32_t padded_height = AlignUp(height_elements, block_height); + const uint32_t size = padded_width * padded_height * bytes_per_element; + + if (level_sizes != nullptr) { + level_sizes[level].size = size; + level_sizes[level].offset = offset; + level_sizes[level].src_size = size; + level_sizes[level].src_offset = offset; + level_sizes[level].x = 0; + level_sizes[level].y = 0; + } + if (padded_size != nullptr) { + padded_size[level].width = padded_width * texels_per_element_wide; + padded_size[level].height = padded_height * texels_per_element_tall; + } + + offset += size; + } + + uint32_t tail_linear_offset = 0; + for (uint32_t l = first_tail_level; l < levels; l++) { + const uint32_t row_elements = std::max(ShiftCeil(row_elements0, l), 1u); + const uint32_t height_elements = std::max(ShiftCeil(height_elements0, l), 1u); + const uint32_t linear_size = row_elements * height_elements * bytes_per_element; + const auto& tail_location = + GEN5_MIP_TAIL_LOCATIONS_THIN_4KB[bytes_log2][l - first_tail_level]; + if (level_sizes != nullptr) { + level_sizes[l].size = linear_size; + level_sizes[l].offset = tail_linear_offset; + level_sizes[l].src_size = 4096u; + level_sizes[l].src_offset = 0; + level_sizes[l].x = tail_location.x; + level_sizes[l].y = tail_location.y; + } + if (padded_size != nullptr) { + padded_size[l].width = block_width * texels_per_element_wide; + padded_size[l].height = block_height * texels_per_element_tall; + } + tail_linear_offset += AlignUp(linear_size, bytes_per_element); + } + EXIT_NOT_IMPLEMENTED(first_tail_level < levels && tail_linear_offset > 4096u); + + if (total_size != nullptr) { + total_size->size = AlignUp(offset, 4096u); + total_size->align = 4096; + } + + return; + } + + if (tile == 24 && levels == 1) { + const uint32_t bytes_per_element = Prospero::NumBytesPerElement(format); + if (bytes_per_element != 0) { + uint32_t block_width = 0; + switch (bytes_per_element) { + case 1: block_width = 512; break; + case 2: block_width = 256; break; + case 4: block_width = 128; break; + default: break; + } + + if (block_width != 0) { + const uint32_t padded_width = (pitch + block_width - 1u) & ~(block_width - 1u); + const uint32_t padded_height = (height + 127u) & ~127u; + const uint32_t size = padded_width * padded_height * bytes_per_element; + + if (total_size != nullptr) { + total_size->size = size; + total_size->align = 65536; + } + + if (level_sizes != nullptr) { + level_sizes[0].size = size; + level_sizes[0].offset = 0; + } + + if (padded_size != nullptr) { + padded_size[0].width = padded_width; + padded_size[0].height = padded_height; + } + + return; + } + } + } + + if (tile == 24 && levels > 1) { + const uint32_t bytes_per_element = Prospero::NumBytesPerElement(format); + if (bytes_per_element != 0) { + uint32_t block_width = 0; + switch (bytes_per_element) { + case 1: block_width = 512; break; + case 2: block_width = 256; break; + case 4: block_width = 128; break; + default: break; + } + + if (block_width != 0) { + static bool logged = false; + if (!logged) { + LOGF("\t temporary: sizing PS5 depth tiled texture with mips, format = %u, " + "levels = %u\n", + format, levels); + logged = true; + } + + uint32_t offset = 0; + uint32_t mip_pitch = pitch; + uint32_t mip_height = height; + for (uint32_t l = 0; l < levels; l++) { + offset = AlignUp(offset, 65536u); + + const uint32_t padded_width = AlignUp(mip_pitch, block_width); + const uint32_t padded_height = AlignUp(mip_height, 128u); + const uint32_t size = padded_width * padded_height * bytes_per_element; + + if (level_sizes != nullptr) { + level_sizes[l].size = size; + level_sizes[l].offset = offset; + } + if (padded_size != nullptr) { + padded_size[l].width = padded_width; + padded_size[l].height = padded_height; + } + + offset += size; + mip_pitch = std::max(mip_pitch / 2u, 1u); + mip_height = std::max(mip_height / 2u, 1u); + } + + if (total_size != nullptr) { + total_size->size = AlignUp(offset, 65536u); + total_size->align = 65536; + } + + return; + } + } + } + + uint32_t std64_bytes_per_element = 0; + uint32_t std64_texels_per_element_wide = 0; + uint32_t std64_texels_per_element_tall = 0; + uint32_t std64_block_width_log2 = 0; + uint32_t std64_block_height_log2 = 0; + if (tile == 9 && levels > 1 && + Gen5Standard64KBLayout(format, &std64_bytes_per_element, &std64_texels_per_element_wide, + &std64_texels_per_element_tall, &std64_block_width_log2, + &std64_block_height_log2)) { + const uint32_t row_elements0 = std::max( + (pitch + std64_texels_per_element_wide - 1u) / std64_texels_per_element_wide, 1u); + const uint32_t height_elements0 = std::max( + (height + std64_texels_per_element_tall - 1u) / std64_texels_per_element_tall, 1u); + const uint32_t block_width = 1u << std64_block_width_log2; + const uint32_t block_height = 1u << std64_block_height_log2; + const uint32_t bytes_log2 = IntLog2(std64_bytes_per_element); + const uint32_t tail_width_limit = block_width >> 1u; + const uint32_t tail_height_limit = block_height; + constexpr uint32_t max_tail_levels = 12u; + + uint32_t first_tail_level = levels; + for (uint32_t l = 0; l < levels; l++) { + const uint32_t row_elements = std::max(ShiftCeil(row_elements0, l), 1u); + const uint32_t height_elements = std::max(ShiftCeil(height_elements0, l), 1u); + if (row_elements <= tail_width_limit && height_elements <= tail_height_limit && + levels - l <= max_tail_levels) { + first_tail_level = l; + break; + } + } + + uint32_t offset = (first_tail_level < levels ? 65536u : 0u); + + for (int32_t l = static_cast(first_tail_level) - 1; l >= 0; l--) { + const uint32_t level = static_cast(l); + const uint32_t row_elements = std::max(ShiftCeil(row_elements0, level), 1u); + const uint32_t height_elements = std::max(ShiftCeil(height_elements0, level), 1u); + const uint32_t padded_width = AlignUp(row_elements, block_width); + const uint32_t padded_height = AlignUp(height_elements, block_height); + const uint32_t size = padded_width * padded_height * std64_bytes_per_element; + + if (level_sizes != nullptr) { + level_sizes[level].size = size; + level_sizes[level].offset = offset; + level_sizes[level].src_size = size; + level_sizes[level].src_offset = offset; + level_sizes[level].x = 0; + level_sizes[level].y = 0; + } + if (padded_size != nullptr) { + padded_size[level].width = padded_width * std64_texels_per_element_wide; + padded_size[level].height = padded_height * std64_texels_per_element_tall; + } + + offset += size; + } + + uint32_t tail_linear_offset = 0; + for (uint32_t l = first_tail_level; l < levels; l++) { + const uint32_t row_elements = std::max(ShiftCeil(row_elements0, l), 1u); + const uint32_t height_elements = std::max(ShiftCeil(height_elements0, l), 1u); + const uint32_t linear_size = row_elements * height_elements * std64_bytes_per_element; + const auto& tail_location = + GEN5_MIP_TAIL_LOCATIONS_THIN_64KB[bytes_log2][l - first_tail_level]; + if (level_sizes != nullptr) { + level_sizes[l].size = linear_size; + level_sizes[l].offset = tail_linear_offset; + level_sizes[l].src_size = 65536u; + level_sizes[l].src_offset = 0; + level_sizes[l].x = tail_location.x; + level_sizes[l].y = tail_location.y; + } + if (padded_size != nullptr) { + padded_size[l].width = block_width * std64_texels_per_element_wide; + padded_size[l].height = block_height * std64_texels_per_element_tall; + } + tail_linear_offset += AlignUp(linear_size, std64_bytes_per_element); + } + EXIT_NOT_IMPLEMENTED(first_tail_level < levels && tail_linear_offset > 65536u); + + if (total_size != nullptr) { + total_size->size = AlignUp(offset, 65536u); + total_size->align = 65536; + } + + return; + } + if (tile == 9 && levels == 1 && + Gen5Standard64KBLayout(format, &std64_bytes_per_element, &std64_texels_per_element_wide, + &std64_texels_per_element_tall, &std64_block_width_log2, + &std64_block_height_log2)) { + const uint32_t row_elements = std::max( + (pitch + std64_texels_per_element_wide - 1u) / std64_texels_per_element_wide, 1u); + const uint32_t height_elements = std::max( + (height + std64_texels_per_element_tall - 1u) / std64_texels_per_element_tall, 1u); + const uint32_t block_width = 1u << std64_block_width_log2; + const uint32_t block_height = 1u << std64_block_height_log2; + const uint32_t padded_width = AlignUp(row_elements, block_width); + const uint32_t padded_height = AlignUp(height_elements, block_height); + const uint32_t size = padded_width * padded_height * std64_bytes_per_element; + + if (total_size != nullptr) { + total_size->size = size; + total_size->align = 65536; + } + + if (level_sizes != nullptr) { + level_sizes[0].size = size; + level_sizes[0].offset = 0; + } + + if (padded_size != nullptr) { + padded_size[0].width = padded_width * std64_texels_per_element_wide; + padded_size[0].height = padded_height * std64_texels_per_element_tall; + } + + return; + } + + if (total_size != nullptr && total_size->size == 0) { + std::vector list; + list.push_back(fmt::format("format = {}", format)); + list.push_back(fmt::format("width = {}", width)); + list.push_back(fmt::format("height = {}", height)); + list.push_back(fmt::format("pitch = {}", pitch)); + list.push_back(fmt::format("levels = {}", levels)); + list.push_back(fmt::format("tile = {}", tile)); + EXIT("unknown format:\n%s\n", Common::Concat(list, '\n').c_str()); + } +} + +void TileGetTextureTotalSize(uint32_t format, uint32_t width, uint32_t height, uint32_t depth, + uint32_t pitch, uint32_t levels, uint32_t tile, bool volume_texture, + TileSizeAlign* total_size) { + EXIT_NOT_IMPLEMENTED(depth == 0); + + TileSizeAlign slice_size {}; + TileGetTextureSize(format, width, height, pitch, levels, tile, &slice_size, nullptr, nullptr); + + *total_size = slice_size; + uint64_t total = static_cast(slice_size.size) * depth; + + uint32_t bytes_per_element = 0; + uint32_t texels_per_element_wide = 0; + uint32_t texels_per_element_tall = 0; + uint32_t block_width_log2 = 0; + uint32_t block_height_log2 = 0; + uint32_t block_depth_log2 = 0; + + if (volume_texture && depth > 1 && tile == 5 && + TileGetStandard4KBVolumeLayout(format, &bytes_per_element, &texels_per_element_wide, + &texels_per_element_tall, &block_width_log2, + &block_height_log2, &block_depth_log2)) { + const uint32_t row_elements0 = + std::max((pitch + texels_per_element_wide - 1u) / texels_per_element_wide, 1u); + const uint32_t height_elements0 = + std::max((height + texels_per_element_tall - 1u) / texels_per_element_tall, 1u); + const uint32_t block_width = 1u << block_width_log2; + const uint32_t block_height = 1u << block_height_log2; + const uint32_t block_depth = 1u << block_depth_log2; + uint64_t block_slice_size = 0; + + if (levels > 1) { + const uint32_t tail_width_limit = block_width; + const uint32_t tail_height_limit = block_height >> 1u; + constexpr uint32_t max_tail_levels = 5u; + + for (uint32_t l = 0; l < levels; l++) { + const uint32_t row_elements = std::max(ShiftCeil(row_elements0, l), 1u); + const uint32_t height_elements = std::max(ShiftCeil(height_elements0, l), 1u); + if (row_elements <= tail_width_limit && height_elements <= tail_height_limit && + levels - l <= max_tail_levels) { + block_slice_size += 4096u; + break; + } + + block_slice_size += static_cast(block_depth) * + AlignUp(row_elements, block_width) * + AlignUp(height_elements, block_height) * bytes_per_element; + } + } else { + block_slice_size = static_cast(block_depth) * + AlignUp(row_elements0, block_width) * + AlignUp(height_elements0, block_height) * bytes_per_element; + } + + total = block_slice_size * ShiftCeil(depth, block_depth_log2); + total_size->align = 4096; + } + + EXIT_NOT_IMPLEMENTED(total > 0xffffffffull); + total_size->size = static_cast(total); +} + +uint32_t TileGetTexturePitch(uint32_t format, uint32_t width, uint32_t levels, uint32_t tile) { + uint32_t pitch = width; + + if (tile == 27) { + if (const uint32_t bytes_per_element = Prospero::NumBytesPerElement(format); bytes_per_element != 0) { + uint32_t block_width = 0; + uint32_t block_height = 0; + EXIT_NOT_IMPLEMENTED(!Gen5RenderTargetBlockSizeFromElementBytes( + bytes_per_element, &block_width, &block_height)); + pitch = AlignUp(pitch, block_width); + } + } + + if (tile == 0) { + if (const auto bytes_per_element = Prospero::NumBytesPerElement(format); bytes_per_element != 0) { + pitch = AlignUp(pitch * bytes_per_element, 256u) / bytes_per_element; + } + } + uint32_t bytes_per_element = 0; + uint32_t texels_per_element_wide = 0; + uint32_t texels_per_element_tall = 0; + uint32_t block_width_log2 = 0; + uint32_t block_height_log2 = 0; + if (tile == 9 && + Gen5Standard64KBLayout(format, &bytes_per_element, &texels_per_element_wide, + &texels_per_element_tall, &block_width_log2, &block_height_log2)) { + pitch = AlignUp(pitch, (1u << block_width_log2) * texels_per_element_wide); + } + if (tile == 24) { + switch (Prospero::NumBytesPerElement(format)) { + case 1: pitch = (pitch + 511u) & ~511u; break; + case 2: pitch = (pitch + 255u) & ~255u; break; + case 4: pitch = (pitch + 127u) & ~127u; break; + default: break; + } + } + + return pitch; +} + +} // namespace Libs::Graphics diff --git a/src/graphics/guest_gpu/tile.h b/src/graphics/guest_gpu/tile.h new file mode 100644 index 000000000..e4079359e --- /dev/null +++ b/src/graphics/guest_gpu/tile.h @@ -0,0 +1,104 @@ +#ifndef EMULATOR_INCLUDE_EMULATOR_GRAPHICS_TILE_H_ +#define EMULATOR_INCLUDE_EMULATOR_GRAPHICS_TILE_H_ + +#include "common/abi.h" +#include "common/common.h" + +namespace Libs::Graphics { + +enum class TileMode { + VideoOutLinear, + VideoOutTiled, + TextureLinear, + TextureTiled, + // RenderTextureLinear, + // RenderTextureTiled, +}; + +struct TileSizeAlign { + uint32_t size = 0; + uint32_t align = 0; +}; + +struct TileSizeOffset { + uint32_t size = 0; + uint32_t offset = 0; + uint32_t src_size = 0; + uint32_t src_offset = 0; + uint32_t x = 0; + uint32_t y = 0; +}; + +struct TilePaddedSize { + uint32_t width = 0; + uint32_t height = 0; +}; + +void TileInit(); +void TileConvertTiledToLinear(void* dst, const void* src, TileMode mode, uint32_t width, + uint32_t height); +void TileConvertTiledToLinearRenderTarget(void* dst, const void* src, uint32_t width, + uint32_t height, uint32_t pitch, + uint32_t bytes_per_element, uint64_t size, + uint64_t src_size = 0, uint32_t src_x = 0, + uint32_t src_y = 0); +void TileConvertLinearToTiledRenderTarget(void* dst, const void* src, uint32_t width, + uint32_t height, uint32_t pitch, + uint32_t bytes_per_element, uint64_t size); +void TileConvertTiledToLinearStandard64KB(void* dst, const void* src, uint32_t format, + uint32_t width, uint32_t height, uint32_t pitch, + uint64_t size, uint64_t src_size = 0, uint32_t src_x = 0, + uint32_t src_y = 0); +void TileConvertTiledToLinearStandard64KB32(void* dst, const void* src, uint32_t width, + uint32_t height, uint32_t pitch, uint64_t size, + uint64_t src_size = 0, uint32_t src_x = 0, + uint32_t src_y = 0); +void TileConvertTiledToLinearStandard64KB16(void* dst, const void* src, uint32_t width, + uint32_t height, uint32_t pitch, uint64_t size, + uint64_t src_size = 0, uint32_t src_x = 0, + uint32_t src_y = 0); +void TileConvertTiledToLinearDepth(void* dst, const void* src, uint32_t format, uint32_t width, + uint32_t height, uint32_t pitch, uint64_t size); +void TileConvertLinearToTiledDepth(void* dst, const void* src, uint32_t format, uint32_t width, + uint32_t height, uint32_t pitch, uint64_t size); +void TileConvertTiledToLinearStandard4KB(void* dst, const void* src, uint32_t format, + uint32_t width, uint32_t height, uint32_t pitch, + uint64_t dst_size, uint64_t src_size, uint32_t src_x = 0, + uint32_t src_y = 0); +void TileConvertTiledToLinearStandard256B(void* dst, const void* src, uint32_t format, + uint32_t width, uint32_t height, uint32_t pitch, + uint64_t dst_size, uint64_t src_size); +bool TileIsStandard256BTextureSupported(uint32_t format); +bool TileIsStandard4KBTextureSupported(uint32_t format); +bool TileIsStandard64KBTextureSupported(uint32_t format); +bool TileGetStandard4KBVolumeLayout(uint32_t format, uint32_t* bytes_per_element, + uint32_t* texels_per_element_wide, + uint32_t* texels_per_element_tall, uint32_t* block_width_log2, + uint32_t* block_height_log2, uint32_t* block_depth_log2); + +bool TileGetDepthSize(uint32_t width, uint32_t height, uint32_t pitch, uint32_t z_format, + uint32_t stencil_format, bool htile, TileSizeAlign* stencil_size, + TileSizeAlign* htile_size, TileSizeAlign* depth_size); +uint32_t TileGetRenderTargetPitch(uint32_t width, uint32_t bytes_per_element); +bool TileGetRenderTargetSize(uint32_t width, uint32_t height, uint32_t pitch, + uint32_t bytes_per_element, TileSizeAlign* total_size); +bool TileGetRenderTargetMipLayout(uint32_t width, uint32_t height, uint32_t pitch, + uint32_t bytes_per_element, uint32_t levels, + TileSizeAlign* total_size, TileSizeOffset* level_sizes, + TilePaddedSize* padded_size); +void TileGetTextureSize(uint32_t format, uint32_t width, uint32_t height, uint32_t pitch, + uint32_t levels, uint32_t tile, TileSizeAlign* total_size, + TileSizeOffset* level_sizes, TilePaddedSize* padded_size); +void TileGetTextureTotalSize(uint32_t format, uint32_t width, uint32_t height, uint32_t depth, + uint32_t pitch, uint32_t levels, uint32_t tile, bool volume_texture, + TileSizeAlign* total_size); +uint32_t TileGetTexturePitch(uint32_t format, uint32_t width, uint32_t levels, uint32_t tile); +void TileConvertTiledToLinearStandard4KB3D(void* dst, const void* src, uint32_t format, + uint32_t width, uint32_t height, uint32_t depth, + uint32_t pitch, uint64_t dst_slice_stride, + uint64_t dst_size, uint64_t src_size, + bool clear_dst = true); + +} // namespace Libs::Graphics + +#endif /* EMULATOR_INCLUDE_EMULATOR_GRAPHICS_TILE_H_ */ diff --git a/src/graphics/host_gpu/graphicContext.h b/src/graphics/host_gpu/graphicContext.h new file mode 100644 index 000000000..6b693d87b --- /dev/null +++ b/src/graphics/host_gpu/graphicContext.h @@ -0,0 +1,179 @@ +#ifndef EMULATOR_INCLUDE_EMULATOR_GRAPHICS_GRAPHICCONTEXT_H_ +#define EMULATOR_INCLUDE_EMULATOR_GRAPHICS_GRAPHICCONTEXT_H_ + +#include "common/abi.h" +#include "common/common.h" +#include "common/threads.h" +#include "graphics/host_gpu/vulkanInstance.h" + +#include +#include +#include +#include +#include // IWYU pragma: export + +namespace Libs::Graphics { + +class CommandBuffer; + +struct VulkanSwapchain { + ~VulkanSwapchain(); + + VkSwapchainKHR swapchain = nullptr; + VkFormat swapchain_format = VK_FORMAT_UNDEFINED; + VkExtent2D swapchain_extent = {}; + std::unique_ptr swapchain_images; + std::unique_ptr swapchain_image_views; + uint32_t swapchain_images_count = 0; + std::unique_ptr image_acquired_semaphores; + std::unique_ptr render_complete_semaphores; + uint32_t current_index = 0; + uint32_t present_frame = 0; + std::vector> present_command_buffers; +}; + +struct VulkanCommandPool { + Common::Mutex mutex; + VkCommandPool pool = nullptr; + std::unique_ptr buffers; + std::unique_ptr fences; + std::unique_ptr semaphores; + std::unique_ptr busy; + uint32_t buffers_count = 0; +}; + +struct GraphicContext: public VulkanInstance { + uint32_t screen_width = 0; + uint32_t screen_height = 0; +}; + +struct VulkanMemory { + VkMemoryRequirements requirements = {}; + VkMemoryPropertyFlags property = 0; + VkDeviceMemory memory = nullptr; + VmaAllocation allocation = nullptr; + VmaAllocationInfo allocation_info = {}; + VkDeviceSize offset = 0; + uint32_t type = 0; + uint64_t unique_id = 0; +}; + +enum class VulkanImageType { + Unknown, + VideoOut, + DepthStencil, + Texture, + StorageTexture, + RenderTexture +}; + +struct VulkanImage { + static constexpr int VIEW_MAX = 21; + static constexpr int VIEW_DEFAULT = 0; + static constexpr int VIEW_BGRA = 1; + static constexpr int VIEW_DEPTH_TEXTURE = 2; + static constexpr int VIEW_R001 = 3; + static constexpr int VIEW_RGB1 = 4; + static constexpr int VIEW_R000 = 5; + static constexpr int VIEW_RG01 = 6; + static constexpr int VIEW_000R = 7; + static constexpr int VIEW_DEFAULT_ARRAY = 8; + static constexpr int VIEW_BGRA_ARRAY = 9; + static constexpr int VIEW_DEPTH_TEXTURE_ARRAY = 10; + static constexpr int VIEW_R001_ARRAY = 11; + static constexpr int VIEW_RGB1_ARRAY = 12; + static constexpr int VIEW_R000_ARRAY = 13; + static constexpr int VIEW_RG01_ARRAY = 14; + static constexpr int VIEW_000R_ARRAY = 15; + static constexpr int VIEW_STENCIL_TEXTURE = 16; + static constexpr int VIEW_STENCIL_TEXTURE_ARRAY = 17; + static constexpr int VIEW_STORAGE = 18; + static constexpr int VIEW_STORAGE_ARRAY = 19; + static constexpr int VIEW_RGBA8_BGRA = 20; + + explicit VulkanImage(VulkanImageType type): type(type) {} + + VulkanImageType type = VulkanImageType::Unknown; + VkFormat format = VK_FORMAT_UNDEFINED; + VkExtent2D extent = {}; + uint32_t guest_pitch = 0; + uint32_t layers = 1; + uint32_t mip_levels = 1; + VkImage image = nullptr; + VkImageView image_view[VIEW_MAX] = {}; + VkImageLayout layout = VK_IMAGE_LAYOUT_UNDEFINED; + Graphics::VulkanMemory memory; +}; + +struct VideoOutVulkanImage: public VulkanImage { + VideoOutVulkanImage(): VulkanImage(VulkanImageType::VideoOut) {} +}; + +struct DepthStencilVulkanImage: public VulkanImage { + DepthStencilVulkanImage(): VulkanImage(VulkanImageType::DepthStencil) {} + bool compressed = false; +}; + +struct GpuTextureVulkanImage: public VulkanImage { + explicit GpuTextureVulkanImage(VulkanImageType type): VulkanImage(type) {} +}; + +struct TextureVulkanImage: public GpuTextureVulkanImage { + TextureVulkanImage(): GpuTextureVulkanImage(VulkanImageType::Texture) {} +}; + +struct StorageTextureVulkanImage: public GpuTextureVulkanImage { + struct SampledView { + VkFormat format = VK_FORMAT_UNDEFINED; + uint32_t swizzle = 0; + uint32_t type = 0; + uint32_t base_level = 0; + uint32_t level_count = 0; + VkImageView view = nullptr; + }; + StorageTextureVulkanImage(): GpuTextureVulkanImage(VulkanImageType::StorageTexture) {} + std::mutex sampled_view_mutex; + std::vector sampled_views; +}; + +struct RenderTextureVulkanImage: public VulkanImage { + struct AttachmentView { + VkFormat format = VK_FORMAT_UNDEFINED; + uint32_t level = 0; + VkImageView view = nullptr; + }; + struct SampledView { + VkFormat format = VK_FORMAT_UNDEFINED; + uint32_t base_level = 0; + uint32_t level_count = 0; + int variant = VIEW_DEFAULT; + VkImageView view = nullptr; + }; + struct StorageView { + VkFormat format = VK_FORMAT_UNDEFINED; + uint32_t base_level = 0; + uint32_t level_count = 0; + VkImageView view = nullptr; + }; + RenderTextureVulkanImage(): VulkanImage(VulkanImageType::RenderTexture) {} + VkImageView render_view[16] = {}; + std::mutex attachment_view_mutex; + std::vector attachment_views; + std::mutex sampled_view_mutex; + std::vector sampled_views; + std::mutex storage_view_mutex; + std::vector storage_views; +}; + +struct VulkanBuffer { + VkBuffer buffer = nullptr; + VulkanMemory memory; + VkBufferUsageFlags usage = 0; + uint64_t buffer_size = 0; +}; + +struct StorageVulkanBuffer: public VulkanBuffer {}; + +} // namespace Libs::Graphics + +#endif /* EMULATOR_INCLUDE_EMULATOR_GRAPHICS_GRAPHICCONTEXT_H_ */ diff --git a/src/graphics/host_gpu/hostMemory.cpp b/src/graphics/host_gpu/hostMemory.cpp new file mode 100644 index 000000000..c440728d3 --- /dev/null +++ b/src/graphics/host_gpu/hostMemory.cpp @@ -0,0 +1,141 @@ +#include "graphics/host_gpu/hostMemory.h" + +#include +#include +#include + +#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS +#ifndef NOMINMAX +#define NOMINMAX +#endif +#include +#undef min +#undef max +#endif + +namespace Libs::Graphics { +namespace { + +#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS +bool IsAccessible(DWORD protect, HostMemoryAccess access) { + constexpr DWORD blocked = PAGE_NOACCESS | PAGE_GUARD; + constexpr DWORD readable = PAGE_READONLY | PAGE_READWRITE | PAGE_WRITECOPY | PAGE_EXECUTE_READ | + PAGE_EXECUTE_READWRITE | PAGE_EXECUTE_WRITECOPY; + constexpr DWORD writable = + PAGE_READWRITE | PAGE_WRITECOPY | PAGE_EXECUTE_READWRITE | PAGE_EXECUTE_WRITECOPY; + return (protect & blocked) == 0 && + (protect & (access == HostMemoryAccess::Write ? writable : readable)) != 0; +} +#endif + +} // namespace + +bool HostMemoryQueryRange(uint64_t addr, uint64_t requested_size, HostMemoryAccess access, + uint64_t* accessible_size) { + if (accessible_size == nullptr) { + return false; + } + *accessible_size = 0; + if (addr == 0 || requested_size == 0) { + return false; + } + + const auto end = UINT64_MAX - addr < requested_size ? UINT64_MAX : addr + requested_size; + uint64_t current = addr; +#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS + while (current < end) { + MEMORY_BASIC_INFORMATION region {}; + if (::VirtualQuery(reinterpret_cast(static_cast(current)), ®ion, + sizeof(region)) == 0) { + break; + } + const auto begin = reinterpret_cast(region.BaseAddress); + auto finish = begin + region.RegionSize; + if (finish < begin) { + finish = UINT64_MAX; + } + if (finish <= current || (region.State & MEM_COMMIT) == 0 || + !IsAccessible(region.Protect, access)) { + break; + } + current = finish < end ? finish : end; + } +#elif KYTY_PLATFORM == KYTY_PLATFORM_LINUX + auto* maps = std::fopen("/proc/self/maps", "r"); + if (maps == nullptr) { + return false; + } + char line[1024] = {}; + while (current < end && std::fgets(line, sizeof(line), maps) != nullptr) { + uint64_t begin = 0; + uint64_t finish = 0; + char permissions[5] = {}; + if (std::sscanf(line, "%" SCNx64 "-%" SCNx64 " %4s", &begin, &finish, permissions) != 3 || + finish <= current) { + continue; + } + if (begin > current) { + break; + } + const bool allowed = + access == HostMemoryAccess::Write ? permissions[1] == 'w' : permissions[0] == 'r'; + if (!allowed) { + break; + } + current = finish < end ? finish : end; + } + std::fclose(maps); +#else + (void)access; +#endif + + *accessible_size = current - addr; + return *accessible_size != 0; +} + +bool HostMemoryQueryEnd(uint64_t addr, HostMemoryAccess access, uint64_t* end) { + if (addr == 0 || end == nullptr) { + return false; + } + uint64_t accessible_size = 0; + if (!HostMemoryQueryRange(addr, UINT64_MAX - addr, access, &accessible_size) || + accessible_size == 0 || UINT64_MAX - addr < accessible_size) { + return false; + } + *end = addr + accessible_size; + return true; +} + +bool HostMemoryQueryReadable(uint64_t addr, uint64_t requested_size, uint64_t* readable_size) { + return HostMemoryQueryRange(addr, requested_size, HostMemoryAccess::Read, readable_size); +} + +bool HostMemoryQueryWritable(uint64_t addr, uint64_t requested_size, uint64_t* writable_size) { + return HostMemoryQueryRange(addr, requested_size, HostMemoryAccess::Write, writable_size); +} + +bool HostMemoryReadDword(void*, uint64_t addr, uint32_t* value) { + if (value == nullptr) { + return false; + } + uint64_t readable = 0; + if (!HostMemoryQueryReadable(addr, sizeof(*value), &readable) || readable < sizeof(*value)) { + return false; + } + std::memcpy(value, reinterpret_cast(static_cast(addr)), sizeof(*value)); + return true; +} + +bool HostMemoryIsReadable(uint64_t addr) { + return HostMemoryRangeIsReadable(addr, 1); +} + +bool HostMemoryRangeIsReadable(uint64_t addr, uint64_t size) { + if (addr == 0 || size == 0 || UINT64_MAX - addr < size) { + return false; + } + uint64_t readable_size = 0; + return HostMemoryQueryReadable(addr, size, &readable_size) && readable_size >= size; +} + +} // namespace Libs::Graphics diff --git a/src/graphics/host_gpu/hostMemory.h b/src/graphics/host_gpu/hostMemory.h new file mode 100644 index 000000000..7463d5018 --- /dev/null +++ b/src/graphics/host_gpu/hostMemory.h @@ -0,0 +1,21 @@ +#ifndef EMULATOR_SRC_GRAPHICS_HOST_GPU_HOSTMEMORY_H_ +#define EMULATOR_SRC_GRAPHICS_HOST_GPU_HOSTMEMORY_H_ + +#include "common/common.h" + +namespace Libs::Graphics { + +enum class HostMemoryAccess { Read, Write }; + +bool HostMemoryQueryRange(uint64_t addr, uint64_t requested_size, HostMemoryAccess access, + uint64_t* accessible_size); +bool HostMemoryQueryEnd(uint64_t addr, HostMemoryAccess access, uint64_t* end); +bool HostMemoryQueryReadable(uint64_t addr, uint64_t requested_size, uint64_t* readable_size); +bool HostMemoryQueryWritable(uint64_t addr, uint64_t requested_size, uint64_t* writable_size); +bool HostMemoryReadDword(void* userdata, uint64_t addr, uint32_t* value); +bool HostMemoryIsReadable(uint64_t addr); +bool HostMemoryRangeIsReadable(uint64_t addr, uint64_t size); + +} // namespace Libs::Graphics + +#endif // EMULATOR_SRC_GRAPHICS_HOST_GPU_HOSTMEMORY_H_ diff --git a/src/graphics/host_gpu/memoryTracker.cpp b/src/graphics/host_gpu/memoryTracker.cpp new file mode 100644 index 000000000..9479c37e8 --- /dev/null +++ b/src/graphics/host_gpu/memoryTracker.cpp @@ -0,0 +1,255 @@ +#include "graphics/host_gpu/memoryTracker.h" + +#include "common/assert.h" + +namespace Libs::Graphics { + +#if defined(KYTY_MEMORY_TRACKER_TESTS) +namespace { +std::atomic g_unmap_contention_hook {nullptr}; +} + +void MemoryTracker::SetUnmapContentionHook(UnmapContentionHook hook) noexcept { + g_unmap_contention_hook.store(hook, std::memory_order_release); +} +#endif + +static_assert(std::atomic::is_always_lock_free); + +MemoryTracker::MemoryTracker(PageManager& page_manager, PageWatchMode gpu_watch_mode) + : m_page_manager(page_manager), m_gpu_watch_mode(gpu_watch_mode) { + switch (m_gpu_watch_mode) { + case PageWatchMode::Write: + case PageWatchMode::ReadWrite: break; + default: EXIT("unsupported memory tracker GPU page-watch mode\n"); + } + m_regions = std::make_unique[]>(REGION_COUNT); + for (size_t i = 0; i < REGION_COUNT; i++) { + m_regions[i].store(nullptr, std::memory_order_relaxed); + } +} + +MemoryTracker::~MemoryTracker() = default; + +void MemoryTracker::ValidateRange(uint64_t vaddr, uint64_t size) { + if (vaddr == 0 || size == 0 || vaddr >= TRACKER_ADDRESS_SIZE || + size > TRACKER_ADDRESS_SIZE - vaddr) { + EXIT("invalid memory tracker range\n"); + } +} + +RegionManager* MemoryTracker::GetOrCreateRegion(uint64_t index) { + if (auto* manager = m_regions[index].load(std::memory_order_acquire); manager != nullptr) { + return manager; + } + std::lock_guard lock(m_region_mutex); + if (auto* manager = m_regions[index].load(std::memory_order_acquire); manager != nullptr) { + return manager; + } + auto manager = std::make_unique(m_page_manager, index * TRACKER_REGION_SIZE); + auto* ptr = manager.get(); + m_region_storage.push_back(std::move(manager)); + m_regions[index].store(ptr, std::memory_order_release); + return ptr; +} + +bool MemoryTracker::IsRegionCpuModified(uint64_t vaddr, uint64_t size) { + CheckNotInUploadCallback(); + std::lock_guard access(m_access_mutex); + RequireMapped(vaddr, size); + return Iterate(vaddr, size, [](RegionManager* manager, uint64_t offset, uint64_t bytes) { + std::scoped_lock lock(manager->lock); + return manager->IsModified(offset, bytes); + }); +} + +bool MemoryTracker::IsRegionGpuModified(uint64_t vaddr, uint64_t size) { + CheckNotInUploadCallback(); + std::lock_guard access(m_access_mutex); + RequireMapped(vaddr, size); + return Iterate(vaddr, size, [](RegionManager* manager, uint64_t offset, uint64_t bytes) { + std::scoped_lock lock(manager->lock); + return manager->IsModified(offset, bytes); + }); +} + +void MemoryTracker::MarkRegionAsCpuModified(uint64_t vaddr, uint64_t size) { + CheckNotInUploadCallback(); + std::lock_guard access(m_access_mutex); + RequireMapped(vaddr, size); + Iterate(vaddr, size, [](RegionManager* manager, uint64_t offset, uint64_t bytes) { + std::scoped_lock lock(manager->lock); + const auto changed = + manager->ChangeState(manager->GetCpuAddr() + offset, bytes); + manager->ApplyProtection(changed, false); + }); +} + +void MemoryTracker::MarkRegionAsGpuModified(uint64_t vaddr, uint64_t size) { + CheckNotInUploadCallback(); + std::lock_guard access(m_access_mutex); + RequireMapped(vaddr, size); + Iterate(vaddr, size, [this](RegionManager* manager, uint64_t offset, uint64_t bytes) { + std::scoped_lock lock(manager->lock); + const auto changed = + manager->ChangeState(manager->GetCpuAddr() + offset, bytes); + manager->ApplyGpuProtection(changed, true, m_gpu_watch_mode); + }); +} + +void MemoryTracker::UnmarkRegionAsGpuModified(uint64_t vaddr, uint64_t size) { + CheckNotInUploadCallback(); + std::lock_guard access(m_access_mutex); + RequireMapped(vaddr, size); + Iterate(vaddr, size, [this](RegionManager* manager, uint64_t offset, uint64_t bytes) { + std::scoped_lock lock(manager->lock); + if (!manager->IsFullyModified(offset, bytes)) { + EXIT("cannot clear partially GPU-dirty tracking range\n"); + } + const auto changed = + manager->ChangeState(manager->GetCpuAddr() + offset, bytes); + manager->ApplyGpuProtection(changed, false, m_gpu_watch_mode); + }); +} + +void MemoryTracker::UntrackMemoryLocked(uint64_t vaddr, uint64_t size) { + RequireMapped(vaddr, size); + + std::vector managers; + managers.reserve((vaddr % TRACKER_REGION_SIZE + size + TRACKER_REGION_SIZE - 1) / + TRACKER_REGION_SIZE); + Iterate(vaddr, size, [&](RegionManager* manager, uint64_t, uint64_t) { + managers.push_back(manager); + }); + + std::vector> locks; + locks.reserve(managers.size()); + for (auto* manager: managers) { + locks.emplace_back(manager->lock); + } + if (Iterate(vaddr, size, [](RegionManager* manager, uint64_t offset, uint64_t bytes) { + return manager->IsModified(offset, bytes); + })) { + EXIT("cannot untrack GPU-dirty memory\n"); + } + Iterate(vaddr, size, [](RegionManager* manager, uint64_t offset, uint64_t bytes) { + const auto changed = + manager->ChangeState(manager->GetCpuAddr() + offset, bytes); + manager->ApplyProtection(changed, false); + manager->Untrack(manager->GetCpuAddr() + offset, bytes); + }); + locks.clear(); +} + +void MemoryTracker::UntrackMemory(uint64_t vaddr, uint64_t size) { + CheckNotInUploadCallback(); + std::lock_guard access(m_access_mutex); + UntrackMemoryLocked(vaddr, size); +} + +void MemoryTracker::UnmapMemory(uint64_t vaddr, uint64_t size) { + CheckNotInUploadCallback(); + std::unique_lock access(m_access_mutex, std::try_to_lock); + if (!access.owns_lock()) { +#if defined(KYTY_MEMORY_TRACKER_TESTS) + if (const auto hook = g_unmap_contention_hook.load(std::memory_order_acquire); + hook != nullptr) { + hook(); + } +#endif + access.lock(); + } + UntrackMemoryLocked(vaddr, size); + m_page_manager.OnGpuUnmap(vaddr, size); +} + +bool MemoryTracker::InvalidateRegion(uint64_t vaddr, uint64_t size, PageFaultPhase phase) noexcept { + switch (phase) { + case PageFaultPhase::Release: return true; + case PageFaultPhase::Invalidate: { + const auto action = BeginCpuFault(vaddr, size); + switch (action) { + case CpuFaultAction::Untracked: return false; + case CpuFaultAction::Continue: return true; + case CpuFaultAction::Download: + EXIT("generic region invalidation cannot download GPU-dirty memory\n"); + } + } + case PageFaultPhase::Complete: + return CompleteCpuFault(vaddr, size, PageFaultAccess::Write, false); + } + EXIT("unsupported region invalidation phase\n"); +} + +bool MemoryTracker::InvalidateVirtualGpuWrite(PageFaultAccess access, uint64_t vaddr, uint64_t size, + PageFaultPhase phase) noexcept { + switch (phase) { + case PageFaultPhase::Release: return true; + case PageFaultPhase::Invalidate: { + const bool gpu_modified = Iterate( + vaddr, size, [](RegionManager* manager, uint64_t offset, uint64_t bytes) { + std::scoped_lock lock(manager->lock); + return manager->IsModified(offset, bytes); + }); + if (!gpu_modified) { + return false; + } + const auto action = BeginCpuFault(vaddr, size); + if (access != PageFaultAccess::Write || action != CpuFaultAction::Download) { + EXIT("virtual GPU write fault requires write access to GPU-dirty memory\n"); + } + return true; + } + case PageFaultPhase::Complete: { + if (access != PageFaultAccess::Write) { + EXIT("virtual GPU write completion requires write access\n"); + } + bool completed = false; + Iterate( + vaddr, size, [&completed](RegionManager* manager, uint64_t offset, uint64_t bytes) { + std::scoped_lock lock(manager->lock); + if (completed) { + EXIT("virtual GPU write fault spans multiple tracked regions\n"); + } + completed = + manager->CompleteVirtualGpuWrite(manager->GetCpuAddr() + offset, bytes); + }); + return completed; + } + } + EXIT("unsupported virtual GPU write invalidation phase\n"); +} + +CpuFaultAction MemoryTracker::BeginCpuFault(uint64_t vaddr, uint64_t size, + PageFaultAccess access) noexcept { + CheckNotInUploadCallback(); + CpuFaultAction action = CpuFaultAction::Untracked; + Iterate( + vaddr, size, [&action, access](RegionManager* manager, uint64_t offset, uint64_t bytes) { + std::scoped_lock lock(manager->lock); + if (action != CpuFaultAction::Untracked) { + EXIT("CPU fault spans multiple tracked regions\n"); + } + action = manager->BeginCpuFault(manager->GetCpuAddr() + offset, bytes, access); + }); + return action; +} + +bool MemoryTracker::CompleteCpuFault(uint64_t vaddr, uint64_t size, PageFaultAccess access, + bool downloaded) noexcept { + CheckNotInUploadCallback(); + bool found = false; + Iterate( + vaddr, size, + [&found, access, downloaded](RegionManager* manager, uint64_t offset, uint64_t bytes) { + std::scoped_lock lock(manager->lock); + if (found) { + EXIT("CPU fault completion spans multiple tracked regions\n"); + } + found = manager->CompleteCpuFault(manager->GetCpuAddr() + offset, bytes, access, + downloaded); + }); + return found; +} + +} // namespace Libs::Graphics diff --git a/src/graphics/host_gpu/memoryTracker.h b/src/graphics/host_gpu/memoryTracker.h new file mode 100644 index 000000000..08fab1d16 --- /dev/null +++ b/src/graphics/host_gpu/memoryTracker.h @@ -0,0 +1,185 @@ +#ifndef EMULATOR_SRC_GRAPHICS_HOST_GPU_MEMORYTRACKER_H_ +#define EMULATOR_SRC_GRAPHICS_HOST_GPU_MEMORYTRACKER_H_ + +#include "common/assert.h" +#include "graphics/host_gpu/pageManager.h" +#include "graphics/host_gpu/regionManager.h" + +#include +#include +#include +#include +#include +#include + +namespace Libs::Graphics { + +class MemoryTracker final { +public: + explicit MemoryTracker(PageManager& page_manager, + PageWatchMode gpu_watch_mode = PageWatchMode::ReadWrite); + ~MemoryTracker(); + + KYTY_CLASS_NO_COPY(MemoryTracker); + + [[nodiscard]] bool IsRegionCpuModified(uint64_t vaddr, uint64_t size); + [[nodiscard]] bool IsRegionGpuModified(uint64_t vaddr, uint64_t size); + void MarkRegionAsCpuModified(uint64_t vaddr, uint64_t size); + void MarkRegionAsGpuModified(uint64_t vaddr, uint64_t size); + void UnmarkRegionAsGpuModified(uint64_t vaddr, uint64_t size); + void UntrackMemory(uint64_t vaddr, uint64_t size); + void UnmapMemory(uint64_t vaddr, uint64_t size); + [[nodiscard]] CpuFaultAction + BeginCpuFault(uint64_t vaddr, uint64_t size, + PageFaultAccess access = PageFaultAccess::Write) noexcept; + [[nodiscard]] bool CompleteCpuFault(uint64_t vaddr, uint64_t size, PageFaultAccess access, + bool downloaded) noexcept; + [[nodiscard]] bool InvalidateRegion(uint64_t vaddr, uint64_t size, + PageFaultPhase phase) noexcept; + [[nodiscard]] bool InvalidateVirtualGpuWrite(PageFaultAccess access, uint64_t vaddr, + uint64_t size, PageFaultPhase phase) noexcept; + + template + void ForEachDownloadRange(uint64_t vaddr, uint64_t size, Preflight&& preflight, Func&& func) { + static_assert(std::is_nothrow_invocable_v); + static_assert(std::is_nothrow_invocable_v); + CheckNotInUploadCallback(); + std::lock_guard access(m_access_mutex); + RequireMapped(vaddr, size); + std::vector managers; + Iterate(vaddr, size, [&](RegionManager* manager, uint64_t, uint64_t) { + managers.push_back(manager); + }); + std::vector> locks; + locks.reserve(managers.size()); + for (auto* manager: managers) { + locks.emplace_back(manager->lock); + } + Iterate(vaddr, size, [&](RegionManager* manager, uint64_t offset, uint64_t bytes) { + const auto address = manager->GetCpuAddr() + offset; + if (manager->HasPendingFault(address, bytes)) { + EXIT("GPU download synchronization raced a pending CPU fault\n"); + } + manager->template ForEachModifiedRange(address, bytes, + preflight); + }); + Iterate(vaddr, size, [&](RegionManager* manager, uint64_t offset, uint64_t bytes) { + manager->template ForEachModifiedRange( + manager->GetCpuAddr() + offset, bytes, func); + }); + if constexpr (clear) { + Iterate(vaddr, size, + [&](RegionManager* manager, uint64_t offset, uint64_t bytes) { + const auto address = manager->GetCpuAddr() + offset; + const auto changed = + manager->template ForEachModifiedRange( + address, bytes, [](uint64_t, uint64_t) noexcept {}); + manager->ApplyGpuProtection(changed, false, m_gpu_watch_mode); + }); + } + } + + template + void ForEachDownloadRange(uint64_t vaddr, uint64_t size, Func&& func) { + ForEachDownloadRange( + vaddr, size, [](uint64_t, uint64_t) noexcept {}, std::forward(func)); + } + +#if defined(KYTY_MEMORY_TRACKER_TESTS) + using UnmapContentionHook = void (*)() noexcept; + static void SetUnmapContentionHook(UnmapContentionHook hook) noexcept; +#endif + + template + void ForEachUploadRange(uint64_t vaddr, uint64_t size, bool is_written, RangeFunc&& range_func, + UploadFunc&& upload_func) { + static_assert(std::is_nothrow_invocable_v); + static_assert(std::is_nothrow_invocable_v); + CheckNotInUploadCallback(); + std::unique_lock access(m_access_mutex); + RequireMapped(vaddr, size); + Iterate(vaddr, size, [](RegionManager*, uint64_t, uint64_t) {}); + s_upload_owner = this; + Iterate(vaddr, size, [&](RegionManager* manager, uint64_t offset, uint64_t bytes) { + manager->lock.lock(); + manager->Track(manager->GetCpuAddr() + offset, bytes); + manager->ForEachModifiedRange(manager->GetCpuAddr() + offset, + bytes, range_func); + if (!is_written) { + manager->lock.unlock(); + } + }); + upload_func(); + if (is_written) { + Iterate( + vaddr, size, [this](RegionManager* manager, uint64_t offset, uint64_t bytes) { + const auto changed = manager->template ChangeState( + manager->GetCpuAddr() + offset, bytes); + manager->ApplyGpuProtection(changed, true, m_gpu_watch_mode); + manager->lock.unlock(); + }); + } + s_upload_owner = nullptr; + } + +private: + static constexpr size_t REGION_COUNT = TRACKER_ADDRESS_SIZE / TRACKER_REGION_SIZE; + inline static thread_local const MemoryTracker* s_upload_owner = nullptr; + + static void CheckNotInUploadCallback() noexcept { + if (s_upload_owner != nullptr) { + EXIT("memory tracker re-entered from upload callback\n"); + } + } + + template + bool Iterate(uint64_t vaddr, uint64_t size, Func&& func) { + ValidateRange(vaddr, size); + using Result = std::invoke_result_t; + constexpr bool returns_bool = std::is_same_v; + uint64_t remaining = size; + uint64_t index = vaddr / TRACKER_REGION_SIZE; + uint64_t offset = vaddr % TRACKER_REGION_SIZE; + while (remaining != 0) { + const auto bytes = std::min(TRACKER_REGION_SIZE - offset, remaining); + auto* manager = m_regions[index].load(std::memory_order_acquire); + if (manager == nullptr && create) { + manager = GetOrCreateRegion(index); + } + if (manager != nullptr) { + if constexpr (returns_bool) { + if (func(manager, offset, bytes)) { + return true; + } + } else { + func(manager, offset, bytes); + } + } + remaining -= bytes; + offset = 0; + index++; + } + return false; + } + + static void ValidateRange(uint64_t vaddr, uint64_t size); + void UntrackMemoryLocked(uint64_t vaddr, uint64_t size); + void RequireMapped(uint64_t vaddr, uint64_t size) const { + ValidateRange(vaddr, size); + if (!m_page_manager.IsMapped(vaddr, size)) { + EXIT("memory tracker range is not mapped\n"); + } + } + RegionManager* GetOrCreateRegion(uint64_t index); + + std::unique_ptr[]> m_regions; + std::vector> m_region_storage; + std::mutex m_region_mutex; + std::mutex m_access_mutex; + PageManager& m_page_manager; + PageWatchMode m_gpu_watch_mode = PageWatchMode::ReadWrite; +}; + +} // namespace Libs::Graphics + +#endif // EMULATOR_SRC_GRAPHICS_HOST_GPU_MEMORYTRACKER_H_ diff --git a/src/graphics/host_gpu/objects/label.cpp b/src/graphics/host_gpu/objects/label.cpp new file mode 100644 index 000000000..b44b90898 --- /dev/null +++ b/src/graphics/host_gpu/objects/label.cpp @@ -0,0 +1,432 @@ +#include "graphics/host_gpu/objects/label.h" + +#include "common/assert.h" +#include "common/logging/log.h" +#include "kernel/memory.h" +#include "common/threads.h" +#include "graphics/host_gpu/graphicContext.h" +#include "graphics/host_gpu/renderer/render.h" + +#include +#include +#include +#include + +namespace Libs::Graphics { + +enum LabelStatus { + New, + Active, + ActiveDeleted, + NotActive, +}; + +struct LabelCallbacks { + uint64_t* dst_gpu_addr64 = nullptr; + uint64_t value64 = 0; + uint32_t* dst_gpu_addr32 = nullptr; + uint32_t value32 = 0; + LabelCallback callback_1 = nullptr; + LabelCallback callback_2 = nullptr; + uint64_t args[LABEL_ARGS_MAX] = {}; +}; + +struct LabelEvent final { + VkDevice device = nullptr; + VkEvent event = nullptr; + + ~LabelEvent() { + if (event != nullptr) { + vkDestroyEvent(device, event, nullptr); + } + } +}; + +struct LabelSubmission { + std::shared_ptr completion; + LabelCallbacks callbacks; +}; + +struct Label { + VkDevice device = nullptr; + LabelStatus status = LabelStatus::New; + LabelCallbacks callbacks; + std::vector submissions; +}; + +class LabelManager { +public: + LabelManager() { + EXIT_NOT_IMPLEMENTED(!Common::Thread::IsMainThread()); + Common::Thread t(ThreadRun, this); + t.Detach(); + } + ~LabelManager() { KYTY_NOT_IMPLEMENTED; } + KYTY_CLASS_NO_COPY(LabelManager); + + Label* Create64(GraphicContext* ctx, uint64_t* dst_gpu_addr, uint64_t value, + LabelCallback callback_1, LabelCallback callback_2, const uint64_t* args); + Label* Create32(GraphicContext* ctx, uint32_t* dst_gpu_addr, uint32_t value, + LabelCallback callback_1, LabelCallback callback_2, const uint64_t* args); + void Delete(Label* label); + void Set(CommandBuffer* buffer, Label* label); + void Drain(); + +private: + static void ThreadRun(void* data); + + template + Label* Create(GraphicContext* ctx, T* dst_gpu_addr, T value, LabelCallback callback_1, + LabelCallback callback_2, const uint64_t* args); + bool Remove(Label* label); + static void Destroy(Label* label); + + Common::Mutex m_mutex; + Common::CondVar m_cond_var; + std::vector m_labels; + uint64_t m_callbacks_in_flight = 0; +}; + +static LabelManager* g_label_manager = nullptr; +static thread_local bool g_in_label_callback = false; + +template +void WriteGuestLabel(T* address, T value) { + LabelWriteGuestMemory(address, &value, sizeof(value)); +} + +class LabelCallbackScope final { +public: + LabelCallbackScope() { + if (g_in_label_callback) { + EXIT("recursive GPU label callback\n"); + } + g_in_label_callback = true; + } + ~LabelCallbackScope() { + if (!g_in_label_callback) { + EXIT("GPU label callback scope is not active\n"); + } + g_in_label_callback = false; + } +}; + +void LabelManager::ThreadRun(void* data) { + auto* manager = static_cast(data); + + for (;;) { + manager->m_mutex.Lock(); + + uint64_t active_count = 0; + + std::vector deleted_labels; + std::vector fired_labels; + std::vector finished_submissions; + deleted_labels.reserve(manager->m_labels.size()); + + for (auto& label: manager->m_labels) { + for (auto it = label->submissions.begin(); it != label->submissions.end();) { + active_count++; + + if (it->completion == nullptr || it->completion->device == nullptr || + it->completion->event == nullptr) { + EXIT("GPU label submission has no completion event\n"); + } + const auto status = + vkGetEventStatus(it->completion->device, it->completion->event); + switch (status) { + case VK_EVENT_SET: + fired_labels.push_back(it->callbacks); + finished_submissions.push_back(*it); + it = label->submissions.erase(it); + break; + case VK_EVENT_RESET: ++it; break; + default: EXIT("vkGetEventStatus returned an unexpected result\n"); + } + } + + if (label->submissions.empty()) { + switch (label->status) { + case LabelStatus::ActiveDeleted: deleted_labels.push_back(label); break; + case LabelStatus::Active: label->status = LabelStatus::NotActive; break; + default: break; + } + } + } + + if (active_count == 0) { + manager->m_cond_var.Wait(&manager->m_mutex); + } + if (fired_labels.size() > UINT64_MAX - manager->m_callbacks_in_flight) { + EXIT("GPU label callback count overflow\n"); + } + manager->m_callbacks_in_flight += fired_labels.size(); + + for (auto& label: deleted_labels) { + bool removed = manager->Remove(label); + EXIT_NOT_IMPLEMENTED(!removed); + } + + manager->m_mutex.Unlock(); + + // Each completion event is shared with the recording command buffer's fence retainer. + // The event is destroyed only after both the label thread observed it and that command + // buffer completed, even when either side wins the race. + (void)finished_submissions; + + for (auto& label: deleted_labels) { + Destroy(label); + } + + for (auto& label: fired_labels) { + LabelCallbackScope callback_scope; + bool write = true; + + if (label.callback_1 != nullptr) { + write = label.callback_1(label.args); + } + + if (write && label.dst_gpu_addr64 != nullptr) { + // Publishing EOP fences through physical backing keeps a fence word coherent even + // when unrelated bytes on its host page are + // protected for BufferCache GPU ownership. + WriteGuestLabel(label.dst_gpu_addr64, label.value64); + + static std::atomic log_count {0}; + if (log_count.fetch_add(1) < 256) { + LOGF_COLOR(Log::Color::BrightGreen, + "EndOfPipe Signal!!! [0x%016" PRIx64 "] <- 0x%016" PRIx64 "\n", + reinterpret_cast(label.dst_gpu_addr64), label.value64); + } + } + + if (write && label.dst_gpu_addr32 != nullptr) { + WriteGuestLabel(label.dst_gpu_addr32, label.value32); + + static std::atomic log_count {0}; + if (log_count.fetch_add(1) < 256) { + LOGF_COLOR(Log::Color::BrightGreen, + "EndOfPipe Signal!!! [0x%016" PRIx64 "] <- 0x%08" PRIx32 "\n", + reinterpret_cast(label.dst_gpu_addr32), label.value32); + } + } + + if (label.callback_2 != nullptr) { + label.callback_2(label.args); + } + } + + if (!fired_labels.empty()) { + Common::LockGuard lock(manager->m_mutex); + if (manager->m_callbacks_in_flight < fired_labels.size()) { + EXIT("GPU label callback count underflow\n"); + } + manager->m_callbacks_in_flight -= fired_labels.size(); + manager->m_cond_var.SignalAll(); + } + + Common::Thread::SleepMicro(100); + } +} + +template +Label* LabelManager::Create(GraphicContext* ctx, T* dst_gpu_addr, T value, LabelCallback callback_1, + LabelCallback callback_2, const uint64_t* args) { + static_assert(sizeof(T) == sizeof(uint32_t) || sizeof(T) == sizeof(uint64_t)); + + EXIT_IF(ctx == nullptr); + + Common::LockGuard lock(m_mutex); + + auto* label = new Label; + + label->status = LabelStatus::New; + if constexpr (sizeof(T) == sizeof(uint64_t)) { + label->callbacks.dst_gpu_addr64 = dst_gpu_addr; + label->callbacks.value64 = value; + label->callbacks.dst_gpu_addr32 = nullptr; + label->callbacks.value32 = 0; + } else { + label->callbacks.dst_gpu_addr32 = dst_gpu_addr; + label->callbacks.value32 = value; + label->callbacks.dst_gpu_addr64 = nullptr; + label->callbacks.value64 = 0; + } + label->device = ctx->device; + label->callbacks.callback_1 = callback_1; + label->callbacks.callback_2 = callback_2; + + if (args != nullptr) { + for (int i = 0; i < LABEL_ARGS_MAX; i++) { + label->callbacks.args[i] = args[i]; + } + } + + m_labels.push_back(label); + + return label; +} + +Label* LabelManager::Create64(GraphicContext* ctx, uint64_t* dst_gpu_addr, uint64_t value, + LabelCallback callback_1, LabelCallback callback_2, + const uint64_t* args) { + return Create(ctx, dst_gpu_addr, value, callback_1, callback_2, args); +} + +Label* LabelManager::Create32(GraphicContext* ctx, uint32_t* dst_gpu_addr, uint32_t value, + LabelCallback callback_1, LabelCallback callback_2, + const uint64_t* args) { + return Create(ctx, dst_gpu_addr, value, callback_1, callback_2, args); +} + +bool LabelManager::Remove(Label* label) { + EXIT_IF(label == nullptr); + EXIT_IF(label->device == nullptr); + + Common::LockGuard lock(m_mutex); + + const auto it = std::find(m_labels.begin(), m_labels.end(), label); + EXIT_NOT_IMPLEMENTED(it == m_labels.end()); + + EXIT_NOT_IMPLEMENTED(label->status != LabelStatus::NotActive && + label->status != LabelStatus::Active && + label->status != LabelStatus::ActiveDeleted); + + if (!label->submissions.empty()) { + label->status = LabelStatus::ActiveDeleted; + + return false; + } + + m_labels.erase(it); + + return true; +} + +void LabelManager::Destroy(Label* label) { + EXIT_IF(label == nullptr); + EXIT_IF(label->device == nullptr); + + EXIT_NOT_IMPLEMENTED(!label->submissions.empty()); + + delete label; +} + +void LabelManager::Delete(Label* label) { + if (Remove(label)) { + Destroy(label); + } +} + +void LabelManager::Set(CommandBuffer* buffer, Label* label) { + EXIT_IF(label == nullptr); + EXIT_IF(buffer == nullptr); + EXIT_IF(buffer->IsInvalid()); + EXIT_IF(label->device == nullptr); + + Common::LockGuard lock(m_mutex); + + const auto it = std::find(m_labels.begin(), m_labels.end(), label); + EXIT_NOT_IMPLEMENTED(it == m_labels.end()); + + EXIT_NOT_IMPLEMENTED(label->status != LabelStatus::New && + label->status != LabelStatus::NotActive && + label->status != LabelStatus::Active); + + label->status = LabelStatus::Active; + + LabelSubmission submission {}; + submission.callbacks = label->callbacks; + submission.completion = std::make_shared(); + submission.completion->device = label->device; + + auto* vk_buffer = buffer->GetPool()->buffers[buffer->GetIndex()]; + + EXIT_NOT_IMPLEMENTED(vk_buffer == nullptr); + + VkEventCreateInfo create_info {}; + create_info.sType = VK_STRUCTURE_TYPE_EVENT_CREATE_INFO; + create_info.pNext = nullptr; + create_info.flags = 0; + + vkCreateEvent(label->device, &create_info, nullptr, &submission.completion->event); + EXIT_NOT_IMPLEMENTED(submission.completion->event == nullptr); + buffer->RetainResourceUntilFence(submission.completion); + + // Labels can be reused before an earlier end-of-pipe event has been + // observed by the polling thread. Capture a separate Vulkan event and + // callback snapshot for each set so older writes are not lost. + vkResetEvent(label->device, submission.completion->event); + vkCmdSetEvent(vk_buffer, submission.completion->event, VK_PIPELINE_STAGE_BOTTOM_OF_PIPE_BIT); + + label->submissions.push_back(submission); + + m_cond_var.SignalAll(); +} + +void LabelManager::Drain() { + m_mutex.Lock(); + for (;;) { + const bool pending = std::any_of(m_labels.begin(), m_labels.end(), [](const Label* label) { + return !label->submissions.empty(); + }); + if (!pending && m_callbacks_in_flight == 0) { + m_mutex.Unlock(); + return; + } + m_cond_var.SignalAll(); + m_cond_var.Wait(&m_mutex); + } +} + +void LabelInit() { + EXIT_IF(g_label_manager != nullptr); + + g_label_manager = new LabelManager; +} + +Label* LabelCreate64(GraphicContext* ctx, uint64_t* dst_gpu_addr, uint64_t value, + LabelCallback callback_1, LabelCallback callback_2, const uint64_t* args) { + EXIT_IF(g_label_manager == nullptr); + + return g_label_manager->Create64(ctx, dst_gpu_addr, value, callback_1, callback_2, args); +} + +Label* LabelCreate32(GraphicContext* ctx, uint32_t* dst_gpu_addr, uint32_t value, + LabelCallback callback_1, LabelCallback callback_2, const uint64_t* args) { + EXIT_IF(g_label_manager == nullptr); + + return g_label_manager->Create32(ctx, dst_gpu_addr, value, callback_1, callback_2, args); +} + +void LabelDelete(Label* label) { + EXIT_IF(g_label_manager == nullptr); + + g_label_manager->Delete(label); +} + +void LabelSet(CommandBuffer* buffer, Label* label) { + EXIT_IF(g_label_manager == nullptr); + + g_label_manager->Set(buffer, label); +} + +void LabelDrain() { + EXIT_IF(g_label_manager == nullptr); + g_label_manager->Drain(); +} + +void LabelWriteGuestMemory(void* address, const void* data, uint64_t size) { + if (address == nullptr || data == nullptr || size == 0) { + EXIT("invalid guest label write\n"); + } + if (!Libs::LibKernel::Memory::TryWriteBacking(reinterpret_cast(address), data, size)) { + std::memcpy(address, data, static_cast(size)); + } +} + +bool LabelInCallback() noexcept { + return g_in_label_callback; +} + +} // namespace Libs::Graphics diff --git a/src/graphics/host_gpu/objects/label.h b/src/graphics/host_gpu/objects/label.h new file mode 100644 index 000000000..2c1bf6244 --- /dev/null +++ b/src/graphics/host_gpu/objects/label.h @@ -0,0 +1,30 @@ +#ifndef EMULATOR_INCLUDE_EMULATOR_GRAPHICS_OBJECTS_LABEL_H_ +#define EMULATOR_INCLUDE_EMULATOR_GRAPHICS_OBJECTS_LABEL_H_ + +#include "common/abi.h" +#include "common/common.h" + +namespace Libs::Graphics { + +struct Label; +class CommandBuffer; +struct GraphicContext; + +void LabelInit(); + +constexpr int LABEL_ARGS_MAX = 5; +using LabelCallback = bool (*)(const uint64_t* args); + +Label* LabelCreate64(GraphicContext* ctx, uint64_t* dst_gpu_addr, uint64_t value, + LabelCallback callback_1, LabelCallback callback_2, const uint64_t* args); +Label* LabelCreate32(GraphicContext* ctx, uint32_t* dst_gpu_addr, uint32_t value, + LabelCallback callback_1, LabelCallback callback_2, const uint64_t* args); +void LabelDelete(Label* label); +void LabelSet(CommandBuffer* buffer, Label* label); +void LabelDrain(); +void LabelWriteGuestMemory(void* address, const void* data, uint64_t size); +[[nodiscard]] bool LabelInCallback() noexcept; + +} // namespace Libs::Graphics + +#endif /* EMULATOR_INCLUDE_EMULATOR_GRAPHICS_OBJECTS_LABEL_H_ */ diff --git a/src/graphics/host_gpu/objects/textureCommon.cpp b/src/graphics/host_gpu/objects/textureCommon.cpp new file mode 100644 index 000000000..8cc4a3a5e --- /dev/null +++ b/src/graphics/host_gpu/objects/textureCommon.cpp @@ -0,0 +1,1108 @@ +#include "graphics/host_gpu/objects/textureCommon.h" + +#include "common/assert.h" +#include "common/logging/log.h" +#include "graphics/guest_gpu/gpu_defs.h" +#include "graphics/guest_gpu/gpu_format.h" +#include "graphics/host_gpu/graphicContext.h" +#include "graphics/host_gpu/utils.h" +#include "graphics/host_gpu/vma.h" + +#include +#include +#include +#include + +namespace Libs::Graphics { + +// TODO: cleanup! +RenderTargetFormatInfo TextureGetRenderTargetFormat(uint32_t raw_layout, uint32_t raw_type, + uint32_t raw_order) { + const auto layout = static_cast(raw_layout); + const auto type = static_cast(raw_type); + const auto order = static_cast(raw_order); + const auto is = [=](Prospero::ChannelLayout l, Prospero::ChannelType t, Prospero::ChannelOrder o) { + return layout == l && type == t && order == o; + }; + + if (layout == Prospero::ChannelLayout::k8 && type == Prospero::ChannelType::kUNorm && + raw_order <= Prospero::GpuEnumValue(Prospero::ChannelOrder::kAltReversed)) { + return {VK_FORMAT_R8_UNORM, 1}; + } + if (is(Prospero::ChannelLayout::k8_8, Prospero::ChannelType::kUNorm, Prospero::ChannelOrder::kStandard)) { + return {VK_FORMAT_R8G8_UNORM, 2}; + } + if (is(Prospero::ChannelLayout::k8_8_8_8, Prospero::ChannelType::kUNorm, Prospero::ChannelOrder::kStandard)) { + return {VK_FORMAT_R8G8B8A8_UNORM, 4}; + } + if (is(Prospero::ChannelLayout::k8_8_8_8, Prospero::ChannelType::kSNorm, Prospero::ChannelOrder::kStandard)) { + return {VK_FORMAT_R8G8B8A8_SNORM, 4}; + } + if (is(Prospero::ChannelLayout::k8_8_8_8, Prospero::ChannelType::kSrgb, Prospero::ChannelOrder::kStandard)) { + return {VK_FORMAT_R8G8B8A8_SRGB, 4}; + } + if (is(Prospero::ChannelLayout::k8_8_8_8, Prospero::ChannelType::kUNorm, Prospero::ChannelOrder::kAlt)) { + return {VK_FORMAT_B8G8R8A8_UNORM, 4}; + } + if (is(Prospero::ChannelLayout::k8_8_8_8, Prospero::ChannelType::kSNorm, Prospero::ChannelOrder::kAlt)) { + return {VK_FORMAT_B8G8R8A8_SNORM, 4}; + } + if (is(Prospero::ChannelLayout::k8_8_8_8, Prospero::ChannelType::kSrgb, Prospero::ChannelOrder::kAlt)) { + return {VK_FORMAT_B8G8R8A8_SRGB, 4}; + } + if (is(Prospero::ChannelLayout::k5_5_5_1, Prospero::ChannelType::kUNorm, Prospero::ChannelOrder::kStandard)) { + return {VK_FORMAT_R5G5B5A1_UNORM_PACK16, 2}; + } + if (is(Prospero::ChannelLayout::k4_4_4_4, Prospero::ChannelType::kUNorm, Prospero::ChannelOrder::kReversed)) { + return {VK_FORMAT_B4G4R4A4_UNORM_PACK16, 2}; + } + if (is(Prospero::ChannelLayout::k10_10_10_2, Prospero::ChannelType::kUNorm, Prospero::ChannelOrder::kStandard)) { + return {VK_FORMAT_A2B10G10R10_UNORM_PACK32, 4}; + } + if (is(Prospero::ChannelLayout::k10_10_10_2, Prospero::ChannelType::kUNorm, Prospero::ChannelOrder::kAlt)) { + return {VK_FORMAT_A2R10G10B10_UNORM_PACK32, 4}; + } + if (is(Prospero::ChannelLayout::k11_11_10, Prospero::ChannelType::kFloat, Prospero::ChannelOrder::kStandard)) { + return {VK_FORMAT_B10G11R11_UFLOAT_PACK32, 4}; + } + if (is(Prospero::ChannelLayout::k16, Prospero::ChannelType::kUNorm, Prospero::ChannelOrder::kStandard)) { + return {VK_FORMAT_R16_UNORM, 2}; + } + if (is(Prospero::ChannelLayout::k16, Prospero::ChannelType::kUInt, Prospero::ChannelOrder::kStandard)) { + return {VK_FORMAT_R16_UINT, 2}; + } + if (is(Prospero::ChannelLayout::k16, Prospero::ChannelType::kFloat, Prospero::ChannelOrder::kStandard)) { + return {VK_FORMAT_R16_SFLOAT, 2}; + } + if (is(Prospero::ChannelLayout::k16_16, Prospero::ChannelType::kUNorm, Prospero::ChannelOrder::kStandard)) { + return {VK_FORMAT_R16G16_UNORM, 4}; + } + if (is(Prospero::ChannelLayout::k16_16, Prospero::ChannelType::kSNorm, Prospero::ChannelOrder::kStandard)) { + return {VK_FORMAT_R16G16_SNORM, 4}; + } + if (is(Prospero::ChannelLayout::k16_16, Prospero::ChannelType::kUInt, Prospero::ChannelOrder::kStandard)) { + return {VK_FORMAT_R16G16_UINT, 4}; + } + if (is(Prospero::ChannelLayout::k16_16, Prospero::ChannelType::kFloat, Prospero::ChannelOrder::kStandard)) { + return {VK_FORMAT_R16G16_SFLOAT, 4}; + } + if (is(Prospero::ChannelLayout::k16_16_16_16, Prospero::ChannelType::kUNorm, Prospero::ChannelOrder::kStandard)) { + return {VK_FORMAT_R16G16B16A16_UNORM, 8}; + } + if (is(Prospero::ChannelLayout::k16_16_16_16, Prospero::ChannelType::kFloat, Prospero::ChannelOrder::kStandard)) { + return {VK_FORMAT_R16G16B16A16_SFLOAT, 8}; + } + if (is(Prospero::ChannelLayout::k32, Prospero::ChannelType::kFloat, Prospero::ChannelOrder::kStandard)) { + return {VK_FORMAT_R32_SFLOAT, 4}; + } + if (is(Prospero::ChannelLayout::k32_32, Prospero::ChannelType::kUInt, Prospero::ChannelOrder::kStandard)) { + return {VK_FORMAT_R32G32_UINT, 8}; + } + if (is(Prospero::ChannelLayout::k32_32, Prospero::ChannelType::kFloat, Prospero::ChannelOrder::kStandard)) { + return {VK_FORMAT_R32G32_SFLOAT, 8}; + } + if (is(Prospero::ChannelLayout::k32_32_32_32, Prospero::ChannelType::kFloat, Prospero::ChannelOrder::kStandard)) { + return {VK_FORMAT_R32G32B32A32_SFLOAT, 16}; + } + EXIT("unsupported render-target format combination: layout=%u type=%u order=%u\n", raw_layout, + raw_type, raw_order); +} + +static uint64_t TextureUploadLevelSrcOffset(const TileSizeOffset& level_size) { + return (level_size.src_size != 0 ? level_size.src_offset : level_size.offset); +} + +static uint64_t TextureUploadLevelSrcSize(const TileSizeOffset& level_size) { + return (level_size.src_size != 0 ? level_size.src_size : level_size.size); +} + +uint64_t TextureUploadSliceSourceOffset(const TextureUploadLayout& layout, uint32_t level, + uint32_t slice, + TextureUploadSliceLayout source_slice_layout) { + if (level >= 16 || layout.level_sizes[level].size == 0) { + EXIT("invalid texture upload slice source, level=%u slice=%u\n", level, slice); + } + const auto level_offset = TextureUploadLevelSrcOffset(layout.level_sizes[level]); + const auto slice_stride = + source_slice_layout == TextureUploadSliceLayout::MipChainPerSlice + ? (layout.source_slice_stride != 0 ? layout.source_slice_stride : layout.slice_stride) + : TextureUploadLevelSrcSize(layout.level_sizes[level]); + if (slice_stride != 0 && slice > (UINT64_MAX - level_offset) / slice_stride) { + EXIT("texture upload slice source offset overflow, level=%u slice=%u\n", level, slice); + } + return level_offset + static_cast(slice) * slice_stride; +} + +uint64_t TextureCalcUploadSize(const TextureUploadLayout& layout, + const std::vector& regions, uint64_t levels, + uint32_t depth, TextureUploadSliceLayout source_slice_layout) { + uint64_t size = 0; + + for (const auto& r: regions) { + size = std::max(size, static_cast(r.offset) + + layout.level_sizes[r.dst_level].size); + } + + for (uint32_t level = 0; level < levels; level++) { + const auto src_size = TextureUploadLevelSrcSize(layout.level_sizes[level]); + for (uint32_t z = 0; z < depth; z++) { + size = std::max( + size, + TextureUploadSliceSourceOffset(layout, level, z, source_slice_layout) + src_size); + } + } + + return size; +} + +void TextureCopyBufferBytes(GraphicContext* ctx, VulkanBuffer* src_buffer, + uint64_t src_buffer_offset, uint64_t copy_size, + UtilScratchBuffer* dst) { + EXIT_IF(ctx == nullptr); + EXIT_IF(src_buffer == nullptr); + EXIT_IF(dst == nullptr); + EXIT_IF(dst->Data() == nullptr); + + std::memset(dst->Data(), 0, copy_size); + + if (copy_size == 0 || src_buffer_offset >= src_buffer->buffer_size) { + return; + } + + const auto available = + std::min(copy_size, src_buffer->buffer_size - src_buffer_offset); + if (available == 0) { + return; + } + + if ((src_buffer->memory.property & VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT) != 0) { + void* data = nullptr; + VulkanMapMemory(ctx, &src_buffer->memory, &data); + std::memcpy(dst->Data(), static_cast(data) + src_buffer_offset, available); + VulkanUnmapMemory(ctx, &src_buffer->memory); + return; + } + + VulkanBuffer readback {}; + readback.usage = VK_BUFFER_USAGE_TRANSFER_DST_BIT; + readback.memory.property = static_cast(VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT) | + VK_MEMORY_PROPERTY_HOST_COHERENT_BIT | + VK_MEMORY_PROPERTY_HOST_CACHED_BIT; + VulkanCreateBuffer(ctx, (src_buffer_offset + available + 3u) & ~uint64_t {3}, &readback); + UtilCopyBuffer(src_buffer, &readback, src_buffer_offset + available); + + void* data = nullptr; + VulkanMapMemory(ctx, &readback.memory, &data); + std::memcpy(dst->Data(), static_cast(data) + src_buffer_offset, available); + VulkanUnmapMemory(ctx, &readback.memory); + + VulkanDeleteBuffer(ctx, &readback); +} + +VkComponentSwizzle TextureGetComponentSwizzle(uint8_t s) { + switch (static_cast(s)) { + case Prospero::CompSwizzle::kZero: return VK_COMPONENT_SWIZZLE_ZERO; + case Prospero::CompSwizzle::kOne: return VK_COMPONENT_SWIZZLE_ONE; + case Prospero::CompSwizzle::kRed: return VK_COMPONENT_SWIZZLE_R; + case Prospero::CompSwizzle::kGreen: return VK_COMPONENT_SWIZZLE_G; + case Prospero::CompSwizzle::kBlue: return VK_COMPONENT_SWIZZLE_B; + case Prospero::CompSwizzle::kAlpha: return VK_COMPONENT_SWIZZLE_A; + default: EXIT("unknown swizzle: %d\n", static_cast(s)); + } + return VK_COMPONENT_SWIZZLE_IDENTITY; +} + +static uint32_t TextureGetDstSel(uint32_t swizzle, uint32_t channel) { + return (swizzle >> (channel * 3u)) & 0x7u; +} + +VkComponentMapping TextureGetComponentMapping(uint32_t swizzle) { + VkComponentMapping components {}; + components.r = TextureGetComponentSwizzle(static_cast(TextureGetDstSel(swizzle, 0))); + components.g = TextureGetComponentSwizzle(static_cast(TextureGetDstSel(swizzle, 1))); + components.b = TextureGetComponentSwizzle(static_cast(TextureGetDstSel(swizzle, 2))); + components.a = TextureGetComponentSwizzle(static_cast(TextureGetDstSel(swizzle, 3))); + return components; +} + +bool TextureCheckFormat(GraphicContext* ctx, VkImageCreateInfo* image_info) { + VkImageFormatProperties props {}; + if (vkGetPhysicalDeviceImageFormatProperties( + ctx->physical_device, image_info->format, image_info->imageType, image_info->tiling, + image_info->usage, image_info->flags, &props) == VK_ERROR_FORMAT_NOT_SUPPORTED) { + auto apply_fallback = [&](VkFormat replacement, const char* message) { + image_info->format = replacement; + const bool result = TextureCheckFormat(ctx, image_info); + LOGF("%s [%s]\n", message, (!result ? "FAIL" : "SUCCESS")); + return result; + }; + + if (image_info->format == VK_FORMAT_R8G8B8A8_SRGB) { + // TODO() convert SRGB -> LINEAR in shader + return apply_fallback(VK_FORMAT_R8G8B8A8_UNORM, + "replace VK_FORMAT_R8G8B8A8_SRGB => VK_FORMAT_R8G8B8A8_UNORM"); + } + if (image_info->format == VK_FORMAT_B8G8R8A8_SRGB) { + // TODO() convert SRGB -> LINEAR in shader + return apply_fallback(VK_FORMAT_B8G8R8A8_UNORM, + "replace VK_FORMAT_B8G8R8A8_SRGB => VK_FORMAT_B8G8R8A8_UNORM"); + } + return false; + } + return true; +} + +static bool TextureCheckFormatExact(GraphicContext* ctx, const VkImageCreateInfo& image_info) { + VkImageFormatProperties props {}; + return vkGetPhysicalDeviceImageFormatProperties( + ctx->physical_device, image_info.format, image_info.imageType, image_info.tiling, + image_info.usage, image_info.flags, &props) != VK_ERROR_FORMAT_NOT_SUPPORTED; +} + +bool TextureCheckStorageSwizzle(VkImageCreateInfo* image_info, VkComponentMapping* components) { + if ((image_info->usage & VK_IMAGE_USAGE_STORAGE_BIT) != 0) { + if (components->r == VK_COMPONENT_SWIZZLE_R && components->g == VK_COMPONENT_SWIZZLE_G && + components->b == VK_COMPONENT_SWIZZLE_B && components->a == VK_COMPONENT_SWIZZLE_A) { + return true; + } + + if (components->r == VK_COMPONENT_SWIZZLE_B && components->g == VK_COMPONENT_SWIZZLE_G && + components->b == VK_COMPONENT_SWIZZLE_R && components->a == VK_COMPONENT_SWIZZLE_A && + image_info->format == VK_FORMAT_R8G8B8A8_SRGB) { + LOGF("replace VK_FORMAT_R8G8B8A8_SRGB => VK_FORMAT_B8G8R8A8_SRGB\n"); + + components->r = VK_COMPONENT_SWIZZLE_R; + components->g = VK_COMPONENT_SWIZZLE_G; + components->b = VK_COMPONENT_SWIZZLE_B; + components->a = VK_COMPONENT_SWIZZLE_A; + image_info->format = VK_FORMAT_B8G8R8A8_SRGB; + return true; + } + + // TODO() swizzle channels in shader + + return false; + } + return true; +} + +VkImageUsageFlags TextureGetUsage(TextureFormatUsage usage) { + VkImageUsageFlags vk_usage = VK_IMAGE_USAGE_TRANSFER_DST_BIT | VK_IMAGE_USAGE_TRANSFER_SRC_BIT; + if (TextureHasFormatUsage(usage, TextureFormatUsage::Sampled)) { + vk_usage |= VK_IMAGE_USAGE_SAMPLED_BIT; + } + if (TextureHasFormatUsage(usage, TextureFormatUsage::Storage)) { + vk_usage |= VK_IMAGE_USAGE_STORAGE_BIT; + } + return vk_usage; +} + +VkImageUsageFlags TextureGetViewUsage(TextureFormatUsage usage) { + VkImageUsageFlags vk_usage = 0; + if (TextureHasFormatUsage(usage, TextureFormatUsage::Sampled)) { + vk_usage |= VK_IMAGE_USAGE_SAMPLED_BIT; + } + if (TextureHasFormatUsage(usage, TextureFormatUsage::Storage)) { + vk_usage |= VK_IMAGE_USAGE_STORAGE_BIT; + } + return vk_usage; +} + +VkFormat TextureGetFormat(uint32_t fmt, [[maybe_unused]] TextureFormatUsage usage) { + const auto vk_format = Prospero::SurfaceFormat(fmt); + if (vk_format != VK_FORMAT_UNDEFINED) { + return vk_format; + } + EXIT("unknown format: fmt = %u\n", fmt); + return VK_FORMAT_UNDEFINED; +} + +static uint32_t AlignUpU32(uint32_t value, uint32_t alignment) { + return (value + alignment - 1u) & ~(alignment - 1u); +} + +static uint32_t ShiftCeilU32(uint32_t value, uint32_t shift) { + return static_cast((static_cast(value) + (1ull << shift) - 1ull) >> shift); +} + +struct Standard4KBVolumeMipLayout { + uint32_t first_tail_level = 0; + uint32_t block_depth = 1; + uint64_t block_slice_size = 0; + uint64_t level_offsets[16] = {}; + uint64_t level_sizes[16] = {}; +}; + +static bool CalcStandard4kbVolumeMipLayout(uint32_t format, uint32_t pitch, uint32_t height, + uint64_t levels, Standard4KBVolumeMipLayout* out) { + EXIT_IF(out == nullptr); + EXIT_NOT_IMPLEMENTED(levels == 0 || levels > 16); + + uint32_t bytes_per_element = 0; + uint32_t texels_per_element_wide = 0; + uint32_t texels_per_element_tall = 0; + uint32_t block_width_log2 = 0; + uint32_t block_height_log2 = 0; + uint32_t block_depth_log2 = 0; + + if (!TileGetStandard4KBVolumeLayout(format, &bytes_per_element, &texels_per_element_wide, + &texels_per_element_tall, &block_width_log2, + &block_height_log2, &block_depth_log2)) { + return false; + } + + const uint32_t row_elements0 = + std::max((pitch + texels_per_element_wide - 1u) / texels_per_element_wide, 1u); + const uint32_t height_elements0 = + std::max((height + texels_per_element_tall - 1u) / texels_per_element_tall, 1u); + const uint32_t block_width = 1u << block_width_log2; + const uint32_t block_height = 1u << block_height_log2; + const uint32_t block_depth = 1u << block_depth_log2; + const uint32_t tail_width_limit = block_width; + const uint32_t tail_height_limit = block_height >> 1u; + constexpr uint32_t max_tail_levels = 5u; + + out->first_tail_level = static_cast(levels); + out->block_depth = block_depth; + out->block_slice_size = 0; + + for (uint32_t level = 0; level < levels; level++) { + const uint32_t row_elements = std::max(ShiftCeilU32(row_elements0, level), 1u); + const uint32_t height_elements = std::max(ShiftCeilU32(height_elements0, level), 1u); + + out->level_offsets[level] = out->block_slice_size; + if (row_elements <= tail_width_limit && height_elements <= tail_height_limit && + levels - level <= max_tail_levels) { + out->first_tail_level = level; + out->level_sizes[level] = 4096u; + out->block_slice_size += 4096u; + for (uint32_t tail_level = level + 1; tail_level < levels; tail_level++) { + out->level_offsets[tail_level] = out->level_offsets[level]; + out->level_sizes[tail_level] = 4096u; + } + break; + } + + out->level_sizes[level] = static_cast(block_depth) * + AlignUpU32(row_elements, block_width) * + AlignUpU32(height_elements, block_height) * bytes_per_element; + out->block_slice_size += out->level_sizes[level]; + } + + return out->block_slice_size != 0; +} + +uint32_t TextureGetAtlasSliceYStride(VkFormat format, uint32_t mip_height, uint32_t depth, + uint64_t levels) { + return (depth > 1 && levels > 1 && UtilIsBcFormat(format) ? AlignUpU32(mip_height, 4u) + : mip_height); +} + +uint32_t TextureCalcStackedImageHeight(VkFormat format, uint32_t height, uint32_t depth, + uint64_t levels) { + auto image_height = height * depth; + if (depth <= 1 || levels <= 1 || !UtilIsBcFormat(format)) { + return image_height; + } + + uint32_t mip_height = height; + for (uint32_t level = 0; level < levels; level++) { + const auto mip_image_height = + TextureGetAtlasSliceYStride(format, mip_height, depth, levels) * depth; + image_height = std::max(image_height, mip_image_height << level); + if (mip_height > 1) { + mip_height /= 2; + } + } + + return image_height; +} + +uint32_t TextureCalcMipmapAtlasImageHeight(VkFormat format, uint32_t width, uint32_t height, + uint32_t depth, uint64_t levels) { + auto image_height = height * depth; + if (levels <= 1) { + return image_height; + } + + uint32_t mip_height = height; + for (uint32_t level = 0; level < levels; level++) { + const auto mipmap_offset = UtilCalcMipmapOffset(level, width, height); + const auto mip_bottom = + static_cast(mipmap_offset.second) + + TextureGetAtlasSliceYStride(format, mip_height, depth, levels) * depth; + image_height = std::max(image_height, mip_bottom); + if (mip_height > 1) { + mip_height /= 2; + } + } + + return image_height; +} + +bool TextureIs3DTexture(uint64_t type) { + return static_cast(type) == Prospero::ImageType::kColor3D; +} + +bool TextureIsCubeTexture(uint64_t type) { + return static_cast(type) == Prospero::ImageType::kCube; +} + +bool TextureIsLayeredTexture(uint64_t type) { + const auto image_type = static_cast(type); + return image_type == Prospero::ImageType::kCube || image_type == Prospero::ImageType::kColor1DArray || + image_type == Prospero::ImageType::kColor2DArray || + image_type == Prospero::ImageType::kColor2DMsaaArray; +} + +bool TextureCanCreateCubeView(uint64_t type, uint32_t base_array, uint32_t layer_count) { + return TextureIsCubeTexture(type) && base_array % 6u == 0 && layer_count >= 6 && + layer_count % 6u == 0; +} + +VkComponentMapping TextureCreateImage(GraphicContext* ctx, VulkanImage* vk_obj, VulkanMemory* mem, + const TextureImageCreateParams& params) { + EXIT_IF(ctx == nullptr); + EXIT_IF(vk_obj == nullptr); + EXIT_IF(mem == nullptr); + EXIT_IF(params.owner == nullptr); + + const bool array_texture = TextureIsLayeredTexture(params.type); + const bool volume_texture = TextureIs3DTexture(params.type); + + auto pixel_format = TextureGetFormat(params.fmt, params.format_usage); + EXIT_NOT_IMPLEMENTED(pixel_format == VK_FORMAT_UNDEFINED); + EXIT_NOT_IMPLEMENTED(params.width == 0); + EXIT_NOT_IMPLEMENTED(params.height == 0); + EXIT_NOT_IMPLEMENTED(params.levels == 0 || params.levels > 16); + + uint32_t image_height = 0; + uint32_t image_mips = 0; + if (params.image_layout == TextureUploadDestination::MipAtlas) { + EXIT_NOT_IMPLEMENTED(params.base_level != 0); + const auto atlas_depth = (array_texture || volume_texture ? 1u : params.depth); + image_height = TextureCalcMipmapAtlasImageHeight( + pixel_format, static_cast(params.width), static_cast(params.height), + atlas_depth, params.levels); + image_mips = 1; + } else { + image_height = + (array_texture || volume_texture + ? static_cast(params.height) + : TextureCalcStackedImageHeight(pixel_format, static_cast(params.height), + params.depth, params.levels)); + image_mips = static_cast(params.levels); + } + + VkComponentMapping components = + TextureGetComponentMapping(static_cast(params.swizzle)); + + VkImageCreateInfo image_info {}; + image_info.sType = VK_STRUCTURE_TYPE_IMAGE_CREATE_INFO; + image_info.pNext = nullptr; + image_info.flags = + (params.allow_cube_view && TextureCanCreateCubeView(params.type, 0, params.depth) + ? VK_IMAGE_CREATE_CUBE_COMPATIBLE_BIT + : 0) | + (volume_texture ? VK_IMAGE_CREATE_2D_ARRAY_COMPATIBLE_BIT : 0) | + (params.compatible_format_views ? VK_IMAGE_CREATE_MUTABLE_FORMAT_BIT : 0); + image_info.imageType = (volume_texture ? VK_IMAGE_TYPE_3D : VK_IMAGE_TYPE_2D); + image_info.extent.width = static_cast(params.width); + image_info.extent.height = image_height; + image_info.extent.depth = (volume_texture ? params.depth : 1); + image_info.mipLevels = image_mips; + image_info.arrayLayers = (array_texture ? params.depth : 1); + image_info.format = pixel_format; + image_info.tiling = VK_IMAGE_TILING_OPTIMAL; + image_info.initialLayout = VK_IMAGE_LAYOUT_UNDEFINED; + image_info.usage = TextureGetUsage(params.format_usage); + image_info.sharingMode = VK_SHARING_MODE_EXCLUSIVE; + image_info.samples = VK_SAMPLE_COUNT_1_BIT; + + const bool storage_view = TextureHasFormatUsage(params.view_usage, TextureFormatUsage::Storage); + if (storage_view && !TextureCheckStorageSwizzle(&image_info, &components)) { + if (!params.storage_swizzle_fallback) { + EXIT("swizzle is not supported"); + } + + static std::atomic_uint log_count {0}; + if (log_count.fetch_add(1, std::memory_order_relaxed) < 16) { + LOGF("\t %s swizzle 0x%08" PRIx64 " is not supported, using identity mapping\n", + params.owner, params.swizzle); + } + components.r = VK_COMPONENT_SWIZZLE_R; + components.g = VK_COMPONENT_SWIZZLE_G; + components.b = VK_COMPONENT_SWIZZLE_B; + components.a = VK_COMPONENT_SWIZZLE_A; + } + + const auto view_checked_format = image_info.format; + const auto requested_usage = params.format_usage; + const auto required_usage = params.required_format_usage; + const bool has_optional_usage = static_cast(requested_usage & ~required_usage) != 0; + if (!TextureCheckFormatExact(ctx, image_info) && has_optional_usage) { + auto required_info = image_info; + required_info.usage = TextureGetUsage(required_usage); + required_info.format = view_checked_format; + if (TextureCheckFormatExact(ctx, required_info)) { + static std::atomic_uint log_count {0}; + if (log_count.fetch_add(1, std::memory_order_relaxed) < 16) { + LOGF("\t %s usage 0x%08x is not supported for format %d, using required " + "usage 0x%08x\n", + params.owner, static_cast(TextureGetUsage(requested_usage)), + static_cast(image_info.format), + static_cast(TextureGetUsage(required_usage))); + } + image_info = required_info; + } + } + + if (!TextureCheckFormat(ctx, &image_info)) { + if (has_optional_usage) { + image_info.format = view_checked_format; + image_info.usage = TextureGetUsage(required_usage); + } + if (!has_optional_usage || !TextureCheckFormat(ctx, &image_info)) { + EXIT("format is not supported"); + } + } + + vk_obj->extent.width = image_info.extent.width; + vk_obj->extent.height = image_info.extent.height; + vk_obj->layers = image_info.arrayLayers; + vk_obj->mip_levels = image_info.mipLevels; + vk_obj->format = image_info.format; + vk_obj->image = nullptr; + vk_obj->layout = image_info.initialLayout; + + UtilResetImageViews(vk_obj); + + mem->property = VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT; + + bool created = VulkanCreateImage(ctx, &image_info, vk_obj, mem); + EXIT_NOT_IMPLEMENTED(!created); + + vk_obj->memory = *mem; + + return components; +} + +void TextureCreateImageViews(GraphicContext* ctx, VulkanImage* vk_obj, + VkComponentMapping components, uint64_t type, uint32_t base_array, + uint32_t base_level, uint32_t level_count, uint32_t depth, + bool allow_cube_view, TextureFormatUsage view_usage) { + EXIT_IF(ctx == nullptr); + EXIT_IF(vk_obj == nullptr); + EXIT_IF(level_count == 0 || base_level + level_count > vk_obj->mip_levels); + + const bool layered_texture = TextureIsLayeredTexture(type); + const bool volume_texture = TextureIs3DTexture(type); + const auto layer_count = (layered_texture && base_array < depth ? depth - base_array : 1u); + const auto volume_slices = std::max(depth >> base_level, 1u); + + VkImageViewUsageCreateInfo usage_info {}; + usage_info.sType = VK_STRUCTURE_TYPE_IMAGE_VIEW_USAGE_CREATE_INFO; + usage_info.pNext = nullptr; + usage_info.usage = TextureGetViewUsage(view_usage); + + VkImageViewCreateInfo create_info {}; + create_info.sType = VK_STRUCTURE_TYPE_IMAGE_VIEW_CREATE_INFO; + create_info.pNext = (usage_info.usage != 0 ? &usage_info : nullptr); + create_info.flags = 0; + create_info.image = vk_obj->image; + create_info.viewType = VK_IMAGE_VIEW_TYPE_2D; + create_info.format = vk_obj->format; + create_info.components = components; + create_info.subresourceRange.aspectMask = VK_IMAGE_ASPECT_COLOR_BIT; + create_info.subresourceRange.baseArrayLayer = (layered_texture ? base_array : 0); + create_info.subresourceRange.baseMipLevel = base_level; + create_info.subresourceRange.layerCount = layer_count; + create_info.subresourceRange.levelCount = level_count; + if (volume_texture) { + create_info.viewType = VK_IMAGE_VIEW_TYPE_3D; + create_info.subresourceRange.baseArrayLayer = 0; + create_info.subresourceRange.layerCount = 1; + } else if (allow_cube_view && TextureCanCreateCubeView(type, base_array, layer_count)) { + create_info.viewType = + (layer_count > 6 ? VK_IMAGE_VIEW_TYPE_CUBE_ARRAY : VK_IMAGE_VIEW_TYPE_CUBE); + } else if (layered_texture) { + create_info.viewType = VK_IMAGE_VIEW_TYPE_2D_ARRAY; + } + + vkCreateImageView(ctx->device, &create_info, nullptr, + &vk_obj->image_view[VulkanImage::VIEW_DEFAULT]); + EXIT_NOT_IMPLEMENTED(vk_obj->image_view[VulkanImage::VIEW_DEFAULT] == nullptr); + + if (!volume_texture || depth >= 1) { + create_info.viewType = VK_IMAGE_VIEW_TYPE_2D_ARRAY; + create_info.subresourceRange.layerCount = (volume_texture ? volume_slices : layer_count); + vkCreateImageView(ctx->device, &create_info, nullptr, + &vk_obj->image_view[VulkanImage::VIEW_DEFAULT_ARRAY]); + EXIT_NOT_IMPLEMENTED(vk_obj->image_view[VulkanImage::VIEW_DEFAULT_ARRAY] == nullptr); + } +} + +static uint64_t CalcTextureSliceStride(const TileSizeOffset* level_sizes, uint64_t levels, + uint64_t total_size, uint32_t depth) { + uint64_t stride = 0; + for (uint32_t i = 0; i < levels; i++) { + stride = + std::max(stride, static_cast(level_sizes[i].offset) + level_sizes[i].size); + } + + if (depth > 1 && total_size != 0 && total_size % depth == 0) { + const auto guest_stride = total_size / depth; + if (guest_stride >= stride) { + stride = guest_stride; + } + } + + return stride; +} + +static uint64_t CalcLinearUploadLevelSize(uint32_t fmt, uint32_t pitch, uint32_t height) { + if (const uint32_t bytes_per_element = Prospero::NumBytesPerElement(fmt); bytes_per_element != 0) { + return static_cast(pitch) * height * bytes_per_element; + } + + if (const uint32_t bytes_per_block = Prospero::BlockCompressedBytesPerBlock(fmt); bytes_per_block != 0) { + const uint32_t blocks_w = std::max((pitch + 3u) / 4u, 1u); + const uint32_t blocks_h = std::max((height + 3u) / 4u, 1u); + return static_cast(blocks_w) * blocks_h * bytes_per_block; + } + + return 0; +} + +static uint64_t FillVolumeLinearUploadLevels(TileSizeOffset* level_sizes, uint32_t fmt, + uint64_t height, uint64_t levels, + uint32_t base_pitch) { + uint64_t offset = 0; + auto pitch = base_pitch; + auto h = static_cast(height); + + for (uint32_t i = 0; i < levels; i++) { + const auto size = CalcLinearUploadLevelSize(fmt, pitch, h); + EXIT_NOT_IMPLEMENTED(size > 0xffffffffull); + EXIT_NOT_IMPLEMENTED(offset > 0xffffffffull); + + level_sizes[i].size = static_cast(size); + level_sizes[i].offset = static_cast(offset); + level_sizes[i].src_size = 0; + level_sizes[i].src_offset = 0; + level_sizes[i].x = 0; + level_sizes[i].y = 0; + + offset += size; + if (pitch > 1) { + pitch /= 2; + } + if (h > 1) { + h /= 2; + } + } + + return offset; +} + +TextureUploadLayout TextureCalcUploadLayout(uint32_t fmt, uint64_t width, uint64_t height, + uint64_t levels, uint32_t depth, uint64_t pitch, + uint64_t tile, uint64_t upload_size, + bool allow_depth_tile, + bool require_single_mip_small_tiles, + bool volume_texture, const char* owner) { + TextureUploadLayout layout {}; + layout.tile = static_cast(tile); + layout.pitch = static_cast(pitch); + layout.volume_texture = volume_texture; + + if (fmt != 0) { + if (layout.tile != 0) { + const auto tile_mode = static_cast(layout.tile); + layout.fmt_tiled_render_target = + (Prospero::RenderTargetBytesPerElement(static_cast(fmt)) != 0 && + tile_mode == Prospero::TileMode::kRenderTarget); + layout.fmt_tiled_standard256b = + (TileIsStandard256BTextureSupported(static_cast(fmt)) && + tile_mode == Prospero::TileMode::kStandard256B && + (!require_single_mip_small_tiles || levels == 1)); + layout.fmt_tiled_standard4kb = + (TileIsStandard4KBTextureSupported(static_cast(fmt)) && + tile_mode == Prospero::TileMode::kStandard4KB && + (!require_single_mip_small_tiles || levels == 1)); + layout.fmt_tiled_standard64kb = + (TileIsStandard64KBTextureSupported(static_cast(fmt)) && + tile_mode == Prospero::TileMode::kStandard64KB); + layout.fmt_tiled_depth = + (allow_depth_tile && Prospero::RenderTargetBytesPerElement(static_cast(fmt)) != 0 && + tile_mode == Prospero::TileMode::kDepth); + if (!layout.fmt_tiled_render_target && !layout.fmt_tiled_standard256b && + !layout.fmt_tiled_standard4kb && !layout.fmt_tiled_standard64kb && + !layout.fmt_tiled_depth) { + EXIT("%s: unsupported typed tiled upload, using linear fallback: fmt=%u tile=%u " + "size=%" PRIu64 " extent=%" PRIu64 "x%" PRIu64 " pitch=%" PRIu64 + " levels=%" PRIu64 "\n", + owner, static_cast(fmt), layout.tile, upload_size, width, height, + pitch, levels); + layout.tile = 0; + } + } + + layout.pitch = TileGetTexturePitch(fmt, width, levels, layout.tile); + + TileGetTextureSize(fmt, width, height, layout.pitch, levels, layout.tile, nullptr, + layout.level_sizes, layout.padded_sizes); + + if (layout.volume_texture) { + TileSizeOffset source_levels[16] {}; + std::copy_n(layout.level_sizes, levels, source_levels); + layout.slice_stride = + FillVolumeLinearUploadLevels(layout.level_sizes, fmt, height, levels, layout.pitch); + if (layout.fmt_tiled_render_target) { + layout.source_slice_stride = + CalcTextureSliceStride(source_levels, levels, upload_size, depth); + for (uint32_t level = 0; level < levels; level++) { + layout.level_sizes[level].src_offset = source_levels[level].offset; + layout.level_sizes[level].src_size = source_levels[level].size; + layout.level_sizes[level].x = source_levels[level].x; + layout.level_sizes[level].y = source_levels[level].y; + } + } + } + + if (layout.fmt_tiled_depth) { + for (uint32_t i = 0; i < levels; i++) { + layout.level_sizes[i].x = layout.padded_sizes[i].width; + layout.level_sizes[i].y = layout.padded_sizes[i].height; + } + } + } else { + EXIT("%s: legacy texture upload format unsupported: fmt=0 tile=%u size=%" PRIu64 + " extent=%" PRIu64 "x%" PRIu64 " pitch=%" PRIu64 " levels=%" PRIu64 "\n", + owner, layout.tile, upload_size, width, height, pitch, levels); + } + + if (!layout.volume_texture) { + layout.slice_stride = + CalcTextureSliceStride(layout.level_sizes, levels, upload_size, depth); + } + return layout; +} + +std::vector +TextureBuildUploadRegions(const TextureUploadLayout& layout, VkFormat image_format, uint32_t width, + uint32_t height, uint32_t depth, uint64_t levels, bool array_texture, + bool volume_texture, TextureUploadDestination destination, + TextureUploadSliceLayout slice_layout) { + uint32_t mip_width = width; + uint32_t mip_height = height; + uint32_t mip_pitch = layout.pitch; + + std::vector regions(levels * depth); + for (uint32_t i = 0; i < levels; i++) { + EXIT_NOT_IMPLEMENTED(layout.level_sizes[i].size == 0); + + const auto mipmap_offset = UtilCalcMipmapOffset(i, width, height); + + for (uint32_t z = 0; z < depth; z++) { + const auto region_index = i * depth + z; + const auto slice_offset = (slice_layout == TextureUploadSliceLayout::MipChainPerSlice + ? z * layout.slice_stride + : z * static_cast(layout.level_sizes[i].size)); + + regions[region_index].offset = + static_cast(layout.level_sizes[i].offset + slice_offset); + regions[region_index].width = mip_width; + regions[region_index].height = mip_height; + regions[region_index].copy_height = + (!array_texture && !volume_texture && depth > 1 && levels > 1 && + UtilIsBcFormat(image_format) + ? TextureGetAtlasSliceYStride(image_format, mip_height, depth, levels) + : 0); + regions[region_index].dst_layer = (array_texture ? z : 0); + regions[region_index].dst_z = (volume_texture ? static_cast(z) : 0); + if (layout.fmt_tiled_depth && layout.level_sizes[i].x != 0) { + regions[region_index].pitch = layout.level_sizes[i].x; + } else if (!layout.volume_texture && + static_cast(layout.tile) == Prospero::TileMode::kLinear && + layout.padded_sizes[i].width != 0) { + regions[region_index].pitch = layout.padded_sizes[i].width; + } else { + regions[region_index].pitch = mip_pitch; + } + + if (destination == TextureUploadDestination::MipLevels) { + regions[region_index].dst_level = i; + regions[region_index].dst_x = 0; + regions[region_index].dst_y = + (array_texture || volume_texture + ? 0 + : static_cast(z * TextureGetAtlasSliceYStride( + image_format, mip_height, depth, levels))); + } else { + regions[region_index].dst_level = 0; + regions[region_index].dst_x = mipmap_offset.first; + regions[region_index].dst_y = + (array_texture || volume_texture + ? mipmap_offset.second + : mipmap_offset.second + + static_cast(z * TextureGetAtlasSliceYStride( + image_format, mip_height, depth, levels))); + } + } + + if (mip_width > 1) { + mip_width /= 2; + } + if (mip_height > 1) { + mip_height /= 2; + } + if (mip_pitch > 1) { + mip_pitch /= 2; + } + } + + return regions; +} + +static uint64_t GetLevelSrcOffset(const TileSizeOffset& level_size) { + return (level_size.src_size != 0 ? level_size.src_offset : level_size.offset); +} + +static uint64_t GetLevelSrcSize(const TileSizeOffset& level_size) { + return (level_size.src_size != 0 ? level_size.src_size : level_size.size); +} + +static uint64_t GetSliceSrcStride(const TextureUploadLayout& layout, uint32_t level, + TextureUploadSliceLayout source_slice_layout) { + return ( + source_slice_layout == TextureUploadSliceLayout::MipChainPerSlice + ? (layout.source_slice_stride != 0 ? layout.source_slice_stride : layout.slice_stride) + : GetLevelSrcSize(layout.level_sizes[level])); +} + +static uint64_t FmaskRegionCopySize(const BufferImageCopy& region) { + const uint64_t row_length = (region.pitch != 0 ? region.pitch : region.width); + if (region.height == 0 || region.width == 0) { + return 0; + } + return ((static_cast(region.height - 1u) * row_length) + region.width) * + sizeof(uint32_t); +} + +static void UploadFmaskIdentity(GraphicContext* ctx, VulkanImage* vk_obj, + const std::vector& regions, uint64_t dst_layout, + const char* owner) { + constexpr uint32_t kIdentityFmaskPattern = 0x76543210u; + + std::vector upload_regions = regions; + uint64_t upload_size = 0; + for (auto& region: upload_regions) { + upload_size = (upload_size + 255u) & ~uint64_t {255}; + EXIT_NOT_IMPLEMENTED(upload_size > UINT32_MAX); + region.offset = static_cast(upload_size); + upload_size += FmaskRegionCopySize(region); + } + + LOGF("%s: temporary: decoding PS5 FMASK8_S4_F4 as identity pattern 0x%08" PRIx32 + ", upload_size=%" PRIu64 " regions=%zu\n", + owner, kIdentityFmaskPattern, upload_size, upload_regions.size()); + + UtilScratchBuffer temp_buf(upload_size); + auto* words = static_cast(temp_buf.Data()); + for (uint64_t i = 0; i < upload_size / sizeof(uint32_t); i++) { + words[i] = kIdentityFmaskPattern; + } + + UtilFillImage(ctx, vk_obj, temp_buf.Data(), upload_size, upload_regions, dst_layout); +} + +void TextureUploadGuestImage(GraphicContext* ctx, VulkanImage* vk_obj, const void* src_data, + uint64_t size, const std::vector& regions, + const TextureUploadLayout& layout, uint32_t fmt, uint64_t width, + uint64_t height, uint32_t depth, uint64_t levels, + TextureUploadSliceLayout source_slice_layout, const char* owner, + uint64_t dst_layout) { + if (fmt == 0) { + EXIT("%s: legacy texture upload format unsupported: fmt=0 tile=%u size=%" PRIu64 + " extent=%" PRIu64 "x%" PRIu64 " depth=%u pitch=%u levels=%" PRIu64 "\n", + owner, layout.tile, size, width, height, depth, layout.pitch, levels); + } else if (static_cast(layout.tile) == Prospero::TileMode::kLinear) { + UtilFillImage(ctx, vk_obj, src_data, size, regions, dst_layout); + } else if (layout.fmt_tiled_render_target) { + LOGF("%s: detiling typed render-target texture: fmt=%u tile=%u size=%" PRIu64 + " extent=%" PRIu64 "x%" PRIu64 " depth=%" PRIu64 " pitch=%u levels=%" PRIu64 "\n", + owner, static_cast(fmt), layout.tile, size, width, height, + static_cast(depth), layout.pitch, levels); + UtilScratchBuffer temp_buf(size); + for (uint32_t i = 0; i < levels; i++) { + for (uint32_t z = 0; z < depth; z++) { + const auto region_index = i * depth + z; + auto* dst = static_cast(temp_buf.Data()) + regions[region_index].offset; + const auto* src = static_cast(src_data) + + TextureUploadSliceSourceOffset(layout, i, z, source_slice_layout); + TileConvertTiledToLinearRenderTarget( + dst, src, regions[region_index].width, regions[region_index].height, + regions[region_index].pitch, + Prospero::RenderTargetBytesPerElement(static_cast(fmt)), + layout.level_sizes[i].size, GetLevelSrcSize(layout.level_sizes[i]), + layout.level_sizes[i].x, layout.level_sizes[i].y); + } + } + UtilFillImage(ctx, vk_obj, temp_buf.Data(), size, regions, dst_layout); + } else if (layout.fmt_tiled_depth) { + if (Prospero::IsFmaskTextureFormat(static_cast(fmt))) { + UploadFmaskIdentity(ctx, vk_obj, regions, dst_layout, owner); + return; + } + const uint32_t bytes_per_element = Prospero::RenderTargetBytesPerElement(static_cast(fmt)); + if (bytes_per_element == 1) { + UtilFillImage(ctx, vk_obj, src_data, size, regions, dst_layout); + } else { + LOGF("%s: detiling typed depth texture: fmt=%u tile=%u size=%" PRIu64 " extent=%" PRIu64 + "x%" PRIu64 " depth=%" PRIu64 " pitch=%u levels=%" PRIu64 "\n", + owner, static_cast(fmt), layout.tile, size, width, height, + static_cast(depth), layout.pitch, levels); + UtilScratchBuffer temp_buf(size); + for (uint32_t i = 0; i < levels; i++) { + for (uint32_t z = 0; z < depth; z++) { + const auto region_index = i * depth + z; + auto* dst = + static_cast(temp_buf.Data()) + regions[region_index].offset; + const auto* src = static_cast(src_data) + + GetLevelSrcOffset(layout.level_sizes[i]) + + z * GetSliceSrcStride(layout, i, source_slice_layout); + TileConvertTiledToLinearDepth( + dst, src, static_cast(fmt), regions[region_index].width, + regions[region_index].height, regions[region_index].pitch, + layout.level_sizes[i].size); + } + } + UtilFillImage(ctx, vk_obj, temp_buf.Data(), size, regions, dst_layout); + } + } else if (layout.fmt_tiled_standard256b) { + UtilScratchBuffer temp_buf(size); + for (uint32_t i = 0; i < levels; i++) { + for (uint32_t z = 0; z < depth; z++) { + const auto region_index = i * depth + z; + auto* dst = static_cast(temp_buf.Data()) + regions[region_index].offset; + const auto* src = static_cast(src_data) + + layout.level_sizes[i].offset + + z * GetSliceSrcStride(layout, i, source_slice_layout); + TileConvertTiledToLinearStandard256B( + dst, src, static_cast(fmt), regions[region_index].width, + regions[region_index].height, regions[region_index].pitch, + layout.level_sizes[i].size, layout.level_sizes[i].size); + } + } + UtilFillImage(ctx, vk_obj, temp_buf.Data(), size, regions, dst_layout); + } else if (layout.fmt_tiled_standard4kb && layout.volume_texture) { + LOGF("%s: detiling typed Standard4KB 3D texture: fmt=%u tile=%u size=%" PRIu64 + " extent=%" PRIu64 "x%" PRIu64 " depth=%" PRIu64 " pitch=%u levels=%" PRIu64 "\n", + owner, static_cast(fmt), layout.tile, size, width, height, + static_cast(depth), layout.pitch, levels); + UtilScratchBuffer temp_buf(size); + if (levels == 1) { + TileConvertTiledToLinearStandard4KB3D( + temp_buf.Data(), src_data, static_cast(fmt), static_cast(width), + static_cast(height), depth, layout.pitch, layout.slice_stride, size, + size); + } else { + std::memset(temp_buf.Data(), 0, static_cast(size)); + + Standard4KBVolumeMipLayout volume_layout {}; + EXIT_NOT_IMPLEMENTED(!CalcStandard4kbVolumeMipLayout( + static_cast(fmt), layout.pitch, static_cast(height), levels, + &volume_layout)); + + auto* dst_base = static_cast(temp_buf.Data()); + const auto* src_base = static_cast(src_data); + uint32_t mip_width = static_cast(width); + uint32_t mip_height = static_cast(height); + uint32_t mip_pitch = layout.pitch; + + for (uint32_t level = 0; level < levels; level++) { + if (level < volume_layout.first_tail_level) { + for (uint32_t z = 0; z < depth; z += volume_layout.block_depth) { + const uint32_t copy_depth = std::min(volume_layout.block_depth, depth - z); + const auto region_index = level * depth + z; + auto* dst = dst_base + regions[region_index].offset; + const auto* src = src_base + + (static_cast(z / volume_layout.block_depth) * + volume_layout.block_slice_size) + + volume_layout.level_offsets[level]; + const uint64_t dst_size = + (static_cast(copy_depth - 1u) * layout.slice_stride) + + layout.level_sizes[level].size; + TileConvertTiledToLinearStandard4KB3D( + dst, src, static_cast(fmt), mip_width, mip_height, copy_depth, + mip_pitch, layout.slice_stride, dst_size, + volume_layout.level_sizes[level], false); + } + } + + if (mip_width > 1) { + mip_width /= 2; + } + if (mip_height > 1) { + mip_height /= 2; + } + if (mip_pitch > 1) { + mip_pitch /= 2; + } + } + } + UtilFillImage(ctx, vk_obj, temp_buf.Data(), size, regions, dst_layout); + } else if (layout.fmt_tiled_standard4kb) { + LOGF("%s: detiling typed Standard4KB texture: fmt=%u tile=%u size=%" PRIu64 + " extent=%" PRIu64 "x%" PRIu64 " depth=%" PRIu64 " pitch=%u\n", + owner, static_cast(fmt), layout.tile, size, width, height, + static_cast(depth), layout.pitch); + UtilScratchBuffer temp_buf(size); + for (uint32_t i = 0; i < levels; i++) { + for (uint32_t z = 0; z < depth; z++) { + const auto region_index = i * depth + z; + auto* dst = static_cast(temp_buf.Data()) + regions[region_index].offset; + const auto* src = static_cast(src_data) + + GetLevelSrcOffset(layout.level_sizes[i]) + + z * GetSliceSrcStride(layout, i, source_slice_layout); + TileConvertTiledToLinearStandard4KB( + dst, src, static_cast(fmt), regions[region_index].width, + regions[region_index].height, regions[region_index].pitch, + layout.level_sizes[i].size, GetLevelSrcSize(layout.level_sizes[i]), + layout.level_sizes[i].x, layout.level_sizes[i].y); + } + } + UtilFillImage(ctx, vk_obj, temp_buf.Data(), size, regions, dst_layout); + } else if (layout.fmt_tiled_standard64kb) { + LOGF("%s: detiling typed Standard64KB texture: fmt=%u tile=%u size=%" PRIu64 + " extent=%" PRIu64 "x%" PRIu64 " depth=%" PRIu64 " pitch=%u\n", + owner, static_cast(fmt), layout.tile, size, width, height, + static_cast(depth), layout.pitch); + UtilScratchBuffer temp_buf(size); + for (uint32_t i = 0; i < levels; i++) { + for (uint32_t z = 0; z < depth; z++) { + const auto region_index = i * depth + z; + auto* dst = static_cast(temp_buf.Data()) + regions[region_index].offset; + const auto* src = static_cast(src_data) + + GetLevelSrcOffset(layout.level_sizes[i]) + + z * GetSliceSrcStride(layout, i, source_slice_layout); + TileConvertTiledToLinearStandard64KB( + dst, src, static_cast(fmt), regions[region_index].width, + regions[region_index].height, regions[region_index].pitch, + layout.level_sizes[i].size, GetLevelSrcSize(layout.level_sizes[i]), + layout.level_sizes[i].x, layout.level_sizes[i].y); + } + } + UtilFillImage(ctx, vk_obj, temp_buf.Data(), size, regions, dst_layout); + } else if (layout.tile != 0) { + EXIT("%s: typed tiled upload still unsupported after sizing, using linear fallback: fmt=%u " + "tile=%u size=%" PRIu64 " extent=%" PRIu64 "x%" PRIu64 " pitch=%u levels=%" PRIu64 + "\n", + owner, static_cast(fmt), layout.tile, size, width, height, layout.pitch, + levels); + UtilFillImage(ctx, vk_obj, src_data, size, regions, dst_layout); + } +} + +} // namespace Libs::Graphics diff --git a/src/graphics/host_gpu/objects/textureCommon.h b/src/graphics/host_gpu/objects/textureCommon.h new file mode 100644 index 000000000..0e7d713f9 --- /dev/null +++ b/src/graphics/host_gpu/objects/textureCommon.h @@ -0,0 +1,140 @@ +#ifndef EMULATOR_INCLUDE_EMULATOR_GRAPHICS_OBJECTS_TEXTURECOMMON_H_ +#define EMULATOR_INCLUDE_EMULATOR_GRAPHICS_OBJECTS_TEXTURECOMMON_H_ + +#include "common/abi.h" +#include "common/common.h" +#include "graphics/guest_gpu/tile.h" +#include "graphics/host_gpu/utils.h" + +#include +#include + +namespace Libs::Graphics { + +struct GraphicContext; +struct VulkanImage; +struct VulkanMemory; + +enum class TextureFormatUsage : uint32_t { + None = 0, + Sampled = 1u << 0u, + Storage = 1u << 1u, +}; + +constexpr TextureFormatUsage operator|(TextureFormatUsage lhs, TextureFormatUsage rhs) { + return static_cast(static_cast(lhs) | static_cast(rhs)); +} + +constexpr TextureFormatUsage operator&(TextureFormatUsage lhs, TextureFormatUsage rhs) { + return static_cast(static_cast(lhs) & static_cast(rhs)); +} + +constexpr TextureFormatUsage operator~(TextureFormatUsage usage) { + return static_cast(~static_cast(usage)); +} + +constexpr TextureFormatUsage& operator|=(TextureFormatUsage& lhs, TextureFormatUsage rhs) { + lhs = lhs | rhs; + return lhs; +} + +constexpr bool TextureHasFormatUsage(TextureFormatUsage usage, TextureFormatUsage flag) { + return (static_cast(usage & flag) == static_cast(flag)); +} + +enum class TextureUploadDestination { MipLevels, MipAtlas }; + +enum class TextureUploadSliceLayout { MipChainPerSlice, MipLevelPerSlice }; + +struct RenderTargetFormatInfo { + VkFormat format = VK_FORMAT_UNDEFINED; + uint32_t bytes_per_element = 0; +}; + +struct TextureUploadLayout { + uint32_t tile = 0; + uint32_t pitch = 0; + uint64_t slice_stride = 0; + uint64_t source_slice_stride = 0; + bool fmt_tiled_render_target = false; + bool fmt_tiled_standard256b = false; + bool fmt_tiled_standard4kb = false; + bool fmt_tiled_standard64kb = false; + bool fmt_tiled_depth = false; + bool volume_texture = false; + TileSizeOffset level_sizes[16] = {}; + TilePaddedSize padded_sizes[16] = {}; +}; + +struct TextureImageCreateParams { + uint32_t fmt = 0; + uint64_t width = 0; + uint64_t height = 0; + uint32_t base_level = 0; + uint64_t levels = 1; + uint32_t depth = 1; + uint64_t type = 0; + uint64_t swizzle = 0; + TextureFormatUsage format_usage = TextureFormatUsage::Sampled; + TextureFormatUsage required_format_usage = TextureFormatUsage::Sampled; + TextureFormatUsage view_usage = TextureFormatUsage::Sampled; + TextureUploadDestination image_layout = TextureUploadDestination::MipLevels; + bool allow_cube_view = false; + bool compatible_format_views = false; + bool storage_swizzle_fallback = false; + const char* owner = nullptr; +}; + +VkComponentSwizzle TextureGetComponentSwizzle(uint8_t s); +VkComponentMapping TextureGetComponentMapping(uint32_t swizzle); +bool TextureCheckFormat(GraphicContext* ctx, VkImageCreateInfo* image_info); +bool TextureCheckStorageSwizzle(VkImageCreateInfo* image_info, VkComponentMapping* components); +VkImageUsageFlags TextureGetUsage(TextureFormatUsage usage); +VkImageUsageFlags TextureGetViewUsage(TextureFormatUsage usage); +VkFormat TextureGetFormat(uint32_t fmt, TextureFormatUsage usage); +RenderTargetFormatInfo TextureGetRenderTargetFormat(uint32_t layout, uint32_t type, uint32_t order); +uint32_t TextureGetAtlasSliceYStride(VkFormat format, uint32_t mip_height, uint32_t depth, + uint64_t levels); +uint32_t TextureCalcStackedImageHeight(VkFormat format, uint32_t height, uint32_t depth, + uint64_t levels); +uint32_t TextureCalcMipmapAtlasImageHeight(VkFormat format, uint32_t width, uint32_t height, + uint32_t depth, uint64_t levels); +bool TextureIs3DTexture(uint64_t type); +bool TextureIsCubeTexture(uint64_t type); +bool TextureIsLayeredTexture(uint64_t type); +bool TextureCanCreateCubeView(uint64_t type, uint32_t base_array, uint32_t layer_count); +VkComponentMapping TextureCreateImage(GraphicContext* ctx, VulkanImage* vk_obj, VulkanMemory* mem, + const TextureImageCreateParams& params); +void TextureCreateImageViews(GraphicContext* ctx, VulkanImage* vk_obj, + VkComponentMapping components, uint64_t type, uint32_t base_array, + uint32_t base_level, uint32_t level_count, uint32_t depth, + bool allow_cube_view, TextureFormatUsage view_usage); +TextureUploadLayout TextureCalcUploadLayout(uint32_t fmt, uint64_t width, uint64_t height, + uint64_t levels, uint32_t depth, uint64_t pitch, + uint64_t tile, uint64_t upload_size, + bool allow_depth_tile, + bool require_single_mip_small_tiles, + bool volume_texture, const char* owner); +uint64_t TextureUploadSliceSourceOffset(const TextureUploadLayout& layout, uint32_t level, + uint32_t slice, + TextureUploadSliceLayout source_slice_layout); +uint64_t TextureCalcUploadSize(const TextureUploadLayout& layout, + const std::vector& regions, uint64_t levels, + uint32_t depth, TextureUploadSliceLayout source_slice_layout); +std::vector +TextureBuildUploadRegions(const TextureUploadLayout& layout, VkFormat image_format, uint32_t width, + uint32_t height, uint32_t depth, uint64_t levels, bool array_texture, + bool volume_texture, TextureUploadDestination destination, + TextureUploadSliceLayout slice_layout); +void TextureCopyBufferBytes(GraphicContext* ctx, VulkanBuffer* src_buffer, + uint64_t src_buffer_offset, uint64_t copy_size, UtilScratchBuffer* dst); +void TextureUploadGuestImage(GraphicContext* ctx, VulkanImage* vk_obj, const void* src_data, + uint64_t size, const std::vector& regions, + const TextureUploadLayout& layout, uint32_t fmt, uint64_t width, + uint64_t height, uint32_t depth, uint64_t levels, + TextureUploadSliceLayout source_slice_layout, const char* owner, + uint64_t dst_layout); + +} // namespace Libs::Graphics + +#endif /* EMULATOR_INCLUDE_EMULATOR_GRAPHICS_OBJECTS_TEXTURECOMMON_H_ */ diff --git a/src/graphics/host_gpu/pageManager.cpp b/src/graphics/host_gpu/pageManager.cpp new file mode 100644 index 000000000..3240bd61b --- /dev/null +++ b/src/graphics/host_gpu/pageManager.cpp @@ -0,0 +1,647 @@ +#include "graphics/host_gpu/pageManager.h" + +#include "graphics/host_gpu/regionDefinitions.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS +#ifndef NOMINMAX +#define NOMINMAX +#endif +#include +#undef min +#undef max +#endif + +namespace Libs::Graphics { +namespace { + +constexpr uint64_t PAGE_SIZE = TRACKER_PAGE_SIZE; +constexpr uint64_t REGION_SIZE = TRACKER_REGION_SIZE; +constexpr uint64_t ADDRESS_SIZE = TRACKER_ADDRESS_SIZE; +constexpr uint64_t REGION_COUNT = ADDRESS_SIZE / REGION_SIZE; +constexpr uint64_t REGION_PAGES = REGION_SIZE / PAGE_SIZE; + +thread_local bool g_in_fault_resolution = false; + +[[noreturn]] void FailFast(const char* reason = nullptr) noexcept { + std::fputs("PageManager fail-fast: ", stderr); + std::fputs(reason != nullptr ? reason : "invalid page state", stderr); + std::fputc('\n', stderr); +#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS + void* frames[16] {}; + const auto frame_count = CaptureStackBackTrace(0, static_cast(std::size(frames)), frames, + nullptr); + const auto image_base = reinterpret_cast(GetModuleHandleW(nullptr)); + for (uint16_t i = 0; i < frame_count; i++) { + const auto address = reinterpret_cast(frames[i]); + std::fprintf(stderr, " frame[%u]=0x%016" PRIxPTR " image_rva=0x%016" PRIxPTR "\n", i, + address, address >= image_base ? address - image_base : 0); + } +#endif + std::fflush(stderr); +#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS + TerminateProcess(GetCurrentProcess(), static_cast(EXCEPTION_NONCONTINUABLE_EXCEPTION)); +#endif + std::_Exit(322); +} + +[[noreturn]] void Fatal(const char* format, ...) { + std::fputs("PageManager fatal: ", stderr); + va_list args; + va_start(args, format); + std::vfprintf(stderr, format, args); + va_end(args); + std::fputc('\n', stderr); + std::fflush(stderr); + std::_Exit(322); +} + +uint32_t CurrentThread() noexcept { +#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS + return GetCurrentThreadId(); +#else + FailFast(); +#endif +} + +class SpinGuard final { +public: + explicit SpinGuard(std::atomic_flag& lock): m_lock(lock) { + while (m_lock.test_and_set(std::memory_order_acquire)) { + std::atomic_signal_fence(std::memory_order_seq_cst); + } + } + ~SpinGuard() { m_lock.clear(std::memory_order_release); } + KYTY_CLASS_NO_COPY(SpinGuard); + +private: + std::atomic_flag& m_lock; +}; + +void ValidateRange(uint64_t vaddr, uint64_t size) { + if (vaddr == 0 || size == 0 || vaddr >= ADDRESS_SIZE || size > ADDRESS_SIZE - vaddr) { + Fatal("invalid range vaddr=0x%016" PRIx64 ", size=0x%016" PRIx64, vaddr, size); + } +} + +uint64_t PageStart(uint64_t vaddr) { + return vaddr & ~(PAGE_SIZE - 1); +} + +uint64_t PageEnd(uint64_t vaddr, uint64_t size) { + ValidateRange(vaddr, size); + return PageStart(vaddr + size - 1) + PAGE_SIZE; +} + +} // namespace + +struct PageManager::Impl { + struct PageState { + std::atomic_flag lock = ATOMIC_FLAG_INIT; + uint32_t mappings = 0; + uint32_t gpu_read_mappings = 0; + uint32_t gpu_write_mappings = 0; + uint32_t write_watchers = 0; + uint32_t access_watchers = 0; + uint32_t original_protection = 0; + uint32_t backing_writer = 0; + bool resolving = false; + bool resolving_read_write = false; + bool late_read_pending = false; + bool late_write_pending = false; + }; + + struct Region { + std::array pages; + }; + + Impl(PageFaultHandler handler, void* context): fault_handler(handler), fault_context(context) { + if (fault_handler == nullptr) { + Fatal("null fault handler"); + } +#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS + SYSTEM_INFO info {}; + GetSystemInfo(&info); + if (info.dwPageSize != PAGE_SIZE) { + Fatal("unsupported host page size 0x%08" PRIx32, + static_cast(info.dwPageSize)); + } +#else + Fatal("page-fault invalidation is not implemented on this platform"); +#endif + regions = std::make_unique[]>(REGION_COUNT); + for (uint64_t i = 0; i < REGION_COUNT; i++) { + regions[i].store(nullptr, std::memory_order_relaxed); + } + } + + ~Impl() { + for (const auto& region: region_storage) { + for (auto& page: region->pages) { + SpinGuard lock(page.lock); + if (page.mappings != 0 || page.gpu_read_mappings != 0 || + page.gpu_write_mappings != 0 || page.write_watchers != 0 || + page.access_watchers != 0 || page.backing_writer != 0 || page.resolving) { + FailFast("PageManager destroyed with live page state"); + } + } + } + } + + Region* FindRegion(uint64_t vaddr) const noexcept { + return vaddr < ADDRESS_SIZE ? regions[vaddr / REGION_SIZE].load(std::memory_order_acquire) + : nullptr; + } + + Region* GetOrCreateRegion(uint64_t vaddr) { + const auto index = vaddr / REGION_SIZE; + if (auto* region = regions[index].load(std::memory_order_acquire); region != nullptr) { + return region; + } + std::lock_guard lock(region_mutex); + if (auto* region = regions[index].load(std::memory_order_acquire); region != nullptr) { + return region; + } + auto region = std::make_unique(); + auto* ptr = region.get(); + region_storage.push_back(std::move(region)); + regions[index].store(ptr, std::memory_order_release); + return ptr; + } + + PageState& GetPage(Region& region, uint64_t vaddr) const { + return region.pages[(vaddr % REGION_SIZE) / PAGE_SIZE]; + } + + static uint32_t WatcherProtection(const PageState& page) { + if (page.access_watchers != 0) { + return PAGE_NOACCESS; + } + if (page.write_watchers != 0) { + return PAGE_READONLY; + } + return page.original_protection; + } + + static void PublishDelayedFaults(PageState& page, uint32_t old_protection, + uint32_t new_protection) { + if (old_protection == PAGE_NOACCESS && new_protection != PAGE_NOACCESS) { + page.late_read_pending = true; + } + if ((old_protection == PAGE_NOACCESS || old_protection == PAGE_READONLY) && + new_protection == PAGE_READWRITE) { + page.late_write_pending = true; + } + } + + static uint32_t QueryProtection(uint64_t vaddr) { +#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS + MEMORY_BASIC_INFORMATION info {}; + if (VirtualQuery(reinterpret_cast(static_cast(vaddr)), &info, + sizeof(info)) == 0 || + info.State != MEM_COMMIT || info.Protect != PAGE_READWRITE) { + Fatal("basic path requires PAGE_READWRITE at 0x%016" PRIx64 " (state=0x%08" PRIx32 + ", protection=0x%08" PRIx32 ")", + vaddr, static_cast(info.State), static_cast(info.Protect)); + } + return info.Protect; +#else + (void)vaddr; + Fatal("page query is unsupported on this platform"); +#endif + } + + static bool AllowsAccess(uint64_t vaddr, PageFaultAccess access) noexcept { +#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS + MEMORY_BASIC_INFORMATION info {}; + if (VirtualQuery(reinterpret_cast(static_cast(vaddr)), &info, + sizeof(info)) == 0 || + info.State != MEM_COMMIT) { + return false; + } + switch (access) { + case PageFaultAccess::Read: + return info.Protect == PAGE_READONLY || info.Protect == PAGE_READWRITE; + case PageFaultAccess::Write: return info.Protect == PAGE_READWRITE; + default: return false; + } +#else + (void)vaddr; + return false; +#endif + } + + static void Protect(uint64_t vaddr, uint32_t protection, uint32_t expected_old, + bool fault_path) noexcept { +#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS + DWORD old_protection = 0; + if (VirtualProtect(reinterpret_cast(static_cast(vaddr)), PAGE_SIZE, + protection, &old_protection) == 0 || + old_protection != expected_old) { + if (fault_path) { + FailFast("VirtualProtect fault transition did not match expected protection"); + } + Fatal("invalid protection transition at 0x%016" PRIx64 ", old=0x%08" PRIx32 + ", expected=0x%08" PRIx32 ", new=0x%08" PRIx32, + vaddr, static_cast(old_protection), expected_old, protection); + } +#else + (void)vaddr; + (void)protection; + (void)fault_path; + FailFast("page protection is unsupported on this platform"); +#endif + } + + std::unique_ptr[]> regions; + std::vector> region_storage; + std::mutex region_mutex; + PageFaultHandler fault_handler = nullptr; + void* fault_context = nullptr; +}; + +static_assert(std::atomic::is_always_lock_free); + +PageManager::PageManager(PageFaultHandler fault_handler, void* fault_context) + : m_impl(std::make_unique(fault_handler, fault_context)) {} + +PageManager::~PageManager() = default; + +uint64_t PageManager::GetPageSize() const { + if (g_in_fault_resolution) { + FailFast("nested page fault while resolving a watched page"); + } + return PAGE_SIZE; +} + +bool PageManager::IsTracked(uint64_t vaddr) const noexcept { + if (g_in_fault_resolution) { + FailFast("IsTracked called during fault resolution"); + } + auto* region = m_impl->FindRegion(vaddr); + if (region == nullptr) { + return false; + } + auto& page = m_impl->GetPage(*region, vaddr); + SpinGuard lock(page.lock); + return page.write_watchers != 0 || page.access_watchers != 0; +} + +bool PageManager::IsMapped(uint64_t vaddr, uint64_t size) const noexcept { + if (g_in_fault_resolution || vaddr == 0 || size == 0 || vaddr >= ADDRESS_SIZE || + size > ADDRESS_SIZE - vaddr) { + return false; + } + const auto end = PageStart(vaddr + size - 1) + PAGE_SIZE; + for (auto page_vaddr = PageStart(vaddr); page_vaddr < end; page_vaddr += PAGE_SIZE) { + auto* region = m_impl->FindRegion(page_vaddr); + if (region == nullptr) { + return false; + } + auto& page = m_impl->GetPage(*region, page_vaddr); + SpinGuard lock(page.lock); + if (page.mappings == 0) { + return false; + } + } + return true; +} + +bool PageManager::HasGpuAccess(uint64_t vaddr, uint64_t size, GpuAccess access) const noexcept { + if (access != GpuAccess::Read && access != GpuAccess::Write && access != GpuAccess::ReadWrite) { + FailFast("HasGpuAccess received an invalid GPU access mode"); + } + const bool need_read = access == GpuAccess::Read || access == GpuAccess::ReadWrite; + const bool need_write = access == GpuAccess::Write || access == GpuAccess::ReadWrite; + if (vaddr == 0 || size == 0 || vaddr >= ADDRESS_SIZE || size > ADDRESS_SIZE - vaddr) { + return false; + } + const auto end = PageEnd(vaddr, size); + for (auto addr = PageStart(vaddr); addr < end; addr += PAGE_SIZE) { + auto* region = m_impl->FindRegion(addr); + if (region == nullptr) { + return false; + } + auto& page = m_impl->GetPage(*region, addr); + SpinGuard lock(page.lock); + if ((need_read && page.gpu_read_mappings == 0) || + (need_write && page.gpu_write_mappings == 0)) { + return false; + } + } + return true; +} + +void PageManager::UpdatePageWatchers(bool track, uint64_t vaddr, uint64_t size, + PageWatchMode mode) { + if (g_in_fault_resolution) { + FailFast("page watchers changed during fault resolution"); + } + if (mode != PageWatchMode::Write && mode != PageWatchMode::ReadWrite) { + Fatal("invalid watcher mode"); + } + const auto end = PageEnd(vaddr, size); + for (auto page_vaddr = PageStart(vaddr); page_vaddr < end; page_vaddr += PAGE_SIZE) { + auto* region = + track ? m_impl->GetOrCreateRegion(page_vaddr) : m_impl->FindRegion(page_vaddr); + if (region == nullptr) { + Fatal("untracking unknown page 0x%016" PRIx64, page_vaddr); + } + auto& page = m_impl->GetPage(*region, page_vaddr); + SpinGuard lock(page.lock); + if (page.resolving && track) { + FailFast("new page watcher raced active fault resolution"); + } + if (page.mappings == 0) { + Fatal("watching unmapped page 0x%016" PRIx64, page_vaddr); + } + auto& watchers = + (mode == PageWatchMode::ReadWrite ? page.access_watchers : page.write_watchers); + if (track) { + if (watchers == std::numeric_limits::max()) { + Fatal("watcher overflow at 0x%016" PRIx64, page_vaddr); + } + const bool first_watcher = page.write_watchers == 0 && page.access_watchers == 0; + if (first_watcher) { + page.original_protection = Impl::QueryProtection(page_vaddr); + } + const auto old_protection = Impl::WatcherProtection(page); + watchers++; + const auto new_protection = Impl::WatcherProtection(page); + if (new_protection != old_protection) { + Impl::Protect(page_vaddr, new_protection, old_protection, false); + } + switch (new_protection) { + case PAGE_NOACCESS: + page.late_read_pending = false; + page.late_write_pending = false; + break; + case PAGE_READONLY: page.late_write_pending = false; break; + default: break; + } + } else { + if (watchers == 0) { + Fatal("watcher underflow at 0x%016" PRIx64, page_vaddr); + } + if (page.backing_writer != 0 && page.backing_writer != CurrentThread()) { + Fatal("backing write ownership changed at 0x%016" PRIx64, page_vaddr); + } + const auto old_protection = Impl::WatcherProtection(page); + watchers--; + const auto new_protection = Impl::WatcherProtection(page); + if (page.backing_writer == 0 && new_protection != old_protection) { + Impl::Protect(page_vaddr, new_protection, old_protection, false); + } + if (page.backing_writer == 0) { + Impl::PublishDelayedFaults(page, old_protection, new_protection); + } + if (page.backing_writer == 0 && page.write_watchers == 0 && page.access_watchers == 0) { + page.original_protection = 0; + } + } + } +} + +void PageManager::OnGpuMap(uint64_t vaddr, uint64_t size, GpuAccess access) { + if (g_in_fault_resolution) { + FailFast("GPU mapping changed during fault resolution"); + } + if (access != GpuAccess::Read && access != GpuAccess::Write && access != GpuAccess::ReadWrite) { + FailFast("GPU map received an invalid access mode"); + } + const bool gpu_read = access == GpuAccess::Read || access == GpuAccess::ReadWrite; + const bool gpu_write = access == GpuAccess::Write || access == GpuAccess::ReadWrite; + const auto end = PageEnd(vaddr, size); + for (auto addr = PageStart(vaddr); addr < end; addr += PAGE_SIZE) { + auto& page = m_impl->GetPage(*m_impl->GetOrCreateRegion(addr), addr); + SpinGuard lock(page.lock); + if (page.resolving || page.mappings == std::numeric_limits::max() || + (gpu_read && page.gpu_read_mappings == std::numeric_limits::max()) || + (gpu_write && page.gpu_write_mappings == std::numeric_limits::max())) { + Fatal("invalid map state at 0x%016" PRIx64, addr); + } + page.mappings++; + page.gpu_read_mappings += gpu_read ? 1u : 0u; + page.gpu_write_mappings += gpu_write ? 1u : 0u; + } +} + +void PageManager::OnGpuUnmap(uint64_t vaddr, uint64_t size, GpuAccess access) { + if (g_in_fault_resolution) { + FailFast("GPU unmapping changed during fault resolution"); + } + if (access != GpuAccess::Read && access != GpuAccess::Write && access != GpuAccess::ReadWrite) { + FailFast("GPU unmap received an invalid access mode"); + } + const bool gpu_read = access == GpuAccess::Read || access == GpuAccess::ReadWrite; + const bool gpu_write = access == GpuAccess::Write || access == GpuAccess::ReadWrite; + const auto end = PageEnd(vaddr, size); + for (auto page_vaddr = PageStart(vaddr); page_vaddr < end; page_vaddr += PAGE_SIZE) { + auto* region = m_impl->FindRegion(page_vaddr); + if (region == nullptr) { + Fatal("unmapping unknown page 0x%016" PRIx64, page_vaddr); + } + auto& page = m_impl->GetPage(*region, page_vaddr); + SpinGuard lock(page.lock); + if (page.resolving || page.mappings == 0 || (gpu_read && page.gpu_read_mappings == 0) || + (gpu_write && page.gpu_write_mappings == 0) || + (page.mappings == 1 && (page.write_watchers != 0 || page.access_watchers != 0))) { + Fatal("invalid unmap state at 0x%016" PRIx64, page_vaddr); + } + page.mappings--; + page.gpu_read_mappings -= gpu_read ? 1u : 0u; + page.gpu_write_mappings -= gpu_write ? 1u : 0u; + if (page.mappings == 0) { + if (page.gpu_read_mappings != 0 || page.gpu_write_mappings != 0) { + FailFast("GPU unmap left nonzero GPU mapping counts"); + } + page.late_read_pending = false; + page.late_write_pending = false; + } + } +} + +PageManager::BackingWrite::BackingWrite(PageManager& manager, uint64_t vaddr, + uint64_t size) noexcept + : m_manager(manager), m_vaddr(vaddr), m_size(size) { + m_manager.BeginBackingWrite(vaddr, size); +} + +PageManager::BackingWrite::~BackingWrite() { + m_manager.EndBackingWrite(m_vaddr, m_size); +} + +void PageManager::BeginBackingWrite(uint64_t vaddr, uint64_t size) noexcept { + if (g_in_fault_resolution) { + FailFast("backing write began during fault resolution"); + } + const auto end = PageEnd(vaddr, size); + const auto writer = CurrentThread(); + for (auto address = PageStart(vaddr); address < end; address += PAGE_SIZE) { + auto* region = m_impl->FindRegion(address); + if (region == nullptr) { + Fatal("backing write reserves an unknown page at 0x%016" PRIx64, address); + } + auto& page = m_impl->GetPage(*region, address); + SpinGuard lock(page.lock); + if (page.mappings == 0 || page.resolving || page.backing_writer != 0 || + page.access_watchers == 0) { + Fatal("backing write races page resolution at 0x%016" PRIx64, address); + } + page.resolving = true; + page.resolving_read_write = true; + page.backing_writer = writer; + } +} + +void PageManager::EndBackingWrite(uint64_t vaddr, uint64_t size) noexcept { + if (g_in_fault_resolution) { + FailFast("backing write ended during fault resolution"); + } + const auto end = PageEnd(vaddr, size); + const auto writer = CurrentThread(); + for (auto address = PageStart(vaddr); address < end; address += PAGE_SIZE) { + auto* region = m_impl->FindRegion(address); + if (region == nullptr) { + FailFast("backing write ended for an unknown page"); + } + auto& page = m_impl->GetPage(*region, address); + SpinGuard lock(page.lock); + if (!page.resolving || page.backing_writer != writer) { + FailFast("backing write ended without matching owner and resolving state"); + } + const auto old_protection = PAGE_NOACCESS; + const auto new_protection = Impl::WatcherProtection(page); + if (new_protection != old_protection) { + Impl::Protect(address, new_protection, old_protection, false); + } + Impl::PublishDelayedFaults(page, old_protection, new_protection); + if (page.write_watchers == 0 && page.access_watchers == 0) { + page.original_protection = 0; + } + page.backing_writer = 0; + page.resolving = false; + page.resolving_read_write = false; + } +} + +bool PageManager::HandleFault(PageFaultAccess access, uint64_t fault_vaddr) noexcept { + if (g_in_fault_resolution) { + FailFast("nested HandleFault call"); + } + auto* region = m_impl->FindRegion(fault_vaddr); + if (region == nullptr) { + return false; + } + auto& page = m_impl->GetPage(*region, fault_vaddr); + bool waited = false; + while (true) { + SpinGuard lock(page.lock); + if (access == PageFaultAccess::Read && page.late_read_pending && + Impl::AllowsAccess(fault_vaddr, access)) { + page.late_read_pending = false; + return true; + } + if (access == PageFaultAccess::Write && page.late_write_pending && + Impl::AllowsAccess(fault_vaddr, access)) { + page.late_write_pending = false; + return true; + } + if (page.resolving) { + if (page.backing_writer == CurrentThread()) { + FailFast("backing writer faulted on its own reserved page"); + } + if ((!page.resolving_read_write && access != PageFaultAccess::Write) || + (page.resolving_read_write && access != PageFaultAccess::Read && + access != PageFaultAccess::Write)) { + FailFast("fault access is incompatible with the active resolver"); + } + waited = true; + continue; + } + if (page.write_watchers == 0 && page.access_watchers == 0) { + if (access != PageFaultAccess::Read && access != PageFaultAccess::Write) { + return false; + } + bool& pending = (access == PageFaultAccess::Read ? page.late_read_pending + : page.late_write_pending); + const bool allowed = Impl::AllowsAccess(fault_vaddr, access); + pending = false; + if (waited && !allowed) { + FailFast("page remained inaccessible after waiting for its resolver"); + } + // More than one CPU can fault before a protection transition becomes visible. The first + // delayed fault consumes the hint bit; later faults must also resume once the mapped page + // already permits the requested access. A genuinely read-only/no-access page still falls + // through to the guest exception path. + return allowed; + } + if ((access != PageFaultAccess::Read && access != PageFaultAccess::Write) || + (access == PageFaultAccess::Read && page.access_watchers == 0)) { + FailFast("fault access is incompatible with active page watchers"); + } + page.resolving = true; + page.resolving_read_write = page.access_watchers != 0; + break; + } + g_in_fault_resolution = true; + const bool handled = m_impl->fault_handler(m_impl->fault_context, access, fault_vaddr, 1, + PageFaultPhase::Invalidate); + g_in_fault_resolution = false; + { + SpinGuard lock(page.lock); + if (!handled || !page.resolving) { + FailFast("fault invalidation did not preserve the resolving state"); + } + } + g_in_fault_resolution = true; + const bool completed = m_impl->fault_handler(m_impl->fault_context, access, fault_vaddr, 1, + PageFaultPhase::Complete); + g_in_fault_resolution = false; + { + SpinGuard lock(page.lock); + if (!completed || !page.resolving) { + FailFast("fault completion did not preserve the resolving state"); + } + if (page.write_watchers != 0 || page.access_watchers != 0) { + const auto old_protection = Impl::WatcherProtection(page); + const bool read_only_fault = access == PageFaultAccess::Read; + if (read_only_fault && page.access_watchers == 0) { + FailFast("read fault completed without a read/write watcher"); + } + page.access_watchers = 0; + if (!read_only_fault) { + page.write_watchers = 0; + } + const auto restored_protection = Impl::WatcherProtection(page); + Impl::Protect(PageStart(fault_vaddr), restored_protection, old_protection, true); + if (page.write_watchers == 0) { + page.original_protection = 0; + } + Impl::PublishDelayedFaults(page, old_protection, restored_protection); + } else if (!Impl::AllowsAccess(fault_vaddr, access)) { + FailFast("fault completion left the page inaccessible"); + } + page.resolving = false; + page.resolving_read_write = false; + } + g_in_fault_resolution = true; + const bool released = m_impl->fault_handler(m_impl->fault_context, access, fault_vaddr, 1, + PageFaultPhase::Release); + g_in_fault_resolution = false; + if (!released) { + FailFast("fault release callback failed"); + } + return true; +} + +} // namespace Libs::Graphics diff --git a/src/graphics/host_gpu/pageManager.h b/src/graphics/host_gpu/pageManager.h new file mode 100644 index 000000000..511eb2243 --- /dev/null +++ b/src/graphics/host_gpu/pageManager.h @@ -0,0 +1,60 @@ +#ifndef EMULATOR_SRC_GRAPHICS_HOST_GPU_PAGEMANAGER_H_ +#define EMULATOR_SRC_GRAPHICS_HOST_GPU_PAGEMANAGER_H_ + +#include "common/common.h" + +#include + +namespace Libs::Graphics { + +enum class PageFaultAccess { Read, Write, Execute, Unknown }; +enum class PageFaultPhase { Invalidate, Complete, Release }; +enum class PageWatchMode { Write, ReadWrite }; +enum class GpuAccess { Read, Write, ReadWrite }; + +using PageFaultHandler = bool (*)(void* context, PageFaultAccess access, uint64_t vaddr, + uint64_t size, PageFaultPhase phase) noexcept; + +class PageManager final { +public: + class BackingWrite final { + public: + BackingWrite(PageManager& manager, uint64_t vaddr, uint64_t size) noexcept; + ~BackingWrite(); + KYTY_CLASS_NO_COPY(BackingWrite); + + private: + PageManager& m_manager; + uint64_t m_vaddr = 0; + uint64_t m_size = 0; + }; + + PageManager(PageFaultHandler fault_handler, void* fault_context); + // The owner must stop all PageManager callers before destruction. + ~PageManager(); + + KYTY_CLASS_NO_COPY(PageManager); + + [[nodiscard]] uint64_t GetPageSize() const; + [[nodiscard]] bool IsTracked(uint64_t vaddr) const noexcept; + [[nodiscard]] bool IsMapped(uint64_t vaddr, uint64_t size) const noexcept; + [[nodiscard]] bool HasGpuAccess(uint64_t vaddr, uint64_t size, GpuAccess access) const noexcept; + + void UpdatePageWatchers(bool track, uint64_t vaddr, uint64_t size, + PageWatchMode mode = PageWatchMode::Write); + void OnGpuMap(uint64_t vaddr, uint64_t size, GpuAccess access = GpuAccess::ReadWrite); + void OnGpuUnmap(uint64_t vaddr, uint64_t size, GpuAccess access = GpuAccess::ReadWrite); + + [[nodiscard]] bool HandleFault(PageFaultAccess access, uint64_t fault_vaddr) noexcept; + +private: + void BeginBackingWrite(uint64_t vaddr, uint64_t size) noexcept; + void EndBackingWrite(uint64_t vaddr, uint64_t size) noexcept; + + struct Impl; + std::unique_ptr m_impl; +}; + +} // namespace Libs::Graphics + +#endif // EMULATOR_SRC_GRAPHICS_HOST_GPU_PAGEMANAGER_H_ diff --git a/src/graphics/host_gpu/rangeSet.h b/src/graphics/host_gpu/rangeSet.h new file mode 100644 index 000000000..203775bc5 --- /dev/null +++ b/src/graphics/host_gpu/rangeSet.h @@ -0,0 +1,93 @@ +#ifndef EMULATOR_SRC_GRAPHICS_HOST_GPU_RANGESET_H_ +#define EMULATOR_SRC_GRAPHICS_HOST_GPU_RANGESET_H_ + +#include "common/assert.h" + +#include +#include +#include +#include + +namespace Libs::Graphics { + +class RangeSet final { +public: + struct Range { + uint64_t address = 0; + uint64_t size = 0; + }; + + void Add(uint64_t address, uint64_t size) { + const auto end = End(address, size); + auto it = m_ranges.lower_bound(address); + if (it != m_ranges.begin() && std::prev(it)->second >= address) { + it = std::prev(it); + } + uint64_t begin = address; + uint64_t last = end; + while (it != m_ranges.end() && it->first <= last) { + begin = std::min(begin, it->first); + last = std::max(last, it->second); + it = m_ranges.erase(it); + } + m_ranges.emplace(begin, last); + } + + void Subtract(uint64_t address, uint64_t size) { + const auto end = End(address, size); + auto it = m_ranges.lower_bound(address); + if (it != m_ranges.begin() && std::prev(it)->second > address) { + it = std::prev(it); + } + while (it != m_ranges.end() && it->first < end) { + const auto begin = it->first; + const auto last = it->second; + it = m_ranges.erase(it); + if (begin < address) { + m_ranges.emplace(begin, address); + } + if (last > end) { + m_ranges.emplace(end, last); + break; + } + } + } + + [[nodiscard]] std::vector Intersections(uint64_t address, uint64_t size) const { + std::vector result; + ForEachIntersection(address, size, [&result](Range range) { result.push_back(range); }); + return result; + } + + template + void ForEachIntersection(uint64_t address, uint64_t size, Func&& func) const { + const auto end = End(address, size); + auto it = m_ranges.upper_bound(address); + if (it != m_ranges.begin()) { + --it; + } + for (; it != m_ranges.end() && it->first < end; ++it) { + const auto begin = std::max(address, it->first); + const auto last = std::min(end, it->second); + if (begin < last) { + func(Range {begin, last - begin}); + } + } + } + + [[nodiscard]] bool Empty() const { return m_ranges.empty(); } + +private: + static uint64_t End(uint64_t address, uint64_t size) { + if (address == 0 || size == 0 || size > UINT64_MAX - address) { + EXIT("invalid range-set address or size\n"); + } + return address + size; + } + + std::map m_ranges; +}; + +} // namespace Libs::Graphics + +#endif // EMULATOR_SRC_GRAPHICS_HOST_GPU_RANGESET_H_ diff --git a/src/graphics/host_gpu/regionDefinitions.h b/src/graphics/host_gpu/regionDefinitions.h new file mode 100644 index 000000000..2b478283f --- /dev/null +++ b/src/graphics/host_gpu/regionDefinitions.h @@ -0,0 +1,20 @@ +#ifndef EMULATOR_SRC_GRAPHICS_HOST_GPU_REGIONDEFINITIONS_H_ +#define EMULATOR_SRC_GRAPHICS_HOST_GPU_REGIONDEFINITIONS_H_ + +#include "common/common.h" + +#include + +namespace Libs::Graphics { + +constexpr uint64_t TRACKER_PAGE_SIZE = 4ull * 1024ull; +constexpr uint64_t TRACKER_REGION_SIZE = 4ull * 1024ull * 1024ull; +constexpr uint64_t TRACKER_ADDRESS_SIZE = 1ull << 40u; +constexpr size_t TRACKER_REGION_PAGES = TRACKER_REGION_SIZE / TRACKER_PAGE_SIZE; + +enum class DirtySource { Cpu, Gpu }; +using RegionBits = std::bitset; + +} // namespace Libs::Graphics + +#endif // EMULATOR_SRC_GRAPHICS_HOST_GPU_REGIONDEFINITIONS_H_ diff --git a/src/graphics/host_gpu/regionManager.h b/src/graphics/host_gpu/regionManager.h new file mode 100644 index 000000000..b30a470b8 --- /dev/null +++ b/src/graphics/host_gpu/regionManager.h @@ -0,0 +1,349 @@ +#ifndef EMULATOR_SRC_GRAPHICS_HOST_GPU_REGIONMANAGER_H_ +#define EMULATOR_SRC_GRAPHICS_HOST_GPU_REGIONMANAGER_H_ + +#include "common/assert.h" +#include "graphics/host_gpu/pageManager.h" +#include "graphics/host_gpu/regionDefinitions.h" + +#include +#include +#include + +#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS +#ifndef NOMINMAX +#define NOMINMAX +#endif +#include +#undef min +#undef max +#endif + +namespace Libs::Graphics { + +enum class CpuFaultAction { Untracked, Continue, Download }; + +class TrackingSpinLock final { +public: + void lock() noexcept { + const auto thread = CurrentThread(); + if (m_owner.load(std::memory_order_relaxed) == thread) { + EXIT("recursive region tracking lock\n"); + } + while (m_lock.test_and_set(std::memory_order_acquire)) { + if (m_owner.load(std::memory_order_relaxed) == thread) { + EXIT("recursive region tracking lock while contended\n"); + } + std::atomic_signal_fence(std::memory_order_seq_cst); + } + m_owner.store(thread, std::memory_order_relaxed); + } + void unlock() noexcept { + if (m_owner.load(std::memory_order_relaxed) != CurrentThread()) { + EXIT("region tracking lock released by non-owner\n"); + } + m_owner.store(0, std::memory_order_relaxed); + m_lock.clear(std::memory_order_release); + } + +private: + static uint32_t CurrentThread() noexcept { +#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS + return GetCurrentThreadId(); +#else + EXIT("region tracking thread identity is unsupported on this platform\n"); +#endif + } + + std::atomic_flag m_lock = ATOMIC_FLAG_INIT; + std::atomic_uint32_t m_owner {0}; +}; + +static_assert(std::atomic_uint32_t::is_always_lock_free); + +class RegionManager final { +public: + RegionManager(PageManager& page_manager, uint64_t cpu_addr) + : m_page_manager(page_manager), m_cpu_addr(cpu_addr) { + if (m_cpu_addr % TRACKER_REGION_SIZE != 0) { + EXIT("invalid region tracking manager construction\n"); + } + m_cpu_dirty.set(); + m_writable.set(); + } + + KYTY_CLASS_NO_COPY(RegionManager); + + [[nodiscard]] uint64_t GetCpuAddr() const { return m_cpu_addr; } + void Track(uint64_t vaddr, uint64_t size) { + const auto [start, end] = GetPageRange(vaddr, size); + for (auto page = start; page < end; page++) { + m_tracked.set(page); + } + } + void Untrack(uint64_t vaddr, uint64_t size) { + const auto [start, end] = GetPageRange(vaddr, size); + for (auto page = start; page < end; page++) { + m_tracked.reset(page); + } + } + template + [[nodiscard]] bool IsModified(uint64_t offset, uint64_t size) const { + const auto [start, end] = GetPageRange(m_cpu_addr + offset, size); + const auto& bits = GetBits(); + for (auto page = start; page < end; page++) { + if (bits.test(page)) { + return true; + } + } + return false; + } + + template + [[nodiscard]] bool IsFullyModified(uint64_t offset, uint64_t size) const { + const auto [start, end] = GetPageRange(m_cpu_addr + offset, size); + const auto& bits = GetBits(); + for (auto page = start; page < end; page++) { + if (!bits.test(page)) { + return false; + } + } + return true; + } + + template + RegionBits ChangeState(uint64_t vaddr, uint64_t size) { + const auto [start, end] = GetPageRange(vaddr, size); + if constexpr (source == DirtySource::Cpu && enable) { + for (auto page = start; page < end; page++) { + if (m_gpu_dirty.test(page) || m_fault_pending.test(page)) { + EXIT("CPU dirty state conflicts with GPU dirty or pending fault state\n"); + } + } + } + if constexpr (source == DirtySource::Gpu && enable) { + for (auto page = start; page < end; page++) { + if (m_cpu_dirty.test(page) || m_fault_pending.test(page)) { + EXIT("GPU dirty state conflicts with CPU dirty or pending fault state\n"); + } + } + } + auto& bits = GetBits(); + auto changed = bits; + for (auto page = start; page < end; page++) { + bits.set(page, enable); + } + changed ^= bits; + if constexpr (source == DirtySource::Cpu) { + changed = m_cpu_dirty ^ m_writable; + m_writable = m_cpu_dirty; + } + return changed; + } + + [[nodiscard]] CpuFaultAction BeginCpuFault(uint64_t vaddr, uint64_t size, + PageFaultAccess access = PageFaultAccess::Write) { + if (access != PageFaultAccess::Read && access != PageFaultAccess::Write) { + EXIT("unsupported CPU fault access while beginning ownership transfer\n"); + } + const auto [start, end] = GetPageRange(vaddr, size); + const bool tracked = m_tracked.test(start); + for (auto page = start; page < end; page++) { + if (m_tracked.test(page) != tracked) { + EXIT("CPU fault spans mixed tracked and untracked pages\n"); + } + if (m_fault_pending.test(page)) { + return CpuFaultAction::Untracked; + } + if (m_cpu_dirty.test(page) != m_writable.test(page) || + (m_gpu_dirty.test(page) && (m_cpu_dirty.test(page) || m_writable.test(page)))) { + EXIT("inconsistent CPU fault page state\n"); + } + } + if (!tracked) { + return CpuFaultAction::Untracked; + } + bool gpu_dirty = m_gpu_dirty.test(start); + bool writable = m_writable.test(start); + for (auto page = start + 1; page < end; page++) { + if (m_gpu_dirty.test(page) != gpu_dirty || m_writable.test(page) != writable) { + EXIT("CPU fault spans pages with incompatible dirty or writable state\n"); + } + } + for (auto page = start; page < end; page++) { + if (!gpu_dirty && access == PageFaultAccess::Write) { + m_cpu_dirty.set(page); + m_writable.set(page); + } + m_fault_pending.set(page); + } + return gpu_dirty ? CpuFaultAction::Download : CpuFaultAction::Continue; + } + + [[nodiscard]] bool CompleteCpuFault(uint64_t vaddr, uint64_t size, PageFaultAccess access, + bool downloaded) { + const auto [start, end] = GetPageRange(vaddr, size); + for (auto page = start; page < end; page++) { + if (!m_fault_pending.test(page)) { + return false; + } + } + for (auto page = start; page < end; page++) { + const bool gpu_dirty = m_gpu_dirty.test(page); + if (gpu_dirty != downloaded) { + EXIT("CPU fault download result disagrees with GPU dirty state\n"); + } + if (gpu_dirty) { + m_gpu_dirty.reset(page); + switch (access) { + case PageFaultAccess::Read: break; + case PageFaultAccess::Write: + m_cpu_dirty.set(page); + m_writable.set(page); + break; + default: EXIT("unsupported CPU fault access after GPU download\n"); + } + } + m_fault_pending.reset(page); + } + return true; + } + + [[nodiscard]] bool HasPendingFault(uint64_t vaddr, uint64_t size) const { + const auto [start, end] = GetPageRange(vaddr, size); + for (auto page = start; page < end; page++) { + if (m_fault_pending.test(page)) { + return true; + } + } + return false; + } + + [[nodiscard]] bool CompleteVirtualGpuWrite(uint64_t vaddr, uint64_t size) { + const auto [start, end] = GetPageRange(vaddr, size); + for (auto page = start; page < end; page++) { + if (!m_fault_pending.test(page)) { + return false; + } + if (!m_gpu_dirty.test(page)) { + EXIT("virtual GPU write completion found a non-GPU-dirty page\n"); + } + } + for (auto page = start; page < end; page++) { + m_gpu_dirty.reset(page); + m_cpu_dirty.set(page); + m_writable.set(page); + m_fault_pending.reset(page); + } + return true; + } + + template + RegionBits ForEachModifiedRange(uint64_t vaddr, uint64_t size, Func&& func) { + const auto [start, end] = GetPageRange(vaddr, size); + auto mask = GetBits(); + if constexpr (source == DirtySource::Cpu) { + mask &= ~m_fault_pending; + } + for (auto page = 0u; page < start; page++) { + mask.reset(page); + } + for (auto page = end; page < TRACKER_REGION_PAGES; page++) { + mask.reset(page); + } + if constexpr (clear) { + auto& bits = GetBits(); + for (auto page = start; page < end; page++) { + if (mask.test(page)) { + bits.reset(page); + } + } + } + if constexpr (source == DirtySource::Cpu && clear) { + auto changed = m_cpu_dirty ^ m_writable; + m_writable = m_cpu_dirty; + ApplyProtection(changed, true); + ForEachRange(mask, std::forward(func)); + return changed; + } + ForEachRange(mask, std::forward(func)); + if constexpr (clear) { + return mask; + } + return {}; + } + + void ApplyProtection(const RegionBits& changed, bool track) { + ForEachRange(changed, [this, track](uint64_t vaddr, uint64_t size) { + m_page_manager.UpdatePageWatchers(track, vaddr, size); + }); + } + + void ApplyGpuProtection(const RegionBits& changed, bool track, PageWatchMode mode) { + if (mode != PageWatchMode::Write && mode != PageWatchMode::ReadWrite) { + EXIT("unsupported GPU page-watch mode\n"); + } + ForEachRange(changed, [this, track, mode](uint64_t vaddr, uint64_t size) { + m_page_manager.UpdatePageWatchers(track, vaddr, size, mode); + }); + } + + TrackingSpinLock lock; + +private: + template + RegionBits& GetBits() { + if constexpr (source == DirtySource::Cpu) { + return m_cpu_dirty; + } else { + return m_gpu_dirty; + } + } + + template + const RegionBits& GetBits() const { + if constexpr (source == DirtySource::Cpu) { + return m_cpu_dirty; + } else { + return m_gpu_dirty; + } + } + + [[nodiscard]] std::pair GetPageRange(uint64_t vaddr, uint64_t size) const { + if (size == 0 || vaddr < m_cpu_addr || vaddr >= m_cpu_addr + TRACKER_REGION_SIZE || + size > m_cpu_addr + TRACKER_REGION_SIZE - vaddr) { + EXIT("range lies outside its tracking region\n"); + } + const auto offset = vaddr - m_cpu_addr; + return {static_cast(offset / TRACKER_PAGE_SIZE), + static_cast((offset + size + TRACKER_PAGE_SIZE - 1) / TRACKER_PAGE_SIZE)}; + } + + template + void ForEachRange(const RegionBits& bits, Func&& func) const { + size_t page = 0; + while (page < TRACKER_REGION_PAGES) { + while (page < TRACKER_REGION_PAGES && !bits.test(page)) { + page++; + } + const auto start = page; + while (page < TRACKER_REGION_PAGES && bits.test(page)) { + page++; + } + if (start != page) { + func(m_cpu_addr + start * TRACKER_PAGE_SIZE, (page - start) * TRACKER_PAGE_SIZE); + } + } + } + + PageManager& m_page_manager; + uint64_t m_cpu_addr = 0; + RegionBits m_cpu_dirty; + RegionBits m_gpu_dirty; + RegionBits m_writable; + RegionBits m_fault_pending; + RegionBits m_tracked; +}; + +} // namespace Libs::Graphics + +#endif // EMULATOR_SRC_GRAPHICS_HOST_GPU_REGIONMANAGER_H_ diff --git a/src/graphics/host_gpu/renderer/bufferCache.cpp b/src/graphics/host_gpu/renderer/bufferCache.cpp new file mode 100644 index 000000000..a36563e0b --- /dev/null +++ b/src/graphics/host_gpu/renderer/bufferCache.cpp @@ -0,0 +1,1632 @@ +#include "graphics/host_gpu/renderer/bufferCache.h" + +#include "common/assert.h" +#include "common/logging/log.h" +#include "common/profiler.h" +#include "graphics/guest_gpu/graphicsRun.h" +#include "graphics/host_gpu/graphicContext.h" +#include "graphics/host_gpu/objects/label.h" +#include "graphics/host_gpu/renderer/render.h" +#include "graphics/host_gpu/renderer/resourceMutex.h" +#include "graphics/host_gpu/renderer/textureCache.h" +#include "graphics/host_gpu/utils.h" +#include "graphics/host_gpu/vma.h" +#include "kernel/memory.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace Libs::Graphics { + +namespace { +thread_local const void* g_cache_lock_owner = nullptr; + +constexpr uint32_t READBACK_MAX_RANGES = 256; +constexpr uint64_t READBACK_COPY_ALIGNMENT = 64; +constexpr uint64_t READBACK_CAPACITY = + TRACKER_PAGE_SIZE + READBACK_MAX_RANGES * (READBACK_COPY_ALIGNMENT - 1); + +constexpr uint64_t AlignReadbackCopySize(uint64_t size) noexcept { + return (size + READBACK_COPY_ALIGNMENT - 1) & ~(READBACK_COPY_ALIGNMENT - 1); +} + +constexpr bool ShouldLogCommandProcessorReadback(uint64_t serial) noexcept { + return serial <= 32 || (serial % 256) == 0; +} + +struct CommandProcessorReadbackState { + BufferCache* cache = nullptr; + PageFaultAccess access = PageFaultAccess::Unknown; + uint64_t vaddr = 0; + uint64_t size = 0; + uint64_t page = 0; + uint64_t serial = 0; + uint64_t dirty_bytes = 0; + uint64_t packed_bytes = 0; + uint32_t dirty_count = 0; + std::chrono::steady_clock::time_point started {}; + bool installed = false; +}; + +thread_local CommandProcessorReadbackState g_command_processor_readback; +std::atomic g_command_processor_readback_serial {0}; + +struct SharedVulkanBufferOwner { + explicit SharedVulkanBufferOwner(GraphicContext* context): ctx(context) { EXIT_IF(ctx == nullptr); } + + ~SharedVulkanBufferOwner() { + if (buffer.buffer != nullptr) { + VulkanDeleteBuffer(ctx, &buffer); + } + } + + GraphicContext* ctx = nullptr; + VulkanBuffer buffer; +}; + +std::shared_ptr MakeSharedVulkanBuffer(GraphicContext* ctx) { + auto owner = std::make_shared(ctx); + return {owner, &owner->buffer}; +} + +std::vector> +ReserveBackingWrites(PageManager& page_manager, const std::vector& ranges) { + if (ranges.empty()) { + EXIT("BufferCache: cannot reserve empty backing-write ranges\n"); + } + std::vector> writes; + writes.reserve(ranges.size()); + uint64_t begin = 0; + uint64_t end = 0; + for (const auto& range: ranges) { + if (range.address == 0 || range.size == 0 || range.size > UINT64_MAX - range.address || + range.address + range.size > UINT64_MAX - (TRACKER_PAGE_SIZE - 1)) { + EXIT("BufferCache: invalid backing-write range, addr=0x%016" PRIx64 + " size=0x%016" PRIx64 "\n", + range.address, range.size); + } + const auto page_begin = range.address & ~(TRACKER_PAGE_SIZE - 1); + const auto page_end = + (range.address + range.size + TRACKER_PAGE_SIZE - 1) & ~(TRACKER_PAGE_SIZE - 1); + if (begin != 0 && page_begin > end) { + writes.push_back( + std::make_unique(page_manager, begin, end - begin)); + begin = 0; + } + if (begin == 0) { + begin = page_begin; + end = page_end; + } else { + end = std::max(end, page_end); + } + } + writes.push_back(std::make_unique(page_manager, begin, end - begin)); + return writes; +} + +void ValidateDirtyPages(const RangeSet& dirty, uint64_t vaddr, uint64_t size, + const char* operation) noexcept { + if (vaddr == 0 || size == 0 || size > UINT64_MAX - vaddr || + (vaddr & (TRACKER_PAGE_SIZE - 1)) != 0 || (size & (TRACKER_PAGE_SIZE - 1)) != 0) { + EXIT("BufferCache: invalid dirty-page validation range\n"); + } + for (auto page = vaddr; page < vaddr + size; page += TRACKER_PAGE_SIZE) { + bool found = false; + dirty.ForEachIntersection(page, TRACKER_PAGE_SIZE, + [&found](RangeSet::Range) { found = true; }); + if (!found) { + EXIT("BufferCache: GPU-dirty tracker page has no dirty bytes, operation=%s " + "addr=0x%016" PRIx64 "\n", + operation, page); + } + } +} + +class FaultSafeCacheLock final { +public: + FaultSafeCacheLock(const void* owner, Common::Mutex& mutex): m_mutex(mutex) { + if (g_cache_lock_owner != nullptr) { + EXIT("BufferCache: recursive cache lock acquisition, current=%p\n", + static_cast(g_cache_lock_owner)); + } + g_cache_lock_owner = owner; + m_mutex.Lock(); + } + ~FaultSafeCacheLock() { + m_mutex.Unlock(); + g_cache_lock_owner = nullptr; + } + +private: + Common::Mutex& m_mutex; +}; + +uint64_t AlignDown(uint64_t value) { + return value & ~(BufferCache::CACHING_PAGE_SIZE - 1); +} + +uint64_t AlignUp(uint64_t value) { + if (value > UINT64_MAX - (BufferCache::CACHING_PAGE_SIZE - 1)) { + EXIT("BufferCache: address alignment overflow, value=0x%016" PRIx64 "\n", value); + } + return (value + BufferCache::CACHING_PAGE_SIZE - 1) & ~(BufferCache::CACHING_PAGE_SIZE - 1); +} + +bool PageOverlaps(uint64_t left, uint64_t left_size, uint64_t right, uint64_t right_size) { + const auto left_begin = left & ~(TRACKER_PAGE_SIZE - 1); + const auto left_end = (left + left_size + TRACKER_PAGE_SIZE - 1) & ~(TRACKER_PAGE_SIZE - 1); + const auto right_begin = right & ~(TRACKER_PAGE_SIZE - 1); + const auto right_end = (right + right_size + TRACKER_PAGE_SIZE - 1) & ~(TRACKER_PAGE_SIZE - 1); + return left_begin < right_end && right_begin < left_end; +} +} // namespace + +bool MergeOverlappingBufferCacheRange(BufferCacheRange* merged, + BufferCacheRange candidate) noexcept { + if (merged == nullptr || merged->address == 0 || merged->size == 0 || candidate.address == 0 || + candidate.size == 0 || merged->size > UINT64_MAX - merged->address || + candidate.size > UINT64_MAX - candidate.address) { + EXIT("BufferCache: invalid overlap-merge range\n"); + } + const auto merged_end = merged->address + merged->size; + const auto candidate_end = candidate.address + candidate.size; + if (merged->address >= candidate_end || candidate.address >= merged_end) { + return false; + } + const auto address = std::min(merged->address, candidate.address); + const auto end = std::max(merged_end, candidate_end); + *merged = {.address = address, .size = end - address}; + return true; +} + +bool CanMergeBufferCacheQueueMask(uint64_t queue_mask, uint32_t queue) noexcept { + return queue < 64 && (queue_mask & ~(uint64_t {1} << queue)) == 0; +} + +bool CanReadbackBufferCacheQueueMask(uint64_t queue_mask, uint32_t queue) noexcept { + return queue < 64 && queue_mask == (uint64_t {1} << queue); +} + +struct BufferCache::CachedBuffer { + uint64_t vaddr = 0; + uint64_t size = 0; + uint64_t queue_mask = 0; + GraphicContext* ctx = nullptr; + std::shared_ptr buffer; +}; + +struct BufferCache::CommandProcessorReadbackResources { + GraphicContext* ctx = nullptr; + int queue = -1; + VulkanBuffer buffer {}; + void* mapped = nullptr; + std::unique_ptr command; +}; + +struct BufferCache::ReadbackWorker { + static constexpr uint32_t MAX_RANGES = READBACK_MAX_RANGES; + enum class State : uint32_t { + Uninitialized, + InitRequested, + Idle, + Claimed, + Requested, + Ready, + Installed, + Completed, + Stopping, + Stopped + }; + static_assert(std::atomic::is_always_lock_free); + + struct Range { + uint64_t address = 0; + uint32_t size = 0; + uint32_t offset = 0; + }; + + explicit ReadbackWorker(BufferCache& owner): cache(owner), thread([this] { Run(); }) {} + + ~ReadbackWorker() { + auto expected = State::Idle; + if (!state.compare_exchange_strong(expected, State::Stopping, std::memory_order_acq_rel)) { + expected = State::Uninitialized; + if (!state.compare_exchange_strong(expected, State::Stopping, + std::memory_order_acq_rel)) { + EXIT("BufferCache: cannot stop readback worker from state %u\n", + static_cast(expected)); + } + } + state.notify_all(); + thread.join(); + if (state.load(std::memory_order_acquire) != State::Stopped) { + EXIT("BufferCache: readback worker did not reach stopped state\n"); + } + } + + void Prepare(GraphicContext* graphic_context) { + if (graphic_context == nullptr) { + EXIT("BufferCache: cannot prepare readback without a graphics context\n"); + } + auto current = state.load(std::memory_order_acquire); + while (current == State::InitRequested || + (current == State::Claimed && (command == nullptr || mapped == nullptr || + readback.buffer == nullptr))) { + state.wait(current, std::memory_order_acquire); + current = state.load(std::memory_order_acquire); + } + if (current != State::Uninitialized) { + if (current == State::Stopping || current == State::Stopped) { + EXIT("BufferCache: cannot prepare a stopping readback worker, state=%u\n", + static_cast(current)); + } + if (ctx != graphic_context || command == nullptr || mapped == nullptr || + readback.buffer == nullptr) { + EXIT("BufferCache: initialized readback worker has invalid resources, state=%u " + "ctx=%p requested=%p " + "command=%p mapped=%p buffer=%p\n", + static_cast(current), static_cast(ctx), + static_cast(graphic_context), + static_cast(command.get()), static_cast(mapped), + static_cast(readback.buffer)); + } + return; + } + auto expected = State::Uninitialized; + if (!state.compare_exchange_strong(expected, State::Claimed, std::memory_order_acq_rel)) { + EXIT("BufferCache: readback prepare requires uninitialized state, state=%u\n", + static_cast(expected)); + } + ctx = graphic_context; + state.store(State::InitRequested, std::memory_order_release); + state.notify_all(); + while ((current = state.load(std::memory_order_acquire)) != State::Idle) { + if (current != State::InitRequested) { + EXIT("BufferCache: invalid readback initialization state %u\n", + static_cast(current)); + } + state.wait(current, std::memory_order_acquire); + } + if (command == nullptr || mapped == nullptr || readback.buffer == nullptr) { + EXIT("BufferCache: readback initialization produced invalid resources, command=%p " + "mapped=%p buffer=%p\n", + static_cast(command.get()), static_cast(mapped), + static_cast(readback.buffer)); + } + } + + void Request(PageFaultAccess fault_access, uint64_t fault_vaddr, uint64_t fault_size) noexcept { + const bool submissions_prepaused_now = GraphicsRunSubmissionLockHeld(); + const bool unsafe_gpu_lock = GraphicsRunGpuLockHeld() && !submissions_prepaused_now; + if (GraphicsRunIsCommandProcessorThread() || unsafe_gpu_lock || LabelInCallback() || + g_cache_lock_owner != nullptr || + ctx == nullptr || command == nullptr || mapped == nullptr || + readback.buffer == nullptr) { + EXIT("BufferCache: unsafe readback request context, command_thread=%d " + "submission_lock=%d " + "gpu_lock=%d label_callback=%d cache_lock=%p ctx=%p command=%p mapped=%p " + "buffer=%p\n", + GraphicsRunIsCommandProcessorThread(), GraphicsRunSubmissionLockHeld(), + GraphicsRunGpuLockHeld(), LabelInCallback(), + static_cast(g_cache_lock_owner), static_cast(ctx), + static_cast(command.get()), static_cast(mapped), + static_cast(readback.buffer)); + } + State expected = State::Idle; + while (!state.compare_exchange_weak(expected, State::Claimed, std::memory_order_acq_rel)) { + if (expected == State::Stopping || expected == State::Stopped) { + EXIT("BufferCache: readback requested while worker is stopping, state=%u\n", + static_cast(expected)); + } + state.wait(expected, std::memory_order_acquire); + expected = State::Idle; + } + access = fault_access; + vaddr = fault_vaddr; + size = fault_size; + range_count = 0; + submissions_prepaused = submissions_prepaused_now; + state.store(State::Requested, std::memory_order_release); + state.notify_all(); + while (true) { + const auto current = state.load(std::memory_order_acquire); + if (current == State::Ready) { + break; + } + if (current != State::Requested) { + EXIT("BufferCache: invalid state while waiting for readback, state=%u\n", + static_cast(current)); + } + state.wait(current, std::memory_order_acquire); + } + } + + [[nodiscard]] bool Complete(PageFaultAccess fault_access, uint64_t fault_vaddr, + uint64_t fault_size) noexcept { + const auto current = state.load(std::memory_order_acquire); + if (current == State::Uninitialized || current == State::InitRequested || + current == State::Idle || current == State::Stopping || current == State::Stopped) { + return false; + } + if (current != State::Ready) { + EXIT("BufferCache: active readback has invalid completion state %u\n", + static_cast(current)); + } + if (access != fault_access || vaddr != fault_vaddr || size != fault_size) { + EXIT("BufferCache: mismatched active readback completion\n"); + } + if (range_count == 0 || range_count > MAX_RANGES) { + EXIT("BufferCache: invalid completed readback range count %u\n", range_count); + } + for (uint32_t i = 0; i < range_count; i++) { + const auto& range = ranges[i]; + Libs::LibKernel::Memory::WriteBacking(range.address, data.data() + range.offset, + range.size); + } + if (!cache.m_memory_tracker.CompleteCpuFault(vaddr, size, access, true)) { + EXIT("BufferCache: failed to complete downloaded CPU fault, addr=0x%016" PRIx64 + " size=0x%016" PRIx64 " access=%u\n", + vaddr, size, static_cast(access)); + } + state.store(State::Installed, std::memory_order_release); + state.notify_all(); + return true; + } + + void Release(PageFaultAccess fault_access, uint64_t fault_vaddr, uint64_t fault_size) noexcept { + const auto current = state.load(std::memory_order_acquire); + if (current == State::Uninitialized || current == State::InitRequested || + current == State::Idle || current == State::Stopping || current == State::Stopped) { + return; + } + if (current != State::Installed) { + EXIT("BufferCache: active readback has invalid release state %u\n", + static_cast(current)); + } + if (access != fault_access || vaddr != fault_vaddr || size != fault_size) { + EXIT("BufferCache: mismatched active readback release\n"); + } + state.store(State::Completed, std::memory_order_release); + state.notify_all(); + while (true) { + const auto current = state.load(std::memory_order_acquire); + if (current == State::Idle) { + break; + } + if (current != State::Completed) { + EXIT("BufferCache: invalid state while releasing readback, state=%u\n", + static_cast(current)); + } + state.wait(current, std::memory_order_acquire); + } + } + + void Run() noexcept { + while (true) { + auto current = state.load(std::memory_order_acquire); + while (current == State::Uninitialized || current == State::Idle) { + state.wait(current, std::memory_order_acquire); + current = state.load(std::memory_order_acquire); + } + if (current == State::Stopping) { + command.reset(); + if (mapped != nullptr) { + VulkanUnmapMemory(ctx, &readback.memory); + mapped = nullptr; + } + if (readback.buffer != nullptr) { + VulkanDeleteBuffer(ctx, &readback); + } + state.store(State::Stopped, std::memory_order_release); + state.notify_all(); + return; + } + if (current == State::InitRequested) { + if (ctx == nullptr || command != nullptr || mapped != nullptr || + readback.buffer != nullptr) { + EXIT("BufferCache: invalid resources before readback initialization, ctx=%p " + "command=%p mapped=%p buffer=%p\n", + static_cast(ctx), static_cast(command.get()), + static_cast(mapped), + static_cast(readback.buffer)); + } + const auto family = ctx->queues[GraphicContext::QUEUE_UTIL].family; + if (family == static_cast(-1) || + ctx->queues[GraphicContext::QUEUE_GFX].family != family) { + EXIT("BufferCache: utility and graphics queues must share a valid family, " + "util=%u " + "gfx=%u\n", + family, ctx->queues[GraphicContext::QUEUE_GFX].family); + } + for (int i = GraphicContext::QUEUE_COMPUTE_START; + i < GraphicContext::QUEUE_COMPUTE_START + GraphicContext::QUEUE_COMPUTE_NUM; + i++) { + if (ctx->queues[i].family != family) { + EXIT("BufferCache: compute queue %d family mismatch, expected=%u " + "actual=%u\n", + i, family, ctx->queues[i].family); + } + } + readback.usage = VK_BUFFER_USAGE_TRANSFER_DST_BIT; + readback.memory.property = VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT | + VK_MEMORY_PROPERTY_HOST_COHERENT_BIT | + VK_MEMORY_PROPERTY_HOST_CACHED_BIT; + VulkanCreateBuffer(ctx, READBACK_CAPACITY, &readback); + VulkanMapMemory(ctx, &readback.memory, &mapped); + command = std::make_unique(GraphicContext::QUEUE_UTIL); + state.store(State::Idle, std::memory_order_release); + state.notify_all(); + continue; + } + if (current != State::Requested) { + EXIT("BufferCache: worker received unsupported state %u\n", + static_cast(current)); + } + + std::optional submissions; + if (!submissions_prepaused) { + submissions.emplace(); + } + const auto page = vaddr & ~(TRACKER_PAGE_SIZE - 1); + FaultSafeCacheLock lock(&cache, cache.m_mutex); + auto it = cache.m_buffers.upper_bound(vaddr); + if (it == cache.m_buffers.begin()) { + EXIT("BufferCache: readback address has no cached buffer, addr=0x%016" PRIx64 "\n", + vaddr); + } + --it; + auto& cached = *it->second; + if (vaddr < cached.vaddr || vaddr >= cached.vaddr + cached.size || + page < cached.vaddr || TRACKER_PAGE_SIZE > cached.size - (page - cached.vaddr)) { + EXIT("BufferCache: readback page is outside cached buffer, fault=0x%016" PRIx64 + " page=0x%016" PRIx64 " buffer=0x%016" PRIx64 " size=0x%016" PRIx64 "\n", + vaddr, page, cached.vaddr, cached.size); + } + uint32_t data_offset = 0; + std::array copies {}; + std::array barriers {}; + cache.m_gpu_modified_ranges.ForEachIntersection( + page, TRACKER_PAGE_SIZE, [&](RangeSet::Range range) { + if (range_count == MAX_RANGES || (range.address - cached.vaddr) % 4 != 0 || + range.size % 4 != 0) { + EXIT("BufferCache: invalid GPU-modified readback range, addr=0x%016" PRIx64 + " size=0x%016" PRIx64 " offset=%u count=%u\n", + range.address, range.size, data_offset, range_count); + } + auto& out = ranges[range_count]; + out.address = range.address; + out.size = static_cast(range.size); + out.offset = data_offset; + copies[range_count] = {.srcOffset = range.address - cached.vaddr, + .dstOffset = data_offset, + .size = range.size}; + auto& barrier = barriers[range_count]; + barrier.sType = VK_STRUCTURE_TYPE_BUFFER_MEMORY_BARRIER; + barrier.srcAccessMask = VK_ACCESS_MEMORY_WRITE_BIT; + barrier.dstAccessMask = VK_ACCESS_TRANSFER_READ_BIT; + barrier.srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; + barrier.dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; + barrier.buffer = cached.buffer->buffer; + barrier.offset = copies[range_count].srcOffset; + barrier.size = range.size; + range_count++; + const auto slot_size = AlignReadbackCopySize(out.size); + if (slot_size > data.size() - data_offset) { + EXIT("BufferCache: aligned readback range exceeds persistent capacity, " + "offset=%u size=%" PRIu64 " capacity=%" PRIu64 "\n", + data_offset, slot_size, static_cast(data.size())); + } + data_offset += static_cast(slot_size); + }); + if (range_count == 0 || data_offset == 0) { + EXIT( + "BufferCache: GPU-modified page produced no readback ranges, page=0x%016" PRIx64 + " count=%u bytes=%u\n", + page, range_count, data_offset); + } + auto* vk_buffer = command->GetPool()->buffers[command->GetIndex()]; + command->Begin(); + vkCmdPipelineBarrier(vk_buffer, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, + VK_PIPELINE_STAGE_TRANSFER_BIT, 0, 0, nullptr, range_count, + barriers.data(), 0, nullptr); + vkCmdCopyBuffer(vk_buffer, cached.buffer->buffer, readback.buffer, range_count, + copies.data()); + command->End(); + command->Execute(); + command->WaitForFenceAndReset(); + std::memcpy(data.data(), mapped, data_offset); + state.store(State::Ready, std::memory_order_release); + state.notify_all(); + while ((current = state.load(std::memory_order_acquire)) != State::Completed) { + if (current != State::Ready && current != State::Installed) { + EXIT("BufferCache: invalid readback completion state %u\n", + static_cast(current)); + } + state.wait(current, std::memory_order_acquire); + } + cache.m_gpu_modified_ranges.Subtract(page, TRACKER_PAGE_SIZE); + submissions_prepaused = false; + state.store(State::Idle, std::memory_order_release); + state.notify_all(); + } + } + + BufferCache& cache; + std::atomic state {State::Uninitialized}; + GraphicContext* ctx = nullptr; + VulkanBuffer readback {}; + void* mapped = nullptr; + std::unique_ptr command; + PageFaultAccess access = PageFaultAccess::Unknown; + uint64_t vaddr = 0; + uint64_t size = 0; + uint32_t range_count = 0; + bool submissions_prepaused = false; + std::array ranges {}; + std::array data {}; + std::thread thread; +}; + +BufferCache::BufferCache(PageManager& page_manager, ResourceMutex& resource_mutex) + : m_memory_tracker(page_manager), m_page_manager(page_manager), + m_resource_mutex(resource_mutex) { + EXIT_NOT_IMPLEMENTED(!Common::Thread::IsMainThread()); + m_readback = std::make_unique(*this); +} + +BufferCache::~BufferCache() { + m_readback.reset(); + if (m_cp_readback_resources != nullptr) { + auto& resources = *m_cp_readback_resources; + resources.command.reset(); + if (resources.mapped != nullptr) { + VulkanUnmapMemory(resources.ctx, &resources.buffer.memory); + resources.mapped = nullptr; + } + if (resources.buffer.buffer != nullptr) { + VulkanDeleteBuffer(resources.ctx, &resources.buffer); + } + m_cp_readback_resources.reset(); + } + if (!m_gpu_modified_ranges.Empty()) { + EXIT("BufferCache: destroyed with pending GPU-modified ranges\n"); + } + for (const auto& [vaddr, cached]: m_buffers) { + (void)vaddr; + if (m_memory_tracker.IsRegionGpuModified(cached->vaddr, cached->size)) { + EXIT("BufferCache: destroyed with GPU-modified buffer, addr=0x%016" PRIx64 + " size=0x%016" PRIx64 "\n", + cached->vaddr, cached->size); + } + } + if (!m_buffers.empty()) { + VulkanDeviceWaitIdle(m_buffers.begin()->second->ctx); + } + m_buffers.clear(); +} + +bool BufferCache::InvalidateMemory(PageFaultAccess access, uint64_t vaddr, uint64_t size, + PageFaultPhase phase) noexcept { + const auto page = vaddr & ~(TRACKER_PAGE_SIZE - 1); + if (size == 0 || size > page + TRACKER_PAGE_SIZE - vaddr) { + EXIT("BufferCache: invalid page-fault range, addr=0x%016" PRIx64 " size=0x%016" PRIx64 "\n", + vaddr, size); + } + switch (phase) { + case PageFaultPhase::Invalidate: break; + case PageFaultPhase::Complete: + return CompleteCommandProcessorReadback(access, vaddr, size) || + m_readback->Complete(access, vaddr, size) || + m_memory_tracker.CompleteCpuFault(vaddr, size, access, false); + case PageFaultPhase::Release: + if (!ReleaseCommandProcessorReadback(access, vaddr, size)) { + m_readback->Release(access, vaddr, size); + } + return true; + default: + EXIT("BufferCache: unsupported page-fault phase %u\n", static_cast(phase)); + } + const auto action = m_memory_tracker.BeginCpuFault(vaddr, size, access); + if (action != CpuFaultAction::Download) { + return action == CpuFaultAction::Continue; + } + if (GraphicsRunIsCommandProcessorThread()) { + if (!GraphicsRunIsCommandProcessorGuestAccess()) { + EXIT("BufferCache: command-processor fault outside a coherent guest-memory access, " + "addr=0x%016" PRIx64 " size=0x%016" PRIx64 "\n", + vaddr, size); + } + RequestCommandProcessorReadback(access, vaddr, size); + } else { + m_readback->Request(access, vaddr, size); + } + return true; +} + +void BufferCache::RequestCommandProcessorReadback(PageFaultAccess access, uint64_t vaddr, + uint64_t size) { + if (!GraphicsRunIsCommandProcessorThread() || g_command_processor_readback.cache != nullptr) { + EXIT("BufferCache: invalid nested command-processor readback\n"); + } + const auto serial = + g_command_processor_readback_serial.fetch_add(1, std::memory_order_relaxed) + 1; + if (serial == 0) { + EXIT("BufferCache: command-processor readback serial overflow\n"); + } + const bool log_detail = ShouldLogCommandProcessorReadback(serial); + if (log_detail) { + LOGF("BufferCache: command-processor readback request serial=%" PRIu64 + " addr=0x%016" PRIx64 " size=0x%016" PRIx64 " access=%u poll=%d\n", + serial, vaddr, size, static_cast(access), GraphicsRunIsGuestMemoryPoll()); + Log::Flush(); + } + const auto page = vaddr & ~(TRACKER_PAGE_SIZE - 1); + const auto queue = static_cast(GraphicsRunBeginCommandProcessorReadback()); + if (log_detail) { + LOGF("BufferCache: command-processor queue synchronized serial=%" PRIu64 + " queue=%u page=0x%016" PRIx64 "\n", + serial, queue, page); + Log::Flush(); + } + if (g_cache_lock_owner != nullptr) { + EXIT("BufferCache: command-processor readback entered with a cache lock\n"); + } + g_cache_lock_owner = this; + m_mutex.Lock(); + if (log_detail) { + LOGF("BufferCache: command-processor cache locked serial=%" PRIu64 + " page=0x%016" PRIx64 "\n", + serial, page); + Log::Flush(); + } + + auto it = m_buffers.upper_bound(vaddr); + if (it == m_buffers.begin()) { + EXIT("BufferCache: command-processor readback address has no cached buffer, " + "addr=0x%016" PRIx64 "\n", + vaddr); + } + --it; + auto& cached = *it->second; + if (vaddr < cached.vaddr || vaddr >= cached.vaddr + cached.size || page < cached.vaddr || + TRACKER_PAGE_SIZE > cached.size - (page - cached.vaddr) || cached.ctx == nullptr || + cached.buffer == nullptr || cached.buffer->buffer == nullptr) { + EXIT("BufferCache: invalid command-processor readback owner, fault=0x%016" PRIx64 + " page=0x%016" PRIx64 " buffer=0x%016" PRIx64 "+0x%016" PRIx64 "\n", + vaddr, page, cached.vaddr, cached.size); + } + if (!CanReadbackBufferCacheQueueMask(cached.queue_mask, queue)) { + EXIT("BufferCache: command-processor readback cannot synchronize cross-queue ownership, " + "addr=0x%016" PRIx64 " used_queues=0x%016" PRIx64 " current_queue=%u\n", + vaddr, cached.queue_mask, queue); + } + const auto dirty = m_gpu_modified_ranges.Intersections(page, TRACKER_PAGE_SIZE); + if (dirty.empty()) { + EXIT("BufferCache: command-processor GPU-dirty page has no dirty byte ranges, " + "page=0x%016" PRIx64 "\n", + page); + } + ValidateDirtyPages(m_gpu_modified_ranges, page, TRACKER_PAGE_SIZE, "command-processor"); + if (m_cp_readback_resources == nullptr) { + auto resources = std::make_unique(); + resources->ctx = cached.ctx; + resources->buffer.usage = VK_BUFFER_USAGE_TRANSFER_DST_BIT; + resources->buffer.memory.property = VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT | + VK_MEMORY_PROPERTY_HOST_COHERENT_BIT | + VK_MEMORY_PROPERTY_HOST_CACHED_BIT; + VulkanCreateBuffer(cached.ctx, READBACK_CAPACITY, &resources->buffer); + VulkanMapMemory(cached.ctx, &resources->buffer.memory, &resources->mapped); + m_cp_readback_resources = std::move(resources); + } + auto& resources = *m_cp_readback_resources; + if (resources.ctx != cached.ctx || resources.mapped == nullptr || + resources.buffer.buffer == nullptr) { + EXIT("BufferCache: invalid command-processor readback resources\n"); + } + if (queue >= GraphicContext::QUEUES_NUM || cached.ctx->queues[queue].family == UINT32_MAX || + cached.ctx->queues[queue].vk_queue == nullptr) { + EXIT("BufferCache: command-processor readback has invalid Vulkan queue %u\n", queue); + } + if (resources.command == nullptr || resources.queue != static_cast(queue)) { + resources.command.reset(); + resources.command = std::make_unique(static_cast(queue)); + resources.queue = static_cast(queue); + } + if (resources.command->IsInvalid() || resources.command->GetQueue() != static_cast(queue)) { + EXIT("BufferCache: failed to initialize owning-queue readback command buffer, queue=%u\n", + queue); + } + if (dirty.size() > ReadbackWorker::MAX_RANGES) { + EXIT("BufferCache: too many command-processor dirty ranges, page=0x%016" PRIx64 + " count=%" PRIu64 " limit=%u\n", + page, static_cast(dirty.size()), ReadbackWorker::MAX_RANGES); + } + std::array copies {}; + std::array barriers {}; + uint64_t dirty_bytes = 0; + uint64_t packed_bytes = 0; + uint32_t dirty_count = 0; + for (const auto& range: dirty) { + if ((range.address - cached.vaddr) % 4 != 0 || range.size % 4 != 0) { + EXIT("BufferCache: invalid command-processor dirty range, addr=0x%016" PRIx64 + " size=0x%016" PRIx64 " page=0x%016" PRIx64 " packed=0x%016" PRIx64 "\n", + range.address, range.size, page, dirty_bytes); + } + copies[dirty_count] = {.srcOffset = range.address - cached.vaddr, + .dstOffset = packed_bytes, + .size = range.size}; + auto& barrier = barriers[dirty_count]; + barrier.sType = VK_STRUCTURE_TYPE_BUFFER_MEMORY_BARRIER; + barrier.srcAccessMask = VK_ACCESS_MEMORY_WRITE_BIT; + barrier.dstAccessMask = VK_ACCESS_TRANSFER_READ_BIT; + barrier.srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; + barrier.dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; + barrier.buffer = cached.buffer->buffer; + barrier.offset = copies[dirty_count].srcOffset; + barrier.size = range.size; + const auto slot_size = AlignReadbackCopySize(range.size); + if (packed_bytes > READBACK_CAPACITY || slot_size > READBACK_CAPACITY - packed_bytes) { + EXIT("BufferCache: aligned command-processor readback exceeds persistent capacity, " + "packed=%" PRIu64 " size=%" PRIu64 " capacity=%" PRIu64 "\n", + packed_bytes, slot_size, READBACK_CAPACITY); + } + packed_bytes += slot_size; + dirty_bytes += range.size; + dirty_count++; + } + if (dirty_count == 0 || dirty_bytes == 0) { + EXIT("BufferCache: command-processor readback produced no packed ranges\n"); + } + const auto started = std::chrono::steady_clock::now(); + if (log_detail) { + LOGF("BufferCache: command-processor readback begin serial=%" PRIu64 + " queue=%u page=0x%016" PRIx64 " ranges=%u bytes=%" PRIu64 + " packed=%" PRIu64 "\n", + serial, queue, page, dirty_count, dirty_bytes, packed_bytes); + Log::Flush(); + } + auto* vk_command = resources.command->GetPool()->buffers[resources.command->GetIndex()]; + resources.command->Begin(); + vkCmdPipelineBarrier(vk_command, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, + VK_PIPELINE_STAGE_TRANSFER_BIT, 0, 0, nullptr, dirty_count, + barriers.data(), 0, nullptr); + vkCmdCopyBuffer(vk_command, cached.buffer->buffer, resources.buffer.buffer, dirty_count, + copies.data()); + resources.command->End(); + resources.command->Execute(); + resources.command->WaitForFenceAndReset(); + if (log_detail) { + LOGF("BufferCache: command-processor GPU copy complete serial=%" PRIu64 "\n", serial); + Log::Flush(); + } + for (uint32_t i = 0; i < dirty_count; i++) { + Libs::LibKernel::Memory::WriteBacking( + dirty[i].address, + static_cast(resources.mapped) + copies[i].dstOffset, dirty[i].size); + } + g_command_processor_readback = { + .cache = this, + .access = access, + .vaddr = vaddr, + .size = size, + .page = page, + .serial = serial, + .dirty_bytes = dirty_bytes, + .packed_bytes = packed_bytes, + .dirty_count = dirty_count, + .started = started}; +} + +bool BufferCache::CompleteCommandProcessorReadback(PageFaultAccess access, uint64_t vaddr, + uint64_t size) noexcept { + auto& pending = g_command_processor_readback; + if (pending.cache == nullptr) { + return false; + } + if (pending.cache != this || pending.access != access || pending.vaddr != vaddr || + pending.size != size || pending.installed) { + EXIT("BufferCache: mismatched active command-processor readback completion\n"); + } + if (!m_memory_tracker.CompleteCpuFault(vaddr, size, access, true)) { + EXIT("BufferCache: failed to complete command-processor readback, addr=0x%016" PRIx64 + " size=0x%016" PRIx64 " access=%u\n", + vaddr, size, static_cast(access)); + } + m_gpu_modified_ranges.Subtract(pending.page, TRACKER_PAGE_SIZE); + pending.installed = true; + if (ShouldLogCommandProcessorReadback(pending.serial)) { + LOGF("BufferCache: command-processor readback complete serial=%" PRIu64 + " page=0x%016" PRIx64 " ranges=%u bytes=%" PRIu64 "\n", + pending.serial, pending.page, pending.dirty_count, pending.dirty_bytes); + } + return true; +} + +bool BufferCache::ReleaseCommandProcessorReadback(PageFaultAccess access, uint64_t vaddr, + uint64_t size) noexcept { + auto& pending = g_command_processor_readback; + if (pending.cache == nullptr) { + return false; + } + if (pending.cache != this || pending.access != access || pending.vaddr != vaddr || + pending.size != size || !pending.installed) { + EXIT("BufferCache: mismatched active command-processor readback release\n"); + } + const auto serial = pending.serial; + const auto page = pending.page; + const auto dirty_count = pending.dirty_count; + const auto dirty_bytes = pending.dirty_bytes; + const auto elapsed_micro = std::chrono::duration_cast( + std::chrono::steady_clock::now() - pending.started) + .count(); + pending = {}; + m_mutex.Unlock(); + g_cache_lock_owner = nullptr; + GraphicsRunEndCommandProcessorReadback(); + if (ShouldLogCommandProcessorReadback(serial)) { + LOGF("BufferCache: command-processor readback release serial=%" PRIu64 + " page=0x%016" PRIx64 " ranges=%u bytes=%" PRIu64 " elapsed_us=%" PRIi64 "\n", + serial, page, dirty_count, dirty_bytes, static_cast(elapsed_micro)); + } + return true; +} + +void BufferCache::UnmapMemory(uint64_t vaddr, uint64_t size) { + GraphicsRunSubmissionLock submissions; + FaultSafeCacheLock lock(this, m_mutex); + for (const auto& [begin, cached]: m_buffers) { + if (vaddr < begin + cached->size && begin < vaddr + size && + (vaddr > begin || size < cached->size || vaddr + size < begin + cached->size)) { + EXIT("BufferCache: partial buffer unmap is unsupported, unmap=0x%016" PRIx64 + "+0x%016" PRIx64 " buffer=0x%016" PRIx64 "+0x%016" PRIx64 "\n", + vaddr, size, begin, cached->size); + } + } + for (const auto& [begin, cached]: m_buffers) { + const auto offset = begin >= vaddr ? begin - vaddr : UINT64_MAX; + if (offset > size || cached->size > size - offset || + !m_memory_tracker.IsRegionGpuModified(begin, cached->size)) { + continue; + } + const auto dirty = m_gpu_modified_ranges.Intersections(begin, cached->size); + if (dirty.empty()) { + EXIT("BufferCache: GPU-modified buffer has no dirty ranges, addr=0x%016" PRIx64 + " size=0x%016" PRIx64 "\n", + begin, cached->size); + } + auto backing_writes = ReserveBackingWrites(m_page_manager, dirty); + uint64_t downloaded = 0; + m_memory_tracker.ForEachDownloadRange( + begin, cached->size, + [&](uint64_t address, uint64_t bytes) noexcept { + ValidateDirtyPages(m_gpu_modified_ranges, address, bytes, "unmap"); + }, + [&](uint64_t address, uint64_t bytes) noexcept { + const auto ranges = m_gpu_modified_ranges.Intersections(address, bytes); + for (const auto& range: ranges) { + std::vector data(range.size); + UtilDownloadBuffer(cached->ctx, cached->buffer.get(), range.address - begin, + data.data(), range.size); + Libs::LibKernel::Memory::WriteBacking(range.address, data.data(), data.size()); + downloaded += range.size; + } + }); + if (downloaded == 0) { + EXIT("BufferCache: GPU-modified buffer downloaded no bytes, addr=0x%016" PRIx64 + " size=0x%016" PRIx64 "\n", + begin, cached->size); + } + m_memory_tracker.MarkRegionAsCpuModified(begin, cached->size); + m_gpu_modified_ranges.Subtract(begin, cached->size); + } + if (!m_gpu_modified_ranges.Intersections(vaddr, size).empty()) { + EXIT("BufferCache: unmap retained dirty byte ranges, addr=0x%016" PRIx64 + " size=0x%016" PRIx64 "\n", + vaddr, size); + } + m_memory_tracker.UntrackMemory(vaddr, size); + for (auto it = m_buffers.begin(); it != m_buffers.end();) { + const auto offset = (it->first >= vaddr ? it->first - vaddr : UINT64_MAX); + if (offset <= size && it->second->size <= size - offset) { + it = m_buffers.erase(it); + } else { + ++it; + } + } +} + +std::pair BufferCache::ObtainBuffer(CommandBuffer* command, + GraphicContext* ctx, uint64_t vaddr, + uint64_t size, bool is_written, + bool is_read, bool is_formatted) { + if (command == nullptr || command->IsInvalid() || command->IsExecute() || ctx == nullptr || + vaddr == 0 || size == 0 || size > UINT64_MAX - vaddr || command->GetQueue() < 0 || + command->GetQueue() >= 64) { + EXIT("BufferCache: invalid buffer request, command=%p queue=%d ctx=%p addr=0x%016" PRIx64 + " size=0x%016" PRIx64 "\n", + static_cast(command), command == nullptr ? -1 : command->GetQueue(), + static_cast(ctx), vaddr, size); + } + const auto queue = static_cast(command->GetQueue()); + const auto queue_mask = uint64_t {1} << queue; + ValidateGpuAccess(vaddr, size, is_read, is_written); + const auto begin = AlignDown(vaddr); + const auto end = AlignUp(vaddr + size); + std::lock_guard transaction(m_resource_mutex); + // Use the stream-buffer fast path before image/buffer alias handling. Clean image and metadata + // views may coexist with a small CPU-current read; Kyty's separate image trackers require + // GPU-dirty ownership guards here. Read physical backing so + // host page protection is irrelevant. + if (is_read && !is_written && size <= CACHING_PAGE_SIZE && + !m_memory_tracker.IsRegionGpuModified(vaddr, size) && + m_memory_tracker.IsRegionCpuModified(vaddr, size) && + !m_texture_cache->HasGpuModifiedRangeOverlap(vaddr, size) && + !m_texture_cache->IsMetaGpuModified(vaddr, size)) { + std::array guest_data; + if (Libs::LibKernel::Memory::TryReadBacking(vaddr, guest_data.data(), size)) { + VulkanBuffer* stream_buffer = nullptr; + VkDeviceSize stream_offset = 0; + VkDeviceSize stream_range = 0; + const auto stream_alignment = std::max(ctx->StorageMinAlignment(), 16); + if (UploadHostData(command, ctx, guest_data.data(), size, stream_alignment, + &stream_buffer, &stream_offset, &stream_range)) { + return {stream_buffer, stream_offset}; + } + } + } + if (m_texture_cache->HasMetaOverlap(begin, end - begin)) { + EXIT("BufferCache: buffer aliases metadata pages, addr=0x%016" PRIx64 " size=0x%016" PRIx64 + "\n", + begin, end - begin); + } + // Cache allocations are tracker-page aligned, but byte-disjoint buffers and images may share + // an edge page. Clean read-only buffer and image views may coexist; Kyty retains a hard failure + // when either cache owns newer GPU bytes. Only a formatted buffer write + // performs the image ownership transition below. + if (m_texture_cache->HasPageOverlap(begin, end - begin) && + m_texture_cache->HasRangeOverlap(vaddr, size)) { + const bool coherent_read = + is_read && !is_written && !m_memory_tracker.IsRegionGpuModified(vaddr, size) && + !m_texture_cache->HasGpuModifiedRangeOverlap(vaddr, size); + if (!coherent_read) { + if (!is_written || !is_formatted) { + EXIT("BufferCache: unsupported buffer/image alias, addr=0x%016" PRIx64 + " size=0x%016" PRIx64 " read=%d written=%d formatted=%d\n", + vaddr, size, is_read, is_written, is_formatted); + } + m_texture_cache->InvalidateMemoryFromGPU(vaddr, size, true); + } + } + if (is_written) { + m_readback->Prepare(ctx); + } + FaultSafeCacheLock lock(this, m_mutex); + + auto it = m_buffers.upper_bound(vaddr); + if (it != m_buffers.begin()) { + auto previous = std::prev(it); + const auto offset = vaddr - previous->second->vaddr; + if (offset <= previous->second->size && size <= previous->second->size - offset) { + it = previous; + } + } + if (it == m_buffers.end() || it->second->vaddr > vaddr || + size > it->second->size - (vaddr - it->second->vaddr)) { + BufferCacheRange merged {.address = begin, .size = end - begin}; + using BufferIterator = decltype(m_buffers.begin()); + std::vector overlaps; + auto first = m_buffers.lower_bound(begin); + if (first != m_buffers.begin()) { + auto previous = std::prev(first); + if (MergeOverlappingBufferCacheRange( + &merged, {previous->second->vaddr, previous->second->size})) { + first = previous; + } + } + for (auto candidate = first; candidate != m_buffers.end(); ++candidate) { + if (candidate->first >= merged.address + merged.size) { + break; + } + if (MergeOverlappingBufferCacheRange( + &merged, {candidate->second->vaddr, candidate->second->size})) { + overlaps.push_back(candidate); + } + } + + if (!overlaps.empty()) { + for (const auto overlap: overlaps) { + auto& old = *overlap->second; + if (old.ctx != ctx || old.buffer == nullptr || old.buffer->buffer == nullptr) { + EXIT("BufferCache: invalid overlapping buffer owner, addr=0x%016" PRIx64 + " size=0x%016" PRIx64 " owner_ctx=%p requested_ctx=%p buffer=%p\n", + old.vaddr, old.size, static_cast(old.ctx), + static_cast(ctx), static_cast(old.buffer.get())); + } + if (!CanMergeBufferCacheQueueMask(old.queue_mask, queue)) { + EXIT("BufferCache: cross-queue overlap merge is unsupported, " + "addr=0x%016" PRIx64 " size=0x%016" PRIx64 " used_queues=0x%016" PRIx64 + " requested_queue=%u\n", + old.vaddr, old.size, old.queue_mask, queue); + } + std::vector> uploads; + m_memory_tracker.ForEachUploadRange( + old.vaddr, old.size, false, + [&](uint64_t upload_vaddr, uint64_t upload_size) noexcept { + uploads.emplace_back(upload_vaddr, upload_size); + }, + [&]() noexcept { + for (const auto& [upload_vaddr, upload_size]: uploads) { + UtilUploadBuffer(ctx, StagingBufferType::Vertex, old.buffer.get(), + upload_vaddr - old.vaddr, + reinterpret_cast(upload_vaddr), + upload_size); + } + }); + } + } + auto cached = std::make_unique(); + cached->vaddr = merged.address; + cached->size = merged.size; + cached->ctx = ctx; + cached->buffer = MakeSharedVulkanBuffer(ctx); + cached->buffer->usage = + VK_BUFFER_USAGE_TRANSFER_SRC_BIT | VK_BUFFER_USAGE_TRANSFER_DST_BIT | + VK_BUFFER_USAGE_VERTEX_BUFFER_BIT | VK_BUFFER_USAGE_INDEX_BUFFER_BIT | + VK_BUFFER_USAGE_STORAGE_BUFFER_BIT; + cached->buffer->memory.property = VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT; + VulkanCreateBuffer(ctx, cached->size, cached->buffer.get()); + if (!overlaps.empty()) { + auto* vk_buffer = command->GetPool()->buffers[command->GetIndex()]; + std::vector before; + before.reserve(overlaps.size() + 1); + for (const auto overlap: overlaps) { + VkBufferMemoryBarrier barrier {}; + barrier.sType = VK_STRUCTURE_TYPE_BUFFER_MEMORY_BARRIER; + barrier.srcAccessMask = VK_ACCESS_MEMORY_WRITE_BIT; + barrier.dstAccessMask = VK_ACCESS_TRANSFER_READ_BIT; + barrier.srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; + barrier.dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; + barrier.buffer = overlap->second->buffer->buffer; + barrier.offset = 0; + barrier.size = overlap->second->size; + before.push_back(barrier); + } + VkBufferMemoryBarrier destination {}; + destination.sType = VK_STRUCTURE_TYPE_BUFFER_MEMORY_BARRIER; + destination.dstAccessMask = VK_ACCESS_TRANSFER_WRITE_BIT; + destination.srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; + destination.dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; + destination.buffer = cached->buffer->buffer; + destination.offset = 0; + destination.size = cached->size; + before.push_back(destination); + vkCmdPipelineBarrier(vk_buffer, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, + VK_PIPELINE_STAGE_TRANSFER_BIT, VK_DEPENDENCY_BY_REGION_BIT, 0, + nullptr, static_cast(before.size()), before.data(), 0, + nullptr); + for (const auto overlap: overlaps) { + const auto& old = *overlap->second; + const VkBufferCopy copy { + .srcOffset = 0, .dstOffset = old.vaddr - cached->vaddr, .size = old.size}; + vkCmdCopyBuffer(vk_buffer, old.buffer->buffer, cached->buffer->buffer, 1, ©); + } + VkBufferMemoryBarrier after {}; + after.sType = VK_STRUCTURE_TYPE_BUFFER_MEMORY_BARRIER; + after.srcAccessMask = VK_ACCESS_TRANSFER_WRITE_BIT; + after.dstAccessMask = VK_ACCESS_MEMORY_READ_BIT | VK_ACCESS_MEMORY_WRITE_BIT; + after.srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; + after.dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; + after.buffer = cached->buffer->buffer; + after.offset = 0; + after.size = cached->size; + vkCmdPipelineBarrier(vk_buffer, VK_PIPELINE_STAGE_TRANSFER_BIT, + VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, VK_DEPENDENCY_BY_REGION_BIT, 0, + nullptr, 1, &after, 0, nullptr); + for (const auto overlap: overlaps) { + command->RetainResourceUntilFence(overlap->second->buffer); + m_buffers.erase(overlap); + } + } + it = m_buffers.emplace(cached->vaddr, std::move(cached)).first; + } + + auto& cached = *it->second; + std::vector> ranges; + ranges.reserve((size + 2 * TRACKER_PAGE_SIZE - 2) / TRACKER_PAGE_SIZE); + m_memory_tracker.ForEachUploadRange( + vaddr, size, is_written, + [&](uint64_t upload_vaddr, uint64_t upload_size) noexcept { + ranges.emplace_back(upload_vaddr, upload_size); + }, + [&]() noexcept { + for (const auto& [upload_vaddr, upload_size]: ranges) { + UtilUploadBuffer(ctx, StagingBufferType::Vertex, cached.buffer.get(), + upload_vaddr - cached.vaddr, + reinterpret_cast(upload_vaddr), upload_size); + } + }); + if (is_written) { + m_gpu_modified_ranges.Add(vaddr, size); + } + cached.queue_mask |= queue_mask; + command->RetainResourceUntilFence(cached.buffer); + return {cached.buffer.get(), vaddr - cached.vaddr}; +} + +bool BufferCache::UploadHostData(CommandBuffer* command, GraphicContext* ctx, const void* src, + uint64_t size, uint64_t alignment, VulkanBuffer** out_buffer, + uint64_t* out_offset, uint64_t* out_range) { + if (command == nullptr || command->IsInvalid() || command->IsExecute() || ctx == nullptr) { + EXIT("BufferCache: host stream upload requires a recording command buffer\n"); + } + return command->m_host_stream.Copy(ctx, src, size, alignment, out_buffer, out_offset, out_range); +} + +VulkanBuffer* BufferCache::ObtainNullBuffer(CommandBuffer* command, GraphicContext* ctx) { + if (command == nullptr || command->IsInvalid() || command->IsExecute() || ctx == nullptr) { + EXIT("BufferCache: null buffer requires a graphics context\n"); + } + FaultSafeCacheLock lock(this, m_mutex); + if (m_null_buffer == nullptr) { + // robustBufferAccess makes every fetch safe; TODO: Use a + // persistent 16-byte fallback when Vulkan null vertex descriptors are unavailable. + auto buffer = MakeSharedVulkanBuffer(ctx); + buffer->usage = VK_BUFFER_USAGE_STORAGE_BUFFER_BIT | VK_BUFFER_USAGE_VERTEX_BUFFER_BIT | + VK_BUFFER_USAGE_INDEX_BUFFER_BIT; + buffer->memory.property = static_cast(VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT) | + VK_MEMORY_PROPERTY_HOST_COHERENT_BIT | + VK_MEMORY_PROPERTY_HOST_CACHED_BIT; + VulkanCreateBuffer(ctx, 16, buffer.get()); + void* data = nullptr; + VulkanMapMemory(ctx, &buffer->memory, &data); + std::memset(data, 0, 16); + VulkanUnmapMemory(ctx, &buffer->memory); + m_null_buffer = std::move(buffer); + } + + // The buffer remains cache-persistent, while every recorded consumer keeps the allocation + // alive until its own fence even if an unrelated command processor resets the global cache. + command->RetainResourceUntilFence(m_null_buffer); + return m_null_buffer.get(); +} + +BufferImageCopySource BufferCache::ObtainBufferForImage(uint64_t vaddr, uint64_t size) { + if (vaddr == 0 || size == 0 || vaddr >= TRACKER_ADDRESS_SIZE || + size > TRACKER_ADDRESS_SIZE - vaddr || (vaddr & (TRACKER_PAGE_SIZE - 1)) != 0 || + (size & (TRACKER_PAGE_SIZE - 1)) != 0) { + EXIT("BufferCache: invalid image source, addr=0x%016" PRIx64 " size=0x%016" PRIx64 + " page_aligned=%d\n", + vaddr, size, + (vaddr & (TRACKER_PAGE_SIZE - 1)) == 0 && (size & (TRACKER_PAGE_SIZE - 1)) == 0); + } + FaultSafeCacheLock lock(this, m_mutex); + auto use_coherent_guest_backing = [&](const CachedBuffer& overlap) { + const bool gpu_modified = m_memory_tracker.IsRegionGpuModified(vaddr, size); + const auto dirty_ranges = m_gpu_modified_ranges.Intersections(vaddr, size); + if (gpu_modified || !dirty_ranges.empty()) { + EXIT("BufferCache: GPU-modified image source requires a containing cached buffer, " + "addr=0x%016" PRIx64 " size=0x%016" PRIx64 + " buffer=0x%016" PRIx64 "+0x%016" PRIx64 " tracker_dirty=%d byte_ranges=%zu\n", + vaddr, size, overlap.vaddr, overlap.size, gpu_modified, dirty_ranges.size()); + } + // A clean/CPU-current image range may coexist with a partial cached-buffer view because + // guest backing is authoritative. The PS5 detiler consumes that backing at the existing + // Tiler seam. + return BufferImageCopySource {nullptr, 0, vaddr, size, true}; + }; + auto it = m_buffers.upper_bound(vaddr); + if (it == m_buffers.begin()) { + if (it != m_buffers.end() && + PageOverlaps(vaddr, size, it->second->vaddr, it->second->size)) { + return use_coherent_guest_backing(*it->second); + } + return {nullptr, 0, vaddr, size, true}; + } + --it; + auto& cached = *it->second; + const auto offset = vaddr - cached.vaddr; + if (offset > cached.size || size > cached.size - offset) { + const auto next = std::next(it); + if (PageOverlaps(vaddr, size, cached.vaddr, cached.size)) { + return use_coherent_guest_backing(cached); + } + if (next != m_buffers.end() && + PageOverlaps(vaddr, size, next->second->vaddr, next->second->size)) { + return use_coherent_guest_backing(*next->second); + } + return {nullptr, 0, vaddr, size, true}; + } + if (cached.ctx == nullptr || cached.buffer == nullptr || cached.buffer->buffer == nullptr) { + EXIT("BufferCache: inconsistent image source, addr=0x%016" PRIx64 " size=0x%016" PRIx64 + " buffer=0x%016" PRIx64 "+0x%016" PRIx64 " ctx=%p native=%p\n", + vaddr, size, cached.vaddr, cached.size, static_cast(cached.ctx), + static_cast(cached.buffer.get())); + } + const bool gpu_modified = m_memory_tracker.IsRegionGpuModified(vaddr, size); + const bool cpu_modified = m_memory_tracker.IsRegionCpuModified(vaddr, size); + if (gpu_modified && cpu_modified) { + EXIT("BufferCache: image source is both CPU- and GPU-modified, addr=0x%016" PRIx64 + " size=0x%016" PRIx64 "\n", + vaddr, size); + } + const auto ranges = m_gpu_modified_ranges.Intersections(vaddr, size); + if (gpu_modified) { + if (!GraphicsRunIsCommandProcessorThread() && !GraphicsRunSubmissionLockHeld() && + !LabelInCallback()) { + EXIT("BufferCache: GPU-dirty image source requires ordered GPU context, " + "addr=0x%016" PRIx64 " size=0x%016" PRIx64 + " command_thread=%d submission_lock=%d completion_callback=%d\n", + vaddr, size, GraphicsRunIsCommandProcessorThread(), + GraphicsRunSubmissionLockHeld(), LabelInCallback()); + } + if (ranges.empty()) { + EXIT("BufferCache: GPU-dirty image read has no dirty byte ranges, addr=0x%016" PRIx64 + " size=0x%016" PRIx64 "\n", + vaddr, size); + } + VulkanDeviceWaitIdle(cached.ctx); + auto backing_writes = ReserveBackingWrites(m_page_manager, ranges); + uint64_t downloaded = 0; + m_memory_tracker.ForEachDownloadRange( + vaddr, size, + [&](uint64_t address, uint64_t bytes) noexcept { + ValidateDirtyPages(m_gpu_modified_ranges, address, bytes, "image"); + }, + [&](uint64_t address, uint64_t bytes) noexcept { + const auto dirty = m_gpu_modified_ranges.Intersections(address, bytes); + if (dirty.empty()) { + EXIT("BufferCache: GPU-dirty image pages have no dirty byte ranges, " + "addr=0x%016" PRIx64 " size=0x%016" PRIx64 "\n", + address, bytes); + } + for (const auto& range: dirty) { + const auto buffer_offset = range.address - cached.vaddr; + std::vector data(range.size); + UtilDownloadBuffer(cached.ctx, cached.buffer.get(), buffer_offset, data.data(), + range.size); + Libs::LibKernel::Memory::WriteBacking(range.address, data.data(), data.size()); + downloaded += range.size; + } + }); + if (downloaded == 0) { + EXIT("BufferCache: image source cleared no tracked GPU pages, addr=0x%016" PRIx64 + " size=0x%016" PRIx64 "\n", + vaddr, size); + } + m_gpu_modified_ranges.Subtract(vaddr, size); + } else if (!ranges.empty()) { + EXIT("BufferCache: CPU-current image source retains GPU-dirty bytes, addr=0x%016" PRIx64 + " size=0x%016" PRIx64 "\n", + vaddr, size); + } + if (cpu_modified) { + std::vector> uploads; + m_memory_tracker.ForEachUploadRange( + vaddr, size, false, + [&](uint64_t address, uint64_t bytes) noexcept { + uploads.emplace_back(address, bytes); + }, + [&]() noexcept { + for (const auto& [address, bytes]: uploads) { + UtilUploadBuffer(cached.ctx, StagingBufferType::Vertex, cached.buffer.get(), + address - cached.vaddr, reinterpret_cast(address), + bytes); + } + }); + if (uploads.empty()) { + EXIT("BufferCache: CPU-modified image source produced no uploads, addr=0x%016" PRIx64 + " size=0x%016" PRIx64 "\n", + vaddr, size); + } + } + if (m_memory_tracker.IsRegionCpuModified(vaddr, size) || + m_memory_tracker.IsRegionGpuModified(vaddr, size) || + !m_gpu_modified_ranges.Intersections(vaddr, size).empty()) { + EXIT("BufferCache: image source did not become coherent, addr=0x%016" PRIx64 + " size=0x%016" PRIx64 "\n", + vaddr, size); + } + return {cached.buffer.get(), offset, vaddr, size, true}; +} + +namespace { +VkBufferMemoryBarrier MakeDmaBarrier(VulkanBuffer* buffer, uint64_t offset, uint64_t size, + VkAccessFlags source, VkAccessFlags destination) { + if (buffer == nullptr || buffer->buffer == nullptr || size == 0 || + offset > buffer->buffer_size || size > buffer->buffer_size - offset) { + EXIT("BufferCache: invalid DMA barrier, buffer=%p handle=%p offset=0x%016" PRIx64 + " size=0x%016" PRIx64 " buffer_size=0x%016" PRIx64 "\n", + static_cast(buffer), + buffer == nullptr ? nullptr : static_cast(buffer->buffer), offset, size, + buffer == nullptr ? 0 : buffer->buffer_size); + } + VkBufferMemoryBarrier barrier {}; + barrier.sType = VK_STRUCTURE_TYPE_BUFFER_MEMORY_BARRIER; + barrier.srcAccessMask = source; + barrier.dstAccessMask = destination; + barrier.srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; + barrier.dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; + barrier.buffer = buffer->buffer; + barrier.offset = offset; + barrier.size = size; + return barrier; +} + +VkCommandBuffer GetDmaCommandBuffer(CommandBuffer* command) { + if (command == nullptr || command->IsInvalid() || command->GetPool() == nullptr || + command->GetIndex() >= command->GetPool()->buffers_count) { + EXIT("BufferCache: invalid DMA command buffer, command=%p\n", + static_cast(command)); + } + const auto vk_buffer = command->GetPool()->buffers[command->GetIndex()]; + if (vk_buffer == nullptr) { + EXIT("BufferCache: DMA command buffer handle is null, index=%u\n", command->GetIndex()); + } + return vk_buffer; +} +} // namespace + +void BufferCache::FillBuffer(CommandBuffer* command, GraphicContext* ctx, uint64_t vaddr, + uint64_t size, uint32_t value) { + if ((vaddr & 3u) != 0 || (size & 3u) != 0) { + EXIT("BufferCache: fill range must be dword aligned, addr=0x%016" PRIx64 + " size=0x%016" PRIx64 "\n", + vaddr, size); + } + ValidateGpuAccess(vaddr, size, false, true); + { + std::lock_guard transaction(m_resource_mutex); + const bool image_overlap = m_texture_cache->HasPageOverlap(vaddr, size); + const bool buffer_overlap = HasPageOverlap(vaddr, size); + const bool buffer_gpu_modified = IsRegionGpuModified(vaddr, size); + if (!buffer_overlap && !buffer_gpu_modified) { + if (image_overlap) { + m_texture_cache->PrepareHostWrite(vaddr, size); + } + auto* dst = reinterpret_cast(vaddr); + std::fill(dst, dst + size / sizeof(uint32_t), value); + return; + } + if (image_overlap) { + EXIT("BufferCache: GPU fill aliases image pages, addr=0x%016" PRIx64 + " size=0x%016" PRIx64 "\n", + vaddr, size); + } + if (m_texture_cache->HasMetaOverlap(vaddr, size)) { + LOGF("BufferCache: GPU fill overlaps virtual metadata, addr=0x%016" PRIx64 + " size=0x%016" PRIx64 "\n", + vaddr, size); + EXIT("BufferCache: GPU fill of virtual metadata is unsupported, addr=0x%016" PRIx64 + " size=0x%016" PRIx64 "\n", + vaddr, size); + } + } + const auto [dst, dst_offset] = ObtainBuffer(command, ctx, vaddr, size, true, false); + const auto before = MakeDmaBarrier(dst, dst_offset, size, + VK_ACCESS_MEMORY_READ_BIT | VK_ACCESS_MEMORY_WRITE_BIT, + VK_ACCESS_TRANSFER_WRITE_BIT); + const auto vk_buffer = GetDmaCommandBuffer(command); + vkCmdPipelineBarrier(vk_buffer, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, + VK_PIPELINE_STAGE_TRANSFER_BIT, VK_DEPENDENCY_BY_REGION_BIT, 0, nullptr, 1, + &before, 0, nullptr); + vkCmdFillBuffer(vk_buffer, dst->buffer, dst_offset, size, value); + const auto after = MakeDmaBarrier(dst, dst_offset, size, VK_ACCESS_TRANSFER_WRITE_BIT, + VK_ACCESS_MEMORY_READ_BIT | VK_ACCESS_MEMORY_WRITE_BIT); + vkCmdPipelineBarrier(vk_buffer, VK_PIPELINE_STAGE_TRANSFER_BIT, + VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, VK_DEPENDENCY_BY_REGION_BIT, 0, + nullptr, 1, &after, 0, nullptr); +} + +void BufferCache::CopyBuffer(CommandBuffer* command, GraphicContext* ctx, uint64_t dst_vaddr, + uint64_t src_vaddr, uint64_t size) { + if (dst_vaddr == 0 || src_vaddr == 0 || size == 0 || + ((dst_vaddr | src_vaddr | size) & 3u) != 0 || size > UINT64_MAX - dst_vaddr || + size > UINT64_MAX - src_vaddr || + (src_vaddr < dst_vaddr + size && dst_vaddr < src_vaddr + size)) { + EXIT("BufferCache: invalid or overlapping copy range, src=0x%016" PRIx64 + " dst=0x%016" PRIx64 " size=0x%016" PRIx64 "\n", + src_vaddr, dst_vaddr, size); + } + ValidateGpuAccess(src_vaddr, size, true, false); + ValidateGpuAccess(dst_vaddr, size, false, true); + { + std::lock_guard transaction(m_resource_mutex); + const bool src_image = m_texture_cache->HasPageOverlap(src_vaddr, size); + const bool dst_image = m_texture_cache->HasPageOverlap(dst_vaddr, size); + if (m_texture_cache->HasGpuTargetPageOverlap(src_vaddr, size)) { + EXIT("BufferCache: GPU copy aliases target pages, src=0x%016" PRIx64 + " dst=0x%016" PRIx64 " size=0x%016" PRIx64 "\n", + src_vaddr, dst_vaddr, size); + } + if (!HasPageOverlap(dst_vaddr, size) && !IsRegionGpuModified(src_vaddr, size) && + !IsRegionGpuModified(dst_vaddr, size)) { + if (m_texture_cache->IsMetaGpuModified(src_vaddr, size)) { + LOGF("BufferCache: host copy reads virtual metadata, src=0x%016" PRIx64 + " size=0x%016" PRIx64 "\n", + src_vaddr, size); + EXIT("BufferCache: host copy from GPU-modified metadata is unsupported, " + "src=0x%016" PRIx64 " size=0x%016" PRIx64 "\n", + src_vaddr, size); + } + if (dst_image) { + m_texture_cache->PrepareHostWrite(dst_vaddr, size); + } + std::memcpy(reinterpret_cast(dst_vaddr), + reinterpret_cast(src_vaddr), size); + return; + } + // A clean target destination is handled above like a protected host write. Target + // aliases that require an actual GPU buffer copy remain unsupported. + if (m_texture_cache->HasGpuTargetPageOverlap(dst_vaddr, size)) { + EXIT("BufferCache: GPU copy aliases target pages, src=0x%016" PRIx64 + " dst=0x%016" PRIx64 " size=0x%016" PRIx64 "\n", + src_vaddr, dst_vaddr, size); + } + if (src_image || dst_image) { + EXIT("BufferCache: GPU copy requires image-buffer synchronization, src=0x%016" PRIx64 + " dst=0x%016" PRIx64 " size=0x%016" PRIx64 "\n", + src_vaddr, dst_vaddr, size); + } + if (m_texture_cache->HasMetaOverlap(src_vaddr, size) || + m_texture_cache->HasMetaOverlap(dst_vaddr, size)) { + LOGF("BufferCache: GPU copy overlaps virtual metadata, src=0x%016" PRIx64 + " dst=0x%016" PRIx64 " size=0x%016" PRIx64 "\n", + src_vaddr, dst_vaddr, size); + EXIT("BufferCache: GPU copy involving virtual metadata is unsupported, " + "src=0x%016" PRIx64 " dst=0x%016" PRIx64 " size=0x%016" PRIx64 "\n", + src_vaddr, dst_vaddr, size); + } + } + const auto [src, src_offset] = ObtainBuffer(command, ctx, src_vaddr, size, false, true); + const auto [dst, dst_offset] = ObtainBuffer(command, ctx, dst_vaddr, size, true, false); + if (src == dst && src_offset < dst_offset + size && dst_offset < src_offset + size) { + EXIT("BufferCache: resolved Vulkan copy ranges overlap, src_offset=0x%016" PRIx64 + " dst_offset=0x%016" PRIx64 " size=0x%016" PRIx64 "\n", + src_offset, dst_offset, size); + } + const VkBufferMemoryBarrier before[] = { + MakeDmaBarrier(dst, dst_offset, size, + VK_ACCESS_MEMORY_READ_BIT | VK_ACCESS_MEMORY_WRITE_BIT, + VK_ACCESS_TRANSFER_WRITE_BIT), + MakeDmaBarrier(src, src_offset, size, VK_ACCESS_MEMORY_WRITE_BIT, + VK_ACCESS_TRANSFER_READ_BIT), + }; + const auto vk_buffer = GetDmaCommandBuffer(command); + vkCmdPipelineBarrier(vk_buffer, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, + VK_PIPELINE_STAGE_TRANSFER_BIT, VK_DEPENDENCY_BY_REGION_BIT, 0, nullptr, 2, + before, 0, nullptr); + const VkBufferCopy copy {src_offset, dst_offset, size}; + vkCmdCopyBuffer(vk_buffer, src->buffer, dst->buffer, 1, ©); + const VkBufferMemoryBarrier after[] = { + MakeDmaBarrier(dst, dst_offset, size, VK_ACCESS_TRANSFER_WRITE_BIT, + VK_ACCESS_MEMORY_READ_BIT | VK_ACCESS_MEMORY_WRITE_BIT), + MakeDmaBarrier(src, src_offset, size, VK_ACCESS_TRANSFER_READ_BIT, + VK_ACCESS_MEMORY_WRITE_BIT), + }; + vkCmdPipelineBarrier(vk_buffer, VK_PIPELINE_STAGE_TRANSFER_BIT, + VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, VK_DEPENDENCY_BY_REGION_BIT, 0, + nullptr, 2, after, 0, nullptr); +} + +bool BufferCache::HasPageOverlap(uint64_t vaddr, uint64_t size) { + if (vaddr == 0 || size == 0 || vaddr >= TRACKER_ADDRESS_SIZE || + size > TRACKER_ADDRESS_SIZE - vaddr) { + EXIT("BufferCache: invalid page-overlap query, addr=0x%016" PRIx64 " size=0x%016" PRIx64 + "\n", + vaddr, size); + } + FaultSafeCacheLock lock(this, m_mutex); + for (const auto& [address, cached]: m_buffers) { + if (PageOverlaps(vaddr, size, address, cached->size)) { + return true; + } + } + return false; +} + +bool BufferCache::IsRegionGpuModified(uint64_t vaddr, uint64_t size) { + return m_memory_tracker.IsRegionGpuModified(vaddr, size); +} + +bool BufferCache::IsRegionCpuModified(uint64_t vaddr, uint64_t size) { + return m_memory_tracker.IsRegionCpuModified(vaddr, size); +} + +void BufferCache::PublishImageBacking(uint64_t vaddr, uint64_t size) { + if (vaddr == 0 || size == 0 || vaddr >= TRACKER_ADDRESS_SIZE || + size > TRACKER_ADDRESS_SIZE - vaddr) { + EXIT("BufferCache: invalid image-backing publication, addr=0x%016" PRIx64 + " size=0x%016" PRIx64 "\n", + vaddr, size); + } + FaultSafeCacheLock lock(this, m_mutex); + auto owner = m_buffers.end(); + for (auto it = m_buffers.begin(); it != m_buffers.end(); ++it) { + if (!PageOverlaps(vaddr, size, it->second->vaddr, it->second->size)) { + continue; + } + const auto offset = vaddr >= it->second->vaddr ? vaddr - it->second->vaddr : UINT64_MAX; + if (owner != m_buffers.end() || offset > it->second->size || + size > it->second->size - offset) { + EXIT("BufferCache: image backing aliases a non-containing cached buffer, " + "addr=0x%016" PRIx64 " size=0x%016" PRIx64 " buffer=0x%016" PRIx64 "+0x%016" PRIx64 + "\n", + vaddr, size, it->second->vaddr, it->second->size); + } + owner = it; + } + if ((owner != m_buffers.end() && + (owner->second->ctx == nullptr || owner->second->buffer == nullptr || + owner->second->buffer->buffer == nullptr || + m_memory_tracker.IsRegionCpuModified(vaddr, size))) || + m_memory_tracker.IsRegionGpuModified(vaddr, size) || + !m_gpu_modified_ranges.Intersections(vaddr, size).empty()) { + EXIT("BufferCache: image backing requires clean buffer ownership, addr=0x%016" PRIx64 + " size=0x%016" PRIx64 " cached=%d\n", + vaddr, size, owner != m_buffers.end()); + } + // A fresh tracker is CPU-dirty by construction when no cached buffer exists. Keep the range + // CPU-dirty so subsequently-created buffer uploads the just-published image backing. + m_memory_tracker.MarkRegionAsCpuModified(vaddr, size); +} + +void BufferCache::ValidateGpuAccess(uint64_t vaddr, uint64_t size, bool is_read, + bool is_written) const { + if ((!is_read && !is_written) || vaddr == 0 || size == 0 || size > UINT64_MAX - vaddr) { + EXIT("BufferCache: invalid GPU access request, addr=0x%016" PRIx64 " size=0x%016" PRIx64 + " read=%d write=%d\n", + vaddr, size, is_read, is_written); + } + if (is_read && !m_page_manager.HasGpuAccess(vaddr, size, GpuAccess::Read)) { + LOGF("BufferCache: GPU-read access denied, addr=0x%016" PRIx64 " size=0x%016" PRIx64 "\n", + vaddr, size); + EXIT("BufferCache: GPU-read access denied, addr=0x%016" PRIx64 " size=0x%016" PRIx64 "\n", + vaddr, size); + } + if (is_written && !m_page_manager.HasGpuAccess(vaddr, size, GpuAccess::Write)) { + LOGF("BufferCache: GPU-write access denied, addr=0x%016" PRIx64 " size=0x%016" PRIx64 "\n", + vaddr, size); + EXIT("BufferCache: GPU-write access denied, addr=0x%016" PRIx64 " size=0x%016" PRIx64 "\n", + vaddr, size); + } +} + +void BufferCache::SetTextureCache(TextureCache& texture_cache) { + if (m_texture_cache != nullptr) { + EXIT("BufferCache: texture cache already connected\n"); + } + m_texture_cache = &texture_cache; +} + +//TODO: add LRU cache +void BufferCache::RegisterForDelete(VulkanBuffer* buffer) { + FaultSafeCacheLock lock(this, m_mutex); + + m_delete_later.push_back(buffer); +} + +void BufferCache::DeleteAll(GraphicContext* ctx) { + KYTY_PROFILER_BLOCK("BufferCache::DeleteAll"); + + FaultSafeCacheLock lock(this, m_mutex); + + for (auto* buffer: m_delete_later) { + EXIT_IF(buffer == nullptr); + EXIT_IF(buffer->buffer == nullptr); + EXIT_IF(ctx == nullptr); + + VulkanDeleteBuffer(ctx, buffer); + delete buffer; + } + m_delete_later.clear(); + m_null_buffer.reset(); +} + +} // namespace Libs::Graphics diff --git a/src/graphics/host_gpu/renderer/bufferCache.h b/src/graphics/host_gpu/renderer/bufferCache.h new file mode 100644 index 000000000..4a9fafede --- /dev/null +++ b/src/graphics/host_gpu/renderer/bufferCache.h @@ -0,0 +1,106 @@ +#ifndef EMULATOR_SRC_GRAPHICS_HOST_GPU_RENDERER_BUFFERCACHE_H_ +#define EMULATOR_SRC_GRAPHICS_HOST_GPU_RENDERER_BUFFERCACHE_H_ + +#include "common/abi.h" +#include "common/common.h" +#include "common/threads.h" +#include "graphics/host_gpu/memoryTracker.h" +#include "graphics/host_gpu/rangeSet.h" + +#include +#include +#include +#include + +namespace Libs::Graphics { + +struct GraphicContext; +struct VulkanBuffer; +class CommandBuffer; +class TextureCache; +class ResourceMutex; + +struct BufferImageCopySource { + VulkanBuffer* buffer = nullptr; + uint64_t offset = 0; + uint64_t address = 0; + uint64_t size = 0; + bool cpu_current = false; +}; + +struct BufferCacheRange { + uint64_t address = 0; + uint64_t size = 0; +}; + +[[nodiscard]] bool MergeOverlappingBufferCacheRange(BufferCacheRange* merged, + BufferCacheRange candidate) noexcept; +[[nodiscard]] bool CanMergeBufferCacheQueueMask(uint64_t queue_mask, uint32_t queue) noexcept; +[[nodiscard]] bool CanReadbackBufferCacheQueueMask(uint64_t queue_mask, uint32_t queue) noexcept; + +class BufferCache { +public: + static constexpr uint64_t CACHING_PAGE_SIZE = 16ull * 1024ull; + static constexpr uint64_t GetBufferOffset(uint64_t vaddr) { + return vaddr & (CACHING_PAGE_SIZE - 1); + } + + BufferCache(PageManager& page_manager, ResourceMutex& resource_mutex); + ~BufferCache(); + KYTY_CLASS_NO_COPY(BufferCache); + + [[nodiscard]] bool InvalidateMemory(PageFaultAccess access, uint64_t vaddr, uint64_t size, + PageFaultPhase phase) noexcept; + void UnmapMemory(uint64_t vaddr, uint64_t size); + [[nodiscard]] std::pair + ObtainBuffer(CommandBuffer* command, GraphicContext* ctx, uint64_t vaddr, uint64_t size, + bool is_written = false, bool is_read = true, bool is_formatted = false); + // Emulator-owned, CPU-current scratch only. Guest ranges must use ObtainBuffer so page + // ownership is resolved before any CPU access. + [[nodiscard]] bool UploadHostData(CommandBuffer* command, GraphicContext* ctx, const void* src, + uint64_t size, uint64_t alignment, + VulkanBuffer** out_buffer, uint64_t* out_offset, + uint64_t* out_range); + [[nodiscard]] VulkanBuffer* ObtainNullBuffer(CommandBuffer* command, + GraphicContext* ctx); + [[nodiscard]] BufferImageCopySource ObtainBufferForImage(uint64_t vaddr, uint64_t size); + void FillBuffer(CommandBuffer* command, GraphicContext* ctx, uint64_t vaddr, uint64_t size, + uint32_t value); + void CopyBuffer(CommandBuffer* command, GraphicContext* ctx, uint64_t dst_vaddr, + uint64_t src_vaddr, uint64_t size); + [[nodiscard]] bool HasPageOverlap(uint64_t vaddr, uint64_t size); + [[nodiscard]] bool IsRegionCpuModified(uint64_t vaddr, uint64_t size); + [[nodiscard]] bool IsRegionGpuModified(uint64_t vaddr, uint64_t size); + void PublishImageBacking(uint64_t vaddr, uint64_t size); + void ValidateGpuAccess(uint64_t vaddr, uint64_t size, bool is_read, bool is_written) const; + void SetTextureCache(TextureCache& texture_cache); + + void RegisterForDelete(VulkanBuffer* buffer); + void DeleteAll(GraphicContext* ctx); + +private: + struct CachedBuffer; + struct ReadbackWorker; + struct CommandProcessorReadbackResources; + void RequestCommandProcessorReadback(PageFaultAccess access, uint64_t vaddr, uint64_t size); + [[nodiscard]] bool CompleteCommandProcessorReadback(PageFaultAccess access, uint64_t vaddr, + uint64_t size) noexcept; + [[nodiscard]] bool ReleaseCommandProcessorReadback(PageFaultAccess access, uint64_t vaddr, + uint64_t size) noexcept; + + Common::Mutex m_mutex; + std::vector m_delete_later; + std::shared_ptr m_null_buffer; + std::map> m_buffers; + std::unique_ptr m_readback; + std::unique_ptr m_cp_readback_resources; + RangeSet m_gpu_modified_ranges; + MemoryTracker m_memory_tracker; + PageManager& m_page_manager; + TextureCache* m_texture_cache = nullptr; + ResourceMutex& m_resource_mutex; +}; + +} // namespace Libs::Graphics + +#endif // EMULATOR_SRC_GRAPHICS_HOST_GPU_RENDERER_BUFFERCACHE_H_ diff --git a/src/graphics/host_gpu/renderer/colorRenderTarget.cpp b/src/graphics/host_gpu/renderer/colorRenderTarget.cpp new file mode 100644 index 000000000..54f93b0cd --- /dev/null +++ b/src/graphics/host_gpu/renderer/colorRenderTarget.cpp @@ -0,0 +1,515 @@ +#include "common/assert.h" +#include "common/common.h" +#include "common/logging/log.h" +#include "common/profiler.h" +#include "common/stringUtils.h" +#include "common/threads.h" +#include "graphics/guest_gpu/gpu_defs.h" +#include "graphics/guest_gpu/hardwareContext.h" +#include "graphics/guest_gpu/tile.h" +#include "graphics/host_gpu/graphicContext.h" +#include "graphics/host_gpu/objects/textureCommon.h" +#include "graphics/host_gpu/renderer/descriptorCache.h" +#include "graphics/host_gpu/renderer/framebufferCache.h" +#include "graphics/host_gpu/renderer/render.h" +#include "graphics/host_gpu/renderer/renderContext.h" +#include "graphics/host_gpu/renderer/renderState.h" +#include "graphics/host_gpu/utils.h" +#include "graphics/presentation/displayBuffer.h" + +#include +#include +#include +#include +#include +#include +#include + +namespace Libs::Graphics { + +static std::atomic g_render_color_log_count = 0; + +static float ColorClearUnorm(uint32_t value, uint32_t bits) { + return static_cast(value) / static_cast((1u << bits) - 1u); +} + +static float ColorClearSnorm(uint32_t value, uint32_t bits) { + const uint32_t mask = (1u << bits) - 1u; + const uint32_t sign_bit = 1u << (bits - 1u); + value &= mask; + + auto signed_value = static_cast(value); + if ((value & sign_bit) != 0) { + signed_value = static_cast(value | ~mask); + } + + const auto denominator = static_cast((1u << (bits - 1u)) - 1u); + return std::max(-1.0f, static_cast(signed_value) / denominator); +} + +static float ColorClearF16(uint32_t value) { + const uint32_t sign = (value >> 15u) & 0x1u; + const uint32_t exponent = (value >> 10u) & 0x1fu; + const uint32_t mantissa = value & 0x3ffu; + const float sign_mul = (sign != 0 ? -1.0f : 1.0f); + + if (exponent == 0) { + if (mantissa == 0) { + return sign_mul * 0.0f; + } + return sign_mul * std::ldexp(static_cast(mantissa), -24); + } + + if (exponent == 0x1fu) { + return mantissa == 0 ? sign_mul * std::numeric_limits::infinity() + : std::numeric_limits::quiet_NaN(); + } + + return sign_mul * + std::ldexp(static_cast(0x400u | mantissa), static_cast(exponent) - 25); +} + +[[maybe_unused]] static VkClearColorValue ColorClearValue(const HW::RenderTarget& rt) { + const uint32_t c0 = rt.clear_word0.word0; + const uint32_t c1 = rt.clear_word1.word1; + + VkClearColorValue color {}; + + switch (static_cast(rt.info.channel_type)) { + case Prospero::ChannelType::kUNorm: + case Prospero::ChannelType::kSrgb: + // SRGB clear words are still encoded as normalized component values. + switch (static_cast(rt.info.format)) { + case Prospero::ChannelLayout::k8: + color.float32[0] = ColorClearUnorm(c0 & 0xffu, 8); + color.float32[1] = 0.0f; + color.float32[2] = 0.0f; + color.float32[3] = 1.0f; + break; + case Prospero::ChannelLayout::k8_8: + color.float32[0] = ColorClearUnorm(c0 & 0xffu, 8); + color.float32[1] = ColorClearUnorm((c0 >> 8u) & 0xffu, 8); + color.float32[2] = 0.0f; + color.float32[3] = 1.0f; + break; + case Prospero::ChannelLayout::k8_8_8_8: + color.float32[0] = ColorClearUnorm(c0 & 0xffu, 8); + color.float32[1] = ColorClearUnorm((c0 >> 8u) & 0xffu, 8); + color.float32[2] = ColorClearUnorm((c0 >> 16u) & 0xffu, 8); + color.float32[3] = ColorClearUnorm((c0 >> 24u) & 0xffu, 8); + if (rt.info.channel_order == + Prospero::GpuEnumValue(Prospero::ChannelOrder::kAlt)) { + std::swap(color.float32[0], color.float32[2]); + } + break; + case Prospero::ChannelLayout::k11_11_10: + color.float32[0] = 0.0f; + color.float32[1] = 0.0f; + color.float32[2] = 0.0f; + color.float32[3] = 1.0f; + break; + case Prospero::ChannelLayout::k10_10_10_2: + color.float32[0] = ColorClearUnorm(c0 & 0x3ffu, 10); + color.float32[1] = ColorClearUnorm((c0 >> 10u) & 0x3ffu, 10); + color.float32[2] = ColorClearUnorm((c0 >> 20u) & 0x3ffu, 10); + color.float32[3] = ColorClearUnorm((c0 >> 30u) & 0x3u, 2); + if (rt.info.channel_order == + Prospero::GpuEnumValue(Prospero::ChannelOrder::kAlt)) { + std::swap(color.float32[0], color.float32[2]); + } + break; + case Prospero::ChannelLayout::k16_16: + color.float32[0] = 0.0f; + color.float32[1] = 0.0f; + color.float32[2] = 0.0f; + color.float32[3] = 1.0f; + break; + case Prospero::ChannelLayout::k32_32_32_32: + color.float32[0] = 0.0f; + color.float32[1] = 0.0f; + color.float32[2] = 0.0f; + color.float32[3] = 1.0f; + break; + default: + EXIT("%s\n unsupported color clear format\n", + Common::Concat(rt_print("RenderTarget", rt), "").c_str()); + } + break; + case Prospero::ChannelType::kSNorm: + switch (static_cast(rt.info.format)) { + case Prospero::ChannelLayout::k16_16: + color.float32[0] = ColorClearSnorm(c0 & 0xffffu, 16); + color.float32[1] = ColorClearSnorm((c0 >> 16u) & 0xffffu, 16); + color.float32[2] = 0.0f; + color.float32[3] = 1.0f; + break; + case Prospero::ChannelLayout::k8_8_8_8: + color.float32[0] = ColorClearSnorm(c0 & 0xffu, 8); + color.float32[1] = ColorClearSnorm((c0 >> 8u) & 0xffu, 8); + color.float32[2] = ColorClearSnorm((c0 >> 16u) & 0xffu, 8); + color.float32[3] = ColorClearSnorm((c0 >> 24u) & 0xffu, 8); + if (rt.info.channel_order == + Prospero::GpuEnumValue(Prospero::ChannelOrder::kAlt)) { + std::swap(color.float32[0], color.float32[2]); + } + break; + default: + EXIT("%s\n unsupported snorm color clear format\n", + Common::Concat(rt_print("RenderTarget", rt), "").c_str()); + } + break; + case Prospero::ChannelType::kFloat: + switch (static_cast(rt.info.format)) { + case Prospero::ChannelLayout::k16: + color.float32[0] = ColorClearF16(c0 & 0xffffu); + color.float32[1] = 0.0f; + color.float32[2] = 0.0f; + color.float32[3] = 1.0f; + break; + case Prospero::ChannelLayout::k16_16: + color.float32[0] = ColorClearF16(c0 & 0xffffu); + color.float32[1] = ColorClearF16((c0 >> 16u) & 0xffffu); + color.float32[2] = 0.0f; + color.float32[3] = 1.0f; + break; + case Prospero::ChannelLayout::k16_16_16_16: + color.float32[0] = ColorClearF16(c0 & 0xffffu); + color.float32[1] = ColorClearF16((c0 >> 16u) & 0xffffu); + color.float32[2] = ColorClearF16(c1 & 0xffffu); + color.float32[3] = ColorClearF16((c1 >> 16u) & 0xffffu); + break; + case Prospero::ChannelLayout::k32: + case Prospero::ChannelLayout::k11_11_10: + case Prospero::ChannelLayout::k32_32_32_32: + color.float32[0] = 0.0f; + color.float32[1] = 0.0f; + color.float32[2] = 0.0f; + color.float32[3] = 1.0f; + break; + default: + EXIT("%s\n unsupported float color clear format\n", + Common::Concat(rt_print("RenderTarget", rt), "").c_str()); + } + break; + case Prospero::ChannelType::kUInt: + case Prospero::ChannelType::kSInt: + switch (static_cast(rt.info.format)) { + case Prospero::ChannelLayout::k8: + color.uint32[0] = c0; + color.uint32[1] = 0; + color.uint32[2] = 0; + color.uint32[3] = 1; + break; + case Prospero::ChannelLayout::k16: + color.uint32[0] = c0 & 0xffffu; + color.uint32[1] = 0; + color.uint32[2] = 0; + color.uint32[3] = 1; + break; + case Prospero::ChannelLayout::k16_16: + color.uint32[0] = c0 & 0xffffu; + color.uint32[1] = (c0 >> 16u) & 0xffffu; + color.uint32[2] = 0; + color.uint32[3] = 1; + break; + case Prospero::ChannelLayout::k8_8_8_8: + color.uint32[0] = c0 & 0xffu; + color.uint32[1] = (c0 >> 8u) & 0xffu; + color.uint32[2] = (c0 >> 16u) & 0xffu; + color.uint32[3] = c0 >> 24u; + if (rt.info.channel_order == + Prospero::GpuEnumValue(Prospero::ChannelOrder::kAlt)) { + std::swap(color.uint32[0], color.uint32[2]); + } + break; + case Prospero::ChannelLayout::k32_32: + color.uint32[0] = c0; + color.uint32[1] = c1; + color.uint32[2] = 0; + color.uint32[3] = 1; + break; + default: + EXIT("%s\n unsupported integer color clear format\n", + Common::Concat(rt_print("RenderTarget", rt), "").c_str()); + } + break; + default: + EXIT("%s\n unsupported color clear number type\n", + Common::Concat(rt_print("RenderTarget", rt), "").c_str()); + } + + EXIT_NOT_IMPLEMENTED( + c1 != 0 && + !(rt.info.channel_type == Prospero::GpuEnumValue(Prospero::ChannelType::kFloat) && + rt.info.format == Prospero::GpuEnumValue(Prospero::ChannelLayout::k16_16_16_16)) && + !((rt.info.channel_type == Prospero::GpuEnumValue(Prospero::ChannelType::kUInt) || + rt.info.channel_type == Prospero::GpuEnumValue(Prospero::ChannelType::kSInt)) && + rt.info.format == Prospero::GpuEnumValue(Prospero::ChannelLayout::k32_32))); + + return color; +} + +static uint32_t RenderGetColorBpp(uint32_t dfmt) { + switch (static_cast(dfmt)) { + case Prospero::ChannelLayout::k8: return 8; + case Prospero::ChannelLayout::k16: + case Prospero::ChannelLayout::k8_8: return 16; + case Prospero::ChannelLayout::k32: + case Prospero::ChannelLayout::k16_16: + case Prospero::ChannelLayout::k11_11_10: + case Prospero::ChannelLayout::k10_10_10_2: + case Prospero::ChannelLayout::k8_8_8_8: return 32; + case Prospero::ChannelLayout::k32_32: + case Prospero::ChannelLayout::k16_16_16_16: return 64; + case Prospero::ChannelLayout::k32_32_32_32: return 128; + case Prospero::ChannelLayout::k5_5_5_1: + case Prospero::ChannelLayout::k4_4_4_4: return 16; + default: EXIT("unsupported render-target channel layout: %u\n", dfmt); + } +} + +// NOLINTNEXTLINE(readability-function-cognitive-complexity) +void ResolveRenderColorTarget(uint64_t submit_id, CommandBuffer* buffer, const HW::Context& hw, + RenderColorInfo* r, uint32_t render_target_slice_offset, + uint32_t render_target_slot, bool ignore_target_mask, + bool reuse_existing_render_texture) { + KYTY_PROFILER_FUNCTION(); + + EXIT_IF(r == nullptr); + + const auto rt_slot = (render_target_slot == UINT32_MAX ? render_target_first_bound_slot(hw) + : render_target_slot); + const auto& rt = hw.GetRenderTarget(rt_slot); + auto mask = render_target_mask_slot(hw.GetRenderTargetMask(), rt_slot); + if (ignore_target_mask && rt.base.addr != 0 && mask == 0) { + mask = 0x0f; + } + + r->target_slot = rt_slot; + + if (rt.base.addr == 0 || mask == 0) { + if (graphics_debug_dump_enabled()) { + static std::atomic_uint log_count = 0; + const auto log_id = log_count.fetch_add(1, std::memory_order_relaxed); + if (log_id < 128) { + LOGF("RenderColorTarget: no color output slot=%" PRIu32 " base=0x%010" PRIx64 + " slot_mask=0x%01" PRIx32 " target_mask=0x%08" PRIx32 + " rt_slice_offset=%" PRIu32 "\n", + rt_slot, rt.base.addr, mask, hw.GetRenderTargetMask(), + render_target_slice_offset); + } + } + + // No color output + r->type = RenderColorType::NoColorOutput; + r->base_addr = 0; + r->vulkan_buffer = nullptr; + r->vulkan_view = nullptr; + r->format = VK_FORMAT_UNDEFINED; + r->extent = {}; + r->base_mip_level = 0; + r->buffer_size = 0; + r->color_clear_enable = false; + r->color_clear_value = {}; + return; + } + const bool msaa_compat = + color_msaa_single_sample_compatible(rt.attrib.num_samples, rt.attrib.num_fragments); + if ((!msaa_compat && (rt.attrib.num_samples != 0 || rt.attrib.num_fragments != 0)) || + rt.view.base_array_slice_index != 0 || render_target_slice_offset != 0) { + EXIT("multisampled or sliced render targets are unsupported\n"); + } + const uint32_t levels = rt.attrib2.num_mip_levels + 1u; + if (levels == 0 || levels > 16 || rt.view.current_mip_level >= levels) { + EXIT("unsupported render-target mip range: current=%u levels=%u\n", + rt.view.current_mip_level, levels); + } + if (msaa_compat) { + static std::atomic logged_fragments = 0; + const uint32_t bit = 1u << rt.attrib.num_fragments; + if ((logged_fragments.fetch_or(bit, std::memory_order_relaxed) & bit) == 0) { + LOGF("RenderColorTarget: compatibility: rendering PS5 %ux samples/fragments as " + "single-sample\n", + bit); + } + } + + if (graphics_debug_dump_enabled()) { + static std::atomic_uint log_count = 0; + const auto log_id = log_count.fetch_add(1, std::memory_order_relaxed); + if (log_id < 128) { + LOGF("RenderColorTarget: inspect slot=%" PRIu32 " base=0x%010" PRIx64 + " mask=0x%01" PRIx32 " attrib2_width=%" PRIu32 " attrib2_height=%" PRIu32 + " attrib3_tile=0x%08" PRIx32 " attrib3_dim=0x%08" PRIx32 " fmt=0x%08" PRIx32 + " nfmt=0x%08" PRIx32 " order=0x%08" PRIx32 "\n", + rt_slot, rt.base.addr, mask, rt.attrib2.width, rt.attrib2.height, + rt.attrib3.tile_mode, rt.attrib3.dimension, rt.info.format, rt.info.channel_type, + rt.info.channel_order); + } + } + + // CB_COLOR_CONTROL describes the color-buffer operation / ROP3 logic op. + // ROP3 Copy is the normal color write path, not a render-target clear. + // Fast color clears are metadata driven and must be handled explicitly when + // that metadata path is implemented; render-pass load must preserve contents. + r->color_clear_enable = false; + r->color_clear_value = {}; + + uint32_t width = 0; + uint32_t height = 0; + uint32_t pitch = 0; + uint64_t size = 0; + bool tile = false; + + switch (rt.attrib3.tile_mode) { + case Prospero::GpuEnumValue(Prospero::TileMode::kLinear): + case Prospero::GpuEnumValue(Prospero::TileMode::kRenderTarget): + tile = !RenderIsColorTileModeLinear(rt.attrib3.tile_mode); + break; + default: EXIT("unknown tile mode: %u\n", rt.attrib3.tile_mode); + } + if (!tile && levels > 1) { + EXIT("linear mipmapped render targets are unsupported\n"); + } + + width = rt.attrib2.width + 1; + height = rt.attrib2.height + 1; + const auto bytes_per_element = RenderGetColorBpp(rt.info.format) / 8u; + if (bytes_per_element == 0) { + EXIT("render-target format has no valid element size\n"); + } + if (rt.pitch.pitch_div8_minus1 != 0) { + pitch = (rt.pitch.pitch_div8_minus1 + 1u) << 3u; + } else if (tile) { + pitch = TileGetRenderTargetPitch(width, bytes_per_element); + if (pitch == 0) { + EXIT("unsupported render-target pitch: width=%u bytes=%u\n", width, bytes_per_element); + } + } else { + pitch = width; + } + + if (tile) { + TileSizeAlign layout {}; + const bool valid_layout = + levels == 1 ? TileGetRenderTargetSize(width, height, pitch, bytes_per_element, &layout) + : TileGetRenderTargetMipLayout(width, height, pitch, bytes_per_element, + levels, &layout, nullptr, nullptr); + if (!valid_layout) { + EXIT("unsupported render-target layout: %ux%u pitch=%u bytes=%u levels=%u\n", width, + height, pitch, bytes_per_element, levels); + } + size = layout.size; + if (rt.slice.slice_div64_minus1 != 0 && + (static_cast(rt.slice.slice_div64_minus1) + 1u) * 64u != size) { + EXIT("render-target slice span mismatch: encoded=0x%016" PRIx64 " derived=0x%016" PRIx64 + "\n", + (static_cast(rt.slice.slice_div64_minus1) + 1u) * 64u, size); + } + } else { + size = static_cast(pitch) * height * bytes_per_element; + } + if (size == 0 || size > TRACKER_ADDRESS_SIZE - rt.base.addr) { + EXIT("render-target memory footprint is invalid\n"); + } + + const auto effective_slice = rt.view.base_array_slice_index + render_target_slice_offset; + const auto effective_base = rt.base.addr + static_cast(size) * effective_slice; + + auto video_image = Presentation::DisplayBufferFind(effective_base, true); + bool render_to_texture = (video_image.image == nullptr); + if (!render_to_texture && (levels != 1 || rt.view.current_mip_level != 0)) { + EXIT("mipmapped display render targets are unsupported\n"); + } + const VkExtent2D view_extent = {std::max(width >> rt.view.current_mip_level, 1u), + std::max(height >> rt.view.current_mip_level, 1u)}; + + auto decision_log_id = g_render_color_log_count.fetch_add(1); + if (decision_log_id < 128 || !render_to_texture) { + LOGF("RenderColorTarget: slot=%" PRIu32 " addr=0x%010" PRIx64 " size=0x%016" PRIx64 + " extent=%ux%u view_mip=%u view_extent=%ux%u levels=%u pitch=%u" + " fmt=0x%08" PRIx32 " nfmt=0x%08" PRIx32 " order=0x%08" PRIx32 + " tile=%s target=%s video_size=0x%016" PRIx64 " video_pitch=%" PRIu64 "\n", + rt_slot, effective_base, size, width, height, rt.view.current_mip_level, + view_extent.width, view_extent.height, levels, pitch, rt.info.format, + rt.info.channel_type, rt.info.channel_order, tile ? "tiled" : "linear", + render_to_texture ? "RenderTexture" : "DisplayBuffer", video_image.size, + video_image.pitch); + } + + if (render_to_texture) { + const auto rt_format = TextureGetRenderTargetFormat(rt.info.format, rt.info.channel_type, + rt.info.channel_order); + + (void)reuse_existing_render_texture; + RenderTargetInfo target {}; + target.address = effective_base; + target.size = size; + target.format = rt_format.format; + target.width = width; + target.height = height; + target.pitch = pitch; + target.bytes_per_element = rt_format.bytes_per_element; + target.tile_mode = rt.attrib3.tile_mode; + target.levels = levels; + auto* texture_cache = g_render_ctx->GetTextureCache(); + auto* buffer_vulkan = + texture_cache->FindRenderTarget(buffer, g_render_ctx->GetGraphicCtx(), target); + r->type = RenderColorType::RenderTexture; + r->base_addr = effective_base; + r->vulkan_buffer = buffer_vulkan; + r->vulkan_view = texture_cache->GetRenderTargetAttachmentView( + g_render_ctx->GetGraphicCtx(), buffer_vulkan, target.format, rt.view.current_mip_level); + r->format = target.format; + r->extent = view_extent; + r->base_mip_level = rt.view.current_mip_level; + r->buffer_size = size; + } else { + const auto layout = static_cast(rt.info.format); + const auto type = static_cast(rt.info.channel_type); + const auto order = static_cast(rt.info.channel_order); + + bool supported_display_format = + (layout == Prospero::ChannelLayout::k8_8_8_8 && + (type == Prospero::ChannelType::kSrgb || type == Prospero::ChannelType::kUNorm) && + (order == Prospero::ChannelOrder::kStandard || + order == Prospero::ChannelOrder::kAlt)) || + (layout == Prospero::ChannelLayout::k10_10_10_2 && + type == Prospero::ChannelType::kUNorm && + (order == Prospero::ChannelOrder::kStandard || order == Prospero::ChannelOrder::kAlt)); + + EXIT_NOT_IMPLEMENTED(!supported_display_format); + + // Display buffer + if (video_image.size != size) { + LOGF("RenderColorTarget: display buffer size differs from render target span, " + "video_size=0x%016" PRIx64 " render_size=0x%016" PRIx64 "\n", + video_image.size, size); + } + EXIT_NOT_IMPLEMENTED(video_image.size < size); + EXIT_NOT_IMPLEMENTED(video_image.pitch != pitch); + r->type = RenderColorType::DisplayBuffer; + r->base_addr = effective_base; + r->vulkan_buffer = video_image.image; + r->vulkan_view = video_image.image->image_view[VulkanImage::VIEW_DEFAULT]; + r->format = video_image.image->format; + r->extent = video_image.image->extent; + r->base_mip_level = 0; + r->buffer_size = video_image.size; + } +} + +void MarkRenderTargetGpuWritten(const RenderColorInfo& target) { + const bool with_color = target.vulkan_buffer != nullptr; + + if (with_color) { + if (target.type == RenderColorType::RenderTexture || + target.type == RenderColorType::DisplayBuffer) { + g_render_ctx->GetTextureCache()->MarkGpuWritten(target.vulkan_buffer); + } else { + EXIT("unknown writable render-color resource type\n"); + } + } +} + +} // namespace Libs::Graphics diff --git a/src/graphics/host_gpu/renderer/context.cpp b/src/graphics/host_gpu/renderer/context.cpp new file mode 100644 index 000000000..743f68b1a --- /dev/null +++ b/src/graphics/host_gpu/renderer/context.cpp @@ -0,0 +1,674 @@ +#include "common/assert.h" +#include "common/common.h" +#include "common/emulatorConfig.h" +#include "common/logging/log.h" +#include "common/profiler.h" +#include "common/threads.h" +#include "graphics/host_gpu/graphicContext.h" +#include "graphics/host_gpu/renderer/descriptorCache.h" +#include "graphics/host_gpu/renderer/framebufferCache.h" +#include "graphics/host_gpu/renderer/render.h" +#include "graphics/host_gpu/renderer/renderContext.h" +#include "graphics/host_gpu/renderer/renderState.h" +#include "graphics/host_gpu/utils.h" +#include "graphics/host_gpu/vma.h" +#include "graphics/presentation/window.h" + +#include +#include +#include +#include +#include +namespace Libs::Graphics { +static std::atomic g_command_buffer_submit_seq = 0; +class CommandPool { +public: + CommandPool() = default; + ~CommandPool() // NOLINT + { + // TODO(): check if destructor is called from std::_Exit() + // DeleteAll(); + } + + KYTY_CLASS_NO_COPY(CommandPool); + + VulkanCommandPool* GetPool(int id) { + if (m_pool[id] == nullptr) { + Create(id); + } + return m_pool[id]; + } + +private: + void Create(int id); + void DeleteAll(); + + VulkanCommandPool* m_pool[GraphicContext::QUEUES_NUM] = {}; +}; + +RenderContext* g_render_ctx = nullptr; +static thread_local CommandPool g_command_pool; + +FenceResourceRetainer::~FenceResourceRetainer() { + if (!m_resources.empty()) { + EXIT("fence resource retainer destroyed before release\n"); + } +} + +void FenceResourceRetainer::Retain(std::shared_ptr resource) { + if (resource == nullptr) { + EXIT("cannot retain a null fence resource\n"); + } + if (std::ranges::none_of(m_resources, [&resource](const auto& retained) { + return retained.get() == resource.get(); + })) { + m_resources.push_back(std::move(resource)); + } +} + +void FenceResourceRetainer::ReleaseAfterFence() noexcept { + m_resources.clear(); +} + +void GraphicsRenderInit() { + EXIT_IF(g_render_ctx != nullptr); + + g_render_ctx = new RenderContext; +} + +void GraphicsRenderCreateContext() { + EXIT_IF(g_render_ctx == nullptr); + + g_render_ctx->SetGraphicCtx(WindowGetGraphicContext()); +} + +void CommandPool::Create(int id) { + auto* ctx = g_render_ctx->GetGraphicCtx(); + + EXIT_IF(id < 0 || id >= GraphicContext::QUEUES_NUM); + auto*& pool = m_pool[id]; + EXIT_IF(pool != nullptr); + + EXIT_IF(ctx == nullptr); + EXIT_IF(ctx->device == nullptr); + EXIT_IF(ctx->queues[id].family == static_cast(-1)); + + pool = new VulkanCommandPool; + + VkCommandPoolCreateInfo pool_info {}; + pool_info.sType = VK_STRUCTURE_TYPE_COMMAND_POOL_CREATE_INFO; + pool_info.pNext = nullptr; + pool_info.queueFamilyIndex = ctx->queues[id].family; + pool_info.flags = VK_COMMAND_POOL_CREATE_RESET_COMMAND_BUFFER_BIT; + + vkCreateCommandPool(ctx->device, &pool_info, nullptr, &pool->pool); + + EXIT_NOT_IMPLEMENTED(pool->pool == nullptr); + + pool->buffers_count = 8; + pool->buffers = std::make_unique(pool->buffers_count); + pool->fences = std::make_unique(pool->buffers_count); + pool->semaphores = std::make_unique(pool->buffers_count); + pool->busy = std::make_unique(pool->buffers_count); + + VkCommandBufferAllocateInfo alloc_info {}; + alloc_info.sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_ALLOCATE_INFO; + alloc_info.commandPool = pool->pool; + alloc_info.level = VK_COMMAND_BUFFER_LEVEL_PRIMARY; + alloc_info.commandBufferCount = pool->buffers_count; + + if (vkAllocateCommandBuffers(ctx->device, &alloc_info, pool->buffers.get()) != VK_SUCCESS) { + EXIT("Can't allocate command buffers"); + } + + for (uint32_t i = 0; i < pool->buffers_count; i++) { + pool->busy[i] = false; + + VkFenceCreateInfo fence_info {}; + fence_info.sType = VK_STRUCTURE_TYPE_FENCE_CREATE_INFO; + fence_info.pNext = nullptr; + fence_info.flags = VK_FENCE_CREATE_SIGNALED_BIT; + + if (vkCreateFence(ctx->device, &fence_info, nullptr, &pool->fences[i]) != VK_SUCCESS) { + EXIT("Can't create fence"); + } + + VkSemaphoreCreateInfo semaphore_info {}; + semaphore_info.sType = VK_STRUCTURE_TYPE_SEMAPHORE_CREATE_INFO; + semaphore_info.pNext = nullptr; + semaphore_info.flags = 0; + + if (vkCreateSemaphore(ctx->device, &semaphore_info, nullptr, &pool->semaphores[i]) != + VK_SUCCESS) { + EXIT("Can't create semaphore"); + } + + EXIT_IF(pool->buffers[i] == nullptr); + EXIT_IF(pool->fences[i] == nullptr); + EXIT_IF(pool->semaphores[i] == nullptr); + } +} + +void CommandPool::DeleteAll() { + auto* ctx = g_render_ctx->GetGraphicCtx(); + + for (auto& pool: m_pool) { + if (pool != nullptr) { + EXIT_IF(ctx == nullptr); + EXIT_IF(ctx->device == nullptr); + + for (uint32_t i = 0; i < pool->buffers_count; i++) { + vkDestroySemaphore(ctx->device, pool->semaphores[i], nullptr); + vkDestroyFence(ctx->device, pool->fences[i], nullptr); + } + + vkFreeCommandBuffers(ctx->device, pool->pool, pool->buffers_count, pool->buffers.get()); + + vkDestroyCommandPool(ctx->device, pool->pool, nullptr); + + delete pool; + pool = nullptr; + } + } +} + +bool CommandBuffer::IsInvalid() const { + EXIT_IF(g_render_ctx == nullptr); + + if (m_pool != nullptr) { + Common::LockGuard lock(m_pool->mutex); + + return (m_index == static_cast(-1) || m_index >= m_pool->buffers_count); + } + + return true; +} + +void CommandBuffer::Allocate() { + EXIT_IF(!IsInvalid()); + + m_pool = g_command_pool.GetPool(m_queue); + + Common::LockGuard lock(m_pool->mutex); + + for (uint32_t i = 0; i < m_pool->buffers_count; i++) { + if (!m_pool->busy[i]) { + m_pool->busy[i] = true; + vkResetCommandBuffer(m_pool->buffers[i], VK_COMMAND_BUFFER_RESET_RELEASE_RESOURCES_BIT); + m_index = i; + break; + } + } + + EXIT_NOT_IMPLEMENTED(IsInvalid()); +} + +void CommandBuffer::Free() { + EXIT_IF(IsInvalid()); + + Common::LockGuard lock(m_pool->mutex); + + WaitForFence(); + + m_host_stream.Release(); + + m_pool->busy[m_index] = false; + vkResetCommandBuffer(m_pool->buffers[m_index], VK_COMMAND_BUFFER_RESET_RELEASE_RESOURCES_BIT); + RecycleDescriptorsAfterFence(); + m_fence_resources.ReleaseAfterFence(); + m_index = static_cast(-1); + + EXIT_NOT_IMPLEMENTED(!IsInvalid()); +} + +void CommandBuffer::DeleteAfterFence(VulkanBuffer* buffer) { + EXIT_IF(buffer == nullptr); + + m_delete_after_fence.push_back(buffer); +} + +void CommandBuffer::RetainResourceUntilFence(std::shared_ptr resource) { + if (IsInvalid() || m_execute) { + EXIT("cannot retain a resource on an invalid or submitted command buffer\n"); + } + m_fence_resources.Retain(std::move(resource)); +} + +void CommandBuffer::RecycleDescriptorAfterFence(VulkanDescriptorSet* set) { + EXIT_IF(set == nullptr); + + m_descriptor_sets_after_fence.push_back(set); +} + +void CommandBuffer::RecycleDescriptorsAfterFence() { + for (auto* set: m_descriptor_sets_after_fence) { + g_render_ctx->GetDescriptorCache()->Recycle(set); + } + m_descriptor_sets_after_fence.clear(); +} + +void CommandBuffer::Begin() const { + EXIT_IF(IsInvalid()); + + auto* buffer = m_pool->buffers[m_index]; + + VkCommandBufferBeginInfo begin_info {}; + begin_info.sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_BEGIN_INFO; + begin_info.pNext = nullptr; + begin_info.flags = 0; + begin_info.pInheritanceInfo = nullptr; + + auto result = vkBeginCommandBuffer(buffer, &begin_info); + + EXIT_NOT_IMPLEMENTED(result != VK_SUCCESS); +} + +void CommandBuffer::End() const { + EXIT_IF(IsInvalid()); + + auto* buffer = m_pool->buffers[m_index]; + + auto result = vkEndCommandBuffer(buffer); + + EXIT_NOT_IMPLEMENTED(result != VK_SUCCESS); +} + +void CommandBuffer::SetDebugInfo(uint32_t op, uint64_t submit_id, uint32_t arg0, uint32_t arg1, + uint32_t arg2, uint32_t arg3, uint64_t arg4) { + m_debug_op = op; + m_debug_submit_id = submit_id; + m_debug_arg0 = arg0; + m_debug_arg1 = arg1; + m_debug_arg2 = arg2; + m_debug_arg3 = arg3; + m_debug_arg4 = arg4; +} + +void CommandBuffer::Execute() { + EXIT_IF(IsInvalid()); + EXIT_IF(m_execute); + + auto* buffer = m_pool->buffers[m_index]; + auto* fence = m_pool->fences[m_index]; + + VkSubmitInfo submit_info {}; + submit_info.sType = VK_STRUCTURE_TYPE_SUBMIT_INFO; + submit_info.pNext = nullptr; + submit_info.waitSemaphoreCount = 0; + submit_info.pWaitSemaphores = nullptr; + submit_info.pWaitDstStageMask = nullptr; + submit_info.commandBufferCount = 1; + submit_info.pCommandBuffers = &buffer; + submit_info.signalSemaphoreCount = 0; + submit_info.pSignalSemaphores = nullptr; + + EXIT_IF(m_queue < 0 || m_queue >= GraphicContext::QUEUES_NUM); + + const auto& queue = g_render_ctx->GetGraphicCtx()->queues[m_queue]; + + auto result = vkResetFences(g_render_ctx->GetGraphicCtx()->device, 1, &fence); + if (result != VK_SUCCESS) { + LOGF("vkResetFences failed before submit: %s (%d)\n", string_VkResult(result), + static_cast(result)); + } + EXIT_NOT_IMPLEMENTED(result != VK_SUCCESS); + + if (queue.mutex != nullptr) { + queue.mutex->Lock(); + } + + if (Config::GraphicsDebugDumpEnabled()) { + LOGF("vkQueueSubmit begin: queue=%d index=%u debug_op=%u debug_submit=%" PRIu64 + " args=%u,%u,%u,%u,0x%016" PRIx64 "\n", + m_queue, m_index, m_debug_op, m_debug_submit_id, m_debug_arg0, m_debug_arg1, + m_debug_arg2, m_debug_arg3, m_debug_arg4); + } + + result = vkQueueSubmit(queue.vk_queue, 1, &submit_info, fence); + + if (queue.mutex != nullptr) { + queue.mutex->Unlock(); + } + + m_execute = true; + m_submit_seq = g_command_buffer_submit_seq.fetch_add(1, std::memory_order_relaxed) + 1; + + if (result != VK_SUCCESS) { + LOGF("vkQueueSubmit failed: %s (%d), queue=%d index=%u submit_seq=%" PRIu64 + " debug_op=%u debug_submit=%" PRIu64 " args=%u,%u,%u,%u,0x%016" PRIx64 "\n", + string_VkResult(result), static_cast(result), m_queue, m_index, m_submit_seq, + m_debug_op, m_debug_submit_id, m_debug_arg0, m_debug_arg1, m_debug_arg2, m_debug_arg3, + m_debug_arg4); + } + EXIT_NOT_IMPLEMENTED(result != VK_SUCCESS); +} + +void CommandBuffer::ExecuteWithSemaphore(VkSemaphore signal_semaphore) { + EXIT_IF(IsInvalid()); + EXIT_IF(m_execute); + + auto* buffer = m_pool->buffers[m_index]; + auto* fence = m_pool->fences[m_index]; + + if (signal_semaphore == nullptr) { + signal_semaphore = m_pool->semaphores[m_index]; + } + + VkSubmitInfo submit_info {}; + submit_info.sType = VK_STRUCTURE_TYPE_SUBMIT_INFO; + submit_info.pNext = nullptr; + submit_info.waitSemaphoreCount = 0; + submit_info.pWaitSemaphores = nullptr; + submit_info.pWaitDstStageMask = nullptr; + submit_info.commandBufferCount = 1; + submit_info.pCommandBuffers = &buffer; + submit_info.signalSemaphoreCount = 1; + submit_info.pSignalSemaphores = &signal_semaphore; + + EXIT_IF(m_queue < 0 || m_queue >= GraphicContext::QUEUES_NUM); + + const auto& queue = g_render_ctx->GetGraphicCtx()->queues[m_queue]; + + auto result = vkResetFences(g_render_ctx->GetGraphicCtx()->device, 1, &fence); + if (result != VK_SUCCESS) { + LOGF("vkResetFences failed before submit: %s (%d)\n", string_VkResult(result), + static_cast(result)); + } + EXIT_NOT_IMPLEMENTED(result != VK_SUCCESS); + + if (queue.mutex != nullptr) { + queue.mutex->Lock(); + } + + if (Config::GraphicsDebugDumpEnabled()) { + LOGF("vkQueueSubmit begin: queue=%d index=%u signal_semaphore=%p debug_op=%u" + " debug_submit=%" PRIu64 " args=%u,%u,%u,%u,0x%016" PRIx64 "\n", + m_queue, m_index, static_cast(signal_semaphore), m_debug_op, m_debug_submit_id, + m_debug_arg0, m_debug_arg1, m_debug_arg2, m_debug_arg3, m_debug_arg4); + } + + result = vkQueueSubmit(queue.vk_queue, 1, &submit_info, fence); + + if (queue.mutex != nullptr) { + queue.mutex->Unlock(); + } + + m_execute = true; + m_submit_seq = g_command_buffer_submit_seq.fetch_add(1, std::memory_order_relaxed) + 1; + + if (result != VK_SUCCESS) { + LOGF("vkQueueSubmit failed: %s (%d), queue=%d index=%u submit_seq=%" PRIu64 + " debug_op=%u debug_submit=%" PRIu64 " args=%u,%u,%u,%u,0x%016" PRIx64 "\n", + string_VkResult(result), static_cast(result), m_queue, m_index, m_submit_seq, + m_debug_op, m_debug_submit_id, m_debug_arg0, m_debug_arg1, m_debug_arg2, m_debug_arg3, + m_debug_arg4); + } + EXIT_NOT_IMPLEMENTED(result != VK_SUCCESS); +} + +void CommandBuffer::ExecuteWithSemaphore(VkSemaphore wait_semaphore, + VkPipelineStageFlags wait_stage, + VkSemaphore signal_semaphore) { + EXIT_IF(IsInvalid()); + EXIT_IF(m_execute); + EXIT_IF(wait_semaphore == nullptr); + + auto* buffer = m_pool->buffers[m_index]; + auto* fence = m_pool->fences[m_index]; + + if (signal_semaphore == nullptr) { + signal_semaphore = m_pool->semaphores[m_index]; + } + + VkSubmitInfo submit_info {}; + submit_info.sType = VK_STRUCTURE_TYPE_SUBMIT_INFO; + submit_info.pNext = nullptr; + submit_info.waitSemaphoreCount = 1; + submit_info.pWaitSemaphores = &wait_semaphore; + submit_info.pWaitDstStageMask = &wait_stage; + submit_info.commandBufferCount = 1; + submit_info.pCommandBuffers = &buffer; + submit_info.signalSemaphoreCount = 1; + submit_info.pSignalSemaphores = &signal_semaphore; + + EXIT_IF(m_queue < 0 || m_queue >= GraphicContext::QUEUES_NUM); + + const auto& queue = g_render_ctx->GetGraphicCtx()->queues[m_queue]; + + auto result = vkResetFences(g_render_ctx->GetGraphicCtx()->device, 1, &fence); + if (result != VK_SUCCESS) { + LOGF("vkResetFences failed before submit: %s (%d)\n", string_VkResult(result), + static_cast(result)); + } + EXIT_NOT_IMPLEMENTED(result != VK_SUCCESS); + + if (queue.mutex != nullptr) { + queue.mutex->Lock(); + } + + if (Config::GraphicsDebugDumpEnabled()) { + LOGF("vkQueueSubmit begin: queue=%d index=%u wait_semaphore=%p signal_semaphore=%p" + " debug_op=%u debug_submit=%" PRIu64 " args=%u,%u,%u,%u,0x%016" PRIx64 "\n", + m_queue, m_index, static_cast(wait_semaphore), + static_cast(signal_semaphore), m_debug_op, m_debug_submit_id, m_debug_arg0, + m_debug_arg1, m_debug_arg2, m_debug_arg3, m_debug_arg4); + } + + result = vkQueueSubmit(queue.vk_queue, 1, &submit_info, fence); + + if (queue.mutex != nullptr) { + queue.mutex->Unlock(); + } + + m_execute = true; + m_submit_seq = g_command_buffer_submit_seq.fetch_add(1, std::memory_order_relaxed) + 1; + + if (result != VK_SUCCESS) { + LOGF("vkQueueSubmit failed: %s (%d), queue=%d index=%u submit_seq=%" PRIu64 + " debug_op=%u debug_submit=%" PRIu64 " args=%u,%u,%u,%u,0x%016" PRIx64 "\n", + string_VkResult(result), static_cast(result), m_queue, m_index, m_submit_seq, + m_debug_op, m_debug_submit_id, m_debug_arg0, m_debug_arg1, m_debug_arg2, m_debug_arg3, + m_debug_arg4); + } + EXIT_NOT_IMPLEMENTED(result != VK_SUCCESS); +} + +void CommandBuffer::WaitForFence() { + EXIT_IF(IsInvalid()); + + const bool was_executed = m_execute; + if (m_execute) { + auto* device = g_render_ctx->GetGraphicCtx()->device; + + auto result = vkWaitForFences(device, 1, &m_pool->fences[m_index], VK_TRUE, UINT64_MAX); + if (result != VK_SUCCESS) { + LOGF("vkWaitForFences failed: %s (%d), wait=WaitForFence queue=%d index=%u " + "submit_seq=%" PRIu64 " debug_op=%u debug_submit=%" PRIu64 + " args=%u,%u,%u,%u,0x%016" PRIx64 "\n", + string_VkResult(result), static_cast(result), m_queue, m_index, m_submit_seq, + m_debug_op, m_debug_submit_id, m_debug_arg0, m_debug_arg1, m_debug_arg2, + m_debug_arg3, m_debug_arg4); + } + EXIT_NOT_IMPLEMENTED(result != VK_SUCCESS); + + m_execute = false; + } + + if (was_executed) { + RecycleDescriptorsAfterFence(); + m_fence_resources.ReleaseAfterFence(); + } + for (auto* buffer: m_delete_after_fence) { + VulkanDeleteBuffer(g_render_ctx->GetGraphicCtx(), buffer); + delete buffer; + } + m_delete_after_fence.clear(); +} + +void CommandBuffer::WaitForFenceAndReset() { + EXIT_IF(IsInvalid()); + + const bool was_executed = m_execute; + if (m_execute) { + auto* device = g_render_ctx->GetGraphicCtx()->device; + + auto result = vkWaitForFences(device, 1, &m_pool->fences[m_index], VK_TRUE, UINT64_MAX); + if (result != VK_SUCCESS) { + LOGF("vkWaitForFences failed: %s (%d), wait=WaitForFenceAndReset queue=%d index=%u " + "submit_seq=%" PRIu64 " debug_op=%u debug_submit=%" PRIu64 + " args=%u,%u,%u,%u,0x%016" PRIx64 "\n", + string_VkResult(result), static_cast(result), m_queue, m_index, m_submit_seq, + m_debug_op, m_debug_submit_id, m_debug_arg0, m_debug_arg1, m_debug_arg2, + m_debug_arg3, m_debug_arg4); + } + EXIT_NOT_IMPLEMENTED(result != VK_SUCCESS); + + m_execute = false; + + vkResetCommandBuffer(m_pool->buffers[m_index], + VK_COMMAND_BUFFER_RESET_RELEASE_RESOURCES_BIT); + } + + m_host_stream.Reset(); + + if (was_executed) { + RecycleDescriptorsAfterFence(); + m_fence_resources.ReleaseAfterFence(); + } + for (auto* buffer: m_delete_after_fence) { + VulkanDeleteBuffer(g_render_ctx->GetGraphicCtx(), buffer); + delete buffer; + } + m_delete_after_fence.clear(); +} + +void CommandBuffer::BeginRenderPass(VulkanFramebuffer* framebuffer, RenderColorInfo* colors, + uint32_t requested_color_count, RenderDepthInfo* depth) const { + EXIT_IF(IsInvalid()); + + auto* buffer = m_pool->buffers[m_index]; + + EXIT_IF(framebuffer == nullptr); + EXIT_IF(colors == nullptr); + EXIT_IF(requested_color_count > RENDER_COLOR_ATTACHMENTS_MAX); + + bool with_depth = (depth->format != VK_FORMAT_UNDEFINED && depth->vulkan_buffer != nullptr); + uint32_t color_count = 0; + for (uint32_t i = 0; i < requested_color_count; i++) { + if (colors[i].vulkan_buffer == nullptr) { + break; + } + color_count++; + } + bool with_color = (color_count != 0); + + EXIT_NOT_IMPLEMENTED(!with_depth && !with_color); + + VkClearValue clears[RENDER_COLOR_ATTACHMENTS_MAX + 1] = {}; + for (uint32_t i = 0; i < color_count; i++) { + clears[i].color = colors[i].color_clear_value; + } + clears[color_count].depthStencil = {depth->depth_clear_value, depth->stencil_clear_value}; + + VkExtent2D extent = (with_color ? colors[0].extent : depth->vulkan_buffer->extent); + + VkRenderPassBeginInfo render_pass_info {}; + render_pass_info.sType = VK_STRUCTURE_TYPE_RENDER_PASS_BEGIN_INFO; + render_pass_info.pNext = nullptr; + render_pass_info.renderPass = framebuffer->render_pass; + render_pass_info.framebuffer = framebuffer->framebuffer; + render_pass_info.renderArea.offset = {0, 0}; + render_pass_info.renderArea.extent = extent; + render_pass_info.clearValueCount = color_count + (with_depth ? 1u : 0u); + render_pass_info.pClearValues = clears; + + for (uint32_t i = 0; i < color_count; i++) { + const auto color_initial_layout = framebuffer->color_layout[i]; + if (colors[i].vulkan_buffer->layout != color_initial_layout) { + if (graphics_debug_dump_enabled()) { + LOGF("BeginRenderPass: color%u initial barrier image=%p mem=%" PRIu64 " %s -> %s\n", + i, reinterpret_cast(colors[i].vulkan_buffer->image), + colors[i].vulkan_buffer->memory.unique_id, + string_VkImageLayout(colors[i].vulkan_buffer->layout), + string_VkImageLayout(color_initial_layout)); + } + + VkImageMemoryBarrier image_memory_barrier {}; + image_memory_barrier.sType = VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER; + image_memory_barrier.pNext = nullptr; + image_memory_barrier.srcAccessMask = 0; + image_memory_barrier.dstAccessMask = + VK_ACCESS_COLOR_ATTACHMENT_READ_BIT | VK_ACCESS_COLOR_ATTACHMENT_WRITE_BIT; + image_memory_barrier.oldLayout = colors[i].vulkan_buffer->layout; + image_memory_barrier.newLayout = color_initial_layout; + image_memory_barrier.srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; + image_memory_barrier.dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; + image_memory_barrier.image = colors[i].vulkan_buffer->image; + image_memory_barrier.subresourceRange.aspectMask = VK_IMAGE_ASPECT_COLOR_BIT; + image_memory_barrier.subresourceRange.baseMipLevel = 0; + image_memory_barrier.subresourceRange.levelCount = VK_REMAINING_MIP_LEVELS; + image_memory_barrier.subresourceRange.baseArrayLayer = 0; + image_memory_barrier.subresourceRange.layerCount = 1; + + vkCmdPipelineBarrier(buffer, VK_PIPELINE_STAGE_TOP_OF_PIPE_BIT, + VK_PIPELINE_STAGE_COLOR_ATTACHMENT_OUTPUT_BIT, 0, 0, nullptr, 0, + nullptr, 1, &image_memory_barrier); + + colors[i].vulkan_buffer->layout = image_memory_barrier.newLayout; + } else if (graphics_debug_dump_enabled()) { + LOGF("BeginRenderPass: color%u initial image=%p mem=%" PRIu64 " layout=%s\n", i, + reinterpret_cast(colors[i].vulkan_buffer->image), + colors[i].vulkan_buffer->memory.unique_id, + string_VkImageLayout(colors[i].vulkan_buffer->layout)); + } + } + + const auto depth_layout = + (with_depth && framebuffer != nullptr ? framebuffer->depth_layout + : VK_IMAGE_LAYOUT_DEPTH_STENCIL_ATTACHMENT_OPTIMAL); + + if (with_depth && depth->vulkan_buffer->layout != depth_layout) { + VkImageMemoryBarrier image_memory_barrier {}; + image_memory_barrier.sType = VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER; + image_memory_barrier.pNext = nullptr; + image_memory_barrier.srcAccessMask = VK_ACCESS_MEMORY_READ_BIT | VK_ACCESS_MEMORY_WRITE_BIT; + image_memory_barrier.dstAccessMask = + (depth_layout == VK_IMAGE_LAYOUT_DEPTH_STENCIL_READ_ONLY_OPTIMAL + ? VK_ACCESS_MEMORY_READ_BIT + : VK_ACCESS_MEMORY_WRITE_BIT); + image_memory_barrier.oldLayout = depth->vulkan_buffer->layout; + image_memory_barrier.newLayout = depth_layout; + image_memory_barrier.srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; + image_memory_barrier.dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; + image_memory_barrier.image = depth->vulkan_buffer->image; + image_memory_barrier.subresourceRange.aspectMask = + DepthStencilAspectMask(depth->vulkan_buffer->format); + image_memory_barrier.subresourceRange.baseMipLevel = 0; + image_memory_barrier.subresourceRange.levelCount = 1; + image_memory_barrier.subresourceRange.baseArrayLayer = 0; + image_memory_barrier.subresourceRange.layerCount = 1; + + vkCmdPipelineBarrier( + buffer, VK_PIPELINE_STAGE_ALL_GRAPHICS_BIT | VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT, + VK_PIPELINE_STAGE_ALL_GRAPHICS_BIT | VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT, 0, 0, + nullptr, 0, nullptr, 1, &image_memory_barrier); + + depth->vulkan_buffer->layout = image_memory_barrier.newLayout; + } + + vkCmdBeginRenderPass(buffer, &render_pass_info, VK_SUBPASS_CONTENTS_INLINE); + + for (uint32_t i = 0; i < color_count; i++) { + colors[i].vulkan_buffer->layout = RENDER_COLOR_IMAGE_LAYOUT; + } +} + +void CommandBuffer::EndRenderPass() const { + EXIT_IF(IsInvalid()); + + auto* buffer = m_pool->buffers[m_index]; + + vkCmdEndRenderPass(buffer); +} + +} // namespace Libs::Graphics diff --git a/src/graphics/host_gpu/renderer/debug.cpp b/src/graphics/host_gpu/renderer/debug.cpp new file mode 100644 index 000000000..a1a985be3 --- /dev/null +++ b/src/graphics/host_gpu/renderer/debug.cpp @@ -0,0 +1,1364 @@ +#include "common/assert.h" +#include "common/common.h" +#include "common/emulatorConfig.h" +#include "common/logging/log.h" +#include "common/stringUtils.h" +#include "graphics/guest_gpu/hardwareContext.h" +#include "graphics/guest_gpu/gpu_defs.h" +#include "graphics/host_gpu/renderer/render.h" +#include "graphics/host_gpu/renderer/renderState.h" + +#include +#include +#include + +namespace Libs::Graphics { + +static std::atomic g_scissor_default_log_count = 0; + +uint32_t render_target_mask_slot(uint32_t mask, uint32_t slot) { + return (mask >> (slot * 4u)) & 0x0fu; +} + +static bool RenderTargetMaskHasMrt(uint32_t mask) { + return (mask & ~0x0fu) != 0; +} + +static bool RenderTargetMaskHasBoundMrt(const HW::Context& hw) { + const auto mask = hw.GetRenderTargetMask(); + + if (!RenderTargetMaskHasMrt(mask)) { + return false; + } + + uint32_t bound_targets = 0; + for (uint32_t i = 0; i < 8; i++) { + if (render_target_mask_slot(mask, i) != 0 && hw.GetRenderTarget(i).base.addr != 0) { + bound_targets++; + } + } + + return bound_targets > 1; +} + +uint32_t render_target_first_bound_slot(const HW::Context& hw) { + const auto mask = hw.GetRenderTargetMask(); + for (uint32_t i = 0; i < 8; i++) { + if (render_target_mask_slot(mask, i) != 0 && hw.GetRenderTarget(i).base.addr != 0) { + return i; + } + } + + return 0; +} + +bool graphics_debug_dump_enabled() { + return Config::GraphicsDebugDumpEnabled() && + Config::GetPrintfDirection() != Config::OutputDirection::Silent; +} + +void uc_print(const char* func, const HW::UserConfig& uc) { + LOGF("%s\n", func); + + const auto& ge_cntl = uc.GetGeControl(); + const auto& user_en = uc.GetGeUserVgprEn(); + + LOGF("\t GetPrimType() = 0x%08" PRIx32 "\n" + "\t GetIndexOffset() = 0x%08" PRIx32 "\n" + "\t GetObjectId() = 0x%08" PRIx32 "\n" + "\t primitive_group_size = 0x%04" PRIx16 "\n" + "\t vertex_group_size = 0x%04" PRIx16 "\n" + "\t en_user_vgpr1 = %s\n" + "\t en_user_vgpr2 = %s\n" + "\t en_user_vgpr3 = %s\n", + uc.GetPrimType(), uc.GetIndexOffset(), uc.GetObjectId(), ge_cntl.primitive_group_size, + ge_cntl.vertex_group_size, user_en.vgpr1 ? "true" : "false", + user_en.vgpr2 ? "true" : "false", user_en.vgpr3 ? "true" : "false"); +} + +void uc_check(const HW::UserConfig& uc) { + const auto& ge_cntl = uc.GetGeControl(); + const auto& user_en = uc.GetGeUserVgprEn(); + + if (ge_cntl.primitive_group_size > 0x0040) { + LOGF("\t warning: unsupported GE_CNTL primitive_group_size = 0x%04" PRIx16 "\n", + ge_cntl.primitive_group_size); + } + if (ge_cntl.vertex_group_size > 0x0040) { + LOGF("\t warning: unsupported GE_CNTL vertex_group_size = 0x%04" PRIx16 "\n", + ge_cntl.vertex_group_size); + } + EXIT_NOT_IMPLEMENTED(user_en.vgpr1 != false); + EXIT_NOT_IMPLEMENTED(user_en.vgpr2 != false); + EXIT_NOT_IMPLEMENTED(user_en.vgpr3 != false); +} + +void sh_print(const char* func, const HW::Shader& /*uc*/) { + LOGF("%s\n", func); +} + +void sh_check(const HW::Shader& /*uc*/) {} + +std::vector rt_print(const char* func, const HW::RenderTarget& rt) { + std::vector dst; + dst.reserve(53); + + dst.push_back(fmt::format("{}\n", func)); + + dst.push_back(fmt::format("\t base.addr = 0x{:016x}\n", rt.base.addr)); + dst.push_back( + fmt::format("\t pitch.pitch_div8_minus1 = 0x{:08x}\n", rt.pitch.pitch_div8_minus1)); + dst.push_back(fmt::format("\t pitch.fmask_pitch_div8_minus1 = 0x{:08x}\n", + rt.pitch.fmask_pitch_div8_minus1)); + dst.push_back(fmt::format("\t slice.slice_div64_minus1 = 0x{:08x}\n", + rt.slice.slice_div64_minus1)); + dst.push_back(fmt::format("\t view.base_array_slice_index = 0x{:08x}\n", + rt.view.base_array_slice_index)); + dst.push_back(fmt::format("\t view.last_array_slice_index = 0x{:08x}\n", + rt.view.last_array_slice_index)); + dst.push_back( + fmt::format("\t view.current_mip_level = 0x{:08x}\n", rt.view.current_mip_level)); + dst.push_back(fmt::format("\t info.fmask_compression_enable = {}\n", + rt.info.fmask_compression_enable ? "true" : "false")); + + // dst.push_back(fmt::format("\t info.fmask_compression_mode = 0x{:08x}\n", // + // rt.info.fmask_compression_mode)); + dst.push_back(fmt::format("\t info.fmask_data_compression_disable = {}\n", + rt.info.fmask_data_compression_disable ? "true" : "false")); + dst.push_back(fmt::format("\t info.fmask_one_frag_mode = {}\n", + rt.info.fmask_one_frag_mode ? "true" : "false")); + + dst.push_back(fmt::format("\t info.cmask_fast_clear_enable = {}\n", + rt.info.cmask_fast_clear_enable ? "true" : "false")); + dst.push_back(fmt::format("\t info.dcc_compression_enable = {}\n", + rt.info.dcc_compression_enable ? "true" : "false")); + dst.push_back( + fmt::format("\t info.cmask_is_linear = 0x{:08x}\n", rt.info.cmask_is_linear)); + dst.push_back( + fmt::format("\t info.cmask_addr_type = 0x{:08x}\n", rt.info.cmask_addr_type)); + dst.push_back(fmt::format("\t info.alt_tile_mode = {}\n", + rt.info.alt_tile_mode ? "true" : "false")); + dst.push_back(fmt::format("\t info.format = 0x{:08x}\n", rt.info.format)); + dst.push_back( + fmt::format("\t info.channel_type = 0x{:08x}\n", rt.info.channel_type)); + dst.push_back( + fmt::format("\t info.channel_order = 0x{:08x}\n", rt.info.channel_order)); + dst.push_back(fmt::format("\t info.blend_bypa = {}\n", + rt.info.blend_bypass ? "true" : "false")); + dst.push_back(fmt::format("\t info.blend_clamp = {}\n", + rt.info.blend_clamp ? "true" : "false")); + dst.push_back(fmt::format("\t info.round_mode = {}\n", + rt.info.round_mode ? "true" : "false")); + dst.push_back(fmt::format("\t attrib.force_dest_alpha_to_one = {}\n", + rt.attrib.force_dest_alpha_to_one ? "true" : "false")); + dst.push_back( + fmt::format("\t attrib.tile_mode = 0x{:08x}\n", rt.attrib.tile_mode)); + dst.push_back( + fmt::format("\t attrib.fmask_tile_mode = 0x{:08x}\n", rt.attrib.fmask_tile_mode)); + dst.push_back( + fmt::format("\t attrib.num_samples = 0x{:08x}\n", rt.attrib.num_samples)); + dst.push_back( + fmt::format("\t attrib.num_fragments = 0x{:08x}\n", rt.attrib.num_fragments)); + dst.push_back(fmt::format("\t attrib2.width = 0x{:08x}\n", rt.attrib2.width)); + dst.push_back( + fmt::format("\t attrib2.height = 0x{:08x}\n", rt.attrib2.height)); + dst.push_back( + fmt::format("\t attrib2.num_mip_levels = 0x{:08x}\n", rt.attrib2.num_mip_levels)); + dst.push_back(fmt::format("\t attrib3.depth = 0x{:08x}\n", rt.attrib3.depth)); + dst.push_back( + fmt::format("\t attrib3.tile_mode = 0x{:08x}\n", rt.attrib3.tile_mode)); + dst.push_back( + fmt::format("\t attrib3.dimension = 0x{:08x}\n", rt.attrib3.dimension)); + dst.push_back(fmt::format("\t attrib3.cmask_pipe_aligned = {}\n", + rt.attrib3.cmask_pipe_aligned ? "true" : "false")); + dst.push_back(fmt::format("\t attrib3.dcc_pipe_aligned = {}\n", + rt.attrib3.dcc_pipe_aligned ? "true" : "false")); + dst.push_back(fmt::format("\t dcc.max_uncompressed_block_size = 0x{:08x}\n", + rt.dcc.max_uncompressed_block_size)); + dst.push_back(fmt::format("\t dcc.max_compressed_block_size = 0x{:08x}\n", + rt.dcc.max_compressed_block_size)); + dst.push_back(fmt::format("\t dcc.min_compressed_block_size = 0x{:08x}\n", + rt.dcc.min_compressed_block_size)); + dst.push_back( + fmt::format("\t dcc.color_transform = 0x{:08x}\n", rt.dcc.color_transform)); + dst.push_back(fmt::format("\t dcc.overwrite_combiner_disable = {}\n", + rt.dcc.overwrite_combiner_disable ? "true" : "false")); + dst.push_back(fmt::format("\t dcc.independent_64b_blocks = {}\n", + rt.dcc.independent_64b_blocks ? "true" : "false")); + dst.push_back(fmt::format("\t dcc.independent_128b_blocks = {}\n", + rt.dcc.independent_128b_blocks ? "true" : "false")); + dst.push_back(fmt::format("\t data_write_on_dcc_clear_to_reg = {}\n", + rt.dcc.data_write_on_dcc_clear_to_reg ? "true" : "false")); + dst.push_back(fmt::format("\t dcc.dcc_clear_key_enable = {}\n", + rt.dcc.dcc_clear_key_enable ? "true" : "false")); + dst.push_back(fmt::format("\t cmask.addr = 0x{:016x}\n", rt.cmask.addr)); + dst.push_back(fmt::format("\t cmask_slice.slice_minus1 = 0x{:08x}\n", + rt.cmask_slice.slice_minus1)); + dst.push_back(fmt::format("\t fmask.addr = 0x{:016x}\n", rt.fmask.addr)); + dst.push_back(fmt::format("\t fmask_slice.slice_minus1 = 0x{:08x}\n", + rt.fmask_slice.slice_minus1)); + dst.push_back( + fmt::format("\t clear_word0.word0 = 0x{:08x}\n", rt.clear_word0.word0)); + dst.push_back( + fmt::format("\t clear_word1.word1 = 0x{:08x}\n", rt.clear_word1.word1)); + dst.push_back( + fmt::format("\t dcc_addr.addr = 0x{:016x}\n", rt.dcc_addr.addr)); + dst.push_back(fmt::format("\t size.width = 0x{:08x}\n", rt.size.width)); + dst.push_back(fmt::format("\t size.height = 0x{:08x}\n", rt.size.height)); + + return dst; +} + +static bool RenderIsColorTileMode(uint32_t tile_mode) { + // AGC CxRenderTarget::TileMode shifted by CB_COLOR*_ATTRIB3.COLOR_SW_MODE. + switch (tile_mode) { + case Prospero::GpuEnumValue(Prospero::TileMode::kLinear): + case Prospero::GpuEnumValue(Prospero::TileMode::kStandard256B): + case Prospero::GpuEnumValue(Prospero::TileMode::kStandard4KB): + case Prospero::GpuEnumValue(Prospero::TileMode::kStandard64KB): + case Prospero::GpuEnumValue(Prospero::TileMode::kPrt): + case Prospero::GpuEnumValue(Prospero::TileMode::kDepth): + case Prospero::GpuEnumValue(Prospero::TileMode::kRenderTarget): return true; + default: return false; + } +} + +bool RenderIsColorTileModeLinear(uint32_t tile_mode) { + return tile_mode == Prospero::GpuEnumValue(Prospero::TileMode::kLinear); +} + +static bool RenderIsColorDimension(uint32_t dimension) { + return dimension <= 0x02; +} + +// NOLINTNEXTLINE(readability-function-cognitive-complexity) +static void RtCheck(const HW::RenderTarget& rt) { + if (rt.base.addr != 0) { + // bool render_to_texture = (rt.attrib.tile_mode == 0x0d); + // EXIT_NOT_IMPLEMENTED(rt.base_addr == 0); + + EXIT_NOT_IMPLEMENTED(rt.pitch.pitch_div8_minus1 != 0); + EXIT_NOT_IMPLEMENTED(rt.pitch.fmask_pitch_div8_minus1 != 0); + EXIT_NOT_IMPLEMENTED(rt.slice.slice_div64_minus1 != 0); + + EXIT_NOT_IMPLEMENTED(rt.view.base_array_slice_index > rt.view.last_array_slice_index); + if (rt.view.base_array_slice_index != 0x00000000 || + rt.view.last_array_slice_index != 0x00000000) { + static bool logged = false; + if (!logged) { + LOGF("RenderTarget: using color target array slice range %" PRIu32 "..%" PRIu32 + "\n", + rt.view.base_array_slice_index, rt.view.last_array_slice_index); + logged = true; + } + } + if (rt.view.current_mip_level != 0x00000000) { + static bool logged = false; + if (!logged) { + LOGF("RenderTarget: using PS5 color target mip level %" PRIu32 "\n", + rt.view.current_mip_level); + logged = true; + } + } + if (rt.info.fmask_compression_enable) { + EXIT_NOT_IMPLEMENTED(rt.attrib.num_samples == 0 && rt.attrib.num_fragments == 0); + static bool logged = false; + if (!logged) { + LOGF("RenderTarget: temporary: ignoring PS5 FMASK metadata for single-sample MSAA " + "fallback, fmask=0x%016" PRIx64 "\n", + rt.fmask.addr); + logged = true; + } + } + + // EXIT_NOT_IMPLEMENTED(rt.info.fmask_compression_mode != 0x00000000); + EXIT_NOT_IMPLEMENTED(rt.info.fmask_data_compression_disable != false); + EXIT_NOT_IMPLEMENTED(rt.info.fmask_one_frag_mode != false); + + if (rt.info.cmask_fast_clear_enable || rt.info.dcc_compression_enable) { + static bool logged = false; + if (!logged) { + LOGF("RenderTarget: temporary: ignoring PS5 color metadata fast_clear=%s dcc=%s " + "cmask=0x%016" PRIx64 " dcc_addr=0x%016" PRIx64 "\n", + rt.info.cmask_fast_clear_enable ? "true" : "false", + rt.info.dcc_compression_enable ? "true" : "false", rt.cmask.addr, + rt.dcc_addr.addr); + logged = true; + } + } + if (rt.info.cmask_is_linear != 0x00000000 || rt.info.cmask_addr_type != 0x00000000) { + static bool logged = false; + if (!logged) { + LOGF("RenderTarget: temporary: ignoring PS5 CMASK metadata fields " + "is_linear=%u addr_type=%u\n", + rt.info.cmask_is_linear, rt.info.cmask_addr_type); + logged = true; + } + } + if (rt.info.alt_tile_mode) { + static bool logged = false; + if (!logged) { + LOGF("RenderTarget: temporary: ignoring alternate tile mode flag\n"); + logged = true; + } + } + if (rt.info.blend_bypass) { + static bool logged = false; + if (!logged) { + LOGF("RenderTarget: temporary: using PS5 blend bypass as disabled Vulkan " + "blending\n"); + logged = true; + } + } + // EXIT_NOT_IMPLEMENTED(rt.info.blend_clamp != false); + if (rt.info.round_mode) { + static bool logged = false; + if (!logged) { + LOGF("RenderTarget: temporary: ignoring PS5 color round mode\n"); + logged = true; + } + } + // EXIT_NOT_IMPLEMENTED(rt.format != 0x0000000a); + // EXIT_NOT_IMPLEMENTED(rt.channel_type != 0x00000006); + // EXIT_NOT_IMPLEMENTED(rt.channel_order != 0x00000001); + if (rt.attrib.force_dest_alpha_to_one) { + static bool logged = false; + if (!logged) { + LOGF("RenderTarget: temporary: accepting PS5 force destination alpha-to-one\n"); + logged = true; + } + } + // EXIT_NOT_IMPLEMENTED(rt.tile_mode != 0x0000000a); + // EXIT_NOT_IMPLEMENTED(rt.fmask_tile_mode != 0x0000000a); + if (rt.attrib.num_samples != 0x00000000 || rt.attrib.num_fragments != 0x00000000) { + static bool logged = false; + if (!logged) { + LOGF("RenderTarget: temporary: rendering PS5 MSAA color target as single-sample, " + "samples=0x%08" PRIx32 " fragments=0x%08" PRIx32 "\n", + rt.attrib.num_samples, rt.attrib.num_fragments); + logged = true; + } + } + + if (rt.attrib2.width == 0x00000000 || rt.attrib2.height == 0x00000000) { + static bool logged = false; + if (!logged) { + LOGF("RenderTarget: temporary: accepting PS5 raw 1-pixel color target extent " + "fields width_minus1=0x%08" PRIx32 " height_minus1=0x%08" PRIx32 "\n", + rt.attrib2.width, rt.attrib2.height); + logged = true; + } + } + if (rt.attrib2.num_mip_levels != 0x00000000) { + static bool logged = false; + if (!logged) { + LOGF("RenderTarget: using PS5 color target mip count field 0x%08" PRIx32 "\n", + rt.attrib2.num_mip_levels); + logged = true; + } + } + if (rt.attrib3.depth != 0x00000000) { + static bool logged = false; + if (!logged) { + LOGF("RenderTarget: temporary: ignoring PS5 color target depth_minus1=0x%08" PRIx32 + "\n", + rt.attrib3.depth); + logged = true; + } + } + if (!RenderIsColorTileMode(rt.attrib3.tile_mode)) { + EXIT("unknown PS5 render-target tile mode: 0x%08" PRIx32 "\n", rt.attrib3.tile_mode); + } + if (rt.attrib3.tile_mode != 0x0000001b) { + static bool logged = false; + if (!logged) { + LOGF("RenderTarget: temporary: using generic PS5 color tile fallback for " + "tile_mode=0x%08" PRIx32 "\n", + rt.attrib3.tile_mode); + logged = true; + } + } + if (!RenderIsColorDimension(rt.attrib3.dimension)) { + EXIT("unknown PS5 render-target dimension: 0x%08" PRIx32 "\n", rt.attrib3.dimension); + } + if (rt.attrib3.dimension != 0x00000001) { + static bool logged = false; + if (!logged) { + LOGF("RenderTarget: temporary: using 2D fallback for PS5 color " + "dimension=0x%08" PRIx32 "\n", + rt.attrib3.dimension); + logged = true; + } + } + if (!rt.attrib3.cmask_pipe_aligned) { + static bool logged = false; + if (!logged) { + LOGF("RenderTarget: temporary: accepting unaligned PS5 CMASK pipe flag\n"); + logged = true; + } + } + if (!rt.attrib3.dcc_pipe_aligned) { + static bool logged = false; + if (!logged) { + LOGF("RenderTarget: temporary: accepting unaligned PS5 DCC pipe flag\n"); + logged = true; + } + } + + // EXIT_NOT_IMPLEMENTED(rt.dcc_max_uncompressed_block_size != 0x00000002); + // EXIT_NOT_IMPLEMENTED(rt.dcc.max_compressed_block_size != 0x00000000); + EXIT_NOT_IMPLEMENTED(rt.dcc.min_compressed_block_size != 0x00000000); + // EXIT_NOT_IMPLEMENTED(rt.dcc.color_transform != 0x00000000); + EXIT_NOT_IMPLEMENTED(rt.dcc.overwrite_combiner_disable != false); + // EXIT_NOT_IMPLEMENTED(rt.dcc.force_independent_blocks != false); + // EXIT_NOT_IMPLEMENTED(rt.dcc.independent_128b_blocks != false); + // EXIT_NOT_IMPLEMENTED(rt.dcc.data_write_on_dcc_clear_to_reg != false); + EXIT_NOT_IMPLEMENTED(rt.dcc.dcc_clear_key_enable != false); + if (rt.cmask.addr != 0x0000000000000000 || rt.cmask_slice.slice_minus1 != 0x00000000 || + rt.fmask.addr != 0x0000000000000000 || + (rt.fmask_slice.slice_minus1 != 0x00000000 && + rt.fmask_slice.slice_minus1 != rt.slice.slice_div64_minus1) || + rt.dcc_addr.addr != 0x0000000000000000) { + static bool logged = false; + if (!logged) { + LOGF("RenderTarget: temporary: ignoring PS5 metadata addresses cmask=0x%016" PRIx64 + " fmask=0x%016" PRIx64 " dcc=0x%016" PRIx64 "\n", + rt.cmask.addr, rt.fmask.addr, rt.dcc_addr.addr); + logged = true; + } + } + + EXIT_NOT_IMPLEMENTED(rt.size.width != 0); + EXIT_NOT_IMPLEMENTED(rt.size.height != 0); + } +} + +static void ZPrint(const char* func, const HW::DepthRenderTarget& z) { + LOGF("%s\n", func); + + LOGF("\t z_info.format = 0x%08" PRIx32 "\n" + "\t z_info.tile_mode_index = 0x%08" PRIx32 "\n" + "\t z_info.num_samples = 0x%08" PRIx32 "\n" + "\t z_info.tile_surface_enable = %s\n" + "\t z_info.expclear_enabled = %s\n" + "\t z_info.zrange_precision = 0x%08" PRIx32 "\n" + "\t z_info.embedded_sample_locations = %s\n" + "\t z_info.partially_resident = %s\n" + "\t z_info.num_mip_levels = 0x%02" PRIx8 "\n" + "\t z_info.plane_compression = 0x%02" PRIx8 "\n" + "\t stencil_info.format = 0x%08" PRIx32 "\n" + "\t stencil_info.tile_stencil_disable = %s\n" + "\t stencil_info.expclear_enabled = %s\n" + "\t stencil_info.tile_mode_index = 0x%08" PRIx32 "\n" + "\t stencil_info.tile_split = 0x%08" PRIx32 "\n" + "\t stencil_info.texture_compatible_stencil = %s\n" + "\t stencil_info.partially_resident = %s\n" + "\t depth_info.addr5_swizzle_mask = 0x%08" PRIx32 "\n" + "\t depth_info.array_mode = 0x%08" PRIx32 "\n" + "\t depth_info.pipe_config = 0x%08" PRIx32 "\n" + "\t depth_info.bank_width = 0x%08" PRIx32 "\n" + "\t depth_info.bank_height = 0x%08" PRIx32 "\n" + "\t depth_info.macro_tile_aspect = 0x%08" PRIx32 "\n" + "\t depth_info.num_banks = 0x%08" PRIx32 "\n" + "\t depth_view.slice_start = 0x%08" PRIx32 "\n" + "\t depth_view.slice_max = 0x%08" PRIx32 "\n" + "\t depth_view.current_mip_level = 0x%02" PRIx8 "\n" + "\t depth_view.depth_write_disable = %s\n" + "\t depth_view.stencil_write_disable = %s\n" + "\t htile_surface.linear = 0x%08" PRIx32 "\n" + "\t htile_surface.full_cache = 0x%08" PRIx32 "\n" + "\t htile_surface.htile_uses_preload_win = 0x%08" PRIx32 "\n" + "\t htile_surface.preload = 0x%08" PRIx32 "\n" + "\t htile_surface.prefetch_width = 0x%08" PRIx32 "\n" + "\t htile_surface.prefetch_height = 0x%08" PRIx32 "\n" + "\t htile_surface.dst_outside_zero_to_one = 0x%08" PRIx32 "\n" + "\t z_read_base_addr = 0x%016" PRIx64 "\n" + "\t stencil_read_base_addr = 0x%016" PRIx64 "\n" + "\t z_write_base_addr = 0x%016" PRIx64 "\n" + "\t stencil_write_base_addr = 0x%016" PRIx64 "\n" + "\t pitch_div8_minus1 = 0x%08" PRIx32 "\n" + "\t height_div8_minus1 = 0x%08" PRIx32 "\n" + "\t slice_div64_minus1 = 0x%08" PRIx32 "\n" + "\t htile_data_base_addr = 0x%016" PRIx64 "\n" + "\t width = 0x%08" PRIx32 "\n" + "\t height = 0x%08" PRIx32 "\n" + "\t size.x_max = 0x%04" PRIx16 "\n" + "\t size.y_max = 0x%04" PRIx16 "\n", + z.z_info.format, z.z_info.tile_mode_index, z.z_info.num_samples, + z.z_info.tile_surface_enable ? "true" : "false", + z.z_info.expclear_enabled ? "true" : "false", z.z_info.zrange_precision, + z.z_info.embedded_sample_locations ? "true" : "false", + z.z_info.partially_resident ? "true" : "false", z.z_info.num_mip_levels, + z.z_info.plane_compression, z.stencil_info.format, + z.stencil_info.tile_stencil_disable ? "true" : "false", + z.stencil_info.expclear_enabled ? "true" : "false", z.stencil_info.tile_mode_index, + z.stencil_info.tile_split, z.stencil_info.texture_compatible_stencil ? "true" : "false", + z.stencil_info.partially_resident ? "true" : "false", z.depth_info.addr5_swizzle_mask, + z.depth_info.array_mode, z.depth_info.pipe_config, z.depth_info.bank_width, + z.depth_info.bank_height, z.depth_info.macro_tile_aspect, z.depth_info.num_banks, + z.depth_view.slice_start, z.depth_view.slice_max, z.depth_view.current_mip_level, + z.depth_view.depth_write_disable ? "true" : "false", + z.depth_view.stencil_write_disable ? "true" : "false", z.htile_surface.linear, + z.htile_surface.full_cache, z.htile_surface.htile_uses_preload_win, + z.htile_surface.preload, z.htile_surface.prefetch_width, z.htile_surface.prefetch_height, + z.htile_surface.dst_outside_zero_to_one, z.z_read_base_addr, z.stencil_read_base_addr, + z.z_write_base_addr, z.stencil_write_base_addr, z.pitch_div8_minus1, z.height_div8_minus1, + z.slice_div64_minus1, z.htile_data_base_addr, z.width, z.height, z.size.x_max, + z.size.y_max); +} + +// NOLINTNEXTLINE(readability-function-cognitive-complexity) +static void ZCheck(const HW::DepthRenderTarget& z) { + if (z.z_info.format == 0) { + EXIT_NOT_IMPLEMENTED(z.z_info.format != 0); + EXIT_NOT_IMPLEMENTED(z.z_info.tile_mode_index != 0); + EXIT_NOT_IMPLEMENTED(z.z_info.num_samples != 0); + EXIT_NOT_IMPLEMENTED(z.z_info.tile_surface_enable != false); + EXIT_NOT_IMPLEMENTED(z.z_info.expclear_enabled != false); + if (z.z_info.zrange_precision != 0) { + LOGF("Warning: zrange_precision != 0\n"); + // z.z_info.zrange_precision = 0; + } + if (z.z_info.embedded_sample_locations) { + static bool logged = false; + if (!logged) { + LOGF("DepthTarget: temporary: ignoring embedded sample locations\n"); + logged = true; + } + } + EXIT_NOT_IMPLEMENTED(z.z_info.partially_resident != false); + EXIT_NOT_IMPLEMENTED(z.z_info.num_mip_levels != 0); + if (z.z_info.plane_compression != 0) { + static bool logged = false; + if (!logged) { + LOGF("DepthTarget: temporary: ignoring PS5 plane_compression=0x%02" PRIx8 "\n", + z.z_info.plane_compression); + logged = true; + } + } + } else { + EXIT_NOT_IMPLEMENTED(z.z_info.format != 0x00000001 && z.z_info.format != 0x00000003); + // EXIT_NOT_IMPLEMENTED(z.z_info.tile_mode_index != 0x00000002); + if (z.z_info.num_samples != 0x00000000) { + static bool logged = false; + if (!logged) { + LOGF("DepthTarget: temporary: ignoring num_samples=0x%08" PRIx32 "\n", + z.z_info.num_samples); + logged = true; + } + } + // EXIT_NOT_IMPLEMENTED(z.z_info.tile_surface_enable != true); + EXIT_NOT_IMPLEMENTED(z.z_info.expclear_enabled != false); + if (z.z_info.zrange_precision != 0x00000001) { + static bool logged = false; + if (!logged) { + LOGF("DepthTarget: temporary: ignoring zrange_precision=0x%08" PRIx32 "\n", + z.z_info.zrange_precision); + logged = true; + } + } + if (z.z_info.embedded_sample_locations) { + static bool logged = false; + if (!logged) { + LOGF("DepthTarget: temporary: ignoring embedded sample locations\n"); + logged = true; + } + } + EXIT_NOT_IMPLEMENTED(z.z_info.partially_resident != false); + EXIT_NOT_IMPLEMENTED(z.z_info.num_mip_levels != 0); + if (z.z_info.plane_compression != 0) { + static bool logged = false; + if (!logged) { + LOGF("DepthTarget: temporary: ignoring PS5 plane_compression=0x%02" PRIx8 "\n", + z.z_info.plane_compression); + logged = true; + } + } + } + + if (z.stencil_info.format == 0) { + // EXIT_NOT_IMPLEMENTED(z.stencil_info.format != 0); + // EXIT_NOT_IMPLEMENTED(z.stencil_info.tile_stencil_disable != false); + EXIT_NOT_IMPLEMENTED(z.stencil_info.expclear_enabled != false); + // EXIT_NOT_IMPLEMENTED(z.stencil_info.tile_mode_index != 0); + // EXIT_NOT_IMPLEMENTED(z.stencil_info.tile_split != 0); + // EXIT_NOT_IMPLEMENTED(z.stencil_info.texture_compatible_stencil != true); + EXIT_NOT_IMPLEMENTED(z.stencil_info.partially_resident != false); + } else { + // EXIT_NOT_IMPLEMENTED(z.stencil_info.format != 0x00000001); + if (z.stencil_info.tile_stencil_disable != true) { + + static std::atomic log_count {0}; + if (log_count.fetch_add(1) < 16) { + LOGF("DepthTarget: temporary: ignoring PS5 HTILE stencil acceleration\n"); + } + } + EXIT_NOT_IMPLEMENTED(z.stencil_info.expclear_enabled != false); + // EXIT_NOT_IMPLEMENTED(z.stencil_info.tile_mode_index != 0x00000002); + // EXIT_NOT_IMPLEMENTED(z.stencil_info.tile_split != 0x00000002); + // EXIT_NOT_IMPLEMENTED(z.stencil_info.texture_compatible_stencil != true); + EXIT_NOT_IMPLEMENTED(z.stencil_info.partially_resident != false); + } + + if (z.z_info.format != 0 || z.stencil_info.format != 0) { + + EXIT_NOT_IMPLEMENTED(z.depth_info.addr5_swizzle_mask != 0x00000000); + EXIT_NOT_IMPLEMENTED(z.depth_info.array_mode != 0x00000000); + EXIT_NOT_IMPLEMENTED(z.depth_info.pipe_config != 0x00000000); + EXIT_NOT_IMPLEMENTED(z.depth_info.bank_width != 0x00000000); + EXIT_NOT_IMPLEMENTED(z.depth_info.bank_height != 0x00000000); + EXIT_NOT_IMPLEMENTED(z.depth_info.macro_tile_aspect != 0x00000000); + EXIT_NOT_IMPLEMENTED(z.depth_info.num_banks != 0x00000000); + EXIT_NOT_IMPLEMENTED(z.htile_surface.linear != 0x00000000); + EXIT_NOT_IMPLEMENTED(z.htile_surface.full_cache != 0x00000000); + EXIT_NOT_IMPLEMENTED(z.htile_surface.htile_uses_preload_win != 0x00000000); + EXIT_NOT_IMPLEMENTED(z.htile_surface.preload != 0x00000000); + EXIT_NOT_IMPLEMENTED(z.htile_surface.prefetch_width != 0x00000000); + EXIT_NOT_IMPLEMENTED(z.htile_surface.prefetch_height != 0x00000000); + EXIT_NOT_IMPLEMENTED(z.htile_surface.dst_outside_zero_to_one != 0x00000000); + + if (z.depth_view.slice_start != 0x00000000 || z.depth_view.slice_max != 0x00000000) { + static std::atomic log_count {0}; + if (log_count.fetch_add(1, std::memory_order_relaxed) < 16) { + LOGF("DepthTarget: temporary: ignoring PS5 array slice view start=0x%08" PRIx32 + ", max=0x%08" PRIx32 "\n", + z.depth_view.slice_start, z.depth_view.slice_max); + } + } + if (z.depth_view.current_mip_level != 0x00000000) { + static std::atomic log_count {0}; + if (log_count.fetch_add(1, std::memory_order_relaxed) < 16) { + LOGF("DepthTarget: temporary: ignoring PS5 current mip level=0x%08" PRIx32 "\n", + z.depth_view.current_mip_level); + } + } + if (z.depth_view.depth_write_disable || z.depth_view.stencil_write_disable) { + static std::atomic log_count {0}; + if (log_count.fetch_add(1, std::memory_order_relaxed) < 16) { + LOGF("DepthTarget: honoring write disable depth=%s, stencil=%s\n", + z.depth_view.depth_write_disable ? "true" : "false", + z.depth_view.stencil_write_disable ? "true" : "false"); + } + } + EXIT_NOT_IMPLEMENTED(z.z_read_base_addr != z.z_write_base_addr); + EXIT_NOT_IMPLEMENTED(z.stencil_read_base_addr != z.stencil_write_base_addr); + EXIT_NOT_IMPLEMENTED(z.z_write_base_addr == 0); + if (z.stencil_info.format != 0 && z.stencil_write_base_addr == 0) { + LOGF("\t warning: stencil format is set without a stencil base address, continuing " + "without stencil attachment\n"); + } + // EXIT_NOT_IMPLEMENTED(z.pitch_div8_minus1 != 0x000000ff); + // EXIT_NOT_IMPLEMENTED(z.height_div8_minus1 != 0x0000008f); + // EXIT_NOT_IMPLEMENTED(z.slice_div64_minus1 != 0x00008fff); + // EXIT_NOT_IMPLEMENTED(z.htile_data_base_addr == 0); + // EXIT_NOT_IMPLEMENTED(z.width != 0x00000780); + // EXIT_NOT_IMPLEMENTED(z.height != 0x00000438); + + EXIT_NOT_IMPLEMENTED(z.width != 0); + EXIT_NOT_IMPLEMENTED(z.height != 0); + if (z.size.x_max == 0 || z.size.y_max == 0) { + static std::atomic log_count {0}; + if (log_count.fetch_add(1) < 16) { + LOGF("DepthTarget: temporary: inferring PS5 default depth extent x_max=0x%04" PRIx16 + ", y_max=0x%04" PRIx16 "\n", + z.size.x_max, z.size.y_max); + } + } + } +} + +static void ClipPrint(const char* func, const HW::ClipControl& c) { + LOGF("%s\n", func); + + LOGF("\t user_clip_planes = 0x%02" PRIx8 "\n" + "\t user_clip_plane_mode = 0x%02" PRIx8 "\n" + "\t dx_clip_space = %s\n" + "\t vertex_kill_any = %s\n" + "\t min_z_clip_disable = %s\n" + "\t max_z_clip_disable = %s\n" + "\t user_clip_plane_negate_y = %s\n" + "\t clip_disable = %s\n" + "\t user_clip_plane_cull_only = %s\n" + "\t cull_on_clipping_error_disable = %s\n" + "\t linear_attribute_clip_enable = %s\n" + "\t force_viewport_index_from_vs_enable = %s\n", + c.user_clip_planes, c.user_clip_plane_mode, c.dx_clip_space ? "true" : "false", + c.vertex_kill_any ? "true" : "false", c.min_z_clip_disable ? "true" : "false", + c.max_z_clip_disable ? "true" : "false", c.user_clip_plane_negate_y ? "true" : "false", + c.clip_disable ? "true" : "false", c.user_clip_plane_cull_only ? "true" : "false", + c.cull_on_clipping_error_disable ? "true" : "false", + c.linear_attribute_clip_enable ? "true" : "false", + c.force_viewport_index_from_vs_enable ? "true" : "false"); +} + +static void ClipCheck(const HW::ClipControl& c) { + EXIT_NOT_IMPLEMENTED(c.user_clip_planes != 0 || c.user_clip_plane_mode != 0 || + c.vertex_kill_any || c.min_z_clip_disable || c.max_z_clip_disable || + c.user_clip_plane_negate_y || c.clip_disable || + c.user_clip_plane_cull_only || c.cull_on_clipping_error_disable || + c.linear_attribute_clip_enable || c.force_viewport_index_from_vs_enable); +} + +static void RcPrint(const char* func, const HW::RenderControl& c) { + LOGF("%s\n", func); + + LOGF("\t depth_clear_enable = %s\n" + "\t stencil_clear_enable = %s\n" + "\t resummarize_enable = %s\n" + "\t stencil_compress_disable = %s\n" + "\t depth_compress_disable = %s\n" + "\t copy_centroid = %s\n" + "\t copy_sample = %" PRIu8 "\n", + c.depth_clear_enable ? "true" : "false", c.stencil_clear_enable ? "true" : "false", + c.resummarize_enable ? "true" : "false", c.stencil_compress_disable ? "true" : "false", + c.depth_compress_disable ? "true" : "false", c.copy_centroid ? "true" : "false", + c.copy_sample); +} + +static void RcCheck(const HW::RenderControl& c) { + // EXIT_NOT_IMPLEMENTED(c.depth_clear_enable != false); + // EXIT_NOT_IMPLEMENTED(c.stencil_clear_enable != false); + if ((c.resummarize_enable || c.copy_centroid || c.copy_sample != 0)) { + static std::atomic log_count {0}; + if (log_count.fetch_add(1) < 16) { + LOGF("\t warning: unsupported PS5 render-control metadata state, continuing: " + "resummarize=%s copy_centroid=%s copy_sample=%" PRIu8 "\n", + c.resummarize_enable ? "true" : "false", c.copy_centroid ? "true" : "false", + c.copy_sample); + } + return; + } + EXIT_NOT_IMPLEMENTED(c.resummarize_enable != false); + // EXIT_NOT_IMPLEMENTED(c.stencil_compress_disable != false); + // EXIT_NOT_IMPLEMENTED(c.depth_compress_disable != false); + EXIT_NOT_IMPLEMENTED(c.copy_centroid != false); + EXIT_NOT_IMPLEMENTED(c.copy_sample != 0); +} + +static void McPrint(const char* func, const HW::ModeControl& c) { + LOGF("%s\n", func); + + LOGF("\t cull_front = %s\n" + "\t cull_back = %s\n" + "\t face = %s\n" + "\t poly_mode = %" PRIu8 "\n" + "\t polymode_front_ptype = %" PRIu8 "\n" + "\t polymode_back_ptype = %" PRIu8 "\n" + "\t poly_offset_front_enable = %s\n" + "\t poly_offset_back_enable = %s\n" + "\t vtx_window_offset_enable = %s\n" + "\t provoking_vtx_last = %s\n" + "\t persp_corr_dis = %s\n", + c.cull_front ? "true" : "false", c.cull_back ? "true" : "false", c.face ? "true" : "false", + c.poly_mode, c.polymode_front_ptype, c.polymode_back_ptype, + c.poly_offset_front_enable ? "true" : "false", + c.poly_offset_back_enable ? "true" : "false", + c.vtx_window_offset_enable ? "true" : "false", c.provoking_vtx_last ? "true" : "false", + c.persp_corr_dis ? "true" : "false"); +} + +static void McCheck(const HW::ModeControl& c) { + // EXIT_NOT_IMPLEMENTED(c.cull_front != false); + // EXIT_NOT_IMPLEMENTED(c.cull_back != false); + // EXIT_NOT_IMPLEMENTED(c.face != false); + if (c.poly_mode != 0) { + static bool logged = false; + if (!logged) { + LOGF("\t temporary: PA_SU_SC_MODE_CNTL.POLY_MODE is not fully implemented; continuing " + "with filled polygons\n"); + logged = true; + } + } + EXIT_NOT_IMPLEMENTED(c.polymode_front_ptype != 0 && c.polymode_front_ptype != 2); + EXIT_NOT_IMPLEMENTED(c.polymode_back_ptype != 0 && c.polymode_back_ptype != 2); + if (c.poly_offset_front_enable || c.poly_offset_back_enable) { + static bool logged = false; + if (!logged) { + LOGF("\t temporary: PA_SU_SC_MODE_CNTL.POLY_OFFSET_*_ENABLE is not implemented; " + "continuing without depth bias\n"); + logged = true; + } + } + if (c.vtx_window_offset_enable) { + static bool logged = false; + if (!logged) { + LOGF("\t temporary: PA_SU_SC_MODE_CNTL.VTX_WINDOW_OFFSET_ENABLE is not fully " + "implemented; continuing without vertex window " + "offset adjustment\n"); + logged = true; + } + } + EXIT_NOT_IMPLEMENTED(c.provoking_vtx_last != false); + EXIT_NOT_IMPLEMENTED(c.persp_corr_dis != false); +} + +static void BcPrint(const char* func, const HW::BlendControl& c, const HW::BlendColor& color, + const HW::ColorControl& cc) { + LOGF("%s\n", func); + + LOGF("\t color_srcblend = %" PRIu8 "\n" + "\t color_comb_fcn = %" PRIu8 "\n" + "\t color_destblend = %" PRIu8 "\n" + "\t alpha_srcblend = %" PRIu8 "\n" + "\t alpha_comb_fcn = %" PRIu8 "\n" + "\t alpha_destblend = %" PRIu8 "\n" + "\t separate_alpha_blend = %s\n" + "\t enable = %s\n" + "\t red = %f\n" + "\t green = %f\n" + "\t blue = %f\n" + "\t alpha = %f\n" + "\t cc.mode = %" PRIu8 "\n" + "\t cc.op = %" PRIu8 "\n", + c.color_srcblend, c.color_comb_fcn, c.color_destblend, c.alpha_srcblend, c.alpha_comb_fcn, + c.alpha_destblend, c.separate_alpha_blend ? "true" : "false", c.enable ? "true" : "false", + color.red, color.green, color.blue, color.alpha, cc.mode, cc.op); +} + +static void BcCheck(const HW::BlendControl& /*c*/, const HW::BlendColor& color, + const HW::ColorControl& cc) { + // EXIT_NOT_IMPLEMENTED(c.color_srcblend != 0); + // EXIT_NOT_IMPLEMENTED(c.color_comb_fcn != 0); + // EXIT_NOT_IMPLEMENTED(c.color_destblend != 0); + // EXIT_NOT_IMPLEMENTED(c.alpha_srcblend != 0); + // EXIT_NOT_IMPLEMENTED(c.alpha_comb_fcn != 0); + // EXIT_NOT_IMPLEMENTED(c.alpha_destblend != 0); + // EXIT_NOT_IMPLEMENTED(c.separate_alpha_blend != false); + // EXIT_NOT_IMPLEMENTED(c.enable != false); + if (color.red != 0.0f || color.green != 0.0f || color.blue != 0.0f || color.alpha != 0.0f) { + static bool logged = false; + if (!logged) { + LOGF("BlendControl: temporary: accepting nonzero blend constants (%f, %f, %f, %f)\n", + color.red, color.green, color.blue, color.alpha); + logged = true; + } + } + if (cc.mode != 1 && cc.mode != 0 && cc.mode != 2 && cc.mode != 3 && cc.mode != 5 && + cc.mode != 6) { + static bool logged = false; + if (!logged) { + LOGF("BlendControl: temporary: accepting unsupported color-control mode %" PRIu8 "\n", + cc.mode); + logged = true; + } + } + if (cc.op != 0xCC) { + static bool logged = false; + if (!logged) { + LOGF("BlendControl: temporary: accepting unsupported raster op 0x%02" PRIx8 "\n", + cc.op); + logged = true; + } + } +} + +static void DPrint(const char* func, const HW::DepthControl& c, const HW::StencilControl& s, + const HW::StencilMask& sm) { + LOGF("%s\n", func); + + LOGF("\t stencil_enable = %s\n" + "\t z_enable = %s\n" + "\t z_write_enable = %s\n" + "\t depth_bounds_enable = %s\n" + "\t zfunc = %" PRIu8 "\n" + "\t backface_enable = %s\n" + "\t stencilfunc = %" PRIu8 "\n" + "\t stencilfunc_bf = %" PRIu8 "\n" + "\t color_writes_on_depth_fail_enable = %s\n" + "\t color_writes_on_depth_pass_disable = %s\n" + "\t stencil_fail = %" PRIu8 "\n" + "\t stencil_zpass = %" PRIu8 "\n" + "\t stencil_zfail = %" PRIu8 "\n" + "\t stencil_fail_bf = %" PRIu8 "\n" + "\t stencil_zpass_bf = %" PRIu8 "\n" + "\t stencil_zfail_bf = %" PRIu8 "\n" + "\t stencil_testval = %" PRIu8 "\n" + "\t stencil_mask = %" PRIu8 "\n" + "\t stencil_writemask = %" PRIu8 "\n" + "\t stencil_opval = %" PRIu8 "\n" + "\t stencil_testval_bf = %" PRIu8 "\n" + "\t stencil_mask_bf = %" PRIu8 "\n" + "\t stencil_writemask_bf = %" PRIu8 "\n" + "\t stencil_opval_bf = %" PRIu8 "\n", + c.stencil_enable ? "true" : "false", c.z_enable ? "true" : "false", + c.z_write_enable ? "true" : "false", c.depth_bounds_enable ? "true" : "false", c.zfunc, + c.backface_enable ? "true" : "false", c.stencilfunc, c.stencilfunc_bf, + c.color_writes_on_depth_fail_enable ? "true" : "false", + c.color_writes_on_depth_pass_disable ? "true" : "false", s.stencil_fail, s.stencil_zpass, + s.stencil_zfail, s.stencil_fail_bf, s.stencil_zpass_bf, s.stencil_zfail_bf, + sm.stencil_testval, sm.stencil_mask, sm.stencil_writemask, sm.stencil_opval, + sm.stencil_testval_bf, sm.stencil_mask_bf, sm.stencil_writemask_bf, sm.stencil_opval_bf); +} + +static void DCheck(const HW::DepthControl& c, const HW::StencilControl& s, + const HW::StencilMask& /*sm*/) { + // EXIT_NOT_IMPLEMENTED(c.stencil_enable != false); + // EXIT_NOT_IMPLEMENTED(c.z_enable != false); + // EXIT_NOT_IMPLEMENTED(c.z_write_enable != false); + // EXIT_NOT_IMPLEMENTED(c.zfunc != 0); + // Back-face stencil state is handled separately when enabled. + // EXIT_NOT_IMPLEMENTED(c.stencilfunc != 0); + // EXIT_NOT_IMPLEMENTED(c.stencilfunc_bf != 0); + EXIT_NOT_IMPLEMENTED(c.color_writes_on_depth_fail_enable != false); + EXIT_NOT_IMPLEMENTED(c.color_writes_on_depth_pass_disable != false); + // EXIT_NOT_IMPLEMENTED(s.stencil_fail != 0); + // EXIT_NOT_IMPLEMENTED(s.stencil_zpass != 0); + // EXIT_NOT_IMPLEMENTED(s.stencil_zfail != 0); + // Back-face stencil ops may legitimately differ from the front-face ops. + // EXIT_NOT_IMPLEMENTED(sm.stencil_testval != 0); + // EXIT_NOT_IMPLEMENTED(sm.stencil_mask != 0); + // EXIT_NOT_IMPLEMENTED(sm.stencil_writemask != 0); + // EXIT_NOT_IMPLEMENTED(sm.stencil_opval != 0); + // EXIT_NOT_IMPLEMENTED(sm.stencil_testval_bf != 0); + // EXIT_NOT_IMPLEMENTED(sm.stencil_mask_bf != 0); + // EXIT_NOT_IMPLEMENTED(sm.stencil_writemask_bf != 0); + // EXIT_NOT_IMPLEMENTED(sm.stencil_opval_bf != 0); +} + +static void EqaaPrint(const char* func, const HW::EqaaControl& c) { + LOGF("%s\n", func); + + LOGF("\t max_anchor_samples = %" PRIu8 "\n" + "\t ps_iter_samples = %" PRIu8 "\n" + "\t mask_export_num_samples = %" PRIu8 "\n" + "\t alpha_to_mask_num_samples = %" PRIu8 "\n" + "\t high_quality_intersections = %s\n" + "\t incoherent_eqaa_reads = %s\n" + "\t interpolate_comp_z = %s\n" + "\t static_anchor_associations = %s\n", + c.max_anchor_samples, c.ps_iter_samples, c.mask_export_num_samples, + c.alpha_to_mask_num_samples, c.high_quality_intersections ? "true" : "false", + c.incoherent_eqaa_reads ? "true" : "false", c.interpolate_comp_z ? "true" : "false", + c.static_anchor_associations ? "true" : "false"); +} + +static void EqaaCheck(const HW::EqaaControl& c) { + if (c.max_anchor_samples != 0 || c.ps_iter_samples != 0 || c.mask_export_num_samples != 0 || + c.alpha_to_mask_num_samples != 0 || c.high_quality_intersections || + c.incoherent_eqaa_reads || c.interpolate_comp_z || c.static_anchor_associations) { + static std::atomic log_count {0}; + if (log_count.fetch_add(1) < 16) { + LOGF("\t warning: unsupported PS5 EQAA state, rendering with single-sample fallback\n"); + } + } +} + +static void AaPrint(const char* func, const HW::AaSampleControl& c, const HW::AaConfig& cf) { + LOGF("%s\n", func); + + LOGF("\t centroid_priority = %016" PRIx64 "\n", c.centroid_priority); + for (int i = 0; i < 16; i++) { + LOGF("\t locations[%d] = %08" PRIx32 "\n", i, c.locations[i]); + } + LOGF("\t msaa_num_samples = %" PRIu8 "\n" + "\t aa_mask_centroid_dtmn = %s\n" + "\t max_sample_dist = %" PRIu8 "\n" + "\t msaa_exposed_samples = %" PRIu8 "\n", + cf.msaa_num_samples, cf.aa_mask_centroid_dtmn ? "true" : "false", cf.max_sample_dist, + cf.msaa_exposed_samples); +} + +static void AaCheck(const HW::AaSampleControl& c, const HW::AaConfig& cf) { + bool non_default_locations = (c.centroid_priority != 0); + for (uint32_t l: c.locations) { + non_default_locations |= (l != 0); + } + + if (non_default_locations || cf.msaa_num_samples != 0 || cf.aa_mask_centroid_dtmn || + cf.max_sample_dist != 0 || cf.msaa_exposed_samples != 0) { + static std::atomic log_count {0}; + if (log_count.fetch_add(1) < 16) { + LOGF("\t warning: unsupported PS5 AA/MSAA state, rendering with single-sample " + "fallback: samples=%" PRIu8 ", exposed=%" PRIu8 ", max_dist=%" PRIu8 "\n", + cf.msaa_num_samples, cf.msaa_exposed_samples, cf.max_sample_dist); + } + } +} + +void LogDrawPhase(const char* draw_name, const char* phase) { + if (graphics_debug_dump_enabled()) { + static std::atomic log_count {0}; + if (log_count.fetch_add(1, std::memory_order_relaxed) < 1024) { + LOGF("DrawPhase: %s %s\n", draw_name, phase); + } + } +} + +void LogPipelineTrace(const char* phase, uint32_t vs_hash0, uint32_t vs_crc32, uint32_t ps_hash0, + uint32_t ps_crc32) { + if (graphics_debug_dump_enabled()) { + LOGF("PipelineTrace: %s VS=0x%08" PRIx32 "/0x%08" PRIx32 " PS=0x%08" PRIx32 "/0x%08" PRIx32 + "\n", + phase, vs_hash0, vs_crc32, ps_hash0, ps_crc32); + } +} + +static void VpPrint(const char* func, const HW::ScreenViewport& vp, + const HW::ScanModeControl& smc) { + LOGF("%s\n", func); + + LOGF("\t msaa_enable = %s\n" + "\t vport_scissor_enable = %s\n" + "\t line_stipple_enable = %s\n" + "\t vp[0].zmin = %f\n" + "\t vp[0].zmax = %f\n" + "\t vp[0].xscale = %f\n" + "\t vp[0].xoffset = %f\n" + "\t vp[0].yscale = %f\n" + "\t vp[0].yoffset = %f\n" + "\t vp[0].zscale = %f\n" + "\t vp[0].zoffset = %f\n" + "\t vp[0].viewport_scissor_left = %d\n" + "\t vp[0].viewport_scissor_top = %d\n" + "\t vp[0].viewport_scissor_right = %d\n" + "\t vp[0].viewport_scissor_bottom = %d\n" + "\t transform_control = 0x%08" PRIx32 "\n" + "\t screen_scissor_left = %d\n" + "\t screen_scissor_top = %d\n" + "\t screen_scissor_right = %d\n" + "\t screen_scissor_bottom = %d\n" + "\t window_scissor_left = %d\n" + "\t window_scissor_top = %d\n" + "\t window_scissor_right = %d\n" + "\t window_scissor_bottom = %d\n" + "\t generic_scissor_left = %d\n" + "\t generic_scissor_top = %d\n" + "\t generic_scissor_right = %d\n" + "\t generic_scissor_bottom = %d\n" + "\t window_offset_x = %d\n" + "\t window_offset_y = %d\n" + "\t hw_offset_x = %u\n" + "\t hw_offset_y = %u\n" + "\t guard_band_horz_clip = %f\n" + "\t guard_band_vert_clip = %f\n" + "\t guard_band_horz_discard = %f\n" + "\t guard_band_vert_discard = %f\n" + "\t clip_rect_rule = 0x%04" PRIx16 "\n" + "\t clip_rect_0 = (%d,%d)-(%d,%d), window_offset = %s\n" + "\t window_scissor_window_offset_enable = %s\n" + "\t generic_scissor_window_offset_enable = %s\n", + smc.msaa_enable ? "true" : "false", smc.vport_scissor_enable ? "true" : "false", + smc.line_stipple_enable ? "true" : "false", vp.viewports[0].zmin, vp.viewports[0].zmax, + vp.viewports[0].xscale, vp.viewports[0].xoffset, vp.viewports[0].yscale, + vp.viewports[0].yoffset, vp.viewports[0].zscale, vp.viewports[0].zoffset, + vp.viewports[0].viewport_scissor_left, vp.viewports[0].viewport_scissor_top, + vp.viewports[0].viewport_scissor_right, vp.viewports[0].viewport_scissor_bottom, + vp.transform_control, vp.screen_scissor_left, vp.screen_scissor_top, + vp.screen_scissor_right, vp.screen_scissor_bottom, vp.window_scissor_left, + vp.window_scissor_top, vp.window_scissor_right, vp.window_scissor_bottom, + vp.generic_scissor_left, vp.generic_scissor_top, vp.generic_scissor_right, + vp.generic_scissor_bottom, vp.window_offset_x, vp.window_offset_y, vp.hw_offset_x, + vp.hw_offset_y, vp.guard_band_horz_clip, vp.guard_band_vert_clip, + vp.guard_band_horz_discard, vp.guard_band_vert_discard, vp.clip_rect_rule, + vp.clip_rect_left[0], vp.clip_rect_top[0], vp.clip_rect_right[0], vp.clip_rect_bottom[0], + vp.clip_rect_window_offset_enable[0] ? "true" : "false", + vp.window_scissor_window_offset_enable ? "true" : "false", + vp.generic_scissor_window_offset_enable ? "true" : "false"); + LOGF("\t viewports[0].viewport_scissor_window_offset_enable = %s\n", + vp.viewports[0].viewport_scissor_window_offset_enable ? "true" : "false"); +} + +static void VpCheck(const HW::ScreenViewport& vp, const HW::ScanModeControl& smc) { + + if (smc.msaa_enable) { + + static std::atomic log_count {0}; + if (log_count.fetch_add(1) < 16) { + LOGF("\t warning: unsupported PS5 MSAA raster state, rendering with single-sample " + "fallback\n"); + } + } + // EXIT_NOT_IMPLEMENTED(smc.vport_scissor_enable); + EXIT_NOT_IMPLEMENTED(smc.line_stipple_enable); + + if (vp.viewports[0].zmin > 0.000000 || vp.viewports[0].zmax != 1.000000) { + static bool logged = false; + if (!logged) { + LOGF("\t warning: non-default viewport depth clamp zmin = %f, zmax = %f; using " + "viewport scale/offset for Vulkan depth range\n", + vp.viewports[0].zmin, vp.viewports[0].zmax); + logged = true; + } + } + // EXIT_NOT_IMPLEMENTED(vp.viewports[0].xscale != 960.000000); + // EXIT_NOT_IMPLEMENTED(vp.viewports[0].xoffset != 960.000000); + // EXIT_NOT_IMPLEMENTED(vp.viewports[0].yscale != -540.000000); + // EXIT_NOT_IMPLEMENTED(vp.viewports[0].yoffset != 540.000000); + // EXIT_NOT_IMPLEMENTED(vp.viewports[0].zscale != 0.500000); + // EXIT_NOT_IMPLEMENTED(vp.viewports[0].zoffset != 0.500000); + if (vp.transform_control != 1087) { + static std::atomic log_count {0}; + if (log_count.fetch_add(1, std::memory_order_relaxed) < 16) { + LOGF("\t warning: non-default viewport transform control = 0x%08" PRIx32 + ", applying enabled scale/offset bits\n", + vp.transform_control); + } + } + // EXIT_NOT_IMPLEMENTED(vp.hw_offset_x != 60); + // EXIT_NOT_IMPLEMENTED(vp.hw_offset_y != 32); + // EXIT_NOT_IMPLEMENTED(fabsf(vp.guard_band_horz_clip - 33.133327f) > 0.001f); + // EXIT_NOT_IMPLEMENTED(fabsf(vp.guard_band_vert_clip - 59.629623f) > 0.001f); + + if (vp.guard_band_horz_discard != 0.0f || vp.guard_band_vert_discard != 0.0f) { + static std::atomic log_count {0}; + if (log_count.fetch_add(1) < 16) { + LOGF("\t warning: unsupported PS5 guard band discard = %f, %f, continuing\n", + vp.guard_band_horz_discard, vp.guard_band_vert_discard); + } + } + + // EXIT_NOT_IMPLEMENTED(vp.viewports[0].viewport_scissor_left != 0); + // EXIT_NOT_IMPLEMENTED(vp.viewports[0].viewport_scissor_top != 0); + // EXIT_NOT_IMPLEMENTED(vp.viewports[0].viewport_scissor_right != 0); + // EXIT_NOT_IMPLEMENTED(vp.viewports[0].viewport_scissor_bottom != 0); + // EXIT_NOT_IMPLEMENTED(viewport_scissor && + // vp.viewports[0].viewport_scissor_window_offset_enable != true); +} + +static bool ScissorRectValid(const ScissorRect& r) { + return r.right > r.left && r.bottom > r.top; +} + +static bool ScissorRectSet(const ScissorRect& r) { + return r.left != 0 || r.top != 0 || r.right != 0 || r.bottom != 0; +} + +static ScissorRect ScissorRectOffset(ScissorRect r, int x, int y) { + r.left += x; + r.right += x; + r.top += y; + r.bottom += y; + return r; +} + +static ScissorRect ScissorRectIntersect(const ScissorRect& a, const ScissorRect& b) { + return {a.left > b.left ? a.left : b.left, a.top > b.top ? a.top : b.top, + a.right < b.right ? a.right : b.right, a.bottom < b.bottom ? a.bottom : b.bottom}; +} + +static ScissorRect ScissorRectClamp(ScissorRect r, uint32_t width, uint32_t height) { + int max_right = static_cast(width); + int max_bottom = static_cast(height); + + r.left = (r.left < 0 ? 0 : (r.left > max_right ? max_right : r.left)); + r.right = (r.right < 0 ? 0 : (r.right > max_right ? max_right : r.right)); + r.top = (r.top < 0 ? 0 : (r.top > max_bottom ? max_bottom : r.top)); + r.bottom = (r.bottom < 0 ? 0 : (r.bottom > max_bottom ? max_bottom : r.bottom)); + + if (!ScissorRectValid(r)) { + r.right = r.left; + r.bottom = r.top; + } + + return r; +} + +static bool ScissorClipRuleToIntersectionMask(uint16_t rule, uint8_t* mask) { + EXIT_IF(mask == nullptr); + + for (uint32_t candidate = 0; candidate < 16; candidate++) { + uint16_t expected = 0; + for (uint32_t combination = 0; combination < 16; combination++) { + if ((combination & candidate) == candidate) { + expected |= static_cast(1u << combination); + } + } + + if (expected == rule) { + *mask = static_cast(candidate); + return true; + } + } + + return false; +} + +ScissorRect calc_final_scissor(const HW::ScreenViewport& vp, const HW::ScanModeControl& smc, + VkExtent2D extent) { + ScissorRect screen {vp.screen_scissor_left, vp.screen_scissor_top, vp.screen_scissor_right, + vp.screen_scissor_bottom}; + ScissorRect final = screen; + + if (!ScissorRectSet(screen)) { + final = {0, 0, static_cast(extent.width), static_cast(extent.height)}; + + auto log_id = g_scissor_default_log_count.fetch_add(1); + if (log_id < 32) { + LOGF("temporary: default unset screen scissor to framebuffer extent %ux%u\n", + extent.width, extent.height); + } + } + + ScissorRect window {vp.window_scissor_left, vp.window_scissor_top, vp.window_scissor_right, + vp.window_scissor_bottom}; + if (ScissorRectSet(window)) { + if (vp.window_scissor_window_offset_enable) { + window = ScissorRectOffset(window, vp.window_offset_x, vp.window_offset_y); + } + final = ScissorRectIntersect(final, window); + } + + ScissorRect generic {vp.generic_scissor_left, vp.generic_scissor_top, vp.generic_scissor_right, + vp.generic_scissor_bottom}; + if (ScissorRectSet(generic)) { + if (vp.generic_scissor_window_offset_enable) { + generic = ScissorRectOffset(generic, vp.window_offset_x, vp.window_offset_y); + } + final = ScissorRectIntersect(final, generic); + } + + const auto& viewport = vp.viewports[0]; + ScissorRect viewport_scissor {viewport.viewport_scissor_left, viewport.viewport_scissor_top, + viewport.viewport_scissor_right, + viewport.viewport_scissor_bottom}; + if (smc.vport_scissor_enable && ScissorRectSet(viewport_scissor)) { + if (viewport.viewport_scissor_window_offset_enable) { + viewport_scissor = + ScissorRectOffset(viewport_scissor, vp.window_offset_x, vp.window_offset_y); + } + final = ScissorRectIntersect(final, viewport_scissor); + } + + if (vp.clip_rect_rule == 0) { + final = {0, 0, 0, 0}; + } else if (vp.clip_rect_rule != 0xffffu) { + uint8_t clip_rect_mask = 0; + if (ScissorClipRuleToIntersectionMask(vp.clip_rect_rule, &clip_rect_mask)) { + for (uint32_t i = 0; i < 4; i++) { + if ((clip_rect_mask & (1u << i)) == 0) { + continue; + } + + ScissorRect clip {vp.clip_rect_left[i], vp.clip_rect_top[i], vp.clip_rect_right[i], + vp.clip_rect_bottom[i]}; + if (vp.clip_rect_window_offset_enable[i]) { + clip = ScissorRectOffset(clip, vp.window_offset_x, vp.window_offset_y); + } + final = ScissorRectIntersect(final, clip); + } + } else { + static std::atomic log_count {0}; + if (log_count.fetch_add(1, std::memory_order_relaxed) < 32) { + LOGF("unsupported clip-rect rule 0x%04" PRIx16 ", leaving scissor unchanged\n", + vp.clip_rect_rule); + } + } + } + + return ScissorRectClamp(final, extent.width, extent.height); +} + +void hw_check(const HW::Context& hw) { + const auto rt_slot = render_target_first_bound_slot(hw); + const auto& rt = hw.GetRenderTarget(rt_slot); + const auto& bc = hw.GetBlendControl(rt_slot); + const auto& bclr = hw.GetBlendColor(); + const auto& vp = hw.GetScreenViewport(); + const auto& z = hw.GetDepthRenderTarget(); + const auto& c = hw.GetClipControl(); + const auto& rc = hw.GetRenderControl(); + const auto& d = hw.GetDepthControl(); + const auto& s = hw.GetStencilControl(); + const auto& sm = hw.GetStencilMask(); + const auto& mc = hw.GetModeControl(); + const auto& eqaa = hw.GetEqaaControl(); + const auto& cc = hw.GetColorControl(); + const auto& smc = hw.GetScanModeControl(); + const auto& aa = hw.GetAaSampleControl(); + const auto& ac = hw.GetAaConfig(); + + auto log_phase = [](const char* phase) { + if (graphics_debug_dump_enabled()) { + static std::atomic log_count {0}; + if (log_count.fetch_add(1, std::memory_order_relaxed) < 512) { + LOGF("HwCheckPhase: %s\n", phase); + } + } + }; + + log_phase("rt"); + RtCheck(rt); + log_phase("vp"); + VpCheck(vp, smc); + log_phase("z"); + ZCheck(z); + log_phase("clip"); + ClipCheck(c); + log_phase("rc"); + RcCheck(rc); + log_phase("depth"); + DCheck(d, s, sm); + log_phase("mode"); + McCheck(mc); + log_phase("blend"); + BcCheck(bc, bclr, cc); + log_phase("eqaa"); + EqaaCheck(eqaa); + log_phase("aa"); + AaCheck(aa, ac); + log_phase("done"); + + if (RenderTargetMaskHasBoundMrt(hw)) { + LOGF("MRT render target mask: 0x%08" PRIx32 "\n", hw.GetRenderTargetMask()); + for (uint32_t i = 0; i < 8; i++) { + const auto& mrt = hw.GetRenderTarget(i); + LOGF("\t RT%u addr=0x%010" PRIx64 " mask=0x%x fmt=0x%08" PRIx32 + " width=%u height=%u tile=%u\n", + i, mrt.base.addr, (hw.GetRenderTargetMask() >> (i * 4u)) & 0x0fu, mrt.info.format, + mrt.attrib2.width + 1, mrt.attrib2.height + 1, mrt.attrib3.tile_mode); + } + } + if (rc.depth_clear_enable && hw.GetDepthClearValue() != 0.0f && + hw.GetDepthClearValue() != 1.0f) { + static std::atomic log_count {0}; + if (log_count.fetch_add(1) < 16) { + LOGF("\t temporary: accepting non-default depth clear value %f\n", + hw.GetDepthClearValue()); + } + } + // EXIT_NOT_IMPLEMENTED(hw.GetStencilClearValue() != 0); +} + +void hw_print(const HW::Context& hw) { + const auto rt_slot = render_target_first_bound_slot(hw); + const auto& rt = hw.GetRenderTarget(rt_slot); + const auto& bc = hw.GetBlendControl(rt_slot); + const auto& bclr = hw.GetBlendColor(); + const auto& vp = hw.GetScreenViewport(); + const auto& z = hw.GetDepthRenderTarget(); + const auto& c = hw.GetClipControl(); + const auto& rc = hw.GetRenderControl(); + const auto& d = hw.GetDepthControl(); + const auto& s = hw.GetStencilControl(); + const auto& sm = hw.GetStencilMask(); + const auto& mc = hw.GetModeControl(); + const auto& eqaa = hw.GetEqaaControl(); + const auto& cc = hw.GetColorControl(); + const auto& smc = hw.GetScanModeControl(); + const auto& aa = hw.GetAaSampleControl(); + const auto& ac = hw.GetAaConfig(); + + if (graphics_debug_dump_enabled()) { + LOGF("Context\n" + "\t GetRenderTargetMask() = 0x%08" PRIx32 "\n" + "\t GetDepthClearValue() = %f\n" + "\t GetStencilClearValue() = %" PRIu8 "\n" + "\t GetLineWidth() = %f\n", + hw.GetRenderTargetMask(), hw.GetDepthClearValue(), hw.GetStencilClearValue(), + hw.GetLineWidth()); + + LOGF("%s", Common::Concat(rt_print("RenderTraget:", rt), "").c_str()); + + ZPrint("DepthRenderTraget:", z); + VpPrint("ScreenViewport:", vp, smc); + ClipPrint("ClipControl:", c); + RcPrint("RenderControl:", rc); + DPrint("DepthStencilControlMask:", d, s, sm); + McPrint("ModeControl:", mc); + BcPrint("BlendColorControl:", bc, bclr, cc); + EqaaPrint("EqaaControl:", eqaa); + AaPrint("AaSampleControl:", aa, ac); + } +} + +} // namespace Libs::Graphics diff --git a/src/graphics/host_gpu/renderer/depthRenderTarget.cpp b/src/graphics/host_gpu/renderer/depthRenderTarget.cpp new file mode 100644 index 000000000..469ed89cd --- /dev/null +++ b/src/graphics/host_gpu/renderer/depthRenderTarget.cpp @@ -0,0 +1,357 @@ +#include "common/assert.h" +#include "common/common.h" +#include "common/logging/log.h" +#include "common/profiler.h" +#include "common/stringUtils.h" +#include "common/threads.h" +#include "graphics/guest_gpu/gpu_defs.h" +#include "graphics/guest_gpu/hardwareContext.h" +#include "graphics/guest_gpu/tile.h" +#include "graphics/host_gpu/graphicContext.h" +#include "graphics/host_gpu/objects/textureCommon.h" +#include "graphics/host_gpu/renderer/descriptorCache.h" +#include "graphics/host_gpu/renderer/framebufferCache.h" +#include "graphics/host_gpu/renderer/render.h" +#include "graphics/host_gpu/renderer/renderContext.h" +#include "graphics/host_gpu/renderer/renderState.h" +#include "graphics/host_gpu/utils.h" +#include "graphics/presentation/displayBuffer.h" + +#include +#include +#include +#include +#include +#include +#include + +namespace Libs::Graphics { + +[[noreturn]] static void DepthFatal(const char* format, ...) { + std::fputs("Depth target fatal: ", stderr); + va_list args; + va_start(args, format); + std::vfprintf(stderr, format, args); + va_end(args); + std::fputc('\n', stderr); + std::fflush(stderr); + EXIT("unsupported render state; details were printed above\n"); +} + +static VkStencilOp ConvertStencilOp(uint8_t value) { + switch (static_cast(value)) { + case Prospero::StencilOp::kKeep: return VK_STENCIL_OP_KEEP; + case Prospero::StencilOp::kZero: return VK_STENCIL_OP_ZERO; + case Prospero::StencilOp::kReplaceTest: + case Prospero::StencilOp::kReplaceOp: return VK_STENCIL_OP_REPLACE; + case Prospero::StencilOp::kAddClamp: return VK_STENCIL_OP_INCREMENT_AND_CLAMP; + case Prospero::StencilOp::kSubClamp: return VK_STENCIL_OP_DECREMENT_AND_CLAMP; + case Prospero::StencilOp::kInvert: return VK_STENCIL_OP_INVERT; + case Prospero::StencilOp::kAddWrap: return VK_STENCIL_OP_INCREMENT_AND_WRAP; + case Prospero::StencilOp::kSubWrap: return VK_STENCIL_OP_DECREMENT_AND_WRAP; + default: DepthFatal("unsupported stencil operation"); + } +} + +static bool UsesStencilOpValue(uint8_t fail, uint8_t pass, uint8_t depth_fail) { + return fail == Prospero::GpuEnumValue(Prospero::StencilOp::kReplaceOp) || + pass == Prospero::GpuEnumValue(Prospero::StencilOp::kReplaceOp) || + depth_fail == Prospero::GpuEnumValue(Prospero::StencilOp::kReplaceOp); +} + +// NOLINTNEXTLINE(readability-function-cognitive-complexity) +void ResolveRenderDepthTarget(uint64_t submit_id, CommandBuffer* buffer, const HW::Context& hw, + RenderDepthInfo* r) { + KYTY_PROFILER_FUNCTION(); + EXIT_IF(r == nullptr); + (void)submit_id; + (void)buffer; + const auto& z = hw.GetDepthRenderTarget(); + const auto& rc = hw.GetRenderControl(); + const auto& dc = hw.GetDepthControl(); + const auto& sc = hw.GetStencilControl(); + const auto& sm = hw.GetStencilMask(); + const bool depth_active = + dc.z_enable || dc.z_write_enable || dc.depth_bounds_enable || rc.depth_clear_enable; + const bool stencil_active = dc.stencil_enable || rc.stencil_clear_enable; + if (!depth_active && !stencil_active) { + return; + } + const bool attachment_unbound = + z.z_info.format == Prospero::GpuEnumValue(Prospero::DepthFormat::kInvalid) && + z.stencil_info.format == Prospero::GpuEnumValue(Prospero::StencilFormat::kInvalid) && + z.z_info.tile_mode_index == 0 && z.z_info.num_samples == 0 && + z.z_info.zrange_precision <= 1 && !z.z_info.expclear_enabled && + !z.z_info.embedded_sample_locations && !z.z_info.partially_resident && + z.z_info.num_mip_levels == 0 && z.z_info.plane_compression == 0 && + z.stencil_info.tile_mode_index == 0 && z.stencil_info.tile_split == 0 && + !z.stencil_info.expclear_enabled && !z.stencil_info.texture_compatible_stencil && + !z.stencil_info.partially_resident && z.depth_view.slice_start == 0 && + z.depth_view.slice_max == 0 && z.depth_view.current_mip_level == 0 && + !z.depth_view.depth_write_disable && !z.depth_view.stencil_write_disable && + z.depth_info.addr5_swizzle_mask == 0 && z.depth_info.array_mode == 0 && + z.depth_info.pipe_config == 0 && z.depth_info.bank_width == 0 && + z.depth_info.bank_height == 0 && z.depth_info.macro_tile_aspect == 0 && + z.depth_info.num_banks == 0 && z.htile_surface.linear == 0 && + z.htile_surface.full_cache == 0 && z.htile_surface.htile_uses_preload_win == 0 && + z.htile_surface.preload == 0 && z.htile_surface.prefetch_width == 0 && + z.htile_surface.prefetch_height == 0 && z.htile_surface.dst_outside_zero_to_one == 0 && + z.z_read_base_addr == 0 && z.z_write_base_addr == 0 && z.stencil_read_base_addr == 0 && + z.stencil_write_base_addr == 0 && z.htile_data_base_addr == 0 && + !z.z_info.tile_surface_enable && !z.size.valid && !z.width_height_valid && + !z.pitch_height_valid && z.size.x_max == 0 && z.size.y_max == 0 && + z.pitch_div8_minus1 == 0 && z.height_div8_minus1 == 0 && z.slice_div64_minus1 == 0 && + z.width == 0 && z.height == 0; + if (attachment_unbound) { + static std::atomic_bool logged = false; + if (!logged.exchange(true, std::memory_order_relaxed)) { + LOGF("DepthTarget: ignoring enabled depth/stencil state without a bound attachment\n"); + } + return; + } + const bool has_stencil = + z.stencil_info.format != Prospero::GpuEnumValue(Prospero::StencilFormat::kInvalid); + const bool has_htile = z.z_info.tile_surface_enable; + const bool msaa_compat = depth_msaa_single_sample_compatible(z.z_info.num_samples); + const bool htile_stencil_compat = depth_htile_stencil_acceleration_compatible( + has_stencil, has_htile, z.stencil_info.tile_stencil_disable); + // Prospero defines the compression-disable bits as tile writeback policy. Vulkan attachments + // expose the same logical depth/stencil values regardless of the driver's backing compression. + if ((stencil_active && !has_stencil) || rc.resummarize_enable || rc.copy_centroid || + rc.copy_sample != 0 || z.z_info.expclear_enabled || z.stencil_info.expclear_enabled || + z.z_info.embedded_sample_locations || z.z_info.partially_resident || + z.stencil_info.partially_resident || z.z_info.plane_compression != 0 || + (z.z_info.num_samples != 0 && !msaa_compat) || z.z_info.num_mip_levels != 0 || + z.z_info.tile_mode_index != 0 || z.z_info.zrange_precision > 1 || + z.depth_view.current_mip_level != 0 || z.depth_view.slice_start != 0 || + z.depth_view.slice_max != 0 || z.depth_info.addr5_swizzle_mask != 0 || + z.depth_info.array_mode != 0 || z.depth_info.pipe_config != 0 || + z.depth_info.bank_width != 0 || z.depth_info.bank_height != 0 || + z.depth_info.macro_tile_aspect != 0 || z.depth_info.num_banks != 0 || + z.htile_surface.linear != 0 || z.htile_surface.full_cache != 0 || + z.htile_surface.htile_uses_preload_win != 0 || z.htile_surface.preload != 0 || + z.htile_surface.prefetch_width != 0 || z.htile_surface.prefetch_height != 0 || + z.htile_surface.dst_outside_zero_to_one != 0 || z.z_read_base_addr == 0 || + z.z_write_base_addr != z.z_read_base_addr || (z.z_read_base_addr & 0xffffu) != 0 || + dc.zfunc > static_cast(VK_COMPARE_OP_ALWAYS)) { + DepthFatal("unsupported depth register state"); + } + if (msaa_compat) { + static std::atomic logged_fragments = 0; + const uint32_t bit = 1u << z.z_info.num_samples; + if ((logged_fragments.fetch_or(bit, std::memory_order_relaxed) & bit) == 0) { + LOGF("DepthTarget: compatibility: rendering PS5 %ux depth fragments as single-sample\n", + bit); + } + } + if (has_stencil) { + // Prospero defines Hi-Stencil as HTile-backed acceleration of the logical stencil plane. + // Keep the plane native in Vulkan while tracking HTile separately. + if (z.stencil_info.format != Prospero::GpuEnumValue(Prospero::StencilFormat::k8UInt) || + z.stencil_info.tile_mode_index != 0 || z.stencil_info.tile_split != 0 || + !htile_stencil_compat || z.stencil_info.texture_compatible_stencil || + z.stencil_read_base_addr == 0 || + z.stencil_write_base_addr != z.stencil_read_base_addr || + (z.stencil_read_base_addr & 0xffffu) != 0 || z.depth_view.stencil_write_disable) { + DepthFatal("unsupported stencil attachment state"); + } + if (!z.stencil_info.tile_stencil_disable) { + static std::atomic_bool logged = false; + if (!logged.load(std::memory_order_relaxed) && + !logged.exchange(true, std::memory_order_relaxed)) { + LOGF("DepthTarget: compatibility: using native stencil with PS5 HTILE " + "acceleration\n"); + } + } + } else if (z.stencil_read_base_addr != 0 || z.stencil_write_base_addr != 0 || + z.stencil_info.tile_mode_index != 0 || z.stencil_info.tile_split != 0 || + !htile_stencil_compat || z.stencil_info.texture_compatible_stencil) { + DepthFatal("stencil state without an active stencil attachment"); + } + if (has_htile) { + if (z.htile_data_base_addr == 0 || (z.htile_data_base_addr & 0x7fffu) != 0) { + DepthFatal("invalid HTile metadata address"); + } + } else if (z.htile_data_base_addr != 0) { + DepthFatal("HTile address without an enabled tile surface"); + } + const bool size_xy_valid = z.size.valid && (z.size.x_max != 0 || z.size.y_max != 0); + const bool wh_valid = z.width_height_valid && z.width != 0 && z.height != 0; + if (!size_xy_valid && !wh_valid) { + DepthFatal("missing depth extent"); + } + const uint32_t width = size_xy_valid ? static_cast(z.size.x_max) + 1u : z.width; + const uint32_t height = size_xy_valid ? static_cast(z.size.y_max) + 1u : z.height; + if (width > 16384 || height > 16384 || + (size_xy_valid && wh_valid && (width != z.width || height != z.height)) || + (!z.pitch_height_valid && + (z.pitch_div8_minus1 != 0 || z.height_div8_minus1 != 0 || z.slice_div64_minus1 != 0))) { + DepthFatal("inconsistent depth extent or encoded layout"); + } + uint32_t guest_format = 0; + uint32_t bytes = 0; + switch (static_cast(z.z_info.format)) { + case Prospero::DepthFormat::kZ16: + if (has_stencil) { + DepthFatal("Z16 plus stencil is unsupported"); + } + r->format = VK_FORMAT_D16_UNORM; + guest_format = Prospero::GpuEnumValue(Prospero::BufferFormat::k16UNorm); + bytes = 2; + break; + case Prospero::DepthFormat::kZ32F: + r->format = has_stencil ? VK_FORMAT_D32_SFLOAT_S8_UINT : VK_FORMAT_D32_SFLOAT; + guest_format = Prospero::GpuEnumValue(Prospero::BufferFormat::k32Float); + bytes = 4; + break; + default: DepthFatal("unsupported depth format"); + } + const auto pitch = TileGetTexturePitch(guest_format, width, 1, + Prospero::GpuEnumValue(Prospero::TileMode::kDepth)); + if (z.pitch_height_valid && ((static_cast(z.pitch_div8_minus1) + 1u) * 8u != pitch || + (static_cast(z.height_div8_minus1) + 1u) * 8u != + ((static_cast(height) + 7u) & ~7ull))) { + DepthFatal("encoded depth pitch or height mismatch"); + } + const uint32_t block_width = bytes == 2 ? 256u : 128u; + const uint64_t padded_width = + (static_cast(pitch) + block_width - 1u) & ~(block_width - 1u); + const uint64_t padded_height = + (static_cast(height) + 127u) & ~static_cast(127u); + if (padded_width > UINT64_MAX / padded_height || + padded_width * padded_height > UINT64_MAX / bytes) { + DepthFatal("depth footprint overflow"); + } + const uint64_t expected_size = padded_width * padded_height * bytes; + TileSizeAlign depth_size {}; + TileSizeAlign stencil_size {}; + TileSizeAlign htile_size {}; + if (has_stencil || has_htile) { + if (!TileGetDepthSize(width, height, 0, z.z_info.format, z.stencil_info.format, has_htile, + &stencil_size, &htile_size, &depth_size) || + depth_size.align != 65536 || depth_size.size != expected_size || + (has_stencil != (stencil_size.align == 65536 && stencil_size.size != 0)) || + (has_htile != (htile_size.align == 32768 && htile_size.size != 0))) { + DepthFatal("unsupported depth/stencil/HTile footprint"); + } + if (has_htile) { + g_render_ctx->GetTextureCache()->RegisterMeta(z.htile_data_base_addr, htile_size.size); + } + } else { + TileGetTextureTotalSize(guest_format, width, height, 1, pitch, 1, + Prospero::GpuEnumValue(Prospero::TileMode::kDepth), false, + &depth_size); + } + if (expected_size == 0 || expected_size > UINT32_MAX || depth_size.align != 65536 || + depth_size.size != expected_size || + (z.pitch_height_valid && + (static_cast(z.slice_div64_minus1) + 1u) * 64u != expected_size)) { + DepthFatal("depth footprint mismatch: extent=%ux%u pitch=%u expected=0x%016" PRIx64 + " calculated=0x%016" PRIx64 "/0x%016" PRIx64 + " encoded_valid=%u encoded=0x%016" PRIx64, + width, height, pitch, expected_size, depth_size.size, depth_size.align, + z.pitch_height_valid ? 1u : 0u, + (static_cast(z.slice_div64_minus1) + 1u) * 64u); + } + r->htile = has_htile; + r->width = width; + r->height = height; + r->depth_buffer_size = depth_size.size; + r->depth_buffer_vaddr = z.z_read_base_addr; + r->stencil_buffer_size = has_stencil ? stencil_size.size : 0; + r->stencil_buffer_vaddr = has_stencil ? z.stencil_read_base_addr : 0; + r->htile_buffer_size = has_htile ? htile_size.size : 0; + r->htile_buffer_vaddr = has_htile ? z.htile_data_base_addr : 0; + auto* cache = g_render_ctx->GetTextureCache(); + if (has_htile && rc.depth_clear_enable && !cache->ClearMeta(z.htile_data_base_addr)) { + DepthFatal("failed to acquire HTile metadata for a depth clear"); + } + const bool meta_clear = + has_htile && cache->IsMetaCleared(z.htile_data_base_addr, z.depth_view.slice_start); + r->depth_clear_enable = rc.depth_clear_enable; + r->depth_meta_clear_enable = meta_clear; + r->depth_load_clear_enable = r->depth_clear_enable || r->depth_meta_clear_enable; + r->depth_clear_value = hw.GetDepthClearValue(); + r->depth_test_enable = dc.z_enable; + r->depth_write_enable = dc.z_write_enable && !z.depth_view.depth_write_disable; + r->depth_compare_op = static_cast(dc.zfunc); + + r->depth_bounds_test_enable = dc.depth_bounds_enable; + r->depth_min_bounds = hw.GetDepthBoundsMin(); + r->depth_max_bounds = hw.GetDepthBoundsMax(); + + r->stencil_clear_enable = rc.stencil_clear_enable; + r->stencil_clear_value = hw.GetStencilClearValue(); + r->stencil_test_enable = dc.stencil_enable; + if (dc.stencil_enable) { + if (dc.stencilfunc > static_cast(VK_COMPARE_OP_ALWAYS) || + (dc.backface_enable && + dc.stencilfunc_bf > static_cast(VK_COMPARE_OP_ALWAYS)) || + (UsesStencilOpValue(sc.stencil_fail, sc.stencil_zpass, sc.stencil_zfail) && + sm.stencil_opval != sm.stencil_testval) || + (dc.backface_enable && + UsesStencilOpValue(sc.stencil_fail_bf, sc.stencil_zpass_bf, sc.stencil_zfail_bf) && + sm.stencil_opval_bf != sm.stencil_testval_bf)) { + DepthFatal("unsupported stencil compare or replacement state"); + } + r->stencil_static_front = { + ConvertStencilOp(sc.stencil_fail), ConvertStencilOp(sc.stencil_zpass), + ConvertStencilOp(sc.stencil_zfail), static_cast(dc.stencilfunc)}; + r->stencil_dynamic_front = {sm.stencil_mask, + rc.stencil_clear_enable ? 0u : sm.stencil_writemask, + sm.stencil_testval}; + if (dc.backface_enable) { + r->stencil_static_back = { + ConvertStencilOp(sc.stencil_fail_bf), ConvertStencilOp(sc.stencil_zpass_bf), + ConvertStencilOp(sc.stencil_zfail_bf), static_cast(dc.stencilfunc_bf)}; + r->stencil_dynamic_back = {sm.stencil_mask_bf, + rc.stencil_clear_enable ? 0u : sm.stencil_writemask_bf, + sm.stencil_testval_bf}; + } else { + r->stencil_static_back = r->stencil_static_front; + r->stencil_dynamic_back = r->stencil_dynamic_front; + } + } + r->vaddr_num = has_stencil ? 2 : 1; + r->vaddr[0] = r->depth_buffer_vaddr; + r->size[0] = r->depth_buffer_size; + if (has_stencil) { + r->vaddr[1] = r->stencil_buffer_vaddr; + r->size[1] = r->stencil_buffer_size; + } + DepthTargetInfo info {}; + info.address = r->depth_buffer_vaddr; + info.size = r->depth_buffer_size; + info.stencil_address = r->stencil_buffer_vaddr; + info.stencil_size = r->stencil_buffer_size; + info.htile_address = r->htile_buffer_vaddr; + info.htile_size = r->htile_buffer_size; + info.format = r->format; + info.guest_format = guest_format; + info.width = width; + info.height = height; + info.pitch = pitch; + info.bytes_per_element = bytes; + info.tile_mode = Prospero::GpuEnumValue(Prospero::TileMode::kDepth); + info.depth_load_clear = r->depth_load_clear_enable; + info.stencil_load_clear = rc.stencil_clear_enable; + info.stencil_access = + r->stencil_clear_enable || + (r->stencil_test_enable && + (stencil_face_accesses_attachment(r->stencil_static_front, r->stencil_dynamic_front) || + stencil_face_accesses_attachment(r->stencil_static_back, r->stencil_dynamic_back))); + r->vulkan_buffer = cache->FindDepthTarget(buffer, g_render_ctx->GetGraphicCtx(), info); + if (meta_clear && !cache->TouchMeta(z.htile_data_base_addr, z.depth_view.slice_start, false)) { + DepthFatal("failed to consume HTile clear state"); + } +} + +void MarkRenderTargetGpuWritten(const RenderDepthInfo& target) { + const bool with_depth = target.format != VK_FORMAT_UNDEFINED && target.vulkan_buffer != nullptr; + + if (with_depth && !depth_attachment_read_only(&target)) { + g_render_ctx->GetTextureCache()->MarkGpuWritten(target.vulkan_buffer); + } +} + +} // namespace Libs::Graphics diff --git a/src/graphics/host_gpu/renderer/descriptorCache.cpp b/src/graphics/host_gpu/renderer/descriptorCache.cpp new file mode 100644 index 000000000..392042161 --- /dev/null +++ b/src/graphics/host_gpu/renderer/descriptorCache.cpp @@ -0,0 +1,303 @@ +#include "graphics/host_gpu/renderer/descriptorCache.h" + +#include "common/assert.h" +#include "common/profiler.h" +#include "graphics/host_gpu/renderer/renderContext.h" +#include "graphics/shader/recompiler/ShaderIR.h" + +#include + +namespace Libs::Graphics { +namespace { + +using BindingKind = ShaderRecompiler::IR::DescriptorBindingKind; + +VkShaderStageFlags StageFlags(DescriptorCache::Stage stage) { + switch (stage) { + case DescriptorCache::Stage::Vertex: return VK_SHADER_STAGE_VERTEX_BIT; + case DescriptorCache::Stage::Pixel: return VK_SHADER_STAGE_FRAGMENT_BIT; + case DescriptorCache::Stage::Compute: return VK_SHADER_STAGE_COMPUTE_BIT; + default: EXIT("unknown descriptor stage\n"); + } +} + +bool IsSampledImage(BindingKind kind) { + switch (kind) { + case BindingKind::Sampled2D: + case BindingKind::Sampled2DArray: + case BindingKind::Sampled3D: + case BindingKind::SampledUint2D: + case BindingKind::SampledUint2DArray: + case BindingKind::SampledUint3D: return true; + default: return false; + } +} + +bool IsStorageImage(BindingKind kind) { + switch (kind) { + case BindingKind::Storage2D: + case BindingKind::Storage2DArray: + case BindingKind::Storage3D: + case BindingKind::StorageUint2D: + case BindingKind::StorageUint2DArray: + case BindingKind::StorageUint3D: return true; + default: return false; + } +} + +VkDescriptorType DescriptorType(BindingKind kind) { + if (kind == BindingKind::Samplers) { + return VK_DESCRIPTOR_TYPE_SAMPLER; + } + if (IsSampledImage(kind)) { + return VK_DESCRIPTOR_TYPE_SAMPLED_IMAGE; + } + if (IsStorageImage(kind)) { + return VK_DESCRIPTOR_TYPE_STORAGE_IMAGE; + } + return VK_DESCRIPTOR_TYPE_STORAGE_BUFFER; +} + +uint32_t DescriptorCount(const ShaderRecompiler::IR::DescriptorBinding& binding) { + return binding.resources.empty() ? 1u : static_cast(binding.resources.size()); +} + +std::vector LayoutKey(DescriptorCache::Stage stage, + const ShaderRecompiler::IR::Program& program) { + std::vector key; + key.reserve(1u + program.bindings.descriptors.size() * 4u); + key.push_back(static_cast(stage)); + for (const auto& binding: program.bindings.descriptors) { + key.push_back(static_cast(binding.kind)); + key.push_back(binding.binding); + key.push_back(DescriptorCount(binding)); + key.push_back(static_cast(DescriptorType(binding.kind))); + } + return key; +} + +VkImageLayout SampledLayout(const VulkanImage* image) { + EXIT_IF(image == nullptr); + switch (image->layout) { + case VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL: + case VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL: + case VK_IMAGE_LAYOUT_COLOR_ATTACHMENT_OPTIMAL: return VK_IMAGE_LAYOUT_GENERAL; + default: return image->layout; + } +} + +VkDescriptorBufferInfo BufferInfo(const BufferView& view) { + EXIT_IF(view.buffer == nullptr || view.buffer->buffer == nullptr); + return {view.buffer->buffer, view.offset, view.range}; +} + +VkDescriptorImageInfo MakeDescriptorImageInfo(const DescriptorCache::TextureBinding& texture, + VkImageLayout layout) { + EXIT_IF(texture.image == nullptr || texture.view < 0 || texture.view >= VulkanImage::VIEW_MAX); + const auto view = texture.image_view != nullptr ? texture.image_view + : texture.image->image_view[texture.view]; + EXIT_IF(view == nullptr); + return {nullptr, view, layout}; +} + +} // namespace + +void DescriptorCache::Init() { + m_initialized = true; +} + +VkDescriptorSetLayout +DescriptorCache::GetDescriptorSetLayoutInternal(GraphicContext* gctx, Stage stage, + const ShaderRecompiler::IR::Program& program) { + EXIT_IF(gctx == nullptr); + Init(); + const auto key = LayoutKey(stage, program); + if (const auto found = m_descriptor_set_layouts.find(key); + found != m_descriptor_set_layouts.end()) { + return found->second; + } + + std::vector bindings; + bindings.reserve(program.bindings.descriptors.size()); + for (const auto& descriptor: program.bindings.descriptors) { + bindings.push_back({descriptor.binding, DescriptorType(descriptor.kind), + DescriptorCount(descriptor), StageFlags(stage), nullptr}); + } + if (bindings.empty()) { + return nullptr; + } + + VkDescriptorSetLayoutCreateInfo info {}; + info.sType = VK_STRUCTURE_TYPE_DESCRIPTOR_SET_LAYOUT_CREATE_INFO; + info.bindingCount = static_cast(bindings.size()); + info.pBindings = bindings.data(); + VkDescriptorSetLayout layout = nullptr; + vkCreateDescriptorSetLayout(gctx->device, &info, nullptr, &layout); + EXIT_NOT_IMPLEMENTED(layout == nullptr); + m_descriptor_set_layouts.emplace(key, layout); + return layout; +} + +void DescriptorCache::CreatePool(GraphicContext* gctx) { + KYTY_PROFILER_FUNCTION(); + EXIT_IF(gctx == nullptr); + constexpr uint32_t MaxSets = 512; + const VkDescriptorPoolSize sizes[] = { + {VK_DESCRIPTOR_TYPE_STORAGE_BUFFER, + MaxSets * (ShaderRecompiler::IR::ShaderInfo::MaxBuffers + + ShaderRecompiler::IR::ShaderInfo::MaxAddresses + 3u)}, + {VK_DESCRIPTOR_TYPE_SAMPLED_IMAGE, MaxSets * ShaderRecompiler::IR::ShaderInfo::MaxImages}, + {VK_DESCRIPTOR_TYPE_STORAGE_IMAGE, MaxSets * ShaderRecompiler::IR::ShaderInfo::MaxImages}, + {VK_DESCRIPTOR_TYPE_SAMPLER, MaxSets * ShaderRecompiler::IR::ShaderInfo::MaxSamplers}, + }; + VkDescriptorPoolCreateInfo info {}; + info.sType = VK_STRUCTURE_TYPE_DESCRIPTOR_POOL_CREATE_INFO; + info.poolSizeCount = static_cast(std::size(sizes)); + info.pPoolSizes = sizes; + info.maxSets = MaxSets; + const auto pool_id = static_cast(m_pools.size()); + auto& pool = m_pools.emplace_back(); + vkCreateDescriptorPool(gctx->device, &info, nullptr, &pool.pool); + EXIT_NOT_IMPLEMENTED(pool.pool == nullptr); + pool.next_free_pool = m_first_free_pool; + m_first_free_pool = pool_id; +} + +VulkanDescriptorSet* DescriptorCache::Allocate(Stage stage, + const ShaderRecompiler::IR::Program& program) { + KYTY_PROFILER_FUNCTION(); + auto* gctx = g_render_ctx->GetGraphicCtx(); + EXIT_IF(gctx == nullptr); + Common::LockGuard lock(m_mutex); + const auto layout = GetDescriptorSetLayoutInternal(gctx, stage, program); + EXIT_IF(layout == nullptr); + auto& free_sets = m_free_sets_by_layout[layout]; + if (!free_sets.empty()) { + auto* result = free_sets.back(); + free_sets.pop_back(); + return result; + } + + constexpr uint32_t Batch = 32; + for (int attempt = 0; attempt < 2; attempt++) { + for (int pool_id = m_first_free_pool; pool_id != -1; + pool_id = m_pools[pool_id].next_free_pool) { + auto& pool = m_pools[pool_id]; + std::array layouts; + layouts.fill(layout); + std::array sets {}; + VkDescriptorSetAllocateInfo info {}; + info.sType = VK_STRUCTURE_TYPE_DESCRIPTOR_SET_ALLOCATE_INFO; + info.descriptorPool = pool.pool; + info.descriptorSetCount = Batch; + info.pSetLayouts = layouts.data(); + if (vkAllocateDescriptorSets(gctx->device, &info, sets.data()) == VK_SUCCESS) { + free_sets.reserve(free_sets.size() + Batch - 1u); + for (uint32_t i = 0; i + 1u < Batch; i++) { + free_sets.push_back(new VulkanDescriptorSet {sets[i], layout, pool_id}); + } + return new VulkanDescriptorSet {sets.back(), layout, pool_id}; + } + pool.free = false; + m_first_free_pool = pool.next_free_pool; + break; + } + CreatePool(gctx); + } + return nullptr; +} + +void DescriptorCache::Recycle(VulkanDescriptorSet* set) { + EXIT_IF(set == nullptr || set->set == nullptr || set->layout == nullptr); + Common::LockGuard lock(m_mutex); + m_free_sets_by_layout[set->layout].push_back(set); +} + +VulkanDescriptorSet* DescriptorCache::GetDescriptor(Stage stage, + const ShaderRecompiler::IR::Program& program, + const NativeDescriptors& data) { + KYTY_PROFILER_FUNCTION(); + EXIT_IF(data.buffers.size() != program.info.buffers.size() || + data.images.size() != program.info.images.size() || + data.samplers.size() != program.info.samplers.size() || + data.addresses.size() != program.info.addresses.size()); + auto* gctx = g_render_ctx->GetGraphicCtx(); + EXIT_IF(gctx == nullptr); + auto* set = Allocate(stage, program); + EXIT_NOT_IMPLEMENTED(set == nullptr); + + const auto descriptor_count = program.info.buffers.size() + program.info.images.size() + + program.info.samplers.size() + program.info.addresses.size() + 3u; + std::vector buffer_infos; + std::vector image_infos; + std::vector writes; + buffer_infos.reserve(descriptor_count); + image_infos.reserve(descriptor_count); + writes.reserve(program.bindings.descriptors.size()); + + for (const auto& binding: program.bindings.descriptors) { + VkWriteDescriptorSet write {}; + write.sType = VK_STRUCTURE_TYPE_WRITE_DESCRIPTOR_SET; + write.dstSet = set->set; + write.dstBinding = binding.binding; + write.descriptorType = DescriptorType(binding.kind); + write.descriptorCount = DescriptorCount(binding); + const auto buffer_start = buffer_infos.size(); + const auto image_start = image_infos.size(); + switch (binding.kind) { + case BindingKind::Buffers: + for (const auto resource: binding.resources) { + buffer_infos.push_back(BufferInfo(data.buffers.at(resource))); + } + break; + case BindingKind::AddressMemory: + for (const auto resource: binding.resources) { + buffer_infos.push_back(BufferInfo(data.addresses.at(resource))); + } + break; + case BindingKind::FlattenedSrt: + buffer_infos.push_back(BufferInfo(data.flattened_srt)); + break; + case BindingKind::UserData: buffer_infos.push_back(BufferInfo(data.user_data)); break; + case BindingKind::Gds: buffer_infos.push_back(BufferInfo(data.gds)); break; + case BindingKind::Samplers: + for (const auto resource: binding.resources) { + const auto sampler = data.samplers.at(resource); + EXIT_IF(sampler == nullptr); + image_infos.push_back({sampler, nullptr, VK_IMAGE_LAYOUT_UNDEFINED}); + } + break; + default: { + const auto layout = IsStorageImage(binding.kind) ? VK_IMAGE_LAYOUT_GENERAL + : VK_IMAGE_LAYOUT_UNDEFINED; + for (const auto resource: binding.resources) { + const auto& texture = data.images.at(resource); + image_infos.push_back(MakeDescriptorImageInfo( + texture, + IsSampledImage(binding.kind) ? SampledLayout(texture.image) : layout)); + } + break; + } + } + if (buffer_infos.size() != buffer_start) { + write.pBufferInfo = buffer_infos.data() + buffer_start; + } + if (image_infos.size() != image_start) { + write.pImageInfo = image_infos.data() + image_start; + } + writes.push_back(write); + } + vkUpdateDescriptorSets(gctx->device, static_cast(writes.size()), writes.data(), 0, + nullptr); + return set; +} + +VkDescriptorSetLayout +DescriptorCache::GetDescriptorSetLayout(Stage stage, const ShaderRecompiler::IR::Program& program) { + auto* gctx = g_render_ctx->GetGraphicCtx(); + EXIT_IF(gctx == nullptr); + Common::LockGuard lock(m_mutex); + return GetDescriptorSetLayoutInternal(gctx, stage, program); +} + +} // namespace Libs::Graphics diff --git a/src/graphics/host_gpu/renderer/descriptorCache.h b/src/graphics/host_gpu/renderer/descriptorCache.h new file mode 100644 index 000000000..1f2080fc4 --- /dev/null +++ b/src/graphics/host_gpu/renderer/descriptorCache.h @@ -0,0 +1,118 @@ +#ifndef EMULATOR_SRC_GRAPHICS_HOST_GPU_RENDERER_DESCRIPTORCACHE_H_ +#define EMULATOR_SRC_GRAPHICS_HOST_GPU_RENDERER_DESCRIPTORCACHE_H_ + +#include "common/abi.h" +#include "common/assert.h" +#include "common/common.h" +#include "common/threads.h" +#include "graphics/host_gpu/graphicContext.h" +#include "graphics/shader/shaderBindings.h" + +#include +#include +#include +#include + +namespace Libs::Graphics { + +namespace ShaderRecompiler::IR { +struct Program; +} + +class CommandBuffer; +struct ShaderStageRuntime; + +struct VulkanDescriptor { + VkDescriptorSet descriptor_set = nullptr; +}; + +struct VulkanDescriptorSet { + VkDescriptorSet set = nullptr; + VkDescriptorSetLayout layout = nullptr; + int pool_id = -1; +}; + +struct BufferView { + VulkanBuffer* buffer = nullptr; + VkDeviceSize offset = 0; + VkDeviceSize range = VK_WHOLE_SIZE; +}; + +struct ShaderAddressWriteRange { + VulkanBuffer* buffer = nullptr; + uint64_t address = 0; + uint64_t size = 0; +}; + +class DescriptorCache { +public: + enum class Stage { Unknown, Vertex, Pixel, Compute }; + + struct TextureBinding { + VulkanImage* image = nullptr; + int view = VulkanImage::VIEW_DEFAULT; + VkImageView image_view = nullptr; + }; + + enum class TextureVariant : int { + Float2D = 0, + Uint2D, + FloatArray, + UintArray, + Float3D, + Uint3D, + }; + + struct NativeDescriptors { + std::vector buffers; + std::vector images; + std::vector samplers; + std::vector addresses; + BufferView gds; + BufferView flattened_srt; + BufferView user_data; + }; + + DescriptorCache() { EXIT_NOT_IMPLEMENTED(!Common::Thread::IsMainThread()); } + virtual ~DescriptorCache() { KYTY_NOT_IMPLEMENTED; } + KYTY_CLASS_NO_COPY(DescriptorCache); + + VkDescriptorSetLayout GetDescriptorSetLayout(Stage stage, + const ShaderRecompiler::IR::Program& program); + void Recycle(VulkanDescriptorSet* set); + VulkanDescriptorSet* GetDescriptor(Stage stage, const ShaderRecompiler::IR::Program& program, + const NativeDescriptors& descriptors); + +private: + struct Pool { + VkDescriptorPool pool = nullptr; + int next_free_pool = -1; + bool free = true; + }; + + void Init(); + void CreatePool(GraphicContext* gctx); + VulkanDescriptorSet* Allocate(Stage stage, const ShaderRecompiler::IR::Program& program); + VkDescriptorSetLayout + GetDescriptorSetLayoutInternal(GraphicContext* gctx, Stage stage, + const ShaderRecompiler::IR::Program& program); + + Common::Mutex m_mutex; + std::vector m_pools; + int m_first_free_pool = -1; + std::unordered_map> + m_free_sets_by_layout; + std::map, VkDescriptorSetLayout> m_descriptor_set_layouts; + bool m_initialized = false; +}; + +const char* storage_usage_name(ShaderStorageUsage usage); +VkImageAspectFlags DepthStencilAspectMask(VkFormat format); +std::vector +BindDescriptors(uint64_t submit_id, CommandBuffer* buffer, VkPipelineBindPoint pipeline_bind_point, + VkPipelineLayout layout, const ShaderStageRuntime& runtime, + VkShaderStageFlags vk_stage, DescriptorCache::Stage stage); + +} // namespace Libs::Graphics + +#endif // EMULATOR_SRC_GRAPHICS_HOST_GPU_RENDERER_DESCRIPTORCACHE_H_ diff --git a/src/graphics/host_gpu/renderer/descriptors.cpp b/src/graphics/host_gpu/renderer/descriptors.cpp new file mode 100644 index 000000000..35f8c07d1 --- /dev/null +++ b/src/graphics/host_gpu/renderer/descriptors.cpp @@ -0,0 +1,698 @@ +#include "graphics/host_gpu/renderer/descriptors.h" + +#include "common/assert.h" +#include "common/common.h" +#include "common/file.h" +#include "common/logging/log.h" +#include "common/profiler.h" +#include "common/stringUtils.h" +#include "common/threads.h" +#include "graphics/guest_gpu/gpu_defs.h" +#include "graphics/guest_gpu/gpu_format.h" +#include "graphics/guest_gpu/graphicsRun.h" +#include "graphics/guest_gpu/hardwareContext.h" +#include "graphics/guest_gpu/tile.h" +#include "graphics/host_gpu/graphicContext.h" +#include "graphics/host_gpu/hostMemory.h" +#include "graphics/host_gpu/objects/textureCommon.h" +#include "graphics/host_gpu/renderer/descriptorCache.h" +#include "graphics/host_gpu/renderer/framebufferCache.h" +#include "graphics/host_gpu/renderer/imageView.h" +#include "graphics/host_gpu/renderer/render.h" +#include "graphics/host_gpu/renderer/renderContext.h" +#include "graphics/host_gpu/renderer/renderState.h" +#include "graphics/host_gpu/renderer/shaderResourceBarrier.h" +#include "graphics/host_gpu/utils.h" +#include "graphics/host_gpu/vma.h" +#include "graphics/presentation/displayBuffer.h" +#include "graphics/shader/recompiler/BindingLayout.h" +#include "graphics/shader/recompiler/ResourceMaterialization.h" +#include "graphics/shader/recompiler/ShaderIR.h" +#include "graphics/shader/shader.h" + +#include +#include +#include +#include +#include +#include + +#ifdef min +#undef min +#endif +#ifdef max +#undef max +#endif + +namespace Libs::Graphics { + +using TextureVariant = DescriptorCache::TextureVariant; + +static void BindNullStorageBuffer(CommandBuffer* cmd_buffer, BufferView* dst) { + EXIT_IF(cmd_buffer == nullptr || dst == nullptr); + + dst->buffer = + g_render_ctx->GetBufferCache()->ObtainNullBuffer(cmd_buffer, g_render_ctx->GetGraphicCtx()); + dst->offset = 0; + dst->range = 16; +} + +// NOLINTNEXTLINE(readability-function-cognitive-complexity) +static const char* VulkanImageTypeName(VulkanImageType type) { + switch (type) { + case VulkanImageType::VideoOut: return "VideoOut"; + case VulkanImageType::DepthStencil: return "DepthStencil"; + case VulkanImageType::Texture: return "Texture"; + case VulkanImageType::StorageTexture: return "StorageTexture"; + case VulkanImageType::RenderTexture: return "RenderTexture"; + case VulkanImageType::Unknown: + default: return "Unknown"; + } +} + +VkImageAspectFlags DepthStencilAspectMask(VkFormat format) { + auto ret = static_cast(VK_IMAGE_ASPECT_DEPTH_BIT); + if (format == VK_FORMAT_D24_UNORM_S8_UINT || format == VK_FORMAT_D32_SFLOAT_S8_UINT) { + ret |= VK_IMAGE_ASPECT_STENCIL_BIT; + } + return ret; +} + +static int SampledArrayViewIndex(const VulkanImage* image, int view_index) { + EXIT_IF(image == nullptr); + + switch (view_index) { + case VulkanImage::VIEW_DEFAULT: return VulkanImage::VIEW_DEFAULT_ARRAY; + case VulkanImage::VIEW_BGRA: return VulkanImage::VIEW_BGRA_ARRAY; + case VulkanImage::VIEW_DEPTH_TEXTURE: return VulkanImage::VIEW_DEPTH_TEXTURE_ARRAY; + case VulkanImage::VIEW_STENCIL_TEXTURE: return VulkanImage::VIEW_STENCIL_TEXTURE_ARRAY; + case VulkanImage::VIEW_R001: return VulkanImage::VIEW_R001_ARRAY; + case VulkanImage::VIEW_RGB1: return VulkanImage::VIEW_RGB1_ARRAY; + case VulkanImage::VIEW_R000: return VulkanImage::VIEW_R000_ARRAY; + case VulkanImage::VIEW_RG01: return VulkanImage::VIEW_RG01_ARRAY; + case VulkanImage::VIEW_000R: return VulkanImage::VIEW_000R_ARRAY; + default: return view_index; + } +} + +static int SelectSampledTextureArrayView(const VulkanImage* image, int base_view) { + const int array_view = SampledArrayViewIndex(image, base_view); + if (image->image_view[array_view] == nullptr) { + EXIT("missing sampled array image view: image_type=%s base_view=%d array_view=%d " + "layers=%u\n", + VulkanImageTypeName(image->type), base_view, array_view, image->layers); + } + return array_view; +} + +static bool TextureVariantIsUint(TextureVariant variant) { + return variant == TextureVariant::Uint2D || variant == TextureVariant::UintArray || + variant == TextureVariant::Uint3D; +} + +static bool TextureVariantIsArray(TextureVariant variant) { + return variant == TextureVariant::FloatArray || variant == TextureVariant::UintArray; +} + +static bool TextureVariantIs3D(TextureVariant variant) { + return variant == TextureVariant::Float3D || variant == TextureVariant::Uint3D; +} + +static int TextureVariantDefaultView(TextureVariant variant) { + return TextureVariantIsArray(variant) ? VulkanImage::VIEW_DEFAULT_ARRAY + : VulkanImage::VIEW_DEFAULT; +} + +static VulkanImage* GetDummySampledTexture(TextureVariant variant) { + return g_render_ctx->GetTextureCache()->GetDummySampledTexture(TextureVariantIsUint(variant), + TextureVariantIs3D(variant)); +} + +static VulkanImage* GetDummyStorageTexture(TextureVariant variant) { + return g_render_ctx->GetTextureCache()->GetDummyStorageTexture(TextureVariantIsUint(variant), + TextureVariantIs3D(variant)); +} + +static ShaderBufferResource +NativeBufferDescriptor(const ShaderRecompiler::IR::DescriptorValue& descriptor) { + ShaderBufferResource result; + std::copy_n(descriptor.dwords.begin(), std::size(result.fields), result.fields); + return result; +} + +static ShaderTextureResource +NativeTextureDescriptor(const ShaderRecompiler::IR::DescriptorValue& descriptor) { + ShaderTextureResource result; + std::copy_n(descriptor.dwords.begin(), std::size(result.fields), result.fields); + return result; +} + +static ShaderSamplerResource +NativeSamplerDescriptor(const ShaderRecompiler::IR::DescriptorValue& descriptor) { + ShaderSamplerResource result; + std::copy_n(descriptor.dwords.begin(), std::size(result.fields), result.fields); + return result; +} + +static BufferView NativeStorageBuffer(uint64_t submit_id, CommandBuffer* command_buffer, + const ShaderBufferResource& descriptor, + const ShaderRecompiler::IR::BufferResource& resource) { + BufferView result; + const auto address = descriptor.Base48(); + const auto stride = descriptor.Stride(); + const auto records = descriptor.NumRecords(); + if (stride != 0 && records > UINT64_MAX / stride) { + EXIT("storage buffer descriptor footprint overflow\n"); + } + const auto size = stride != 0 ? static_cast(stride) * records : records; + if (address == 0 || size == 0) { + BindNullStorageBuffer(command_buffer, &result); + return result; + } + auto* const ctx = g_render_ctx->GetGraphicCtx(); + const auto alignment = ctx->StorageMinAlignment(); + if (alignment == 0 || size > ctx->GetPhysicalDeviceProperties().limits.maxStorageBufferRange || + BufferCache::CACHING_PAGE_SIZE % alignment != 0) { + EXIT("storage buffer range or device alignment is unsupported\n"); + } + (void)submit_id; + auto binding = g_render_ctx->GetBufferCache()->ObtainBuffer( + command_buffer, ctx, address, size, resource.written, resource.read, resource.formatted); + if (binding.second % alignment != 0) { + EXIT("storage buffer binding is not device-aligned\n"); + } + result.buffer = binding.first; + result.offset = binding.second; + result.range = static_cast(size); + return result; +} + +static BufferView +NativeAddressBuffer(uint64_t submit_id, CommandBuffer* command_buffer, + const ShaderRecompiler::IR::AddressResource& resource, + const ShaderRecompiler::IR::ResourceSnapshot::Address& address) { + BufferView result; + if (address.binding_base == 0) { + BindNullStorageBuffer(command_buffer, &result); + return result; + } + if (resource.written) { + EXIT("writable address resources are unsupported\n"); + } + const auto limit = resource.kind == ShaderRecompiler::IR::ResourceKind::Flat + ? ShaderRecompiler::IR::FlatAddressWindowSize + : static_cast(g_render_ctx->GetGraphicCtx() + ->GetPhysicalDeviceProperties() + .limits.maxStorageBufferRange); + uint64_t size = 0; + const auto access = HostMemoryAccess::Read; + if (!HostMemoryQueryRange(address.binding_base, limit, access, &size)) { + EXIT("address resource is not host-accessible: base=0x%016" PRIx64 "\n", + address.binding_base); + } + auto* const ctx = g_render_ctx->GetGraphicCtx(); + const auto alignment = ctx->StorageMinAlignment(); + if (alignment == 0 || size > ctx->GetPhysicalDeviceProperties().limits.maxStorageBufferRange || + BufferCache::GetBufferOffset(address.binding_base) % alignment != 0) { + EXIT("address resource range or alignment is unsupported\n"); + } + (void)submit_id; + auto binding = g_render_ctx->GetBufferCache()->ObtainBuffer(command_buffer, ctx, + address.binding_base, size); + result.buffer = binding.first; + result.offset = binding.second; + result.range = static_cast(size); + return result; +} + +static TextureVariant NativeTextureVariant(const ShaderRecompiler::IR::ImageResource& resource) { + const bool uint_image = resource.kind == ShaderRecompiler::IR::ResourceKind::ImageUint || + resource.kind == ShaderRecompiler::IR::ResourceKind::StorageImageUint; + switch (resource.dimension) { + case ShaderRecompiler::Decoder::ImageDimension::Dim3D: + return uint_image ? TextureVariant::Uint3D : TextureVariant::Float3D; + case ShaderRecompiler::Decoder::ImageDimension::Dim2DArray: + return uint_image ? TextureVariant::UintArray : TextureVariant::FloatArray; + default: return uint_image ? TextureVariant::Uint2D : TextureVariant::Float2D; + } +} + +static bool IsSupportedSampledColorResource(const ShaderRecompiler::IR::ImageResource& resource) { + return resource.kind == ShaderRecompiler::IR::ResourceKind::Image && + resource.dimension == ShaderRecompiler::Decoder::ImageDimension::Dim2D && + resource.mip_mode == ShaderRecompiler::IR::ImageMipMode::None && resource.read && + !resource.written && !resource.atomic && !resource.depth_compare; +} + +static bool IsSupportedSampledDepthFormat(VkFormat image_format, uint32_t guest_format, + VkFormat view_format) { + const bool d16 = image_format == VK_FORMAT_D16_UNORM && + guest_format == Prospero::GpuEnumValue(Prospero::BufferFormat::k16UNorm) && + view_format == VK_FORMAT_R16_UNORM; + const bool d32 = + (image_format == VK_FORMAT_D32_SFLOAT || image_format == VK_FORMAT_D32_SFLOAT_S8_UINT) && + guest_format == Prospero::GpuEnumValue(Prospero::BufferFormat::k32Float) && + view_format == VK_FORMAT_R32_SFLOAT; + return d16 || d32; +} + +bool IsSupportedDepthTargetDescriptor(const ShaderTextureResource& descriptor, + const VulkanImage& image) { + const auto width = static_cast(descriptor.Width5()) + 1u; + const auto height = static_cast(descriptor.Height5()) + 1u; + const auto pitch = TileGetTexturePitch(descriptor.Format(), width, 1, descriptor.TileMode()); + return image.type == VulkanImageType::DepthStencil && image.layers == 1 && + width == image.extent.width && height == image.extent.height && + descriptor.Depth() == 0 && descriptor.BaseLevel() == 0 && descriptor.LastLevel() == 0 && + descriptor.MaxMip() == 0 && descriptor.MinLod() == 0 && descriptor.BaseArray5() == 0 && + descriptor.TileMode() == Prospero::GpuEnumValue(Prospero::TileMode::kDepth) && + descriptor.Type() == Prospero::GpuEnumValue(Prospero::ImageType::kColor2D) && + descriptor.BCSwizzle() == 0 && !descriptor.MsaaDepth() && pitch >= width && + pitch == image.guest_pitch; +} + +static bool IsSupportedDepthTextureEncoding(const ShaderTextureResource& descriptor) { + constexpr uint32_t field1_reserved_mask = 0x200fff00u; + constexpr uint32_t field2_reserved_mask = 0xf0003000u; + constexpr uint32_t field3_base = 0x91800000u; + constexpr uint32_t field5_expected = 0x00700000u; + return (descriptor.fields[1] & field1_reserved_mask) == 0 && + (descriptor.fields[2] & field2_reserved_mask) == 0 && + descriptor.fields[3] == (field3_base | descriptor.DstSelXYZW()) && + descriptor.fields[4] == 0 && descriptor.fields[5] == field5_expected && + descriptor.fields[6] == 0 && descriptor.fields[7] == 0; +} + +static void ValidateDepthTargetBinding(const ShaderRecompiler::IR::ImageResource& resource, + const ShaderTextureResource& descriptor, + const VulkanImage* image, VkFormat view_format, + uint64_t size) { + const bool resource_ok = IsSupportedSampledDepthResource(resource); + const bool descriptor_ok = + image != nullptr && IsSupportedDepthTargetDescriptor(descriptor, *image); + const bool encoding_ok = IsSupportedDepthTextureEncoding(descriptor); + const bool format_ok = image != nullptr && IsSupportedSampledDepthFormat( + image->format, descriptor.Format(), view_format); + if (resource_ok && descriptor_ok && encoding_ok && format_ok && size != 0) { + return; + } + const auto descriptor_pitch = + TileGetTexturePitch(descriptor.Format(), static_cast(descriptor.Width5()) + 1u, 1, + descriptor.TileMode()); + EXIT("unsupported sampled depth target: resource=%d descriptor=%d encoding=%d format=%d " + "image_format=%d view_format=%d descriptor_pitch=%u target_pitch=%u " + "addr=0x%016" PRIx64 " size=0x%016" PRIx64 "\n", + resource_ok, descriptor_ok, encoding_ok, format_ok, + image == nullptr ? static_cast(VK_FORMAT_UNDEFINED) : static_cast(image->format), + static_cast(view_format), descriptor_pitch, + image == nullptr ? 0u : image->guest_pitch, descriptor.Base40(), size); +} + +static bool IsSupportedStorageTextureDescriptor(const ShaderRecompiler::IR::ImageResource& resource, + const ShaderTextureResource& descriptor) { + const auto tile = descriptor.TileMode(); + const bool is_2d = resource.dimension == ShaderRecompiler::Decoder::ImageDimension::Dim2D && + descriptor.Type() == Prospero::GpuEnumValue(Prospero::ImageType::kColor2D) && + descriptor.Depth() == 0; + const bool is_2d_array = + resource.dimension == ShaderRecompiler::Decoder::ImageDimension::Dim2DArray && + descriptor.Type() == Prospero::GpuEnumValue(Prospero::ImageType::kColor2DArray) && + descriptor.BaseArray5() <= descriptor.Depth(); + const bool is_3d = resource.dimension == ShaderRecompiler::Decoder::ImageDimension::Dim3D && + descriptor.Type() == Prospero::GpuEnumValue(Prospero::ImageType::kColor3D); + const bool supported_tile = tile == Prospero::GpuEnumValue(Prospero::TileMode::kLinear) || + tile == Prospero::GpuEnumValue(Prospero::TileMode::kRenderTarget); + const bool supported_swizzle = + IsSupportedStorageSwizzle(descriptor.Format(), descriptor.DstSelXYZW()) && + (descriptor.DstSelXYZW() == DstSel(4, 5, 6, 7) || !resource.read); + return (is_2d || is_2d_array || is_3d) && supported_tile && descriptor.BaseLevel() == 0 && + descriptor.LastLevel() == 0 && descriptor.MaxMip() == 0 && descriptor.MinLod() == 0 && + descriptor.BaseArray5() == 0 && supported_swizzle && descriptor.BCSwizzle() == 0 && + !descriptor.MsaaDepth(); +} + +static bool IsSupportedStorageTextureEncoding(const ShaderTextureResource& descriptor) { + constexpr uint32_t field1_reserved_mask = 0x200fff00u; + constexpr uint32_t field2_reserved_mask = 0xf0003000u; + constexpr uint32_t field5_expected = 0x00700000u; + const uint32_t expected_field3 = descriptor.DstSelXYZW() | + (static_cast(descriptor.TileMode()) << 20u) | + (static_cast(descriptor.Type()) << 28u); + return (descriptor.fields[1] & field1_reserved_mask) == 0 && + (descriptor.fields[2] & field2_reserved_mask) == 0 && + descriptor.fields[3] == expected_field3 && descriptor.fields[4] == descriptor.Depth() && + descriptor.fields[5] == field5_expected && descriptor.fields[6] == 0 && + descriptor.fields[7] == 0; +} + +void ValidateStorageTexture(const ShaderRecompiler::IR::ImageResource& resource, + const ShaderTextureResource& descriptor, uint64_t size) { + const auto format = descriptor.Format(); + const bool resource_ok = IsSupportedStorageImageResource(resource); + const bool descriptor_ok = IsSupportedStorageTextureDescriptor(resource, descriptor); + const bool encoding_ok = IsSupportedStorageTextureEncoding(descriptor); + const bool uint_resource = + resource.kind == ShaderRecompiler::IR::ResourceKind::StorageImageUint; + const bool format_ok = Prospero::IsSupportedTextureFormat(format) && + uint_resource == Prospero::IsUintTextureFormat(format); + if (resource_ok && descriptor_ok && encoding_ok && format_ok && size != 0) { + return; + } + EXIT("unsupported storage texture: resource=%d descriptor=%d encoding=%d format=%d " + "addr=0x%016" PRIx64 " size=0x%016" PRIx64 + " extent=%ux%ux%u type=%u format=%u tile=%u swizzle=0x%03x read=%d written=%d\n", + resource_ok, descriptor_ok, encoding_ok, format_ok, descriptor.Base40(), size, + static_cast(descriptor.Width5()) + 1u, + static_cast(descriptor.Height5()) + 1u, + static_cast(descriptor.Depth()) + 1u, descriptor.Type(), format, + descriptor.TileMode(), descriptor.DstSelXYZW(), resource.read, resource.written); +} + +void ValidateMetadataReuseTexture(const ShaderRecompiler::IR::ImageResource& resource, + const ShaderTextureResource& descriptor, uint64_t size) { + constexpr uint32_t field1_reserved = 0x200fff00u; + constexpr uint32_t field2_reserved = 0xf0003000u; + const auto format = descriptor.Format(); + if (!IsSupportedSampledColorResource(resource) || size == 0 || + (descriptor.fields[1] & field1_reserved) != 0 || + (descriptor.fields[2] & field2_reserved) != 0 || descriptor.fields[3] != 0x90500facu || + descriptor.fields[4] != 0 || descriptor.fields[5] != 0x00700000u || + descriptor.fields[6] != 0 || descriptor.fields[7] != 0 || + !Prospero::IsSupportedTextureFormat(format) || Prospero::IsUintTextureFormat(format)) { + EXIT("unsupported storage texture descriptor encoding\n"); + } +} + +static DescriptorCache::TextureBinding +NativeTexture(uint64_t submit_id, CommandBuffer* command_buffer, + const ShaderRecompiler::IR::ImageResource& resource, + const ShaderRecompiler::IR::DescriptorValue& value) { + auto descriptor = NativeTextureDescriptor(value); + const bool storage = resource.kind == ShaderRecompiler::IR::ResourceKind::StorageImage || + resource.kind == ShaderRecompiler::IR::ResourceKind::StorageImageUint; + const auto variant = NativeTextureVariant(resource); + if (storage) { + ValidateStorageImageResource(resource); + } + if (descriptor.IsNull()) { + return storage ? DescriptorCache::TextureBinding {GetDummyStorageTexture(variant), + TextureVariantDefaultView(variant)} + : DescriptorCache::TextureBinding {GetDummySampledTexture(variant), + TextureVariantDefaultView(variant)}; + } + const auto address = descriptor.Base40(); + const auto width = static_cast(descriptor.Width5()) + 1u; + const auto height = static_cast(descriptor.Height5()) + 1u; + const auto base_level = descriptor.BaseLevel(); + const auto last_level = descriptor.LastLevel(); + const auto levels = static_cast(descriptor.MaxMip()) + 1u; + if (base_level > last_level || last_level >= levels) { + EXIT("unsupported texture mip view: base=%u last=%u levels=%u\n", base_level, last_level, + levels); + } + const auto view_levels = last_level - base_level + 1u; + const auto depth = static_cast(descriptor.Depth()) + 1u; + const auto tile = descriptor.TileMode(); + const auto format = descriptor.Format(); + const auto view_format = TextureGetFormat(format, storage ? TextureFormatUsage::Storage + : TextureFormatUsage::Sampled); + const auto type = static_cast(descriptor.Type()); + const auto pitch = TileGetTexturePitch(format, width, levels, tile); + const auto swizzle = descriptor.DstSelXYZW(); + TileSizeAlign size; + TileGetTextureTotalSize(format, width, height, depth, pitch, levels, tile, + type == Prospero::ImageType::kColor3D, &size); + EXIT_NOT_IMPLEMENTED(size.size == 0 || + (address & (static_cast(size.align) - 1u)) != 0); + if (storage) { + ValidateStorageTexture(resource, descriptor, size.size); + g_render_ctx->GetBufferCache()->ValidateGpuAccess(address, size.size, resource.read, + resource.written); + } + + VulkanImage* image = nullptr; + int view = VulkanImage::VIEW_DEFAULT; + VkImageView image_view = nullptr; + const bool check_depth = static_cast(tile) == Prospero::TileMode::kDepth || + descriptor.MsaaDepth(); + if (image == nullptr) { + if (check_depth) { + image = g_render_ctx->GetTextureCache()->FindDepthTargetByRange(address, size.size); + } else { + image = g_render_ctx->GetTextureCache()->FindRenderTargetByRange(command_buffer, + address, size.size); + } + if (image != nullptr) { + if (check_depth) { + ValidateDepthTargetBinding(resource, descriptor, image, view_format, size.size); + view = SelectSampledDepthView(image->format, view_format, swizzle); + if (graphics_debug_dump_enabled()) { + static std::atomic_uint log_count {0}; + if (log_count.fetch_add(1, std::memory_order_relaxed) < 64) { + LOGF("SampledDepthTarget: bind addr=0x%016" PRIx64 " size=0x%016" PRIx64 + " format=%d view=%d\n", + address, size.size, static_cast(image->format), view); + } + } + } else { + if (!(storage ? IsSupportedStorageImageResource(resource) + : IsSupportedSampledColorResource(resource)) || + image->type != VulkanImageType::RenderTexture || image->layers != 1 || + width != image->extent.width || height != image->extent.height || depth != 1 || + (storage ? levels != 1 || base_level != 0 + : levels != image->mip_levels || base_level >= levels) || + descriptor.BaseArray5() != 0 || type != Prospero::ImageType::kColor2D) { + EXIT("unsupported cached render-target image view: storage=%d resource=%u" + " image_type=%u layers=%u extent=%ux%u/%ux%u depth=%u" + " levels=%u/%u base_level=%u base_array=%u descriptor_type=%u\n", + storage, static_cast(resource.kind), + static_cast(image->type), image->layers, width, height, + image->extent.width, image->extent.height, depth, levels, + image->mip_levels, base_level, descriptor.BaseArray5(), + static_cast(type)); + } + view = storage ? SelectStorageColorView(image->format, view_format, swizzle) + : SelectSampledColorView(image->format, view_format, swizzle); + if (storage) { + image_view = g_render_ctx->GetTextureCache()->GetRenderTargetStorageView( + g_render_ctx->GetGraphicCtx(), + static_cast(image), view_format, base_level, + view_levels); + } else { + image_view = g_render_ctx->GetTextureCache()->GetRenderTargetSampledView( + g_render_ctx->GetGraphicCtx(), + static_cast(image), view_format, view, + base_level, view_levels); + } + } + if (image_view == nullptr && image->image_view[view] == nullptr) { + EXIT("required cached texture image view is missing\n"); + } + if (storage) { + g_render_ctx->GetTextureCache()->MarkGpuWritten(image); + } + } + } + if (image == nullptr) { + const auto video = Presentation::DisplayBufferFind(address); + if (video.image != nullptr) { + if (storage) { + EXIT("storage access to a video-out surface is unsupported\n"); + } + image = video.image; + if (!storage && swizzle == DstSel(6, 5, 4, 7)) { + view = VulkanImage::VIEW_BGRA; + } + } + } + if (image == nullptr) { + auto* texture_cache = g_render_ctx->GetTextureCache(); + const bool metadata_read = texture_cache->HasMetaOverlap(address, size.size); + if (storage && metadata_read) { + EXIT("storage texture overlaps surface metadata\n"); + } + if (!storage && metadata_read) { + ValidateMetadataReuseTexture(resource, descriptor, size.size); + } + (void)submit_id; + (void)command_buffer; + ImageInfo info {}; + info.address = address; + info.size = size.size; + info.format = format; + info.width = width; + info.height = height; + info.pitch = pitch; + info.base_level = base_level; + info.levels = levels; + info.view_levels = view_levels; + info.tile = tile; + info.swizzle = swizzle; + info.depth = depth; + info.type = descriptor.Type(); + info.base_array = descriptor.BaseArray5(); + if (storage) { + image = texture_cache->FindStorageTexture(command_buffer, g_render_ctx->GetGraphicCtx(), + info); + view = VulkanImage::VIEW_DEFAULT; + } else { + image = texture_cache->FindTexture(command_buffer, g_render_ctx->GetGraphicCtx(), info, + metadata_read); + if (image->type == VulkanImageType::StorageTexture) { + image_view = texture_cache->GetStorageTextureSampledView( + g_render_ctx->GetGraphicCtx(), static_cast(image), + info); + } + } + } + EXIT_NOT_IMPLEMENTED(image == nullptr); + if (resource.dimension == ShaderRecompiler::Decoder::ImageDimension::Dim2DArray) { + view = SelectSampledTextureArrayView(image, view); + } + return {image, view, image_view}; +} + +static VkSampler NativeSampler(const ShaderRecompiler::IR::Program& program, uint32_t index, + const ShaderRecompiler::IR::DescriptorValue& value) { + auto descriptor = NativeSamplerDescriptor(value); + const bool depth_compare = std::any_of(program.info.sampled_pairs.begin(), + program.info.sampled_pairs.end(), [&](const auto& pair) { + return pair.sampler == index && + pair.image < program.info.images.size() && + program.info.images[pair.image].depth_compare; + }); + if (!depth_compare) { + descriptor.fields[0] &= ~(0x7u << 12u); + } + return g_render_ctx->GetSamplerCache()->GetSampler(descriptor); +} + +static BufferView NativeUpload(CommandBuffer* command_buffer, std::span data) { + EXIT_IF(data.empty()); + BufferView result; + EXIT_IF(!g_render_ctx->GetBufferCache()->UploadHostData( + command_buffer, g_render_ctx->GetGraphicCtx(), data.data(), data.size_bytes(), 256, + &result.buffer, &result.offset, &result.range)); + return result; +} + +std::vector +BindDescriptors(uint64_t submit_id, CommandBuffer* buffer, VkPipelineBindPoint pipeline_bind_point, + VkPipelineLayout layout, const ShaderStageRuntime& runtime, + VkShaderStageFlags vk_stage, DescriptorCache::Stage stage) { + KYTY_PROFILER_FUNCTION(); + EXIT_IF(buffer == nullptr || !runtime); + const auto& program = *runtime.program; + const auto& snapshot = *runtime.resources; + std::string error; + if (!ShaderRecompiler::IR::ValidateResourceSpecialization(program, snapshot, &error)) { + EXIT("invalid native shader runtime snapshot: %s\n", error.c_str()); + } + auto* vk_buffer = buffer->GetPool()->buffers[buffer->GetIndex()]; + const auto shader_stages = ShaderPipelineStages(vk_stage); + + DescriptorCache::NativeDescriptors descriptors; + descriptors.buffers.reserve(program.info.buffers.size()); + for (uint32_t i = 0; i < program.info.buffers.size(); i++) { + descriptors.buffers.push_back( + NativeStorageBuffer(submit_id, buffer, NativeBufferDescriptor(snapshot.buffers[i]), + program.info.buffers[i])); + } + descriptors.images.reserve(program.info.images.size()); + for (uint32_t i = 0; i < program.info.images.size(); i++) { + const auto kind = program.info.images[i].kind; + if ((kind == ShaderRecompiler::IR::ResourceKind::StorageImage || + kind == ShaderRecompiler::IR::ResourceKind::StorageImageUint) && + vk_stage != VK_SHADER_STAGE_COMPUTE_BIT) { + EXIT("storage images are unsupported outside compute shaders\n"); + } + descriptors.images.push_back( + NativeTexture(submit_id, buffer, program.info.images[i], snapshot.images[i])); + } + descriptors.samplers.reserve(program.info.samplers.size()); + for (uint32_t i = 0; i < program.info.samplers.size(); i++) { + descriptors.samplers.push_back(NativeSampler(program, i, snapshot.samplers[i])); + } + descriptors.addresses.reserve(program.info.addresses.size()); + std::vector address_writes; + for (uint32_t i = 0; i < program.info.addresses.size(); i++) { + auto view = NativeAddressBuffer(submit_id, buffer, program.info.addresses[i], + snapshot.addresses[i]); + descriptors.addresses.push_back(view); + if (program.info.addresses[i].written && snapshot.addresses[i].binding_base != 0) { + EXIT_IF(view.buffer == nullptr || view.offset != 0 || view.range == VK_WHOLE_SIZE || + view.range == 0); + address_writes.push_back({view.buffer, snapshot.addresses[i].binding_base, + static_cast(view.range)}); + } + } + if (ShaderRecompiler::IR::FindBinding( + program.bindings, ShaderRecompiler::IR::DescriptorBindingKind::FlattenedSrt) != + nullptr) { + descriptors.flattened_srt = NativeUpload(buffer, snapshot.flattened_srt); + } + + std::vector user_data; + user_data.reserve(program.bindings.user_data_registers.size()); + for (const auto reg: program.bindings.user_data_registers) { + user_data.push_back(snapshot.user_data[reg - program.user_data_base]); + } + if (ShaderRecompiler::IR::FindBinding( + program.bindings, ShaderRecompiler::IR::DescriptorBindingKind::UserData) != nullptr) { + descriptors.user_data = NativeUpload(buffer, user_data); + } + if (ShaderRecompiler::IR::FindBinding( + program.bindings, ShaderRecompiler::IR::DescriptorBindingKind::Gds) != nullptr) { + descriptors.gds.buffer = + g_render_ctx->GetGdsBuffer()->GetBuffer(g_render_ctx->GetGraphicCtx()); + const auto barrier = MakeGdsDependency(*descriptors.gds.buffer); + vkCmdPipelineBarrier(vk_buffer, + VK_PIPELINE_STAGE_HOST_BIT | VK_PIPELINE_STAGE_TRANSFER_BIT | + VK_PIPELINE_STAGE_ALL_GRAPHICS_BIT | + VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT, + shader_stages, 0, 0, nullptr, 1, &barrier, 0, nullptr); + } + + for (uint32_t i = 0; i < program.info.images.size(); i++) { + auto* image = descriptors.images[i].image; + const auto& resource = program.info.images[i]; + if (resource.kind == ShaderRecompiler::IR::ResourceKind::Image || + resource.kind == ShaderRecompiler::IR::ResourceKind::ImageUint) { + switch (image->type) { + case VulkanImageType::DepthStencil: + GraphicsRenderDepthStencilBarrier(vk_buffer, image); + break; + case VulkanImageType::RenderTexture: + case VulkanImageType::StorageTexture: + GraphicsRenderTextureBarrier(vk_buffer, image); + break; + case VulkanImageType::VideoOut: + GraphicsRenderColorImageBarrier(vk_buffer, image, RENDER_COLOR_IMAGE_LAYOUT); + break; + default: break; + } + } else { + const auto barrier = + MakeStorageImageDependency(*image, resource.read, resource.written); + vkCmdPipelineBarrier(vk_buffer, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, shader_stages, 0, 0, + nullptr, 0, nullptr, 1, &barrier); + image->layout = VK_IMAGE_LAYOUT_GENERAL; + } + } + + if (!program.bindings.descriptors.empty()) { + auto* set = g_render_ctx->GetDescriptorCache()->GetDescriptor(stage, program, descriptors); + EXIT_IF(set == nullptr); + vkCmdBindDescriptorSets(vk_buffer, pipeline_bind_point, layout, + program.bindings.descriptor_set, 1, &set->set, 0, nullptr); + buffer->RecycleDescriptorAfterFence(set); + } + if (program.bindings.push_constant_size != 0) { + EXIT_IF(program.bindings.push_constant_size != user_data.size() * sizeof(uint32_t)); + vkCmdPushConstants(vk_buffer, layout, vk_stage, program.bindings.push_constant_offset, + program.bindings.push_constant_size, user_data.data()); + } + return address_writes; +} + +} // namespace Libs::Graphics diff --git a/src/graphics/host_gpu/renderer/descriptors.h b/src/graphics/host_gpu/renderer/descriptors.h new file mode 100644 index 000000000..b2b87eb96 --- /dev/null +++ b/src/graphics/host_gpu/renderer/descriptors.h @@ -0,0 +1,22 @@ +#ifndef EMULATOR_SRC_GRAPHICS_HOST_GPU_RENDERER_DESCRIPTORS_H_ +#define EMULATOR_SRC_GRAPHICS_HOST_GPU_RENDERER_DESCRIPTORS_H_ + +#include "graphics/shader/recompiler/ShaderIR.h" +#include "graphics/shader/shaderBindings.h" + +#include + +namespace Libs::Graphics { + +struct VulkanImage; + +[[nodiscard]] bool IsSupportedDepthTargetDescriptor(const ShaderTextureResource& descriptor, + const VulkanImage& image); +void ValidateMetadataReuseTexture(const ShaderRecompiler::IR::ImageResource& resource, + const ShaderTextureResource& descriptor, uint64_t size); +void ValidateStorageTexture(const ShaderRecompiler::IR::ImageResource& resource, + const ShaderTextureResource& descriptor, uint64_t size); + +} // namespace Libs::Graphics + +#endif // EMULATOR_SRC_GRAPHICS_HOST_GPU_RENDERER_DESCRIPTORS_H_ diff --git a/src/graphics/host_gpu/renderer/framebufferCache.cpp b/src/graphics/host_gpu/renderer/framebufferCache.cpp new file mode 100644 index 000000000..57a59f1bd --- /dev/null +++ b/src/graphics/host_gpu/renderer/framebufferCache.cpp @@ -0,0 +1,422 @@ +#include "graphics/host_gpu/renderer/framebufferCache.h" + +#include "common/assert.h" +#include "common/logging/log.h" +#include "common/profiler.h" +#include "graphics/host_gpu/renderer/renderContext.h" +#include "graphics/host_gpu/utils.h" + +#include +#include +#include + +namespace Libs::Graphics { + +VulkanFramebuffer* FramebufferCache::CreateFramebuffer(RenderColorInfo* colors, + uint32_t requested_color_count, + RenderDepthInfo* depth) { + KYTY_PROFILER_FUNCTION(); + + Common::LockGuard lock(m_mutex); + + EXIT_IF(colors == nullptr); + EXIT_IF(depth == nullptr); + EXIT_IF(g_render_ctx == nullptr); + EXIT_IF(requested_color_count > RENDER_COLOR_ATTACHMENTS_MAX); + + bool with_depth = (depth->format != VK_FORMAT_UNDEFINED && depth->vulkan_buffer != nullptr); + bool with_color[RENDER_COLOR_ATTACHMENTS_MAX] = {}; + uint32_t color_count = 0; + VulkanImage* first_color = nullptr; + VkExtent2D first_color_extent = {}; + for (uint32_t i = 0; i < requested_color_count; i++) { + with_color[i] = (colors[i].vulkan_buffer != nullptr); + if (!with_color[i]) { + break; + } + if (first_color == nullptr) { + first_color = colors[i].vulkan_buffer; + first_color_extent = colors[i].extent; + } else if (colors[i].extent.width != first_color_extent.width || + colors[i].extent.height != first_color_extent.height) { + LOGF("Framebuffer: temporary: dropping mismatched MRT%u attachment color0=%ux%u " + "color%u=%ux%u\n", + i, first_color_extent.width, first_color_extent.height, i, colors[i].extent.width, + colors[i].extent.height); + with_color[i] = false; + break; + } + color_count++; + } + VkImageLayout color_layout[RENDER_COLOR_ATTACHMENTS_MAX] = {}; + for (auto& layout: color_layout) { + layout = RENDER_COLOR_IMAGE_LAYOUT; + } + auto depth_layout = (with_depth ? depth_attachment_layout(depth) + : VK_IMAGE_LAYOUT_DEPTH_STENCIL_ATTACHMENT_OPTIMAL); + auto depth_read_only = + (with_depth && depth_layout == VK_IMAGE_LAYOUT_DEPTH_STENCIL_READ_ONLY_OPTIMAL); + + if (with_depth && first_color != nullptr && + (first_color_extent.width != depth->vulkan_buffer->extent.width || + first_color_extent.height != depth->vulkan_buffer->extent.height)) { + static std::atomic log_count {0}; + if (log_count.fetch_add(1, std::memory_order_relaxed) < 16) { + LOGF("Framebuffer: temporary: dropping mismatched PS5 depth attachment color=%ux%u " + "depth=%ux%u format=%s\n", + first_color_extent.width, first_color_extent.height, + depth->vulkan_buffer->extent.width, depth->vulkan_buffer->extent.height, + string_VkFormat(depth->format)); + } + depth->format = VK_FORMAT_UNDEFINED; + depth->vulkan_buffer = nullptr; + depth->depth_test_enable = false; + depth->depth_write_enable = false; + depth->depth_bounds_test_enable = false; + depth->stencil_test_enable = false; + depth->depth_clear_enable = false; + depth->depth_load_clear_enable = false; + depth->stencil_clear_enable = false; + with_depth = false; + depth_layout = VK_IMAGE_LAYOUT_DEPTH_STENCIL_ATTACHMENT_OPTIMAL; + depth_read_only = false; + } + + for (auto& f: m_framebuffers) { + bool color_match = (f.framebuffer != nullptr); + for (uint32_t i = 0; color_match && i < RENDER_COLOR_ATTACHMENTS_MAX; i++) { + const uint64_t image_id = + (i < color_count && with_color[i] ? colors[i].vulkan_buffer->memory.unique_id : 0); + color_match = + color_match && f.image_id[i] == image_id && + f.color_view[i] == + (i < color_count && with_color[i] ? colors[i].vulkan_view : nullptr) && + f.color_clear_enable[i] == + (i < color_count && with_color[i] && colors[i].color_clear_enable) && + f.color_layout[i] == color_layout[i]; + } + if (color_match && + f.depth_id == (with_depth ? depth->vulkan_buffer->memory.unique_id : 0) && + f.depth_clear_enable == depth->depth_load_clear_enable && + f.stencil_clear_enable == depth->stencil_clear_enable && + f.depth_read_only == depth_read_only) { + return f.framebuffer; + } + } + + auto* framebuffer = new VulkanFramebuffer; + framebuffer->render_pass = nullptr; + framebuffer->framebuffer = nullptr; + for (uint32_t i = 0; i < RENDER_COLOR_ATTACHMENTS_MAX; i++) { + framebuffer->color_layout[i] = color_layout[i]; + } + framebuffer->depth_layout = depth_layout; + + auto* gctx = g_render_ctx->GetGraphicCtx(); + + EXIT_IF(gctx == nullptr); + + if (!with_depth && color_count == 0) { + LOGF("Framebuffer: warning: no color or depth attachment\n"); + return nullptr; + } + + EXIT_NOT_IMPLEMENTED(!with_depth && color_count == 0); + EXIT_NOT_IMPLEMENTED(with_depth && first_color != nullptr && + (first_color_extent.width != depth->vulkan_buffer->extent.width || + first_color_extent.height != depth->vulkan_buffer->extent.height)); + + if (first_color == nullptr) { + first_color = CreateDummyBuffer(VK_FORMAT_B8G8R8A8_SRGB, depth->vulkan_buffer->extent.width, + depth->vulkan_buffer->extent.height); + first_color_extent = first_color->extent; + } + + VkAttachmentDescription attachments[RENDER_COLOR_ATTACHMENTS_MAX + 1] = {}; + for (uint32_t i = 0; i < color_count; i++) { + attachments[i].flags = 0; + attachments[i].format = colors[i].format; + attachments[i].samples = VK_SAMPLE_COUNT_1_BIT; + attachments[i].loadOp = (colors[i].color_clear_enable ? VK_ATTACHMENT_LOAD_OP_CLEAR + : VK_ATTACHMENT_LOAD_OP_LOAD); + attachments[i].storeOp = VK_ATTACHMENT_STORE_OP_STORE; + attachments[i].stencilLoadOp = VK_ATTACHMENT_LOAD_OP_DONT_CARE; + attachments[i].stencilStoreOp = VK_ATTACHMENT_STORE_OP_DONT_CARE; + attachments[i].initialLayout = color_layout[i]; + attachments[i].finalLayout = RENDER_COLOR_IMAGE_LAYOUT; + } + + const uint32_t depth_attachment = color_count; + attachments[depth_attachment].flags = 0; + attachments[depth_attachment].format = depth->format; + attachments[depth_attachment].samples = VK_SAMPLE_COUNT_1_BIT; + attachments[depth_attachment].loadOp = + (depth->depth_load_clear_enable ? VK_ATTACHMENT_LOAD_OP_CLEAR : VK_ATTACHMENT_LOAD_OP_LOAD); + attachments[depth_attachment].storeOp = VK_ATTACHMENT_STORE_OP_STORE; + attachments[depth_attachment].stencilLoadOp = + (depth->stencil_clear_enable ? VK_ATTACHMENT_LOAD_OP_CLEAR : VK_ATTACHMENT_LOAD_OP_LOAD); + attachments[depth_attachment].stencilStoreOp = VK_ATTACHMENT_STORE_OP_STORE; + attachments[depth_attachment].initialLayout = depth_layout; + attachments[depth_attachment].finalLayout = depth_layout; + + VkAttachmentReference color_attachment_ref[RENDER_COLOR_ATTACHMENTS_MAX] = {}; + for (uint32_t i = 0; i < RENDER_COLOR_ATTACHMENTS_MAX; i++) { + color_attachment_ref[i].attachment = (i < color_count ? i : VK_ATTACHMENT_UNUSED); + color_attachment_ref[i].layout = RENDER_COLOR_IMAGE_LAYOUT; + } + + VkAttachmentReference depth_attachment_ref {}; + depth_attachment_ref.attachment = depth_attachment; + depth_attachment_ref.layout = depth_layout; + + VkSubpassDescription subpass {}; + subpass.flags = 0; + subpass.pipelineBindPoint = VK_PIPELINE_BIND_POINT_GRAPHICS; + subpass.inputAttachmentCount = 0; + subpass.pInputAttachments = nullptr; + subpass.colorAttachmentCount = color_count; + subpass.pColorAttachments = (color_count > 0 ? color_attachment_ref : nullptr); + subpass.pResolveAttachments = nullptr; + subpass.pDepthStencilAttachment = (with_depth ? &depth_attachment_ref : nullptr); + subpass.preserveAttachmentCount = 0; + subpass.pPreserveAttachments = nullptr; + + const auto attachment_stage_mask = static_cast( + VK_PIPELINE_STAGE_COLOR_ATTACHMENT_OUTPUT_BIT | VK_PIPELINE_STAGE_EARLY_FRAGMENT_TESTS_BIT | + VK_PIPELINE_STAGE_LATE_FRAGMENT_TESTS_BIT); + const auto attachment_access_mask = static_cast( + VK_ACCESS_COLOR_ATTACHMENT_READ_BIT | VK_ACCESS_COLOR_ATTACHMENT_WRITE_BIT | + VK_ACCESS_DEPTH_STENCIL_ATTACHMENT_READ_BIT | VK_ACCESS_DEPTH_STENCIL_ATTACHMENT_WRITE_BIT); + + VkSubpassDependency dependencies[2] = {}; + dependencies[0].srcSubpass = VK_SUBPASS_EXTERNAL; + dependencies[0].dstSubpass = 0; + dependencies[0].srcStageMask = VK_PIPELINE_STAGE_ALL_COMMANDS_BIT; + dependencies[0].dstStageMask = attachment_stage_mask; + dependencies[0].srcAccessMask = VK_ACCESS_MEMORY_READ_BIT | VK_ACCESS_MEMORY_WRITE_BIT; + dependencies[0].dstAccessMask = attachment_access_mask; + dependencies[0].dependencyFlags = VK_DEPENDENCY_BY_REGION_BIT; + + dependencies[1].srcSubpass = 0; + dependencies[1].dstSubpass = VK_SUBPASS_EXTERNAL; + dependencies[1].srcStageMask = attachment_stage_mask; + dependencies[1].dstStageMask = VK_PIPELINE_STAGE_ALL_COMMANDS_BIT; + dependencies[1].srcAccessMask = attachment_access_mask; + dependencies[1].dstAccessMask = VK_ACCESS_MEMORY_READ_BIT | VK_ACCESS_MEMORY_WRITE_BIT; + dependencies[1].dependencyFlags = VK_DEPENDENCY_BY_REGION_BIT; + + VkRenderPassCreateInfo render_pass_info {}; + render_pass_info.sType = VK_STRUCTURE_TYPE_RENDER_PASS_CREATE_INFO; + render_pass_info.pNext = nullptr; + render_pass_info.flags = 0; + render_pass_info.attachmentCount = color_count + (with_depth ? 1u : 0u); + render_pass_info.pAttachments = attachments; + render_pass_info.subpassCount = 1; + render_pass_info.pSubpasses = &subpass; + render_pass_info.dependencyCount = 2; + render_pass_info.pDependencies = dependencies; + + auto result = + vkCreateRenderPass(gctx->device, &render_pass_info, nullptr, &framebuffer->render_pass); + EXIT_NOT_IMPLEMENTED(result != VK_SUCCESS); + + VkFormat color_formats[RENDER_COLOR_ATTACHMENTS_MAX] = {}; + for (uint32_t i = 0; i < color_count; i++) { + color_formats[i] = colors[i].format; + } + framebuffer->render_pass_id = + render_pass_compat_id(color_count, color_formats, with_depth, depth->format, depth_layout); + + EXIT_NOT_IMPLEMENTED(framebuffer->render_pass == nullptr); + + VkImageView views[RENDER_COLOR_ATTACHMENTS_MAX + 1] = {}; + for (uint32_t i = 0; i < color_count; i++) { + if (colors[i].vulkan_view == nullptr) { + EXIT("Framebuffer: color attachment view is missing at slot %u\n", i); + } + views[i] = colors[i].vulkan_view; + } + if (with_depth) { + views[color_count] = depth->vulkan_buffer->image_view[VulkanImage::VIEW_DEFAULT]; + } + + VkFramebufferCreateInfo framebuffer_info {}; + framebuffer_info.sType = VK_STRUCTURE_TYPE_FRAMEBUFFER_CREATE_INFO; + framebuffer_info.pNext = nullptr; + framebuffer_info.flags = 0; + framebuffer_info.renderPass = framebuffer->render_pass; + framebuffer_info.attachmentCount = color_count + (with_depth ? 1u : 0u); + framebuffer_info.pAttachments = views; + framebuffer_info.width = first_color_extent.width; + framebuffer_info.height = first_color_extent.height; + framebuffer_info.layers = 1; + + result = + vkCreateFramebuffer(gctx->device, &framebuffer_info, nullptr, &framebuffer->framebuffer); + EXIT_NOT_IMPLEMENTED(result != VK_SUCCESS); + + EXIT_NOT_IMPLEMENTED(framebuffer->framebuffer == nullptr); + + Framebuffer fnew; + fnew.framebuffer = framebuffer; + for (uint32_t i = 0; i < RENDER_COLOR_ATTACHMENTS_MAX; i++) { + fnew.image_id[i] = + (i < color_count && with_color[i] ? colors[i].vulkan_buffer->memory.unique_id : 0); + fnew.color_view[i] = (i < color_count && with_color[i] ? colors[i].vulkan_view : nullptr); + fnew.color_clear_enable[i] = + (i < color_count && with_color[i] && colors[i].color_clear_enable); + fnew.color_layout[i] = color_layout[i]; + } + fnew.depth_id = (with_depth ? depth->vulkan_buffer->memory.unique_id : 0); + fnew.depth_clear_enable = depth->depth_load_clear_enable; + fnew.stencil_clear_enable = depth->stencil_clear_enable; + fnew.depth_read_only = depth_read_only; + + bool updated = false; + + for (auto& f: m_framebuffers) { + if (f.framebuffer == nullptr) { + f = fnew; + updated = true; + break; + } + } + + if (!updated) { + m_framebuffers.push_back(fnew); + } + + return framebuffer; +} + +void FramebufferCache::FreeFramebufferByColor(VulkanImage* image) { + EXIT_IF(g_render_ctx == nullptr); + EXIT_IF(image == nullptr); + + Common::LockGuard lock(m_mutex); + + for (auto& f: m_framebuffers) { + bool uses_image = false; + for (auto image_id: f.image_id) { + if (image_id == image->memory.unique_id) { + uses_image = true; + break; + } + } + if (f.framebuffer != nullptr && uses_image) { + auto* gctx = g_render_ctx->GetGraphicCtx(); + + EXIT_IF(gctx == nullptr); + + vkDestroyFramebuffer(gctx->device, f.framebuffer->framebuffer, nullptr); + + vkDestroyRenderPass(gctx->device, f.framebuffer->render_pass, nullptr); + + delete f.framebuffer; + + f.framebuffer = nullptr; + } + } +} + +void FramebufferCache::FreeFramebufferByDepth(DepthStencilVulkanImage* image) { + EXIT_IF(g_render_ctx == nullptr); + EXIT_IF(image == nullptr); + + Common::LockGuard lock(m_mutex); + + for (auto& f: m_framebuffers) { + if (f.framebuffer != nullptr && f.depth_id == image->memory.unique_id) { + auto* gctx = g_render_ctx->GetGraphicCtx(); + + EXIT_IF(gctx == nullptr); + + vkDestroyFramebuffer(gctx->device, f.framebuffer->framebuffer, nullptr); + + vkDestroyRenderPass(gctx->device, f.framebuffer->render_pass, nullptr); + + delete f.framebuffer; + + f.framebuffer = nullptr; + } + } +} + +VideoOutVulkanImage* FramebufferCache::CreateDummyBuffer(VkFormat format, uint32_t width, + uint32_t height) { + for (auto* b: m_dummy_buffers) { + if (b->extent.width == width && b->extent.height == height && b->format == format) { + return b; + } + } + + auto* ctx = g_render_ctx->GetGraphicCtx(); + + EXIT_IF(ctx == nullptr); + + auto* vk_obj = new VideoOutVulkanImage; + + vk_obj->extent.width = width; + vk_obj->extent.height = height; + vk_obj->format = format; + vk_obj->image = nullptr; + vk_obj->layout = VK_IMAGE_LAYOUT_UNDEFINED; + + UtilResetImageViews(vk_obj); + + VkImageCreateInfo image_info {}; + image_info.sType = VK_STRUCTURE_TYPE_IMAGE_CREATE_INFO; + image_info.pNext = nullptr; + image_info.flags = 0; + image_info.imageType = VK_IMAGE_TYPE_2D; + image_info.extent.width = vk_obj->extent.width; + image_info.extent.height = vk_obj->extent.height; + image_info.extent.depth = 1; + image_info.mipLevels = 1; + image_info.arrayLayers = 1; + image_info.format = vk_obj->format; + image_info.tiling = VK_IMAGE_TILING_OPTIMAL; + image_info.initialLayout = vk_obj->layout; + image_info.usage = VK_IMAGE_USAGE_COLOR_ATTACHMENT_BIT; + image_info.sharingMode = VK_SHARING_MODE_EXCLUSIVE; + image_info.samples = VK_SAMPLE_COUNT_1_BIT; + + VulkanMemory mem; + + mem.property = VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT; + + bool created = VulkanCreateImage(ctx, &image_info, vk_obj, &mem); + EXIT_NOT_IMPLEMENTED(!created); + + vk_obj->memory = mem; + + VkImageViewCreateInfo create_info {}; + create_info.sType = VK_STRUCTURE_TYPE_IMAGE_VIEW_CREATE_INFO; + create_info.pNext = nullptr; + create_info.flags = 0; + create_info.image = vk_obj->image; + create_info.viewType = VK_IMAGE_VIEW_TYPE_2D; + create_info.format = vk_obj->format; + create_info.components.r = VK_COMPONENT_SWIZZLE_IDENTITY; + create_info.components.g = VK_COMPONENT_SWIZZLE_IDENTITY; + create_info.components.b = VK_COMPONENT_SWIZZLE_IDENTITY; + create_info.components.a = VK_COMPONENT_SWIZZLE_IDENTITY; + create_info.subresourceRange.aspectMask = VK_IMAGE_ASPECT_COLOR_BIT; + create_info.subresourceRange.baseArrayLayer = 0; + create_info.subresourceRange.baseMipLevel = 0; + create_info.subresourceRange.layerCount = 1; + create_info.subresourceRange.levelCount = 1; + + vkCreateImageView(ctx->device, &create_info, nullptr, + &vk_obj->image_view[VulkanImage::VIEW_DEFAULT]); + + EXIT_NOT_IMPLEMENTED(vk_obj->image_view[VulkanImage::VIEW_DEFAULT] == nullptr); + + UtilSetImageLayoutOptimal(vk_obj); + + m_dummy_buffers.push_back(vk_obj); + + return vk_obj; +} + +} // namespace Libs::Graphics diff --git a/src/graphics/host_gpu/renderer/framebufferCache.h b/src/graphics/host_gpu/renderer/framebufferCache.h new file mode 100644 index 000000000..64e36abac --- /dev/null +++ b/src/graphics/host_gpu/renderer/framebufferCache.h @@ -0,0 +1,80 @@ +#ifndef EMULATOR_SRC_GRAPHICS_HOST_GPU_RENDERER_FRAMEBUFFERCACHE_H_ +#define EMULATOR_SRC_GRAPHICS_HOST_GPU_RENDERER_FRAMEBUFFERCACHE_H_ + +#include "common/abi.h" +#include "common/assert.h" +#include "common/common.h" +#include "common/threads.h" +#include "graphics/host_gpu/graphicContext.h" +#include "graphics/host_gpu/renderer/renderState.h" + +#include +#include + +namespace Libs::Graphics { + +static constexpr VkImageLayout RENDER_COLOR_IMAGE_LAYOUT = VK_IMAGE_LAYOUT_GENERAL; + +struct VulkanFramebuffer { + VkRenderPass render_pass = nullptr; + uint64_t render_pass_id = 0; + VkFramebuffer framebuffer = nullptr; + VkImageLayout color_layout[RENDER_COLOR_ATTACHMENTS_MAX] = {}; + VkImageLayout depth_layout = VK_IMAGE_LAYOUT_DEPTH_STENCIL_ATTACHMENT_OPTIMAL; +}; + +inline uint64_t render_pass_compat_id(uint32_t color_count, const VkFormat* color_formats, + bool with_depth, VkFormat depth_format, + VkImageLayout depth_layout) { + uint64_t id = 0xcbf29ce484222325ull; + auto mix = [&id](uint64_t v) { + id ^= v; + id *= 0x100000001b3ull; + }; + + mix(color_count); + for (uint32_t i = 0; i < color_count; i++) { + mix(static_cast(color_formats[i])); + } + mix(with_depth ? 1u : 0u); + mix(static_cast(depth_format)); + mix(static_cast(depth_layout)); + mix(VK_SAMPLE_COUNT_1_BIT); + + return id; +} + +class FramebufferCache { +public: + FramebufferCache() { EXIT_NOT_IMPLEMENTED(!Common::Thread::IsMainThread()); } + virtual ~FramebufferCache() { KYTY_NOT_IMPLEMENTED; } + KYTY_CLASS_NO_COPY(FramebufferCache); + + VulkanFramebuffer* CreateFramebuffer(RenderColorInfo* colors, uint32_t color_count, + RenderDepthInfo* depth); + void FreeFramebufferByColor(VulkanImage* image); + void FreeFramebufferByDepth(DepthStencilVulkanImage* image); + +private: + VideoOutVulkanImage* CreateDummyBuffer(VkFormat format, uint32_t width, uint32_t height); + + struct Framebuffer { + VulkanFramebuffer* framebuffer = nullptr; + uint64_t image_id[RENDER_COLOR_ATTACHMENTS_MAX] = {}; + VkImageView color_view[RENDER_COLOR_ATTACHMENTS_MAX] = {}; + uint64_t depth_id = 0; + bool color_clear_enable[RENDER_COLOR_ATTACHMENTS_MAX] = {}; + VkImageLayout color_layout[RENDER_COLOR_ATTACHMENTS_MAX] = {}; + bool depth_clear_enable = false; + bool stencil_clear_enable = false; + bool depth_read_only = false; + }; + + Common::Mutex m_mutex; + std::vector m_framebuffers; + std::vector m_dummy_buffers; +}; + +} // namespace Libs::Graphics + +#endif // EMULATOR_SRC_GRAPHICS_HOST_GPU_RENDERER_FRAMEBUFFERCACHE_H_ diff --git a/src/graphics/host_gpu/renderer/gdsBuffer.cpp b/src/graphics/host_gpu/renderer/gdsBuffer.cpp new file mode 100644 index 000000000..f389905e6 --- /dev/null +++ b/src/graphics/host_gpu/renderer/gdsBuffer.cpp @@ -0,0 +1,70 @@ +#include "graphics/host_gpu/renderer/gdsBuffer.h" + +#include "common/assert.h" +#include "graphics/host_gpu/utils.h" +#include "graphics/host_gpu/vma.h" + +namespace Libs::Graphics { + +void GdsBuffer::Init(GraphicContext* ctx) { + if (m_buffer == nullptr) { + m_buffer = std::make_unique(); + + m_buffer->usage = VK_BUFFER_USAGE_STORAGE_BUFFER_BIT; + m_buffer->memory.property = static_cast(VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT) | + VK_MEMORY_PROPERTY_HOST_COHERENT_BIT | + VK_MEMORY_PROPERTY_HOST_CACHED_BIT; + VulkanCreateBuffer(ctx, DW_SIZE * 4, m_buffer.get()); + } +} + +void GdsBuffer::Clear(GraphicContext* ctx, uint64_t dw_offset, uint32_t dw_num, + uint32_t clear_value) { + EXIT_IF(ctx == nullptr); + + Common::LockGuard lock(m_mutex); + + Init(ctx); + + EXIT_NOT_IMPLEMENTED(dw_offset >= DW_SIZE); + EXIT_NOT_IMPLEMENTED(dw_offset + dw_num > DW_SIZE); + + void* data = nullptr; + VulkanMapMemory(ctx, &m_buffer->memory, &data); + + for (uint32_t i = 0; i < dw_num; i++) { + static_cast(data)[dw_offset + i] = clear_value; + } + + VulkanUnmapMemory(ctx, &m_buffer->memory); +} + +void GdsBuffer::Read(GraphicContext* ctx, uint32_t* dst, uint32_t dw_offset, uint32_t dw_size) { + EXIT_IF(dst == nullptr); + + Common::LockGuard lock(m_mutex); + + Init(ctx); + + EXIT_NOT_IMPLEMENTED(dw_offset >= DW_SIZE); + EXIT_NOT_IMPLEMENTED(dw_offset + dw_size > DW_SIZE); + + void* data = nullptr; + VulkanMapMemory(ctx, &m_buffer->memory, &data); + + for (uint32_t i = 0; i < dw_size; i++) { + dst[i] = static_cast(data)[dw_offset + i]; + } + + VulkanUnmapMemory(ctx, &m_buffer->memory); +} + +VulkanBuffer* GdsBuffer::GetBuffer(GraphicContext* ctx) { + Common::LockGuard lock(m_mutex); + + Init(ctx); + + return m_buffer.get(); +} + +} // namespace Libs::Graphics diff --git a/src/graphics/host_gpu/renderer/gdsBuffer.h b/src/graphics/host_gpu/renderer/gdsBuffer.h new file mode 100644 index 000000000..872d7fd9b --- /dev/null +++ b/src/graphics/host_gpu/renderer/gdsBuffer.h @@ -0,0 +1,36 @@ +#ifndef EMULATOR_SRC_GRAPHICS_HOST_GPU_RENDERER_GDSBUFFER_H_ +#define EMULATOR_SRC_GRAPHICS_HOST_GPU_RENDERER_GDSBUFFER_H_ + +#include "common/abi.h" +#include "common/assert.h" +#include "common/common.h" +#include "common/threads.h" +#include "graphics/host_gpu/graphicContext.h" + +#include + +namespace Libs::Graphics { + +class GdsBuffer { +public: + GdsBuffer() { EXIT_NOT_IMPLEMENTED(!Common::Thread::IsMainThread()); } + virtual ~GdsBuffer() { KYTY_NOT_IMPLEMENTED; } + KYTY_CLASS_NO_COPY(GdsBuffer); + + void Clear(GraphicContext* ctx, uint64_t dw_offset, uint32_t dw_num, uint32_t clear_value); + void Read(GraphicContext* ctx, uint32_t* dst, uint32_t dw_offset, uint32_t dw_size); + + VulkanBuffer* GetBuffer(GraphicContext* ctx); + +private: + static constexpr uint64_t DW_SIZE = 0x3000; + + void Init(GraphicContext* ctx); + + Common::Mutex m_mutex; + std::unique_ptr m_buffer; +}; + +} // namespace Libs::Graphics + +#endif // EMULATOR_SRC_GRAPHICS_HOST_GPU_RENDERER_GDSBUFFER_H_ diff --git a/src/graphics/host_gpu/renderer/gpuResourceManager.cpp b/src/graphics/host_gpu/renderer/gpuResourceManager.cpp new file mode 100644 index 000000000..cb27702a5 --- /dev/null +++ b/src/graphics/host_gpu/renderer/gpuResourceManager.cpp @@ -0,0 +1,120 @@ +#include "graphics/host_gpu/renderer/gpuResourceManager.h" + +#include "common/assert.h" +#include "graphics/guest_gpu/graphicsRun.h" +#include "graphics/host_gpu/objects/label.h" +#include "graphics/host_gpu/renderer/render.h" +#include "graphics/host_gpu/renderer/renderContext.h" + +namespace Libs::Graphics { + +GpuResourceManager::GpuResourceManager() + : m_page_manager(FaultThunk, this), m_buffer_cache(m_page_manager, m_resource_mutex), + m_texture_cache(m_page_manager, m_buffer_cache, m_resource_mutex) { + m_buffer_cache.SetTextureCache(m_texture_cache); +} + +GpuResourceManager::~GpuResourceManager() = default; + +bool GpuResourceManager::FaultThunk(void* context, PageFaultAccess access, uint64_t vaddr, + uint64_t size, PageFaultPhase phase) noexcept { + return static_cast(context)->InvalidateMemory(access, vaddr, size, phase); +} + +bool GpuResourceManager::InvalidateMemory(PageFaultAccess access, uint64_t vaddr, uint64_t size, + PageFaultPhase phase) noexcept { + const bool buffer_handled = m_buffer_cache.InvalidateMemory(access, vaddr, size, phase); + const bool image_handled = m_texture_cache.InvalidateMemory(access, vaddr, size, phase); + return buffer_handled || image_handled; +} + +bool GpuResourceManager::HandleFault(PageFaultAccess access, uint64_t fault_vaddr) noexcept { + if (!m_page_manager.IsMapped(fault_vaddr, 1)) { + return false; + } + if (LabelInCallback()) { + EXIT("unsupported guest-memory fault from an asynchronous GPU label callback, " + "addr=0x%016" PRIx64 " access=%u\n", + fault_vaddr, static_cast(access)); + } + if (m_resource_mutex.IsOwnedByCurrentThread()) { + if (!GraphicsRunIsCommandProcessorGuestAccess()) { + EXIT("unsupported page fault from a pre-owned resource transaction, addr=0x%016" PRIx64 + " access=%u\n", + fault_vaddr, static_cast(access)); + } + ResourceMutex::FaultScope fault(m_resource_mutex); + return m_page_manager.HandleFault(access, fault_vaddr); + } + // Stop command-processor jobs before taking the shared cache transaction. External readback + // workers inherit this paused state and therefore never form resource -> submission lock inversion. + GraphicsRunSubmissionLock submissions; + ResourceMutex::FaultScope fault(m_resource_mutex); + return m_page_manager.HandleFault(access, fault_vaddr); +} + +bool GpuResourceManager::IsMapped(uint64_t vaddr, uint64_t size) const noexcept { + return m_page_manager.IsMapped(vaddr, size); +} + +std::unique_lock GpuResourceManager::LockCommandProcessorGuestAccess( + uint64_t vaddr, uint64_t size) { + if (vaddr == 0 || size == 0 || vaddr >= TRACKER_ADDRESS_SIZE || + size > TRACKER_ADDRESS_SIZE - vaddr) { + EXIT("invalid command-processor guest-memory access, addr=0x%016" PRIx64 + " size=0x%016" PRIx64 "\n", + vaddr, size); + } + return std::unique_lock(m_resource_mutex); +} + +std::unique_lock GpuResourceManager::LockAsynchronousGuestWrite(uint64_t vaddr, + uint64_t size) { + if (vaddr == 0 || size == 0 || vaddr >= TRACKER_ADDRESS_SIZE || + size > TRACKER_ADDRESS_SIZE - vaddr) { + EXIT("invalid asynchronous guest-memory write, addr=0x%016" PRIx64 + " size=0x%016" PRIx64 "\n", + vaddr, size); + } + std::unique_lock transaction(m_resource_mutex); + if (m_buffer_cache.HasPageOverlap(vaddr, size) || m_texture_cache.HasPageOverlap(vaddr, size) || + m_texture_cache.HasMetaOverlap(vaddr, size)) { + EXIT("asynchronous guest-memory write to a cached buffer/image range is unsupported, " + "addr=0x%016" PRIx64 " size=0x%016" PRIx64 "\n", + vaddr, size); + } + return transaction; +} + +void GpuResourceManager::MapMemory(uint64_t vaddr, uint64_t size, GpuAccess access) { + m_page_manager.OnGpuMap(vaddr, size, access); +} + +void GpuResourceManager::UnmapMemory(uint64_t vaddr, uint64_t size, GpuAccess access) { + if (!IsMapped(vaddr, size)) { + EXIT("cannot unmap an unmapped GPU resource range\n"); + } + m_texture_cache.UnmapMemory(vaddr, size); + m_buffer_cache.UnmapMemory(vaddr, size); + m_page_manager.OnGpuUnmap(vaddr, size, access); +} + +void GpuResourceManager::FillBuffer(CommandBuffer* command, uint64_t vaddr, uint64_t size, + uint32_t value) { + if (g_render_ctx == nullptr || command == nullptr || command->IsInvalid()) { + EXIT("cannot fill a buffer without a valid render command context\n"); + } + Common::LockGuard lock(g_render_ctx->GetMutex()); + m_buffer_cache.FillBuffer(command, g_render_ctx->GetGraphicCtx(), vaddr, size, value); +} + +void GpuResourceManager::CopyBuffer(CommandBuffer* command, uint64_t dst_vaddr, uint64_t src_vaddr, + uint64_t size) { + if (g_render_ctx == nullptr || command == nullptr || command->IsInvalid()) { + EXIT("cannot copy a buffer without a valid render command context\n"); + } + Common::LockGuard lock(g_render_ctx->GetMutex()); + m_buffer_cache.CopyBuffer(command, g_render_ctx->GetGraphicCtx(), dst_vaddr, src_vaddr, size); +} + +} // namespace Libs::Graphics diff --git a/src/graphics/host_gpu/renderer/gpuResourceManager.h b/src/graphics/host_gpu/renderer/gpuResourceManager.h new file mode 100644 index 000000000..ead9e5e9c --- /dev/null +++ b/src/graphics/host_gpu/renderer/gpuResourceManager.h @@ -0,0 +1,56 @@ +#ifndef EMULATOR_SRC_GRAPHICS_HOST_GPU_RENDERER_GPURESOURCEMANAGER_H_ +#define EMULATOR_SRC_GRAPHICS_HOST_GPU_RENDERER_GPURESOURCEMANAGER_H_ + +#include "common/abi.h" +#include "common/common.h" +#include "graphics/host_gpu/pageManager.h" +#include "graphics/host_gpu/renderer/bufferCache.h" +#include "graphics/host_gpu/renderer/resourceMutex.h" +#include "graphics/host_gpu/renderer/textureCache.h" + +#include +#include + +namespace Libs::Graphics { + +class CommandBuffer; + +class GpuResourceManager { +public: + GpuResourceManager(); + ~GpuResourceManager(); + KYTY_CLASS_NO_COPY(GpuResourceManager); + + [[nodiscard]] BufferCache* GetBufferCache() { return &m_buffer_cache; } + [[nodiscard]] TextureCache* GetTextureCache() { return &m_texture_cache; } + + [[nodiscard]] bool HandleFault(PageFaultAccess access, uint64_t fault_vaddr) noexcept; + [[nodiscard]] bool IsMapped(uint64_t vaddr, uint64_t size) const noexcept; + // Serializes synchronous command-processor guest-memory access against buffer/image ownership + // publication. Address metadata keeps this seam ready for a future page-range lock. + [[nodiscard]] std::unique_lock + LockCommandProcessorGuestAccess(uint64_t vaddr, uint64_t size); + // Keeps an asynchronous host callback's validation and guest write in one cache transaction. + // Cached buffer/image destinations are unsupported and fail before the callback can fault. + [[nodiscard]] std::unique_lock LockAsynchronousGuestWrite(uint64_t vaddr, + uint64_t size); + void MapMemory(uint64_t vaddr, uint64_t size, GpuAccess access); + void UnmapMemory(uint64_t vaddr, uint64_t size, GpuAccess access); + void FillBuffer(CommandBuffer* command, uint64_t vaddr, uint64_t size, uint32_t value); + void CopyBuffer(CommandBuffer* command, uint64_t dst_vaddr, uint64_t src_vaddr, uint64_t size); + +private: + static bool FaultThunk(void* context, PageFaultAccess access, uint64_t vaddr, uint64_t size, + PageFaultPhase phase) noexcept; + [[nodiscard]] bool InvalidateMemory(PageFaultAccess access, uint64_t vaddr, uint64_t size, + PageFaultPhase phase) noexcept; + + PageManager m_page_manager; + ResourceMutex m_resource_mutex; + BufferCache m_buffer_cache; + TextureCache m_texture_cache; +}; + +} // namespace Libs::Graphics + +#endif // EMULATOR_SRC_GRAPHICS_HOST_GPU_RENDERER_GPURESOURCEMANAGER_H_ diff --git a/src/graphics/host_gpu/renderer/image.h b/src/graphics/host_gpu/renderer/image.h new file mode 100644 index 000000000..8099e204e --- /dev/null +++ b/src/graphics/host_gpu/renderer/image.h @@ -0,0 +1,74 @@ +#ifndef EMULATOR_SRC_GRAPHICS_HOST_GPU_RENDERER_IMAGE_H_ +#define EMULATOR_SRC_GRAPHICS_HOST_GPU_RENDERER_IMAGE_H_ + +#include "common/assert.h" +#include "graphics/host_gpu/renderer/imageInfo.h" + +#include +#include + +namespace Libs::Graphics { + +struct Image final: ImageInfo { + Image& operator=(const ImageInfo& value) { + if (m_cpu_dirty) { + EXIT("dirty sampled image cannot be reassigned\n"); + } + static_cast(*this) = value; + return *this; + } + + void InvalidateCpuWrite(uint64_t vaddr, uint64_t size) { + if (ImagePageRangesOverlap(address, this->size, vaddr, size)) { + m_cpu_dirty = true; + } + } + + [[nodiscard]] bool IsCpuDirty() const { return m_cpu_dirty; } + + void RefreshComplete() { + if (!m_cpu_dirty) { + EXIT("clean sampled image cannot complete a refresh\n"); + } + m_cpu_dirty = false; + } + +private: + bool m_cpu_dirty = false; +}; + +struct ImageRetirementRange { + uint64_t address = 0; + uint64_t size = 0; + bool retire = false; +}; + +struct ImageRetirementConflict { + size_t retired = SIZE_MAX; + size_t retained = SIZE_MAX; + + [[nodiscard]] bool Exists() const { return retired != SIZE_MAX; } +}; + +[[nodiscard]] inline ImageRetirementConflict +FindImageRetirementConflict(std::span ranges) { + for (size_t retired = 0; retired < ranges.size(); retired++) { + if (!ranges[retired].retire) { + continue; + } + for (size_t retained = 0; retained < ranges.size(); retained++) { + if (ranges[retained].retire) { + continue; + } + if (ImagePageRangesOverlap(ranges[retired].address, ranges[retired].size, + ranges[retained].address, ranges[retained].size)) { + return {retired, retained}; + } + } + } + return {}; +} + +} // namespace Libs::Graphics + +#endif // EMULATOR_SRC_GRAPHICS_HOST_GPU_RENDERER_IMAGE_H_ diff --git a/src/graphics/host_gpu/renderer/imageInfo.h b/src/graphics/host_gpu/renderer/imageInfo.h new file mode 100644 index 000000000..695c8cf36 --- /dev/null +++ b/src/graphics/host_gpu/renderer/imageInfo.h @@ -0,0 +1,487 @@ +#ifndef EMULATOR_SRC_GRAPHICS_HOST_GPU_RENDERER_IMAGEINFO_H_ +#define EMULATOR_SRC_GRAPHICS_HOST_GPU_RENDERER_IMAGEINFO_H_ + +#include "common/assert.h" +#include "graphics/guest_gpu/gpu_defs.h" +#include "graphics/host_gpu/regionDefinitions.h" + +#include +#include +#include +#include + +namespace Libs::Graphics { + +struct ImageInfo { + uint64_t address = 0; + uint64_t size = 0; + uint32_t format = 0; + uint32_t width = 0; + uint32_t height = 0; + uint32_t pitch = 0; + uint32_t base_level = 0; + uint32_t levels = 1; + uint32_t view_levels = 1; + uint32_t tile = 0; + uint32_t swizzle = 0; + uint32_t depth = 1; + uint32_t type = 0; + uint32_t base_array = 0; +}; + +struct RenderTargetInfo { + uint64_t address = 0; + uint64_t size = 0; + VkFormat format = VK_FORMAT_UNDEFINED; + uint32_t width = 0; + uint32_t height = 0; + uint32_t pitch = 0; + uint32_t bytes_per_element = 0; + uint32_t tile_mode = 0; + uint32_t levels = 1; +}; + +// Common image-to-buffer copy description. Storage images and render targets keep distinct +// cache records today, but downloads should consume one normalized layout just as uploads do. +struct ColorImageTransferInfo { + uint64_t address = 0; + uint64_t size = 0; + VkFormat format = VK_FORMAT_UNDEFINED; + uint32_t width = 0; + uint32_t height = 0; + uint32_t pitch = 0; + uint32_t bytes_per_element = 0; + uint32_t tile_mode = 0; + uint32_t levels = 1; +}; + +[[nodiscard]] inline ColorImageTransferInfo +MakeColorImageTransferInfo(const ImageInfo& info, VkFormat format, + uint32_t bytes_per_element) noexcept { + return {info.address, info.size, format, info.width, info.height, + info.pitch, bytes_per_element, info.tile, info.levels}; +} + +[[nodiscard]] inline ColorImageTransferInfo +MakeColorImageTransferInfo(const RenderTargetInfo& info) noexcept { + return { + info.address, info.size, info.format, info.width, info.height, info.pitch, + info.bytes_per_element, info.tile_mode, info.levels}; +} + +struct DepthTargetInfo { + uint64_t address = 0; + uint64_t size = 0; + uint64_t stencil_address = 0; + uint64_t stencil_size = 0; + uint64_t htile_address = 0; + uint64_t htile_size = 0; + VkFormat format = VK_FORMAT_UNDEFINED; + uint32_t guest_format = 0; + uint32_t width = 0; + uint32_t height = 0; + uint32_t pitch = 0; + uint32_t bytes_per_element = 0; + uint32_t tile_mode = 0; + bool depth_load_clear = false; + bool stencil_load_clear = false; + bool stencil_access = false; +}; + +enum class VideoOutCompression : uint8_t { Uncompressed, Dcc256_64_64, Unsupported }; + +struct VideoOutInfo { + uint64_t address = 0; + uint64_t size = 0; + uint64_t metadata_address = 0; + VkFormat format = VK_FORMAT_UNDEFINED; + uint32_t guest_format = 0; + uint32_t width = 0; + uint32_t height = 0; + uint32_t pitch = 0; + uint32_t bytes_per_element = 0; + uint32_t tile_mode = 0; + uint32_t dcc_control = 0; + VideoOutCompression compression = VideoOutCompression::Unsupported; +}; + +[[nodiscard]] inline VideoOutCompression +ClassifyVideoOutCompression(bool compressed, uint64_t metadata_address, uint32_t dcc_control, + uint64_t dcc_clear_color) noexcept { + constexpr uint32_t VIDEO_OUT_DCC_CONTROL_256_64_64 = 0x00000208u; + if (!compressed) { + return metadata_address == 0 && dcc_control == 0 && dcc_clear_color == 0 + ? VideoOutCompression::Uncompressed + : VideoOutCompression::Unsupported; + } + return metadata_address != 0 && (metadata_address & 0xffu) == 0 && + dcc_control == VIDEO_OUT_DCC_CONTROL_256_64_64 && dcc_clear_color == 0 + ? VideoOutCompression::Dcc256_64_64 + : VideoOutCompression::Unsupported; +} + +[[nodiscard]] inline constexpr bool +CanUseVideoOutNativeWithoutUpload(VideoOutCompression compression, bool render_target, + bool gpu_modified, bool guest_modified) noexcept { + return compression == VideoOutCompression::Dcc256_64_64 && !guest_modified && + (render_target || gpu_modified); +} + +struct VideoOutPixelFormatInfo { + VkFormat format = VK_FORMAT_UNDEFINED; + uint32_t guest_format = 0; +}; + +[[nodiscard]] inline bool DecodeVideoOutPixelFormat(uint64_t pixel_format, + VideoOutPixelFormatInfo* info) { + if (info == nullptr) { + return false; + } + VideoOutPixelFormatInfo next {}; + switch (pixel_format) { + case 0x8000000022000000ull: // SCE_VIDEO_OUT_PIXEL_FORMAT2_R8_G8_B8_A8_SRGB + next.format = VK_FORMAT_R8G8B8A8_SRGB; + next.guest_format = Prospero::GpuEnumValue(Prospero::BufferFormat::k8_8_8_8Srgb); + break; + case 0x8000000000000000ull: // SCE_VIDEO_OUT_PIXEL_FORMAT2_B8_G8_R8_A8_SRGB + next.format = VK_FORMAT_B8G8R8A8_SRGB; + next.guest_format = Prospero::GpuEnumValue(Prospero::BufferFormat::k8_8_8_8Srgb); + break; + case 0x8100000022000000ull: // SCE_VIDEO_OUT_PIXEL_FORMAT2_R10_G10_B10_A2_SRGB + next.format = VK_FORMAT_A2B10G10R10_UNORM_PACK32; + next.guest_format = Prospero::GpuEnumValue(Prospero::BufferFormat::k10_10_10_2UNorm); + break; + case 0x8100000000000000ull: // SCE_VIDEO_OUT_PIXEL_FORMAT2_B10_G10_R10_A2_SRGB + next.format = VK_FORMAT_A2R10G10B10_UNORM_PACK32; + next.guest_format = Prospero::GpuEnumValue(Prospero::BufferFormat::k10_10_10_2UNorm); + break; + default: return false; + } + *info = next; + return true; +} + +[[nodiscard]] inline bool IsSupportedVideoOutFormat(uint32_t guest_format, VkFormat format) { + const bool rgba8 = guest_format == Prospero::GpuEnumValue(Prospero::BufferFormat::k8_8_8_8Srgb) && + (format == VK_FORMAT_R8G8B8A8_SRGB || format == VK_FORMAT_B8G8R8A8_SRGB); + const bool rgb10 = guest_format == Prospero::GpuEnumValue(Prospero::BufferFormat::k10_10_10_2UNorm) && + (format == VK_FORMAT_A2B10G10R10_UNORM_PACK32 || + format == VK_FORMAT_A2R10G10B10_UNORM_PACK32); + return rgba8 || rgb10; +} + +enum class DepthOverlap : uint8_t { None, RetireSampled, Unsupported }; +enum class RenderTargetOverlap : uint8_t { None, RetireSampled, RetireTarget, Unsupported }; +enum class SampledOverlap : uint8_t { None, ReadOnlyAlias, Unsupported }; +enum class StorageSampledOverlap : uint8_t { None, ExactImage, Unsupported }; +enum class HostWriteOverlap : uint8_t { None, InvalidateImage, Unsupported }; +enum class BufferImageAlias : uint8_t { None, VideoOutWrite, RenderTargetWrite, Unsupported }; +enum class MetaImageOverlap : uint8_t { RetainSampled, RetireTarget, Unsupported }; + +[[nodiscard]] inline MetaImageOverlap ClassifyMetaImageOverlap(bool sampled, bool render_target, + bool gpu_modified, + bool buffer_modified) { + if (sampled && !gpu_modified) { + return MetaImageOverlap::RetainSampled; + } + if (render_target && !gpu_modified && !buffer_modified) { + return MetaImageOverlap::RetireTarget; + } + return MetaImageOverlap::Unsupported; +} + +[[nodiscard]] inline float DecodeSrgbClearComponent(uint32_t value) { + const auto encoded = static_cast(value & 0xffu) / 255.0f; + return encoded <= 0.04045f ? encoded / 12.92f : std::pow((encoded + 0.055f) / 1.055f, 2.4f); +} + +[[nodiscard]] inline bool DecodePackedColorClear(VkFormat format, uint32_t packed, + VkClearColorValue* clear) { + if (clear == nullptr) { + return false; + } + VkClearColorValue next {}; + const auto unorm8 = [](uint32_t value) { return static_cast(value & 0xffu) / 255.0f; }; + switch (format) { + case VK_FORMAT_R8G8B8A8_SRGB: + next.float32[0] = DecodeSrgbClearComponent(packed); + next.float32[1] = DecodeSrgbClearComponent(packed >> 8u); + next.float32[2] = DecodeSrgbClearComponent(packed >> 16u); + next.float32[3] = unorm8(packed >> 24u); + break; + case VK_FORMAT_B8G8R8A8_SRGB: + next.float32[0] = DecodeSrgbClearComponent(packed >> 16u); + next.float32[1] = DecodeSrgbClearComponent(packed >> 8u); + next.float32[2] = DecodeSrgbClearComponent(packed); + next.float32[3] = unorm8(packed >> 24u); + break; + case VK_FORMAT_R8G8B8A8_UNORM: + next.float32[0] = unorm8(packed); + next.float32[1] = unorm8(packed >> 8u); + next.float32[2] = unorm8(packed >> 16u); + next.float32[3] = unorm8(packed >> 24u); + break; + case VK_FORMAT_B8G8R8A8_UNORM: + next.float32[0] = unorm8(packed >> 16u); + next.float32[1] = unorm8(packed >> 8u); + next.float32[2] = unorm8(packed); + next.float32[3] = unorm8(packed >> 24u); + break; + case VK_FORMAT_A2B10G10R10_UNORM_PACK32: + next.float32[0] = static_cast(packed & 0x3ffu) / 1023.0f; + next.float32[1] = static_cast((packed >> 10u) & 0x3ffu) / 1023.0f; + next.float32[2] = static_cast((packed >> 20u) & 0x3ffu) / 1023.0f; + next.float32[3] = static_cast((packed >> 30u) & 0x3u) / 3.0f; + break; + case VK_FORMAT_A2R10G10B10_UNORM_PACK32: + next.float32[0] = static_cast((packed >> 20u) & 0x3ffu) / 1023.0f; + next.float32[1] = static_cast((packed >> 10u) & 0x3ffu) / 1023.0f; + next.float32[2] = static_cast(packed & 0x3ffu) / 1023.0f; + next.float32[3] = static_cast((packed >> 30u) & 0x3u) / 3.0f; + break; + default: return false; + } + *clear = next; + return true; +} + +[[nodiscard]] inline bool DecodePackedStencilClear(uint32_t packed, uint8_t* clear) { + if (clear == nullptr) { + return false; + } + const auto value = static_cast(packed); + if (packed != static_cast(value) * 0x01010101u) { + return false; + } + *clear = value; + return true; +} + +[[nodiscard]] inline bool DecodePackedDepthClear(VkFormat format, uint32_t packed, float* clear) { + if (clear == nullptr || + (format != VK_FORMAT_D32_SFLOAT && format != VK_FORMAT_D32_SFLOAT_S8_UINT)) { + return false; + } + const auto value = std::bit_cast(packed); + if (!std::isfinite(value) || value < 0.0f || value > 1.0f) { + return false; + } + *clear = value; + return true; +} + +[[nodiscard]] inline bool CanNativeClearDepthFromBuffer(const DepthTargetInfo& target, + uint64_t address, uint64_t size) { + const bool d32 = + target.format == VK_FORMAT_D32_SFLOAT || target.format == VK_FORMAT_D32_SFLOAT_S8_UINT; + return address == target.address && size == target.size && d32 && + target.guest_format == Prospero::GpuEnumValue(Prospero::BufferFormat::k32Float) && + target.bytes_per_element == 4 && target.tile_mode == Prospero::GpuEnumValue(Prospero::TileMode::kDepth) && + target.htile_address == 0 && target.htile_size == 0; +} + +[[nodiscard]] inline bool CanLoadStencilAttachment(const DepthTargetInfo& target, + bool stencil_initialized) { + const bool has_stencil = target.stencil_address != 0 || target.stencil_size != 0; + return !has_stencil || !target.stencil_access || target.stencil_load_clear || + stencil_initialized; +} + +[[nodiscard]] inline bool IsDepthTargetRangeCompatible(const DepthTargetInfo& target, + uint64_t address, uint64_t size) { + if (address == 0 || size == 0 || address > UINT64_MAX - size) { + EXIT("invalid depth-target view range\n"); + } + const bool depth = address == target.address && size <= target.size; + const bool stencil = target.stencil_address != 0 && address == target.stencil_address && + size == target.stencil_size; + return depth || stencil; +} + +[[nodiscard]] inline bool ImageRangeOverlaps(uint64_t left, uint64_t left_size, uint64_t right, + uint64_t right_size) { + if (left_size == 0 || right_size == 0 || left > UINT64_MAX - left_size || + right > UINT64_MAX - right_size) { + EXIT("invalid image overlap range\n"); + } + return left < right + right_size && right < left + left_size; +} + +[[nodiscard]] inline bool ImagePageRangesOverlap(uint64_t left, uint64_t left_size, uint64_t right, + uint64_t right_size) { + if (left_size == 0 || right_size == 0 || left > UINT64_MAX - left_size || + right > UINT64_MAX - right_size) { + EXIT("invalid image page-overlap range\n"); + } + const auto left_first = left / TRACKER_PAGE_SIZE; + const auto left_last = (left + left_size - 1) / TRACKER_PAGE_SIZE; + const auto right_first = right / TRACKER_PAGE_SIZE; + const auto right_last = (right + right_size - 1) / TRACKER_PAGE_SIZE; + return left_first <= right_last && right_first <= left_last; +} + +[[nodiscard]] inline SampledOverlap ClassifySampledOverlap(const ImageInfo& requested, + const ImageInfo& cached, + bool cached_gpu_modified, + bool same_context) { + if (!ImagePageRangesOverlap(requested.address, requested.size, cached.address, cached.size)) { + return SampledOverlap::None; + } + return !cached_gpu_modified && same_context ? SampledOverlap::ReadOnlyAlias + : SampledOverlap::Unsupported; +} + +[[nodiscard]] inline bool IsRgba8SrgbReinterpretation(VkFormat cached, VkFormat requested) noexcept; + +[[nodiscard]] inline StorageSampledOverlap +ClassifyStorageSampledOverlap(const ImageInfo& requested, const ImageInfo& cached, + VkFormat requested_view_format, VkFormat cached_image_format, + bool cached_gpu_modified, bool cached_cpu_dirty, bool same_context) { + if (!ImagePageRangesOverlap(requested.address, requested.size, cached.address, cached.size)) { + return StorageSampledOverlap::None; + } + const bool same_backing = + requested.address == cached.address && requested.size == cached.size && + requested.width == cached.width && requested.height == cached.height && + requested.pitch == cached.pitch && requested.base_level == cached.base_level && + requested.levels == cached.levels && requested.view_levels == cached.view_levels && + requested.tile == cached.tile && requested.depth == cached.depth && + requested.type == cached.type && requested.base_array == cached.base_array; + const bool compatible_format = + (requested.format == cached.format && requested_view_format == cached_image_format) || + IsRgba8SrgbReinterpretation(cached_image_format, requested_view_format); + return same_backing && compatible_format && cached_gpu_modified && !cached_cpu_dirty && + same_context + ? StorageSampledOverlap::ExactImage + : StorageSampledOverlap::Unsupported; +} + +[[nodiscard]] inline HostWriteOverlap +ClassifyHostWriteOverlap(uint64_t write_address, uint64_t write_size, uint64_t image_address, + uint64_t image_size, bool host_refreshable, bool gpu_modified, + bool metadata_overlap) { + if (!ImagePageRangesOverlap(write_address, write_size, image_address, image_size)) { + return HostWriteOverlap::None; + } + return host_refreshable && !gpu_modified && !metadata_overlap + ? HostWriteOverlap::InvalidateImage + : HostWriteOverlap::Unsupported; +} + +[[nodiscard]] inline bool IsSampledBufferWriteTransition(uint64_t buffer_address, + uint64_t buffer_size, + uint64_t image_address, + uint64_t image_size, + bool image_gpu_modified) { + if (!ImagePageRangesOverlap(buffer_address, buffer_size, image_address, image_size)) { + return false; + } + const bool exact = buffer_address == image_address && buffer_size == image_size; + const bool page_aligned = ((buffer_address | buffer_size) & (TRACKER_PAGE_SIZE - 1)) == 0; + return exact && page_aligned && !image_gpu_modified; +} + +[[nodiscard]] inline BufferImageAlias +ClassifyBufferImageAlias(uint64_t buffer_address, uint64_t buffer_size, uint64_t image_address, + uint64_t image_size, bool video_out, bool render_target, + bool image_gpu_modified, bool buffer_written, bool buffer_formatted) { + if (!ImagePageRangesOverlap(buffer_address, buffer_size, image_address, image_size)) { + return BufferImageAlias::None; + } + const bool exact = buffer_address == image_address && buffer_size == image_size; + if (!exact || !buffer_written || !buffer_formatted) { + return BufferImageAlias::Unsupported; + } + if (video_out && !render_target && !image_gpu_modified) { + return BufferImageAlias::VideoOutWrite; + } + const bool page_aligned = ((buffer_address | buffer_size) & (TRACKER_PAGE_SIZE - 1)) == 0; + if (render_target && !video_out && image_gpu_modified && page_aligned) { + return BufferImageAlias::RenderTargetWrite; + } + return BufferImageAlias::Unsupported; +} + +[[nodiscard]] inline DepthOverlap ClassifyDepthOverlap(const ImageInfo& sampled, + bool sampled_gpu_modified, + const DepthTargetInfo& depth) { + const bool depth_overlap = + ImageRangeOverlaps(sampled.address, sampled.size, depth.address, depth.size); + const bool stencil_overlap = + depth.stencil_address != 0 && ImageRangeOverlaps(sampled.address, sampled.size, + depth.stencil_address, depth.stencil_size); + if (!depth_overlap && !stencil_overlap) { + return DepthOverlap::None; + } + const bool has_stencil = depth.stencil_address != 0 || depth.stencil_size != 0; + if (sampled.address == depth.address && !sampled_gpu_modified && depth.depth_load_clear && + (!has_stencil || + (depth.stencil_address != 0 && depth.stencil_size != 0 && depth.stencil_load_clear))) { + return DepthOverlap::RetireSampled; + } + return DepthOverlap::Unsupported; +} + +[[nodiscard]] inline RenderTargetOverlap +ClassifyRenderTargetOverlap(const ImageInfo& sampled, bool sampled_gpu_modified, bool same_context, + const RenderTargetInfo& target) { + if (!ImagePageRangesOverlap(sampled.address, sampled.size, target.address, target.size)) { + return RenderTargetOverlap::None; + } + const bool exact = sampled.address == target.address && sampled.size == target.size; + const bool page_isolated_overlap = + sampled.address % TRACKER_PAGE_SIZE == 0 && sampled.size % TRACKER_PAGE_SIZE == 0 && + target.address % TRACKER_PAGE_SIZE == 0 && target.size % TRACKER_PAGE_SIZE == 0 && + ImageRangeOverlaps(sampled.address, sampled.size, target.address, target.size); + return !sampled_gpu_modified && same_context && (exact || page_isolated_overlap) + ? RenderTargetOverlap::RetireSampled + : RenderTargetOverlap::Unsupported; +} + +[[nodiscard]] inline bool IsRgba8SrgbViewFormat(VkFormat format) noexcept { + return format == VK_FORMAT_R8G8B8A8_UNORM || format == VK_FORMAT_R8G8B8A8_SRGB || + format == VK_FORMAT_B8G8R8A8_UNORM || format == VK_FORMAT_B8G8R8A8_SRGB; +} + +[[nodiscard]] inline bool IsRgba8SrgbReinterpretation(VkFormat cached, + VkFormat requested) noexcept { + return (cached == VK_FORMAT_R8G8B8A8_UNORM && requested == VK_FORMAT_R8G8B8A8_SRGB) || + (cached == VK_FORMAT_R8G8B8A8_SRGB && requested == VK_FORMAT_R8G8B8A8_UNORM) || + (cached == VK_FORMAT_B8G8R8A8_UNORM && requested == VK_FORMAT_B8G8R8A8_SRGB) || + (cached == VK_FORMAT_B8G8R8A8_SRGB && requested == VK_FORMAT_B8G8R8A8_UNORM); +} + +[[nodiscard]] inline bool IsCompatibleRenderTargetView(const RenderTargetInfo& cached, + const RenderTargetInfo& requested) noexcept { + return cached.address == requested.address && cached.size == requested.size && + cached.width == requested.width && cached.height == requested.height && + cached.pitch == requested.pitch && + cached.bytes_per_element == requested.bytes_per_element && + cached.tile_mode == requested.tile_mode && cached.levels == requested.levels && + IsRgba8SrgbReinterpretation(cached.format, requested.format); +} + +[[nodiscard]] inline RenderTargetOverlap +ClassifyRenderTargetOverlap(const RenderTargetInfo& cached, bool cached_gpu_modified, + bool cached_buffer_modified, bool same_context, + const RenderTargetInfo& requested) { + if (!ImagePageRangesOverlap(cached.address, cached.size, requested.address, requested.size)) { + return RenderTargetOverlap::None; + } + // For an equal-address allocation-pool entry, a changed block raster or block size is a new + // image allocation, not a view of the old image. In Kyty we can only retire an already-published target with page-isolated storage :) + const bool page_isolated = + cached.address % TRACKER_PAGE_SIZE == 0 && cached.size % TRACKER_PAGE_SIZE == 0 && + requested.address % TRACKER_PAGE_SIZE == 0 && requested.size % TRACKER_PAGE_SIZE == 0; + const bool pool_storage_shape_changed = cached.pitch != requested.pitch || + cached.height != requested.height || + cached.bytes_per_element != requested.bytes_per_element; + return cached.address == requested.address && page_isolated && pool_storage_shape_changed && + !cached_gpu_modified && !cached_buffer_modified && same_context + ? RenderTargetOverlap::RetireTarget + : RenderTargetOverlap::Unsupported; +} + +} // namespace Libs::Graphics + +#endif // EMULATOR_SRC_GRAPHICS_HOST_GPU_RENDERER_IMAGEINFO_H_ diff --git a/src/graphics/host_gpu/renderer/imageView.h b/src/graphics/host_gpu/renderer/imageView.h new file mode 100644 index 000000000..207184fd0 --- /dev/null +++ b/src/graphics/host_gpu/renderer/imageView.h @@ -0,0 +1,141 @@ +#ifndef EMULATOR_SRC_GRAPHICS_HOST_GPU_RENDERER_IMAGEVIEW_H_ +#define EMULATOR_SRC_GRAPHICS_HOST_GPU_RENDERER_IMAGEVIEW_H_ + +#include "common/assert.h" +#include "graphics/host_gpu/graphicContext.h" +#include "graphics/shader/recompiler/ShaderIR.h" +#include "graphics/shader/shader.h" + +namespace Libs::Graphics { + +[[nodiscard]] inline bool IsSupportedStorageSwizzle(uint32_t format, uint32_t swizzle) noexcept { + const bool single_channel_uint = format == Prospero::GpuEnumValue(Prospero::BufferFormat::k8UInt) || + format == Prospero::GpuEnumValue(Prospero::BufferFormat::k16UInt) || + format == Prospero::GpuEnumValue(Prospero::BufferFormat::k32UInt); + return swizzle == DstSel(4, 5, 6, 7) || + (single_channel_uint && + (swizzle == DstSel(4, 0, 0, 0) || swizzle == DstSel(4, 0, 0, 1))) || + (format == Prospero::GpuEnumValue(Prospero::BufferFormat::k8_8_8_8UNorm) && + (swizzle == DstSel(4, 5, 6, 1) || swizzle == DstSel(6, 5, 4, 7))) || + (format == Prospero::GpuEnumValue(Prospero::BufferFormat::k32_32_32_32Float) && + swizzle == DstSel(5, 6, 7, 4)); +} + +[[noreturn]] inline void UnsupportedColorView(const char* usage, VkFormat image_format, + VkFormat view_format, uint32_t swizzle) noexcept { + EXIT("unsupported %s color image view: image_format=%d view_format=%d swizzle=0x%03x\n", usage, + static_cast(image_format), static_cast(view_format), swizzle); +} + +[[nodiscard]] inline VkFormat BgraToRgbaSampledViewFormat(VkFormat image_format) noexcept { + switch (image_format) { + case VK_FORMAT_B8G8R8A8_UNORM: return VK_FORMAT_R8G8B8A8_UNORM; + case VK_FORMAT_B8G8R8A8_SRGB: return VK_FORMAT_R8G8B8A8_SRGB; + default: return VK_FORMAT_UNDEFINED; + } +} + +[[nodiscard]] inline bool IsBgraToRgba8SampledView(VkFormat image_format, + VkFormat view_format) noexcept { + const bool bgra8 = + image_format == VK_FORMAT_B8G8R8A8_UNORM || image_format == VK_FORMAT_B8G8R8A8_SRGB; + const bool rgba8 = + view_format == VK_FORMAT_R8G8B8A8_UNORM || view_format == VK_FORMAT_R8G8B8A8_SRGB; + return bgra8 && rgba8; +} + +[[nodiscard]] inline VkFormat BgraSrgbStorageViewFormat(VkFormat image_format) noexcept { + return image_format == VK_FORMAT_B8G8R8A8_SRGB ? VK_FORMAT_R8G8B8A8_UNORM : VK_FORMAT_UNDEFINED; +} + +[[nodiscard]] inline bool IsBgraSrgbStorageView(VkFormat image_format, VkFormat view_format, + uint32_t swizzle) noexcept { + return view_format == BgraSrgbStorageViewFormat(image_format) && swizzle == DstSel(6, 5, 4, 7); +} + +[[nodiscard]] inline int SelectSampledColorView(VkFormat image_format, VkFormat view_format, + uint32_t swizzle) noexcept { + if (image_format == view_format) { + switch (swizzle) { + case DstSel(4, 5, 6, 7): return VulkanImage::VIEW_DEFAULT; + case DstSel(4, 0, 0, 1): return VulkanImage::VIEW_R001; + case DstSel(4, 5, 0, 1): return VulkanImage::VIEW_RG01; + case DstSel(4, 5, 6, 1): return VulkanImage::VIEW_RGB1; + default: break; + } + } + if (IsBgraToRgba8SampledView(image_format, view_format) && swizzle == DstSel(6, 5, 4, 7)) { + return VulkanImage::VIEW_RGBA8_BGRA; + } + UnsupportedColorView("sampled", image_format, view_format, swizzle); +} + +[[nodiscard]] inline int SelectSampledDepthView(VkFormat image_format, VkFormat view_format, + uint32_t swizzle) noexcept { + const bool d16 = image_format == VK_FORMAT_D16_UNORM && view_format == VK_FORMAT_R16_UNORM; + const bool d32 = + (image_format == VK_FORMAT_D32_SFLOAT || image_format == VK_FORMAT_D32_SFLOAT_S8_UINT) && + view_format == VK_FORMAT_R32_SFLOAT; + if (d16 || d32) { + switch (swizzle) { + case DstSel(4, 4, 4, 4): return VulkanImage::VIEW_DEPTH_TEXTURE; + case DstSel(4, 0, 0, 0): return VulkanImage::VIEW_R000; + default: break; + } + } + EXIT("unsupported sampled depth image view: image_format=%d view_format=%d swizzle=0x%03x\n", + static_cast(image_format), static_cast(view_format), swizzle); +} + +[[nodiscard]] inline bool +IsSupportedSampledDepthResource(const ShaderRecompiler::IR::ImageResource& resource) noexcept { + return resource.kind == ShaderRecompiler::IR::ResourceKind::Image && + resource.dimension == ShaderRecompiler::Decoder::ImageDimension::Dim2D && + resource.mip_mode == ShaderRecompiler::IR::ImageMipMode::None && resource.read && + !resource.written && !resource.atomic; +} + +[[nodiscard]] inline int SelectStorageColorView(VkFormat image_format, VkFormat view_format, + uint32_t swizzle) noexcept { + const bool single_channel_uint = view_format == VK_FORMAT_R8_UINT || + view_format == VK_FORMAT_R16_UINT || + view_format == VK_FORMAT_R32_UINT; + const bool swizzle_ok = + swizzle == DstSel(4, 5, 6, 7) || + (single_channel_uint && (swizzle == DstSel(4, 0, 0, 0) || swizzle == DstSel(4, 0, 0, 1))) || + (view_format == VK_FORMAT_R8G8B8A8_UNORM && + (swizzle == DstSel(4, 5, 6, 1) || swizzle == DstSel(6, 5, 4, 7))) || + (view_format == VK_FORMAT_R32G32B32A32_SFLOAT && swizzle == DstSel(5, 6, 7, 4)); + if ((image_format != view_format && + !IsBgraSrgbStorageView(image_format, view_format, swizzle)) || + !swizzle_ok) { + UnsupportedColorView("storage", image_format, view_format, swizzle); + } + return VulkanImage::VIEW_STORAGE; +} + +[[nodiscard]] inline bool +IsSupportedStorageImageResource(const ShaderRecompiler::IR::ImageResource& resource) noexcept { + return (resource.kind == ShaderRecompiler::IR::ResourceKind::StorageImage || + resource.kind == ShaderRecompiler::IR::ResourceKind::StorageImageUint) && + (resource.dimension == ShaderRecompiler::Decoder::ImageDimension::Dim2D || + resource.dimension == ShaderRecompiler::Decoder::ImageDimension::Dim3D || + resource.dimension == ShaderRecompiler::Decoder::ImageDimension::Dim2DArray) && + resource.mip_mode == ShaderRecompiler::IR::ImageMipMode::None && resource.written && + !resource.atomic && !resource.depth_compare; +} + +inline void +ValidateStorageImageResource(const ShaderRecompiler::IR::ImageResource& resource) noexcept { + if (!IsSupportedStorageImageResource(resource)) { + EXIT("unsupported storage color image resource: kind=%u dimension=%u mip=%u " + "read=%d written=%d atomic=%d depth_compare=%d\n", + static_cast(resource.kind), static_cast(resource.dimension), + static_cast(resource.mip_mode), resource.read, resource.written, + resource.atomic, resource.depth_compare); + } +} + +} // namespace Libs::Graphics + +#endif // EMULATOR_SRC_GRAPHICS_HOST_GPU_RENDERER_IMAGEVIEW_H_ diff --git a/src/graphics/host_gpu/renderer/internal.h b/src/graphics/host_gpu/renderer/internal.h new file mode 100644 index 000000000..c801b4812 --- /dev/null +++ b/src/graphics/host_gpu/renderer/internal.h @@ -0,0 +1,12 @@ +#ifndef EMULATOR_SRC_GRAPHICS_HOST_GPU_RENDERER_INTERNAL_H_ +#define EMULATOR_SRC_GRAPHICS_HOST_GPU_RENDERER_INTERNAL_H_ + +#include "graphics/host_gpu/renderer/descriptorCache.h" +#include "graphics/host_gpu/renderer/framebufferCache.h" +#include "graphics/host_gpu/renderer/gdsBuffer.h" +#include "graphics/host_gpu/renderer/pipelineCache.h" +#include "graphics/host_gpu/renderer/renderContext.h" +#include "graphics/host_gpu/renderer/renderState.h" +#include "graphics/host_gpu/renderer/samplerCache.h" + +#endif // EMULATOR_SRC_GRAPHICS_HOST_GPU_RENDERER_INTERNAL_H_ diff --git a/src/graphics/host_gpu/renderer/pipelineCache.cpp b/src/graphics/host_gpu/renderer/pipelineCache.cpp new file mode 100644 index 000000000..ac4c30b52 --- /dev/null +++ b/src/graphics/host_gpu/renderer/pipelineCache.cpp @@ -0,0 +1,296 @@ +#include "graphics/host_gpu/renderer/pipelineCache.h" + +#include "common/assert.h" +#include "common/emulatorConfig.h" +#include "common/logging/log.h" +#include "common/profiler.h" +#include "common/stringUtils.h" +#include "graphics/guest_gpu/graphicsRun.h" +#include "graphics/guest_gpu/hardwareContext.h" +#include "graphics/host_gpu/renderer/framebufferCache.h" +#include "graphics/host_gpu/renderer/renderContext.h" +#include "graphics/host_gpu/utils.h" + +#include +#include +#include +#include +#include + +namespace Libs::Graphics { + +namespace { + +void NormalizeStaticParamsForDynamicState(PipelineStaticParameters& static_params) { + static_params.viewport_scale[0] = 0.5f; + static_params.viewport_scale[1] = 0.5f; + static_params.viewport_scale[2] = 1.0f; + static_params.viewport_offset[0] = 0.5f; + static_params.viewport_offset[1] = 0.5f; + static_params.viewport_offset[2] = 0.0f; + + static_params.scissor_ltrb[0] = 0; + static_params.scissor_ltrb[1] = 0; + static_params.scissor_ltrb[2] = 1; + static_params.scissor_ltrb[3] = 1; +} + +} // namespace + +bool PipelineStaticParameters::operator==(const PipelineStaticParameters& other) const noexcept { + return std::memcmp(this, &other, sizeof(*this)) == 0; +} + +void PipelineCache::DeletePipelineInternal(Pipeline* p) { + EXIT_IF(g_render_ctx == nullptr); + EXIT_IF(p == nullptr); + EXIT_IF(p->pipeline == nullptr); + EXIT_IF(p->pipeline_layout == nullptr); + + DumpPipeline("delete", *p); + + auto* gctx = g_render_ctx->GetGraphicCtx(); + + EXIT_IF(gctx == nullptr); + + VulkanDeviceWaitIdle(gctx); + vkDestroyPipeline(gctx->device, p->pipeline, nullptr); + vkDestroyPipelineLayout(gctx->device, p->pipeline_layout, nullptr); + + p->pipeline = nullptr; + p->pipeline_layout = nullptr; +} + +PipelineCache::GraphicsPipeline* PipelineCache::CreateGraphicsPipeline( + VulkanFramebuffer* framebuffer, RenderColorInfo* colors, uint32_t color_count, + RenderDepthInfo* depth, ShaderVertexInputInfo* vs_input_info, HW::Context* ctx, + HW::Shader* sh_ctx, ShaderPixelInputInfo* ps_input_info, VkPrimitiveTopology topology, + bool ps_active, std::span vs_spirv, std::span ps_spirv) { + KYTY_PROFILER_BLOCK("PipelineCache::CreatePipeline(Gfx)", profiler::colors::DeepOrangeA200); + + EXIT_IF(framebuffer == nullptr); + EXIT_IF(depth == nullptr); + EXIT_IF(colors == nullptr); + EXIT_IF(color_count > RENDER_COLOR_ATTACHMENTS_MAX); + EXIT_IF(vs_spirv.empty()); + EXIT_IF(ps_active && ps_spirv.empty()); + + Common::LockGuard lock(m_mutex); + + const auto& vertex_info = sh_ctx->GetVs(); + const auto& ps_regs = sh_ctx->GetPs(); + const HW::BlendColor& bclr = ctx->GetBlendColor(); + uint32_t color_mask[RENDER_COLOR_ATTACHMENTS_MAX] = {}; + for (uint32_t i = 0; i < color_count; i++) { + color_mask[i] = + (colors[i].vulkan_buffer != nullptr + ? render_target_mask_slot(ctx->GetRenderTargetMask(), colors[i].target_slot) + : 0); + } + const HW::ModeControl& mc = ctx->GetModeControl(); + + auto vs_id = ShaderGetIdVS(&vertex_info, vs_input_info, true); + ShaderId ps_id {}; + if (ps_active) { + ps_id = ShaderGetIdPS(&ps_regs, ps_input_info, true); + } + + PipelineStaticParameters static_params {}; + GraphicsPipeline p {}; + p.render_pass_id = framebuffer->render_pass_id; + p.ps_shader_id = ps_id; + p.vs_shader_id = vs_id; + + static_params.color_count = color_count; + + if (ps_active && depth->depth_test_enable && ps_input_info->ps_execute_on_noop) { + static std::atomic log_count {0}; + if (log_count.fetch_add(1, std::memory_order_relaxed) < 16) { + LOGF("Pipeline: temporary: accepting EXEC_ON_NOOP with depth test enabled\n"); + } + } + + static_params.negative_one_to_one = !ctx->GetClipControl().dx_clip_space; + static_params.topology = topology; + static_params.with_depth = + (depth->format != VK_FORMAT_UNDEFINED && depth->vulkan_buffer != nullptr); + static_params.depth_test_enable = depth->depth_test_enable; + static_params.depth_write_enable = (depth->depth_write_enable && !depth->depth_clear_enable); + static_params.depth_compare_op = depth->depth_compare_op; + static_params.depth_bounds_test_enable = depth->depth_bounds_test_enable; + static_params.depth_min_bounds = depth->depth_min_bounds; + static_params.depth_max_bounds = depth->depth_max_bounds; + static_params.stencil_test_enable = depth->stencil_test_enable; + static_params.stencil_front = depth->stencil_static_front; + static_params.stencil_back = depth->stencil_static_back; + for (uint32_t i = 0; i < RENDER_COLOR_ATTACHMENTS_MAX; i++) { + static_params.color_mask[i] = color_mask[i]; + } + static_params.cull_back = mc.cull_back; + static_params.cull_front = mc.cull_front; + static_params.face = mc.face; + + for (uint32_t i = 0; i < color_count; i++) { + const auto& rt = ctx->GetRenderTarget(colors[i].target_slot); + const auto& bc = ctx->GetBlendControl(colors[i].target_slot); + static_params.color_srcblend[i] = bc.color_srcblend; + static_params.color_comb_fcn[i] = bc.color_comb_fcn; + static_params.color_destblend[i] = bc.color_destblend; + static_params.alpha_srcblend[i] = bc.alpha_srcblend; + static_params.alpha_comb_fcn[i] = bc.alpha_comb_fcn; + static_params.alpha_destblend[i] = bc.alpha_destblend; + static_params.separate_alpha_blend[i] = bc.separate_alpha_blend; + static_params.blend_enable[i] = bc.enable; + static_params.blend_bypass[i] = rt.info.blend_bypass; + } + static_params.blend_color_red = bclr.red; + static_params.blend_color_green = bclr.green; + static_params.blend_color_blue = bclr.blue; + static_params.blend_color_alpha = bclr.alpha; + + NormalizeStaticParamsForDynamicState(static_params); + + GraphicsPipelineKey key {}; + key.render_pass_id = p.render_pass_id; + key.vs_shader_id = p.vs_shader_id; + key.ps_shader_id = p.ps_shader_id; + key.static_params = static_params; + + if (auto iter = m_graphics_pipelines.find(key); iter != m_graphics_pipelines.end()) { + return iter->second.get(); + } + + if (graphics_debug_dump_enabled()) { + ShaderDbgDumpInputInfo(vs_input_info); + if (ps_active) { + ShaderDbgDumpInputInfo(ps_input_info); + } + LOGF("PipelineTrace: shader binaries VS=0x%08" PRIx32 "/0x%08" PRIx32 " words=%" PRIu64 + " PS=0x%08" PRIx32 "/0x%08" PRIx32 " words=%" PRIu64 "\n", + vs_id.hash0, vs_id.crc32, static_cast(vs_spirv.size()), ps_id.hash0, + ps_id.crc32, static_cast(ps_spirv.size())); + } + + auto cached = std::make_unique(p); + LogPipelineTrace("CreatePipelineInternal begin", vs_id.hash0, vs_id.crc32, ps_id.hash0, + ps_id.crc32); + CreatePipelineInternal(cached.get(), framebuffer->render_pass, vs_input_info, vs_spirv, + ps_input_info, ps_spirv, static_params, vs_id.hash0, vs_id.crc32, + ps_id.hash0, ps_id.crc32, ps_active); + LogPipelineTrace("CreatePipelineInternal done", vs_id.hash0, vs_id.crc32, ps_id.hash0, + ps_id.crc32); + + EXIT_NOT_IMPLEMENTED(cached->pipeline == nullptr); + EXIT_NOT_IMPLEMENTED(cached->pipeline_layout == nullptr); + + auto [iter, inserted] = m_graphics_pipelines.emplace(std::move(key), std::move(cached)); + EXIT_IF(!inserted); + DumpPipeline("create", *iter->second); + + return iter->second.get(); +} + +PipelineCache::ComputePipeline* +PipelineCache::CreateComputePipeline(ShaderComputeInputInfo* input_info, + const HW::ComputeShaderInfo* cs_regs, + std::span cs_spirv) { + KYTY_PROFILER_BLOCK("PipelineCache::CreatePipeline(Compute)", profiler::colors::RedA100); + + EXIT_IF(cs_regs == nullptr); + EXIT_IF(cs_spirv.empty()); + + Common::LockGuard lock(m_mutex); + + auto cs_id = ShaderGetIdCS(cs_regs, input_info, true); + + ComputePipeline p {}; + p.cs_shader_id = cs_id; + + ComputePipelineKey key {}; + key.cs_shader_id = p.cs_shader_id; + + if (auto iter = m_compute_pipelines.find(key); iter != m_compute_pipelines.end()) { + return iter->second.get(); + } + + if (graphics_debug_dump_enabled()) { + ShaderDbgDumpInputInfo(input_info); + } + + auto cached = std::make_unique(p); + CreatePipelineInternal(cached.get(), input_info, cs_spirv); + + EXIT_NOT_IMPLEMENTED(cached->pipeline == nullptr); + EXIT_NOT_IMPLEMENTED(cached->pipeline_layout == nullptr); + + auto [iter, inserted] = m_compute_pipelines.emplace(std::move(key), std::move(cached)); + EXIT_IF(!inserted); + + return iter->second.get(); +} + +void PipelineCache::DeletePipeline(Pipeline* pipeline) { + Common::LockGuard lock(m_mutex); + + EXIT_IF(pipeline == nullptr); + + for (auto iter = m_graphics_pipelines.begin(); iter != m_graphics_pipelines.end(); iter++) { + if (iter->second.get() == pipeline) { + DeletePipelineInternal(iter->second.get()); + m_graphics_pipelines.erase(iter); + return; + } + } + + for (auto iter = m_compute_pipelines.begin(); iter != m_compute_pipelines.end(); iter++) { + if (iter->second.get() == pipeline) { + DeletePipelineInternal(iter->second.get()); + m_compute_pipelines.erase(iter); + return; + } + } + + EXIT_IF(true); +} + +void PipelineCache::DeleteAllPipelines() { + Common::LockGuard lock(m_mutex); + + for (auto& item: m_graphics_pipelines) { + DeletePipelineInternal(item.second.get()); + } + m_graphics_pipelines.clear(); + + for (auto& item: m_compute_pipelines) { + DeletePipelineInternal(item.second.get()); + } + m_compute_pipelines.clear(); +} + +void PipelineCache::DumpToFile(Common::File* f, const Pipeline& p) {} + +void PipelineCache::DumpPipeline(const char* action, const Pipeline& p) { + EXIT_IF(action == nullptr); + + static std::atomic_int dump_id = 0; + + if (!Config::PipelineDumpEnabled()) { + return; + } + + Common::File f; + const auto file_name = + Config::GetPipelineDumpFolder() / + fmt::format("{:04d}_{:04d}_pipeline_{}.log", GraphicsRunGetFrameNum(), dump_id++, action); + Common::File::CreateDirectories(file_name.parent_path()); + f.Create(file_name); + if (f.IsInvalid()) { + auto file_name_text = Common::PathToString(file_name); + LOGF_COLOR(Log::Color::BrightRed, "Can't create file: %s\n", file_name_text.c_str()); + return; + } + DumpToFile(&f, p); + f.Close(); +} + +} // namespace Libs::Graphics diff --git a/src/graphics/host_gpu/renderer/pipelineCache.h b/src/graphics/host_gpu/renderer/pipelineCache.h new file mode 100644 index 000000000..b77873688 --- /dev/null +++ b/src/graphics/host_gpu/renderer/pipelineCache.h @@ -0,0 +1,152 @@ +#ifndef EMULATOR_SRC_GRAPHICS_HOST_GPU_RENDERER_PIPELINECACHE_H_ +#define EMULATOR_SRC_GRAPHICS_HOST_GPU_RENDERER_PIPELINECACHE_H_ + +#include "common/abi.h" +#include "common/assert.h" +#include "common/common.h" +#include "common/file.h" +#include "common/threads.h" +#include "graphics/host_gpu/renderer/renderState.h" +#include "graphics/shader/shader.h" + +#include +#include +#include +#include +#include +#include + +namespace Libs::Graphics { + +struct VulkanFramebuffer; + +namespace HW { +class Context; +class Shader; +struct ComputeShaderInfo; +} // namespace HW + +class PipelineCache { +public: + PipelineCache() { EXIT_NOT_IMPLEMENTED(!Common::Thread::IsMainThread()); } + virtual ~PipelineCache() { KYTY_NOT_IMPLEMENTED; } + KYTY_CLASS_NO_COPY(PipelineCache); + + struct Pipeline { + VkPipelineLayout pipeline_layout = nullptr; + VkPipeline pipeline = nullptr; + }; + + struct GraphicsPipeline: Pipeline { + uint64_t render_pass_id = 0; + ShaderId vs_shader_id; + ShaderId ps_shader_id; + }; + + struct ComputePipeline: Pipeline { + ShaderId cs_shader_id; + }; + + GraphicsPipeline* CreateGraphicsPipeline( + VulkanFramebuffer* framebuffer, RenderColorInfo* colors, uint32_t color_count, + RenderDepthInfo* depth, ShaderVertexInputInfo* vs_input_info, HW::Context* ctx, + HW::Shader* sh_ctx, ShaderPixelInputInfo* ps_input_info, VkPrimitiveTopology topology, + bool ps_active, std::span vs_spirv, std::span ps_spirv); + ComputePipeline* CreateComputePipeline(ShaderComputeInputInfo* input_info, + const HW::ComputeShaderInfo* cs_regs, + std::span cs_spirv); + void DeletePipeline(Pipeline* pipeline); + void DeleteAllPipelines(); + +private: + struct GraphicsPipelineKey { + uint64_t render_pass_id = 0; + ShaderId vs_shader_id; + ShaderId ps_shader_id; + PipelineStaticParameters static_params; + + bool operator==(const GraphicsPipelineKey& other) const { + return render_pass_id == other.render_pass_id && vs_shader_id == other.vs_shader_id && + ps_shader_id == other.ps_shader_id && static_params == other.static_params; + } + }; + + struct ComputePipelineKey { + ShaderId cs_shader_id; + + bool operator==(const ComputePipelineKey& other) const { + return cs_shader_id == other.cs_shader_id; + } + }; + + struct PipelineKeyHash { + static void Mix(std::size_t* hash, std::size_t value) { + *hash ^= value + static_cast(0x9e3779b97f4a7c15ull) + (*hash << 6u) + + (*hash >> 2u); + } + + static void MixShaderId(std::size_t* hash, const ShaderId& id) { + Mix(hash, id.hash0); + Mix(hash, id.crc32); + Mix(hash, id.ids.size()); + for (auto value: id.ids) { + Mix(hash, value); + } + } + + static void MixStaticParams(std::size_t* hash, const PipelineStaticParameters& params) { + const auto* bytes = reinterpret_cast(¶ms); + for (std::size_t i = 0; i < sizeof(params); i++) { + Mix(hash, bytes[i]); + } + } + }; + + struct GraphicsPipelineKeyHash { + std::size_t operator()(const GraphicsPipelineKey& key) const { + std::size_t hash = 0; + PipelineKeyHash::Mix(&hash, key.render_pass_id); + PipelineKeyHash::MixShaderId(&hash, key.vs_shader_id); + PipelineKeyHash::MixShaderId(&hash, key.ps_shader_id); + PipelineKeyHash::MixStaticParams(&hash, key.static_params); + return hash; + } + }; + + struct ComputePipelineKeyHash { + std::size_t operator()(const ComputePipelineKey& key) const { + std::size_t hash = 0; + PipelineKeyHash::MixShaderId(&hash, key.cs_shader_id); + return hash; + } + }; + + void DeletePipelineInternal(Pipeline* p); + void DumpToFile(Common::File* f, const Pipeline& p); + void DumpPipeline(const char* action, const Pipeline& p); + + std::unordered_map, + GraphicsPipelineKeyHash> + m_graphics_pipelines; + std::unordered_map, ComputePipelineKeyHash> + m_compute_pipelines; + Common::Mutex m_mutex; +}; + +void LogPipelineTrace(const char* phase, uint32_t vs_hash0, uint32_t vs_crc32, uint32_t ps_hash0, + uint32_t ps_crc32); +void CreatePipelineInternal(PipelineCache::GraphicsPipeline* pipeline, VkRenderPass render_pass, + const ShaderVertexInputInfo* vs_input_info, + std::span vs_shader, + const ShaderPixelInputInfo* ps_input_info, + std::span ps_shader, + const PipelineStaticParameters& static_params, uint32_t vs_hash0, + uint32_t vs_crc32, uint32_t ps_hash0, uint32_t ps_crc32, + bool ps_active); +void CreatePipelineInternal(PipelineCache::ComputePipeline* pipeline, + const ShaderComputeInputInfo* input_info, + std::span cs_shader); + +} // namespace Libs::Graphics + +#endif // EMULATOR_SRC_GRAPHICS_HOST_GPU_RENDERER_PIPELINECACHE_H_ diff --git a/src/graphics/host_gpu/renderer/render.h b/src/graphics/host_gpu/renderer/render.h new file mode 100644 index 000000000..ed07f232e --- /dev/null +++ b/src/graphics/host_gpu/renderer/render.h @@ -0,0 +1,191 @@ +#ifndef EMULATOR_INCLUDE_EMULATOR_GRAPHICS_GRAPHICSRENDER_H_ +#define EMULATOR_INCLUDE_EMULATOR_GRAPHICS_GRAPHICSRENDER_H_ + +#include "common/abi.h" +#include "common/common.h" +#include "graphics/host_gpu/renderer/streamBuffer.h" +#include "kernel/eventQueue.h" + +#include +#include +#include + +namespace Libs::Graphics { + +namespace HW { +class Context; +class UserConfig; +class Shader; +struct DepthRenderTarget; +} // namespace HW + +struct GraphicContext; +struct ShaderBufferResource; +struct ShaderComputeInputInfo; +struct VideoOutVulkanImage; +struct DepthStencilVulkanImage; +struct TextureVulkanImage; +struct StorageTextureVulkanImage; +struct RenderTextureVulkanImage; +struct VulkanCommandPool; +struct VulkanBuffer; +struct VulkanDescriptorSet; +struct VulkanFramebuffer; +struct RenderDepthInfo; +struct RenderColorInfo; +class BufferCache; + +struct HtileClearTarget { + uint64_t address = 0; + uint64_t size = 0; +}; + +class FenceResourceRetainer { +public: + FenceResourceRetainer() = default; + ~FenceResourceRetainer(); + KYTY_CLASS_NO_COPY(FenceResourceRetainer); + + void Retain(std::shared_ptr resource); + void ReleaseAfterFence() noexcept; + [[nodiscard]] bool Empty() const noexcept { return m_resources.empty(); } + +private: + std::vector> m_resources; +}; + +class CommandBuffer { +public: + explicit CommandBuffer(int queue): m_queue(queue) { Allocate(); } + virtual ~CommandBuffer() { Free(); } + + KYTY_CLASS_NO_COPY(CommandBuffer); + + [[nodiscard]] bool IsInvalid() const; + + void Allocate(); + void Free(); + void Begin() const; + void End() const; + void Execute(); + void ExecuteWithSemaphore(VkSemaphore signal_semaphore = nullptr); + void ExecuteWithSemaphore(VkSemaphore wait_semaphore, VkPipelineStageFlags wait_stage, + VkSemaphore signal_semaphore); + void SetDebugInfo(uint32_t op, uint64_t submit_id, uint32_t arg0 = 0, uint32_t arg1 = 0, + uint32_t arg2 = 0, uint32_t arg3 = 0, uint64_t arg4 = 0); + void BeginRenderPass(VulkanFramebuffer* framebuffer, RenderColorInfo* colors, + uint32_t color_count, RenderDepthInfo* depth) const; + void EndRenderPass() const; + void WaitForFence(); + void WaitForFenceAndReset(); + void DeleteAfterFence(VulkanBuffer* buffer); + void RetainResourceUntilFence(std::shared_ptr resource); + void RecycleDescriptorAfterFence(VulkanDescriptorSet* set); + + [[nodiscard]] uint32_t GetIndex() const { return m_index; } + [[nodiscard]] int GetQueue() const { return m_queue; } + VulkanCommandPool* GetPool() { return m_pool; } + [[nodiscard]] bool IsExecute() const { return m_execute; } + +private: + friend class BufferCache; + + void RecycleDescriptorsAfterFence(); + + VulkanCommandPool* m_pool = nullptr; + uint32_t m_index = static_cast(-1); + int m_queue = -1; + bool m_execute = false; + uint64_t m_submit_seq = 0; + uint32_t m_debug_op = 0; + uint64_t m_debug_submit_id = 0; + uint32_t m_debug_arg0 = 0; + uint32_t m_debug_arg1 = 0; + uint32_t m_debug_arg2 = 0; + uint32_t m_debug_arg3 = 0; + uint64_t m_debug_arg4 = 0; + std::vector m_delete_after_fence; + FenceResourceRetainer m_fence_resources; + std::vector m_descriptor_sets_after_fence; + HostStreamBuffer m_host_stream; +}; + +void RenderDrawIndex(uint64_t submit_id, CommandBuffer* buffer, HW::Context* ctx, + HW::UserConfig* ucfg, HW::Shader* sh_ctx, uint32_t index_type_and_size, + uint32_t index_count, const void* index_addr, uint32_t flags, uint32_t type, + uint32_t instance_count = 1, uint32_t render_target_slice_offset = 0, + int32_t vertex_offset_add = 0, uint32_t first_instance = 0); +void RenderDrawIndexAuto(uint64_t submit_id, CommandBuffer* buffer, HW::Context* ctx, + HW::UserConfig* ucfg, HW::Shader* sh_ctx, uint32_t index_count, + uint32_t flags, uint32_t render_target_slice_offset = 0, + uint32_t instance_count = 1, uint32_t first_vertex = 0, + uint32_t first_instance = 0); +void RenderDispatchDirect(uint64_t submit_id, CommandBuffer* buffer, HW::Context* ctx, + HW::Shader* sh_ctx, uint32_t thread_group_x, uint32_t thread_group_y, + uint32_t thread_group_z, uint32_t mode); + +void GraphicsRenderInit(); +void GraphicsRenderCreateContext(); + +[[nodiscard]] bool ResolveComputeImageClear(const ShaderComputeInputInfo& input, uint32_t group_x, + uint32_t group_y, uint32_t group_z, uint32_t mode, + ShaderBufferResource* descriptor, + uint32_t* packed_clear, uint64_t* size); +[[nodiscard]] bool ResolveHtileClearTarget(const HW::DepthRenderTarget& target, + uint64_t descriptor_size, HtileClearTarget* resolved); + +void GraphicsRenderWriteAtEndOfPipe64(uint64_t submit_id, CommandBuffer* buffer, + uint64_t* dst_gpu_addr, uint64_t value); +void GraphicsRenderWriteAtEndOfPipeClockCounter(uint64_t submit_id, CommandBuffer* buffer, + uint64_t* dst_gpu_addr); +void GraphicsRenderWriteAtEndOfPipeClockCounterWithWriteBack(uint64_t submit_id, + CommandBuffer* buffer, + uint64_t* dst_gpu_addr); +void GraphicsRenderWriteAtEndOfPipe32(uint64_t submit_id, CommandBuffer* buffer, + uint32_t* dst_gpu_addr, uint32_t value); +void GraphicsRenderWriteAtEndOfPipeGds32(uint64_t submit_id, CommandBuffer* buffer, + uint32_t* dst_gpu_addr, uint32_t dw_offset, + uint32_t dw_num); +void GraphicsRenderWriteAtEndOfPipeWithInterruptWriteBackFlip32( + uint64_t submit_id, CommandBuffer* buffer, uint32_t* dst_gpu_addr, uint32_t value, int handle, + int index, int flip_mode, int64_t flip_arg); +void GraphicsRenderWriteAtEndOfPipeWithFlip32(uint64_t submit_id, CommandBuffer* buffer, + uint32_t* dst_gpu_addr, uint32_t value, int handle, + int index, int flip_mode, int64_t flip_arg); +void GraphicsRenderWriteAtEndOfPipeOnlyFlip(uint64_t submit_id, CommandBuffer* buffer, int handle, + int index, int flip_mode, int64_t flip_arg); +void GraphicsRenderWriteAtEndOfPipeWithWriteBack64(uint64_t submit_id, CommandBuffer* buffer, + uint64_t* dst_gpu_addr, uint64_t value); +void GraphicsRenderWriteAtEndOfPipeWithWriteBack32(uint64_t submit_id, CommandBuffer* buffer, + uint32_t* dst_gpu_addr, uint32_t value); +void GraphicsRenderWriteAtEndOfPipeWithInterruptWriteBack64(uint64_t submit_id, + CommandBuffer* buffer, + uint64_t* dst_gpu_addr, uint64_t value, + uint32_t context_id = 0); +void GraphicsRenderWriteAtEndOfPipeWithInterruptWriteBack32(uint64_t submit_id, + CommandBuffer* buffer, + uint32_t* dst_gpu_addr, uint32_t value, + uint32_t context_id = 0); +void GraphicsRenderWriteAtEndOfPipeWithInterrupt64(uint64_t submit_id, CommandBuffer* buffer, + uint64_t* dst_gpu_addr, uint64_t value, + uint32_t context_id = 0); +void GraphicsRenderWriteAtEndOfPipeWithInterrupt32(uint64_t submit_id, CommandBuffer* buffer, + uint32_t* dst_gpu_addr, uint32_t value, + uint32_t context_id = 0); +void GraphicsRenderMemoryBarrier(CommandBuffer* buffer); +void GraphicsRenderTextureBarrier(CommandBuffer* buffer, uint64_t vaddr, uint64_t size); +void GraphicsRenderDepthStencilBarrier(CommandBuffer* buffer, uint64_t vaddr, uint64_t size); +void GraphicsRenderDeleteBuffers(); +void GraphicsRenderTriggerEopEventAtEndOfPipe(CommandBuffer* buffer, uint32_t context_id); + +int GraphicsRenderAddEqEvent(LibKernel::EventQueue::KernelEqueue eq, int id, void* udata); +int GraphicsRenderDeleteEqEvent(LibKernel::EventQueue::KernelEqueue eq, int id); +void GraphicsRenderTriggerAgcUserInterrupt(); +void GraphicsRenderTriggerEopEvent(uint32_t context_id); + +void GraphicsRenderClearGds(uint64_t dw_offset, uint32_t dw_num, uint32_t clear_value); +void GraphicsRenderReadGds(uint32_t* dst, uint32_t dw_offset, uint32_t dw_size); + +} // namespace Libs::Graphics + +#endif /* EMULATOR_INCLUDE_EMULATOR_GRAPHICS_GRAPHICSRENDER_H_ */ diff --git a/src/graphics/host_gpu/renderer/renderCompute.cpp b/src/graphics/host_gpu/renderer/renderCompute.cpp new file mode 100644 index 000000000..8988e6bd6 --- /dev/null +++ b/src/graphics/host_gpu/renderer/renderCompute.cpp @@ -0,0 +1,577 @@ +#include "graphics/host_gpu/renderer/render.h" + +#include "graphics/host_gpu/renderer/renderInternal.h" + +#include "common/assert.h" +#include "common/common.h" +#include "common/emulatorConfig.h" +#include "common/file.h" +#include "common/logging/log.h" +#include "common/profiler.h" +#include "common/stringUtils.h" +#include "common/threads.h" +#include "graphics/guest_gpu/gpu_defs.h" +#include "graphics/guest_gpu/graphicsRun.h" +#include "graphics/guest_gpu/hardwareContext.h" +#include "graphics/guest_gpu/tile.h" +#include "graphics/host_gpu/graphicContext.h" +#include "graphics/host_gpu/objects/label.h" +#include "graphics/host_gpu/renderer/descriptorCache.h" +#include "graphics/host_gpu/renderer/framebufferCache.h" +#include "graphics/host_gpu/renderer/pipelineCache.h" +#include "graphics/host_gpu/renderer/renderContext.h" +#include "graphics/host_gpu/renderer/renderState.h" +#include "graphics/host_gpu/renderer/shaderResourceBarrier.h" +#include "graphics/host_gpu/renderer/shaderSubgroup.h" +#include "graphics/host_gpu/utils.h" +#include "graphics/shader/recompiler/ResourceMaterialization.h" +#include "graphics/shader/recompiler/ShaderIR.h" +#include "graphics/shader/shader.h" +#include "kernel/eventQueue.h" +#include "kernel/pthread.h" +#include "libs/errno.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +// IWYU pragma: no_forward_declare VkImageView_T + +namespace Libs::Graphics { +static uint64_t BufferDescriptorSize(const ShaderBufferResource& descriptor) { + const uint64_t records = descriptor.NumRecords(); + const uint64_t stride = descriptor.Stride(); + if (stride != 0 && records > UINT64_MAX / stride) { + EXIT("compute buffer descriptor footprint overflow\n"); + } + return stride == 0 ? records : records * stride; +} + +bool ResolveHtileClearTarget(const HW::DepthRenderTarget& z, uint64_t descriptor_size, + HtileClearTarget* resolved) { + if (resolved == nullptr) { + return false; + } + *resolved = {}; + const bool has_stencil = + z.stencil_info.format != Prospero::GpuEnumValue(Prospero::StencilFormat::kInvalid); + const bool depth_format_supported = + z.z_info.format == Prospero::GpuEnumValue(Prospero::DepthFormat::kZ16) || + z.z_info.format == Prospero::GpuEnumValue(Prospero::DepthFormat::kZ32F); + const bool msaa_compat = depth_msaa_single_sample_compatible(z.z_info.num_samples); + const bool supported_depth_state = + z.z_info.tile_surface_enable && depth_format_supported && + !(z.z_info.format == Prospero::GpuEnumValue(Prospero::DepthFormat::kZ16) && has_stencil) && + z.z_info.tile_mode_index == 0 && (z.z_info.num_samples == 0 || msaa_compat) && + z.z_info.zrange_precision <= 1 && !z.z_info.expclear_enabled && + !z.z_info.embedded_sample_locations && !z.z_info.partially_resident && + z.z_info.num_mip_levels == 0 && z.z_info.plane_compression == 0 && + z.depth_view.current_mip_level == 0 && z.depth_view.slice_start == 0 && + z.depth_view.slice_max == 0 && z.depth_info.addr5_swizzle_mask == 0 && + z.depth_info.array_mode == 0 && z.depth_info.pipe_config == 0 && + z.depth_info.bank_width == 0 && z.depth_info.bank_height == 0 && + z.depth_info.macro_tile_aspect == 0 && z.depth_info.num_banks == 0; + const bool supported_stencil_state = + z.stencil_info.tile_mode_index == 0 && z.stencil_info.tile_split == 0 && + !z.stencil_info.expclear_enabled && + depth_htile_stencil_acceleration_compatible(has_stencil, true, + z.stencil_info.tile_stencil_disable) && + !z.stencil_info.texture_compatible_stencil && !z.stencil_info.partially_resident && + (!has_stencil || + (z.stencil_info.format == Prospero::GpuEnumValue(Prospero::StencilFormat::k8UInt) && + !z.depth_view.stencil_write_disable)); + const bool supported_htile_state = + z.htile_surface.linear == 0 && z.htile_surface.full_cache == 0 && + z.htile_surface.htile_uses_preload_win == 0 && z.htile_surface.preload == 0 && + z.htile_surface.prefetch_width == 0 && z.htile_surface.prefetch_height == 0 && + z.htile_surface.dst_outside_zero_to_one == 0; + const bool supported_addresses = + z.z_read_base_addr != 0 && z.z_write_base_addr == z.z_read_base_addr && + (z.z_read_base_addr & 0xffffu) == 0 && + (has_stencil ? z.stencil_read_base_addr != 0 && + z.stencil_write_base_addr == z.stencil_read_base_addr && + (z.stencil_read_base_addr & 0xffffu) == 0 + : z.stencil_read_base_addr == 0 && z.stencil_write_base_addr == 0) && + z.htile_data_base_addr != 0 && (z.htile_data_base_addr & 0x7fffu) == 0 && + descriptor_size != 0 && (descriptor_size & 0x7fffu) == 0 && + z.htile_data_base_addr < TRACKER_ADDRESS_SIZE && + descriptor_size <= TRACKER_ADDRESS_SIZE - z.htile_data_base_addr; + if (!supported_depth_state || !supported_stencil_state || !supported_htile_state || + !supported_addresses) { + return false; + } + if (msaa_compat) { + static std::atomic logged_fragments = 0; + const uint32_t bit = 1u << z.z_info.num_samples; + if ((logged_fragments.fetch_or(bit, std::memory_order_relaxed) & bit) == 0) { + LOGF("HTileClear: compatibility: treating PS5 %ux depth fragments as single-sample\n", + bit); + } + } + + const bool size_xy_valid = z.size.valid && (z.size.x_max != 0 || z.size.y_max != 0); + const bool wh_valid = z.width_height_valid && z.width != 0 && z.height != 0; + if (!size_xy_valid && !wh_valid) { + // Prospero emits an exact full-surface metadata descriptor even + // when the compute context omits depth extent registers. Admit only that wholly absent + // layout state; partially programmed layouts remain unsupported. + const bool descriptor_backed_state = + !z.size.valid && z.size.x_max == 0 && z.size.y_max == 0 && !z.width_height_valid && + z.width == 0 && z.height == 0 && !z.pitch_height_valid && z.pitch_div8_minus1 == 0 && + z.height_div8_minus1 == 0 && z.slice_div64_minus1 == 0; + if (!descriptor_backed_state) { + return false; + } + *resolved = {.address = z.htile_data_base_addr, .size = descriptor_size}; + return true; + } + const uint32_t width = size_xy_valid ? static_cast(z.size.x_max) + 1u : z.width; + const uint32_t height = size_xy_valid ? static_cast(z.size.y_max) + 1u : z.height; + if (width > 16384 || height > 16384 || + (size_xy_valid && wh_valid && (width != z.width || height != z.height)) || + (!z.pitch_height_valid && + (z.pitch_div8_minus1 != 0 || z.height_div8_minus1 != 0 || z.slice_div64_minus1 != 0))) { + return false; + } + + const bool z16 = z.z_info.format == Prospero::GpuEnumValue(Prospero::DepthFormat::kZ16); + const uint32_t guest_format = Prospero::GpuEnumValue(z16 ? Prospero::BufferFormat::k16UNorm + : Prospero::BufferFormat::k32Float); + const uint32_t bytes = z16 ? 2u : 4u; + const uint32_t pitch = TileGetTexturePitch(guest_format, width, 1, + Prospero::GpuEnumValue(Prospero::TileMode::kDepth)); + if (z.pitch_height_valid && ((static_cast(z.pitch_div8_minus1) + 1u) * 8u != pitch || + (static_cast(z.height_div8_minus1) + 1u) * 8u != + ((static_cast(height) + 7u) & ~7ull))) { + return false; + } + const uint32_t block_width = z16 ? 256u : 128u; + const uint64_t padded_width = + (static_cast(pitch) + block_width - 1u) & ~(block_width - 1u); + const uint64_t padded_height = (static_cast(height) + 127u) & ~127ull; + if (padded_width > UINT64_MAX / padded_height || + padded_width * padded_height > UINT64_MAX / bytes) { + return false; + } + const uint64_t expected_depth_size = padded_width * padded_height * bytes; + TileSizeAlign depth_size {}; + TileSizeAlign stencil_size {}; + TileSizeAlign htile_size {}; + if (!TileGetDepthSize(width, height, 0, z.z_info.format, z.stencil_info.format, true, + &stencil_size, &htile_size, &depth_size) || + expected_depth_size == 0 || expected_depth_size > UINT32_MAX || depth_size.align != 65536 || + depth_size.size != expected_depth_size || + (has_stencil != (stencil_size.align == 65536 && stencil_size.size != 0)) || + htile_size.align != 32768 || htile_size.size == 0 || htile_size.size != descriptor_size || + (z.pitch_height_valid && + (static_cast(z.slice_div64_minus1) + 1u) * 64u != expected_depth_size)) { + return false; + } + *resolved = {.address = z.htile_data_base_addr, .size = htile_size.size}; + return true; +} + +static void ValidateFullHtileClearDispatch(const ShaderComputeInputInfo& input, + const ShaderBufferResource& metadata, uint32_t group_x, + uint32_t group_y, uint32_t group_z, uint32_t mode) { + const bool thread_dimensions = input.dispatch_thread_dimensions; + const bool supported_shape = + input.threads_num[0] == 64 && input.threads_num[1] == 1 && input.threads_num[2] == 1 && + group_x != 0 && group_y == 1 && group_z == 1 && input.group_id[0] && !input.group_id[1] && + !input.group_id[2] && input.thread_ids_num == 1 && input.wave_size == 32 && + !input.tg_size_en && mode == (thread_dimensions ? 0x61u : 0x41u); + const bool dimensions_match = + thread_dimensions + ? input.dispatch_threads_num[0] == group_x && input.dispatch_threads_num[1] == 1 && + input.dispatch_threads_num[2] == 1 && group_x % input.threads_num[0] == 0 + : input.dispatch_threads_num[0] == 0 && input.dispatch_threads_num[1] == 0 && + input.dispatch_threads_num[2] == 0; + const uint64_t launched_threads = + thread_dimensions ? group_x : static_cast(group_x) * input.threads_num[0]; + if (!supported_shape || !dimensions_match || metadata.Stride() == 0 || + launched_threads != metadata.NumRecords()) { + EXIT("HTile compute clear does not cover the complete metadata surface\n"); + } +} + +static bool TryConsumeComputeMetaClear(const ShaderComputeInputInfo& input, const HW::Context& ctx, + uint32_t group_x, uint32_t group_y, uint32_t group_z, + uint32_t mode) { + const auto& program = *input.stage.program; + const auto& resources = *input.stage.resources; + if (resources.buffers.size() != program.info.buffers.size()) { + EXIT("compute runtime buffer count does not match shader metadata\n"); + } + const auto& z = ctx.GetDepthRenderTarget(); + const uint64_t meta_addr = z.htile_data_base_addr; + auto* cache = g_render_ctx->GetTextureCache(); + uint32_t current_references = 0; + uint32_t registered_writes = 0; + uint64_t described_meta_size = 0; + HtileClearTarget registered_target {}; + for (uint32_t i = 0; i < program.info.buffers.size(); i++) { + const auto& resource = program.info.buffers[i]; + const auto descriptor = DecodeNativeDescriptor(resources.buffers[i]); + const auto descriptor_size = BufferDescriptorSize(descriptor); + if (meta_addr != 0 && descriptor.Base48() == meta_addr) { + current_references++; + described_meta_size = descriptor_size; + } + // An exact registered metadata range remains + // identifiable even when it is no longer the currently bound depth target. + if (resource.written && cache->IsMetaRange(descriptor.Base48(), descriptor_size)) { + registered_writes++; + registered_target = {.address = descriptor.Base48(), .size = descriptor_size}; + } + } + if (current_references == 0 && registered_writes == 0) { + return false; + } + if (current_references > 1 || (current_references == 0 && registered_writes > 1)) { + EXIT("HTile clear has ambiguous metadata descriptors: current=%u registered=%u\n", + current_references, registered_writes); + } + HtileClearTarget target {}; + if (current_references != 0) { + if (!ResolveHtileClearTarget(z, described_meta_size, &target)) { + EXIT("unsupported HTile compute-clear target state\n"); + } + cache->RegisterMeta(target.address, target.size); + } else { + target = registered_target; + } + g_render_ctx->GetBufferCache()->ValidateGpuAccess(target.address, target.size, false, true); + + uint32_t metadata_writes = 0; + ShaderBufferResource metadata_descriptor {}; + if (!program.info.images.empty() || !program.info.samplers.empty() || + !program.info.addresses.empty()) { + EXIT("HTile clear with non-buffer resources is unsupported: images=%zu samplers=%zu " + "addresses=%zu\n", + program.info.images.size(), program.info.samplers.size(), + program.info.addresses.size()); + } + for (uint32_t i = 0; i < program.info.buffers.size(); i++) { + const auto& resource = program.info.buffers[i]; + const auto descriptor = DecodeNativeDescriptor(resources.buffers[i]); + const auto descriptor_size = BufferDescriptorSize(descriptor); + if (descriptor.Base48() == target.address) { + if (!resource.written || resource.read || resource.atomic || + descriptor_size != target.size || descriptor.SwizzleEnabled() || + descriptor.IndexStride() != 0 || descriptor.AddTid() || + resource.packed_stride != descriptor.PackedStride()) { + EXIT("unsupported HTile compute metadata access\n"); + } + metadata_descriptor = descriptor; + metadata_writes++; + continue; + } + if (resource.written || !resource.read || resource.atomic || descriptor.Base48() == 0 || + descriptor_size == 0 || cache->HasMetaOverlap(descriptor.Base48(), descriptor_size)) { + EXIT("unsupported HTile clear side-buffer access\n"); + } + g_render_ctx->GetBufferCache()->ValidateGpuAccess(descriptor.Base48(), descriptor_size, + true, false); + } + if (metadata_writes != 1) { + EXIT("HTile clear requires exactly one write-only metadata buffer, writes=%u\n", + metadata_writes); + } + ValidateFullHtileClearDispatch(input, metadata_descriptor, group_x, group_y, group_z, mode); + if (!cache->ClearMeta(target.address)) { + EXIT("failed to record HTile compute clear\n"); + } + const bool descriptor_backed = + current_references != 0 && !z.size.valid && !z.width_height_valid && !z.pitch_height_valid; + const uint32_t source_bit = current_references == 0 ? 1u : descriptor_backed ? 2u : 4u; + static std::atomic logged_sources {0}; + if ((logged_sources.fetch_or(source_bit, std::memory_order_relaxed) & source_bit) == 0) { + LOGF("HTileClear: native virtual clear source=%s shader=0x%016" PRIx64 " addr=0x%016" PRIx64 + " size=0x%016" PRIx64 "\n", + current_references == 0 ? "registered" + : descriptor_backed ? "descriptor" + : "derived", + program.shader_hash, target.address, target.size); + } + return true; +} + +bool ResolveComputeImageClear(const ShaderComputeInputInfo& input, uint32_t group_x, + uint32_t group_y, uint32_t group_z, uint32_t mode, + ShaderBufferResource* resolved_descriptor, uint32_t* resolved_clear, + uint64_t* resolved_size) { + if (resolved_descriptor == nullptr || resolved_clear == nullptr || resolved_size == nullptr) { + return false; + } + const auto& program = *input.stage.program; + const auto& resources = *input.stage.resources; + if (program.info.buffers.size() != 1 || resources.buffers.size() != 1 || + !program.info.images.empty() || !program.info.samplers.empty() || + !program.info.addresses.empty() || !resources.images.empty() || + !resources.samplers.empty() || !resources.addresses.empty()) { + return false; + } + const auto& resource = program.info.buffers.front(); + const auto& raw = resources.buffers.front(); + const auto descriptor = DecodeNativeDescriptor(raw); + if (!resource.formatted || !resource.written || resource.read || resource.atomic || + resource.scalar || resource.max_byte_extent != 16 || descriptor.Stride() != 16 || + descriptor.Format() != Prospero::GpuEnumValue(Prospero::BufferFormat::k32_32_32_32UInt) || + descriptor.SwizzleEnabled() || descriptor.IndexStride() != 0 || descriptor.AddTid() || + resource.packed_stride != descriptor.PackedStride() || raw.dword_count != 4 || + program.user_data_base != 0 || resources.user_data.size() != 8) { + return false; + } + for (uint32_t i = 0; i < raw.dword_count; i++) { + if (raw.dwords[i] != resources.user_data[i]) { + return false; + } + } + const uint32_t clear = resources.user_data[4]; + if (resources.user_data[5] != clear || resources.user_data[6] != clear || + resources.user_data[7] != clear) { + return false; + } + const bool full_dispatch = + input.dispatch_thread_dimensions && input.threads_num[0] == 64 && + input.threads_num[1] == 1 && input.threads_num[2] == 1 && group_x != 0 && group_y == 1 && + group_z == 1 && input.dispatch_threads_num[0] == group_x && + input.dispatch_threads_num[1] == 1 && input.dispatch_threads_num[2] == 1 && + input.group_id[0] && !input.group_id[1] && !input.group_id[2] && + input.thread_ids_num == 1 && input.wave_size == 32 && !input.tg_size_en && mode == 0x61u && + group_x % input.threads_num[0] == 0 && descriptor.NumRecords() == group_x; + const auto size = BufferDescriptorSize(descriptor); + if (!full_dispatch || size == 0) { + return false; + } + *resolved_descriptor = descriptor; + *resolved_clear = clear; + *resolved_size = size; + return true; +} + +static bool TryConsumeComputeImageClear(const ShaderComputeInputInfo& input, CommandBuffer* command, + uint32_t group_x, uint32_t group_y, uint32_t group_z, + uint32_t mode) { + ShaderBufferResource descriptor; + uint32_t packed_clear = 0; + uint64_t size = 0; + if (!ResolveComputeImageClear(input, group_x, group_y, group_z, mode, &descriptor, + &packed_clear, &size)) { + return false; + } + auto* cache = g_render_ctx->GetTextureCache(); + if (!cache->ClearColorImageFromBuffer(command, descriptor.Base48(), size, packed_clear) && + !cache->ClearDepthImageFromBuffer(command, descriptor.Base48(), size, packed_clear) && + !cache->ClearStencilImageFromBuffer(command, descriptor.Base48(), size, packed_clear)) { + return false; + } + static std::atomic logged_clears {0}; + if (logged_clears.fetch_add(1, std::memory_order_relaxed) < 32) { + LOGF("GraphicsRenderDispatchDirect: compute image clear shader=0x%016" PRIx64 + " addr=0x%016" PRIx64 " size=0x%016" PRIx64 " value=0x%08" PRIx32 "\n", + input.stage.program->shader_hash, descriptor.Base48(), size, packed_clear); + } + return true; +} + +void RenderDispatchDirect(uint64_t submit_id, CommandBuffer* buffer, HW::Context* ctx, + HW::Shader* sh_ctx, uint32_t thread_group_x, uint32_t thread_group_y, + uint32_t thread_group_z, uint32_t mode) { + EXIT_IF(ctx == nullptr); + EXIT_IF(g_render_ctx == nullptr); + EXIT_IF(buffer == nullptr); + EXIT_IF(buffer->IsInvalid()); + + buffer->SetDebugInfo(static_cast(CommandBufferDebugOp::DispatchDirect), submit_id, + thread_group_x, thread_group_y, thread_group_z, mode, + sh_ctx != nullptr ? sh_ctx->GetCs().cs_regs.data_addr : 0); + + Common::LockGuard lock(g_render_ctx->GetMutex()); + + if (sh_ctx->GetCs().cs_regs.data_addr == 0) { + LOGF("GraphicsRenderDispatchDirect: temporary: ignoring dispatch with null CS shader, " + "groups=%ux%ux%u mode=%u\n", + thread_group_x, thread_group_y, thread_group_z, mode); + return; + } + + if (!ShaderAddressValid(sh_ctx->GetCs().cs_regs.data_addr)) { + return; + } + + constexpr uint32_t DISPATCH_INITIATOR_USE_THREAD_DIMENSIONS = 1u << 5u; + constexpr uint32_t DISPATCH_INITIATOR_BASE_BITS = 0x41u; + constexpr uint32_t DISPATCH_INITIATOR_MODIFIER_BITS = 0xa038u; + constexpr uint32_t DISPATCH_INITIATOR_KNOWN_MASK = + DISPATCH_INITIATOR_BASE_BITS | DISPATCH_INITIATOR_MODIFIER_BITS; + + const uint32_t unknown_mode_bits = mode & ~DISPATCH_INITIATOR_KNOWN_MASK; + if (unknown_mode_bits != 0) { + static std::atomic log_count {0}; + if (log_count.fetch_add(1, std::memory_order_relaxed) < 32) { + LOGF("GraphicsRenderDispatchDirect: unknown dispatch initiator bits " + "mode=0x%08" PRIx32 " unknown=0x%08" PRIx32 " shader=0x%016" PRIx64 + " groups=%ux%ux%u\n", + mode, unknown_mode_bits, sh_ctx->GetCs().cs_regs.data_addr, thread_group_x, + thread_group_y, thread_group_z); + } + } + + const auto& cs_regs = sh_ctx->GetCs(); + const auto& sh_regs = ctx->GetShaderRegisters(); + + ShaderComputeInputInfo input_info {}; + std::span cs_shader; + if (!ShaderCompileInfoCS(&cs_regs, &sh_regs, &input_info, &cs_shader)) { + EXIT("ShaderCompileInfoCS failed for dispatch with CS shader 0x%016" PRIx64 "\n", + cs_regs.cs_regs.data_addr); + } + + const bool use_thread_dimensions = (mode & DISPATCH_INITIATOR_USE_THREAD_DIMENSIONS) != 0; + if (use_thread_dimensions) { + input_info.dispatch_thread_dimensions = true; + input_info.dispatch_threads_num[0] = thread_group_x; + input_info.dispatch_threads_num[1] = thread_group_y; + input_info.dispatch_threads_num[2] = thread_group_z; + } + + const uint32_t frame_num = GraphicsRunGetFrameNum(); + const bool large_workgroup = + (input_info.threads_num[0] * input_info.threads_num[1] * input_info.threads_num[2] >= 512); + const auto& program = *input_info.stage.program; + const auto& resources = *input_info.stage.resources; + if (TryConsumeComputeMetaClear(input_info, *ctx, thread_group_x, thread_group_y, thread_group_z, + mode)) { + return; + } + if (TryConsumeComputeImageClear(input_info, buffer, thread_group_x, thread_group_y, + thread_group_z, mode)) { + return; + } + const auto sampled_images = std::count_if( + program.info.images.begin(), program.info.images.end(), [](const auto& image) { + return image.kind == ShaderRecompiler::IR::ResourceKind::Image || + image.kind == ShaderRecompiler::IR::ResourceKind::ImageUint; + }); + const bool has_sampler = !program.info.samplers.empty(); + static std::atomic dispatch_log_count {0}; + if ((large_workgroup || has_sampler) && + dispatch_log_count.fetch_add(1, std::memory_order_relaxed) < 512) { + LOGF("GraphicsRenderDispatchDirect: frame=%u shader=0x%016" PRIx64 + " groups=%ux%ux%u mode=0x%08" PRIx32 " local=%ux%ux%u " + "buffers=%zu textures=%zu sampled=%zu storage=%zu samplers=%zu push=%u\n", + frame_num, sh_ctx->GetCs().cs_regs.data_addr, thread_group_x, thread_group_y, + thread_group_z, mode, input_info.threads_num[0], input_info.threads_num[1], + input_info.threads_num[2], program.info.buffers.size(), program.info.images.size(), + sampled_images, program.info.images.size() - sampled_images, + program.info.samplers.size(), program.bindings.push_constant_size); + for (uint32_t i = 0; i < program.info.buffers.size(); i++) { + const auto& buffer = program.info.buffers[i]; + const auto r = DecodeNativeDescriptor(resources.buffers[i]); + LOGF(" CS buffer[%u]: source=%u usage=%s addr=0x%012" PRIx64 + " stride=%u records=%u format=%u\n", + i, buffer.source, buffer.written ? "read-write" : "read-only", r.Base48(), + r.Stride(), r.NumRecords(), r.Format()); + } + for (uint32_t i = 0; i < program.info.images.size(); i++) { + const auto& image = program.info.images[i]; + const auto r = DecodeNativeDescriptor(resources.images[i]); + LOGF(" CS texture[%u]: source=%u usage=%s sampled=%s addr=0x%010" PRIx64 + " type=%u fmt=%u extent=%ux%u depth=%u levels=%u tile=%u\n", + i, image.source, image.written ? "read-write" : "read-only", + (image.kind == ShaderRecompiler::IR::ResourceKind::Image || + image.kind == ShaderRecompiler::IR::ResourceKind::ImageUint) + ? "true" + : "false", + r.Base40(), static_cast(r.Type()), r.Format(), + static_cast(r.Width5()) + 1u, static_cast(r.Height5()) + 1u, + static_cast(r.Depth()) + 1u, + std::max(static_cast(r.LastLevel()), + static_cast(r.MaxMip())) + + 1u, + r.TileMode()); + } + for (uint32_t i = 0; i < program.info.samplers.size(); i++) { + const auto r = DecodeNativeDescriptor(resources.samplers[i]); + LOGF(" CS sampler[%u]: source=%u clamp=%u/%u/%u filter=%u/%u/%u mip=%u " + "lod=%u-%u bias=%d\n", + i, program.info.samplers[i].source, static_cast(r.ClampX()), + static_cast(r.ClampY()), static_cast(r.ClampZ()), + static_cast(r.XyMagFilter()), static_cast(r.XyMinFilter()), + static_cast(r.ZFilter()), static_cast(r.MipFilter()), + static_cast(r.MinLod()), static_cast(r.MaxLod()), + static_cast(r.LodBias())); + } + } + + if (use_thread_dimensions) { + auto groups_from_threads = [](uint32_t threads, uint32_t group_size) { + return (threads == 0 + ? 0u + : (threads + std::max(group_size, 1u) - 1u) / std::max(group_size, 1u)); + }; + + const uint32_t old_x = thread_group_x; + const uint32_t old_y = thread_group_y; + const uint32_t old_z = thread_group_z; + thread_group_x = groups_from_threads(thread_group_x, cs_regs.cs_regs.num_thread_x); + thread_group_y = groups_from_threads(thread_group_y, cs_regs.cs_regs.num_thread_y); + thread_group_z = groups_from_threads(thread_group_z, cs_regs.cs_regs.num_thread_z); + + static std::atomic log_count {0}; + if (log_count.fetch_add(1, std::memory_order_relaxed) < 32) { + LOGF("GraphicsRenderDispatchDirect: use-thread-dimensions %ux%ux%u / %ux%ux%u -> " + "groups %ux%ux%u\n", + old_x, old_y, old_z, std::max(cs_regs.cs_regs.num_thread_x, 1u), + std::max(cs_regs.cs_regs.num_thread_y, 1u), + std::max(cs_regs.cs_regs.num_thread_z, 1u), thread_group_x, thread_group_y, + thread_group_z); + } + } + + if (thread_group_x == 0 || thread_group_y == 0 || thread_group_z == 0) { + static std::atomic log_count {0}; + if (log_count.fetch_add(1, std::memory_order_relaxed) < 32) { + LOGF("GraphicsRenderDispatchDirect: skipping zero-sized dispatch groups=%ux%ux%u " + "mode=0x%08" PRIx32 " shader=0x%016" PRIx64 "\n", + thread_group_x, thread_group_y, thread_group_z, mode, + sh_ctx->GetCs().cs_regs.data_addr); + } + return; + } + + auto* vk_buffer = buffer->GetPool()->buffers[buffer->GetIndex()]; + auto* pipeline = g_render_ctx->GetPipelineCache()->CreateComputePipeline( + &input_info, &sh_ctx->GetCs(), cs_shader); + + vkCmdBindPipeline(vk_buffer, VK_PIPELINE_BIND_POINT_COMPUTE, pipeline->pipeline); + + const auto address_writes = BindDescriptors( + submit_id, buffer, VK_PIPELINE_BIND_POINT_COMPUTE, pipeline->pipeline_layout, + input_info.stage, VK_SHADER_STAGE_COMPUTE_BIT, DescriptorCache::Stage::Compute); + + vkCmdDispatch(vk_buffer, thread_group_x, thread_group_y, thread_group_z); + + bool has_storage_writes = HasShaderBufferWrites(input_info.stage); + has_storage_writes = MarkShaderAddressWrites(address_writes) || has_storage_writes; + has_storage_writes = + std::any_of(program.info.images.begin(), program.info.images.end(), + [](const auto& image) { + return image.written && + (image.kind == ShaderRecompiler::IR::ResourceKind::StorageImage || + image.kind == ShaderRecompiler::IR::ResourceKind::StorageImageUint); + }) || + has_storage_writes; + if (has_storage_writes) { + ShaderWriteBarrier(vk_buffer, VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT); + } +} + +} // namespace Libs::Graphics diff --git a/src/graphics/host_gpu/renderer/renderContext.cpp b/src/graphics/host_gpu/renderer/renderContext.cpp new file mode 100644 index 000000000..8e09dfa56 --- /dev/null +++ b/src/graphics/host_gpu/renderer/renderContext.cpp @@ -0,0 +1,97 @@ +#include "graphics/host_gpu/renderer/renderContext.h" + +#include "common/assert.h" +#include "common/logging/log.h" +#include "graphics/presentation/window.h" +#include "kernel/pthread.h" +#include "libs/errno.h" + +#include + +namespace Libs::Graphics { + +void RenderContext::SetGraphicCtx(GraphicContext* ctx) { + EXIT_IF(ctx == nullptr); + + Common::LockGuard lock(m_graphic_ctx_mutex); + + m_graphic_ctx = ctx; +} + +GraphicContext* RenderContext::GetGraphicCtx() { + EXIT_IF(g_render_ctx == nullptr); + + { + Common::LockGuard lock(m_graphic_ctx_mutex); + if (m_graphic_ctx != nullptr) { + return m_graphic_ctx; + } + } + + WindowWaitForGraphicInitialized(); + + auto* ctx = WindowGetGraphicContext(); + LOGF("Graphic context initialized\n ctx = %p\n", static_cast(ctx)); + EXIT_IF(ctx == nullptr); + + GraphicContext* ret = nullptr; + { + Common::LockGuard lock(m_graphic_ctx_mutex); + if (m_graphic_ctx == nullptr) { + m_graphic_ctx = ctx; + } + ret = m_graphic_ctx; + } + + return ret; +} + +void RenderContext::AddEopEq(LibKernel::EventQueue::KernelEqueue eq, int id) { + Common::LockGuard lock(m_eop_mutex); + + auto it = std::find_if(m_eop_eqs.begin(), m_eop_eqs.end(), [eq, id](const auto& entry) { + return entry.eq == eq && entry.id == id; + }); + if (it != m_eop_eqs.end()) { + it->count++; + return; + } + + m_eop_eqs.push_back({eq, id, 1}); +} + +void RenderContext::DeleteEopEq(LibKernel::EventQueue::KernelEqueue eq, int id) { + Common::LockGuard lock(m_eop_mutex); + + auto it = std::find_if(m_eop_eqs.begin(), m_eop_eqs.end(), [eq, id](const auto& entry) { + return entry.eq == eq && entry.id == id; + }); + if (it == m_eop_eqs.end()) { + return; + } + + if (--it->count == 0) { + m_eop_eqs.erase(it); + } +} + +void RenderContext::TriggerEopEvent(uint32_t context_id) { + Common::LockGuard lock(m_eop_mutex); + + for (auto& eop_entry: m_eop_eqs) { + if (eop_entry.eq != nullptr) { + const auto id = static_cast(eop_entry.id); + auto result = LibKernel::EventQueue::KernelTriggerEvent( + eop_entry.eq, id, LibKernel::EventQueue::KERNEL_EVFILT_GRAPHICS, + reinterpret_cast(static_cast(context_id))); + EXIT_NOT_IMPLEMENTED(result != OK && result != LibKernel::KERNEL_ERROR_ENOENT); + } + } + + auto tsc = LibKernel::KernelReadTsc(); + auto result = LibKernel::EventQueue::KernelTriggerUserEventForAll(AGC_USER_INTERRUPT_EVENT, + reinterpret_cast(tsc)); + EXIT_NOT_IMPLEMENTED(result != OK && result != LibKernel::KERNEL_ERROR_ENOENT); +} + +} // namespace Libs::Graphics diff --git a/src/graphics/host_gpu/renderer/renderContext.h b/src/graphics/host_gpu/renderer/renderContext.h new file mode 100644 index 000000000..400b8026a --- /dev/null +++ b/src/graphics/host_gpu/renderer/renderContext.h @@ -0,0 +1,72 @@ +#ifndef EMULATOR_SRC_GRAPHICS_HOST_GPU_RENDERER_RENDERCONTEXT_H_ +#define EMULATOR_SRC_GRAPHICS_HOST_GPU_RENDERER_RENDERCONTEXT_H_ + +#include "common/abi.h" +#include "common/assert.h" +#include "common/common.h" +#include "common/threads.h" +#include "graphics/host_gpu/renderer/bufferCache.h" +#include "graphics/host_gpu/renderer/descriptorCache.h" +#include "graphics/host_gpu/renderer/framebufferCache.h" +#include "graphics/host_gpu/renderer/gdsBuffer.h" +#include "graphics/host_gpu/renderer/gpuResourceManager.h" +#include "graphics/host_gpu/renderer/pipelineCache.h" +#include "graphics/host_gpu/renderer/samplerCache.h" +#include "graphics/host_gpu/renderer/textureCache.h" +#include "kernel/eventQueue.h" + +#include + +namespace Libs::Graphics { + +constexpr int AGC_USER_INTERRUPT_EVENT = 0x1800; + +class RenderContext { +public: + RenderContext() { EXIT_NOT_IMPLEMENTED(!Common::Thread::IsMainThread()); } + virtual ~RenderContext() { KYTY_NOT_IMPLEMENTED; } + KYTY_CLASS_NO_COPY(RenderContext); + + void SetGraphicCtx(GraphicContext* ctx); + GraphicContext* GetGraphicCtx(); + + Common::Mutex& GetMutex() { return m_mutex; } + PipelineCache* GetPipelineCache() { return &m_pipeline_cache; } + DescriptorCache* GetDescriptorCache() { return &m_descriptor_cache; } + FramebufferCache* GetFramebufferCache() { return &m_framebuffer_cache; } + SamplerCache* GetSamplerCache() { return &m_sampler_cache; } + GdsBuffer* GetGdsBuffer() { return &m_gds_buffer; } + GpuResourceManager* GetGpuResources() { return &m_gpu_resources; } + BufferCache* GetBufferCache() { return m_gpu_resources.GetBufferCache(); } + TextureCache* GetTextureCache() { return m_gpu_resources.GetTextureCache(); } + + void AddEopEq(LibKernel::EventQueue::KernelEqueue eq, int id); + void DeleteEopEq(LibKernel::EventQueue::KernelEqueue eq, int id); + void TriggerEopEvent(uint32_t context_id); + +private: + struct EopEqRegistration { + LibKernel::EventQueue::KernelEqueue eq = nullptr; + int id = 0; + uint32_t count = 0; + }; + + Common::Mutex m_mutex; + Common::Mutex m_graphic_ctx_mutex; + PipelineCache m_pipeline_cache; + DescriptorCache m_descriptor_cache; + FramebufferCache m_framebuffer_cache; + SamplerCache m_sampler_cache; + GraphicContext* m_graphic_ctx = nullptr; + GdsBuffer m_gds_buffer; + GpuResourceManager m_gpu_resources; + + Common::Mutex m_eop_mutex; + std::vector m_eop_eqs; +}; + +extern RenderContext* g_render_ctx; + +} // namespace Libs::Graphics + +#endif // EMULATOR_SRC_GRAPHICS_HOST_GPU_RENDERER_RENDERCONTEXT_H_ diff --git a/src/graphics/host_gpu/renderer/renderDraw.cpp b/src/graphics/host_gpu/renderer/renderDraw.cpp new file mode 100644 index 000000000..a6380979b --- /dev/null +++ b/src/graphics/host_gpu/renderer/renderDraw.cpp @@ -0,0 +1,1295 @@ +#include "graphics/host_gpu/renderer/render.h" + +#include "graphics/host_gpu/renderer/renderInternal.h" + +#include "common/assert.h" +#include "common/common.h" +#include "common/emulatorConfig.h" +#include "common/file.h" +#include "common/logging/log.h" +#include "common/profiler.h" +#include "common/stringUtils.h" +#include "common/threads.h" +#include "graphics/guest_gpu/gpu_defs.h" +#include "graphics/guest_gpu/graphicsRun.h" +#include "graphics/guest_gpu/hardwareContext.h" +#include "graphics/guest_gpu/tile.h" +#include "graphics/host_gpu/graphicContext.h" +#include "graphics/host_gpu/objects/label.h" +#include "graphics/host_gpu/renderer/descriptorCache.h" +#include "graphics/host_gpu/renderer/framebufferCache.h" +#include "graphics/host_gpu/renderer/pipelineCache.h" +#include "graphics/host_gpu/renderer/renderContext.h" +#include "graphics/host_gpu/renderer/renderState.h" +#include "graphics/host_gpu/renderer/shaderResourceBarrier.h" +#include "graphics/host_gpu/renderer/shaderSubgroup.h" +#include "graphics/host_gpu/utils.h" +#include "graphics/shader/recompiler/ResourceMaterialization.h" +#include "graphics/shader/recompiler/ShaderIR.h" +#include "graphics/shader/shader.h" +#include "kernel/eventQueue.h" +#include "kernel/pthread.h" +#include "libs/errno.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +// IWYU pragma: no_forward_declare VkImageView_T + +namespace Libs::Graphics { + +static std::atomic g_draw_state_log_count = 0; +static std::atomic g_draw_input_log_count = 0; +static std::atomic g_mrt_state_log_count = 0; +static std::atomic g_shader_stage_log_count = 0; +static std::atomic g_framebuffer_skip_log_count = 0; + +static bool ResolveColorTargets(uint64_t submit_id, CommandBuffer* buffer, const HW::Context& hw, + uint32_t render_target_slice_offset); + +static const char* RenderColorTypeName(RenderColorType type) { + switch (type) { + case RenderColorType::NoColorOutput: return "NoColorOutput"; + case RenderColorType::DisplayBuffer: return "DisplayBuffer"; + case RenderColorType::RenderTexture: return "RenderTexture"; + default: return "Unknown"; + } +} + +static bool IsDualSourceBlendFactor(uint32_t factor) { + return factor >= 0x0fu && factor <= 0x12u; +} + +static void LogFramebufferSkip(const char* draw_name, const RenderColorInfo& color, + const RenderDepthInfo& depth, const HW::Context* ctx, + const HW::UserConfig* ucfg, uint32_t index_count, uint32_t flags) { + if (!graphics_debug_dump_enabled()) { + return; + } + + auto log_id = g_framebuffer_skip_log_count.fetch_add(1, std::memory_order_relaxed); + if (log_id >= 128) { + return; + } + + LOGF( + "DrawFramebufferSkip[%u]: %s color=%s color_addr=0x%010" PRIx64 " color_size=0x%016" PRIx64 + " color_image=%s depth_format=%s depth_image=%s depth_vaddr_num=%d target_mask=0x%08" PRIx32 + " prim=%u index_count=%u flags=0x%08" PRIx32 "\n", + log_id, draw_name, RenderColorTypeName(color.type), color.base_addr, color.buffer_size, + color.vulkan_buffer != nullptr ? "yes" : "no", string_VkFormat(depth.format), + depth.vulkan_buffer != nullptr ? "yes" : "no", depth.vaddr_num, + ctx != nullptr ? ctx->GetRenderTargetMask() : 0, ucfg != nullptr ? ucfg->GetPrimType() : 0, + index_count, flags); +} + +static void LogMrtState(const char* draw_name, const HW::Context* ctx, + const ShaderPixelInputInfo& ps_input_info) { + EXIT_IF(ctx == nullptr); + + const auto& sh_regs = ctx->GetShaderRegisters(); + const auto rt_mask = ctx->GetRenderTargetMask(); + const auto cb_shader_mask = sh_regs.m_cbShaderMask; + const auto& bc0 = ctx->GetBlendControl(0); + + bool interesting = rt_mask != 0x0f || (cb_shader_mask & ~0x0fu) != 0 || + IsDualSourceBlendFactor(bc0.color_srcblend) || + IsDualSourceBlendFactor(bc0.color_destblend) || + (bc0.separate_alpha_blend && (IsDualSourceBlendFactor(bc0.alpha_srcblend) || + IsDualSourceBlendFactor(bc0.alpha_destblend))); + + for (uint32_t i = 1; i < 8; i++) { + const auto& rt = ctx->GetRenderTarget(i); + if (rt.base.addr != 0 || ps_input_info.target_output_mode[i] != 0 || + ((rt_mask >> (i * 4u)) & 0x0fu) != 0 || ((cb_shader_mask >> (i * 4u)) & 0x0fu) != 0) { + interesting = true; + } + } + + if (!interesting) { + return; + } + + auto log_id = g_mrt_state_log_count.fetch_add(1); + if (log_id >= 32) { + return; + } + + LOGF("MrtState[%u]: %s rt_mask=0x%08" PRIx32 " cb_shader_mask=0x%08" PRIx32 + " blend0=%s src=%u dst=%u alpha_src=%u alpha_dst=%u sep_alpha=%s\n", + log_id, draw_name, rt_mask, cb_shader_mask, bc0.enable ? "true" : "false", + bc0.color_srcblend, bc0.color_destblend, bc0.alpha_srcblend, bc0.alpha_destblend, + bc0.separate_alpha_blend ? "true" : "false"); + + for (uint32_t i = 0; i < 8; i++) { + const auto& rt = ctx->GetRenderTarget(i); + const auto& bc = ctx->GetBlendControl(i); + const auto ctm = (rt_mask >> (i * 4u)) & 0x0fu; + const auto csm = (cb_shader_mask >> (i * 4u)) & 0x0fu; + + if (rt.base.addr == 0 && ps_input_info.target_output_mode[i] == 0 && ctm == 0 && csm == 0 && + !bc.enable) { + continue; + } + + LOGF("MrtState[%u]: slot=%u addr=0x%010" PRIx64 + " target_mask=0x%x shader_mask=0x%x out_mode=%u" + " fmt=0x%08" PRIx32 " nfmt=0x%08" PRIx32 " order=0x%08" PRIx32 + " width=%u height=%u tile=%u" + " blend=%s src=%u dst=%u alpha_src=%u alpha_dst=%u\n", + log_id, i, rt.base.addr, ctm, csm, ps_input_info.target_output_mode[i], rt.info.format, + rt.info.channel_type, rt.info.channel_order, rt.attrib2.width + 1, + rt.attrib2.height + 1, rt.attrib3.tile_mode, bc.enable ? "true" : "false", + bc.color_srcblend, bc.color_destblend, bc.alpha_srcblend, bc.alpha_destblend); + } +} + +static void LogDrawTargetState(const char* draw_name, const RenderColorInfo& color, + const RenderDepthInfo& depth, const HW::Context* ctx, + const HW::UserConfig* ucfg, + const ShaderPixelInputInfo& ps_input_info, uint32_t index_count, + uint32_t flags) { + if (color.type == RenderColorType::NoColorOutput) { + return; + } + + auto log_id = g_draw_state_log_count.fetch_add(1); + if (log_id >= 192 && color.type != RenderColorType::DisplayBuffer) { + return; + } + + EXIT_IF(ctx == nullptr); + EXIT_IF(ucfg == nullptr); + + const auto& cc = ctx->GetColorControl(); + const auto& bc = ctx->GetBlendControl(color.target_slot); + const auto& dc = ctx->GetDepthControl(); + const auto& vp = ctx->GetScreenViewport(); + const auto& vp0 = vp.viewports[0]; + const auto& ps_resources = ps_input_info.stage.program->info; + const auto sampled_images = std::count_if( + ps_resources.images.begin(), ps_resources.images.end(), [](const auto& image) { + return image.kind == ShaderRecompiler::IR::ResourceKind::Image || + image.kind == ShaderRecompiler::IR::ResourceKind::ImageUint; + }); + + VkExtent2D extent = color.vulkan_buffer != nullptr ? color.extent : VkExtent2D {}; + auto sc = calc_final_scissor(vp, ctx->GetScanModeControl(), extent); + + LOGF( + "DrawTargetState[%u]: frame=%d %s target=%s addr=0x%010" PRIx64 + " extent=%ux%u prim=%u index_count=%u flags=0x%08" PRIx32 " color_mask=0x%08" PRIx32 + " clear=%s clear_rgba=(%.3f,%.3f,%.3f,%.3f) cc_mode=%u cc_op=0x%02x" + " blend=%s src=%u dst=%u comb=%u ps_tex=%d sampled=%d storage=%d ps_kill=%s target_mode0=%u" + " depth_test=%s depth_write=%s depth_func=%u depth_clear=%s viewport=(%.1f,%.1f %.1fx%.1f) " + "scissor=(%d,%d)-(%d,%d)\n", + log_id, GraphicsRunGetFrameNum(), draw_name, RenderColorTypeName(color.type), + color.base_addr, extent.width, extent.height, ucfg->GetPrimType(), index_count, flags, + ctx->GetRenderTargetMask(), color.color_clear_enable ? "true" : "false", + color.color_clear_value.float32[0], color.color_clear_value.float32[1], + color.color_clear_value.float32[2], color.color_clear_value.float32[3], cc.mode, cc.op, + bc.enable ? "true" : "false", bc.color_srcblend, bc.color_destblend, bc.color_comb_fcn, + static_cast(ps_resources.images.size()), static_cast(sampled_images), + static_cast(ps_resources.images.size() - sampled_images), + ps_input_info.ps_pixel_kill_enable ? "true" : "false", ps_input_info.target_output_mode[0], + dc.z_enable ? "true" : "false", dc.z_write_enable ? "true" : "false", dc.zfunc, + depth.depth_clear_enable ? "true" : "false", vp0.xoffset - vp0.xscale, + vp0.yoffset - vp0.yscale, vp0.xscale * 2.0f, vp0.yscale * 2.0f, sc.left, sc.top, sc.right, + sc.bottom); + + LogMrtState(draw_name, ctx, ps_input_info); +} + +static void LogDrawInputState(const RenderColorInfo& color, + const ShaderVertexInputInfo& vs_input_info, + uint32_t index_type_and_size, uint32_t index_count, + const void* index_addr) { + auto log_id = g_draw_input_log_count.fetch_add(1); + if (log_id >= 64 && color.type != RenderColorType::DisplayBuffer) { + return; + } + + LOGF("DrawInputState[%u]: frame=%d target=%s addr=0x%010" PRIx64 + " index_type=%u index_count=%u index_addr=0x%016" PRIx64 + " vs_resources=%d vs_buffers=%d\n", + log_id, GraphicsRunGetFrameNum(), RenderColorTypeName(color.type), color.base_addr, + index_type_and_size, index_count, reinterpret_cast(index_addr), + vs_input_info.resources_num, vs_input_info.buffers_num); + + for (int bi = 0; bi < vs_input_info.buffers_num; bi++) { + const auto& b = vs_input_info.buffers[bi]; + LOGF("DrawInputState[%u]: vb[%d] addr=0x%010" PRIx64 + " stride=%u records=%u fetch_index=%u attr_num=%d\n", + log_id, bi, b.addr, b.stride, b.num_records, b.fetch_index, b.attr_num); + + const auto* bytes = reinterpret_cast(b.addr); + if (bytes != nullptr && b.stride != 0) { + const uint32_t records = std::min(b.num_records, 4u); + for (uint32_t rec = 0; rec < records; rec++) { + const auto* rec_bytes = bytes + static_cast(rec) * b.stride; + const auto dword_num = std::min(b.stride / 4u, 12u); + uint32_t raw[12] = {}; + float flt[12] = {}; + for (uint32_t i = 0; i < dword_num; i++) { + std::memcpy(&raw[i], rec_bytes + i * 4u, sizeof(raw[i])); + std::memcpy(&flt[i], rec_bytes + i * 4u, sizeof(flt[i])); + } + LOGF("DrawInputState[%u]: vb[%d].rec[%u] stride=%u dwords=%u raw=%08" PRIx32 + " %08" PRIx32 " %08" PRIx32 " %08" PRIx32 " %08" PRIx32 " %08" PRIx32 + " %08" PRIx32 " %08" PRIx32 " %08" PRIx32 + " f=(%.3f,%.3f,%.3f,%.3f,%.3f,%.3f,%.3f,%.3f,%.3f)\n", + log_id, bi, rec, b.stride, dword_num, raw[0], raw[1], raw[2], raw[3], raw[4], + raw[5], raw[6], raw[7], raw[8], flt[0], flt[1], flt[2], flt[3], flt[4], flt[5], + flt[6], flt[7], flt[8]); + + for (int ai = 0; ai < b.attr_num; ai++) { + const auto res_index = b.attr_indices[ai]; + const auto& r = vs_input_info.resources[res_index]; + const auto& rd = vs_input_info.resources_dst[res_index]; + const auto offset = b.attr_offsets[ai]; + if (offset + 4u <= b.stride && + r.Format() == + Prospero::GpuEnumValue(Prospero::BufferFormat::k8_8_8_8UNorm)) { + uint32_t packed = 0; + std::memcpy(&packed, rec_bytes + offset, sizeof(packed)); + const auto r8 = (packed >> 0u) & 0xffu; + const auto g8 = (packed >> 8u) & 0xffu; + const auto b8 = (packed >> 16u) & 0xffu; + const auto a8 = (packed >> 24u) & 0xffu; + LOGF("DrawInputState[%u]: vb[%d].rec[%u].attr[%d] dst=v%d fmt=56 " + "rgba8=%02" PRIx32 "%02" PRIx32 "%02" PRIx32 "%02" PRIx32 + " rgba=(%.3f,%.3f,%.3f,%.3f)\n", + log_id, bi, rec, ai, rd.register_start, r8, g8, b8, a8, + static_cast(r8) / 255.0, static_cast(g8) / 255.0, + static_cast(b8) / 255.0, static_cast(a8) / 255.0); + } + } + } + } + + for (int ai = 0; ai < b.attr_num; ai++) { + const auto res_index = b.attr_indices[ai]; + const auto& r = vs_input_info.resources[res_index]; + const auto& rd = vs_input_info.resources_dst[res_index]; + LOGF("DrawInputState[%u]: attr[%d] res=%d offset=%u dst=v%d regs=%d fetch_index=%u " + "sharp=%08" PRIx32 " %08" PRIx32 " %08" PRIx32 " %08" PRIx32 "\n", + log_id, ai, res_index, b.attr_offsets[ai], rd.register_start, rd.registers_num, + rd.fetch_index, r.fields[0], r.fields[1], r.fields[2], r.fields[3]); + } + } +} + +[[maybe_unused]] static void LogDrawTextureState(const char* draw_name, + const RenderColorInfo& color, + const ShaderPixelInputInfo& ps_input_info) { + static std::atomic log_count {0}; + const auto log_id = log_count.fetch_add(1, std::memory_order_relaxed); + if (log_id >= 256) { + return; + } + + const auto& ps_program = *ps_input_info.stage.program; + const auto& ps_resources = *ps_input_info.stage.resources; + const auto sampled_images = std::count_if( + ps_program.info.images.begin(), ps_program.info.images.end(), [](const auto& image) { + return image.kind == ShaderRecompiler::IR::ResourceKind::Image || + image.kind == ShaderRecompiler::IR::ResourceKind::ImageUint; + }); + LOGF("DrawTextureState[%u]: frame=%d %s target=%s addr=0x%010" PRIx64 + " textures=%zu sampled=%zu storage=%zu samplers=%zu\n", + log_id, GraphicsRunGetFrameNum(), draw_name, RenderColorTypeName(color.type), + color.base_addr, ps_program.info.images.size(), sampled_images, + ps_program.info.images.size() - sampled_images, ps_program.info.samplers.size()); + + for (uint32_t i = 0; i < ps_program.info.images.size(); i++) { + const auto& image = ps_program.info.images[i]; + const auto r = DecodeNativeDescriptor(ps_resources.images[i]); + LOGF("DrawTextureState[%u]: tex[%u] source=%u usage=%s sampled=%s " + "addr=0x%010" PRIx64 " type=%u fmt=%u extent=%ux%u depth=%u levels=%u base_level=%u " + "tile=%u swizzle=0x%03" PRIx32 " fields=%08" PRIx32 " %08" PRIx32 " %08" PRIx32 + " %08" PRIx32 " %08" PRIx32 " %08" PRIx32 " %08" PRIx32 " %08" PRIx32 "\n", + log_id, i, image.source, image.written ? "read-write" : "read-only", + (image.kind == ShaderRecompiler::IR::ResourceKind::Image || + image.kind == ShaderRecompiler::IR::ResourceKind::ImageUint) + ? "true" + : "false", + r.Base40(), static_cast(r.Type()), static_cast(r.Format()), + static_cast(r.Width5()) + 1u, static_cast(r.Height5()) + 1u, + static_cast(r.Depth()) + 1u, + std::max(static_cast(r.LastLevel()), + static_cast(r.MaxMip())) + + 1u, + static_cast(r.BaseLevel()), static_cast(r.TileMode()), + r.DstSelXYZW(), r.fields[0], r.fields[1], r.fields[2], r.fields[3], r.fields[4], + r.fields[5], r.fields[6], r.fields[7]); + } + + for (uint32_t i = 0; i < ps_program.info.samplers.size(); i++) { + const auto& sampler = ps_program.info.samplers[i]; + const auto r = DecodeNativeDescriptor(ps_resources.samplers[i]); + LOGF("DrawTextureState[%u]: samp[%u] source=%u clamp=%u/%u/%u filter=%u/%u/%u " + "mip=%u lod=%u-%u" + " bias=%d fields=%08" PRIx32 " %08" PRIx32 " %08" PRIx32 " %08" PRIx32 "\n", + log_id, i, sampler.source, static_cast(r.ClampX()), + static_cast(r.ClampY()), static_cast(r.ClampZ()), + static_cast(r.XyMagFilter()), static_cast(r.XyMinFilter()), + static_cast(r.ZFilter()), static_cast(r.MipFilter()), + static_cast(r.MinLod()), static_cast(r.MaxLod()), + static_cast(r.LodBias()), r.fields[0], r.fields[1], r.fields[2], r.fields[3]); + } +} + +static void VulkanCmdSetColorWriteEnableEXT(GraphicContext* ctx, VkCommandBuffer command_buffer, + uint32_t attachment_count, + const VkBool32* p_color_write_enables) { + EXIT_IF(ctx == nullptr); + EXIT_IF(ctx->instance == nullptr); + + static auto func = reinterpret_cast( + vkGetInstanceProcAddr(ctx->instance, "vkCmdSetColorWriteEnableEXT")); + + if (func != nullptr) { + func(command_buffer, attachment_count, p_color_write_enables); + } else { + EXIT("vkCmdSetColorWriteEnableEXT not present\n"); + } +} + +static PipelineDynamicParameters BuildGraphicsDynamicParams(const HW::Context& ctx, + const RenderColorInfo* colors, + uint32_t color_count, + const RenderDepthInfo& depth) { + EXIT_IF(colors == nullptr); + + PipelineDynamicParameters ret {}; + ret.color_write_count = color_count; + ret.stencil_test_enable = depth.stencil_test_enable; + + const auto& vp = ctx.GetScreenViewport(); + VkExtent2D framebuffer_extent {}; + if (color_count > 0 && colors[0].vulkan_buffer != nullptr) { + framebuffer_extent = colors[0].extent; + } else if (depth.vulkan_buffer != nullptr) { + framebuffer_extent = depth.vulkan_buffer->extent; + } + + const auto final_scissor = calc_final_scissor(vp, ctx.GetScanModeControl(), framebuffer_extent); + + ret.viewport_scale[0] = vp.viewports[0].xscale; + ret.viewport_scale[1] = vp.viewports[0].yscale; + ret.viewport_scale[2] = vp.viewports[0].zscale; + ret.viewport_offset[0] = vp.viewports[0].xoffset; + ret.viewport_offset[1] = vp.viewports[0].yoffset; + ret.viewport_offset[2] = vp.viewports[0].zoffset; + ret.scissor_ltrb[0] = final_scissor.left; + ret.scissor_ltrb[1] = final_scissor.top; + ret.scissor_ltrb[2] = final_scissor.right; + ret.scissor_ltrb[3] = final_scissor.bottom; + ret.line_width = ctx.GetLineWidth(); + ret.stencil_front = depth.stencil_dynamic_front; + ret.stencil_back = depth.stencil_dynamic_back; + + // CB_COLOR_CONTROL.operation controls special CB operations, not the normal color component + // write mask. Use CB_TARGET_MASK here so scanout passes with mode=Disable do not go black. + for (uint32_t i = 0; i < RENDER_COLOR_ATTACHMENTS_MAX; i++) { + ret.color_write_enable[i] = + (i < color_count && + render_target_mask_slot(ctx.GetRenderTargetMask(), colors[i].target_slot) != 0); + } + + return ret; +} + +static void SetDynamicParams(VkCommandBuffer vk_buffer, + const PipelineDynamicParameters& dynamic_params) { + KYTY_PROFILER_FUNCTION(); + + VkViewport viewport {}; + viewport.x = dynamic_params.viewport_offset[0] - dynamic_params.viewport_scale[0]; + viewport.y = dynamic_params.viewport_offset[1] - dynamic_params.viewport_scale[1]; + viewport.width = dynamic_params.viewport_scale[0] * 2.0f; + viewport.height = dynamic_params.viewport_scale[1] * 2.0f; + viewport.minDepth = dynamic_params.viewport_offset[2]; + viewport.maxDepth = dynamic_params.viewport_scale[2] + dynamic_params.viewport_offset[2]; + vkCmdSetViewport(vk_buffer, 0, 1, &viewport); + + VkRect2D scissor {}; + scissor.offset = {dynamic_params.scissor_ltrb[0], dynamic_params.scissor_ltrb[1]}; + scissor.extent = { + static_cast(dynamic_params.scissor_ltrb[2] - dynamic_params.scissor_ltrb[0]), + static_cast(dynamic_params.scissor_ltrb[3] - dynamic_params.scissor_ltrb[1])}; + vkCmdSetScissor(vk_buffer, 0, 1, &scissor); + + float line_width = dynamic_params.line_width; + if (line_width != 1.0f) { + static bool logged = false; + if (!logged) { + LOGF("Render: temporary: clamping Vulkan line width %f to 1.0 because wideLines is " + "not enabled\n", + line_width); + logged = true; + } + line_width = 1.0f; + } + vkCmdSetLineWidth(vk_buffer, line_width); + + if (dynamic_params.stencil_test_enable) { + vkCmdSetStencilCompareMask(vk_buffer, VK_STENCIL_FACE_FRONT_BIT, + dynamic_params.stencil_front.compareMask); + vkCmdSetStencilCompareMask(vk_buffer, VK_STENCIL_FACE_BACK_BIT, + dynamic_params.stencil_back.compareMask); + vkCmdSetStencilWriteMask(vk_buffer, VK_STENCIL_FACE_FRONT_BIT, + dynamic_params.stencil_front.writeMask); + vkCmdSetStencilWriteMask(vk_buffer, VK_STENCIL_FACE_BACK_BIT, + dynamic_params.stencil_back.writeMask); + vkCmdSetStencilReference(vk_buffer, VK_STENCIL_FACE_FRONT_BIT, + dynamic_params.stencil_front.reference); + vkCmdSetStencilReference(vk_buffer, VK_STENCIL_FACE_BACK_BIT, + dynamic_params.stencil_back.reference); + } + + VkBool32 enable[RENDER_COLOR_ATTACHMENTS_MAX] = {}; + for (uint32_t i = 0; i < dynamic_params.color_write_count; i++) { + enable[i] = (dynamic_params.color_write_enable[i] ? VK_TRUE : VK_FALSE); + } + VulkanCmdSetColorWriteEnableEXT(g_render_ctx->GetGraphicCtx(), vk_buffer, + dynamic_params.color_write_count, enable); +} + +static bool DrawHasValidVertexShader(const HW::Shader* sh_ctx) { + EXIT_IF(sh_ctx == nullptr); + + const auto& vs = sh_ctx->GetVs(); + return vs.gs_regs.chksum != 0 && ShaderAddressValid(vs.es_regs.data_addr); +} + +static bool PixelShaderHasDepthOrCoverageSideEffects(const HW::ShaderRegisters& sh_regs) { + const auto& db = sh_regs.db_shader_control; + return sh_regs.shader_z_format != 0 || db.shader_kill_enable || db.shader_z_export_enable || + db.shader_mask_export_enable || db.shader_dual_export_enable || + db.shader_execute_on_noop; +} + +static bool ShouldSkipGeShader(const HW::Context* ctx, const HW::UserConfig* ucfg, + const HW::Shader* sh_ctx) { + EXIT_IF(ctx == nullptr); + EXIT_IF(ucfg == nullptr); + EXIT_IF(sh_ctx == nullptr); + + const auto& sh_regs = ctx->GetShaderRegisters(); + const auto& ge_cntl = ucfg->GetGeControl(); + const auto& vertex_info = sh_ctx->GetVs(); + const auto stages = ctx->GetShaderStages(); + + const auto is_known_gs_out_prim_type = [](uint32_t value) { + switch (static_cast(value)) { + case Prospero::GsOutputPrimitiveType::kPoints: + case Prospero::GsOutputPrimitiveType::kLines: + case Prospero::GsOutputPrimitiveType::kTriangles: + case Prospero::GsOutputPrimitiveType::k2dRectangle: + case Prospero::GsOutputPrimitiveType::kRectList: return true; + } + + return false; + }; + + const bool ps5_ngg_vertex_path = stages == 0x02002000 && vertex_info.es_regs.data_addr != 0 && + vertex_info.gs_regs.chksum != 0 && + sh_regs.m_vgtGsMaxVertOut == 0x00000000 && + is_known_gs_out_prim_type(sh_regs.m_vgtGsOutPrimType); + + const bool unsupported_stage_mask = (stages != 0 && stages != 0x02002000); + const bool unsupported_gs_stage = (vertex_info.es_regs.data_addr != 0 && + vertex_info.gs_regs.data_addr != 0 && !ps5_ngg_vertex_path); + const bool ge_group_size = + ge_cntl.primitive_group_size > 0x0040 || ge_cntl.vertex_group_size > 0x0040; + const bool ge_shader_regs = + (sh_regs.m_geNggSubgrpCntl != 0x00000000 && sh_regs.m_geNggSubgrpCntl != 0x00000001) || + sh_regs.m_vgtGsMaxVertOut != 0x00000000 || + !is_known_gs_out_prim_type(sh_regs.m_vgtGsOutPrimType) || + sh_regs.m_geMaxOutputPerSubgroup > 0x00000040; + + if (unsupported_stage_mask || unsupported_gs_stage || ge_group_size || ge_shader_regs) { + const auto log_id = g_shader_stage_log_count.fetch_add(1); + if (log_id < 32) { + LOGF("Skipping unsupported GE shader draw: stages=0x%08" PRIx32 + " prim_group=0x%04" PRIx16 " vert_group=0x%04" PRIx16 " ngg=0x%08" PRIx32 + " max_out=0x%08" PRIx32 " gs_max_vert=0x%08" PRIx32 " gs_out_prim=0x%08" PRIx32 + " es=0x%016" PRIx64 " gs=0x%016" PRIx64 "\n", + stages, ge_cntl.primitive_group_size, ge_cntl.vertex_group_size, + sh_regs.m_geNggSubgrpCntl, sh_regs.m_geMaxOutputPerSubgroup, + sh_regs.m_vgtGsMaxVertOut, sh_regs.m_vgtGsOutPrimType, + vertex_info.es_regs.data_addr, vertex_info.gs_regs.data_addr); + } + return true; + } + + return false; +} + +struct DrawRenderState { + RenderDepthInfo depth_info; + RenderColorInfo color_info[RENDER_COLOR_ATTACHMENTS_MAX] = {}; + uint32_t color_count = 0; + bool ps_active = true; + VulkanFramebuffer* framebuffer = nullptr; + VkCommandBuffer vk_buffer = nullptr; + ShaderVertexInputInfo vs_input_info; + ShaderPixelInputInfo ps_input_info; + std::span vs_shader; + std::span ps_shader; +}; + +struct DrawCallInfo { + const char* name = nullptr; + CommandBufferDebugOp debug_op = CommandBufferDebugOp::DrawIndex; + uint32_t index_count = 0; + uint32_t flags = 0; + uint32_t instance_count = 0; + uint32_t first_instance = 0; +}; + +static bool DrawHasActivePixelShader(const HW::Context* ctx, const HW::Shader* sh_ctx, + const DrawRenderState& state, const DrawCallInfo& draw) { + EXIT_IF(ctx == nullptr); + EXIT_IF(sh_ctx == nullptr); + EXIT_IF(draw.name == nullptr); + + const bool with_depth = (state.depth_info.format != VK_FORMAT_UNDEFINED && + state.depth_info.vulkan_buffer != nullptr); + if (state.color_count != 0 || !with_depth) { + return true; + } + + const auto& sh_regs = ctx->GetShaderRegisters(); + const auto& ps = sh_ctx->GetPs(); + return ShaderAddressValid(ps.ps_regs.data_addr) && + PixelShaderHasDepthOrCoverageSideEffects(sh_regs); +} + +enum class CbColorMode : uint8_t { + Disable = 0, + Normal = 1, + EliminateFastClear = 2, + Resolve = 3, + FmaskDecompress = 5, + DccDecompress = 6, +}; + +static bool ConsumeMetadataColorOperation(const HW::Context& ctx) { + const auto mode = ctx.GetColorControl().mode; + // These AGC CB modes run color-buffer metadata/decompression operations. The shader is a + // dummy vehicle for the CB, and its exported color must not be applied as a normal draw. + // Kyty currently stores host images as expanded Vulkan images and does not track CMASK/DCC + // metadata state, so the matching host operation is a no-op. + return mode == static_cast(CbColorMode::EliminateFastClear) || + mode == static_cast(CbColorMode::FmaskDecompress) || + mode == static_cast(CbColorMode::DccDecompress); +} + +struct DrawEmitInfo { + bool indexed = false; + bool draw_prim7_as_ngg = false; + uint32_t draw_vertex_count = 0; + int32_t vertex_offset = 0; + uint32_t first_vertex = 0; +}; + +struct DrawIndexBufferBind { + bool enabled = false; + VulkanBuffer* buffer = nullptr; + VkDeviceSize offset = 0; + VkIndexType type = VK_INDEX_TYPE_UINT16; +}; + +static uint64_t VertexBufferDescriptorSize(const ShaderVertexInputBuffer& buffer) { + return (buffer.stride != 0 ? static_cast(buffer.stride) * buffer.num_records + : buffer.num_records); +} + +static void SetDrawDebugPhase(CommandBuffer* buffer, uint64_t submit_id, const DrawCallInfo& draw, + uint32_t phase) { + EXIT_IF(buffer == nullptr); + EXIT_IF(draw.name == nullptr); + + buffer->SetDebugInfo(static_cast(draw.debug_op), submit_id, phase, draw.index_count, + draw.flags, draw.instance_count, draw.first_instance); +} + +static bool GetDrawTopology(const HW::UserConfig* ucfg, bool auto_draw, bool use_ngg_rectlist_draw, + VkPrimitiveTopology* topology) { + EXIT_IF(ucfg == nullptr); + EXIT_IF(topology == nullptr); + + *topology = VK_PRIMITIVE_TOPOLOGY_POINT_LIST; + + switch (static_cast(ucfg->GetPrimType())) { + case Prospero::PrimitiveType::kNone: return false; + case Prospero::PrimitiveType::kPointList: + *topology = VK_PRIMITIVE_TOPOLOGY_POINT_LIST; + break; + case Prospero::PrimitiveType::kLineList: *topology = VK_PRIMITIVE_TOPOLOGY_LINE_LIST; break; + case Prospero::PrimitiveType::kLineStrip: + *topology = VK_PRIMITIVE_TOPOLOGY_LINE_STRIP; + break; + case Prospero::PrimitiveType::kTriList: + *topology = VK_PRIMITIVE_TOPOLOGY_TRIANGLE_LIST; + break; + case Prospero::PrimitiveType::kTriFan: + *topology = VK_PRIMITIVE_TOPOLOGY_TRIANGLE_FAN; + break; + case Prospero::PrimitiveType::kTriStrip: + *topology = VK_PRIMITIVE_TOPOLOGY_TRIANGLE_STRIP; + break; + case Prospero::PrimitiveType::kRectList: + *topology = (auto_draw && use_ngg_rectlist_draw ? VK_PRIMITIVE_TOPOLOGY_TRIANGLE_STRIP + : VK_PRIMITIVE_TOPOLOGY_TRIANGLE_LIST); + break; + case Prospero::PrimitiveType::kRectListLegacy: + if (!auto_draw) { + EXIT("unknown primitive type: %u\n", ucfg->GetPrimType()); + } + *topology = VK_PRIMITIVE_TOPOLOGY_TRIANGLE_STRIP; + break; + case Prospero::PrimitiveType::kQuadListLegacy: + *topology = VK_PRIMITIVE_TOPOLOGY_TRIANGLE_FAN; + break; + default: EXIT("unknown primitive type: %u\n", ucfg->GetPrimType()); + } + + return true; +} + +static bool PrepareDrawRenderState(uint64_t submit_id, CommandBuffer* buffer, HW::Context* ctx, + HW::UserConfig* ucfg, HW::Shader* sh_ctx, + const DrawCallInfo& draw, uint32_t render_target_slice_offset, + bool skip_null_framebuffer, bool log_setup_phases, + DrawRenderState* state) { + EXIT_IF(buffer == nullptr); + EXIT_IF(ctx == nullptr); + EXIT_IF(ucfg == nullptr); + EXIT_IF(sh_ctx == nullptr); + EXIT_IF(draw.name == nullptr); + EXIT_IF(state == nullptr); + + if (log_setup_phases) { + LogDrawPhase(draw.name, "ResolveRenderDepthTarget"); + } + ResolveRenderDepthTarget(submit_id, buffer, *ctx, &state->depth_info); + + if (ResolveColorTargets(submit_id, buffer, *ctx, render_target_slice_offset)) { + MarkRenderTargetGpuWritten(state->depth_info); + return false; + } + MarkRenderTargetGpuWritten(state->depth_info); + + if (log_setup_phases) { + LogDrawPhase(draw.name, "ResolveRenderColorTarget"); + } + for (uint32_t slot = 0; slot < RENDER_COLOR_ATTACHMENTS_MAX; slot++) { + if (slot == 0 || (render_target_mask_slot(ctx->GetRenderTargetMask(), slot) != 0 && + ctx->GetRenderTarget(slot).base.addr != 0)) { + ResolveRenderColorTarget(submit_id, buffer, *ctx, + &state->color_info[state->color_count], + render_target_slice_offset, slot); + if (state->color_info[state->color_count].vulkan_buffer != nullptr) { + MarkRenderTargetGpuWritten(state->color_info[state->color_count]); + state->color_count++; + } + } + } + + const bool with_depth = (state->depth_info.format != VK_FORMAT_UNDEFINED && + state->depth_info.vulkan_buffer != nullptr); + if (state->color_count == 0 && !with_depth) { + LogFramebufferSkip(draw.name, state->color_info[0], state->depth_info, ctx, ucfg, + draw.index_count, draw.flags); + return false; + } + state->ps_active = DrawHasActivePixelShader(ctx, sh_ctx, *state, draw); + + if (log_setup_phases) { + LogDrawPhase(draw.name, "CreateFramebuffer"); + } + state->framebuffer = g_render_ctx->GetFramebufferCache()->CreateFramebuffer( + state->color_info, state->color_count, &state->depth_info); + + if (state->framebuffer == nullptr && skip_null_framebuffer) { + LogFramebufferSkip(draw.name, state->color_info[0], state->depth_info, ctx, ucfg, + draw.index_count, draw.flags); + return false; + } + EXIT_NOT_IMPLEMENTED(state->framebuffer == nullptr); + EXIT_NOT_IMPLEMENTED(state->framebuffer->render_pass == nullptr); + + state->vk_buffer = buffer->GetPool()->buffers[buffer->GetIndex()]; + + return true; +} + +static void RefreshShaders(HW::Context* ctx, HW::Shader* sh_ctx, const DrawCallInfo& draw, + bool log_phases, DrawRenderState* state) { + EXIT_IF(ctx == nullptr); + EXIT_IF(sh_ctx == nullptr); + EXIT_IF(draw.name == nullptr); + EXIT_IF(state == nullptr); + + const auto& vertex_shader_info = sh_ctx->GetVs(); + const auto& pixel_shader_info = sh_ctx->GetPs(); + const auto& shader_regs = ctx->GetShaderRegisters(); + + state->vs_shader = {}; + state->ps_shader = {}; + state->ps_input_info = {}; + EXIT_IF(g_render_ctx == nullptr || g_render_ctx->GetGraphicCtx() == nullptr); + const auto lane_mask_mode = SelectGraphicsLaneMaskMode(*g_render_ctx->GetGraphicCtx(), 64u); + + if (log_phases) { + LogDrawPhase(draw.name, "ShaderCompileInfoVS"); + } + if (!ShaderCompileInfoVS(&vertex_shader_info, &shader_regs, lane_mask_mode, + &state->vs_input_info, &state->vs_shader)) { + EXIT("ShaderCompileInfoVS failed for draw %s\n", draw.name); + } + + if (!state->ps_active) { + return; + } + if (log_phases) { + LogDrawPhase(draw.name, "ShaderCompileInfoPS"); + } + if (!ShaderCompileInfoPS(&pixel_shader_info, &shader_regs, lane_mask_mode, + &state->vs_input_info, &state->ps_input_info, &state->ps_shader)) { + EXIT("ShaderCompileInfoPS failed for draw %s\n", draw.name); + } +} + +static void BindDrawVertexBuffers(uint64_t submit_id, CommandBuffer* buffer, + const DrawCallInfo& draw, VkCommandBuffer vk_buffer, + const ShaderVertexInputInfo& vs_input_info) { + EXIT_IF(buffer == nullptr); + EXIT_IF(draw.name == nullptr); + (void)submit_id; + + LogDrawPhase(draw.name, "BindVertexBuffers"); + for (int i = 0; i < vs_input_info.buffers_num; i++) { + const auto& b = vs_input_info.buffers[i]; + uint64_t addr = b.addr; + uint64_t size = VertexBufferDescriptorSize(b); + VulkanBuffer* vertices = nullptr; + VkDeviceSize offset = 0; + + if (size == 0) { + vertices = g_render_ctx->GetBufferCache()->ObtainNullBuffer( + buffer, g_render_ctx->GetGraphicCtx()); + } else { + auto binding = g_render_ctx->GetBufferCache()->ObtainBuffer( + buffer, g_render_ctx->GetGraphicCtx(), addr, size); + vertices = binding.first; + offset = binding.second; + } + EXIT_NOT_IMPLEMENTED(vertices == nullptr); + + vkCmdBindVertexBuffers(vk_buffer, i, 1, &vertices->buffer, &offset); + } +} + +static void LogDrawStateIfNeeded(HW::Context* ctx, HW::UserConfig* ucfg, const DrawCallInfo& draw, + const DrawRenderState& state, bool always_log, + bool force_legacy_rect_log, uint32_t index_type_and_size, + const void* index_addr) { + EXIT_IF(ctx == nullptr); + EXIT_IF(ucfg == nullptr); + EXIT_IF(draw.name == nullptr); + + if (!graphics_debug_dump_enabled()) { + return; + } + + if (!always_log && !force_legacy_rect_log) { + return; + } + + LogDrawTargetState(draw.name, state.color_info[0], state.depth_info, ctx, ucfg, + state.ps_input_info, draw.index_count, draw.flags); + LogDrawInputState(state.color_info[0], state.vs_input_info, index_type_and_size, + draw.index_count, index_addr); + // LogDrawTextureState(draw.name, state.color_info[0], state.ps_input_info); +} + +static bool IsHostExpandedRectListDrawSupported(const ShaderVertexInputInfo& vs_input_info, + const DrawCallInfo& draw, + const DrawEmitInfo& emit) { + if (!emit.draw_prim7_as_ngg) { + return true; + } + + if (vs_input_info.buffers_num != 0) { + return false; + } + + return draw.index_count == 3 || draw.index_count == emit.draw_vertex_count; +} + +static void EmitDrawPrimitives(const HW::UserConfig* ucfg, VkCommandBuffer vk_buffer, + const ShaderVertexInputInfo& vs_input_info, const DrawCallInfo& draw, + const DrawEmitInfo& emit) { + EXIT_IF(ucfg == nullptr); + EXIT_IF(draw.name == nullptr); + + switch (static_cast(ucfg->GetPrimType())) { + case Prospero::PrimitiveType::kPointList: + case Prospero::PrimitiveType::kLineList: + case Prospero::PrimitiveType::kLineStrip: + case Prospero::PrimitiveType::kTriList: + case Prospero::PrimitiveType::kTriFan: + case Prospero::PrimitiveType::kTriStrip: + if (emit.indexed) { + vkCmdDrawIndexed(vk_buffer, draw.index_count, draw.instance_count, 0, + emit.vertex_offset, draw.first_instance); + } else { + vkCmdDraw(vk_buffer, draw.index_count, draw.instance_count, emit.first_vertex, + draw.first_instance); + } + break; + case Prospero::PrimitiveType::kRectList: + if (emit.indexed) { + vkCmdDrawIndexed(vk_buffer, draw.index_count, draw.instance_count, 0, + emit.vertex_offset, draw.first_instance); + } else { + EXIT_NOT_IMPLEMENTED( + !IsHostExpandedRectListDrawSupported(vs_input_info, draw, emit)); + vkCmdDraw(vk_buffer, emit.draw_vertex_count, draw.instance_count, emit.first_vertex, + draw.first_instance); + } + break; + case Prospero::PrimitiveType::kRectListLegacy: + if (emit.indexed) { + EXIT("unknown primitive type: %u\n", ucfg->GetPrimType()); + } + // Sarah + EXIT_NOT_IMPLEMENTED(!(draw.index_count == 3 && vs_input_info.buffers_num == 0)); + vkCmdDraw(vk_buffer, 4, draw.instance_count, emit.first_vertex, draw.first_instance); + break; + case Prospero::PrimitiveType::kQuadListLegacy: + EXIT_NOT_IMPLEMENTED((draw.index_count & 0x3u) != 0); + for (uint32_t i = 0; i < draw.index_count; i += 4) { + if (emit.indexed) { + vkCmdDrawIndexed(vk_buffer, 4, draw.instance_count, i, emit.vertex_offset, + draw.first_instance); + } else { + vkCmdDraw(vk_buffer, 4, draw.instance_count, i + emit.first_vertex, + draw.first_instance); + } + } + break; + default: EXIT("unknown primitive type: %u\n", ucfg->GetPrimType()); + } +} + +static void ExecutePreparedDraw(uint64_t submit_id, CommandBuffer* buffer, HW::Context* ctx, + HW::UserConfig* ucfg, HW::Shader* sh_ctx, const DrawCallInfo& draw, + DrawRenderState* state, VkPrimitiveTopology topology, + const DrawEmitInfo& emit, const DrawIndexBufferBind& index_bind, + bool log_pipeline_phase, bool set_bind_debug, bool set_auto_debug) { + EXIT_IF(buffer == nullptr); + EXIT_IF(ctx == nullptr); + EXIT_IF(ucfg == nullptr); + EXIT_IF(sh_ctx == nullptr); + EXIT_IF(draw.name == nullptr); + EXIT_IF(state == nullptr); + + if (log_pipeline_phase) { + LogDrawPhase(draw.name, "CreatePipeline"); + } + auto* pipeline = g_render_ctx->GetPipelineCache()->CreateGraphicsPipeline( + state->framebuffer, state->color_info, state->color_count, &state->depth_info, + &state->vs_input_info, ctx, sh_ctx, &state->ps_input_info, topology, state->ps_active, + state->vs_shader, state->ps_shader); + + if (set_bind_debug) { + SetDrawDebugPhase(buffer, submit_id, draw, 0x100u); + } + vkCmdBindPipeline(state->vk_buffer, VK_PIPELINE_BIND_POINT_GRAPHICS, pipeline->pipeline); + const auto dynamic_params = + BuildGraphicsDynamicParams(*ctx, state->color_info, state->color_count, state->depth_info); + SetDynamicParams(state->vk_buffer, dynamic_params); + + // EXIT_NOT_IMPLEMENTED(vs_input_info.buffers_num > 1); + BindDrawVertexBuffers(submit_id, buffer, draw, state->vk_buffer, state->vs_input_info); + + LogDrawPhase(draw.name, "BindDescriptorsVS"); + if (set_auto_debug) { + SetDrawDebugPhase(buffer, submit_id, draw, 0x200u); + } + const auto vs_address_writes = BindDescriptors( + submit_id, buffer, VK_PIPELINE_BIND_POINT_GRAPHICS, pipeline->pipeline_layout, + state->vs_input_info.stage, VK_SHADER_STAGE_VERTEX_BIT, DescriptorCache::Stage::Vertex); + + std::vector ps_address_writes; + if (state->ps_active) { + LogDrawPhase(draw.name, "BindDescriptorsPS"); + if (set_auto_debug) { + SetDrawDebugPhase(buffer, submit_id, draw, 0x300u); + } + ps_address_writes = + BindDescriptors(submit_id, buffer, VK_PIPELINE_BIND_POINT_GRAPHICS, + pipeline->pipeline_layout, state->ps_input_info.stage, + VK_SHADER_STAGE_FRAGMENT_BIT, DescriptorCache::Stage::Pixel); + } + + if (index_bind.enabled) { + EXIT_NOT_IMPLEMENTED(index_bind.buffer == nullptr); + vkCmdBindIndexBuffer(state->vk_buffer, index_bind.buffer->buffer, index_bind.offset, + index_bind.type); + } + + LogDrawPhase(draw.name, "BeginRenderPass"); + if (set_auto_debug) { + SetDrawDebugPhase(buffer, submit_id, draw, 0x400u); + } + buffer->BeginRenderPass(state->framebuffer, state->color_info, state->color_count, + &state->depth_info); + if (set_auto_debug) { + SetDrawDebugPhase(buffer, submit_id, draw, 0x500u); + } + + EmitDrawPrimitives(ucfg, state->vk_buffer, state->vs_input_info, draw, emit); + + if (set_auto_debug) { + SetDrawDebugPhase(buffer, submit_id, draw, 0x600u); + } + buffer->EndRenderPass(); + VkPipelineStageFlags shader_write_stages = 0; + const bool vs_wrote_buffers = HasShaderBufferWrites(state->vs_input_info.stage); + const bool vs_wrote_addresses = MarkShaderAddressWrites(vs_address_writes); + if (vs_wrote_buffers || vs_wrote_addresses) { + shader_write_stages |= VK_PIPELINE_STAGE_VERTEX_SHADER_BIT; + } + if (state->ps_active) { + const bool ps_wrote_buffers = HasShaderBufferWrites(state->ps_input_info.stage); + const bool ps_wrote_addresses = MarkShaderAddressWrites(ps_address_writes); + if (ps_wrote_buffers || ps_wrote_addresses) { + shader_write_stages |= VK_PIPELINE_STAGE_FRAGMENT_SHADER_BIT; + } + } + if (shader_write_stages != 0) { + ShaderWriteBarrier(state->vk_buffer, shader_write_stages); + } + LogDrawPhase(draw.name, "EndRenderPass"); + if (set_auto_debug) { + SetDrawDebugPhase(buffer, submit_id, draw, 0x700u); + } +} + +void RenderDrawIndex(uint64_t submit_id, CommandBuffer* buffer, HW::Context* ctx, + HW::UserConfig* ucfg, HW::Shader* sh_ctx, uint32_t index_type_and_size, + uint32_t index_count, const void* index_addr, uint32_t flags, uint32_t type, + uint32_t instance_count, uint32_t render_target_slice_offset, + int32_t vertex_offset_add, uint32_t first_instance) { + KYTY_PROFILER_FUNCTION(); + + EXIT_IF(ctx == nullptr); + EXIT_IF(ucfg == nullptr); + EXIT_IF(g_render_ctx == nullptr); + EXIT_IF(buffer == nullptr); + EXIT_IF(buffer->IsInvalid()); + + buffer->SetDebugInfo(static_cast(CommandBufferDebugOp::DrawIndex), submit_id, + index_count, flags, type, instance_count, + reinterpret_cast(index_addr)); + + Common::LockGuard lock(g_render_ctx->GetMutex()); + + if (index_count == 0) { + return; + } + + if (ConsumeMetadataColorOperation(*ctx)) { + return; + } + + if (!DrawHasValidVertexShader(sh_ctx)) { + return; + } + + if (ShouldSkipGeShader(ctx, ucfg, sh_ctx)) { + return; + } + + if (graphics_debug_dump_enabled()) { + sh_print("GraphicsRenderDrawIndex():Shader:", *sh_ctx); + uc_print("GraphicsRenderDrawIndex():UserConfig:", *ucfg); + hw_print(*ctx); + + LOGF("GraphicsRenderDrawIndex():Parameters:\n" + "\t index_type_and_size = 0x%08" PRIx32 "\n" + "\t index_count = 0x%08" PRIx32 "\n" + "\t index_addr = 0x%016" PRIx64 "\n" + "\t flags = 0x%08" PRIx32 "\n" + "\t type = 0x%08" PRIx32 "\n" + "\t instance_count = 0x%08" PRIx32 "\n" + "\t rt_slice_offset = 0x%08" PRIx32 "\n" + "\t vertex_offset_add = 0x%08" PRIx32 "\n" + "\t first_instance = 0x%08" PRIx32 "\n", + index_type_and_size, index_count, reinterpret_cast(index_addr), flags, type, + instance_count, render_target_slice_offset, static_cast(vertex_offset_add), + first_instance); + } + sh_check(*sh_ctx); + + uc_check(*ucfg); + + hw_check(*ctx); + + VkPrimitiveTopology topology = VK_PRIMITIVE_TOPOLOGY_POINT_LIST; + if (!GetDrawTopology(ucfg, false, false, &topology)) { + return; + } + + VkIndexType index_type = VK_INDEX_TYPE_UINT16; + uint64_t index_size = 0; + bool expand_index8_to_u16 = false; + + switch (static_cast(index_type_and_size)) { + case Prospero::IndexType::kIndex16: + index_type = VK_INDEX_TYPE_UINT16; + index_size = 2 * static_cast(index_count); + break; + case Prospero::IndexType::kIndex32: + index_type = VK_INDEX_TYPE_UINT32; + index_size = 4 * static_cast(index_count); + break; + // Some games use it + case Prospero::IndexType::kIndex8: + index_type = VK_INDEX_TYPE_UINT16; + index_size = static_cast(index_count); + expand_index8_to_u16 = true; + break; + default: EXIT("unknown index_type_and_size: %u\n", index_type_and_size); + } + + EXIT_NOT_IMPLEMENTED(flags != 0); + EXIT_NOT_IMPLEMENTED(type != 1); + if (instance_count == 0) { + instance_count = 1; + } + + const DrawCallInfo draw {"DrawIndex", CommandBufferDebugOp::DrawIndex, + index_count, flags, + instance_count, first_instance}; + + DrawRenderState state {}; + if (!PrepareDrawRenderState(submit_id, buffer, ctx, ucfg, sh_ctx, draw, + render_target_slice_offset, false, true, &state)) { + return; + } + + RefreshShaders(ctx, sh_ctx, draw, true, &state); + + LogDrawStateIfNeeded(ctx, ucfg, draw, state, true, false, index_type_and_size, index_addr); + + const auto vertex_offset = + ResolveVertexOffset(ucfg->GetIndexOffset(), state.vs_input_info) + vertex_offset_add; + + VkDeviceSize index_offset = 0; + VulkanBuffer* indices = nullptr; + if (expand_index8_to_u16) { + EXIT_NOT_IMPLEMENTED(index_addr == nullptr); + const auto* src = static_cast(index_addr); + std::vector expanded_indices(index_count); + for (uint32_t i = 0; i < index_count; i++) { + expanded_indices[i] = src[i]; + } + VkDeviceSize range = 0; + if (!g_render_ctx->GetBufferCache()->UploadHostData( + buffer, g_render_ctx->GetGraphicCtx(), expanded_indices.data(), + expanded_indices.size() * sizeof(uint16_t), 16, &indices, &index_offset, &range)) { + EXIT("failed to upload expanded 8-bit index buffer\n"); + } + } else { + auto binding = g_render_ctx->GetBufferCache()->ObtainBuffer( + buffer, g_render_ctx->GetGraphicCtx(), reinterpret_cast(index_addr), + index_size); + indices = binding.first; + index_offset = binding.second; + } + + EXIT_NOT_IMPLEMENTED(indices == nullptr); + + DrawIndexBufferBind index_bind {}; + index_bind.enabled = true; + index_bind.buffer = indices; + index_bind.offset = index_offset; + index_bind.type = index_type; + + DrawEmitInfo emit {}; + emit.indexed = true; + emit.vertex_offset = vertex_offset; + + ExecutePreparedDraw(submit_id, buffer, ctx, ucfg, sh_ctx, draw, &state, topology, emit, + index_bind, true, true, false); +} + +// NOLINTNEXTLINE(readability-function-cognitive-complexity) +void RenderDrawIndexAuto(uint64_t submit_id, CommandBuffer* buffer, HW::Context* ctx, + HW::UserConfig* ucfg, HW::Shader* sh_ctx, uint32_t index_count, + uint32_t flags, uint32_t render_target_slice_offset, + uint32_t instance_count, uint32_t first_vertex, uint32_t first_instance) { + KYTY_PROFILER_FUNCTION(); + + EXIT_IF(ctx == nullptr || ucfg == nullptr); + EXIT_IF(g_render_ctx == nullptr); + EXIT_IF(buffer == nullptr || buffer->IsInvalid()); + + buffer->SetDebugInfo(static_cast(CommandBufferDebugOp::DrawIndexAuto), submit_id, + index_count, flags, first_vertex, instance_count, first_instance); + + Common::LockGuard lock(g_render_ctx->GetMutex()); + + if (index_count == 0) { + return; + } + + if (ConsumeMetadataColorOperation(*ctx)) { + return; + } + + if (!DrawHasValidVertexShader(sh_ctx)) { + return; + } + + if (ShouldSkipGeShader(ctx, ucfg, sh_ctx)) { + return; + } + + if (graphics_debug_dump_enabled()) { + sh_print("GraphicsRenderDrawIndexAuto():Shader:", *sh_ctx); + uc_print("GraphicsRenderDrawIndexAuto():UserConfig:", *ucfg); + hw_print(*ctx); + + LOGF("GraphicsRenderDrawIndexAuto():Parameters:\n" + "\t index_count = 0x%08" PRIx32 "\n" + "\t flags = 0x%08" PRIx32 "\n" + "\t rt_slice_offset = 0x%08" PRIx32 "\n" + "\t instance_count = 0x%08" PRIx32 "\n" + "\t first_vertex = 0x%08" PRIx32 "\n" + "\t first_instance = 0x%08" PRIx32 "\n", + index_count, flags, render_target_slice_offset, instance_count, first_vertex, + first_instance); + } + sh_check(*sh_ctx); + + uc_check(*ucfg); + + hw_check(*ctx); + + EXIT_NOT_IMPLEMENTED(flags != 0); + if (instance_count == 0) { + instance_count = 1; + } + + const DrawCallInfo draw {"DrawIndexAuto", CommandBufferDebugOp::DrawIndexAuto, + index_count, flags, + instance_count, first_instance}; + + DrawRenderState state {}; + if (!PrepareDrawRenderState(submit_id, buffer, ctx, ucfg, sh_ctx, draw, + render_target_slice_offset, true, false, &state)) { + return; + } + + VkPrimitiveTopology topology = VK_PRIMITIVE_TOPOLOGY_POINT_LIST; + const bool use_ngg_rectlist_draw = Config::NggRectlistDrawEnabled(); + + if (!GetDrawTopology(ucfg, true, use_ngg_rectlist_draw, &topology)) { + return; + } + const bool draw_prim7_as_ngg = + (use_ngg_rectlist_draw && + ucfg->GetPrimType() == Prospero::GpuEnumValue(Prospero::PrimitiveType::kRectList)); + + RefreshShaders(ctx, sh_ctx, draw, false, &state); + + if (draw_prim7_as_ngg && state.vs_input_info.buffers_num == 0 && + state.vs_input_info.param_export_mask == 0 && state.ps_input_info.input_num != 0) { + if (graphics_debug_dump_enabled()) { + LOGF("DrawIndexAuto: skipping rect-list draw with no VS param exports and PS inputs: " + "ps_inputs=%u ps=0x%016" PRIx64 " es=0x%016" PRIx64 " gs=0x%016" PRIx64 "\n", + state.ps_input_info.input_num, sh_ctx->GetPs().ps_regs.chksum, + sh_ctx->GetVs().es_regs.data_addr, sh_ctx->GetVs().gs_regs.data_addr); + } + return; + } + + LogDrawStateIfNeeded(ctx, ucfg, draw, state, false, + ucfg->GetPrimType() == + Prospero::GpuEnumValue(Prospero::PrimitiveType::kRectListLegacy), + 0, nullptr); + + const uint32_t draw_vertex_count = (draw_prim7_as_ngg ? 4u : index_count); + const auto vertex_offset = ResolveVertexOffset(ucfg->GetIndexOffset(), state.vs_input_info) + + static_cast(first_vertex); + DrawEmitInfo emit {}; + emit.draw_prim7_as_ngg = draw_prim7_as_ngg; + emit.draw_vertex_count = draw_vertex_count; + emit.first_vertex = static_cast(vertex_offset); + + DrawIndexBufferBind index_bind {}; + ExecutePreparedDraw(submit_id, buffer, ctx, ucfg, sh_ctx, draw, &state, topology, emit, + index_bind, false, false, true); +} + +static bool ResolveColorTargets(uint64_t submit_id, CommandBuffer* buffer, const HW::Context& hw, + uint32_t render_target_slice_offset) { + if (hw.GetColorControl().mode != 3) { + return false; + } + + const auto& src_rt = hw.GetRenderTarget(0); + const auto& dst_rt = hw.GetRenderTarget(1); + if (src_rt.base.addr == 0 || dst_rt.base.addr == 0) { + return false; + } + + RenderColorInfo src {}; + RenderColorInfo dst {}; + ResolveRenderColorTarget(submit_id, buffer, hw, &src, render_target_slice_offset, 0, true, + true); + ResolveRenderColorTarget(submit_id, buffer, hw, &dst, render_target_slice_offset, 1, true); + if (src.vulkan_buffer == nullptr || dst.vulkan_buffer == nullptr || + src.type == RenderColorType::NoColorOutput || dst.type == RenderColorType::NoColorOutput) { + return false; + } + if (src.base_addr == dst.base_addr && src.base_mip_level == dst.base_mip_level) { + return true; + } + + const uint32_t width = std::min(src.extent.width, dst.extent.width); + const uint32_t height = std::min(src.extent.height, dst.extent.height); + if (width == 0 || height == 0) { + return false; + } + + std::vector regions; + ImageImageCopy region {}; + region.src_image = src.vulkan_buffer; + region.src_level = src.base_mip_level; + region.dst_level = dst.base_mip_level; + region.width = width; + region.height = height; + regions.push_back(region); + + MarkRenderTargetGpuWritten(dst); + UtilImageToImage(buffer, regions, dst.vulkan_buffer, dst.vulkan_buffer->layout); + return true; +} + +} // namespace Libs::Graphics diff --git a/src/graphics/host_gpu/renderer/renderInternal.h b/src/graphics/host_gpu/renderer/renderInternal.h new file mode 100644 index 000000000..14eb6f238 --- /dev/null +++ b/src/graphics/host_gpu/renderer/renderInternal.h @@ -0,0 +1,21 @@ +#ifndef EMULATOR_SRC_GRAPHICS_HOST_GPU_RENDERER_RENDERINTERNAL_H_ +#define EMULATOR_SRC_GRAPHICS_HOST_GPU_RENDERER_RENDERINTERNAL_H_ + +#include "common/assert.h" +#include "graphics/shader/recompiler/ShaderIR.h" + +#include + +namespace Libs::Graphics { + +template +T DecodeNativeDescriptor(const ShaderRecompiler::IR::DescriptorValue& value) { + T result {}; + EXIT_IF(value.dword_count < sizeof(result) / sizeof(uint32_t)); + std::memcpy(&result, value.dwords.data(), sizeof(result)); + return result; +} + +} // namespace Libs::Graphics + +#endif // EMULATOR_SRC_GRAPHICS_HOST_GPU_RENDERER_RENDERINTERNAL_H_ diff --git a/src/graphics/host_gpu/renderer/renderState.h b/src/graphics/host_gpu/renderer/renderState.h new file mode 100644 index 000000000..19e807480 --- /dev/null +++ b/src/graphics/host_gpu/renderer/renderState.h @@ -0,0 +1,281 @@ +#ifndef EMULATOR_SRC_GRAPHICS_HOST_GPU_RENDERER_RENDERSTATE_H_ +#define EMULATOR_SRC_GRAPHICS_HOST_GPU_RENDERER_RENDERSTATE_H_ + +#include "common/abi.h" +#include "common/assert.h" +#include "common/common.h" + +#include +#include +#include +#include +#include + +namespace Libs::Graphics { + +class CommandBuffer; +struct DepthStencilVulkanImage; +struct RenderTextureVulkanImage; +struct ShaderVertexInputInfo; +struct VulkanImage; + +namespace HW { +class Context; +class Shader; +class UserConfig; +struct RenderTarget; +struct ScanModeControl; +struct ScreenViewport; +} // namespace HW + +static constexpr uint32_t RENDER_COLOR_ATTACHMENTS_MAX = 8; + +inline constexpr bool depth_msaa_single_sample_compatible(uint32_t encoded_fragments) { + return encoded_fragments == 1 || encoded_fragments == 2; +} + +inline constexpr bool color_msaa_single_sample_compatible(uint32_t encoded_samples, + uint32_t encoded_fragments) { + return encoded_samples == encoded_fragments && + depth_msaa_single_sample_compatible(encoded_fragments); +} + +inline constexpr bool depth_htile_stencil_acceleration_compatible(bool has_stencil, bool has_htile, + bool acceleration_disabled) { + return acceleration_disabled || (has_stencil && has_htile); +} + +// Pack structs to guarantee the uniqueness of object representation. +#pragma pack(push, 1) + +struct PipelineStencilStaticState { + VkStencilOp failOp = VK_STENCIL_OP_KEEP; + VkStencilOp passOp = VK_STENCIL_OP_KEEP; + VkStencilOp depthFailOp = VK_STENCIL_OP_KEEP; + VkCompareOp compareOp = VK_COMPARE_OP_NEVER; +}; + +struct PipelineStencilDynamicState { + uint32_t compareMask = 0; + uint32_t writeMask = 0; + uint32_t reference = 0; +}; + +inline constexpr bool stencil_face_accesses_attachment(const PipelineStencilStaticState& state, + const PipelineStencilDynamicState& dynamic) { + return state.compareOp != VK_COMPARE_OP_ALWAYS || + (dynamic.writeMask != 0 && + (state.failOp != VK_STENCIL_OP_KEEP || state.passOp != VK_STENCIL_OP_KEEP || + state.depthFailOp != VK_STENCIL_OP_KEEP)); +} + +struct PipelineStaticParameters { + float viewport_scale[3] = {}; + float viewport_offset[3] = {}; + bool negative_one_to_one = false; + int scissor_ltrb[4] = {0}; + VkPrimitiveTopology topology = VK_PRIMITIVE_TOPOLOGY_POINT_LIST; + bool with_depth = false; + bool depth_test_enable = false; + bool depth_write_enable = false; + VkCompareOp depth_compare_op = VK_COMPARE_OP_NEVER; + bool depth_bounds_test_enable = false; + float depth_min_bounds = 0.0f; + float depth_max_bounds = 0.0f; + bool stencil_test_enable = false; + PipelineStencilStaticState stencil_front; + PipelineStencilStaticState stencil_back; + uint32_t color_count = 1; + uint32_t color_mask[RENDER_COLOR_ATTACHMENTS_MAX] = {}; + bool cull_front = false; + bool cull_back = false; + bool face = false; + uint8_t color_srcblend[RENDER_COLOR_ATTACHMENTS_MAX] = {}; + uint8_t color_comb_fcn[RENDER_COLOR_ATTACHMENTS_MAX] = {}; + uint8_t color_destblend[RENDER_COLOR_ATTACHMENTS_MAX] = {}; + uint8_t alpha_srcblend[RENDER_COLOR_ATTACHMENTS_MAX] = {}; + uint8_t alpha_comb_fcn[RENDER_COLOR_ATTACHMENTS_MAX] = {}; + uint8_t alpha_destblend[RENDER_COLOR_ATTACHMENTS_MAX] = {}; + bool separate_alpha_blend[RENDER_COLOR_ATTACHMENTS_MAX] = {}; + bool blend_enable[RENDER_COLOR_ATTACHMENTS_MAX] = {}; + bool blend_bypass[RENDER_COLOR_ATTACHMENTS_MAX] = {}; + float blend_color_red = 0.0f; + float blend_color_green = 0.0f; + float blend_color_blue = 0.0f; + float blend_color_alpha = 0.0f; + + bool operator==(const PipelineStaticParameters& other) const noexcept; +}; + +struct PipelineDynamicParameters { + bool stencil_test_enable = false; + + float viewport_scale[3] = {}; + float viewport_offset[3] = {}; + int scissor_ltrb[4] = {0}; + + float line_width = 1.0f; + uint32_t color_write_count = 1; + bool color_write_enable[RENDER_COLOR_ATTACHMENTS_MAX] = {true, true}; + + PipelineStencilDynamicState stencil_front; + PipelineStencilDynamicState stencil_back; +}; + +#pragma pack(pop) + +static_assert(std::is_trivially_copyable_v); +static_assert(std::is_standard_layout_v); +static_assert(alignof(PipelineStencilStaticState) == 1); +static_assert(sizeof(PipelineStencilStaticState) == sizeof(VkStencilOp) * 3 + sizeof(VkCompareOp)); +static_assert(std::is_trivially_copyable_v); +static_assert(std::is_standard_layout_v); +static_assert(alignof(PipelineStencilDynamicState) == 1); +static_assert(sizeof(PipelineStencilDynamicState) == sizeof(uint32_t) * 3); +static_assert(std::is_trivially_copyable_v); +static_assert(std::is_standard_layout_v); +static_assert(alignof(PipelineStaticParameters) == 1); +static_assert(sizeof(PipelineStaticParameters) == + sizeof(float[3]) + sizeof(float[3]) + sizeof(bool) + sizeof(int[4]) + + sizeof(VkPrimitiveTopology) + sizeof(bool) * 3 + sizeof(VkCompareOp) + + sizeof(bool) + sizeof(float) * 2 + sizeof(bool) + + sizeof(PipelineStencilStaticState) * 2 + sizeof(uint32_t) + + sizeof(uint32_t[RENDER_COLOR_ATTACHMENTS_MAX]) + sizeof(bool) * 3 + + sizeof(uint8_t[RENDER_COLOR_ATTACHMENTS_MAX]) * 6 + + sizeof(bool[RENDER_COLOR_ATTACHMENTS_MAX]) * 3 + sizeof(float) * 4); +static_assert(std::is_trivially_copyable_v); +static_assert(std::is_standard_layout_v); +static_assert(alignof(PipelineDynamicParameters) == 1); +static_assert(sizeof(PipelineDynamicParameters) == + sizeof(bool) + sizeof(float[3]) + sizeof(float[3]) + sizeof(int[4]) + + sizeof(float) + sizeof(uint32_t) + sizeof(bool[RENDER_COLOR_ATTACHMENTS_MAX]) + + sizeof(PipelineStencilDynamicState) * 2); + +struct RenderDepthInfo { + VkFormat format = VK_FORMAT_UNDEFINED; + uint32_t width = 0; + uint32_t height = 0; + bool htile = false; + uint64_t depth_buffer_size = 0; + uint64_t depth_buffer_vaddr = 0; + uint64_t depth_tile_swizzle = 0; + uint64_t stencil_buffer_size = 0; + uint64_t stencil_buffer_vaddr = 0; + uint64_t stencil_tile_swizzle = 0; + uint64_t htile_buffer_size = 0; + uint64_t htile_buffer_vaddr = 0; + uint64_t htile_tile_swizzle = 0; + bool depth_clear_enable = false; + bool depth_load_clear_enable = false; + bool depth_meta_clear_enable = false; + float depth_clear_value = 0.0f; + bool depth_test_enable = false; + bool depth_write_enable = false; + VkCompareOp depth_compare_op = VK_COMPARE_OP_NEVER; + bool depth_bounds_test_enable = false; + float depth_min_bounds = 0.0f; + float depth_max_bounds = 0.0f; + bool stencil_clear_enable = false; + uint8_t stencil_clear_value = 0; + bool stencil_test_enable = false; + PipelineStencilStaticState stencil_static_front; + PipelineStencilStaticState stencil_static_back; + PipelineStencilDynamicState stencil_dynamic_front; + PipelineStencilDynamicState stencil_dynamic_back; + DepthStencilVulkanImage* vulkan_buffer = nullptr; + uint64_t vaddr[3] = {}; + uint64_t size[3] = {}; + int vaddr_num = 0; +}; + +inline bool depth_attachment_read_only(const RenderDepthInfo* depth) { + EXIT_IF(depth == nullptr); + + const bool stencil_write = + depth->stencil_test_enable && + (depth->stencil_dynamic_front.writeMask != 0 || depth->stencil_dynamic_back.writeMask != 0); + + return !depth->depth_load_clear_enable && !depth->stencil_clear_enable && + !depth->depth_write_enable && !stencil_write; +} + +inline VkImageLayout depth_attachment_layout(const RenderDepthInfo* depth) { + return depth_attachment_read_only(depth) ? VK_IMAGE_LAYOUT_DEPTH_STENCIL_READ_ONLY_OPTIMAL + : VK_IMAGE_LAYOUT_DEPTH_STENCIL_ATTACHMENT_OPTIMAL; +} + +enum class RenderColorType { + NoColorOutput, + DisplayBuffer, + RenderTexture, +}; + +struct RenderColorInfo { + RenderColorType type = RenderColorType::NoColorOutput; + VulkanImage* vulkan_buffer = nullptr; + VkImageView vulkan_view = nullptr; + VkFormat format = VK_FORMAT_UNDEFINED; + VkExtent2D extent = {}; + uint32_t base_mip_level = 0; + uint64_t base_addr = 0; + uint64_t buffer_size = 0; + uint32_t target_slot = 0; + bool color_clear_enable = false; + VkClearColorValue color_clear_value {}; +}; + +struct ScissorRect { + int left = 0; + int top = 0; + int right = 0; + int bottom = 0; +}; + +enum class CommandBufferDebugOp : uint32_t { + DispatchDirect, + DrawIndex, + DrawIndexAuto, + EopWrite, + EopInterrupt, + EopWriteBack, + EopFlip, + EopWriteBackFlip, + EopOnlyFlip, + Unknown, +}; + +uint32_t render_target_mask_slot(uint32_t mask, uint32_t slot); +uint32_t render_target_first_bound_slot(const HW::Context& hw); +bool graphics_debug_dump_enabled(); +void uc_print(const char* func, const HW::UserConfig& uc); +void uc_check(const HW::UserConfig& uc); +void sh_print(const char* func, const HW::Shader& uc); +void sh_check(const HW::Shader& uc); +std::vector rt_print(const char* func, const HW::RenderTarget& rt); +bool RenderIsColorTileModeLinear(uint32_t tile_mode); +void hw_print(const HW::Context& hw); +void hw_check(const HW::Context& hw); +void LogDrawPhase(const char* draw_name, const char* phase); +ScissorRect calc_final_scissor(const HW::ScreenViewport& vp, const HW::ScanModeControl& smc, + VkExtent2D extent); +int32_t ResolveVertexOffset(uint32_t index_offset, const ShaderVertexInputInfo& vs_input_info); + +void GraphicsRenderTextureBarrier(VkCommandBuffer vk_buffer, VulkanImage* image); +void GraphicsRenderColorImageBarrier(VkCommandBuffer vk_buffer, VulkanImage* image, + VkImageLayout new_layout); +void GraphicsRenderDepthStencilImageBarrier(VkCommandBuffer vk_buffer, VulkanImage* image, + VkImageLayout new_layout); +void GraphicsRenderDepthStencilBarrier(VkCommandBuffer vk_buffer, VulkanImage* image); +void ResolveRenderDepthTarget(uint64_t submit_id, CommandBuffer* buffer, const HW::Context& hw, + RenderDepthInfo* r); +void ResolveRenderColorTarget(uint64_t submit_id, CommandBuffer* buffer, const HW::Context& hw, + RenderColorInfo* r, uint32_t render_target_slice_offset = 0, + uint32_t render_target_slot = UINT32_MAX, + bool ignore_target_mask = false, + bool reuse_existing_render_texture = false); +void MarkRenderTargetGpuWritten(const RenderColorInfo& target); +void MarkRenderTargetGpuWritten(const RenderDepthInfo& target); + +} // namespace Libs::Graphics + +#endif // EMULATOR_SRC_GRAPHICS_HOST_GPU_RENDERER_RENDERSTATE_H_ diff --git a/src/graphics/host_gpu/renderer/renderTargetBarriers.cpp b/src/graphics/host_gpu/renderer/renderTargetBarriers.cpp new file mode 100644 index 000000000..c77dfd59d --- /dev/null +++ b/src/graphics/host_gpu/renderer/renderTargetBarriers.cpp @@ -0,0 +1,226 @@ +#include "common/assert.h" +#include "common/common.h" +#include "common/logging/log.h" +#include "common/profiler.h" +#include "common/stringUtils.h" +#include "common/threads.h" +#include "graphics/guest_gpu/gpu_defs.h" +#include "graphics/guest_gpu/hardwareContext.h" +#include "graphics/guest_gpu/tile.h" +#include "graphics/host_gpu/graphicContext.h" +#include "graphics/host_gpu/objects/textureCommon.h" +#include "graphics/host_gpu/renderer/descriptorCache.h" +#include "graphics/host_gpu/renderer/framebufferCache.h" +#include "graphics/host_gpu/renderer/render.h" +#include "graphics/host_gpu/renderer/renderContext.h" +#include "graphics/host_gpu/renderer/renderState.h" +#include "graphics/host_gpu/utils.h" +#include "graphics/presentation/displayBuffer.h" + +#include +#include +#include +#include +#include +#include +#include + +namespace Libs::Graphics { + +void GraphicsRenderMemoryBarrier(CommandBuffer* buffer) { + EXIT_IF(buffer == nullptr); + EXIT_IF(buffer->IsInvalid()); + + Common::LockGuard lock(g_render_ctx->GetMutex()); + + auto* vk_buffer = buffer->GetPool()->buffers[buffer->GetIndex()]; + + VkMemoryBarrier mem_barrier {}; + mem_barrier.sType = VK_STRUCTURE_TYPE_MEMORY_BARRIER; + mem_barrier.pNext = nullptr; + mem_barrier.srcAccessMask = VK_ACCESS_MEMORY_WRITE_BIT; + mem_barrier.dstAccessMask = VK_ACCESS_MEMORY_READ_BIT | VK_ACCESS_MEMORY_WRITE_BIT; + + vkCmdPipelineBarrier(vk_buffer, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, + VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, 0, 1, &mem_barrier, 0, nullptr, 0, + nullptr); +} + +void GraphicsRenderTextureBarrier(VkCommandBuffer vk_buffer, VulkanImage* image) { + EXIT_IF(image == nullptr); + + VkImageMemoryBarrier image_memory_barrier {}; + image_memory_barrier.sType = VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER; + image_memory_barrier.pNext = nullptr; + image_memory_barrier.srcAccessMask = VK_ACCESS_COLOR_ATTACHMENT_WRITE_BIT | + VK_ACCESS_TRANSFER_WRITE_BIT | VK_ACCESS_SHADER_WRITE_BIT | + VK_ACCESS_MEMORY_READ_BIT; + image_memory_barrier.dstAccessMask = VK_ACCESS_SHADER_READ_BIT; + image_memory_barrier.oldLayout = image->layout; + image_memory_barrier.newLayout = RENDER_COLOR_IMAGE_LAYOUT; + image_memory_barrier.srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; + image_memory_barrier.dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; + image_memory_barrier.image = image->image; + image_memory_barrier.subresourceRange.aspectMask = VK_IMAGE_ASPECT_COLOR_BIT; + image_memory_barrier.subresourceRange.baseMipLevel = 0; + image_memory_barrier.subresourceRange.levelCount = VK_REMAINING_MIP_LEVELS; + image_memory_barrier.subresourceRange.baseArrayLayer = 0; + image_memory_barrier.subresourceRange.layerCount = image->layers; + + vkCmdPipelineBarrier( + vk_buffer, + VK_PIPELINE_STAGE_COLOR_ATTACHMENT_OUTPUT_BIT | VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT | + VK_PIPELINE_STAGE_TRANSFER_BIT | VK_PIPELINE_STAGE_FRAGMENT_SHADER_BIT, + VK_PIPELINE_STAGE_VERTEX_SHADER_BIT | VK_PIPELINE_STAGE_FRAGMENT_SHADER_BIT | + VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT, + 0, 0, nullptr, 0, nullptr, 1, &image_memory_barrier); + + image->layout = image_memory_barrier.newLayout; +} + +void GraphicsRenderColorImageBarrier(VkCommandBuffer vk_buffer, VulkanImage* image, + VkImageLayout new_layout) { + EXIT_IF(image == nullptr); + + if (image->layout != new_layout) { + if (image->type == VulkanImageType::VideoOut && + image->layout == VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL) { + static std::atomic log_count {0}; + if (log_count.fetch_add(1, std::memory_order_relaxed) < 128) { + LOGF("GraphicsRenderColorImageBarrier: image=%p type=%d layout=%s -> %s\n", + reinterpret_cast(image->image), static_cast(image->type), + string_VkImageLayout(image->layout), string_VkImageLayout(new_layout)); + } + } + VkImageMemoryBarrier image_memory_barrier {}; + image_memory_barrier.sType = VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER; + image_memory_barrier.pNext = nullptr; + image_memory_barrier.srcAccessMask = VK_ACCESS_MEMORY_WRITE_BIT | VK_ACCESS_MEMORY_READ_BIT; + image_memory_barrier.dstAccessMask = VK_ACCESS_MEMORY_WRITE_BIT | VK_ACCESS_MEMORY_READ_BIT; + image_memory_barrier.oldLayout = image->layout; + image_memory_barrier.newLayout = new_layout; + image_memory_barrier.srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; + image_memory_barrier.dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; + image_memory_barrier.image = image->image; + image_memory_barrier.subresourceRange.aspectMask = VK_IMAGE_ASPECT_COLOR_BIT; + image_memory_barrier.subresourceRange.baseMipLevel = 0; + image_memory_barrier.subresourceRange.levelCount = VK_REMAINING_MIP_LEVELS; + image_memory_barrier.subresourceRange.baseArrayLayer = 0; + image_memory_barrier.subresourceRange.layerCount = image->layers; + + const auto stages = static_cast(VK_PIPELINE_STAGE_ALL_GRAPHICS_BIT | + VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT | + VK_PIPELINE_STAGE_TRANSFER_BIT); + vkCmdPipelineBarrier(vk_buffer, stages, stages, 0, 0, nullptr, 0, nullptr, 1, + &image_memory_barrier); + + image->layout = image_memory_barrier.newLayout; + } +} + +VkImageAspectFlags DepthStencilAspectMask(VkFormat format); + +void GraphicsRenderDepthStencilImageBarrier(VkCommandBuffer vk_buffer, VulkanImage* image, + VkImageLayout new_layout) { + EXIT_IF(image == nullptr); + EXIT_IF(image->type != VulkanImageType::DepthStencil); + if (new_layout == VK_IMAGE_LAYOUT_UNDEFINED || new_layout == VK_IMAGE_LAYOUT_PREINITIALIZED) { + EXIT("invalid destination depth/stencil image layout: %s\n", + string_VkImageLayout(new_layout)); + } + if (image->layout == new_layout) { + return; + } + VkImageMemoryBarrier barrier {}; + barrier.sType = VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER; + barrier.srcAccessMask = VK_ACCESS_MEMORY_READ_BIT | VK_ACCESS_MEMORY_WRITE_BIT; + barrier.dstAccessMask = VK_ACCESS_MEMORY_READ_BIT | VK_ACCESS_MEMORY_WRITE_BIT; + barrier.oldLayout = image->layout; + barrier.newLayout = new_layout; + barrier.srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; + barrier.dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; + barrier.image = image->image; + barrier.subresourceRange = {DepthStencilAspectMask(image->format), 0, VK_REMAINING_MIP_LEVELS, + 0, image->layers}; + const auto stages = static_cast(VK_PIPELINE_STAGE_ALL_GRAPHICS_BIT | + VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT | + VK_PIPELINE_STAGE_TRANSFER_BIT); + vkCmdPipelineBarrier(vk_buffer, stages, stages, 0, 0, nullptr, 0, nullptr, 1, &barrier); + image->layout = new_layout; +} + +void GraphicsRenderDepthStencilBarrier(VkCommandBuffer vk_buffer, VulkanImage* image) { + EXIT_IF(image == nullptr); + + EXIT_IF(image->type != VulkanImageType::DepthStencil); + + auto* depth = static_cast(image); + if (depth->compressed) { + static std::atomic log_count {0}; + if (log_count.fetch_add(1, std::memory_order_relaxed) < 16) { + LOGF( + "DepthTexture: decompressing depth target for shader read format=%s extent=%ux%u\n", + string_VkFormat(depth->format), depth->extent.width, depth->extent.height); + } + depth->compressed = false; + } + + if (image->layout != VK_IMAGE_LAYOUT_DEPTH_STENCIL_READ_ONLY_OPTIMAL) { + VkImageMemoryBarrier image_memory_barrier {}; + image_memory_barrier.sType = VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER; + image_memory_barrier.pNext = nullptr; + image_memory_barrier.srcAccessMask = VK_ACCESS_MEMORY_READ_BIT | VK_ACCESS_MEMORY_WRITE_BIT; + image_memory_barrier.dstAccessMask = VK_ACCESS_SHADER_READ_BIT; + image_memory_barrier.oldLayout = image->layout; + image_memory_barrier.newLayout = VK_IMAGE_LAYOUT_DEPTH_STENCIL_READ_ONLY_OPTIMAL; + image_memory_barrier.srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; + image_memory_barrier.dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; + image_memory_barrier.image = image->image; + image_memory_barrier.subresourceRange.aspectMask = DepthStencilAspectMask(image->format); + image_memory_barrier.subresourceRange.baseMipLevel = 0; + image_memory_barrier.subresourceRange.levelCount = VK_REMAINING_MIP_LEVELS; + image_memory_barrier.subresourceRange.baseArrayLayer = 0; + image_memory_barrier.subresourceRange.layerCount = image->layers; + + vkCmdPipelineBarrier( + vk_buffer, + VK_PIPELINE_STAGE_ALL_GRAPHICS_BIT | VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT | + VK_PIPELINE_STAGE_TRANSFER_BIT, + VK_PIPELINE_STAGE_VERTEX_SHADER_BIT | VK_PIPELINE_STAGE_FRAGMENT_SHADER_BIT | + VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT, + 0, 0, nullptr, 0, nullptr, 1, &image_memory_barrier); + + image->layout = image_memory_barrier.newLayout; + } +} + +void GraphicsRenderTextureBarrier(CommandBuffer* buffer, uint64_t vaddr, uint64_t size) { + EXIT_IF(buffer == nullptr); + EXIT_IF(buffer->IsInvalid()); + + Common::LockGuard lock(g_render_ctx->GetMutex()); + + auto* vk_buffer = buffer->GetPool()->buffers[buffer->GetIndex()]; + + auto* native = g_render_ctx->GetTextureCache()->FindRenderTargetByRange(buffer, vaddr, size); + if (native == nullptr) { + EXIT("render-target barrier range has no cached image\n"); + } + GraphicsRenderTextureBarrier(vk_buffer, native); +} + +void GraphicsRenderDepthStencilBarrier(CommandBuffer* buffer, uint64_t vaddr, uint64_t size) { + EXIT_IF(buffer == nullptr); + EXIT_IF(buffer->IsInvalid()); + + Common::LockGuard lock(g_render_ctx->GetMutex()); + + auto* vk_buffer = buffer->GetPool()->buffers[buffer->GetIndex()]; + auto* native = g_render_ctx->GetTextureCache()->FindDepthTargetByRange(vaddr, size); + if (native == nullptr) { + EXIT("depth-target barrier range has no cached image\n"); + } + GraphicsRenderDepthStencilBarrier(vk_buffer, native); +} + +} // namespace Libs::Graphics diff --git a/src/graphics/host_gpu/renderer/renderVertex.cpp b/src/graphics/host_gpu/renderer/renderVertex.cpp new file mode 100644 index 000000000..4de0dbc9a --- /dev/null +++ b/src/graphics/host_gpu/renderer/renderVertex.cpp @@ -0,0 +1,28 @@ +#include "common/assert.h" +#include "graphics/host_gpu/renderer/renderState.h" +#include "graphics/shader/recompiler/ResourceMaterialization.h" +#include "graphics/shader/recompiler/ShaderIR.h" +#include "graphics/shader/shader.h" + +namespace Libs::Graphics { + +int32_t ResolveVertexOffset(uint32_t index_offset, const ShaderVertexInputInfo& vs_input_info) { + if (index_offset != 0 || !vs_input_info.fetch_embedded) { + return static_cast(index_offset); + } + + EXIT_IF(!vs_input_info.stage); + const auto& program = *vs_input_info.stage.program; + const auto& resources = *vs_input_info.stage.resources; + if (program.info.vertex_offset_sgpr >= static_cast(program.user_data_base)) { + const auto index = + static_cast(program.info.vertex_offset_sgpr) - program.user_data_base; + if (index < resources.user_data.size()) { + return static_cast(resources.user_data[index]); + } + } + + return 0; +} + +} // namespace Libs::Graphics diff --git a/src/graphics/host_gpu/renderer/resourceMutex.cpp b/src/graphics/host_gpu/renderer/resourceMutex.cpp new file mode 100644 index 000000000..2a2ac4d7f --- /dev/null +++ b/src/graphics/host_gpu/renderer/resourceMutex.cpp @@ -0,0 +1,93 @@ +#include "graphics/host_gpu/renderer/resourceMutex.h" + +#include "common/assert.h" + +namespace Libs::Graphics { + +ResourceMutex::FaultScope::FaultScope(ResourceMutex& mutex): m_mutex(mutex) { + m_resource_preowned = m_mutex.BeginFault(); +} + +ResourceMutex::FaultScope::~FaultScope() { + m_mutex.EndFault(m_resource_preowned); +} + +ResourceMutex::~ResourceMutex() { + std::lock_guard state(m_state); + if (m_resource_owner != std::thread::id {} || m_fault_owner != std::thread::id {}) { + EXIT("ResourceMutex destroyed with an active owner or fault transaction\n"); + } +} + +void ResourceMutex::lock() { + const auto current = std::this_thread::get_id(); + { + std::lock_guard state(m_state); + if (m_resource_owner == current) { + EXIT("recursive resource transaction\n"); + } + if (m_fault_owner == current) { + EXIT("resource transaction re-entered from a page-fault callback\n"); + } + } + m_resource.lock(); + std::lock_guard state(m_state); + if (m_resource_owner != std::thread::id {}) { + EXIT("resource mutex acquired with a stale owner\n"); + } + m_resource_owner = current; +} + +void ResourceMutex::unlock() { + const auto current = std::this_thread::get_id(); + { + std::lock_guard state(m_state); + if (m_resource_owner != current) { + EXIT("resource transaction released without ownership\n"); + } + m_resource_owner = {}; + } + m_resource.unlock(); +} + +bool ResourceMutex::IsOwnedByCurrentThread() { + std::lock_guard state(m_state); + return m_resource_owner == std::this_thread::get_id(); +} + +bool ResourceMutex::BeginFault() { + const auto current = std::this_thread::get_id(); + { + std::lock_guard state(m_state); + if (m_fault_owner == current) { + EXIT("nested resource page-fault transaction\n"); + } + if (m_resource_owner == current) { + m_fault_owner = current; + return true; + } + } + lock(); + std::lock_guard state(m_state); + if (m_resource_owner != current || m_fault_owner != std::thread::id {}) { + EXIT("resource fault transaction acquired inconsistent ownership\n"); + } + m_fault_owner = current; + return false; +} + +void ResourceMutex::EndFault(bool resource_preowned) { + const auto current = std::this_thread::get_id(); + { + std::lock_guard state(m_state); + if (m_fault_owner != current || m_resource_owner != current) { + EXIT("resource page-fault transaction ended with inconsistent ownership\n"); + } + m_fault_owner = {}; + } + if (!resource_preowned) { + unlock(); + } +} + +} // namespace Libs::Graphics diff --git a/src/graphics/host_gpu/renderer/resourceMutex.h b/src/graphics/host_gpu/renderer/resourceMutex.h new file mode 100644 index 000000000..b6ea1c1a9 --- /dev/null +++ b/src/graphics/host_gpu/renderer/resourceMutex.h @@ -0,0 +1,48 @@ +#ifndef EMULATOR_SRC_GRAPHICS_HOST_GPU_RENDERER_RESOURCEMUTEX_H_ +#define EMULATOR_SRC_GRAPHICS_HOST_GPU_RENDERER_RESOURCEMUTEX_H_ + +#include "common/common.h" + +#include +#include + +namespace Libs::Graphics { + +// Owner-tracked shared buffer/image transaction. External faults pause GPU submissions before +// taking this mutex; synchronous command-processor accesses reuse their already-owned transaction. +class ResourceMutex final { +public: + class FaultScope final { + public: + explicit FaultScope(ResourceMutex& mutex); + ~FaultScope(); + KYTY_CLASS_NO_COPY(FaultScope); + + private: + ResourceMutex& m_mutex; + bool m_resource_preowned = false; + }; + + ResourceMutex() = default; + ~ResourceMutex(); + KYTY_CLASS_NO_COPY(ResourceMutex); + + void lock(); + void unlock(); + [[nodiscard]] bool IsOwnedByCurrentThread(); + +private: + friend class FaultScope; + + [[nodiscard]] bool BeginFault(); + void EndFault(bool resource_preowned); + + std::mutex m_resource; + std::mutex m_state; + std::thread::id m_resource_owner; + std::thread::id m_fault_owner; +}; + +} // namespace Libs::Graphics + +#endif // EMULATOR_SRC_GRAPHICS_HOST_GPU_RENDERER_RESOURCEMUTEX_H_ diff --git a/src/graphics/host_gpu/renderer/samplerCache.cpp b/src/graphics/host_gpu/renderer/samplerCache.cpp new file mode 100644 index 000000000..9c338649b --- /dev/null +++ b/src/graphics/host_gpu/renderer/samplerCache.cpp @@ -0,0 +1,146 @@ +#include "graphics/host_gpu/renderer/samplerCache.h" + +#include "common/assert.h" +#include "common/logging/log.h" +#include "graphics/guest_gpu/gpu_defs.h" +#include "graphics/host_gpu/renderer/renderContext.h" + +namespace Libs::Graphics { + +VkSampler SamplerCache::GetSampler(const ShaderSamplerResource& r) { + Common::LockGuard lock(m_mutex); + + const SamplerKey key {r.fields[0], r.fields[1], r.fields[2], r.fields[3]}; + if (auto iter = m_samplers.find(key); iter != m_samplers.end()) { + return iter->second; + } + + float aniso_ratio = 1.0f; + const auto mag_filter = r.XyMagFilter(); + const auto min_filter = r.XyMinFilter(); + + auto is_aniso_filter = [](uint8_t filter) { + switch (static_cast(filter)) { + case Prospero::SamplerFilter::kAnisoPoint: + case Prospero::SamplerFilter::kAnisoLinear: return true; + case Prospero::SamplerFilter::kPoint: + case Prospero::SamplerFilter::kBilinear: return false; + default: EXIT("unknown sampler filter: %u\n", filter); + } + return false; + }; + + auto to_vk_filter = [](uint8_t filter) { + switch (static_cast(filter)) { + case Prospero::SamplerFilter::kPoint: + case Prospero::SamplerFilter::kAnisoPoint: return VK_FILTER_NEAREST; + case Prospero::SamplerFilter::kBilinear: + case Prospero::SamplerFilter::kAnisoLinear: return VK_FILTER_LINEAR; + default: EXIT("unknown sampler filter: %u\n", filter); + } + return VK_FILTER_NEAREST; + }; + + const bool aniso = is_aniso_filter(mag_filter) || is_aniso_filter(min_filter); + if (aniso) { + switch (static_cast(r.MaxAnisoRatio())) { + case Prospero::SamplerAnisoRatio::kOne: aniso_ratio = 1.0f; break; + case Prospero::SamplerAnisoRatio::kTwo: aniso_ratio = 2.0f; break; + case Prospero::SamplerAnisoRatio::kFour: aniso_ratio = 4.0f; break; + case Prospero::SamplerAnisoRatio::kEight: aniso_ratio = 8.0f; break; + case Prospero::SamplerAnisoRatio::kSixteen: aniso_ratio = 16.0f; break; + default: EXIT("unknown ratio: %d\n", static_cast(r.MaxAnisoRatio())); + } + } + + const auto mip_filter = r.MipFilter(); + float min_lod = 0.0f; + float max_lod = 0.0f; + if (static_cast(mip_filter) != Prospero::SamplerMipFilter::kNone) { + min_lod = static_cast(r.MinLod()) / 256.0f; + max_lod = static_cast(r.MaxLod()) / 256.0f; + } + + VkSamplerCreateInfo sampler_info {}; + + auto to_vk_address_mode = [](uint8_t clamp) { + switch (static_cast(clamp)) { + case Prospero::SamplerClampMode::kWrap: return VK_SAMPLER_ADDRESS_MODE_REPEAT; + case Prospero::SamplerClampMode::kMirror: return VK_SAMPLER_ADDRESS_MODE_MIRRORED_REPEAT; + case Prospero::SamplerClampMode::kClampLastTexel: return VK_SAMPLER_ADDRESS_MODE_CLAMP_TO_EDGE; + case Prospero::SamplerClampMode::kMirrorOnceLastTexel: + return VK_SAMPLER_ADDRESS_MODE_MIRROR_CLAMP_TO_EDGE; + case Prospero::SamplerClampMode::kClampHalfBorder: + return VK_SAMPLER_ADDRESS_MODE_CLAMP_TO_BORDER; + case Prospero::SamplerClampMode::kMirrorOnceHalfBorder: + return VK_SAMPLER_ADDRESS_MODE_MIRROR_CLAMP_TO_EDGE; + case Prospero::SamplerClampMode::kClampBorder: return VK_SAMPLER_ADDRESS_MODE_CLAMP_TO_BORDER; + case Prospero::SamplerClampMode::kMirrorOnceBorder: + return VK_SAMPLER_ADDRESS_MODE_MIRROR_CLAMP_TO_EDGE; + default: EXIT("unknown clamp: %u\n", clamp); + } + return VK_SAMPLER_ADDRESS_MODE_CLAMP_TO_BORDER; + }; + + VkBorderColor border = VK_BORDER_COLOR_INT_TRANSPARENT_BLACK; + switch (static_cast(r.BorderColorType())) { + case Prospero::SamplerBorderColor::kTransBlack: + border = VK_BORDER_COLOR_INT_TRANSPARENT_BLACK; + break; + case Prospero::SamplerBorderColor::kOpaqueBlack: border = VK_BORDER_COLOR_INT_OPAQUE_BLACK; break; + case Prospero::SamplerBorderColor::kOpaqueWhite: border = VK_BORDER_COLOR_INT_OPAQUE_WHITE; break; + case Prospero::SamplerBorderColor::kFromTable: + LOGF( + "temporary: approximating table border color as transparent black, index = %" PRIu16 + "\n", + r.BorderColorPtr()); + border = VK_BORDER_COLOR_INT_TRANSPARENT_BLACK; + break; + default: EXIT("unknown border color: %d", static_cast(r.BorderColorType())); + } + + sampler_info.sType = VK_STRUCTURE_TYPE_SAMPLER_CREATE_INFO; + sampler_info.pNext = nullptr; + sampler_info.flags = 0; + sampler_info.magFilter = to_vk_filter(mag_filter); + sampler_info.minFilter = to_vk_filter(min_filter); + sampler_info.mipmapMode = + (static_cast(mip_filter) == Prospero::SamplerMipFilter::kLinear + ? VK_SAMPLER_MIPMAP_MODE_LINEAR + : VK_SAMPLER_MIPMAP_MODE_NEAREST); + sampler_info.addressModeU = to_vk_address_mode(r.ClampX()); + sampler_info.addressModeV = to_vk_address_mode(r.ClampY()); + sampler_info.addressModeW = to_vk_address_mode(r.ClampZ()); + sampler_info.mipLodBias = + static_cast(static_cast((r.LodBias() ^ 0x2000u) - 0x2000u)) / 256.0f; + sampler_info.anisotropyEnable = (aniso ? VK_TRUE : VK_FALSE); + sampler_info.maxAnisotropy = aniso_ratio; + sampler_info.compareEnable = (r.DepthCompareFunc() != 0 ? VK_TRUE : VK_FALSE); + sampler_info.compareOp = static_cast(r.DepthCompareFunc()); + sampler_info.minLod = min_lod; + sampler_info.maxLod = max_lod; + sampler_info.borderColor = border; + sampler_info.unnormalizedCoordinates = (r.ForceUnormCoords() ? VK_TRUE : VK_FALSE); + + if (r.ForceUnormCoords()) { + sampler_info.addressModeU = VK_SAMPLER_ADDRESS_MODE_CLAMP_TO_EDGE; + sampler_info.addressModeV = VK_SAMPLER_ADDRESS_MODE_CLAMP_TO_EDGE; + sampler_info.addressModeW = VK_SAMPLER_ADDRESS_MODE_CLAMP_TO_EDGE; + sampler_info.mipmapMode = VK_SAMPLER_MIPMAP_MODE_NEAREST; + sampler_info.minLod = 0.0f; + sampler_info.maxLod = 0.0f; + sampler_info.anisotropyEnable = VK_FALSE; + sampler_info.maxAnisotropy = 1.0f; + sampler_info.compareEnable = VK_FALSE; + sampler_info.mipLodBias = 0.0f; + } + + VkSampler vk_sampler = nullptr; + vkCreateSampler(g_render_ctx->GetGraphicCtx()->device, &sampler_info, nullptr, &vk_sampler); + EXIT_NOT_IMPLEMENTED(vk_sampler == nullptr); + + m_samplers.emplace(key, vk_sampler); + return vk_sampler; +} + +} // namespace Libs::Graphics diff --git a/src/graphics/host_gpu/renderer/samplerCache.h b/src/graphics/host_gpu/renderer/samplerCache.h new file mode 100644 index 000000000..65301842f --- /dev/null +++ b/src/graphics/host_gpu/renderer/samplerCache.h @@ -0,0 +1,46 @@ +#ifndef EMULATOR_SRC_GRAPHICS_HOST_GPU_RENDERER_SAMPLERCACHE_H_ +#define EMULATOR_SRC_GRAPHICS_HOST_GPU_RENDERER_SAMPLERCACHE_H_ + +#include "common/abi.h" +#include "common/assert.h" +#include "common/common.h" +#include "common/threads.h" +#include "graphics/shader/shaderBindings.h" + +#include +#include +#include +#include + +namespace Libs::Graphics { + +class SamplerCache { +public: + SamplerCache() { EXIT_NOT_IMPLEMENTED(!Common::Thread::IsMainThread()); } + virtual ~SamplerCache() { KYTY_NOT_IMPLEMENTED; } + KYTY_CLASS_NO_COPY(SamplerCache); + + VkSampler GetSampler(const ShaderSamplerResource& r); + +private: + using SamplerKey = std::array; + + struct SamplerKeyHash { + std::size_t operator()(const SamplerKey& key) const { + std::size_t hash = 0; + for (auto value: key) { + hash ^= static_cast(value) + + static_cast(0x9e3779b97f4a7c15ull) + (hash << 6u) + + (hash >> 2u); + } + return hash; + } + }; + + Common::Mutex m_mutex; + std::unordered_map m_samplers; +}; + +} // namespace Libs::Graphics + +#endif // EMULATOR_SRC_GRAPHICS_HOST_GPU_RENDERER_SAMPLERCACHE_H_ diff --git a/src/graphics/host_gpu/renderer/shaderResourceBarrier.cpp b/src/graphics/host_gpu/renderer/shaderResourceBarrier.cpp new file mode 100644 index 000000000..5865224e6 --- /dev/null +++ b/src/graphics/host_gpu/renderer/shaderResourceBarrier.cpp @@ -0,0 +1,127 @@ +#include "graphics/host_gpu/renderer/shaderResourceBarrier.h" + +#include "common/assert.h" +#include "graphics/host_gpu/renderer/descriptorCache.h" +#include "graphics/shader/shaderBindings.h" +#include "graphics/shader/shader.h" + +#include + +namespace Libs::Graphics { + +VkPipelineStageFlags ShaderPipelineStages(VkShaderStageFlags stages) { + VkPipelineStageFlags result = 0; + if ((stages & VK_SHADER_STAGE_VERTEX_BIT) != 0) { + result |= VK_PIPELINE_STAGE_VERTEX_SHADER_BIT; + } + if ((stages & VK_SHADER_STAGE_FRAGMENT_BIT) != 0) { + result |= VK_PIPELINE_STAGE_FRAGMENT_SHADER_BIT; + } + if ((stages & VK_SHADER_STAGE_COMPUTE_BIT) != 0) { + result |= VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT; + } + EXIT_IF(result == 0); + return result; +} + +VkMemoryBarrier MakeShaderWriteDependency() { + VkMemoryBarrier barrier {}; + barrier.sType = VK_STRUCTURE_TYPE_MEMORY_BARRIER; + barrier.srcAccessMask = VK_ACCESS_SHADER_WRITE_BIT; + barrier.dstAccessMask = VK_ACCESS_SHADER_READ_BIT | VK_ACCESS_SHADER_WRITE_BIT | + VK_ACCESS_VERTEX_ATTRIBUTE_READ_BIT | VK_ACCESS_INDEX_READ_BIT | + VK_ACCESS_UNIFORM_READ_BIT | VK_ACCESS_TRANSFER_READ_BIT | + VK_ACCESS_TRANSFER_WRITE_BIT | VK_ACCESS_COLOR_ATTACHMENT_READ_BIT | + VK_ACCESS_COLOR_ATTACHMENT_WRITE_BIT; + return barrier; +} + +VkImageMemoryBarrier MakeStorageImageDependency(const VulkanImage& image, bool read, bool written) { + EXIT_IF(image.image == nullptr || image.type == VulkanImageType::DepthStencil); + + VkImageMemoryBarrier barrier {}; + barrier.sType = VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER; + barrier.srcAccessMask = VK_ACCESS_MEMORY_WRITE_BIT; + barrier.dstAccessMask = + (read ? VK_ACCESS_SHADER_READ_BIT : 0) | (written ? VK_ACCESS_SHADER_WRITE_BIT : 0); + barrier.oldLayout = image.layout; + barrier.newLayout = VK_IMAGE_LAYOUT_GENERAL; + barrier.srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; + barrier.dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; + barrier.image = image.image; + barrier.subresourceRange.aspectMask = VK_IMAGE_ASPECT_COLOR_BIT; + barrier.subresourceRange.baseMipLevel = 0; + barrier.subresourceRange.levelCount = VK_REMAINING_MIP_LEVELS; + barrier.subresourceRange.baseArrayLayer = 0; + barrier.subresourceRange.layerCount = image.layers; + EXIT_IF(barrier.dstAccessMask == 0 || barrier.subresourceRange.layerCount == 0); + return barrier; +} + +VkBufferMemoryBarrier MakeGdsDependency(const VulkanBuffer& buffer) { + EXIT_IF(buffer.buffer == nullptr); + + VkBufferMemoryBarrier barrier {}; + barrier.sType = VK_STRUCTURE_TYPE_BUFFER_MEMORY_BARRIER; + barrier.srcAccessMask = + VK_ACCESS_HOST_WRITE_BIT | VK_ACCESS_TRANSFER_WRITE_BIT | VK_ACCESS_SHADER_WRITE_BIT; + barrier.dstAccessMask = VK_ACCESS_SHADER_READ_BIT | VK_ACCESS_SHADER_WRITE_BIT; + barrier.srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; + barrier.dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; + barrier.buffer = buffer.buffer; + barrier.offset = 0; + barrier.size = VK_WHOLE_SIZE; + return barrier; +} + +std::vector +CollectShaderBufferWrites(const ShaderRecompiler::IR::Program& program, + const ShaderRecompiler::IR::ResourceSnapshot& resources) { + EXIT_IF(resources.buffers.size() != program.info.buffers.size()); + std::vector writes; + for (uint32_t i = 0; i < program.info.buffers.size(); i++) { + if (!program.info.buffers[i].written) { + continue; + } + const auto& value = resources.buffers[i]; + EXIT_IF(value.dword_count < 4); + ShaderBufferResource descriptor; + std::memcpy(descriptor.fields, value.dwords.data(), sizeof(descriptor.fields)); + const auto address = descriptor.Base48(); + const auto records = static_cast(descriptor.NumRecords()); + const auto stride = static_cast(descriptor.Stride()); + if (stride != 0 && records > UINT64_MAX / stride) { + EXIT("shader resource barrier buffer footprint overflow\n"); + } + const auto size = stride == 0 ? records : stride * records; + if (address != 0 && size != 0) { + writes.push_back({address, size}); + } + } + return writes; +} + +bool MarkShaderAddressWrites(const std::vector& writes) { + if (!writes.empty()) { + EXIT("shader address writes are unsupported\n"); + } + return false; +} + +bool HasShaderBufferWrites(const ShaderStageRuntime& runtime) { + EXIT_IF(!runtime); + return !CollectShaderBufferWrites(*runtime.program, *runtime.resources).empty(); +} + +void ShaderWriteBarrier(VkCommandBuffer vk_buffer, VkPipelineStageFlags source_stages) { + EXIT_IF(vk_buffer == nullptr || source_stages == 0); + const auto barrier = MakeShaderWriteDependency(); + vkCmdPipelineBarrier( + vk_buffer, source_stages, + VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT | VK_PIPELINE_STAGE_VERTEX_INPUT_BIT | + VK_PIPELINE_STAGE_VERTEX_SHADER_BIT | VK_PIPELINE_STAGE_FRAGMENT_SHADER_BIT | + VK_PIPELINE_STAGE_TRANSFER_BIT | VK_PIPELINE_STAGE_COLOR_ATTACHMENT_OUTPUT_BIT, + 0, 1, &barrier, 0, nullptr, 0, nullptr); +} + +} // namespace Libs::Graphics diff --git a/src/graphics/host_gpu/renderer/shaderResourceBarrier.h b/src/graphics/host_gpu/renderer/shaderResourceBarrier.h new file mode 100644 index 000000000..2ed32359d --- /dev/null +++ b/src/graphics/host_gpu/renderer/shaderResourceBarrier.h @@ -0,0 +1,34 @@ +#ifndef EMULATOR_SRC_GRAPHICS_HOST_GPU_RENDERER_SHADERRESOURCEBARRIER_H_ +#define EMULATOR_SRC_GRAPHICS_HOST_GPU_RENDERER_SHADERRESOURCEBARRIER_H_ + +#include "graphics/host_gpu/graphicContext.h" +#include "graphics/shader/recompiler/ResourceMaterialization.h" + +#include + +namespace Libs::Graphics { + +struct ShaderAddressWriteRange; +struct ShaderStageRuntime; + +struct ShaderBufferWriteRange { + uint64_t address = 0; + uint64_t size = 0; + + bool operator==(const ShaderBufferWriteRange& other) const = default; +}; + +VkPipelineStageFlags ShaderPipelineStages(VkShaderStageFlags stages); +VkMemoryBarrier MakeShaderWriteDependency(); +VkImageMemoryBarrier MakeStorageImageDependency(const VulkanImage& image, bool read, bool written); +VkBufferMemoryBarrier MakeGdsDependency(const VulkanBuffer& buffer); +std::vector +CollectShaderBufferWrites(const ShaderRecompiler::IR::Program& program, + const ShaderRecompiler::IR::ResourceSnapshot& resources); +bool MarkShaderAddressWrites(const std::vector& writes); +bool HasShaderBufferWrites(const ShaderStageRuntime& runtime); +void ShaderWriteBarrier(VkCommandBuffer vk_buffer, VkPipelineStageFlags source_stages); + +} // namespace Libs::Graphics + +#endif // EMULATOR_SRC_GRAPHICS_HOST_GPU_RENDERER_SHADERRESOURCEBARRIER_H_ diff --git a/src/graphics/host_gpu/renderer/shaderSubgroup.cpp b/src/graphics/host_gpu/renderer/shaderSubgroup.cpp new file mode 100644 index 000000000..96c736d31 --- /dev/null +++ b/src/graphics/host_gpu/renderer/shaderSubgroup.cpp @@ -0,0 +1,50 @@ +#include "graphics/host_gpu/renderer/shaderSubgroup.h" + +#include "graphics/shader/recompiler/SpirvEmitter.h" + +namespace Libs::Graphics { + +ShaderLaneMaskMode SelectGraphicsLaneMaskMode(const GraphicContext& context, + uint32_t guest_wave_size) { + return context.subgroup_size == 32u && guest_wave_size == 64u + ? ShaderLaneMaskMode::PerInvocation + : ShaderLaneMaskMode::NativeWave; +} + +ShaderSubgroupConfiguration ConfigureShaderSubgroup(const GraphicContext& context, + VkShaderStageFlagBits stage, + const ShaderRecompiler::IR::Program& program) { + const auto guest_wave_size = program.wave_size; + if (guest_wave_size != 32u && guest_wave_size != 64u) { + return {}; + } + if (stage != VK_SHADER_STAGE_COMPUTE_BIT) { + const auto expected = SelectGraphicsLaneMaskMode(context, guest_wave_size); + if (program.lane_mask_mode != expected || (context.subgroup_size != guest_wave_size && + expected != ShaderLaneMaskMode::PerInvocation)) { + return {}; + } + return {expected == ShaderLaneMaskMode::NativeWave + ? ShaderSubgroupMode::Natural + : ShaderSubgroupMode::PerInvocationGraphics, + 0}; + } + if (program.lane_mask_mode != ShaderLaneMaskMode::NativeWave) { + return {}; + } + if (context.subgroup_size == guest_wave_size) { + return {ShaderSubgroupMode::Natural, 0}; + } + if (context.subgroup_size_control_enabled && + (context.required_subgroup_size_stages & stage) != 0 && + guest_wave_size >= context.min_subgroup_size && + guest_wave_size <= context.max_subgroup_size) { + return {ShaderSubgroupMode::Controlled, guest_wave_size}; + } + if (!ShaderRecompiler::Spirv::ProgramRequiresExactSubgroupSize(program)) { + return {ShaderSubgroupMode::FlattenedMasks, 0}; + } + return {}; +} + +} // namespace Libs::Graphics diff --git a/src/graphics/host_gpu/renderer/shaderSubgroup.h b/src/graphics/host_gpu/renderer/shaderSubgroup.h new file mode 100644 index 000000000..499530a31 --- /dev/null +++ b/src/graphics/host_gpu/renderer/shaderSubgroup.h @@ -0,0 +1,31 @@ +#ifndef EMULATOR_SRC_GRAPHICS_HOST_GPU_RENDERER_SHADERSUBGROUP_H_ +#define EMULATOR_SRC_GRAPHICS_HOST_GPU_RENDERER_SHADERSUBGROUP_H_ + +#include "graphics/host_gpu/graphicContext.h" +#include "graphics/shader/recompiler/ShaderIR.h" + +namespace Libs::Graphics { + +enum class ShaderSubgroupMode { + Natural, + Controlled, + PerInvocationGraphics, + FlattenedMasks, + Unsupported +}; + +struct ShaderSubgroupConfiguration { + ShaderSubgroupMode mode = ShaderSubgroupMode::Unsupported; + uint32_t required_size = 0; +}; + +ShaderLaneMaskMode SelectGraphicsLaneMaskMode(const GraphicContext& context, + uint32_t guest_wave_size); + +ShaderSubgroupConfiguration ConfigureShaderSubgroup(const GraphicContext& context, + VkShaderStageFlagBits stage, + const ShaderRecompiler::IR::Program& program); + +} // namespace Libs::Graphics + +#endif // EMULATOR_SRC_GRAPHICS_HOST_GPU_RENDERER_SHADERSUBGROUP_H_ diff --git a/src/graphics/host_gpu/renderer/shaders.cpp b/src/graphics/host_gpu/renderer/shaders.cpp new file mode 100644 index 000000000..98990dc6e --- /dev/null +++ b/src/graphics/host_gpu/renderer/shaders.cpp @@ -0,0 +1,1054 @@ +#include "common/assert.h" +#include "common/common.h" +#include "common/emulatorConfig.h" +#include "common/logging/log.h" +#include "common/profiler.h" +#include "graphics/guest_gpu/gpu_defs.h" +#include "graphics/host_gpu/graphicContext.h" +#include "graphics/host_gpu/renderer/descriptorCache.h" +#include "graphics/host_gpu/renderer/pipelineCache.h" +#include "graphics/host_gpu/renderer/render.h" +#include "graphics/host_gpu/renderer/renderContext.h" +#include "graphics/host_gpu/renderer/renderState.h" +#include "graphics/host_gpu/renderer/shaderSubgroup.h" +#include "graphics/shader/recompiler/ShaderIR.h" +#include "graphics/shader/shader.h" + +#include +#include +#include +#include + +namespace Libs::Graphics { + +//IDK: maybe we can remove it? +constexpr uint32_t kTemporaryPs5BufferFormat121 = 121u; + +static bool NarrowInputFormat(VkFormat* format, uint32_t* size, uint32_t used_components) { + EXIT_IF(format == nullptr); + EXIT_IF(size == nullptr); + + if (used_components == 0 || used_components >= *size) { + return false; + } + + switch (*format) { + case VK_FORMAT_R32G32B32A32_SFLOAT: + switch (used_components) { + case 1: *format = VK_FORMAT_R32_SFLOAT; break; + case 2: *format = VK_FORMAT_R32G32_SFLOAT; break; + case 3: *format = VK_FORMAT_R32G32B32_SFLOAT; break; + default: return false; + } + *size = used_components; + return true; + case VK_FORMAT_R32G32B32_SFLOAT: + switch (used_components) { + case 1: *format = VK_FORMAT_R32_SFLOAT; break; + case 2: *format = VK_FORMAT_R32G32_SFLOAT; break; + default: return false; + } + *size = used_components; + return true; + case VK_FORMAT_R16G16B16A16_SFLOAT: + switch (used_components) { + case 1: *format = VK_FORMAT_R16_SFLOAT; break; + case 2: *format = VK_FORMAT_R16G16_SFLOAT; break; + default: return false; + } + *size = used_components; + return true; + case VK_FORMAT_R8G8B8A8_UNORM: + switch (used_components) { + case 1: *format = VK_FORMAT_R8_UNORM; break; + case 2: *format = VK_FORMAT_R8G8_UNORM; break; + default: return false; + } + *size = used_components; + return true; + case VK_FORMAT_R8G8B8A8_SNORM: + if (used_components != 2) { + return false; + } + *format = VK_FORMAT_R8G8_SNORM; + *size = 2; + return true; + case VK_FORMAT_R8G8B8A8_UINT: + switch (used_components) { + case 1: *format = VK_FORMAT_R8_UINT; break; + case 2: *format = VK_FORMAT_R8G8_UINT; break; + default: return false; + } + *size = used_components; + return true; + default: break; + } + + return false; +} + +static void GetInputFormat(const ShaderBufferResource& res, VkFormat* format, uint32_t* size, + uint32_t used_components) { + EXIT_IF(format == nullptr); + EXIT_IF(size == nullptr); + + auto fmt = res.Format(); + if (fmt == Prospero::GpuEnumValue(Prospero::VertexAttribFormat::k16_16SInt)) { + static bool logged_113 = false; + if (!logged_113) { + LOGF("InputFormat: temporary: accepting invalid PS5 buffer format 113 as " + "VK_FORMAT_R32G32B32A32_SFLOAT\n"); + logged_113 = true; + } + *format = VK_FORMAT_R32G32B32A32_SFLOAT; + *size = 4; + if (NarrowInputFormat(format, size, used_components)) { + LOGF("InputFormat: narrowing fmt=%u to %s for used_components=%u\n", fmt, + string_VkFormat(*format), used_components); + } + return; + } + if (fmt == kTemporaryPs5BufferFormat121) { + static bool logged_121 = false; + if (!logged_121) { + LOGF("InputFormat: accepting PS5 buffer format 121 as VK_FORMAT_R16G16_SFLOAT\n"); + logged_121 = true; + } + *format = VK_FORMAT_R16G16_SFLOAT; + *size = 2; + if (NarrowInputFormat(format, size, used_components)) { + LOGF("InputFormat: narrowing fmt=%u to %s for used_components=%u\n", fmt, + string_VkFormat(*format), used_components); + } + return; + } + + switch (static_cast(fmt)) { + case Prospero::BufferFormat::k32_32_32_32Float: + *format = VK_FORMAT_R32G32B32A32_SFLOAT; + *size = 4; + break; + case Prospero::BufferFormat::k32_32_32_32SInt: + *format = VK_FORMAT_R32G32B32A32_SINT; + *size = 4; + break; + case Prospero::BufferFormat::k32_32_32_32UInt: + *format = VK_FORMAT_R32G32B32A32_UINT; + *size = 4; + break; + case Prospero::BufferFormat::k32_32_32Float: + *format = VK_FORMAT_R32G32B32_SFLOAT; + *size = 3; + break; + case Prospero::BufferFormat::k32_32_32SInt: + *format = VK_FORMAT_R32G32B32_SINT; + *size = 3; + break; + case Prospero::BufferFormat::k32_32_32UInt: + *format = VK_FORMAT_R32G32B32_UINT; + *size = 3; + break; + case Prospero::BufferFormat::k16_16_16_16Float: + *format = VK_FORMAT_R16G16B16A16_SFLOAT; + *size = 4; + break; + case Prospero::BufferFormat::k16_16_16_16SInt: + *format = VK_FORMAT_R16G16B16A16_SINT; + *size = 4; + break; + case Prospero::BufferFormat::k16_16_16_16UInt: + *format = VK_FORMAT_R16G16B16A16_UINT; + *size = 4; + break; + case Prospero::BufferFormat::k16_16_16_16SScaled: + *format = VK_FORMAT_R16G16B16A16_SSCALED; + *size = 4; + break; + case Prospero::BufferFormat::k16_16_16_16UScaled: + *format = VK_FORMAT_R16G16B16A16_USCALED; + *size = 4; + break; + case Prospero::BufferFormat::k16_16_16_16SNorm: + *format = VK_FORMAT_R16G16B16A16_SNORM; + *size = 4; + break; + case Prospero::BufferFormat::k16_16_16_16UNorm: + *format = VK_FORMAT_R16G16B16A16_UNORM; + *size = 4; + break; + case Prospero::BufferFormat::k32_32Float: + *format = VK_FORMAT_R32G32_SFLOAT; + *size = 2; + break; + case Prospero::BufferFormat::k32_32SInt: + *format = VK_FORMAT_R32G32_SINT; + *size = 2; + break; + case Prospero::BufferFormat::k32_32UInt: + *format = VK_FORMAT_R32G32_UINT; + *size = 2; + break; + case Prospero::BufferFormat::k8_8_8_8UInt: + *format = VK_FORMAT_R8G8B8A8_UINT; + *size = 4; + break; + case Prospero::BufferFormat::k8_8_8_8SScaled: + *format = VK_FORMAT_R8G8B8A8_SSCALED; + *size = 4; + break; + case Prospero::BufferFormat::k8_8_8_8UScaled: + *format = VK_FORMAT_R8G8B8A8_USCALED; + *size = 4; + break; + case Prospero::BufferFormat::k8_8_8_8SNorm: + *format = VK_FORMAT_R8G8B8A8_SNORM; + *size = 4; + break; + case Prospero::BufferFormat::k8_8_8_8UNorm: + *format = VK_FORMAT_R8G8B8A8_UNORM; + *size = 4; + break; + case Prospero::BufferFormat::k10_10_10_2UNorm: + *format = VK_FORMAT_A2B10G10R10_UNORM_PACK32; + *size = 4; + break; + case Prospero::BufferFormat::k10_10_10_2SNorm: + *format = VK_FORMAT_A2B10G10R10_SNORM_PACK32; + *size = 4; + break; + case Prospero::BufferFormat::k16_16Float: + *format = VK_FORMAT_R16G16_SFLOAT; + *size = 2; + break; + case Prospero::BufferFormat::k16_16SInt: + *format = VK_FORMAT_R16G16_SINT; + *size = 2; + break; + case Prospero::BufferFormat::k16_16UInt: + *format = VK_FORMAT_R16G16_UINT; + *size = 2; + break; + case Prospero::BufferFormat::k16_16SScaled: + *format = VK_FORMAT_R16G16_SSCALED; + *size = 2; + break; + case Prospero::BufferFormat::k16_16UScaled: + *format = VK_FORMAT_R16G16_USCALED; + *size = 2; + break; + case Prospero::BufferFormat::k16_16SNorm: + *format = VK_FORMAT_R16G16_SNORM; + *size = 2; + break; + case Prospero::BufferFormat::k16_16UNorm: + *format = VK_FORMAT_R16G16_UNORM; + *size = 2; + break; + case Prospero::BufferFormat::k32Float: + *format = VK_FORMAT_R32_SFLOAT; + *size = 1; + break; + case Prospero::BufferFormat::k32SInt: + *format = VK_FORMAT_R32_SINT; + *size = 1; + break; + case Prospero::BufferFormat::k32UInt: + *format = VK_FORMAT_R32_UINT; + *size = 1; + break; + case Prospero::BufferFormat::k8_8SInt: + *format = VK_FORMAT_R8G8_SINT; + *size = 2; + break; + case Prospero::BufferFormat::k8_8UInt: + *format = VK_FORMAT_R8G8_UINT; + *size = 2; + break; + case Prospero::BufferFormat::k8_8SScaled: + *format = VK_FORMAT_R8G8_SSCALED; + *size = 2; + break; + case Prospero::BufferFormat::k8_8UScaled: + *format = VK_FORMAT_R8G8_USCALED; + *size = 2; + break; + case Prospero::BufferFormat::k8_8SNorm: + *format = VK_FORMAT_R8G8_SNORM; + *size = 2; + break; + case Prospero::BufferFormat::k8_8UNorm: + *format = VK_FORMAT_R8G8_UNORM; + *size = 2; + break; + case Prospero::BufferFormat::k16Float: + *format = VK_FORMAT_R16_SFLOAT; + *size = 1; + break; + case Prospero::BufferFormat::k16SInt: + *format = VK_FORMAT_R16_SINT; + *size = 1; + break; + case Prospero::BufferFormat::k16UInt: + *format = VK_FORMAT_R16_UINT; + *size = 1; + break; + case Prospero::BufferFormat::k16SScaled: + *format = VK_FORMAT_R16_SSCALED; + *size = 1; + break; + case Prospero::BufferFormat::k16UScaled: + *format = VK_FORMAT_R16_USCALED; + *size = 1; + break; + case Prospero::BufferFormat::k16SNorm: + *format = VK_FORMAT_R16_SNORM; + *size = 1; + break; + case Prospero::BufferFormat::k16UNorm: + *format = VK_FORMAT_R16_UNORM; + *size = 1; + break; + case Prospero::BufferFormat::k8SInt: + *format = VK_FORMAT_R8_SINT; + *size = 1; + break; + case Prospero::BufferFormat::k8UInt: + *format = VK_FORMAT_R8_UINT; + *size = 1; + break; + case Prospero::BufferFormat::k8SScaled: + *format = VK_FORMAT_R8_SSCALED; + *size = 1; + break; + case Prospero::BufferFormat::k8UScaled: + *format = VK_FORMAT_R8_USCALED; + *size = 1; + break; + case Prospero::BufferFormat::k8SNorm: + *format = VK_FORMAT_R8_SNORM; + *size = 1; + break; + case Prospero::BufferFormat::k8UNorm: + *format = VK_FORMAT_R8_UNORM; + *size = 1; + break; + default: + EXIT("unknown format: fmt = %u\n", fmt); + *format = VK_FORMAT_UNDEFINED; + *size = 4; + break; + } + + if (NarrowInputFormat(format, size, used_components)) { + static std::atomic log_count = 0; + auto log_id = log_count.fetch_add(1); + if (log_id < 32) { + LOGF("VertexInput: narrowed vertex format to %" PRIu32 + " component(s) for shader fetch\n", + used_components); + } + } +} + +static VkBlendFactor GetBlendFactor(uint32_t factor) { + switch (static_cast(factor)) { + case Prospero::BlendFactor::kZero: return VK_BLEND_FACTOR_ZERO; + case Prospero::BlendFactor::kOne: return VK_BLEND_FACTOR_ONE; + case Prospero::BlendFactor::kSrcColor: return VK_BLEND_FACTOR_SRC_COLOR; + case Prospero::BlendFactor::kOneMinusSrcColor: return VK_BLEND_FACTOR_ONE_MINUS_SRC_COLOR; + case Prospero::BlendFactor::kSrcAlpha: return VK_BLEND_FACTOR_SRC_ALPHA; + case Prospero::BlendFactor::kOneMinusSrcAlpha: return VK_BLEND_FACTOR_ONE_MINUS_SRC_ALPHA; + case Prospero::BlendFactor::kDstAlpha: return VK_BLEND_FACTOR_DST_ALPHA; + case Prospero::BlendFactor::kOneMinusDstAlpha: return VK_BLEND_FACTOR_ONE_MINUS_DST_ALPHA; + case Prospero::BlendFactor::kDstColor: return VK_BLEND_FACTOR_DST_COLOR; + case Prospero::BlendFactor::kOneMinusDstColor: return VK_BLEND_FACTOR_ONE_MINUS_DST_COLOR; + case Prospero::BlendFactor::kSrcAlphaSaturate: return VK_BLEND_FACTOR_SRC_ALPHA_SATURATE; + case Prospero::BlendFactor::kConstantColor: return VK_BLEND_FACTOR_CONSTANT_COLOR; + case Prospero::BlendFactor::kOneMinusConstantColor: + return VK_BLEND_FACTOR_ONE_MINUS_CONSTANT_COLOR; + case Prospero::BlendFactor::kSrc1Color: return VK_BLEND_FACTOR_SRC1_COLOR; + case Prospero::BlendFactor::kOneMinusSrc1Color: return VK_BLEND_FACTOR_ONE_MINUS_SRC1_COLOR; + case Prospero::BlendFactor::kSrc1Alpha: return VK_BLEND_FACTOR_SRC1_ALPHA; + case Prospero::BlendFactor::kOneMinusSrc1Alpha: return VK_BLEND_FACTOR_ONE_MINUS_SRC1_ALPHA; + case Prospero::BlendFactor::kConstantAlpha: return VK_BLEND_FACTOR_CONSTANT_ALPHA; + case Prospero::BlendFactor::kOneMinusConstantAlpha: + return VK_BLEND_FACTOR_ONE_MINUS_CONSTANT_ALPHA; + default: EXIT("unknown factor: %u\n", factor); + } + return VK_BLEND_FACTOR_ZERO; +} + +static VkBlendOp GetBlendOp(uint32_t op) { + switch (static_cast(op)) { + case Prospero::BlendOp::kAdd: return VK_BLEND_OP_ADD; + case Prospero::BlendOp::kSubtract: return VK_BLEND_OP_SUBTRACT; + case Prospero::BlendOp::kMin: return VK_BLEND_OP_MIN; + case Prospero::BlendOp::kMax: return VK_BLEND_OP_MAX; + case Prospero::BlendOp::kReverseSubtract: return VK_BLEND_OP_REVERSE_SUBTRACT; + default: EXIT("unknown op: %u\n", op); + } + return VK_BLEND_OP_ADD; +} + +static void CreateLayout(VkDescriptorSetLayout* set_layouts, uint32_t* set_layouts_num, + VkPushConstantRange* push_constant_info, uint32_t* push_constant_info_num, + const ShaderRecompiler::IR::Program& program, VkShaderStageFlags vk_stage, + DescriptorCache::Stage stage) { + EXIT_IF(set_layouts == nullptr); + EXIT_IF(set_layouts_num == nullptr); + EXIT_IF(push_constant_info == nullptr); + EXIT_IF(push_constant_info_num == nullptr); + + const auto& bindings = program.bindings; + const bool need_descriptor = !bindings.descriptors.empty(); + + if (bindings.push_constant_size != 0) { + auto index = *push_constant_info_num; + + push_constant_info[index].stageFlags = vk_stage; + push_constant_info[index].offset = bindings.push_constant_offset; + push_constant_info[index].size = bindings.push_constant_size; + (*push_constant_info_num)++; + } + + if (need_descriptor) { + EXIT_IF(bindings.descriptor_set != *set_layouts_num); + + set_layouts[*set_layouts_num] = + g_render_ctx->GetDescriptorCache()->GetDescriptorSetLayout(stage, program); + (*set_layouts_num)++; + } +} + +static void ConfigureSubgroupSize(const GraphicContext* gctx, VkShaderStageFlagBits vk_stage, + const ShaderRecompiler::IR::Program& program, + VkPipelineShaderStageRequiredSubgroupSizeCreateInfo* required, + VkPipelineShaderStageCreateInfo* stage) { + EXIT_IF(gctx == nullptr || required == nullptr || stage == nullptr); + const auto config = ConfigureShaderSubgroup(*gctx, vk_stage, program); + switch (config.mode) { + case ShaderSubgroupMode::Natural: return; + case ShaderSubgroupMode::PerInvocationGraphics: { + static std::atomic log_count {0}; + if (log_count.fetch_add(1, std::memory_order_relaxed) < 16) { + LOGF("Vulkan running %u-lane graphics shader stage 0x%08x with per-invocation " + "EXEC/VCC on the host-native subgroup\n", + program.wave_size, static_cast(vk_stage)); + } + return; + } + case ShaderSubgroupMode::FlattenedMasks: { + static std::atomic log_count {0}; + if (log_count.fetch_add(1, std::memory_order_relaxed) < 16) { + LOGF("Vulkan running %u-lane shader stage 0x%08x with flattened per-invocation " + "EXEC/VCC\n", + program.wave_size, static_cast(vk_stage)); + } + return; + } + case ShaderSubgroupMode::Controlled: break; + case ShaderSubgroupMode::Unsupported: + default: + EXIT("Vulkan cannot run %u-lane shader stage 0x%08x: default=%u min=%u max=%u " + "controlled_stages=0x%08x\n", + program.wave_size, static_cast(vk_stage), gctx->subgroup_size, + gctx->min_subgroup_size, gctx->max_subgroup_size, + gctx->required_subgroup_size_stages); + } + + required->sType = VK_STRUCTURE_TYPE_PIPELINE_SHADER_STAGE_REQUIRED_SUBGROUP_SIZE_CREATE_INFO; + required->requiredSubgroupSize = config.required_size; + stage->pNext = required; +} + +// NOLINTNEXTLINE(readability-function-cognitive-complexity) +void CreatePipelineInternal(PipelineCache::GraphicsPipeline* pipeline, VkRenderPass render_pass, + const ShaderVertexInputInfo* vs_input_info, + std::span vs_shader, + const ShaderPixelInputInfo* ps_input_info, + std::span ps_shader, + const PipelineStaticParameters& static_params, uint32_t vs_hash0, + uint32_t vs_crc32, uint32_t ps_hash0, uint32_t ps_crc32, + bool ps_active) { + EXIT_IF(g_render_ctx == nullptr); + EXIT_IF(pipeline == nullptr); + EXIT_IF(render_pass == nullptr); + EXIT_IF(ps_active && ps_input_info == nullptr); + + auto* gctx = g_render_ctx->GetGraphicCtx(); + + EXIT_IF(gctx == nullptr); + + VkShaderModule vert_shader_module = nullptr; + VkShaderModule frag_shader_module = nullptr; + + VkShaderModuleCreateInfo create_info {}; + + create_info.sType = VK_STRUCTURE_TYPE_SHADER_MODULE_CREATE_INFO; + create_info.pNext = nullptr; + create_info.flags = 0; + + create_info.codeSize = vs_shader.size() * 4; + create_info.pCode = vs_shader.data(); + auto result = vkCreateShaderModule(gctx->device, &create_info, nullptr, &vert_shader_module); + if (graphics_debug_dump_enabled()) { + LOGF("PipelineTrace: vkCreateShaderModule VS done result=%s module=%p\n", + string_VkResult(result), static_cast(vert_shader_module)); + } + EXIT_NOT_IMPLEMENTED(result != VK_SUCCESS); + + if (ps_active) { + create_info.codeSize = ps_shader.size() * 4; + create_info.pCode = ps_shader.data(); + result = vkCreateShaderModule(gctx->device, &create_info, nullptr, &frag_shader_module); + if (graphics_debug_dump_enabled()) { + LOGF("PipelineTrace: vkCreateShaderModule PS done result=%s module=%p\n", + string_VkResult(result), static_cast(frag_shader_module)); + } + EXIT_NOT_IMPLEMENTED(result != VK_SUCCESS); + } + + EXIT_NOT_IMPLEMENTED(vert_shader_module == nullptr); + EXIT_NOT_IMPLEMENTED(ps_active && frag_shader_module == nullptr); + + VkPipelineShaderStageCreateInfo vert_shader_stage_info {}; + VkPipelineShaderStageRequiredSubgroupSizeCreateInfo vert_subgroup_size {}; + + vert_shader_stage_info.sType = VK_STRUCTURE_TYPE_PIPELINE_SHADER_STAGE_CREATE_INFO; + vert_shader_stage_info.pNext = nullptr; + vert_shader_stage_info.flags = 0; + vert_shader_stage_info.stage = VK_SHADER_STAGE_VERTEX_BIT; + vert_shader_stage_info.module = vert_shader_module; + vert_shader_stage_info.pName = "main"; + vert_shader_stage_info.pSpecializationInfo = nullptr; + EXIT_IF(!vs_input_info->stage); + ConfigureSubgroupSize(gctx, VK_SHADER_STAGE_VERTEX_BIT, *vs_input_info->stage.program, + &vert_subgroup_size, &vert_shader_stage_info); + + VkPipelineShaderStageCreateInfo frag_shader_stage_info {}; + VkPipelineShaderStageRequiredSubgroupSizeCreateInfo frag_subgroup_size {}; + frag_shader_stage_info.sType = VK_STRUCTURE_TYPE_PIPELINE_SHADER_STAGE_CREATE_INFO; + frag_shader_stage_info.pNext = nullptr; + frag_shader_stage_info.flags = 0; + frag_shader_stage_info.stage = VK_SHADER_STAGE_FRAGMENT_BIT; + frag_shader_stage_info.module = frag_shader_module; + frag_shader_stage_info.pName = "main"; + frag_shader_stage_info.pSpecializationInfo = nullptr; + if (ps_active) { + EXIT_IF(!ps_input_info->stage); + ConfigureSubgroupSize(gctx, VK_SHADER_STAGE_FRAGMENT_BIT, *ps_input_info->stage.program, + &frag_subgroup_size, &frag_shader_stage_info); + } + + VkPipelineShaderStageCreateInfo shader_stages[] = {vert_shader_stage_info, + frag_shader_stage_info}; + const uint32_t shader_stage_count = ps_active ? 2u : 1u; + + VkVertexInputAttributeDescription input_attr[ShaderVertexInputInfo::RES_MAX]; + VkVertexInputBindingDescription input_desc[ShaderVertexInputInfo::RES_MAX]; + + for (int bi = 0; bi < vs_input_info->buffers_num; bi++) { + const auto& b = vs_input_info->buffers[bi]; + input_desc[bi].binding = bi; + input_desc[bi].stride = b.stride; + input_desc[bi].inputRate = + (b.fetch_index == 0 ? VK_VERTEX_INPUT_RATE_VERTEX : VK_VERTEX_INPUT_RATE_INSTANCE); + + for (int ai = 0; ai < b.attr_num; ai++) { + auto index = b.attr_indices[ai]; + input_attr[index].binding = bi; + input_attr[index].location = index; + input_attr[index].offset = b.attr_offsets[ai]; + + uint32_t attr_size = 4; + auto registers_num = vs_input_info->resources_dst[index].registers_num; + auto used_components = (vs_input_info->resource_fetch_components[index] > 0 + ? vs_input_info->resource_fetch_components[index] + : registers_num); + GetInputFormat(vs_input_info->resources[index], &input_attr[index].format, &attr_size, + static_cast(used_components)); + + if (graphics_debug_dump_enabled()) { + static std::atomic_uint log_count = 0; + const auto log_id = log_count.fetch_add(1, std::memory_order_relaxed); + if (log_id < 128) { + LOGF("VertexInputState[%u]: attr=%u binding=%u offset=%u stride=%u fmt=%d " + "src_fmt=%u dst=v%u regs=%u" + " fetched_components=%u attr_size=%u swizzle=%u,%u,%u,%u\n", + log_id, index, input_attr[index].binding, input_attr[index].offset, + input_desc[bi].stride, static_cast(input_attr[index].format), + static_cast(vs_input_info->resources[index].Format()), + static_cast(vs_input_info->resources_dst[index].register_start), + static_cast(registers_num), + static_cast(used_components), attr_size, + static_cast(vs_input_info->resources[index].DstSelX()), + static_cast(vs_input_info->resources[index].DstSelY()), + static_cast(vs_input_info->resources[index].DstSelZ()), + static_cast(vs_input_info->resources[index].DstSelW())); + } + } + + if (vs_input_info->resources[index].OutOfBounds() != 0) { + static bool logged = false; + if (!logged) { + LOGF("VertexInput: temporary: accepting PS5 out-of-bounds behavior %" PRIu8 + "\n", + vs_input_info->resources[index].OutOfBounds()); + logged = true; + } + } + + EXIT_NOT_IMPLEMENTED(vs_input_info->resources[index].AddTid()); + EXIT_NOT_IMPLEMENTED(vs_input_info->resources[index].SwizzleEnabled()); + + auto log_unsupported_vertex_swizzle = [index, attr_size, registers_num]( + uint32_t swizzle, uint32_t expected) { + static bool logged = false; + if (!logged) { + LOGF( + "VertexInput: temporary: accepting unsupported dst swizzle at attr %" PRIu32 + " (attr_size=%" PRIu32 ", regs=%" PRIu32 ", swizzle=0x%03" PRIx32 + ", expected=0x%03" PRIx32 ")\n", + static_cast(index), attr_size, registers_num, swizzle, expected); + logged = true; + } + }; + + switch (registers_num) { + case 1: { + auto swizzle = vs_input_info->resources[index].DstSelX(); + if (swizzle != DstSel(4)) { + log_unsupported_vertex_swizzle(swizzle, DstSel(4)); + } + break; + } + case 2: { + auto swizzle = vs_input_info->resources[index].DstSelXY(); + auto expected = (attr_size == 1 ? DstSel(4, 0) : DstSel(4, 5)); + if (swizzle != expected) { + log_unsupported_vertex_swizzle(swizzle, expected); + } + break; + } + case 3: { + auto swizzle = vs_input_info->resources[index].DstSelXYZ(); + auto expected = + (attr_size == 1 ? DstSel(4, 0, 0) + : (attr_size == 2 ? DstSel(4, 5, 0) : DstSel(4, 5, 6))); + if (swizzle != expected) { + log_unsupported_vertex_swizzle(swizzle, expected); + } + break; + } + case 4: { + auto swizzle = vs_input_info->resources[index].DstSelXYZW(); + auto expected = DstSel(4, 5, 6, 7); + bool supported = false; + switch (attr_size) { + case 1: + expected = DstSel(4, 0, 0, 1); + supported = (swizzle == expected); + break; + case 2: + expected = DstSel(4, 5, 0, 1); + supported = (swizzle == expected); + break; + case 3: + expected = DstSel(4, 5, 6, 1); + supported = (swizzle == expected || swizzle == DstSel(4, 5, 6, 0)); + break; + default: + supported = (swizzle == expected || swizzle == DstSel(4, 5, 6, 1) || + swizzle == DstSel(4, 5, 6, 0)); + break; + } + if (!supported) { + log_unsupported_vertex_swizzle(swizzle, expected); + } + break; + } + default: EXIT("invalid registers_num"); + } + } + } + + VkPipelineVertexInputStateCreateInfo vertex_input_info {}; + vertex_input_info.sType = VK_STRUCTURE_TYPE_PIPELINE_VERTEX_INPUT_STATE_CREATE_INFO; + vertex_input_info.pNext = nullptr; + vertex_input_info.flags = 0; + vertex_input_info.vertexBindingDescriptionCount = vs_input_info->buffers_num; + vertex_input_info.pVertexBindingDescriptions = input_desc; + vertex_input_info.vertexAttributeDescriptionCount = vs_input_info->resources_num; + vertex_input_info.pVertexAttributeDescriptions = input_attr; + + VkPipelineInputAssemblyStateCreateInfo input_assembly {}; + input_assembly.sType = VK_STRUCTURE_TYPE_PIPELINE_INPUT_ASSEMBLY_STATE_CREATE_INFO; + input_assembly.pNext = nullptr; + input_assembly.flags = 0; + input_assembly.topology = static_params.topology; + input_assembly.primitiveRestartEnable = VK_FALSE; + + VkViewport viewport {}; + viewport.x = static_params.viewport_offset[0] - static_params.viewport_scale[0]; + viewport.y = static_params.viewport_offset[1] - static_params.viewport_scale[1]; + viewport.width = static_params.viewport_scale[0] * 2.0f; + viewport.height = static_params.viewport_scale[1] * 2.0f; + viewport.minDepth = static_params.viewport_offset[2]; + viewport.maxDepth = static_params.viewport_scale[2] + static_params.viewport_offset[2]; + + VkRect2D scissor {}; + scissor.offset = {static_params.scissor_ltrb[0], static_params.scissor_ltrb[1]}; + scissor.extent = { + static_cast(static_params.scissor_ltrb[2] - static_params.scissor_ltrb[0]), + static_cast(static_params.scissor_ltrb[3] - static_params.scissor_ltrb[1])}; + + VkPipelineViewportDepthClipControlCreateInfoEXT depth_clip_control {}; + depth_clip_control.sType = + VK_STRUCTURE_TYPE_PIPELINE_VIEWPORT_DEPTH_CLIP_CONTROL_CREATE_INFO_EXT; + depth_clip_control.pNext = nullptr; + depth_clip_control.negativeOneToOne = (static_params.negative_one_to_one ? VK_TRUE : VK_FALSE); + + VkPipelineViewportStateCreateInfo viewport_state {}; + viewport_state.sType = VK_STRUCTURE_TYPE_PIPELINE_VIEWPORT_STATE_CREATE_INFO; + viewport_state.pNext = &depth_clip_control; + viewport_state.flags = 0; + viewport_state.viewportCount = 1; + viewport_state.pViewports = &viewport; + viewport_state.scissorCount = 1; + viewport_state.pScissors = &scissor; + + VkCullModeFlags cull_mode = VK_CULL_MODE_NONE; + cull_mode |= (static_params.cull_back ? VK_CULL_MODE_BACK_BIT : 0u); + cull_mode |= (static_params.cull_front ? VK_CULL_MODE_FRONT_BIT : 0u); + + VkFrontFace front_face = + (static_params.face ? VK_FRONT_FACE_CLOCKWISE : VK_FRONT_FACE_COUNTER_CLOCKWISE); + + VkPipelineRasterizationDepthClipStateCreateInfoEXT clip_ext {}; + clip_ext.sType = VK_STRUCTURE_TYPE_PIPELINE_RASTERIZATION_DEPTH_CLIP_STATE_CREATE_INFO_EXT; + clip_ext.pNext = nullptr; + clip_ext.flags = 0; + clip_ext.depthClipEnable = VK_FALSE; + + VkPipelineRasterizationStateCreateInfo rasterizer {}; + rasterizer.sType = VK_STRUCTURE_TYPE_PIPELINE_RASTERIZATION_STATE_CREATE_INFO; + rasterizer.pNext = &clip_ext; + rasterizer.flags = 0; + rasterizer.depthClampEnable = VK_FALSE; + rasterizer.rasterizerDiscardEnable = VK_FALSE; + rasterizer.polygonMode = VK_POLYGON_MODE_FILL; + rasterizer.cullMode = cull_mode; + rasterizer.frontFace = front_face; + rasterizer.depthBiasEnable = VK_FALSE; + rasterizer.depthBiasConstantFactor = 0.0f; + rasterizer.depthBiasClamp = 0.0f; + rasterizer.depthBiasSlopeFactor = 0.0f; + rasterizer.lineWidth = 1.0f; + + VkPipelineMultisampleStateCreateInfo multisampling {}; + multisampling.sType = VK_STRUCTURE_TYPE_PIPELINE_MULTISAMPLE_STATE_CREATE_INFO; + multisampling.pNext = nullptr; + multisampling.flags = 0; + multisampling.sampleShadingEnable = VK_FALSE; + multisampling.rasterizationSamples = VK_SAMPLE_COUNT_1_BIT; + multisampling.minSampleShading = 1.0f; + multisampling.pSampleMask = nullptr; + multisampling.alphaToCoverageEnable = VK_FALSE; + multisampling.alphaToOneEnable = VK_FALSE; + + VkPipelineColorBlendAttachmentState color_blend_attachment[RENDER_COLOR_ATTACHMENTS_MAX] = {}; + for (uint32_t i = 0; i < static_params.color_count; i++) { + VkColorComponentFlags color_write_mask = 0; + EXIT_NOT_IMPLEMENTED((static_params.color_mask[i] & ~0x0fu) != 0); + if ((static_params.color_mask[i] & 0x1u) != 0) { + color_write_mask |= static_cast(VK_COLOR_COMPONENT_R_BIT); + } + if ((static_params.color_mask[i] & 0x2u) != 0) { + color_write_mask |= static_cast(VK_COLOR_COMPONENT_G_BIT); + } + if ((static_params.color_mask[i] & 0x4u) != 0) { + color_write_mask |= static_cast(VK_COLOR_COMPONENT_B_BIT); + } + if ((static_params.color_mask[i] & 0x8u) != 0) { + color_write_mask |= static_cast(VK_COLOR_COMPONENT_A_BIT); + } + + color_blend_attachment[i].colorWriteMask = color_write_mask; + color_blend_attachment[i].blendEnable = + (static_params.blend_enable[i] && !static_params.blend_bypass[i]) ? VK_TRUE : VK_FALSE; + color_blend_attachment[i].srcColorBlendFactor = + GetBlendFactor(static_params.color_srcblend[i]); + color_blend_attachment[i].dstColorBlendFactor = + GetBlendFactor(static_params.color_destblend[i]); + color_blend_attachment[i].colorBlendOp = GetBlendOp(static_params.color_comb_fcn[i]); + color_blend_attachment[i].srcAlphaBlendFactor = + (static_params.separate_alpha_blend[i] ? GetBlendFactor(static_params.alpha_srcblend[i]) + : color_blend_attachment[i].srcColorBlendFactor); + color_blend_attachment[i].dstAlphaBlendFactor = + (static_params.separate_alpha_blend[i] + ? GetBlendFactor(static_params.alpha_destblend[i]) + : color_blend_attachment[i].dstColorBlendFactor); + color_blend_attachment[i].alphaBlendOp = + (static_params.separate_alpha_blend[i] ? GetBlendOp(static_params.alpha_comb_fcn[i]) + : color_blend_attachment[i].colorBlendOp); + } + + VkBool32 color_write_enable[RENDER_COLOR_ATTACHMENTS_MAX] = {}; + for (uint32_t i = 0; i < static_params.color_count; i++) { + color_write_enable[i] = VK_TRUE; + } + + VkPipelineColorWriteCreateInfoEXT color_write {}; + color_write.sType = VK_STRUCTURE_TYPE_PIPELINE_COLOR_WRITE_CREATE_INFO_EXT; + color_write.pNext = nullptr; + color_write.attachmentCount = static_params.color_count; + color_write.pColorWriteEnables = color_write_enable; + + VkPipelineColorBlendStateCreateInfo color_blending {}; + color_blending.sType = VK_STRUCTURE_TYPE_PIPELINE_COLOR_BLEND_STATE_CREATE_INFO; + color_blending.pNext = &color_write; + color_blending.flags = 0; + color_blending.logicOpEnable = VK_FALSE; + color_blending.logicOp = VK_LOGIC_OP_COPY; + color_blending.attachmentCount = static_params.color_count; + color_blending.pAttachments = color_blend_attachment; + color_blending.blendConstants[0] = static_params.blend_color_red; + color_blending.blendConstants[1] = static_params.blend_color_green; + color_blending.blendConstants[2] = static_params.blend_color_blue; + color_blending.blendConstants[3] = static_params.blend_color_alpha; + + VkDescriptorSetLayout set_layouts[2] = {}; + uint32_t set_layouts_num = 0; + + VkPushConstantRange push_constant_info[2]; + uint32_t push_constant_info_num = 0; + + EXIT_IF(!vs_input_info->stage); + CreateLayout(set_layouts, &set_layouts_num, push_constant_info, &push_constant_info_num, + *vs_input_info->stage.program, VK_SHADER_STAGE_VERTEX_BIT, + DescriptorCache::Stage::Vertex); + if (ps_active) { + EXIT_IF(!ps_input_info->stage); + CreateLayout(set_layouts, &set_layouts_num, push_constant_info, &push_constant_info_num, + *ps_input_info->stage.program, VK_SHADER_STAGE_FRAGMENT_BIT, + DescriptorCache::Stage::Pixel); + } + + VkPipelineLayoutCreateInfo pipeline_layout_info {}; + pipeline_layout_info.sType = VK_STRUCTURE_TYPE_PIPELINE_LAYOUT_CREATE_INFO; + pipeline_layout_info.pNext = nullptr; + pipeline_layout_info.flags = 0; + pipeline_layout_info.setLayoutCount = set_layouts_num; + pipeline_layout_info.pSetLayouts = (set_layouts_num > 0 ? set_layouts : nullptr); + pipeline_layout_info.pushConstantRangeCount = push_constant_info_num; + pipeline_layout_info.pPushConstantRanges = + push_constant_info_num > 0 ? push_constant_info : nullptr; + + EXIT_IF(pipeline->pipeline_layout != nullptr); + + if (graphics_debug_dump_enabled()) { + LOGF("PipelineTrace: vkCreatePipelineLayout begin VS=0x%08" PRIx32 "/0x%08" PRIx32 + " PS=0x%08" PRIx32 "/0x%08" PRIx32 " set_layouts=%" PRIu32 " push_constants=%" PRIu32 + "\n", + vs_hash0, vs_crc32, ps_hash0, ps_crc32, set_layouts_num, push_constant_info_num); + } + result = vkCreatePipelineLayout(gctx->device, &pipeline_layout_info, nullptr, + &pipeline->pipeline_layout); + if (graphics_debug_dump_enabled()) { + LOGF("PipelineTrace: vkCreatePipelineLayout done result=%s layout=%p\n", + string_VkResult(result), static_cast(pipeline->pipeline_layout)); + } + EXIT_NOT_IMPLEMENTED(result != VK_SUCCESS); + + EXIT_NOT_IMPLEMENTED(pipeline->pipeline_layout == nullptr); + + VkPipelineDepthStencilStateCreateInfo depth_stencil_info {}; + depth_stencil_info.sType = VK_STRUCTURE_TYPE_PIPELINE_DEPTH_STENCIL_STATE_CREATE_INFO; + depth_stencil_info.pNext = nullptr; + depth_stencil_info.flags = 0; + depth_stencil_info.depthTestEnable = (static_params.depth_test_enable ? VK_TRUE : VK_FALSE); + depth_stencil_info.depthWriteEnable = (static_params.depth_write_enable ? VK_TRUE : VK_FALSE); + depth_stencil_info.depthCompareOp = static_params.depth_compare_op; + depth_stencil_info.depthBoundsTestEnable = + (static_params.depth_bounds_test_enable ? VK_TRUE : VK_FALSE); + depth_stencil_info.stencilTestEnable = (static_params.stencil_test_enable ? VK_TRUE : VK_FALSE); + depth_stencil_info.front.failOp = static_params.stencil_front.failOp; + depth_stencil_info.front.passOp = static_params.stencil_front.passOp; + depth_stencil_info.front.depthFailOp = static_params.stencil_front.depthFailOp; + depth_stencil_info.front.compareOp = static_params.stencil_front.compareOp; + depth_stencil_info.back.failOp = static_params.stencil_back.failOp; + depth_stencil_info.back.passOp = static_params.stencil_back.passOp; + depth_stencil_info.back.depthFailOp = static_params.stencil_back.depthFailOp; + depth_stencil_info.back.compareOp = static_params.stencil_back.compareOp; + depth_stencil_info.minDepthBounds = static_params.depth_min_bounds; + depth_stencil_info.maxDepthBounds = static_params.depth_max_bounds; + + const VkDynamicState dynamic_states[] = { + VK_DYNAMIC_STATE_VIEWPORT, + VK_DYNAMIC_STATE_SCISSOR, + VK_DYNAMIC_STATE_LINE_WIDTH, + VK_DYNAMIC_STATE_STENCIL_COMPARE_MASK, + VK_DYNAMIC_STATE_STENCIL_REFERENCE, + VK_DYNAMIC_STATE_STENCIL_WRITE_MASK, + VK_DYNAMIC_STATE_COLOR_WRITE_ENABLE_EXT, + }; + const auto dynamic_states_count = + static_cast(sizeof(dynamic_states) / sizeof(dynamic_states[0])); + + VkPipelineDynamicStateCreateInfo dynamic_state {}; + dynamic_state.sType = VK_STRUCTURE_TYPE_PIPELINE_DYNAMIC_STATE_CREATE_INFO; + dynamic_state.pNext = nullptr; + dynamic_state.flags = 0; + dynamic_state.dynamicStateCount = dynamic_states_count; + dynamic_state.pDynamicStates = dynamic_states; + + VkGraphicsPipelineCreateInfo pipeline_info {}; + pipeline_info.sType = VK_STRUCTURE_TYPE_GRAPHICS_PIPELINE_CREATE_INFO; + pipeline_info.pNext = nullptr; + pipeline_info.flags = 0; + pipeline_info.stageCount = shader_stage_count; + pipeline_info.pStages = shader_stages; + pipeline_info.pVertexInputState = &vertex_input_info; + pipeline_info.pInputAssemblyState = &input_assembly; + pipeline_info.pTessellationState = nullptr; + pipeline_info.pViewportState = &viewport_state; + pipeline_info.pRasterizationState = &rasterizer; + pipeline_info.pMultisampleState = &multisampling; + pipeline_info.pDepthStencilState = (static_params.with_depth ? &depth_stencil_info : nullptr); + pipeline_info.pColorBlendState = &color_blending; + pipeline_info.pDynamicState = &dynamic_state; + pipeline_info.layout = pipeline->pipeline_layout; + pipeline_info.renderPass = render_pass; + pipeline_info.subpass = 0; + pipeline_info.basePipelineHandle = nullptr; + pipeline_info.basePipelineIndex = -1; + + EXIT_IF(pipeline->pipeline != nullptr); + + if (graphics_debug_dump_enabled()) { + LOGF("PipelineTrace: vkCreateGraphicsPipelines begin VS=0x%08" PRIx32 "/0x%08" PRIx32 + " PS=0x%08" PRIx32 "/0x%08" PRIx32 " topology=%" PRIu32 " color_mask=0x%08" PRIx32 + " depth=%s blend=%s dyn_states=%" PRIu32 " viewport=%f,%f %fx%f" + " scissor=%d,%d %ux%u\n", + vs_hash0, vs_crc32, ps_hash0, ps_crc32, static_cast(static_params.topology), + static_params.color_mask[0], (static_params.with_depth ? "true" : "false"), + (static_params.blend_enable[0] ? "true" : "false"), dynamic_states_count, viewport.x, + viewport.y, viewport.width, viewport.height, scissor.offset.x, scissor.offset.y, + scissor.extent.width, scissor.extent.height); + } + result = vkCreateGraphicsPipelines(gctx->device, nullptr, 1, &pipeline_info, nullptr, + &pipeline->pipeline); + if (graphics_debug_dump_enabled()) { + LOGF("PipelineTrace: vkCreateGraphicsPipelines done result=%s pipeline=%p\n", + string_VkResult(result), static_cast(pipeline->pipeline)); + } + EXIT_NOT_IMPLEMENTED(result != VK_SUCCESS); + + EXIT_NOT_IMPLEMENTED(pipeline->pipeline == nullptr); + + if (frag_shader_module != nullptr) { + vkDestroyShaderModule(gctx->device, frag_shader_module, nullptr); + } + vkDestroyShaderModule(gctx->device, vert_shader_module, nullptr); +} + +// NOLINTNEXTLINE(readability-function-cognitive-complexity) +void CreatePipelineInternal(PipelineCache::ComputePipeline* pipeline, + const ShaderComputeInputInfo* input_info, + std::span cs_shader) { + EXIT_IF(g_render_ctx == nullptr); + EXIT_IF(pipeline == nullptr); + + auto* gctx = g_render_ctx->GetGraphicCtx(); + + EXIT_IF(gctx == nullptr); + + VkShaderModule comp_shader_module = nullptr; + + VkShaderModuleCreateInfo create_info {}; + + create_info.sType = VK_STRUCTURE_TYPE_SHADER_MODULE_CREATE_INFO; + create_info.pNext = nullptr; + create_info.flags = 0; + create_info.codeSize = cs_shader.size() * 4; + create_info.pCode = cs_shader.data(); + LOGF("PipelineTrace: vkCreateShaderModule CS begin words=%" PRIu64 "\n", + static_cast(cs_shader.size())); + auto result = vkCreateShaderModule(gctx->device, &create_info, nullptr, &comp_shader_module); + LOGF("PipelineTrace: vkCreateShaderModule CS done result=%s module=%p\n", + string_VkResult(result), static_cast(comp_shader_module)); + EXIT_NOT_IMPLEMENTED(result != VK_SUCCESS); + + EXIT_NOT_IMPLEMENTED(comp_shader_module == nullptr); + + VkPipelineShaderStageCreateInfo comp_shader_stage_info {}; + VkPipelineShaderStageRequiredSubgroupSizeCreateInfo comp_subgroup_size {}; + comp_shader_stage_info.sType = VK_STRUCTURE_TYPE_PIPELINE_SHADER_STAGE_CREATE_INFO; + comp_shader_stage_info.pNext = nullptr; + comp_shader_stage_info.flags = 0; + comp_shader_stage_info.stage = VK_SHADER_STAGE_COMPUTE_BIT; + comp_shader_stage_info.module = comp_shader_module; + comp_shader_stage_info.pName = "main"; + comp_shader_stage_info.pSpecializationInfo = nullptr; + EXIT_IF(!input_info->stage); + ConfigureSubgroupSize(gctx, VK_SHADER_STAGE_COMPUTE_BIT, *input_info->stage.program, + &comp_subgroup_size, &comp_shader_stage_info); + + VkDescriptorSetLayout set_layouts[1] = {}; + uint32_t set_layouts_num = 0; + + VkPushConstantRange push_constant_info[1]; + uint32_t push_constant_info_num = 0; + + EXIT_IF(!input_info->stage); + CreateLayout(set_layouts, &set_layouts_num, push_constant_info, &push_constant_info_num, + *input_info->stage.program, VK_SHADER_STAGE_COMPUTE_BIT, + DescriptorCache::Stage::Compute); + + VkPipelineLayoutCreateInfo pipeline_layout_info {}; + pipeline_layout_info.sType = VK_STRUCTURE_TYPE_PIPELINE_LAYOUT_CREATE_INFO; + pipeline_layout_info.pNext = nullptr; + pipeline_layout_info.flags = 0; + pipeline_layout_info.setLayoutCount = set_layouts_num; + pipeline_layout_info.pSetLayouts = (set_layouts_num > 0 ? set_layouts : nullptr); + pipeline_layout_info.pushConstantRangeCount = push_constant_info_num; + pipeline_layout_info.pPushConstantRanges = + push_constant_info_num > 0 ? push_constant_info : nullptr; + + EXIT_IF(pipeline->pipeline_layout != nullptr); + + LOGF("PipelineTrace: vkCreatePipelineLayout CS begin set_layouts=%u push_constants=%u\n", + set_layouts_num, push_constant_info_num); + result = vkCreatePipelineLayout(gctx->device, &pipeline_layout_info, nullptr, + &pipeline->pipeline_layout); + LOGF("PipelineTrace: vkCreatePipelineLayout CS done result=%s layout=%p\n", + string_VkResult(result), static_cast(pipeline->pipeline_layout)); + EXIT_NOT_IMPLEMENTED(result != VK_SUCCESS); + + EXIT_NOT_IMPLEMENTED(pipeline->pipeline_layout == nullptr); + + VkComputePipelineCreateInfo info {}; + info.sType = VK_STRUCTURE_TYPE_COMPUTE_PIPELINE_CREATE_INFO; + info.pNext = nullptr; + info.flags = 0; + info.stage = comp_shader_stage_info; + info.layout = pipeline->pipeline_layout; + info.basePipelineHandle = nullptr; + info.basePipelineIndex = -1; + + EXIT_IF(pipeline->pipeline != nullptr); + + LOGF("PipelineTrace: vkCreateComputePipelines begin layout=%p\n", + static_cast(pipeline->pipeline_layout)); + result = + vkCreateComputePipelines(gctx->device, nullptr, 1, &info, nullptr, &pipeline->pipeline); + LOGF("PipelineTrace: vkCreateComputePipelines done result=%s pipeline=%p\n", + string_VkResult(result), static_cast(pipeline->pipeline)); + EXIT_NOT_IMPLEMENTED(result != VK_SUCCESS); + + EXIT_NOT_IMPLEMENTED(pipeline->pipeline == nullptr); + + vkDestroyShaderModule(gctx->device, comp_shader_module, nullptr); +} + +} // namespace Libs::Graphics diff --git a/src/graphics/host_gpu/renderer/streamBuffer.cpp b/src/graphics/host_gpu/renderer/streamBuffer.cpp new file mode 100644 index 000000000..3ea081c68 --- /dev/null +++ b/src/graphics/host_gpu/renderer/streamBuffer.cpp @@ -0,0 +1,99 @@ +#include "graphics/host_gpu/renderer/streamBuffer.h" + +#include "common/assert.h" +#include "common/logging/log.h" +#include "common/profiler.h" +#include "graphics/host_gpu/graphicContext.h" +#include "graphics/host_gpu/utils.h" +#include "graphics/host_gpu/vma.h" + +#include +#include + +namespace Libs::Graphics { + +namespace { +std::atomic g_stream_buffer_count {0}; +} // namespace + +HostStreamBuffer::HostStreamBuffer() = default; + +HostStreamBuffer::~HostStreamBuffer() { + Release(); +} + +void HostStreamBuffer::EnsureBuffer(GraphicContext* ctx) { + if (m_buffer != nullptr) { + if (m_ctx != ctx || m_mapped == nullptr) { + EXIT("host stream buffer changed graphics context\n"); + } + return; + } + KYTY_PROFILER_BLOCK("HostStreamBuffer::create"); + m_ctx = ctx; + m_buffer = std::make_unique(); + m_buffer->usage = VK_BUFFER_USAGE_STORAGE_BUFFER_BIT | VK_BUFFER_USAGE_VERTEX_BUFFER_BIT | + VK_BUFFER_USAGE_INDEX_BUFFER_BIT | VK_BUFFER_USAGE_TRANSFER_SRC_BIT | + VK_BUFFER_USAGE_TRANSFER_DST_BIT; + m_buffer->memory.property = static_cast(VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT) | + VK_MEMORY_PROPERTY_HOST_COHERENT_BIT | + VK_MEMORY_PROPERTY_HOST_CACHED_BIT; + VulkanCreateBuffer(ctx, CAPACITY, m_buffer.get()); + VulkanMapMemory(ctx, &m_buffer->memory, &m_mapped); + const auto count = g_stream_buffer_count.fetch_add(1, std::memory_order_relaxed) + 1; + LOGF("HostStreamBuffer: created serial=%" PRIu64 " capacity=%" PRIu64 "\n", count, + CAPACITY); +} + +bool HostStreamBuffer::Copy(GraphicContext* ctx, const void* src, uint64_t size, + uint64_t alignment, VulkanBuffer** out_buffer, uint64_t* out_offset, + uint64_t* out_range) { + KYTY_PROFILER_FUNCTION(); + if (ctx == nullptr || src == nullptr || out_buffer == nullptr || out_offset == nullptr || + out_range == nullptr) { + EXIT("invalid host stream upload\n"); + } + std::lock_guard lock(m_mutex); + if (size == 0) { + return false; + } + if (alignment == 0) { + alignment = 1; + } + if ((alignment & (alignment - 1)) != 0 || m_offset > UINT64_MAX - (alignment - 1)) { + return false; + } + const auto offset = (m_offset + alignment - 1) & ~(alignment - 1); + if (offset > CAPACITY || size > CAPACITY - offset) { + return false; + } + EnsureBuffer(ctx); + std::memcpy(static_cast(m_mapped) + offset, src, static_cast(size)); + m_offset = offset + size; + *out_buffer = m_buffer.get(); + *out_offset = offset; + *out_range = size; + return true; +} + +void HostStreamBuffer::Reset() noexcept { + std::lock_guard lock(m_mutex); + m_offset = 0; +} + +void HostStreamBuffer::Release() noexcept { + std::lock_guard lock(m_mutex); + if (m_buffer != nullptr) { + if (m_ctx == nullptr || m_mapped == nullptr) { + EXIT("invalid host stream buffer release state\n"); + } + VulkanUnmapMemory(m_ctx, &m_buffer->memory); + VulkanDeleteBuffer(m_ctx, m_buffer.get()); + m_buffer.reset(); + } + m_ctx = nullptr; + m_mapped = nullptr; + m_offset = 0; +} + +} // namespace Libs::Graphics diff --git a/src/graphics/host_gpu/renderer/streamBuffer.h b/src/graphics/host_gpu/renderer/streamBuffer.h new file mode 100644 index 000000000..dd9026133 --- /dev/null +++ b/src/graphics/host_gpu/renderer/streamBuffer.h @@ -0,0 +1,48 @@ +#ifndef EMULATOR_SRC_GRAPHICS_HOST_GPU_RENDERER_STREAMBUFFER_H_ +#define EMULATOR_SRC_GRAPHICS_HOST_GPU_RENDERER_STREAMBUFFER_H_ + +#include "common/abi.h" +#include "common/common.h" + +#include +#include +#include + +namespace Libs::Graphics { + +struct GraphicContext; +struct VulkanBuffer; + +// Host-only mapped stream mechanism. BufferCache owns the upload policy/API; one backend remains +// attached to each CommandBuffer so Reset is provably after that command's fence. Replacing this +// backend with scheduler watches will not change descriptor/render call sites. +class HostStreamBuffer final { +public: + HostStreamBuffer(); + ~HostStreamBuffer(); + KYTY_CLASS_NO_COPY(HostStreamBuffer); + + [[nodiscard]] bool Copy(GraphicContext* ctx, const void* src, uint64_t size, + uint64_t alignment, VulkanBuffer** out_buffer, uint64_t* out_offset, + uint64_t* out_range); + void Reset() noexcept; + void Release() noexcept; + +private: + void EnsureBuffer(GraphicContext* ctx); + + // Interim fence-scoped backend: one command processor's eight live command buffers reserve up + // to 128 MiB lazily. Additional active processors scale that aggregate until this backend is + // replaced by a single 64 MiB scheduler-watched ring; the BufferCache seam stays fixed. + static constexpr uint64_t CAPACITY = 16ull * 1024ull * 1024ull; + + mutable std::mutex m_mutex; + GraphicContext* m_ctx = nullptr; + std::unique_ptr m_buffer; + void* m_mapped = nullptr; + uint64_t m_offset = 0; +}; + +} // namespace Libs::Graphics + +#endif // EMULATOR_SRC_GRAPHICS_HOST_GPU_RENDERER_STREAMBUFFER_H_ diff --git a/src/graphics/host_gpu/renderer/sync.cpp b/src/graphics/host_gpu/renderer/sync.cpp new file mode 100644 index 000000000..3637037b6 --- /dev/null +++ b/src/graphics/host_gpu/renderer/sync.cpp @@ -0,0 +1,483 @@ +#include "common/assert.h" +#include "common/common.h" +#include "common/logging/log.h" +#include "common/threads.h" +#include "graphics/host_gpu/graphicContext.h" +#include "graphics/host_gpu/objects/label.h" +#include "graphics/host_gpu/renderer/bufferCache.h" +#include "graphics/host_gpu/renderer/render.h" +#include "graphics/host_gpu/renderer/renderContext.h" +#include "graphics/host_gpu/renderer/renderState.h" +#include "graphics/presentation/displayBuffer.h" +#include "kernel/eventQueue.h" +#include "kernel/pthread.h" +#include "libs/errno.h" + +#include + +namespace Libs::Graphics { + +constexpr int GRAPHICS_EVENT_QUEUED_GRAPHICS_INTERRUPT = 0x00; +constexpr int GRAPHICS_EVENT_EOP = 0x40; + +static void SubmitLabel32(CommandBuffer* buffer, uint32_t* dst, uint32_t value, + LabelCallback callback_1 = nullptr, LabelCallback callback_2 = nullptr, + const uint64_t* args = nullptr) { + auto* label = + LabelCreate32(g_render_ctx->GetGraphicCtx(), dst, value, callback_1, callback_2, args); + LabelSet(buffer, label); + LabelDelete(label); +} + +static void SubmitLabel64(CommandBuffer* buffer, uint64_t* dst, uint64_t value, + LabelCallback callback_1 = nullptr, LabelCallback callback_2 = nullptr, + const uint64_t* args = nullptr) { + auto* label = + LabelCreate64(g_render_ctx->GetGraphicCtx(), dst, value, callback_1, callback_2, args); + LabelSet(buffer, label); + LabelDelete(label); +} + +template +static void PublishImmediateFence(T* dst, T value) { + static_assert(sizeof(T) == sizeof(uint32_t) || sizeof(T) == sizeof(uint64_t)); + EXIT_IF(dst == nullptr); + // EOP/EOS fence payloads are constant and published while parsing the packet. Kyty performs the + // direct mapped write under CommandProcessorGuestAccessScope: a protected access is first resolved by + // BufferCache inline, then this instruction retries so the EOP payload wins over prior GPU bytes. + std::memcpy(dst, &value, sizeof(value)); +} + +static void SubmitDisplayBufferFlip(const uint64_t* args) { + if (g_render_ctx == nullptr || args == nullptr) { + EXIT("GPU flip submission has invalid state, render_ctx=%p args=%p\n", + static_cast(g_render_ctx), static_cast(args)); + } + const auto handle = static_cast(args[0]); + const auto index = static_cast(args[1]); + const auto flip_mode = static_cast(args[2]); + const auto flip_arg = static_cast(args[3]); + const auto result = + Presentation::DisplayBufferSubmitFlipFromGpu(handle, index, flip_mode, flip_arg); + if (result != 0) { + EXIT("GPU flip submission failed, result=%d handle=%d index=%d mode=%d arg=%" PRId64 "\n", + result, handle, index, flip_mode, flip_arg); + } +} + +static bool CompleteDisplayBufferFlip(const uint64_t* args) { + if (g_render_ctx == nullptr || args == nullptr) { + EXIT("GPU flip completion has invalid state, render_ctx=%p args=%p\n", + static_cast(g_render_ctx), static_cast(args)); + } + Presentation::DisplayBufferCompleteFlipFromGpu(static_cast(args[0]), + static_cast(args[1])); + return true; +} + +void GraphicsRenderTriggerAgcUserInterrupt() { + auto tsc = LibKernel::KernelReadTsc(); + auto result = LibKernel::EventQueue::KernelTriggerUserEventForAll(AGC_USER_INTERRUPT_EVENT, + reinterpret_cast(tsc)); + EXIT_NOT_IMPLEMENTED(result != OK && result != LibKernel::KERNEL_ERROR_ENOENT); +} + +void GraphicsRenderTriggerEopEvent(uint32_t context_id) { + EXIT_IF(g_render_ctx == nullptr); + g_render_ctx->TriggerEopEvent(context_id); +} + +void GraphicsRenderWriteAtEndOfPipe32(uint64_t submit_id, CommandBuffer* buffer, + uint32_t* dst_gpu_addr, uint32_t value) { + EXIT_IF(g_render_ctx == nullptr); + EXIT_IF(dst_gpu_addr == nullptr); + EXIT_IF(buffer == nullptr); + EXIT_IF(buffer->IsInvalid()); + + buffer->SetDebugInfo(static_cast(CommandBufferDebugOp::EopWrite), submit_id, 4, value, + 0, 0, reinterpret_cast(dst_gpu_addr)); + + PublishImmediateFence(dst_gpu_addr, value); +} + +void GraphicsRenderWriteAtEndOfPipeGds32(uint64_t submit_id, CommandBuffer* buffer, + uint32_t* dst_gpu_addr, uint32_t dw_offset, + uint32_t dw_num) { + EXIT_IF(g_render_ctx == nullptr); + EXIT_IF(dst_gpu_addr == nullptr); + EXIT_IF(buffer == nullptr); + EXIT_IF(buffer->IsInvalid()); + + buffer->SetDebugInfo(static_cast(CommandBufferDebugOp::EopWrite), submit_id, + dw_offset, dw_num, 0, 0, reinterpret_cast(dst_gpu_addr)); + + uint64_t args[LABEL_ARGS_MAX] = {static_cast(dw_offset), + static_cast(dw_num), + reinterpret_cast(dst_gpu_addr), 0}; + + SubmitLabel32( + buffer, dst_gpu_addr, 0, + [](const uint64_t* args) { + auto dw_offset = static_cast(args[0]); + auto dw_num = static_cast(args[1]); + auto* dst_gpu_addr = reinterpret_cast(args[2]); + if (dst_gpu_addr == nullptr || dw_num == 0) { + EXIT("invalid asynchronous EOP GDS write\n"); + } + const auto size = static_cast(dw_num) * sizeof(uint32_t); + auto transaction = g_render_ctx->GetGpuResources()->LockAsynchronousGuestWrite( + reinterpret_cast(dst_gpu_addr), size); + g_render_ctx->GetGdsBuffer()->Read(g_render_ctx->GetGraphicCtx(), dst_gpu_addr, + dw_offset, dw_num); + return false; + }, + nullptr, args); +} + +void GraphicsRenderWriteAtEndOfPipe64(uint64_t submit_id, CommandBuffer* buffer, + uint64_t* dst_gpu_addr, uint64_t value) { + EXIT_IF(g_render_ctx == nullptr); + EXIT_IF(dst_gpu_addr == nullptr); + EXIT_IF(buffer == nullptr); + EXIT_IF(buffer->IsInvalid()); + + buffer->SetDebugInfo(static_cast(CommandBufferDebugOp::EopWrite), submit_id, 8, + static_cast(value), static_cast(value >> 32u), 0, + reinterpret_cast(dst_gpu_addr)); + + PublishImmediateFence(dst_gpu_addr, value); +} + +void GraphicsRenderWriteAtEndOfPipeClockCounter(uint64_t submit_id, CommandBuffer* buffer, + uint64_t* dst_gpu_addr) { + EXIT_IF(g_render_ctx == nullptr); + EXIT_IF(dst_gpu_addr == nullptr); + EXIT_IF(buffer == nullptr); + EXIT_IF(buffer->IsInvalid()); + + buffer->SetDebugInfo(static_cast(CommandBufferDebugOp::EopWrite), submit_id, 8, 0, 0, + 0, reinterpret_cast(dst_gpu_addr)); + + const auto value = LibKernel::KernelReadTsc(); + PublishImmediateFence(dst_gpu_addr, value); + LOGF_COLOR(Log::Color::BrightGreen, + "EndOfPipe Signal!!! [0x%016" PRIx64 "] <- Clock: 0x%016" PRIx64 "\n", + reinterpret_cast(dst_gpu_addr), value); +} + +void GraphicsRenderWriteAtEndOfPipeClockCounterWithWriteBack(uint64_t submit_id, + CommandBuffer* buffer, + uint64_t* dst_gpu_addr) { + EXIT_IF(g_render_ctx == nullptr); + EXIT_IF(dst_gpu_addr == nullptr); + EXIT_IF(buffer == nullptr); + EXIT_IF(buffer->IsInvalid()); + + buffer->SetDebugInfo(static_cast(CommandBufferDebugOp::EopWriteBack), submit_id, 8, 0, + 0, 0, reinterpret_cast(dst_gpu_addr)); + + const auto value = LibKernel::KernelReadTsc(); + PublishImmediateFence(dst_gpu_addr, value); + LOGF_COLOR(Log::Color::BrightGreen, + "EndOfPipe Signal!!! [0x%016" PRIx64 "] <- Clock: 0x%016" PRIx64 "\n", + reinterpret_cast(dst_gpu_addr), value); +} + +void GraphicsRenderWriteAtEndOfPipeWithWriteBack64(uint64_t submit_id, CommandBuffer* buffer, + uint64_t* dst_gpu_addr, uint64_t value) { + EXIT_IF(g_render_ctx == nullptr); + EXIT_IF(dst_gpu_addr == nullptr); + EXIT_IF(buffer == nullptr); + EXIT_IF(buffer->IsInvalid()); + + buffer->SetDebugInfo(static_cast(CommandBufferDebugOp::EopWriteBack), submit_id, 8, + static_cast(value), static_cast(value >> 32u), 0, + reinterpret_cast(dst_gpu_addr)); + + PublishImmediateFence(dst_gpu_addr, value); +} + +void GraphicsRenderWriteAtEndOfPipeWithWriteBack32(uint64_t submit_id, CommandBuffer* buffer, + uint32_t* dst_gpu_addr, uint32_t value) { + EXIT_IF(g_render_ctx == nullptr); + EXIT_IF(dst_gpu_addr == nullptr); + EXIT_IF(buffer == nullptr); + EXIT_IF(buffer->IsInvalid()); + + buffer->SetDebugInfo(static_cast(CommandBufferDebugOp::EopWriteBack), submit_id, 4, + value, 0, 0, reinterpret_cast(dst_gpu_addr)); + + PublishImmediateFence(dst_gpu_addr, value); +} + +void GraphicsRenderWriteAtEndOfPipeWithInterruptWriteBack64(uint64_t submit_id, + CommandBuffer* buffer, + uint64_t* dst_gpu_addr, uint64_t value, + uint32_t context_id) { + EXIT_IF(g_render_ctx == nullptr); + EXIT_IF(dst_gpu_addr == nullptr); + EXIT_IF(buffer == nullptr); + EXIT_IF(buffer->IsInvalid()); + + buffer->SetDebugInfo(static_cast(CommandBufferDebugOp::EopWriteBack), submit_id, 8, + context_id, static_cast(value), + static_cast(value >> 32u), + reinterpret_cast(dst_gpu_addr)); + PublishImmediateFence(dst_gpu_addr, value); + + uint64_t args[LABEL_ARGS_MAX] = {context_id != 0 ? context_id : value}; + + SubmitLabel64( + buffer, nullptr, 0, nullptr, + [](const uint64_t* args) { + EXIT_IF(g_render_ctx == nullptr); + g_render_ctx->TriggerEopEvent(static_cast(args[0])); + return true; + }, + args); +} + +void GraphicsRenderWriteAtEndOfPipeWithInterruptWriteBack32(uint64_t submit_id, + CommandBuffer* buffer, + uint32_t* dst_gpu_addr, uint32_t value, + uint32_t context_id) { + EXIT_IF(g_render_ctx == nullptr); + EXIT_IF(dst_gpu_addr == nullptr); + EXIT_IF(buffer == nullptr); + EXIT_IF(buffer->IsInvalid()); + + buffer->SetDebugInfo(static_cast(CommandBufferDebugOp::EopWriteBack), submit_id, 4, + context_id, value, 0, reinterpret_cast(dst_gpu_addr)); + PublishImmediateFence(dst_gpu_addr, value); + + uint64_t args[LABEL_ARGS_MAX] = {context_id != 0 ? context_id : value}; + + SubmitLabel32( + buffer, nullptr, 0, nullptr, + [](const uint64_t* args) { + EXIT_IF(g_render_ctx == nullptr); + g_render_ctx->TriggerEopEvent(static_cast(args[0])); + return true; + }, + args); +} + +void GraphicsRenderWriteAtEndOfPipeWithInterrupt64(uint64_t submit_id, CommandBuffer* buffer, + uint64_t* dst_gpu_addr, uint64_t value, + uint32_t context_id) { + EXIT_IF(g_render_ctx == nullptr); + EXIT_IF(dst_gpu_addr == nullptr); + EXIT_IF(buffer == nullptr); + EXIT_IF(buffer->IsInvalid()); + + buffer->SetDebugInfo(static_cast(CommandBufferDebugOp::EopInterrupt), submit_id, 8, + context_id, static_cast(value), + static_cast(value >> 32u), + reinterpret_cast(dst_gpu_addr)); + PublishImmediateFence(dst_gpu_addr, value); + + uint64_t args[LABEL_ARGS_MAX] = {context_id != 0 ? context_id : value}; + + SubmitLabel64( + buffer, nullptr, 0, nullptr, + [](const uint64_t* args) { + EXIT_IF(g_render_ctx == nullptr); + g_render_ctx->TriggerEopEvent(static_cast(args[0])); + return true; + }, + args); +} + +void GraphicsRenderWriteAtEndOfPipeWithInterrupt32(uint64_t submit_id, CommandBuffer* buffer, + uint32_t* dst_gpu_addr, uint32_t value, + uint32_t context_id) { + EXIT_IF(g_render_ctx == nullptr); + EXIT_IF(dst_gpu_addr == nullptr); + EXIT_IF(buffer == nullptr); + EXIT_IF(buffer->IsInvalid()); + + buffer->SetDebugInfo(static_cast(CommandBufferDebugOp::EopInterrupt), submit_id, 4, + context_id, value, 0, reinterpret_cast(dst_gpu_addr)); + PublishImmediateFence(dst_gpu_addr, value); + + uint64_t args[LABEL_ARGS_MAX] = {context_id != 0 ? context_id : value}; + + SubmitLabel32( + buffer, nullptr, 0, nullptr, + [](const uint64_t* args) { + EXIT_IF(g_render_ctx == nullptr); + g_render_ctx->TriggerEopEvent(static_cast(args[0])); + return true; + }, + args); +} + +void GraphicsRenderWriteAtEndOfPipeWithInterruptWriteBackFlip32( + uint64_t submit_id, CommandBuffer* buffer, uint32_t* dst_gpu_addr, uint32_t value, int handle, + int index, int flip_mode, int64_t flip_arg) { + EXIT_IF(g_render_ctx == nullptr); + EXIT_IF(dst_gpu_addr == nullptr); + EXIT_IF(buffer == nullptr); + EXIT_IF(buffer->IsInvalid()); + + buffer->SetDebugInfo(static_cast(CommandBufferDebugOp::EopWriteBackFlip), submit_id, + static_cast(handle), static_cast(index), + static_cast(flip_mode), value, static_cast(flip_arg)); + PublishImmediateFence(dst_gpu_addr, value); + + uint64_t args[LABEL_ARGS_MAX] = {static_cast(handle), static_cast(index), + static_cast(flip_mode), + static_cast(flip_arg)}; + + SubmitDisplayBufferFlip(args); + SubmitLabel32( + buffer, nullptr, 0, CompleteDisplayBufferFlip, + [](const uint64_t* /*args*/) { + EXIT_IF(g_render_ctx == nullptr); + g_render_ctx->TriggerEopEvent(0); + return true; + }, + args); +} + +void GraphicsRenderWriteAtEndOfPipeWithFlip32(uint64_t submit_id, CommandBuffer* buffer, + uint32_t* dst_gpu_addr, uint32_t value, int handle, + int index, int flip_mode, int64_t flip_arg) { + EXIT_IF(g_render_ctx == nullptr); + EXIT_IF(dst_gpu_addr == nullptr); + EXIT_IF(buffer == nullptr); + EXIT_IF(buffer->IsInvalid()); + + buffer->SetDebugInfo(static_cast(CommandBufferDebugOp::EopFlip), submit_id, + static_cast(handle), static_cast(index), + static_cast(flip_mode), value, static_cast(flip_arg)); + PublishImmediateFence(dst_gpu_addr, value); + + uint64_t args[LABEL_ARGS_MAX] = {static_cast(handle), static_cast(index), + static_cast(flip_mode), + static_cast(flip_arg)}; + + SubmitDisplayBufferFlip(args); + SubmitLabel32(buffer, nullptr, 0, CompleteDisplayBufferFlip, nullptr, args); +} + +void GraphicsRenderWriteAtEndOfPipeOnlyFlip(uint64_t submit_id, CommandBuffer* buffer, int handle, + int index, int flip_mode, int64_t flip_arg) { + EXIT_IF(g_render_ctx == nullptr); + EXIT_IF(buffer == nullptr); + EXIT_IF(buffer->IsInvalid()); + + buffer->SetDebugInfo(static_cast(CommandBufferDebugOp::EopOnlyFlip), submit_id, + static_cast(handle), static_cast(index), + static_cast(flip_mode), 0, static_cast(flip_arg)); + + uint64_t args[LABEL_ARGS_MAX] = {static_cast(handle), static_cast(index), + static_cast(flip_mode), + static_cast(flip_arg)}; + + SubmitDisplayBufferFlip(args); + SubmitLabel32(buffer, nullptr, 0, CompleteDisplayBufferFlip, nullptr, args); +} + +void GraphicsRenderTriggerEopEventAtEndOfPipe(CommandBuffer* buffer, uint32_t context_id) { + EXIT_IF(g_render_ctx == nullptr); + EXIT_IF(buffer == nullptr); + EXIT_IF(buffer->IsInvalid()); + + uint64_t args[LABEL_ARGS_MAX] = {static_cast(context_id)}; + + SubmitLabel32( + buffer, nullptr, 0, nullptr, + [](const uint64_t* args) { + EXIT_IF(g_render_ctx == nullptr); + g_render_ctx->TriggerEopEvent(static_cast(args[0])); + return true; + }, + args); +} + +static void EopEventResetFunc(LibKernel::EventQueue::KernelEqueueEvent* event) { + EXIT_IF(event == nullptr); + event->triggered = false; + event->event.fflags = 0; + event->event.data = 0; +} + +static void EopEventDeleteFunc(LibKernel::EventQueue::KernelEqueue eq, + LibKernel::EventQueue::KernelEqueueEvent* event) { + EXIT_IF(event == nullptr); + EXIT_IF(g_render_ctx == nullptr); + EXIT_NOT_IMPLEMENTED(event->event.filter != LibKernel::EventQueue::KERNEL_EVFILT_GRAPHICS); + if (event->event.ident == GRAPHICS_EVENT_QUEUED_GRAPHICS_INTERRUPT || + event->event.ident == GRAPHICS_EVENT_EOP) { + g_render_ctx->DeleteEopEq(eq, static_cast(event->event.ident)); + } +} + +static void EopEventTriggerFunc(LibKernel::EventQueue::KernelEqueueEvent* event, + void* trigger_data) { + EXIT_IF(event == nullptr); + + auto triggered_event = event->event; + triggered_event.fflags++; + triggered_event.data = reinterpret_cast(trigger_data); + if (event->triggered) { + event->pending_events.push_back(triggered_event); + } else { + event->event = triggered_event; + event->triggered = true; + } +} + +int GraphicsRenderAddEqEvent(LibKernel::EventQueue::KernelEqueue eq, int id, void* udata) { + EXIT_IF(g_render_ctx == nullptr); + + LibKernel::EventQueue::KernelEqueueEvent event; + event.triggered = false; + event.event.ident = static_cast(id); + event.event.filter = LibKernel::EventQueue::KERNEL_EVFILT_GRAPHICS; + event.event.udata = udata; + event.event.fflags = 0; + event.event.data = id; + event.filter.delete_event_func = EopEventDeleteFunc; + event.filter.reset_func = EopEventResetFunc; + event.filter.trigger_func = EopEventTriggerFunc; + event.filter.data = nullptr; + + int result = LibKernel::EventQueue::KernelAddEvent(eq, event); + + if (id == GRAPHICS_EVENT_QUEUED_GRAPHICS_INTERRUPT || id == GRAPHICS_EVENT_EOP) { + g_render_ctx->AddEopEq(eq, id); + } + + return result; +} + +int GraphicsRenderDeleteEqEvent(LibKernel::EventQueue::KernelEqueue eq, int id) { + EXIT_IF(g_render_ctx == nullptr); + + int result = LibKernel::EventQueue::KernelDeleteEvent( + eq, static_cast(id), LibKernel::EventQueue::KERNEL_EVFILT_GRAPHICS); + + return result; +} + +void GraphicsRenderClearGds(uint64_t dw_offset, uint32_t dw_num, uint32_t clear_value) { + EXIT_IF(g_render_ctx == nullptr); + + g_render_ctx->GetGdsBuffer()->Clear(g_render_ctx->GetGraphicCtx(), dw_offset, dw_num, + clear_value); +} + +void GraphicsRenderReadGds(uint32_t* dst, uint32_t dw_offset, uint32_t dw_size) { + EXIT_IF(g_render_ctx == nullptr); + + g_render_ctx->GetGdsBuffer()->Read(g_render_ctx->GetGraphicCtx(), dst, dw_offset, dw_size); +} + +void GraphicsRenderDeleteBuffers() { + EXIT_IF(g_render_ctx == nullptr); + g_render_ctx->GetBufferCache()->DeleteAll(g_render_ctx->GetGraphicCtx()); +} + +} // namespace Libs::Graphics diff --git a/src/graphics/host_gpu/renderer/textureCache.cpp b/src/graphics/host_gpu/renderer/textureCache.cpp new file mode 100644 index 000000000..46227466a --- /dev/null +++ b/src/graphics/host_gpu/renderer/textureCache.cpp @@ -0,0 +1,3414 @@ +#include "graphics/host_gpu/renderer/textureCache.h" + +#include "common/assert.h" +#include "common/logging/log.h" +#include "common/profiler.h" +#include "graphics/guest_gpu/graphicsRun.h" +#include "graphics/guest_gpu/gpu_defs.h" +#include "graphics/guest_gpu/gpu_format.h" +#include "graphics/guest_gpu/tile.h" +#include "graphics/host_gpu/graphicContext.h" +#include "graphics/host_gpu/hostMemory.h" +#include "graphics/host_gpu/objects/label.h" +#include "graphics/host_gpu/objects/textureCommon.h" +#include "graphics/host_gpu/renderer/bufferCache.h" +#include "graphics/host_gpu/renderer/framebufferCache.h" +#include "graphics/host_gpu/renderer/image.h" +#include "graphics/host_gpu/renderer/imageView.h" +#include "graphics/host_gpu/renderer/render.h" +#include "graphics/host_gpu/renderer/renderContext.h" +#include "graphics/host_gpu/renderer/resourceMutex.h" +#include "graphics/host_gpu/utils.h" +#include "graphics/shader/shader.h" +#include "kernel/memory.h" + +#include +#include +#include +#include +#include +#include +#include + +namespace Libs::Graphics { + +namespace { + +thread_local const void* g_texture_cache_lock_owner = nullptr; +thread_local const void* g_texture_fault_owner = nullptr; + +class FaultSafeTextureLock final { +public: + FaultSafeTextureLock(const void* owner, TrackingSpinLock& mutex): m_mutex(mutex) { + if (g_texture_cache_lock_owner != nullptr) { + EXIT("TextureCache: recursive cache lock acquisition, current=%p\n", + g_texture_cache_lock_owner); + } + g_texture_cache_lock_owner = owner; + m_mutex.lock(); + } + ~FaultSafeTextureLock() { + m_mutex.unlock(); + g_texture_cache_lock_owner = nullptr; + } + +private: + TrackingSpinLock& m_mutex; +}; + +bool Overlaps(uint64_t left, uint64_t left_size, uint64_t right, uint64_t right_size) { + return left < right + right_size && right < left + left_size; +} + +bool PageOverlaps(uint64_t left, uint64_t left_size, uint64_t right, uint64_t right_size) { + return ImagePageRangesOverlap(left, left_size, right, right_size); +} + +} // namespace + +struct TextureCache::CachedImage { + enum class Kind { + Texture, + StorageTexture, + RenderTarget, + DepthTarget, + VideoOut + } kind = Kind::Texture; + Image info; + RenderTargetInfo target; + DepthTargetInfo depth; + VideoOutInfo video_out; + GraphicContext* ctx = nullptr; + VulkanImage* image = nullptr; + VulkanMemory memory; + bool gpu_modified = false; + bool buffer_modified = false; + bool stencil_initialized = false; + + ~CachedImage() { + if (ctx == nullptr || image == nullptr) { + EXIT("TextureCache: cached image destroyed with invalid resources, ctx=%p image=%p " + "kind=%u\n", + static_cast(ctx), static_cast(image), + static_cast(kind)); + } + switch (kind) { + case Kind::Texture: + case Kind::StorageTexture: + TextureCache::DeleteGpuTexture(ctx, static_cast(image), + &memory); + break; + case Kind::RenderTarget: + TextureCache::DeleteRenderTexture( + ctx, static_cast(image), &memory); + break; + case Kind::DepthTarget: + TextureCache::DeleteDepthStencil(ctx, static_cast(image), + &memory); + break; + case Kind::VideoOut: + TextureCache::DeleteVideoOut(ctx, static_cast(image), + &memory); + break; + } + image = nullptr; + } + + [[nodiscard]] uint32_t RangeCount() const { + return kind == Kind::DepthTarget && depth.stencil_address != 0 ? 2u : 1u; + } + [[nodiscard]] uint64_t Address(uint32_t index = 0) const { + if (index >= RangeCount()) { + EXIT("TextureCache: image address range index out of bounds, index=%u count=%u\n", + index, RangeCount()); + } + if (index == 1) { + return depth.stencil_address; + } + switch (kind) { + case Kind::Texture: return info.address; + case Kind::StorageTexture: return info.address; + case Kind::RenderTarget: return target.address; + case Kind::DepthTarget: return depth.address; + case Kind::VideoOut: return video_out.address; + } + EXIT("TextureCache: unsupported cached image kind %u for address\n", + static_cast(kind)); + } + [[nodiscard]] uint64_t Size(uint32_t index = 0) const { + if (index >= RangeCount()) { + EXIT("TextureCache: image size range index out of bounds, index=%u count=%u\n", index, + RangeCount()); + } + if (index == 1) { + return depth.stencil_size; + } + switch (kind) { + case Kind::Texture: return info.size; + case Kind::StorageTexture: return info.size; + case Kind::RenderTarget: return target.size; + case Kind::DepthTarget: return depth.size; + case Kind::VideoOut: return video_out.size; + } + EXIT("TextureCache: unsupported cached image kind %u for size\n", + static_cast(kind)); + } + [[nodiscard]] bool OverlapsRange(uint64_t address, uint64_t size, bool page) const { + for (uint32_t i = 0; i < RangeCount(); i++) { + if (page ? PageOverlaps(address, size, Address(i), Size(i)) + : Overlaps(address, size, Address(i), Size(i))) { + return true; + } + } + return false; + } + // A page-expanded region lookup identifies candidates before exact byte classification. Kyty's + // tracker retains GPU ownership per 4 KiB page, so an edge-page + // candidate must remain coherent even when the triggering byte lies just outside the image. + [[nodiscard]] bool IsGpuReadbackPageCandidate(uint64_t address, uint64_t size) const { + return gpu_modified && OverlapsRange(address, size, true); + } + [[nodiscard]] bool HasExactRange(uint64_t address, uint64_t size) const { + for (uint32_t i = 0; i < RangeCount(); i++) { + if (address == Address(i) && size == Size(i)) { + return true; + } + } + return false; + } +}; + +struct TextureCache::ReadbackWorker { + enum class State : uint32_t { + Idle, + Claimed, + Requested, + Ready, + Installed, + Completed, + Stopping, + Stopped + }; + struct ReadbackRange { + uint64_t address; + uint64_t size; + }; + static_assert(std::atomic::is_always_lock_free); + + explicit ReadbackWorker(TextureCache& owner): cache(owner), thread([this] { Run(); }) {} + + ~ReadbackWorker() { + auto expected = State::Idle; + if (!state.compare_exchange_strong(expected, State::Stopping, std::memory_order_acq_rel)) { + EXIT("TextureCache: cannot stop readback worker from state %u\n", + static_cast(expected)); + } + state.notify_all(); + thread.join(); + if (state.load(std::memory_order_acquire) != State::Stopped) { + EXIT("TextureCache: readback worker did not reach stopped state\n"); + } + } + + void Request(PageFaultAccess fault_access, uint64_t fault_vaddr, uint64_t fault_size) noexcept { + const bool submissions_prepaused_now = GraphicsRunSubmissionLockHeld(); + const bool unsafe_gpu_lock = GraphicsRunGpuLockHeld() && !submissions_prepaused_now; + if ((fault_access != PageFaultAccess::Read && fault_access != PageFaultAccess::Write) || + GraphicsRunIsCommandProcessorThread() || unsafe_gpu_lock || LabelInCallback() || + g_texture_cache_lock_owner != nullptr || g_texture_fault_owner != &cache) { + EXIT("TextureCache: unsafe image readback request, access=%u command_thread=%d " + "submission_lock=%d gpu_lock=%d label_callback=%d cache_lock=%p fault_owner=%p\n", + static_cast(fault_access), GraphicsRunIsCommandProcessorThread(), + GraphicsRunSubmissionLockHeld(), GraphicsRunGpuLockHeld(), LabelInCallback(), + g_texture_cache_lock_owner, g_texture_fault_owner); + } + State expected = State::Idle; + while (!state.compare_exchange_weak(expected, State::Claimed, std::memory_order_acq_rel)) { + if (expected == State::Stopping || expected == State::Stopped) { + EXIT("TextureCache: image readback requested while worker is stopping, state=%u\n", + static_cast(expected)); + } + state.wait(expected, std::memory_order_acquire); + expected = State::Idle; + } + access = fault_access; + vaddr = fault_vaddr; + size = fault_size; + submissions_prepaused = submissions_prepaused_now; + state.store(State::Requested, std::memory_order_release); + state.notify_all(); + while (true) { + const auto current = state.load(std::memory_order_acquire); + if (current == State::Ready) { + return; + } + if (current != State::Requested) { + EXIT("TextureCache: invalid state while waiting for image readback, state=%u\n", + static_cast(current)); + } + state.wait(current, std::memory_order_acquire); + } + } + + [[nodiscard]] bool Complete(PageFaultAccess fault_access, uint64_t fault_vaddr, + uint64_t fault_size) noexcept { + const auto current = state.load(std::memory_order_acquire); + if (current == State::Idle || current == State::Stopping || current == State::Stopped) { + return false; + } + if (current != State::Ready) { + EXIT("TextureCache: active image readback has invalid completion state %u\n", + static_cast(current)); + } + if (access != fault_access || vaddr != fault_vaddr || size != fault_size) { + EXIT("TextureCache: mismatched active image readback completion\n"); + } + state.store(State::Installed, std::memory_order_release); + state.notify_all(); + return true; + } + + [[nodiscard]] bool IsReady(PageFaultAccess fault_access, uint64_t fault_vaddr, + uint64_t fault_size) const noexcept { + return state.load(std::memory_order_acquire) == State::Ready && access == fault_access && + vaddr == fault_vaddr && size == fault_size; + } + + void Release(PageFaultAccess fault_access, uint64_t fault_vaddr, uint64_t fault_size) noexcept { + const auto current = state.load(std::memory_order_acquire); + if (current == State::Idle || current == State::Stopping || current == State::Stopped) { + return; + } + if (current != State::Installed) { + EXIT("TextureCache: active image readback has invalid release state %u\n", + static_cast(current)); + } + if (access != fault_access || vaddr != fault_vaddr || size != fault_size) { + EXIT("TextureCache: mismatched active image readback release\n"); + } + state.store(State::Completed, std::memory_order_release); + state.notify_all(); + while (true) { + const auto current = state.load(std::memory_order_acquire); + if (current == State::Idle) { + break; + } + if (current != State::Completed) { + EXIT("TextureCache: invalid state while releasing image readback, state=%u\n", + static_cast(current)); + } + state.wait(current, std::memory_order_acquire); + } + } + + [[nodiscard]] ReadbackRange DownloadDepthTarget(CachedImage& cached) { + const auto& info = cached.depth; + const bool has_stencil = info.stencil_address != 0 || info.stencil_size != 0; + const bool has_htile = info.htile_address != 0 || info.htile_size != 0; + const bool fault_in_depth = vaddr >= info.address && vaddr < info.address + info.size && + size <= info.address + info.size - vaddr; + const bool d16 = info.guest_format == Prospero::GpuEnumValue(Prospero::BufferFormat::k16UNorm) && + info.format == VK_FORMAT_D16_UNORM && info.bytes_per_element == 2; + const bool d32 = info.guest_format == Prospero::GpuEnumValue(Prospero::BufferFormat::k32Float) && + info.format == VK_FORMAT_D32_SFLOAT && info.bytes_per_element == 4; + TileSizeAlign expected_stencil {}; + TileSizeAlign expected_htile {}; + TileSizeAlign expected_depth {}; + const bool prospero_layout = + (d16 || d32) && + TileGetDepthSize(info.width, info.height, 0, + Prospero::GpuEnumValue(d16 ? Prospero::DepthFormat::kZ16 : Prospero::DepthFormat::kZ32F), + Prospero::GpuEnumValue(Prospero::StencilFormat::kInvalid), has_htile, &expected_stencil, + &expected_htile, &expected_depth); + const auto expected_pitch = prospero_layout + ? TileGetTexturePitch(info.guest_format, info.width, 1, + Prospero::GpuEnumValue(Prospero::TileMode::kDepth)) + : 0u; + if (!fault_in_depth || has_stencil || info.address == 0 || info.size == 0 || + info.width == 0 || info.height == 0 || info.pitch < info.width || + info.tile_mode != Prospero::GpuEnumValue(Prospero::TileMode::kDepth) || !prospero_layout || + info.pitch != expected_pitch || info.size != expected_depth.size || + expected_depth.align != 65536u || + (has_htile && + (info.htile_size != expected_htile.size || expected_htile.align != 32768u))) { + EXIT("TextureCache: unsupported depth-image readback layout, fault=0x%016" PRIx64 + "+0x%016" PRIx64 " depth=0x%016" PRIx64 "+0x%016" PRIx64 " stencil=0x%016" PRIx64 + "+0x%016" PRIx64 " htile=0x%016" PRIx64 "+0x%016" PRIx64 + " extent=%ux%u pitch=%u/%u tile=%u format=%d guest_format=%u bpe=%u " + "expected_depth=0x%016" PRIx64 " expected_htile=0x%016" PRIx64 "\n", + vaddr, size, info.address, info.size, info.stencil_address, info.stencil_size, + info.htile_address, info.htile_size, info.width, info.height, info.pitch, + expected_pitch, info.tile_mode, static_cast(info.format), info.guest_format, + info.bytes_per_element, expected_depth.size, expected_htile.size); + } + const auto rows = static_cast(info.height - 1); + if (rows > (UINT64_MAX - info.width) / info.pitch) { + EXIT("TextureCache: depth-image readback size overflow\n"); + } + const auto linear_elements = rows * info.pitch + info.width; + if (linear_elements > UINT64_MAX / info.bytes_per_element) { + EXIT("TextureCache: depth-image readback size overflow\n"); + } + const auto linear_size = linear_elements * info.bytes_per_element; + const bool meta_overlap = cache.HasMetaOverlapLocked(info.address, info.size); + const bool buffer_overlap = cache.m_buffer_cache.HasPageOverlap(info.address, info.size); + if (linear_size > info.size || cached.image->format != info.format || + cached.image->extent.width != info.width || + cached.image->extent.height != info.height || meta_overlap || buffer_overlap) { + EXIT("TextureCache: depth-image readback storage is unsupported, " + "depth=0x%016" PRIx64 "+0x%016" PRIx64 " linear=0x%016" PRIx64 + " format=%d/%d extent=%ux%u/%ux%u meta=%d buffer=%d\n", + info.address, info.size, linear_size, static_cast(cached.image->format), + static_cast(info.format), cached.image->extent.width, + cached.image->extent.height, info.width, info.height, meta_overlap, + buffer_overlap); + } + download.resize(info.size); + std::fill(download.begin(), download.end(), 0); + UtilFillBuffer(cached.ctx, download.data(), linear_size, info.pitch, cached.image, + cached.image->layout, VK_IMAGE_ASPECT_DEPTH_BIT); + guest.resize(info.size); + cache.m_tiler.TileImage(guest.data(), download.data(), info); + Libs::LibKernel::Memory::WriteBacking(info.address, guest.data(), info.size); + return {info.address, info.size}; + } + + [[nodiscard]] ReadbackRange DownloadColorImage(CachedImage& cached) { + const bool storage = cached.kind == CachedImage::Kind::StorageTexture; + const bool target = cached.kind == CachedImage::Kind::RenderTarget; + const auto info = + storage ? MakeColorImageTransferInfo(cached.info, Prospero::SurfaceFormat(cached.info.format), + Prospero::NumBytesPerElement(cached.info.format)) + : MakeColorImageTransferInfo(cached.target); + const bool linear = info.tile_mode == Prospero::GpuEnumValue(Prospero::TileMode::kLinear); + const bool tiled = target && info.tile_mode == Prospero::GpuEnumValue(Prospero::TileMode::kRenderTarget); + const bool basic_storage = + !storage || (cached.info.type == Prospero::GpuEnumValue(Prospero::ImageType::kColor2D) && + cached.info.depth == 1 && cached.info.base_level == 0 && + cached.info.levels == 1 && cached.info.base_array == 0 && linear); + const auto rows = static_cast(info.height - 1); + if ((!linear && !tiled) || !basic_storage || info.address == 0 || info.size == 0 || + info.width == 0 || info.height == 0 || info.levels != 1 || info.pitch < info.width || + info.bytes_per_element == 0 || rows > (UINT64_MAX - info.width) / info.pitch) { + EXIT("TextureCache: unsupported color-image readback layout, addr=0x%016" PRIx64 + " size=0x%016" PRIx64 " extent=%ux%u pitch=%u bpe=%u levels=%u tile=%u kind=%u\n", + info.address, info.size, info.width, info.height, info.pitch, + info.bytes_per_element, info.levels, info.tile_mode, + static_cast(cached.kind)); + } + const auto linear_elements = rows * info.pitch + info.width; + if (linear_elements > UINT64_MAX / info.bytes_per_element) { + EXIT("TextureCache: color-image readback size overflow\n"); + } + const auto linear_size = linear_elements * info.bytes_per_element; + const bool meta_overlap = cache.HasMetaOverlapLocked(info.address, info.size); + const bool buffer_overlap = cache.m_buffer_cache.HasPageOverlap(info.address, info.size); + if (linear_size > info.size || cached.image->format != info.format || + cached.image->extent.width != info.width || + cached.image->extent.height != info.height || + (tiled && info.bytes_per_element != 1 && info.bytes_per_element != 2 && + info.bytes_per_element != 4 && info.bytes_per_element != 8) || + meta_overlap || buffer_overlap) { + EXIT("TextureCache: color-image readback storage is unsupported, addr=0x%016" PRIx64 + " size=0x%016" PRIx64 " linear=0x%016" PRIx64 + " format=%d/%d extent=%ux%u/%ux%u bpe=%u tiled=%d meta=%d kind=%u\n", + info.address, info.size, linear_size, static_cast(cached.image->format), + static_cast(info.format), cached.image->extent.width, + cached.image->extent.height, info.width, info.height, info.bytes_per_element, + tiled, meta_overlap, static_cast(cached.kind)); + } + download.resize(info.size); + std::fill(download.begin(), download.end(), 0); + UtilFillBuffer(cached.ctx, download.data(), linear_size, info.pitch, cached.image, + cached.image->layout); + if (tiled) { + guest.resize(info.size); + cache.m_tiler.TileImage(guest.data(), download.data(), cached.target); + Libs::LibKernel::Memory::WriteBacking(info.address, guest.data(), info.size); + } else { + Libs::LibKernel::Memory::WriteBacking(info.address, download.data(), info.size); + } + return {info.address, info.size}; + } + + void Run() noexcept { + while (true) { + auto current = state.load(std::memory_order_acquire); + while (current == State::Idle) { + state.wait(current, std::memory_order_acquire); + current = state.load(std::memory_order_acquire); + } + if (current == State::Stopping) { + state.store(State::Stopped, std::memory_order_release); + state.notify_all(); + return; + } + if (current != State::Requested) { + EXIT("TextureCache: readback worker received unsupported state %u\n", + static_cast(current)); + } + + std::optional submissions; + if (!submissions_prepaused) { + submissions.emplace(); + } + FaultSafeTextureLock lock(&cache, cache.m_lock); + CachedImage* selected = cache.FindGpuReadbackPageCandidateLocked(vaddr, size); + const bool render_target = + selected != nullptr && selected->kind == CachedImage::Kind::RenderTarget; + const bool storage_texture = + selected != nullptr && selected->kind == CachedImage::Kind::StorageTexture; + const bool depth_target = + selected != nullptr && selected->kind == CachedImage::Kind::DepthTarget; + if ((!render_target && !storage_texture && !depth_target) || !selected->gpu_modified || + selected->buffer_modified || selected->ctx == nullptr || + selected->image == nullptr) { + EXIT("TextureCache: unsupported GPU image readback owner, addr=0x%016" PRIx64 + " size=0x%016" PRIx64 " access=%u image=%p kind=%u gpu_modified=%d " + "buffer_modified=%d ctx=%p vulkan_image=%p\n", + vaddr, size, static_cast(access), static_cast(selected), + selected != nullptr ? static_cast(selected->kind) : UINT32_MAX, + selected != nullptr && selected->gpu_modified, + selected != nullptr && selected->buffer_modified, + selected != nullptr ? static_cast(selected->ctx) : nullptr, + selected != nullptr ? static_cast(selected->image) : nullptr); + } + + const auto transfer = + depth_target ? DownloadDepthTarget(*selected) : DownloadColorImage(*selected); + + if (!cache.m_memory_tracker.IsRegionGpuModified(vaddr, size)) { + EXIT("TextureCache: readback fault page is not GPU-modified, addr=0x%016" PRIx64 + " size=0x%016" PRIx64 "\n", + vaddr, size); + } + const auto fault_page = vaddr & ~(TRACKER_PAGE_SIZE - 1); + const auto target_end = transfer.address + transfer.size; + const auto page_end = fault_page + TRACKER_PAGE_SIZE; + if (fault_page > transfer.address) { + cache.m_memory_tracker.ForEachDownloadRange( + transfer.address, fault_page - transfer.address, + [](uint64_t, uint64_t) noexcept {}); + } + if (page_end < target_end) { + cache.m_memory_tracker.ForEachDownloadRange( + page_end, target_end - page_end, [](uint64_t, uint64_t) noexcept {}); + } + + state.store(State::Ready, std::memory_order_release); + state.notify_all(); + while ((current = state.load(std::memory_order_acquire)) != State::Completed) { + if (current != State::Ready && current != State::Installed) { + EXIT("TextureCache: invalid image readback completion state %u\n", + static_cast(current)); + } + state.wait(current, std::memory_order_acquire); + } + if (cache.m_memory_tracker.IsRegionGpuModified(transfer.address, transfer.size)) { + EXIT("TextureCache: completed image readback retained GPU ownership, " + "addr=0x%016" PRIx64 " size=0x%016" PRIx64 "\n", + transfer.address, transfer.size); + } + selected->gpu_modified = false; + submissions_prepaused = false; + state.store(State::Idle, std::memory_order_release); + state.notify_all(); + } + } + + TextureCache& cache; + std::atomic state {State::Idle}; + PageFaultAccess access = PageFaultAccess::Unknown; + uint64_t vaddr = 0; + uint64_t size = 0; + bool submissions_prepaused = false; + std::vector download; + std::vector guest; + std::thread thread; +}; + +void TextureCache::RetireImages(const std::vector& retire) { + if (retire.empty()) { + return; + } + // FindTexture and FindRenderTarget retain cache records on their recording commands. Erasing + // the lookup owner therefore defers Vulkan destruction until the last referencing fence. + size_t removed = 0; + for (auto it = m_images.begin(); it != m_images.end();) { + if (std::find(retire.begin(), retire.end(), it->get()) == retire.end()) { + ++it; + continue; + } + const bool sampled = (*it)->kind == CachedImage::Kind::Texture; + const bool target = (*it)->kind == CachedImage::Kind::RenderTarget; + if ((!sampled && !target) || (*it)->gpu_modified || (target && (*it)->buffer_modified)) { + EXIT("TextureCache: invalid image retirement, kind=%u gpu_modified=%d " + "buffer_modified=%d\n", + static_cast((*it)->kind), (*it)->gpu_modified, (*it)->buffer_modified); + } + for (uint32_t range = 0; range < (*it)->RangeCount(); range++) { + if (target && + m_memory_tracker.IsRegionGpuModified((*it)->Address(range), (*it)->Size(range))) { + EXIT("TextureCache: clean target retirement retained tracker GPU ownership, " + "addr=0x%016" PRIx64 " size=0x%016" PRIx64 "\n", + (*it)->Address(range), (*it)->Size(range)); + } + m_memory_tracker.UntrackMemory((*it)->Address(range), (*it)->Size(range)); + } + it = m_images.erase(it); + removed++; + } + if (removed != retire.size()) { + EXIT("TextureCache: image retirement set mismatch, requested=%zu removed=%zu\n", + retire.size(), removed); + } +} + +namespace { + +bool Equal(const ImageInfo& left, const ImageInfo& right) { + return left.address == right.address && left.size == right.size && + left.format == right.format && left.width == right.width && + left.height == right.height && left.pitch == right.pitch && + left.base_level == right.base_level && left.levels == right.levels && + left.view_levels == right.view_levels && left.tile == right.tile && + left.swizzle == right.swizzle && left.depth == right.depth && left.type == right.type && + left.base_array == right.base_array; +} + +bool Equal(const RenderTargetInfo& left, const RenderTargetInfo& right) { + return left.address == right.address && left.size == right.size && + left.format == right.format && left.width == right.width && + left.height == right.height && left.pitch == right.pitch && + left.bytes_per_element == right.bytes_per_element && left.tile_mode == right.tile_mode && + left.levels == right.levels; +} + +bool Equal(const DepthTargetInfo& left, const DepthTargetInfo& right) { + return left.address == right.address && left.size == right.size && + left.stencil_address == right.stencil_address && + left.stencil_size == right.stencil_size && left.htile_address == right.htile_address && + left.htile_size == right.htile_size && left.format == right.format && + left.guest_format == right.guest_format && left.width == right.width && + left.height == right.height && left.pitch == right.pitch && + left.bytes_per_element == right.bytes_per_element && left.tile_mode == right.tile_mode; +} + +[[nodiscard]] bool IsCoherentGuestImageSource(const BufferImageCopySource& source, + uint64_t address, uint64_t size) { + // The current PS5 Tiler consumes coherent guest backing directly. A native buffer is an + // optional future GPU-detiler source or staging fallback. + return source.cpu_current && source.address == address && source.size == size && + (source.buffer != nullptr || source.offset == 0); +} + +VkImageAspectFlags DepthAspectMask(VkFormat format) { + return VK_IMAGE_ASPECT_DEPTH_BIT | + (format == VK_FORMAT_D16_UNORM_S8_UINT || format == VK_FORMAT_D24_UNORM_S8_UINT || + format == VK_FORMAT_D32_SFLOAT_S8_UINT + ? VK_IMAGE_ASPECT_STENCIL_BIT + : 0u); +} + +TextureImageCreateParams MakeImageParams(const ImageInfo& info, bool storage) { + TextureImageCreateParams params {}; + params.fmt = info.format; + params.width = info.width; + params.height = info.height; + params.base_level = info.base_level; + params.levels = info.levels; + params.depth = info.depth; + params.type = info.type; + // Storage image views use identity component mapping. The guest storage write mapping is + // validated before this point and intentionally does not become a Vulkan view swizzle. + params.swizzle = storage ? DstSel(4, 5, 6, 7) : info.swizzle; + params.format_usage = TextureFormatUsage::Sampled | TextureFormatUsage::Storage; + params.required_format_usage = storage + ? TextureFormatUsage::Sampled | TextureFormatUsage::Storage + : TextureFormatUsage::Sampled; + params.view_usage = storage ? TextureFormatUsage::Sampled | TextureFormatUsage::Storage + : TextureFormatUsage::Sampled; + params.image_layout = TextureUploadDestination::MipLevels; + params.allow_cube_view = !storage; + params.compatible_format_views = + storage && IsRgba8SrgbViewFormat(TextureGetFormat(info.format, params.format_usage)); + params.owner = storage ? "StorageTextureCache" : "TextureCache"; + return params; +} + +bool FormatSupportsStorage(GraphicContext* ctx, VkFormat format) { + VkFormatProperties properties {}; + vkGetPhysicalDeviceFormatProperties(ctx->physical_device, format, &properties); + return (properties.optimalTilingFeatures & VK_FORMAT_FEATURE_STORAGE_IMAGE_BIT) != 0; +} + +bool RenderTargetSupportsStorage(GraphicContext* ctx, VkFormat format, VkImageCreateFlags flags) { + const auto compatible = BgraSrgbStorageViewFormat(format); + const auto required_flags = + VK_IMAGE_CREATE_MUTABLE_FORMAT_BIT | VK_IMAGE_CREATE_EXTENDED_USAGE_BIT; + const bool compatible_views = (flags & required_flags) == required_flags; + return FormatSupportsStorage(ctx, format) || + (compatible_views && compatible != VK_FORMAT_UNDEFINED && + FormatSupportsStorage(ctx, compatible)); +} + +VkImageCreateFlags RenderTargetCreateFlags(VkFormat format) { + return IsRgba8SrgbViewFormat(format) + ? VK_IMAGE_CREATE_MUTABLE_FORMAT_BIT | VK_IMAGE_CREATE_EXTENDED_USAGE_BIT + : VkImageCreateFlags {0}; +} + +VkImageUsageFlags RenderTargetUsage(GraphicContext* ctx, VkFormat format, + VkImageCreateFlags flags) { + auto usage = static_cast(VK_IMAGE_USAGE_COLOR_ATTACHMENT_BIT) | + static_cast(VK_IMAGE_USAGE_TRANSFER_SRC_BIT) | + static_cast(VK_IMAGE_USAGE_TRANSFER_DST_BIT) | + static_cast(VK_IMAGE_USAGE_SAMPLED_BIT); + if (RenderTargetSupportsStorage(ctx, format, flags)) { + usage |= VK_IMAGE_USAGE_STORAGE_BIT; + } + VkImageFormatProperties properties {}; + if (vkGetPhysicalDeviceImageFormatProperties(ctx->physical_device, format, VK_IMAGE_TYPE_2D, + VK_IMAGE_TILING_OPTIMAL, usage, flags, + &properties) != VK_SUCCESS) { + EXIT("TextureCache: render-target format does not support required usage, format=%d " + "usage=0x%x\n", + static_cast(format), usage); + } + return usage; +} + +void CreateRenderTargetView(GraphicContext* ctx, VulkanImage* image, int index, + VkComponentSwizzle r, VkComponentSwizzle g, VkComponentSwizzle b, + VkComponentSwizzle a, VkImageViewType type = VK_IMAGE_VIEW_TYPE_2D, + VkFormat view_format = VK_FORMAT_UNDEFINED, + VkImageUsageFlags view_usage = 0, uint32_t level_count = 0) { + UtilCreateImageView(ctx, image, index, type, VK_IMAGE_ASPECT_COLOR_BIT, {r, g, b, a}, 0, 0, 1, + level_count == 0 ? image->mip_levels : level_count, view_format, + view_usage); +} + +void CreateRenderAttachmentViews(GraphicContext* ctx, RenderTextureVulkanImage* image) { + for (uint32_t level = 0; level < image->mip_levels; level++) { + VkImageViewCreateInfo create {}; + create.sType = VK_STRUCTURE_TYPE_IMAGE_VIEW_CREATE_INFO; + create.image = image->image; + create.viewType = VK_IMAGE_VIEW_TYPE_2D; + create.format = image->format; + create.components = {VK_COMPONENT_SWIZZLE_IDENTITY, VK_COMPONENT_SWIZZLE_IDENTITY, + VK_COMPONENT_SWIZZLE_IDENTITY, VK_COMPONENT_SWIZZLE_IDENTITY}; + create.subresourceRange.aspectMask = VK_IMAGE_ASPECT_COLOR_BIT; + create.subresourceRange.baseMipLevel = level; + create.subresourceRange.levelCount = 1; + create.subresourceRange.baseArrayLayer = 0; + create.subresourceRange.layerCount = 1; + const auto result = + vkCreateImageView(ctx->device, &create, nullptr, &image->render_view[level]); + if (result != VK_SUCCESS || image->render_view[level] == nullptr) { + EXIT("TextureCache: failed to create render-target mip view, result=%d level=%u\n", + static_cast(result), level); + } + } +} + +void CreateRenderTargetViews(GraphicContext* ctx, RenderTextureVulkanImage* image) { + CreateRenderTargetView(ctx, image, VulkanImage::VIEW_DEFAULT, VK_COMPONENT_SWIZZLE_IDENTITY, + VK_COMPONENT_SWIZZLE_IDENTITY, VK_COMPONENT_SWIZZLE_IDENTITY, + VK_COMPONENT_SWIZZLE_IDENTITY); + CreateRenderTargetView(ctx, image, VulkanImage::VIEW_BGRA, VK_COMPONENT_SWIZZLE_B, + VK_COMPONENT_SWIZZLE_G, VK_COMPONENT_SWIZZLE_R, + VK_COMPONENT_SWIZZLE_IDENTITY); + CreateRenderTargetView(ctx, image, VulkanImage::VIEW_R001, VK_COMPONENT_SWIZZLE_R, + VK_COMPONENT_SWIZZLE_ZERO, VK_COMPONENT_SWIZZLE_ZERO, + VK_COMPONENT_SWIZZLE_ONE); + CreateRenderTargetView(ctx, image, VulkanImage::VIEW_RGB1, VK_COMPONENT_SWIZZLE_R, + VK_COMPONENT_SWIZZLE_G, VK_COMPONENT_SWIZZLE_B, + VK_COMPONENT_SWIZZLE_ONE); + CreateRenderTargetView(ctx, image, VulkanImage::VIEW_R000, VK_COMPONENT_SWIZZLE_R, + VK_COMPONENT_SWIZZLE_ZERO, VK_COMPONENT_SWIZZLE_ZERO, + VK_COMPONENT_SWIZZLE_ZERO); + CreateRenderTargetView(ctx, image, VulkanImage::VIEW_RG01, VK_COMPONENT_SWIZZLE_R, + VK_COMPONENT_SWIZZLE_G, VK_COMPONENT_SWIZZLE_ZERO, + VK_COMPONENT_SWIZZLE_ONE); + CreateRenderTargetView(ctx, image, VulkanImage::VIEW_000R, VK_COMPONENT_SWIZZLE_ZERO, + VK_COMPONENT_SWIZZLE_ZERO, VK_COMPONENT_SWIZZLE_ZERO, + VK_COMPONENT_SWIZZLE_R); + if (FormatSupportsStorage(ctx, image->format)) { + CreateRenderTargetView(ctx, image, VulkanImage::VIEW_STORAGE, VK_COMPONENT_SWIZZLE_IDENTITY, + VK_COMPONENT_SWIZZLE_IDENTITY, VK_COMPONENT_SWIZZLE_IDENTITY, + VK_COMPONENT_SWIZZLE_IDENTITY, VK_IMAGE_VIEW_TYPE_2D, + VK_FORMAT_UNDEFINED, 0, 1); + } + const auto rgba_view_format = BgraToRgbaSampledViewFormat(image->format); + if (rgba_view_format != VK_FORMAT_UNDEFINED) { + CreateRenderTargetView(ctx, image, VulkanImage::VIEW_RGBA8_BGRA, VK_COMPONENT_SWIZZLE_B, + VK_COMPONENT_SWIZZLE_G, VK_COMPONENT_SWIZZLE_R, + VK_COMPONENT_SWIZZLE_A, VK_IMAGE_VIEW_TYPE_2D, rgba_view_format, + VK_IMAGE_USAGE_SAMPLED_BIT); + } + CreateRenderAttachmentViews(ctx, image); +} + +void CreateVideoOutViews(GraphicContext* ctx, VideoOutVulkanImage* image) { + CreateRenderTargetView(ctx, image, VulkanImage::VIEW_DEFAULT, VK_COMPONENT_SWIZZLE_IDENTITY, + VK_COMPONENT_SWIZZLE_IDENTITY, VK_COMPONENT_SWIZZLE_IDENTITY, + VK_COMPONENT_SWIZZLE_IDENTITY); + CreateRenderTargetView(ctx, image, VulkanImage::VIEW_BGRA, VK_COMPONENT_SWIZZLE_B, + VK_COMPONENT_SWIZZLE_G, VK_COMPONENT_SWIZZLE_R, + VK_COMPONENT_SWIZZLE_IDENTITY); + CreateRenderTargetView(ctx, image, VulkanImage::VIEW_DEFAULT_ARRAY, + VK_COMPONENT_SWIZZLE_IDENTITY, VK_COMPONENT_SWIZZLE_IDENTITY, + VK_COMPONENT_SWIZZLE_IDENTITY, VK_COMPONENT_SWIZZLE_IDENTITY, + VK_IMAGE_VIEW_TYPE_2D_ARRAY); + CreateRenderTargetView(ctx, image, VulkanImage::VIEW_BGRA_ARRAY, VK_COMPONENT_SWIZZLE_B, + VK_COMPONENT_SWIZZLE_G, VK_COMPONENT_SWIZZLE_R, + VK_COMPONENT_SWIZZLE_IDENTITY, VK_IMAGE_VIEW_TYPE_2D_ARRAY); +} + +void UploadRenderTarget(GraphicContext* ctx, RenderTextureVulkanImage* image, + const RenderTargetInfo& info, bool refresh) { + if (refresh) { + VulkanDeviceWaitIdle(ctx); + } + image->layout = VK_IMAGE_LAYOUT_UNDEFINED; + if (info.levels > 1) { + uint32_t format = 0; + switch (info.bytes_per_element) { + case 1: format = Prospero::GpuEnumValue(Prospero::BufferFormat::k8UNorm); break; + case 2: format = Prospero::GpuEnumValue(Prospero::BufferFormat::k16UNorm); break; + case 4: format = Prospero::GpuEnumValue(Prospero::BufferFormat::k32Float); break; + case 8: format = Prospero::GpuEnumValue(Prospero::BufferFormat::k16_16_16_16Float); break; + case 16: format = Prospero::GpuEnumValue(Prospero::BufferFormat::k32_32_32_32Float); break; + default: + EXIT("TextureCache: unsupported render-target mip element size: %u\n", + info.bytes_per_element); + } + auto layout = TextureCalcUploadLayout(format, info.width, info.height, info.levels, 1, + info.pitch, info.tile_mode, info.size, false, false, + false, "TextureCache render target"); + if (!layout.fmt_tiled_render_target || layout.pitch != info.pitch) { + EXIT("TextureCache: unsupported render-target mip upload layout, pitch=%u/%u tile=%u\n", + info.pitch, layout.pitch, info.tile_mode); + } + auto regions = TextureBuildUploadRegions( + layout, info.format, info.width, info.height, 1, info.levels, false, false, + TextureUploadDestination::MipLevels, TextureUploadSliceLayout::MipChainPerSlice); + TextureUploadGuestImage(ctx, image, reinterpret_cast(info.address), info.size, + regions, layout, format, info.width, info.height, 1, info.levels, + TextureUploadSliceLayout::MipChainPerSlice, + "TextureCache render target", + static_cast(VK_IMAGE_LAYOUT_GENERAL)); + return; + } + if (info.tile_mode == Prospero::GpuEnumValue(Prospero::TileMode::kRenderTarget) && + UtilBufferIsTiled(info.address, info.size)) { + UtilScratchBuffer scratch(info.size); + TileConvertTiledToLinearRenderTarget( + scratch.Data(), reinterpret_cast(info.address), info.width, info.height, + info.pitch, info.bytes_per_element, info.size); + UtilFillImage(ctx, image, scratch.Data(), info.size, info.pitch, + static_cast(VK_IMAGE_LAYOUT_GENERAL)); + } else { + UtilFillImage(ctx, image, reinterpret_cast(info.address), info.size, + info.pitch, static_cast(VK_IMAGE_LAYOUT_GENERAL)); + } +} + +RenderTextureVulkanImage* CreateRenderTarget(GraphicContext* ctx, const RenderTargetInfo& info, + VulkanMemory* memory) { + auto* image = new RenderTextureVulkanImage; + image->extent.width = info.width; + image->extent.height = info.height; + image->format = info.format; + image->mip_levels = info.levels; + image->layout = VK_IMAGE_LAYOUT_UNDEFINED; + UtilResetImageViews(image); + VkImageCreateInfo create {}; + create.sType = VK_STRUCTURE_TYPE_IMAGE_CREATE_INFO; + create.flags = RenderTargetCreateFlags(info.format); + create.imageType = VK_IMAGE_TYPE_2D; + create.extent = {info.width, info.height, 1}; + create.mipLevels = info.levels; + create.arrayLayers = 1; + create.format = info.format; + create.tiling = VK_IMAGE_TILING_OPTIMAL; + create.initialLayout = VK_IMAGE_LAYOUT_UNDEFINED; + create.usage = RenderTargetUsage(ctx, info.format, create.flags); + create.sharingMode = VK_SHARING_MODE_EXCLUSIVE; + create.samples = VK_SAMPLE_COUNT_1_BIT; + memory->property = VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT; + if (!VulkanCreateImage(ctx, &create, image, memory)) { + EXIT("TextureCache: failed to create render target, addr=0x%016" PRIx64 + " extent=%ux%u format=%d\n", + info.address, info.width, info.height, static_cast(info.format)); + } + image->memory = *memory; + CreateRenderTargetViews(ctx, image); + return image; +} + +void CreateDepthViews(GraphicContext* ctx, DepthStencilVulkanImage* image) { + UtilCreateImageView(ctx, image, VulkanImage::VIEW_DEFAULT, VK_IMAGE_VIEW_TYPE_2D, + DepthAspectMask(image->format), + {VK_COMPONENT_SWIZZLE_IDENTITY, VK_COMPONENT_SWIZZLE_IDENTITY, + VK_COMPONENT_SWIZZLE_IDENTITY, VK_COMPONENT_SWIZZLE_IDENTITY}, + 0, 0, 1, 1); + UtilCreateImageView(ctx, image, VulkanImage::VIEW_DEPTH_TEXTURE, VK_IMAGE_VIEW_TYPE_2D, + VK_IMAGE_ASPECT_DEPTH_BIT, + {VK_COMPONENT_SWIZZLE_R, VK_COMPONENT_SWIZZLE_R, VK_COMPONENT_SWIZZLE_R, + VK_COMPONENT_SWIZZLE_R}, + 0, 0, 1, 1); + UtilCreateImageView(ctx, image, VulkanImage::VIEW_R000, VK_IMAGE_VIEW_TYPE_2D, + VK_IMAGE_ASPECT_DEPTH_BIT, + {VK_COMPONENT_SWIZZLE_R, VK_COMPONENT_SWIZZLE_ZERO, + VK_COMPONENT_SWIZZLE_ZERO, VK_COMPONENT_SWIZZLE_ZERO}, + 0, 0, 1, 1); +} + +DepthStencilVulkanImage* CreateDepthTarget(GraphicContext* ctx, const DepthTargetInfo& info, + VulkanMemory* memory) { + VkImageCreateInfo create {}; + create.sType = VK_STRUCTURE_TYPE_IMAGE_CREATE_INFO; + create.imageType = VK_IMAGE_TYPE_2D; + create.extent = {info.width, info.height, 1}; + create.mipLevels = 1; + create.arrayLayers = 1; + create.format = info.format; + create.tiling = VK_IMAGE_TILING_OPTIMAL; + create.initialLayout = VK_IMAGE_LAYOUT_UNDEFINED; + create.usage = VK_IMAGE_USAGE_DEPTH_STENCIL_ATTACHMENT_BIT | VK_IMAGE_USAGE_SAMPLED_BIT | + VK_IMAGE_USAGE_TRANSFER_SRC_BIT | VK_IMAGE_USAGE_TRANSFER_DST_BIT; + create.sharingMode = VK_SHARING_MODE_EXCLUSIVE; + create.samples = VK_SAMPLE_COUNT_1_BIT; + VkImageFormatProperties properties {}; + if (vkGetPhysicalDeviceImageFormatProperties(ctx->physical_device, info.format, + VK_IMAGE_TYPE_2D, VK_IMAGE_TILING_OPTIMAL, + create.usage, 0, &properties) != VK_SUCCESS) { + EXIT("TextureCache: depth format does not support required usage, format=%d usage=0x%x\n", + static_cast(info.format), create.usage); + } + auto* image = new DepthStencilVulkanImage; + image->extent.width = info.width; + image->extent.height = info.height; + image->guest_pitch = info.pitch; + image->format = info.format; + image->layout = VK_IMAGE_LAYOUT_UNDEFINED; + image->compressed = false; + UtilResetImageViews(image); + memory->property = VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT; + if (!VulkanCreateImage(ctx, &create, image, memory)) { + EXIT("TextureCache: failed to create depth target, addr=0x%016" PRIx64 + " extent=%ux%u format=%d\n", + info.address, info.width, info.height, static_cast(info.format)); + } + image->memory = *memory; + CreateDepthViews(ctx, image); + return image; +} + +void ValidateVideoOutInfo(GraphicContext* ctx, const VideoOutInfo& info) { + const auto compression = + ClassifyVideoOutCompression(info.compression != VideoOutCompression::Uncompressed, + info.metadata_address, info.dcc_control, 0); + const bool metadata_invalid = compression == VideoOutCompression::Dcc256_64_64 && + (info.metadata_address >= TRACKER_ADDRESS_SIZE || + (info.metadata_address >= info.address && + info.metadata_address < info.address + info.size)); + if (ctx == nullptr || info.address == 0 || info.size == 0 || + info.address >= TRACKER_ADDRESS_SIZE || info.size > TRACKER_ADDRESS_SIZE - info.address || + (info.address & 0xffffu) != 0 || info.width == 0 || info.height == 0 || + info.width > 16384 || info.height > 16384 || info.pitch < info.width || + info.tile_mode != Prospero::GpuEnumValue(Prospero::TileMode::kRenderTarget) || info.bytes_per_element != 4 || + compression == VideoOutCompression::Unsupported || compression != info.compression || + metadata_invalid || !IsSupportedVideoOutFormat(info.guest_format, info.format)) { + EXIT("TextureCache: unsupported video-out surface, ctx=%p addr=0x%016" PRIx64 + " size=0x%016" PRIx64 " metadata=0x%016" PRIx64 " dcc=0x%08" PRIx32 + " extent=%ux%u pitch=%u tile=%u guest_format=%u bpe=%u vk_format=%d\n", + static_cast(ctx), info.address, info.size, info.metadata_address, + info.dcc_control, info.width, info.height, info.pitch, info.tile_mode, + info.guest_format, info.bytes_per_element, static_cast(info.format)); + } + TileSizeAlign exact {}; + TileGetTextureTotalSize(info.guest_format, info.width, info.height, 1, info.pitch, 1, + info.tile_mode, false, &exact); + if (exact.align != 65536 || exact.size != info.size || + TileGetTexturePitch(info.guest_format, info.width, 1, info.tile_mode) != info.pitch) { + EXIT("TextureCache: video-out tile layout mismatch, addr=0x%016" PRIx64 + " size=0x%016" PRIx64 " expected_size=0x%016" PRIx64 " align=0x%016" PRIx64 + " pitch=%u\n", + info.address, info.size, exact.size, exact.align, info.pitch); + } + (void)RenderTargetUsage(ctx, info.format, 0); +} + +VideoOutVulkanImage* CreateVideoOut(GraphicContext* ctx, const VideoOutInfo& info, + VulkanMemory* memory) { + auto* image = new VideoOutVulkanImage; + image->extent.width = info.width; + image->extent.height = info.height; + image->format = info.format; + image->layout = VK_IMAGE_LAYOUT_UNDEFINED; + UtilResetImageViews(image); + VkImageCreateInfo create {}; + create.sType = VK_STRUCTURE_TYPE_IMAGE_CREATE_INFO; + create.imageType = VK_IMAGE_TYPE_2D; + create.extent = {info.width, info.height, 1}; + create.mipLevels = 1; + create.arrayLayers = 1; + create.format = info.format; + create.tiling = VK_IMAGE_TILING_OPTIMAL; + create.initialLayout = VK_IMAGE_LAYOUT_UNDEFINED; + create.usage = RenderTargetUsage(ctx, info.format, 0); + create.sharingMode = VK_SHARING_MODE_EXCLUSIVE; + create.samples = VK_SAMPLE_COUNT_1_BIT; + memory->property = VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT; + if (!VulkanCreateImage(ctx, &create, image, memory)) { + EXIT("TextureCache: failed to create video-out image, addr=0x%016" PRIx64 + " extent=%ux%u format=%d\n", + info.address, info.width, info.height, static_cast(info.format)); + } + image->memory = *memory; + CreateVideoOutViews(ctx, image); + return image; +} + +void UploadVideoOut(GraphicContext* ctx, VideoOutVulkanImage* image, const VideoOutInfo& info, + bool refresh) { + if (info.compression != VideoOutCompression::Uncompressed) { + EXIT("TextureCache: compressed video-out guest upload is unsupported, " + "addr=0x%016" PRIx64 " metadata=0x%016" PRIx64 " dcc=0x%08" PRIx32 "\n", + info.address, info.metadata_address, info.dcc_control); + } + if (refresh) { + VulkanDeviceWaitIdle(ctx); + } + image->layout = VK_IMAGE_LAYOUT_UNDEFINED; + UtilScratchBuffer scratch(info.size); + TileConvertTiledToLinearRenderTarget( + scratch.Data(), reinterpret_cast(info.address), info.width, info.height, + info.pitch, info.bytes_per_element, info.size); + UtilFillImage(ctx, image, scratch.Data(), info.size, info.pitch, + static_cast(VK_IMAGE_LAYOUT_GENERAL)); +} + +} // namespace + +TextureCache::TextureCache(PageManager& page_manager, BufferCache& buffer_cache, + ResourceMutex& resource_mutex) + : m_memory_tracker(page_manager), m_metadata_tracker(page_manager, PageWatchMode::Write), + m_buffer_cache(buffer_cache), m_resource_mutex(resource_mutex) { + if (!Common::Thread::IsMainThread()) { + EXIT("TextureCache: construction is restricted to the main thread\n"); + } + m_readback = std::make_unique(*this); +} + +VkImageView TextureCache::GetRenderTargetAttachmentView(GraphicContext* ctx, + RenderTextureVulkanImage* image, + VkFormat format, uint32_t level) { + if (ctx == nullptr || image == nullptr || image->image == nullptr || + format == VK_FORMAT_UNDEFINED || level >= image->mip_levels || level >= 16) { + EXIT("TextureCache: invalid render-target attachment view, image=%p format=%d" + " level=%u image_levels=%u\n", + static_cast(image), static_cast(format), level, + image != nullptr ? image->mip_levels : 0); + } + if (format == image->format) { + if (image->render_view[level] == nullptr) { + EXIT("TextureCache: base render-target attachment view is missing, level=%u\n", level); + } + return image->render_view[level]; + } + if (!IsRgba8SrgbReinterpretation(image->format, format)) { + EXIT("TextureCache: incompatible render-target attachment view, image_format=%d" + " view_format=%d level=%u\n", + static_cast(image->format), static_cast(format), level); + } + + std::lock_guard lock(image->attachment_view_mutex); + for (const auto& cached: image->attachment_views) { + if (cached.format == format && cached.level == level) { + return cached.view; + } + } + + VkImageViewUsageCreateInfo usage {}; + usage.sType = VK_STRUCTURE_TYPE_IMAGE_VIEW_USAGE_CREATE_INFO; + usage.usage = VK_IMAGE_USAGE_COLOR_ATTACHMENT_BIT; + VkImageViewCreateInfo create {}; + create.sType = VK_STRUCTURE_TYPE_IMAGE_VIEW_CREATE_INFO; + create.pNext = &usage; + create.image = image->image; + create.viewType = VK_IMAGE_VIEW_TYPE_2D; + create.format = format; + create.components = {VK_COMPONENT_SWIZZLE_IDENTITY, VK_COMPONENT_SWIZZLE_IDENTITY, + VK_COMPONENT_SWIZZLE_IDENTITY, VK_COMPONENT_SWIZZLE_IDENTITY}; + create.subresourceRange.aspectMask = VK_IMAGE_ASPECT_COLOR_BIT; + create.subresourceRange.baseMipLevel = level; + create.subresourceRange.levelCount = 1; + create.subresourceRange.baseArrayLayer = 0; + create.subresourceRange.layerCount = 1; + VkImageView view = nullptr; + const auto result = vkCreateImageView(ctx->device, &create, nullptr, &view); + if (result != VK_SUCCESS || view == nullptr) { + EXIT("TextureCache: failed to create compatible render-target attachment view," + " result=%d image_format=%d view_format=%d level=%u\n", + static_cast(result), static_cast(image->format), static_cast(format), + level); + } + image->attachment_views.push_back({format, level, view}); + LOGF("TextureCache: created compatible render-target attachment view:" + " image_format=%d view_format=%d level=%u extent=%ux%u\n", + static_cast(image->format), static_cast(format), level, image->extent.width, + image->extent.height); + return view; +} + +VkImageView TextureCache::GetRenderTargetSampledView(GraphicContext* ctx, + RenderTextureVulkanImage* image, + VkFormat view_format, int variant, + uint32_t base_level, uint32_t level_count) { + if (ctx == nullptr || image == nullptr || image->image == nullptr || base_level >= 16 || + view_format == VK_FORMAT_UNDEFINED || level_count == 0 || + base_level + level_count > image->mip_levels) { + EXIT("TextureCache: invalid render-target sampled view, image=%p variant=%d" + " view_format=%d base=%u count=%u image_levels=%u\n", + static_cast(image), variant, static_cast(view_format), base_level, + level_count, image != nullptr ? image->mip_levels : 0); + } + const bool compatible_variant_format = + view_format == image->format || (variant == VulkanImage::VIEW_RGBA8_BGRA && + IsBgraToRgba8SampledView(image->format, view_format)); + if (!compatible_variant_format) { + EXIT("TextureCache: incompatible render-target sampled view format, image_format=%d" + " view_format=%d variant=%d\n", + static_cast(image->format), static_cast(view_format), variant); + } + const auto default_view_format = variant == VulkanImage::VIEW_RGBA8_BGRA + ? BgraToRgbaSampledViewFormat(image->format) + : image->format; + if (view_format == default_view_format && base_level == 0 && level_count == image->mip_levels) { + if (variant < 0 || variant >= VulkanImage::VIEW_MAX || + image->image_view[variant] == nullptr) { + EXIT("TextureCache: full render-target sampled view is missing, variant=%d\n", variant); + } + return image->image_view[variant]; + } + + std::lock_guard lock(image->sampled_view_mutex); + for (const auto& cached: image->sampled_views) { + if (cached.format == view_format && cached.variant == variant && + cached.base_level == base_level && cached.level_count == level_count) { + return cached.view; + } + } + + VkComponentMapping components {VK_COMPONENT_SWIZZLE_IDENTITY, VK_COMPONENT_SWIZZLE_IDENTITY, + VK_COMPONENT_SWIZZLE_IDENTITY, VK_COMPONENT_SWIZZLE_IDENTITY}; + switch (variant) { + case VulkanImage::VIEW_DEFAULT: break; + case VulkanImage::VIEW_BGRA: + components = {VK_COMPONENT_SWIZZLE_B, VK_COMPONENT_SWIZZLE_G, VK_COMPONENT_SWIZZLE_R, + VK_COMPONENT_SWIZZLE_IDENTITY}; + break; + case VulkanImage::VIEW_R001: + components = {VK_COMPONENT_SWIZZLE_R, VK_COMPONENT_SWIZZLE_ZERO, + VK_COMPONENT_SWIZZLE_ZERO, VK_COMPONENT_SWIZZLE_ONE}; + break; + case VulkanImage::VIEW_RGB1: + components = {VK_COMPONENT_SWIZZLE_R, VK_COMPONENT_SWIZZLE_G, VK_COMPONENT_SWIZZLE_B, + VK_COMPONENT_SWIZZLE_ONE}; + break; + case VulkanImage::VIEW_R000: + components = {VK_COMPONENT_SWIZZLE_R, VK_COMPONENT_SWIZZLE_ZERO, + VK_COMPONENT_SWIZZLE_ZERO, VK_COMPONENT_SWIZZLE_ZERO}; + break; + case VulkanImage::VIEW_RG01: + components = {VK_COMPONENT_SWIZZLE_R, VK_COMPONENT_SWIZZLE_G, VK_COMPONENT_SWIZZLE_ZERO, + VK_COMPONENT_SWIZZLE_ONE}; + break; + case VulkanImage::VIEW_000R: + components = {VK_COMPONENT_SWIZZLE_ZERO, VK_COMPONENT_SWIZZLE_ZERO, + VK_COMPONENT_SWIZZLE_ZERO, VK_COMPONENT_SWIZZLE_R}; + break; + case VulkanImage::VIEW_RGBA8_BGRA: + if (!IsBgraToRgba8SampledView(image->format, view_format)) { + EXIT("TextureCache: incompatible mutable render-target sampled view\n"); + } + components = {VK_COMPONENT_SWIZZLE_B, VK_COMPONENT_SWIZZLE_G, VK_COMPONENT_SWIZZLE_R, + VK_COMPONENT_SWIZZLE_A}; + break; + default: + EXIT("TextureCache: unsupported render-target sampled view variant: %d\n", variant); + } + + VkImageViewCreateInfo create {}; + VkImageViewUsageCreateInfo usage {}; + usage.sType = VK_STRUCTURE_TYPE_IMAGE_VIEW_USAGE_CREATE_INFO; + usage.usage = VK_IMAGE_USAGE_SAMPLED_BIT; + create.sType = VK_STRUCTURE_TYPE_IMAGE_VIEW_CREATE_INFO; + create.pNext = &usage; + create.image = image->image; + create.viewType = VK_IMAGE_VIEW_TYPE_2D; + create.format = view_format; + create.components = components; + create.subresourceRange.aspectMask = VK_IMAGE_ASPECT_COLOR_BIT; + create.subresourceRange.baseMipLevel = base_level; + create.subresourceRange.levelCount = level_count; + create.subresourceRange.baseArrayLayer = 0; + create.subresourceRange.layerCount = 1; + VkImageView view = nullptr; + const auto result = vkCreateImageView(ctx->device, &create, nullptr, &view); + if (result != VK_SUCCESS || view == nullptr) { + EXIT("TextureCache: failed to create render-target sampled subresource view," + " result=%d view_format=%d variant=%d base=%u count=%u\n", + static_cast(result), static_cast(view_format), variant, base_level, + level_count); + } + image->sampled_views.push_back({view_format, base_level, level_count, variant, view}); + return view; +} + +VkImageView TextureCache::GetRenderTargetStorageView(GraphicContext* ctx, + RenderTextureVulkanImage* image, + VkFormat view_format, uint32_t base_level, + uint32_t level_count) { + if (ctx == nullptr || image == nullptr || image->image == nullptr || + view_format == VK_FORMAT_UNDEFINED || level_count == 0 || + base_level + level_count > image->mip_levels) { + EXIT("TextureCache: invalid render-target storage view, image=%p view_format=%d" + " base=%u count=%u image_levels=%u\n", + static_cast(image), static_cast(view_format), base_level, + level_count, image != nullptr ? image->mip_levels : 0); + } + const bool exact = view_format == image->format; + const bool compatible = view_format == BgraSrgbStorageViewFormat(image->format); + if (!exact && !compatible) { + EXIT("TextureCache: incompatible render-target storage view, image_format=%d" + " view_format=%d base=%u count=%u\n", + static_cast(image->format), static_cast(view_format), base_level, + level_count); + } + if (exact) { + if (image->image_view[VulkanImage::VIEW_STORAGE] == nullptr) { + EXIT("TextureCache: base render-target storage view is missing\n"); + } + if (base_level == 0 && level_count == 1) { + return image->image_view[VulkanImage::VIEW_STORAGE]; + } + } + + std::lock_guard lock(image->storage_view_mutex); + for (const auto& cached: image->storage_views) { + if (cached.format == view_format && cached.base_level == base_level && + cached.level_count == level_count) { + return cached.view; + } + } + if (compatible && !FormatSupportsStorage(ctx, view_format)) { + EXIT("TextureCache: compatible render-target storage format lacks storage support," + " image_format=%d view_format=%d base=%u count=%u\n", + static_cast(image->format), static_cast(view_format), base_level, + level_count); + } + + VkImageViewUsageCreateInfo usage {}; + usage.sType = VK_STRUCTURE_TYPE_IMAGE_VIEW_USAGE_CREATE_INFO; + usage.usage = VK_IMAGE_USAGE_STORAGE_BIT; + VkImageViewCreateInfo create {}; + create.sType = VK_STRUCTURE_TYPE_IMAGE_VIEW_CREATE_INFO; + create.pNext = &usage; + create.image = image->image; + create.viewType = VK_IMAGE_VIEW_TYPE_2D; + create.format = view_format; + create.components = {VK_COMPONENT_SWIZZLE_IDENTITY, VK_COMPONENT_SWIZZLE_IDENTITY, + VK_COMPONENT_SWIZZLE_IDENTITY, VK_COMPONENT_SWIZZLE_IDENTITY}; + create.subresourceRange.aspectMask = VK_IMAGE_ASPECT_COLOR_BIT; + create.subresourceRange.baseMipLevel = base_level; + create.subresourceRange.levelCount = level_count; + create.subresourceRange.baseArrayLayer = 0; + create.subresourceRange.layerCount = 1; + VkImageView view = nullptr; + const auto result = vkCreateImageView(ctx->device, &create, nullptr, &view); + if (result != VK_SUCCESS || view == nullptr) { + EXIT("TextureCache: failed to create compatible render-target storage view," + " result=%d image_format=%d view_format=%d base=%u count=%u\n", + static_cast(result), static_cast(image->format), + static_cast(view_format), base_level, level_count); + } + image->storage_views.push_back({view_format, base_level, level_count, view}); + LOGF("TextureCache: created compatible render-target storage view: image_format=%d" + " view_format=%d base=%u count=%u extent=%ux%u\n", + static_cast(image->format), static_cast(view_format), base_level, level_count, + image->extent.width, image->extent.height); + return view; +} + +VkImageView TextureCache::GetStorageTextureSampledView(GraphicContext* ctx, + StorageTextureVulkanImage* image, + const ImageInfo& info) { + const bool image_2d = info.type == Prospero::GpuEnumValue(Prospero::ImageType::kColor2D) && info.depth == 1; + const bool image_3d = info.type == Prospero::GpuEnumValue(Prospero::ImageType::kColor3D) && info.depth != 0; + if (ctx == nullptr || image == nullptr || image->image == nullptr || (!image_2d && !image_3d) || + info.base_array != 0 || info.levels != image->mip_levels || + info.base_level >= info.levels || info.view_levels == 0 || + info.base_level + info.view_levels > info.levels) { + EXIT("TextureCache: invalid sampled view of storage texture, image=%p type=%u depth=%u" + " base=%u levels=%u view_levels=%u image_levels=%u base_array=%u\n", + static_cast(image), info.type, info.depth, info.base_level, info.levels, + info.view_levels, image != nullptr ? image->mip_levels : 0, info.base_array); + } + const auto view_format = TextureGetFormat(info.format, TextureFormatUsage::Sampled); + if (view_format != image->format && !IsRgba8SrgbReinterpretation(image->format, view_format)) { + EXIT("TextureCache: incompatible sampled view of storage texture, image_format=%d" + " view_format=%d swizzle=0x%03x\n", + static_cast(image->format), static_cast(view_format), info.swizzle); + } + + std::lock_guard lock(image->sampled_view_mutex); + for (const auto& cached: image->sampled_views) { + if (cached.format == view_format && cached.swizzle == info.swizzle && + cached.type == info.type && cached.base_level == info.base_level && + cached.level_count == info.view_levels) { + return cached.view; + } + } + + VkImageViewUsageCreateInfo usage {}; + usage.sType = VK_STRUCTURE_TYPE_IMAGE_VIEW_USAGE_CREATE_INFO; + usage.usage = VK_IMAGE_USAGE_SAMPLED_BIT; + VkImageViewCreateInfo create {}; + create.sType = VK_STRUCTURE_TYPE_IMAGE_VIEW_CREATE_INFO; + create.pNext = &usage; + create.image = image->image; + create.viewType = image_3d ? VK_IMAGE_VIEW_TYPE_3D : VK_IMAGE_VIEW_TYPE_2D; + create.format = view_format; + create.components = TextureGetComponentMapping(info.swizzle); + create.subresourceRange.aspectMask = VK_IMAGE_ASPECT_COLOR_BIT; + create.subresourceRange.baseMipLevel = info.base_level; + create.subresourceRange.levelCount = info.view_levels; + create.subresourceRange.baseArrayLayer = 0; + create.subresourceRange.layerCount = 1; + VkImageView view = nullptr; + const auto result = vkCreateImageView(ctx->device, &create, nullptr, &view); + if (result != VK_SUCCESS || view == nullptr) { + EXIT("TextureCache: failed to create sampled view of storage texture, result=%d" + " image_format=%d view_format=%d swizzle=0x%03x type=%u base=%u count=%u\n", + static_cast(result), static_cast(image->format), + static_cast(view_format), info.swizzle, info.type, info.base_level, + info.view_levels); + } + image->sampled_views.push_back( + {view_format, info.swizzle, info.type, info.base_level, info.view_levels, view}); + LOGF("TextureCache: created sampled view of storage texture: image_format=%d" + " view_format=%d swizzle=0x%03x type=%u base=%u count=%u\n", + static_cast(image->format), static_cast(view_format), info.swizzle, info.type, + info.base_level, info.view_levels); + return view; +} + +TextureCache::CachedImage* TextureCache::FindGpuReadbackPageCandidateLocked(uint64_t vaddr, + uint64_t size) const { + CachedImage* selected = nullptr; + for (const auto& cached: m_images) { + if (!cached->IsGpuReadbackPageCandidate(vaddr, size)) { + continue; + } + if (selected != nullptr) { + EXIT("TextureCache: CPU fault has multiple GPU-modified image page candidates, " + "addr=0x%016" PRIx64 " size=0x%016" PRIx64 " first=%p second=%p\n", + vaddr, size, static_cast(selected), + static_cast(cached.get())); + } + selected = cached.get(); + } + return selected; +} + +void TextureCache::MarkSampledAliasesCpuDirtyLocked(uint64_t vaddr, uint64_t size) { + for (auto& cached: m_images) { + if (cached->kind != CachedImage::Kind::Texture) { + continue; + } + if (cached->gpu_modified && + ImagePageRangesOverlap(cached->info.address, cached->info.size, vaddr, size)) { + EXIT("TextureCache: CPU write overlaps a GPU-modified sampled texture, " + "write=0x%016" PRIx64 "+0x%016" PRIx64 " image=0x%016" PRIx64 "+0x%016" PRIx64 + "\n", + vaddr, size, cached->info.address, cached->info.size); + } + cached->info.InvalidateCpuWrite(vaddr, size); + } +} + +TextureCache::~TextureCache() { + m_readback.reset(); + GraphicContext* ctx = nullptr; + if (!m_images.empty()) { + ctx = m_images.front()->ctx; + } else { + ctx = m_dummy_ctx; + } + if (ctx != nullptr) { + VulkanDeviceWaitIdle(ctx); + } + m_images.clear(); + for (size_t i = 0; i < m_dummy_sampled_textures.size(); i++) { + if (m_dummy_sampled_textures[i] != nullptr) { + DeleteGpuTexture(ctx, static_cast(m_dummy_sampled_textures[i]), + m_dummy_sampled_memory[i]); + delete m_dummy_sampled_memory[i]; + } + if (m_dummy_storage_textures[i] != nullptr) { + DeleteGpuTexture(ctx, static_cast(m_dummy_storage_textures[i]), + m_dummy_storage_memory[i]); + delete m_dummy_storage_memory[i]; + } + } +} + +VulkanImage* TextureCache::FindTexture(CommandBuffer* command, GraphicContext* ctx, + const ImageInfo& info, bool metadata_read) { + if (command == nullptr || ctx == nullptr || info.address == 0 || info.size == 0 || + info.address >= TRACKER_ADDRESS_SIZE || info.size > TRACKER_ADDRESS_SIZE - info.address || + info.width == 0 || info.height == 0 || info.depth == 0 || info.levels == 0 || + info.view_levels == 0 || info.base_level + info.view_levels > info.levels) { + EXIT("TextureCache: invalid sampled-image request, command=%p ctx=%p addr=0x%016" PRIx64 + " size=0x%016" PRIx64 " extent=%ux%ux%u levels=%u\n", + static_cast(command), static_cast(ctx), info.address, + info.size, info.width, info.height, info.depth, info.levels); + } + std::lock_guard transaction(m_resource_mutex); + BufferImageCopySource source {nullptr, 0, info.address, info.size, true}; + if (m_buffer_cache.HasPageOverlap(info.address, info.size)) { + // ObtainBufferForImage publishes dirty native-buffer bytes when necessary and otherwise + // uses a CPU-current staging fallback through guest backing. + source = m_buffer_cache.ObtainBufferForImage(info.address, info.size); + if (!IsCoherentGuestImageSource(source, info.address, info.size)) { + EXIT("TextureCache: sampled-image buffer source is inconsistent, addr=0x%016" PRIx64 + " size=0x%016" PRIx64 " source=%p source_addr=0x%016" PRIx64 + " source_size=0x%016" PRIx64 " current=%d\n", + info.address, info.size, static_cast(source.buffer), source.address, + source.size, source.cpu_current); + } + } + FaultSafeTextureLock lock(this, m_lock); + if (HasMetaOverlapLocked(info.address, info.size) != metadata_read) { + EXIT("TextureCache: sampled-image metadata classification mismatch, addr=0x%016" PRIx64 + " size=0x%016" PRIx64 " requested=%d\n", + info.address, info.size, metadata_read); + } + std::shared_ptr storage_match; + const auto requested_view_format = TextureGetFormat(info.format, TextureFormatUsage::Sampled); + for (auto& cached: m_images) { + if (cached->kind != CachedImage::Kind::StorageTexture) { + continue; + } + switch (ClassifyStorageSampledOverlap(info, cached->info, requested_view_format, + cached->image->format, cached->gpu_modified, + cached->info.IsCpuDirty(), cached->ctx == ctx)) { + case StorageSampledOverlap::None: break; + case StorageSampledOverlap::ExactImage: + if (storage_match != nullptr) { + EXIT("TextureCache: duplicate exact storage image for sampled binding, " + "addr=0x%016" PRIx64 " size=0x%016" PRIx64 "\n", + info.address, info.size); + } + storage_match = cached; + break; + case StorageSampledOverlap::Unsupported: + EXIT("TextureCache: unsupported sampled/storage image alias, " + "requested=0x%016" PRIx64 "+0x%016" PRIx64 " storage=0x%016" PRIx64 + "+0x%016" PRIx64 " gpu_modified=%d cpu_dirty=%d same_context=%d" + " requested_info={format=%u extent=%ux%ux%u pitch=%u base=%u levels=%u" + " view_levels=%u tile=%u swizzle=0x%03x type=%u base_array=%u}" + " storage_info={format=%u extent=%ux%ux%u pitch=%u base=%u levels=%u" + " view_levels=%u tile=%u swizzle=0x%03x type=%u base_array=%u}\n", + info.address, info.size, cached->info.address, cached->info.size, + cached->gpu_modified, cached->info.IsCpuDirty(), cached->ctx == ctx, + info.format, info.width, info.height, info.depth, info.pitch, info.base_level, + info.levels, info.view_levels, info.tile, info.swizzle, info.type, + info.base_array, cached->info.format, cached->info.width, cached->info.height, + cached->info.depth, cached->info.pitch, cached->info.base_level, + cached->info.levels, cached->info.view_levels, cached->info.tile, + cached->info.swizzle, cached->info.type, cached->info.base_array); + } + } + if (storage_match != nullptr) { + if (m_memory_tracker.IsRegionCpuModified(info.address, info.size) || + !m_memory_tracker.IsRegionGpuModified(info.address, info.size) || + storage_match->image->type != VulkanImageType::StorageTexture || + storage_match->image->image_view[VulkanImage::VIEW_DEFAULT] == nullptr) { + EXIT("TextureCache: sampled storage-image ownership or view is inconsistent, " + "addr=0x%016" PRIx64 " size=0x%016" PRIx64 "\n", + info.address, info.size); + } + command->RetainResourceUntilFence(storage_match); + return storage_match->image; + } + if (m_memory_tracker.IsRegionCpuModified(info.address, info.size)) { + MarkSampledAliasesCpuDirtyLocked(info.address, info.size); + } + std::shared_ptr match; + for (auto& cached: m_images) { + if (cached->kind != CachedImage::Kind::Texture || !Equal(info, cached->info)) { + continue; + } + if (match != nullptr || cached->gpu_modified || cached->ctx != ctx) { + EXIT("TextureCache: invalid exact sampled-image cache match, addr=0x%016" PRIx64 + " size=0x%016" PRIx64 " duplicate=%d gpu_modified=%d same_context=%d\n", + info.address, info.size, match != nullptr, cached->gpu_modified, + cached->ctx == ctx); + } + match = cached; + } + if (match != nullptr) { + const bool buffer_dirty = match->buffer_modified; + if (buffer_dirty) { + if (match->gpu_modified || + !IsCoherentGuestImageSource(source, info.address, info.size)) { + EXIT("TextureCache: sampled-image refresh has invalid buffer ownership, " + "addr=0x%016" PRIx64 " size=0x%016" PRIx64 + " source=%p current=%d gpu_modified=%d\n", + info.address, info.size, static_cast(source.buffer), + source.cpu_current, match->gpu_modified); + } + m_memory_tracker.MarkRegionAsCpuModified(info.address, info.size); + } + if (match->info.IsCpuDirty() || buffer_dirty) { + m_memory_tracker.ForEachUploadRange( + info.address, info.size, false, [](uint64_t, uint64_t) noexcept {}, + [&]() noexcept { + m_tiler.DetileImage(match->ctx, + static_cast(match->image), + match->info, source, true, false); + }); + if (match->info.IsCpuDirty()) { + match->info.RefreshComplete(); + } + match->buffer_modified = false; + } + command->RetainResourceUntilFence(match); + return match->image; + } + for (const auto& cached: m_images) { + if (cached->kind != CachedImage::Kind::Texture) { + if (!cached->OverlapsRange(info.address, info.size, true)) { + continue; + } + EXIT("TextureCache: sampled image shares pages with GPU target, addr=0x%016" PRIx64 + " size=0x%016" PRIx64 " target_kind=%u\n", + info.address, info.size, static_cast(cached->kind)); + } + const auto overlap = + ClassifySampledOverlap(info, cached->info, cached->gpu_modified, cached->ctx == ctx); + if (overlap == SampledOverlap::Unsupported) { + EXIT("TextureCache: unsupported sampled-texture alias, requested=0x%016" PRIx64 + "+0x%016" PRIx64 " existing=0x%016" PRIx64 "+0x%016" PRIx64 + " gpu_modified=%d same_context=%d\n", + info.address, info.size, cached->info.address, cached->info.size, + cached->gpu_modified, cached->ctx == ctx); + } + } + uint64_t readable = 0; + if (!HostMemoryQueryReadable(info.address, info.size, &readable) || readable < info.size) { + EXIT("TextureCache: new sampled image has unreadable guest backing, addr=0x%016" PRIx64 + " size=0x%016" PRIx64 " readable=0x%016" PRIx64 + " format=%u extent=%ux%ux%u pitch=%u levels=%u tile=%u type=%u\n", + info.address, info.size, readable, info.format, info.width, info.height, info.depth, + info.pitch, info.levels, info.tile, info.type); + } + + m_images.reserve(m_images.size() + 1); + auto cached = std::make_shared(); + cached->kind = CachedImage::Kind::Texture; + cached->info = info; + cached->ctx = ctx; + cached->image = new TextureVulkanImage; + const auto components = + TextureCreateImage(ctx, cached->image, &cached->memory, MakeImageParams(info, false)); + m_memory_tracker.ForEachUploadRange( + info.address, info.size, false, [](uint64_t, uint64_t) noexcept {}, + [&]() noexcept { + m_tiler.DetileImage(cached->ctx, + static_cast(cached->image), + cached->info, source, false, false); + }); + TextureCreateImageViews(ctx, cached->image, components, info.type, info.base_array, + info.base_level, info.view_levels, info.depth, true, + TextureFormatUsage::Sampled); + auto* image = cached->image; + command->RetainResourceUntilFence(cached); + m_images.push_back(std::move(cached)); + return image; +} + +StorageTextureVulkanImage* TextureCache::FindStorageTexture(CommandBuffer* command, + GraphicContext* ctx, + const ImageInfo& info) { + const bool supported_type = info.type == Prospero::GpuEnumValue(Prospero::ImageType::kColor2D) || + info.type == Prospero::GpuEnumValue(Prospero::ImageType::kColor2DArray) || + info.type == Prospero::GpuEnumValue(Prospero::ImageType::kColor3D); + if (command == nullptr || ctx == nullptr || info.address == 0 || info.size == 0 || + info.address >= TRACKER_ADDRESS_SIZE || info.size > TRACKER_ADDRESS_SIZE - info.address || + info.width == 0 || info.height == 0 || info.depth == 0 || info.levels != 1 || + info.base_level != 0 || info.view_levels != 1 || info.base_array != 0 || !supported_type || + (info.tile != Prospero::GpuEnumValue(Prospero::TileMode::kLinear) && + info.tile != Prospero::GpuEnumValue(Prospero::TileMode::kRenderTarget)) || + !IsSupportedStorageSwizzle(info.format, info.swizzle)) { + EXIT("TextureCache: unsupported storage-image request, command=%p ctx=%p " + "addr=0x%016" PRIx64 " size=0x%016" PRIx64 + " extent=%ux%ux%u levels=%u base_level=%u base_array=%u type=%u tile=%u " + "swizzle=0x%03x\n", + static_cast(command), static_cast(ctx), info.address, + info.size, info.width, info.height, info.depth, info.levels, info.base_level, + info.base_array, info.type, info.tile, info.swizzle); + } + if ((info.type == Prospero::GpuEnumValue(Prospero::ImageType::kColor2D) && info.depth != 1) || + (info.type == Prospero::GpuEnumValue(Prospero::ImageType::kColor3D) && info.depth == 0) || + (info.type == Prospero::GpuEnumValue(Prospero::ImageType::kColor2DArray) && info.base_array >= info.depth)) { + EXIT("TextureCache: storage-image type and depth disagree, type=%u depth=%u\n", info.type, + info.depth); + } + + std::lock_guard transaction(m_resource_mutex); + if (m_buffer_cache.HasPageOverlap(info.address, info.size)) { + EXIT("TextureCache: storage image aliases buffer pages, addr=0x%016" PRIx64 + " size=0x%016" PRIx64 "\n", + info.address, info.size); + } + FaultSafeTextureLock lock(this, m_lock); + RequireNoMetaOverlapLocked(info.address, info.size); + + std::shared_ptr match; + for (auto& cached: m_images) { + if (cached->kind != CachedImage::Kind::StorageTexture || !Equal(info, cached->info)) { + continue; + } + if (match != nullptr || cached->ctx != ctx || cached->info.IsCpuDirty()) { + EXIT("TextureCache: invalid exact storage-image cache match, addr=0x%016" PRIx64 + " size=0x%016" PRIx64 " duplicate=%d same_context=%d cpu_dirty=%d\n", + info.address, info.size, match != nullptr, cached->ctx == ctx, + cached->info.IsCpuDirty()); + } + match = cached; + } + if (match != nullptr) { + const bool cpu_modified = m_memory_tracker.IsRegionCpuModified(info.address, info.size); + const bool gpu_modified = m_memory_tracker.IsRegionGpuModified(info.address, info.size); + if (match->gpu_modified != gpu_modified || (gpu_modified && cpu_modified)) { + EXIT("TextureCache: storage-image ownership is inconsistent, addr=0x%016" PRIx64 + " size=0x%016" PRIx64 + " cached_gpu_modified=%d tracker_gpu_modified=%d cpu_modified=%d\n", + info.address, info.size, match->gpu_modified, gpu_modified, cpu_modified); + } + if (!gpu_modified) { + if (cpu_modified) { + m_memory_tracker.ForEachUploadRange( + info.address, info.size, true, [](uint64_t, uint64_t) noexcept {}, + [&]() noexcept { + m_tiler.DetileImage( + match->ctx, static_cast(match->image), + match->info, {nullptr, 0, match->info.address, match->info.size, true}, + true, true); + }); + } else { + // A readback leaves both copies current. Reclaim ownership without an unnecessary + // upload through the clean UpdateImage storage-binding path. + m_memory_tracker.MarkRegionAsGpuModified(info.address, info.size); + } + match->gpu_modified = true; + } + command->RetainResourceUntilFence(match); + return static_cast(match->image); + } + + for (const auto& cached: m_images) { + if (cached->OverlapsRange(info.address, info.size, true)) { + EXIT("TextureCache: unsupported storage-image alias, requested=0x%016" PRIx64 + "+0x%016" PRIx64 " existing=0x%016" PRIx64 "+0x%016" PRIx64 + " kind=%u gpu_modified=%d\n", + info.address, info.size, cached->Address(), cached->Size(), + static_cast(cached->kind), cached->gpu_modified); + } + } + uint64_t readable = 0; + //For debugging purposes + if (!HostMemoryQueryReadable(info.address, info.size, &readable) || readable < info.size) { + EXIT("TextureCache: new storage image has unreadable guest backing, addr=0x%016" PRIx64 + " size=0x%016" PRIx64 " readable=0x%016" PRIx64 "\n", + info.address, info.size, readable); + } + + m_images.reserve(m_images.size() + 1); + auto cached = std::make_shared(); + cached->kind = CachedImage::Kind::StorageTexture; + cached->info = info; + cached->ctx = ctx; + cached->image = new StorageTextureVulkanImage; + const auto components = + TextureCreateImage(ctx, cached->image, &cached->memory, MakeImageParams(info, true)); + m_memory_tracker.ForEachUploadRange( + info.address, info.size, true, [](uint64_t, uint64_t) noexcept {}, + [&]() noexcept { + m_tiler.DetileImage( + cached->ctx, static_cast(cached->image), cached->info, + {nullptr, 0, cached->info.address, cached->info.size, true}, false, true); + }); + cached->gpu_modified = true; + TextureCreateImageViews(ctx, cached->image, components, info.type, 0, 0, 1, info.depth, false, + TextureFormatUsage::Sampled | TextureFormatUsage::Storage); + auto* image = static_cast(cached->image); + command->RetainResourceUntilFence(cached); + m_images.push_back(std::move(cached)); + return image; +} + +RenderTextureVulkanImage* TextureCache::FindRenderTarget(CommandBuffer* command, + GraphicContext* ctx, + const RenderTargetInfo& info) { + if (command == nullptr || ctx == nullptr || info.address == 0 || info.size == 0 || + info.address >= TRACKER_ADDRESS_SIZE || info.size > TRACKER_ADDRESS_SIZE - info.address || + info.format == VK_FORMAT_UNDEFINED || info.width == 0 || info.height == 0 || + info.pitch < info.width || info.bytes_per_element == 0 || info.levels == 0 || + info.levels > 16 || + (info.tile_mode != Prospero::GpuEnumValue(Prospero::TileMode::kLinear) && + info.tile_mode != Prospero::GpuEnumValue(Prospero::TileMode::kRenderTarget))) { + EXIT("TextureCache: invalid render-target request, ctx=%p addr=0x%016" PRIx64 + " size=0x%016" PRIx64 " extent=%ux%u pitch=%u bpe=%u tile=%u format=%d\n", + static_cast(ctx), info.address, info.size, info.width, info.height, + info.pitch, info.bytes_per_element, info.tile_mode, static_cast(info.format)); + } + if (info.levels > 1) { + TileSizeAlign layout {}; + if (info.tile_mode != Prospero::GpuEnumValue(Prospero::TileMode::kRenderTarget) || + !TileGetRenderTargetMipLayout(info.width, info.height, info.pitch, + info.bytes_per_element, info.levels, &layout, nullptr, + nullptr) || + layout.size != info.size) { + EXIT("TextureCache: unsupported render-target mip layout, size=0x%016" PRIx64 + " expected=0x%08x extent=%ux%u pitch=%u bpe=%u levels=%u tile=%u\n", + info.size, layout.size, info.width, info.height, info.pitch, + info.bytes_per_element, info.levels, info.tile_mode); + } + } + const auto rows = static_cast(info.height - 1); + if (rows > (UINT64_MAX - info.width) / info.pitch) { + EXIT("TextureCache: render-target element count overflow, extent=%ux%u pitch=%u\n", + info.width, info.height, info.pitch); + } + const auto elements = rows * info.pitch + info.width; + if (elements > UINT64_MAX / info.bytes_per_element || + info.size < elements * info.bytes_per_element) { + EXIT("TextureCache: invalid render-target storage, size=0x%016" PRIx64 + " elements=0x%016" PRIx64 " bpe=%u\n", + info.size, elements, info.bytes_per_element); + } + std::lock_guard transaction(m_resource_mutex); + BufferImageCopySource target_source {nullptr, 0, info.address, info.size, true}; + if (m_buffer_cache.HasPageOverlap(info.address, info.size)) { + // A render target may use a containing native buffer after dirty bytes are published or + // coherent guest backing for a clean partial view. ObtainBufferForImage keeps GPU-dirty + // partial ownership and inconsistent state as hard failures. + target_source = m_buffer_cache.ObtainBufferForImage(info.address, info.size); + } + FaultSafeTextureLock lock(this, m_lock); + RequireNoMetaOverlapLocked(info.address, info.size); + std::shared_ptr match; + for (auto& cached: m_images) { + if (cached->kind != CachedImage::Kind::RenderTarget || cached->ctx != ctx || + (!Equal(info, cached->target) && !IsCompatibleRenderTargetView(cached->target, info))) { + continue; + } + match = cached; + } + if (match != nullptr) { + if (match->buffer_modified) { + if (match->gpu_modified || + !IsCoherentGuestImageSource(target_source, info.address, info.size)) { + EXIT("TextureCache: render-target refresh has invalid buffer ownership, " + "addr=0x%016" PRIx64 " size=0x%016" PRIx64 + " source=%p current=%d gpu_modified=%d\n", + info.address, info.size, static_cast(target_source.buffer), + target_source.cpu_current, match->gpu_modified); + } + m_memory_tracker.MarkRegionAsCpuModified(info.address, info.size); + } + if (!match->gpu_modified && m_memory_tracker.IsRegionCpuModified(info.address, info.size)) { + m_memory_tracker.ForEachUploadRange( + info.address, info.size, false, [](uint64_t, uint64_t) noexcept {}, + [&]() noexcept { + UploadRenderTarget(ctx, static_cast(match->image), + match->target, true); + }); + match->buffer_modified = false; + } + command->RetainResourceUntilFence(match); + return static_cast(match->image); + } + std::vector retire; + for (const auto& entry: m_images) { + auto& cached = *entry; + if (!cached.OverlapsRange(info.address, info.size, true)) { + continue; + } + const auto overlap = + cached.kind == CachedImage::Kind::Texture + ? ClassifyRenderTargetOverlap(cached.info, cached.gpu_modified, cached.ctx == ctx, + info) + : cached.kind == CachedImage::Kind::RenderTarget + ? ClassifyRenderTargetOverlap(cached.target, cached.gpu_modified, + cached.buffer_modified, cached.ctx == ctx, info) + : RenderTargetOverlap::Unsupported; + const bool retire_sampled = cached.kind == CachedImage::Kind::Texture && + overlap == RenderTargetOverlap::RetireSampled; + const bool retire_target = cached.kind == CachedImage::Kind::RenderTarget && + overlap == RenderTargetOverlap::RetireTarget; + if (!retire_sampled && !retire_target) { + EXIT("TextureCache: unsupported render-target alias, requested=0x%016" PRIx64 + "+0x%016" PRIx64 " existing_kind=%u existing=0x%016" PRIx64 "+0x%016" PRIx64 + " gpu_modified=%d same_context=%d" + " sampled_format=%u extent=%ux%ux%u pitch=%u levels=%u base_level=%u" + " tile=%u type=%u base_array=%u\n", + info.address, info.size, static_cast(cached.kind), cached.Address(), + cached.Size(), cached.gpu_modified, cached.ctx == ctx, cached.info.format, + cached.info.width, cached.info.height, cached.info.depth, cached.info.pitch, + cached.info.levels, cached.info.base_level, cached.info.tile, cached.info.type, + cached.info.base_array); + } + retire.push_back(&cached); + } + std::vector ranges; + ranges.reserve(m_images.size() * 2); + for (const auto& entry: m_images) { + const bool retiring = std::find(retire.begin(), retire.end(), entry.get()) != retire.end(); + for (uint32_t range = 0; range < entry->RangeCount(); range++) { + ranges.push_back({entry->Address(range), entry->Size(range), retiring}); + } + } + const auto conflict = FindImageRetirementConflict(ranges); + if (conflict.Exists()) { + const auto& retired = ranges[conflict.retired]; + const auto& retained = ranges[conflict.retained]; + EXIT("TextureCache: sampled retirement leaves a tracked page alias, target=0x%016" PRIx64 + "+0x%016" PRIx64 " retired=0x%016" PRIx64 "+0x%016" PRIx64 " retained=0x%016" PRIx64 + "+0x%016" PRIx64 "\n", + info.address, info.size, retired.address, retired.size, retained.address, + retained.size); + } + RetireImages(retire); + auto cached = std::make_shared(); + cached->kind = CachedImage::Kind::RenderTarget; + cached->target = info; + cached->ctx = ctx; + cached->image = CreateRenderTarget(ctx, info, &cached->memory); + m_memory_tracker.ForEachUploadRange( + info.address, info.size, false, [](uint64_t, uint64_t) noexcept {}, + [&]() noexcept { + UploadRenderTarget(ctx, static_cast(cached->image), info, + false); + }); + auto* image = static_cast(cached->image); + command->RetainResourceUntilFence(cached); + m_images.push_back(std::move(cached)); + return image; +} + +DepthStencilVulkanImage* TextureCache::FindDepthTarget(CommandBuffer* command, GraphicContext* ctx, + const DepthTargetInfo& info) { +//same as in findRenderTarget, but with additional checks for stencil and htile + const bool has_stencil = info.stencil_address != 0 || info.stencil_size != 0; + const bool has_htile = info.htile_address != 0 || info.htile_size != 0; + if (command == nullptr || ctx == nullptr || info.address == 0 || info.size == 0 || + info.address >= TRACKER_ADDRESS_SIZE || info.size > TRACKER_ADDRESS_SIZE - info.address || + (info.address & 0xffffu) != 0 || info.width == 0 || info.height == 0 || + info.pitch < info.width || info.tile_mode != Prospero::GpuEnumValue(Prospero::TileMode::kDepth) || + (has_stencil && + (info.stencil_address == 0 || info.stencil_size == 0 || + info.stencil_address >= TRACKER_ADDRESS_SIZE || + info.stencil_size > TRACKER_ADDRESS_SIZE - info.stencil_address || + (info.stencil_address & 0xffffu) != 0 || + PageOverlaps(info.address, info.size, info.stencil_address, info.stencil_size))) || + ((info.htile_address == 0) != (info.htile_size == 0)) || + (has_htile && (info.htile_address >= TRACKER_ADDRESS_SIZE || + info.htile_size > TRACKER_ADDRESS_SIZE - info.htile_address || + (info.htile_address & 0x7fffu) != 0 || + PageOverlaps(info.address, info.size, info.htile_address, info.htile_size) || + (has_stencil && PageOverlaps(info.stencil_address, info.stencil_size, + info.htile_address, info.htile_size)))) || + !(info.guest_format == Prospero::GpuEnumValue(Prospero::BufferFormat::k16UNorm) + ? !has_stencil && info.format == VK_FORMAT_D16_UNORM && info.bytes_per_element == 2 + : info.guest_format == Prospero::GpuEnumValue(Prospero::BufferFormat::k32Float) && + info.bytes_per_element == 4 && + info.format == + (has_stencil ? VK_FORMAT_D32_SFLOAT_S8_UINT : VK_FORMAT_D32_SFLOAT))) { + EXIT("TextureCache: unsupported depth target, command=%p ctx=%p depth=0x%016" PRIx64 + "+0x%016" PRIx64 " stencil=0x%016" PRIx64 "+0x%016" PRIx64 " htile=0x%016" PRIx64 + "+0x%016" PRIx64 " extent=%ux%u pitch=%u tile=%u format=%d guest_format=%u bpe=%u\n", + static_cast(command), static_cast(ctx), info.address, + info.size, info.stencil_address, info.stencil_size, info.htile_address, + info.htile_size, info.width, info.height, info.pitch, info.tile_mode, + static_cast(info.format), info.guest_format, info.bytes_per_element); + } + const auto rows = static_cast(info.height - 1); + if (rows > (UINT64_MAX - info.width) / info.pitch) { + EXIT("TextureCache: depth-target element count overflow, extent=%ux%u pitch=%u\n", + info.width, info.height, info.pitch); + } + const auto elements = rows * info.pitch + info.width; + if (elements > UINT64_MAX / info.bytes_per_element || + info.size < elements * info.bytes_per_element) { + EXIT("TextureCache: depth storage is too small, size=0x%016" PRIx64 + " elements=0x%016" PRIx64 " bpe=%u\n", + info.size, elements, info.bytes_per_element); + } + if (has_stencil && info.stencil_size < elements) { + EXIT("TextureCache: stencil storage is too small, size=0x%016" PRIx64 + " elements=0x%016" PRIx64 "\n", + info.stencil_size, elements); + } + if (has_htile) { + RegisterMeta(info.htile_address, info.htile_size); + } + std::lock_guard transaction(m_resource_mutex); + const auto depth_source = m_buffer_cache.ObtainBufferForImage(info.address, info.size); + if (has_stencil) { + (void)m_buffer_cache.ObtainBufferForImage(info.stencil_address, info.stencil_size); + } + FaultSafeTextureLock lock(this, m_lock); + RequireNoMetaOverlapLocked(info.address, info.size); + if (has_stencil) { + RequireNoMetaOverlapLocked(info.stencil_address, info.stencil_size); + } + CachedImage* match = nullptr; + for (auto& cached: m_images) { + if (cached->kind != CachedImage::Kind::DepthTarget || !Equal(info, cached->depth)) { + continue; + } + match = cached.get(); + } + if (match != nullptr) { + if (!CanLoadStencilAttachment(info, match->stencil_initialized)) { + EXIT("TextureCache: stencil load requires initialized contents, addr=0x%016" PRIx64 + " size=0x%016" PRIx64 "\n", + info.stencil_address, info.stencil_size); + } + if (has_stencil && info.stencil_load_clear) { + match->stencil_initialized = true; + } + if (!match->gpu_modified && m_memory_tracker.IsRegionCpuModified(info.address, info.size)) { + if (has_stencil && match->stencil_initialized && !info.stencil_load_clear) { + EXIT("TextureCache: depth refresh would discard initialized stencil, " + "addr=0x%016" PRIx64 "\n", + info.address); + } + m_memory_tracker.ForEachUploadRange( + info.address, info.size, false, [](uint64_t, uint64_t) noexcept {}, + [&]() noexcept { + m_tiler.DetileImage(ctx, + static_cast(match->image), + match->depth, depth_source, true); + }); + } + if (!match->gpu_modified && has_stencil && + m_memory_tracker.IsRegionCpuModified(info.stencil_address, info.stencil_size)) { + if (!info.stencil_load_clear) { + if (info.stencil_access) { + EXIT("TextureCache: stencil refresh from guest memory is unsupported, " + "addr=0x%016" PRIx64 " size=0x%016" PRIx64 "\n", + info.stencil_address, info.stencil_size); + } + } else { + m_memory_tracker.ForEachUploadRange( + info.stencil_address, info.stencil_size, false, + [](uint64_t, uint64_t) noexcept {}, []() noexcept {}); + } + } + return static_cast(match->image); + } + std::vector retire; + for (const auto& entry: m_images) { + auto& cached = *entry; + const bool overlaps = + cached.OverlapsRange(info.address, info.size, true) || + (has_stencil && cached.OverlapsRange(info.stencil_address, info.stencil_size, true)); + if (!overlaps) { + continue; + } + if (cached.kind != CachedImage::Kind::Texture || cached.ctx != ctx || + ClassifyDepthOverlap(cached.info, cached.gpu_modified, info) != + DepthOverlap::RetireSampled) { + EXIT("TextureCache: unsupported depth-target alias, depth=0x%016" PRIx64 + "+0x%016" PRIx64 " existing_kind=%u existing=0x%016" PRIx64 "+0x%016" PRIx64 "\n", + info.address, info.size, static_cast(cached.kind), cached.Address(), + cached.Size()); + } + retire.push_back(&cached); + } + std::vector ranges; + ranges.reserve(m_images.size() * 2); + for (const auto& entry: m_images) { + const bool retiring = std::find(retire.begin(), retire.end(), entry.get()) != retire.end(); + for (uint32_t range = 0; range < entry->RangeCount(); range++) { + ranges.push_back({entry->Address(range), entry->Size(range), retiring}); + } + } + const auto conflict = FindImageRetirementConflict(ranges); + if (conflict.Exists()) { + const auto& retired = ranges[conflict.retired]; + const auto& retained = ranges[conflict.retained]; + EXIT("TextureCache: sampled retirement leaves a tracked page alias, depth=0x%016" PRIx64 + "+0x%016" PRIx64 " retired=0x%016" PRIx64 "+0x%016" PRIx64 " retained=0x%016" PRIx64 + "+0x%016" PRIx64 "\n", + info.address, info.size, retired.address, retired.size, retained.address, + retained.size); + } + RetireImages(retire); + if (!CanLoadStencilAttachment(info, false)) { + EXIT("TextureCache: new stencil target requires a clear before stencil access, " + "addr=0x%016" PRIx64 " size=0x%016" PRIx64 "\n", + info.stencil_address, info.stencil_size); + } + auto cached = std::make_shared(); + cached->kind = CachedImage::Kind::DepthTarget; + cached->depth = info; + cached->ctx = ctx; + cached->stencil_initialized = !has_stencil || info.stencil_load_clear; + cached->image = CreateDepthTarget(ctx, info, &cached->memory); + m_memory_tracker.ForEachUploadRange( + info.address, info.size, false, [](uint64_t, uint64_t) noexcept {}, + [&]() noexcept { + if (!info.depth_load_clear) { + m_tiler.DetileImage(ctx, + static_cast(cached->image), + info, depth_source, false); + } + }); + if (has_stencil) { + m_memory_tracker.ForEachUploadRange( + info.stencil_address, info.stencil_size, false, [](uint64_t, uint64_t) noexcept {}, + []() noexcept {}); + } + auto* image = static_cast(cached->image); + m_images.push_back(std::move(cached)); + return image; +} + +std::vector +TextureCache::RegisterVideoOutSurfaces(GraphicContext* ctx, + const std::vector& infos) { + if (infos.empty()) { + EXIT("TextureCache: video-out registration requires surfaces\n"); + } + for (const auto& info: infos) { + ValidateVideoOutInfo(ctx, info); + } + for (size_t i = 0; i < infos.size(); i++) { + for (size_t j = i + 1; j < infos.size(); j++) { + if (PageOverlaps(infos[i].address, infos[i].size, infos[j].address, infos[j].size)) { + EXIT("TextureCache: video-out surfaces share pages, first=%zu second=%zu\n", i, j); + } + } + } + std::lock_guard transaction(m_resource_mutex); + for (const auto& info: infos) { + if (m_buffer_cache.HasPageOverlap(info.address, info.size)) { + EXIT("TextureCache: video-out surface aliases buffer pages, addr=0x%016" PRIx64 + " size=0x%016" PRIx64 "\n", + info.address, info.size); + } + } + FaultSafeTextureLock lock(this, m_lock); + for (const auto& info: infos) { + RequireNoMetaOverlapLocked(info.address, info.size); + for (const auto& cached: m_images) { + if (cached->OverlapsRange(info.address, info.size, true)) { + EXIT( + "TextureCache: video-out surface aliases cached image pages, addr=0x%016" PRIx64 + " size=0x%016" PRIx64 " existing_kind=%u\n", + info.address, info.size, static_cast(cached->kind)); + } + } + } + m_images.reserve(m_images.size() + infos.size()); + std::vector result; + result.reserve(infos.size()); + for (const auto& info: infos) { + auto cached = std::make_shared(); + cached->kind = CachedImage::Kind::VideoOut; + cached->video_out = info; + cached->ctx = ctx; + cached->image = CreateVideoOut(ctx, info, &cached->memory); + if (info.compression == VideoOutCompression::Uncompressed) { + m_memory_tracker.ForEachUploadRange( + info.address, info.size, false, [](uint64_t, uint64_t) noexcept {}, + [&]() noexcept { + UploadVideoOut(ctx, static_cast(cached->image), info, + false); + }); + } else { + // A compressed guest surface cannot be decoded without its DCC metadata. Establish the + // normal tracked range, but leave the shared native image to be initialized by a GPU + // render/clear before any sampled or presentation read. + m_memory_tracker.ForEachUploadRange( + info.address, info.size, false, [](uint64_t, uint64_t) noexcept {}, + []() noexcept {}); + } + result.push_back(static_cast(cached->image)); + m_images.push_back(std::move(cached)); + } + return result; +} + +void TextureCache::RefreshVideoOut(VideoOutVulkanImage* image, bool render_target) { + if (image == nullptr) { + EXIT("TextureCache: invalid video-out refresh, image=%p\n", + static_cast(image)); + } + std::lock_guard transaction(m_resource_mutex); + FaultSafeTextureLock lock(this, m_lock); + const auto it = std::find_if(m_images.begin(), m_images.end(), + [image](const auto& cached) { return cached->image == image; }); + if (it == m_images.end() || (*it)->kind != CachedImage::Kind::VideoOut) { + EXIT("TextureCache: video-out image is not registered, image=%p\n", + static_cast(image)); + } + auto& cached = **it; + if (cached.gpu_modified) { + return; + } + const auto& info = cached.video_out; + const bool image_dirty = m_memory_tracker.IsRegionCpuModified(info.address, info.size); + const bool buffer_dirty = cached.buffer_modified || + m_buffer_cache.IsRegionCpuModified(info.address, info.size) || + m_buffer_cache.IsRegionGpuModified(info.address, info.size); + if (!image_dirty && !buffer_dirty) { + if (info.compression == VideoOutCompression::Uncompressed || + CanUseVideoOutNativeWithoutUpload(info.compression, render_target, false, false)) { + return; + } + EXIT("TextureCache: compressed video-out read requires native GPU contents, " + "addr=0x%016" PRIx64 " size=0x%016" PRIx64 "\n", + info.address, info.size); + } + if (info.compression != VideoOutCompression::Uncompressed) { + EXIT("TextureCache: compressed video-out guest refresh is unsupported, " + "addr=0x%016" PRIx64 " size=0x%016" PRIx64 + " image_dirty=%d buffer_dirty=%d render_target=%d\n", + info.address, info.size, image_dirty, buffer_dirty, render_target); + } + if (buffer_dirty) { + const auto source = m_buffer_cache.ObtainBufferForImage(info.address, info.size); + if (!IsCoherentGuestImageSource(source, info.address, info.size)) { + EXIT("TextureCache: invalid video-out source, addr=0x%016" PRIx64 " size=0x%016" PRIx64 + " buffer=%p current=%d source=0x%016" PRIx64 "+0x%016" PRIx64 + " buffer_modified=%d\n", + info.address, info.size, static_cast(source.buffer), + source.cpu_current, source.address, source.size, cached.buffer_modified); + } + cached.buffer_modified = false; + } + m_memory_tracker.ForEachUploadRange( + info.address, info.size, false, [](uint64_t, uint64_t) noexcept {}, + [&]() noexcept { UploadVideoOut(cached.ctx, image, info, true); }); +} + +void TextureCache::UnregisterVideoOutSurfaces(const std::vector& images) { + if (images.empty()) { + EXIT("TextureCache: video-out unregistration requires surfaces\n"); + } + std::lock_guard transaction(m_resource_mutex); + FaultSafeTextureLock lock(this, m_lock); + std::vector selected; + selected.reserve(images.size()); + for (size_t i = 0; i < images.size(); i++) { + auto* image = images[i]; + if (image == nullptr || + std::find(images.begin(), images.begin() + i, image) != images.begin() + i) { + EXIT("TextureCache: invalid or duplicate video-out image at index %zu, image=%p\n", i, + static_cast(image)); + } + auto it = std::find_if(m_images.begin(), m_images.end(), + [image](const auto& cached) { return cached->image == image; }); + if (it == m_images.end() || (*it)->kind != CachedImage::Kind::VideoOut) { + EXIT("TextureCache: video-out image is not registered, index=%zu image=%p\n", i, + static_cast(image)); + } + selected.push_back(it->get()); + } + GraphicContext* ctx = selected.front()->ctx; + for (auto* cached: selected) { + if (cached->ctx != ctx) { + EXIT("TextureCache: video-out surfaces span graphics contexts, expected=%p actual=%p\n", + static_cast(ctx), static_cast(cached->ctx)); + } + if (cached->buffer_modified) { + EXIT("TextureCache: cannot unregister a buffer-dirty video-out surface, " + "addr=0x%016" PRIx64 " size=0x%016" PRIx64 "\n", + cached->Address(), cached->Size()); + } + } + VulkanDeviceWaitIdle(ctx); + for (auto* cached: selected) { + if (cached->gpu_modified) { + m_memory_tracker.UnmarkRegionAsGpuModified(cached->Address(), cached->Size()); + cached->gpu_modified = false; + } + m_memory_tracker.UntrackMemory(cached->Address(), cached->Size()); + } + for (auto* image: images) { + auto it = std::find_if(m_images.begin(), m_images.end(), + [image](const auto& cached) { return cached->image == image; }); + m_images.erase(it); + } +} + +bool TextureCache::ClearColorImageFromBuffer(CommandBuffer* command, uint64_t vaddr, uint64_t size, + uint32_t packed_clear) { + if (command == nullptr || command->IsInvalid() || vaddr == 0 || size == 0 || + vaddr >= TRACKER_ADDRESS_SIZE || size > TRACKER_ADDRESS_SIZE - vaddr) { + EXIT("TextureCache: invalid compute image clear, command=%p addr=0x%016" PRIx64 + " size=0x%016" PRIx64 "\n", + static_cast(command), vaddr, size); + } + m_buffer_cache.ValidateGpuAccess(vaddr, size, false, true); + std::lock_guard transaction(m_resource_mutex); + FaultSafeTextureLock lock(this, m_lock); + std::shared_ptr match; + for (const auto& cached: m_images) { + if (!cached->OverlapsRange(vaddr, size, true)) { + continue; + } + const bool color = cached->kind == CachedImage::Kind::VideoOut || + cached->kind == CachedImage::Kind::RenderTarget; + const bool exact = cached->Address() == vaddr && cached->Size() == size; + if (!color || !exact) { + return false; + } + if (match != nullptr) { + EXIT("TextureCache: compute image clear has ambiguous exact aliases, " + "addr=0x%016" PRIx64 " size=0x%016" PRIx64 " first=%p second=%p\n", + vaddr, size, static_cast(match->image), + static_cast(cached->image)); + } + match = cached; + } + if (match == nullptr) { + return false; + } + RequireNoMetaOverlapLocked(vaddr, size); + const bool buffer_overlap = m_buffer_cache.HasPageOverlap(vaddr, size); + const bool buffer_cpu_modified = + buffer_overlap && m_buffer_cache.IsRegionCpuModified(vaddr, size); + const bool buffer_gpu_modified = + buffer_overlap && m_buffer_cache.IsRegionGpuModified(vaddr, size); + BufferImageCopySource source {nullptr, 0, vaddr, size, true}; + if (buffer_overlap && !buffer_cpu_modified && !buffer_gpu_modified) { + // A prior buffer clear can leave one clean cached buffer after this range becomes a native + // color target. Allow coexisting cache views only after the shared ownership + // helper proves the stale source is coherent native-buffer or guest backing. + source = m_buffer_cache.ObtainBufferForImage(vaddr, size); + } + const bool buffer_source_valid = + !buffer_overlap || IsCoherentGuestImageSource(source, vaddr, size); + const bool image_cpu_modified = m_memory_tracker.IsRegionCpuModified(vaddr, size); + if (!buffer_source_valid || buffer_cpu_modified || buffer_gpu_modified || + match->buffer_modified || image_cpu_modified) { + EXIT("TextureCache: compute image clear requires exclusive GPU image ownership, " + "addr=0x%016" PRIx64 " size=0x%016" PRIx64 + " buffer_overlap=%d source_valid=%d buffer_cpu_modified=%d" + " buffer_gpu_modified=%d buffer_modified=%d image_cpu_modified=%d\n", + vaddr, size, buffer_overlap, buffer_source_valid, buffer_cpu_modified, + buffer_gpu_modified, match->buffer_modified, image_cpu_modified); + } + VkClearColorValue clear {}; + if (!DecodePackedColorClear(match->image->format, packed_clear, &clear)) { + return false; + } + auto* vk_buffer = command->GetPool()->buffers[command->GetIndex()]; + GraphicsRenderColorImageBarrier(vk_buffer, match->image, VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL); + const VkImageSubresourceRange range {VK_IMAGE_ASPECT_COLOR_BIT, 0, VK_REMAINING_MIP_LEVELS, 0, + match->image->layers}; + vkCmdClearColorImage(vk_buffer, match->image->image, match->image->layout, &clear, 1, &range); + GraphicsRenderColorImageBarrier(vk_buffer, match->image, RENDER_COLOR_IMAGE_LAYOUT); + if (!match->gpu_modified) { + m_memory_tracker.MarkRegionAsGpuModified(vaddr, size); + match->gpu_modified = true; + } + command->RetainResourceUntilFence(match); + return true; +} + +bool TextureCache::ClearDepthImageFromBuffer(CommandBuffer* command, uint64_t vaddr, uint64_t size, + uint32_t packed_clear) { + if (command == nullptr || command->IsInvalid() || vaddr == 0 || size == 0 || + vaddr >= TRACKER_ADDRESS_SIZE || size > TRACKER_ADDRESS_SIZE - vaddr) { + EXIT("TextureCache: invalid compute depth clear, command=%p addr=0x%016" PRIx64 + " size=0x%016" PRIx64 "\n", + static_cast(command), vaddr, size); + } + m_buffer_cache.ValidateGpuAccess(vaddr, size, false, true); + std::lock_guard transaction(m_resource_mutex); + FaultSafeTextureLock lock(this, m_lock); + std::shared_ptr match; + for (const auto& cached: m_images) { + if (!cached->OverlapsRange(vaddr, size, true)) { + continue; + } + const bool exact_depth = cached->kind == CachedImage::Kind::DepthTarget && + CanNativeClearDepthFromBuffer(cached->depth, vaddr, size); + if (!exact_depth) { + return false; + } + if (match != nullptr) { + EXIT("TextureCache: compute depth clear has ambiguous exact aliases, " + "addr=0x%016" PRIx64 " size=0x%016" PRIx64 " first=%p second=%p\n", + vaddr, size, static_cast(match->image), + static_cast(cached->image)); + } + match = cached; + } + if (match == nullptr) { + return false; + } + float depth_clear = 0.0f; + if (!DecodePackedDepthClear(match->image->format, packed_clear, &depth_clear)) { + return false; + } + RequireNoMetaOverlapLocked(vaddr, size); + const bool buffer_overlap = m_buffer_cache.HasPageOverlap(vaddr, size); + const bool buffer_cpu_modified = + buffer_overlap && m_buffer_cache.IsRegionCpuModified(vaddr, size); + const bool buffer_gpu_modified = + buffer_overlap && m_buffer_cache.IsRegionGpuModified(vaddr, size); + BufferImageCopySource source {nullptr, 0, vaddr, size, true}; + if (buffer_overlap && !buffer_cpu_modified && !buffer_gpu_modified) { + source = m_buffer_cache.ObtainBufferForImage(vaddr, size); + } + const bool buffer_source_valid = + !buffer_overlap || IsCoherentGuestImageSource(source, vaddr, size); + const bool image_cpu_modified = m_memory_tracker.IsRegionCpuModified(vaddr, size); + if (!match->gpu_modified || !buffer_source_valid || buffer_cpu_modified || + buffer_gpu_modified || match->buffer_modified || image_cpu_modified) { + EXIT("TextureCache: compute depth clear requires a GPU-owned depth image and a clean " + "buffer source, addr=0x%016" PRIx64 " size=0x%016" PRIx64 + " gpu_modified=%d buffer_overlap=%d source_valid=%d buffer_cpu_modified=%d" + " buffer_gpu_modified=%d buffer_modified=%d image_cpu_modified=%d\n", + vaddr, size, match->gpu_modified, buffer_overlap, buffer_source_valid, + buffer_cpu_modified, buffer_gpu_modified, match->buffer_modified, image_cpu_modified); + } + if (match->image->layout == VK_IMAGE_LAYOUT_UNDEFINED || + match->image->layout == VK_IMAGE_LAYOUT_PREINITIALIZED) { + EXIT("TextureCache: compute depth clear has invalid source layout %u\n", + static_cast(match->image->layout)); + } + auto* vk_buffer = command->GetPool()->buffers[command->GetIndex()]; + const auto old_layout = match->image->layout; + GraphicsRenderDepthStencilImageBarrier(vk_buffer, match->image, + VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL); + const VkClearDepthStencilValue clear {depth_clear, 0}; + const VkImageSubresourceRange range {VK_IMAGE_ASPECT_DEPTH_BIT, 0, VK_REMAINING_MIP_LEVELS, 0, + match->image->layers}; + vkCmdClearDepthStencilImage(vk_buffer, match->image->image, match->image->layout, &clear, 1, + &range); + GraphicsRenderDepthStencilImageBarrier(vk_buffer, match->image, old_layout); + command->RetainResourceUntilFence(match); + return true; +} + +bool TextureCache::ClearStencilImageFromBuffer(CommandBuffer* command, uint64_t vaddr, + uint64_t size, uint32_t packed_clear) { + if (command == nullptr || command->IsInvalid() || vaddr == 0 || size == 0 || + vaddr >= TRACKER_ADDRESS_SIZE || size > TRACKER_ADDRESS_SIZE - vaddr) { + EXIT("TextureCache: invalid compute stencil clear, command=%p addr=0x%016" PRIx64 + " size=0x%016" PRIx64 "\n", + static_cast(command), vaddr, size); + } + uint8_t stencil_clear = 0; + if (!DecodePackedStencilClear(packed_clear, &stencil_clear)) { + return false; + } + m_buffer_cache.ValidateGpuAccess(vaddr, size, false, true); + std::lock_guard transaction(m_resource_mutex); + FaultSafeTextureLock lock(this, m_lock); + std::shared_ptr match; + for (const auto& cached: m_images) { + if (!cached->OverlapsRange(vaddr, size, true)) { + continue; + } + const bool exact_stencil = cached->kind == CachedImage::Kind::DepthTarget && + cached->RangeCount() == 2 && cached->Address(1) == vaddr && + cached->Size(1) == size; + if (!exact_stencil) { + return false; + } + if (match != nullptr) { + EXIT("TextureCache: compute stencil clear has ambiguous exact aliases, " + "addr=0x%016" PRIx64 " size=0x%016" PRIx64 " first=%p second=%p\n", + vaddr, size, static_cast(match->image), + static_cast(cached->image)); + } + match = cached; + } + if (match == nullptr) { + return false; + } + RequireNoMetaOverlapLocked(vaddr, size); + const bool buffer_overlap = m_buffer_cache.HasPageOverlap(vaddr, size); + const bool buffer_cpu_modified = + buffer_overlap && m_buffer_cache.IsRegionCpuModified(vaddr, size); + const bool buffer_gpu_modified = + buffer_overlap && m_buffer_cache.IsRegionGpuModified(vaddr, size); + BufferImageCopySource source {nullptr, 0, vaddr, size, true}; + if (buffer_overlap && !buffer_cpu_modified && !buffer_gpu_modified) { + source = m_buffer_cache.ObtainBufferForImage(vaddr, size); + } + const bool buffer_source_valid = + !buffer_overlap || IsCoherentGuestImageSource(source, vaddr, size); + const bool image_cpu_modified = m_memory_tracker.IsRegionCpuModified(vaddr, size); + if (!match->gpu_modified || !buffer_source_valid || buffer_cpu_modified || + buffer_gpu_modified || match->buffer_modified || image_cpu_modified) { + EXIT("TextureCache: compute stencil clear requires a GPU-owned depth image and a clean " + "stencil source, addr=0x%016" PRIx64 " size=0x%016" PRIx64 + " gpu_modified=%d buffer_overlap=%d source_valid=%d buffer_cpu_modified=%d" + " buffer_gpu_modified=%d buffer_modified=%d image_cpu_modified=%d\n", + vaddr, size, match->gpu_modified, buffer_overlap, buffer_source_valid, + buffer_cpu_modified, buffer_gpu_modified, match->buffer_modified, image_cpu_modified); + } + if (match->image->layout == VK_IMAGE_LAYOUT_UNDEFINED || + match->image->layout == VK_IMAGE_LAYOUT_PREINITIALIZED) { + EXIT("TextureCache: compute stencil clear has invalid source layout %u\n", + static_cast(match->image->layout)); + } + auto* vk_buffer = command->GetPool()->buffers[command->GetIndex()]; + const auto old_layout = match->image->layout; + GraphicsRenderDepthStencilImageBarrier(vk_buffer, match->image, + VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL); + const VkClearDepthStencilValue clear {0.0f, stencil_clear}; + const VkImageSubresourceRange range {VK_IMAGE_ASPECT_STENCIL_BIT, 0, VK_REMAINING_MIP_LEVELS, 0, + match->image->layers}; + vkCmdClearDepthStencilImage(vk_buffer, match->image->image, match->image->layout, &clear, 1, + &range); + GraphicsRenderDepthStencilImageBarrier(vk_buffer, match->image, old_layout); + match->stencil_initialized = true; + command->RetainResourceUntilFence(match); + return true; +} + +void TextureCache::MarkGpuWritten(VulkanImage* image) { + if (image == nullptr) { + EXIT("TextureCache: invalid GPU-write notification, image=%p\n", + static_cast(image)); + } + std::lock_guard transaction(m_resource_mutex); + FaultSafeTextureLock lock(this, m_lock); + for (auto& cached: m_images) { + if (cached->image != image) { + continue; + } + if (cached->kind != CachedImage::Kind::RenderTarget && + cached->kind != CachedImage::Kind::DepthTarget && + cached->kind != CachedImage::Kind::VideoOut) { + EXIT("TextureCache: sampled texture cannot be marked GPU-written, image=%p kind=%u\n", + static_cast(image), static_cast(cached->kind)); + } + for (uint32_t i = 0; i < cached->RangeCount(); i++) { + if (m_buffer_cache.HasPageOverlap(cached->Address(i), cached->Size(i))) { + const bool cpu_modified = + m_buffer_cache.IsRegionCpuModified(cached->Address(i), cached->Size(i)); + const bool gpu_modified = + m_buffer_cache.IsRegionGpuModified(cached->Address(i), cached->Size(i)); + if (cpu_modified || gpu_modified) { + EXIT( + "TextureCache: GPU-written image aliases a dirty buffer, addr=0x%016" PRIx64 + " size=0x%016" PRIx64 " cpu_modified=%d gpu_modified=%d\n", + cached->Address(i), cached->Size(i), cpu_modified, gpu_modified); + } + } + if (m_memory_tracker.IsRegionCpuModified(cached->Address(i), cached->Size(i))) { + EXIT("TextureCache: GPU-write begins while image range is CPU-modified, " + "addr=0x%016" PRIx64 " size=0x%016" PRIx64 "\n", + cached->Address(i), cached->Size(i)); + } + } + if (cached->kind == CachedImage::Kind::DepthTarget) { + if ((cached->depth.htile_address == 0) != (cached->depth.htile_size == 0)) { + EXIT("TextureCache: depth target has incomplete HTile range, addr=0x%016" PRIx64 + " size=0x%016" PRIx64 "\n", + cached->depth.htile_address, cached->depth.htile_size); + } + if (cached->depth.htile_address != 0) { + auto meta = m_surface_metas.find(cached->depth.htile_address); + if (meta == m_surface_metas.end() || + meta->second.size != cached->depth.htile_size) { + EXIT("TextureCache: depth target HTile metadata is missing or mismatched, " + "addr=0x%016" PRIx64 " size=0x%016" PRIx64 "\n", + cached->depth.htile_address, cached->depth.htile_size); + } + m_buffer_cache.ValidateGpuAccess(cached->depth.htile_address, + cached->depth.htile_size, false, true); + const bool cpu_modified = m_metadata_tracker.IsRegionCpuModified( + cached->depth.htile_address, cached->depth.htile_size); + if (meta->second.gpu_modified && cpu_modified) { + EXIT("TextureCache: depth write races CPU-modified HTile metadata, " + "addr=0x%016" PRIx64 " size=0x%016" PRIx64 "\n", + cached->depth.htile_address, cached->depth.htile_size); + } + if (!meta->second.gpu_modified) { + m_metadata_tracker.ForEachUploadRange( + cached->depth.htile_address, cached->depth.htile_size, true, + [](uint64_t, uint64_t) noexcept {}, []() noexcept {}); + meta->second.gpu_modified = true; + } + meta->second.clear_mask = 0; + } + } + if (!cached->gpu_modified) { + for (uint32_t i = 0; i < cached->RangeCount(); i++) { + m_memory_tracker.MarkRegionAsGpuModified(cached->Address(i), cached->Size(i)); + } + cached->gpu_modified = true; + } + return; + } + EXIT("TextureCache: GPU-written image is not registered, image=%p\n", + static_cast(image)); +} + +void TextureCache::PrepareHostWrite(uint64_t vaddr, uint64_t size) { + if (vaddr == 0 || size == 0 || vaddr >= TRACKER_ADDRESS_SIZE || + size > TRACKER_ADDRESS_SIZE - vaddr) { + EXIT("TextureCache: invalid host-write range, addr=0x%016" PRIx64 " size=0x%016" PRIx64 + "\n", + vaddr, size); + } + FaultSafeTextureLock lock(this, m_lock); + const bool metadata_overlap = HasMetaOverlapLocked(vaddr, size); + bool found = false; + for (const auto& cached: m_images) { + for (uint32_t range = 0; range < cached->RangeCount(); range++) { + const bool host_refreshable = (cached->kind == CachedImage::Kind::Texture || + cached->kind == CachedImage::Kind::RenderTarget) && + !cached->buffer_modified; + switch (ClassifyHostWriteOverlap(vaddr, size, cached->Address(range), + cached->Size(range), host_refreshable, + cached->gpu_modified, metadata_overlap)) { + case HostWriteOverlap::None: break; + case HostWriteOverlap::InvalidateImage: found = true; break; + case HostWriteOverlap::Unsupported: + EXIT("TextureCache: host write aliases unsupported image, " + "write=0x%016" PRIx64 "+0x%016" PRIx64 " image=0x%016" PRIx64 + "+0x%016" PRIx64 + " kind=%u gpu_modified=%d buffer_modified=%d metadata_overlap=%d\n", + vaddr, size, cached->Address(range), cached->Size(range), + static_cast(cached->kind), cached->gpu_modified, + cached->buffer_modified, metadata_overlap); + } + } + } + if (!found) { + EXIT("TextureCache: host write expected a refreshable image overlap, addr=0x%016" PRIx64 + " size=0x%016" PRIx64 "\n", + vaddr, size); + } + // Explicitly invalidate on a write fault. The resource transaction is already held here, so + // writing first would recursively enter the cache. + m_memory_tracker.MarkRegionAsCpuModified(vaddr, size); + MarkSampledAliasesCpuDirtyLocked(vaddr, size); +} + +void TextureCache::SynchronizeRenderTargetToBufferLocked(CachedImage& cached) { + const auto& target = cached.target; + const bool linear = target.tile_mode == Prospero::GpuEnumValue(Prospero::TileMode::kLinear); + const bool tiled = target.tile_mode == Prospero::GpuEnumValue(Prospero::TileMode::kRenderTarget); + const auto rows = static_cast(target.height - 1); + TileSizeAlign exact {}; + const bool exact_tiled = tiled && + TileGetRenderTargetSize(target.width, target.height, target.pitch, + target.bytes_per_element, &exact) && + exact.align == 65536 && exact.size == target.size; + if (cached.kind != CachedImage::Kind::RenderTarget || !cached.gpu_modified || + cached.buffer_modified || cached.ctx == nullptr || cached.image == nullptr || + (!linear && !exact_tiled) || target.address == 0 || target.size == 0 || target.width == 0 || + target.height == 0 || target.pitch < target.width || target.bytes_per_element == 0 || + target.levels != 1 || rows > (UINT64_MAX - target.width) / target.pitch) { + EXIT("TextureCache: unsupported render-target buffer synchronization, " + "addr=0x%016" PRIx64 " size=0x%016" PRIx64 + " extent=%ux%u pitch=%u bpe=%u levels=%u tile=%u" + " gpu_modified=%d buffer_modified=%d\n", + target.address, target.size, target.width, target.height, target.pitch, + target.bytes_per_element, target.levels, target.tile_mode, cached.gpu_modified, + cached.buffer_modified); + } + const auto linear_elements = rows * target.pitch + target.width; + if (linear_elements > UINT64_MAX / target.bytes_per_element) { + EXIT("TextureCache: render-target buffer synchronization size overflow\n"); + } + const auto linear_size = linear_elements * target.bytes_per_element; + if (linear_size > target.size || cached.image->format != target.format || + cached.image->extent.width != target.width || + cached.image->extent.height != target.height || + (tiled && target.bytes_per_element != 1 && target.bytes_per_element != 2 && + target.bytes_per_element != 4 && target.bytes_per_element != 8) || + HasMetaOverlapLocked(target.address, target.size)) { + EXIT("TextureCache: render-target buffer synchronization storage mismatch, " + "addr=0x%016" PRIx64 " size=0x%016" PRIx64 " linear=0x%016" PRIx64 "\n", + target.address, target.size, linear_size); + } + + // This is the CPU Tiler backend for the image-to-buffer synchronization seam. + // The vectors and Vulkan staging allocation retain capacity; a future PS5 GPU tiler can replace + // this block without changing alias classification or ownership transitions. + VulkanDeviceWaitIdle(cached.ctx); + m_buffer_transition_linear.resize(target.size); + std::fill(m_buffer_transition_linear.begin(), m_buffer_transition_linear.end(), 0); + UtilFillBuffer(cached.ctx, m_buffer_transition_linear.data(), linear_size, target.pitch, + cached.image, cached.image->layout); + if (tiled) { + m_buffer_transition_guest.resize(target.size); + m_tiler.TileImage(m_buffer_transition_guest.data(), + m_buffer_transition_linear.data(), target); + Libs::LibKernel::Memory::WriteBacking(target.address, m_buffer_transition_guest.data(), + target.size); + } else { + Libs::LibKernel::Memory::WriteBacking(target.address, m_buffer_transition_linear.data(), + target.size); + } + m_memory_tracker.ForEachDownloadRange(target.address, target.size, + [](uint64_t, uint64_t) noexcept {}); + if (m_memory_tracker.IsRegionGpuModified(target.address, target.size)) { + EXIT("TextureCache: render-target buffer synchronization retained GPU ownership\n"); + } + m_buffer_cache.PublishImageBacking(target.address, target.size); + cached.gpu_modified = false; + cached.buffer_modified = true; +} + +void TextureCache::InvalidateMemoryFromGPU(uint64_t vaddr, uint64_t size, + bool formatted_buffer_write) { + if (vaddr == 0 || size == 0 || vaddr >= TRACKER_ADDRESS_SIZE || + size > TRACKER_ADDRESS_SIZE - vaddr) { + EXIT("TextureCache: invalid GPU invalidation range, addr=0x%016" PRIx64 + " size=0x%016" PRIx64 "\n", + vaddr, size); + } + FaultSafeTextureLock lock(this, m_lock); + auto match = m_images.end(); + BufferImageAlias action = BufferImageAlias::None; + bool sampled_write = false; + for (auto it = m_images.begin(); it != m_images.end(); ++it) { + auto& cached = **it; + if (!cached.OverlapsRange(vaddr, size, true)) { + continue; + } + const auto alias = + ClassifyBufferImageAlias(vaddr, size, cached.Address(), cached.Size(), + cached.kind == CachedImage::Kind::VideoOut, + cached.kind == CachedImage::Kind::RenderTarget, + cached.gpu_modified, true, formatted_buffer_write); + const bool sampled = cached.kind == CachedImage::Kind::Texture && + IsSampledBufferWriteTransition(vaddr, size, cached.Address(), + cached.Size(), cached.gpu_modified); + if (match != m_images.end() || (!sampled && alias != BufferImageAlias::VideoOutWrite && + alias != BufferImageAlias::RenderTargetWrite)) { + EXIT("TextureCache: unsupported GPU invalidation alias, addr=0x%016" PRIx64 + " size=0x%016" PRIx64 " cached_kind=%u cached=0x%016" PRIx64 "+0x%016" PRIx64 + " gpu_modified=%d buffer_modified=%d formatted=%d ambiguous=%d\n", + vaddr, size, static_cast(cached.kind), cached.Address(), cached.Size(), + cached.gpu_modified, cached.buffer_modified, formatted_buffer_write, + match != m_images.end()); + } + match = it; + action = sampled ? BufferImageAlias::None : alias; + sampled_write = sampled; + } + if (match == m_images.end()) { + return; + } + if (HasMetaOverlapLocked(vaddr, size)) { + EXIT("TextureCache: GPU invalidation overlaps metadata, addr=0x%016" PRIx64 + " size=0x%016" PRIx64 "\n", + vaddr, size); + } + if (sampled_write) { + (**match).buffer_modified = true; + return; + } + if (action == BufferImageAlias::VideoOutWrite) { + auto& cached = **match; + cached.buffer_modified = true; + return; + } + if (action == BufferImageAlias::RenderTargetWrite) { + SynchronizeRenderTargetToBufferLocked(**match); + return; + } + m_memory_tracker.UntrackMemory(vaddr, size); + m_images.erase(match); +} + +DepthStencilVulkanImage* TextureCache::FindDepthTargetByRange(uint64_t vaddr, uint64_t size) { + if (vaddr == 0 || size == 0 || vaddr >= TRACKER_ADDRESS_SIZE || + size > TRACKER_ADDRESS_SIZE - vaddr) { + EXIT("TextureCache: invalid depth-target range query, addr=0x%016" PRIx64 + " size=0x%016" PRIx64 "\n", + vaddr, size); + } + FaultSafeTextureLock lock(this, m_lock); + DepthStencilVulkanImage* found = nullptr; + for (auto& cached: m_images) { + if (cached->kind != CachedImage::Kind::DepthTarget || + !cached->OverlapsRange(vaddr, size, false)) { + continue; + } + if (!IsDepthTargetRangeCompatible(cached->depth, vaddr, size) || found != nullptr) { + EXIT("TextureCache: incompatible or ambiguous depth-target range, addr=0x%016" PRIx64 + " size=0x%016" PRIx64 " cached=0x%016" PRIx64 "+0x%016" PRIx64 " previous=%p\n", + vaddr, size, cached->depth.address, cached->depth.size, + static_cast(found)); + } + const bool stencil_range = + vaddr == cached->depth.stencil_address && size == cached->depth.stencil_size; + if (stencil_range && !cached->stencil_initialized) { + EXIT("TextureCache: sampled stencil range is uninitialized, addr=0x%016" PRIx64 + " size=0x%016" PRIx64 "\n", + vaddr, size); + } + found = static_cast(cached->image); + } + return found; +} + +RenderTextureVulkanImage* TextureCache::FindRenderTargetByRange(CommandBuffer* command, + uint64_t vaddr, uint64_t size) { + if (command == nullptr || vaddr == 0 || size == 0 || vaddr >= TRACKER_ADDRESS_SIZE || + size > TRACKER_ADDRESS_SIZE - vaddr) { + EXIT("TextureCache: invalid render-target range query, addr=0x%016" PRIx64 + " size=0x%016" PRIx64 "\n", + vaddr, size); + } + FaultSafeTextureLock lock(this, m_lock); + std::shared_ptr found; + for (auto& cached: m_images) { + if (cached->kind != CachedImage::Kind::RenderTarget || + !Overlaps(vaddr, size, cached->Address(), cached->Size())) { + continue; + } + if (vaddr != cached->Address() || size != cached->Size() || found != nullptr) { + EXIT("TextureCache: ambiguous or partial render-target range, addr=0x%016" PRIx64 + " size=0x%016" PRIx64 " cached=0x%016" PRIx64 "+0x%016" PRIx64 " previous=%p\n", + vaddr, size, cached->Address(), cached->Size(), + static_cast(found.get())); + } + found = cached; + } + if (found == nullptr) { + return nullptr; + } + command->RetainResourceUntilFence(found); + return static_cast(found->image); +} + +bool TextureCache::HasGpuTargetPageOverlap(uint64_t vaddr, uint64_t size) { + if (vaddr == 0 || size == 0 || vaddr >= TRACKER_ADDRESS_SIZE || + size > TRACKER_ADDRESS_SIZE - vaddr) { + EXIT("TextureCache: invalid GPU-target overlap query, addr=0x%016" PRIx64 + " size=0x%016" PRIx64 "\n", + vaddr, size); + } + FaultSafeTextureLock lock(this, m_lock); + for (const auto& cached: m_images) { + if (cached->kind != CachedImage::Kind::Texture && + cached->OverlapsRange(vaddr, size, true)) { + return true; + } + } + return false; +} + +bool TextureCache::HasPageOverlap(uint64_t vaddr, uint64_t size) { + if (vaddr == 0 || size == 0 || vaddr >= TRACKER_ADDRESS_SIZE || + size > TRACKER_ADDRESS_SIZE - vaddr) { + EXIT("TextureCache: invalid image page-overlap query, addr=0x%016" PRIx64 + " size=0x%016" PRIx64 "\n", + vaddr, size); + } + FaultSafeTextureLock lock(this, m_lock); + for (const auto& cached: m_images) { + if (cached->OverlapsRange(vaddr, size, true)) { + return true; + } + } + return false; +} + +bool TextureCache::HasRangeOverlap(uint64_t vaddr, uint64_t size) { + if (vaddr == 0 || size == 0 || vaddr >= TRACKER_ADDRESS_SIZE || + size > TRACKER_ADDRESS_SIZE - vaddr) { + EXIT("TextureCache: invalid image-range-overlap query, addr=0x%016" PRIx64 + " size=0x%016" PRIx64 "\n", + vaddr, size); + } + FaultSafeTextureLock lock(this, m_lock); + for (const auto& cached: m_images) { + if (cached->OverlapsRange(vaddr, size, false)) { + return true; + } + } + return false; +} + +bool TextureCache::HasGpuModifiedRangeOverlap(uint64_t vaddr, uint64_t size) { + if (vaddr == 0 || size == 0 || vaddr >= TRACKER_ADDRESS_SIZE || + size > TRACKER_ADDRESS_SIZE - vaddr) { + EXIT("TextureCache: invalid GPU-modified image-overlap query, addr=0x%016" PRIx64 + " size=0x%016" PRIx64 "\n", + vaddr, size); + } + FaultSafeTextureLock lock(this, m_lock); + for (const auto& cached: m_images) { + if (cached->gpu_modified && cached->OverlapsRange(vaddr, size, false)) { + return true; + } + } + return false; +} + +void TextureCache::RegisterMeta(uint64_t vaddr, uint64_t size) { + if (vaddr == 0 || size == 0 || vaddr >= TRACKER_ADDRESS_SIZE || + size > TRACKER_ADDRESS_SIZE - vaddr || (vaddr & 0x7fffu) != 0) { + EXIT("TextureCache: invalid metadata registration, addr=0x%016" PRIx64 " size=0x%016" PRIx64 + "\n", + vaddr, size); + } + std::lock_guard transaction(m_resource_mutex); + if (m_buffer_cache.HasPageOverlap(vaddr, size)) { + // Register virtual surface metadata independently of an earlier buffer view. Kyty's split + // caches first publish any dirty buffer bytes; clean partial views use guest backing. + const auto source = m_buffer_cache.ObtainBufferForImage(vaddr, size); + if (!IsCoherentGuestImageSource(source, vaddr, size)) { + EXIT("TextureCache: metadata buffer source is inconsistent, addr=0x%016" PRIx64 + " size=0x%016" PRIx64 " source=%p source_addr=0x%016" PRIx64 + " source_size=0x%016" PRIx64 " current=%d\n", + vaddr, size, static_cast(source.buffer), source.address, source.size, + source.cpu_current); + } + } + FaultSafeTextureLock lock(this, m_lock); + const auto existing = m_surface_metas.find(vaddr); + if (existing != m_surface_metas.end()) { + if (existing->second.size != size) { + EXIT("TextureCache: metadata address reused with a different size, addr=0x%016" PRIx64 + " requested=0x%016" PRIx64 " existing=0x%016" PRIx64 "\n", + vaddr, size, existing->second.size); + } + return; + } + for (const auto& [address, meta]: m_surface_metas) { + if (PageOverlaps(vaddr, size, address, meta.size)) { + LOGF("TextureCache: registering overlapping metadata views, requested=0x%016" PRIx64 + "+0x%016" PRIx64 " existing=0x%016" PRIx64 "+0x%016" PRIx64 "\n", + vaddr, size, address, meta.size); + } + } + std::vector retire; + for (const auto& cached: m_images) { + if (!cached->OverlapsRange(vaddr, size, true)) { + continue; + } + const auto overlap = + ClassifyMetaImageOverlap(cached->kind == CachedImage::Kind::Texture, + cached->kind == CachedImage::Kind::RenderTarget, + cached->gpu_modified, cached->buffer_modified); + if (overlap == MetaImageOverlap::Unsupported) { + EXIT("TextureCache: metadata aliases image pages, addr=0x%016" PRIx64 + " size=0x%016" PRIx64 " image_kind=%u image=0x%016" PRIx64 "+0x%016" PRIx64 + " gpu_modified=%d buffer_modified=%d\n", + vaddr, size, static_cast(cached->kind), cached->Address(), + cached->Size(), cached->gpu_modified, cached->buffer_modified); + } + if (overlap == MetaImageOverlap::RetireTarget) { + retire.push_back(cached.get()); + continue; + } + LOGF("TextureCache: metadata aliases a CPU-current sampled image, metadata=0x%016" PRIx64 + "+0x%016" PRIx64 " image=0x%016" PRIx64 "+0x%016" PRIx64 "\n", + vaddr, size, cached->Address(), cached->Size()); + } + if (!retire.empty()) { + std::vector ranges; + ranges.reserve(m_images.size() * 2); + for (const auto& cached: m_images) { + const bool retiring = + std::find(retire.begin(), retire.end(), cached.get()) != retire.end(); + for (uint32_t range = 0; range < cached->RangeCount(); range++) { + ranges.push_back({cached->Address(range), cached->Size(range), retiring}); + } + } + const auto conflict = FindImageRetirementConflict(ranges); + if (conflict.Exists()) { + const auto& retired = ranges[conflict.retired]; + const auto& retained = ranges[conflict.retained]; + EXIT("TextureCache: metadata retirement leaves a tracked page alias, " + "metadata=0x%016" PRIx64 "+0x%016" PRIx64 " retired=0x%016" PRIx64 "+0x%016" PRIx64 + " retained=0x%016" PRIx64 "+0x%016" PRIx64 "\n", + vaddr, size, retired.address, retired.size, retained.address, retained.size); + } + for (const auto* cached: retire) { + LOGF("TextureCache: retiring a CPU-current render target for metadata reuse, " + "metadata=0x%016" PRIx64 "+0x%016" PRIx64 " target=0x%016" PRIx64 "+0x%016" PRIx64 + "\n", + vaddr, size, cached->Address(), cached->Size()); + } + RetireImages(retire); + } + m_surface_metas.emplace(vaddr, MetaDataInfo {size}); +} + +bool TextureCache::IsMeta(uint64_t vaddr) { + FaultSafeTextureLock lock(this, m_lock); + return m_surface_metas.contains(vaddr); +} + +bool TextureCache::IsMetaRange(uint64_t vaddr, uint64_t size) { + if (vaddr == 0 || size == 0 || vaddr >= TRACKER_ADDRESS_SIZE || + size > TRACKER_ADDRESS_SIZE - vaddr) { + return false; + } + FaultSafeTextureLock lock(this, m_lock); + const auto it = m_surface_metas.find(vaddr); + return it != m_surface_metas.end() && it->second.size == size; +} + +bool TextureCache::HasMetaOverlap(uint64_t vaddr, uint64_t size) { + if (vaddr == 0 || size == 0 || vaddr >= TRACKER_ADDRESS_SIZE || + size > TRACKER_ADDRESS_SIZE - vaddr) { + EXIT("TextureCache: invalid metadata-overlap query, addr=0x%016" PRIx64 + " size=0x%016" PRIx64 "\n", + vaddr, size); + } + FaultSafeTextureLock lock(this, m_lock); + for (const auto& [address, meta]: m_surface_metas) { + if (PageOverlaps(vaddr, size, address, meta.size)) { + return true; + } + } + return false; +} + +bool TextureCache::IsMetaGpuModified(uint64_t vaddr, uint64_t size) { + if (vaddr == 0 || size == 0 || vaddr >= TRACKER_ADDRESS_SIZE || + size > TRACKER_ADDRESS_SIZE - vaddr) { + EXIT("TextureCache: invalid metadata ownership query, addr=0x%016" PRIx64 + " size=0x%016" PRIx64 "\n", + vaddr, size); + } + FaultSafeTextureLock lock(this, m_lock); + for (const auto& [address, meta]: m_surface_metas) { + if (PageOverlaps(vaddr, size, address, meta.size) && + m_metadata_tracker.IsRegionGpuModified(vaddr, size)) { + return true; + } + } + return false; +} + +bool TextureCache::HasMetaOverlapLocked(uint64_t vaddr, uint64_t size) const { + for (const auto& [address, meta]: m_surface_metas) { + if (PageOverlaps(vaddr, size, address, meta.size)) { + return true; + } + } + return false; +} + +void TextureCache::RequireNoMetaOverlapLocked(uint64_t vaddr, uint64_t size) const { + if (HasMetaOverlapLocked(vaddr, size)) { + EXIT("TextureCache: image range overlaps virtual metadata, addr=0x%016" PRIx64 + " size=0x%016" PRIx64 "\n", + vaddr, size); + } +} + +bool TextureCache::IsMetaCleared(uint64_t vaddr, uint32_t slice) { + if (slice >= 32) { + EXIT("TextureCache: metadata clear slice out of range, addr=0x%016" PRIx64 " slice=%u\n", + vaddr, slice); + } + FaultSafeTextureLock lock(this, m_lock); + const auto it = m_surface_metas.find(vaddr); + if (it != m_surface_metas.end() && it->second.gpu_modified && + m_metadata_tracker.IsRegionCpuModified(vaddr, it->second.size)) { + EXIT("TextureCache: GPU-owned metadata became CPU-modified, addr=0x%016" PRIx64 + " size=0x%016" PRIx64 "\n", + vaddr, it->second.size); + } + return it != m_surface_metas.end() && (it->second.clear_mask & (1u << slice)) != 0; +} + +bool TextureCache::ClearMeta(uint64_t vaddr) { + std::lock_guard transaction(m_resource_mutex); + FaultSafeTextureLock lock(this, m_lock); + const auto it = m_surface_metas.find(vaddr); + if (it == m_surface_metas.end()) { + return false; + } + m_metadata_tracker.ForEachUploadRange( + vaddr, it->second.size, true, [](uint64_t, uint64_t) noexcept {}, []() noexcept {}); + it->second.gpu_modified = true; + it->second.clear_mask = UINT32_MAX; + return true; +} + +bool TextureCache::TouchMeta(uint64_t vaddr, uint32_t slice, bool is_clear) { + if (slice >= 32) { + EXIT("TextureCache: metadata update slice out of range, addr=0x%016" PRIx64 " slice=%u\n", + vaddr, slice); + } + FaultSafeTextureLock lock(this, m_lock); + const auto it = m_surface_metas.find(vaddr); + if (it == m_surface_metas.end()) { + return false; + } + if (!it->second.gpu_modified) { + EXIT("TextureCache: metadata update requires GPU ownership, addr=0x%016" PRIx64 "\n", + vaddr); + } + if (m_metadata_tracker.IsRegionCpuModified(vaddr, it->second.size)) { + EXIT("TextureCache: metadata update races CPU modification, addr=0x%016" PRIx64 + " size=0x%016" PRIx64 "\n", + vaddr, it->second.size); + } + if (is_clear) { + it->second.clear_mask |= 1u << slice; + } else { + it->second.clear_mask &= ~(1u << slice); + } + return true; +} + +bool TextureCache::InvalidateMemory(PageFaultAccess access, uint64_t vaddr, uint64_t size, + PageFaultPhase phase) noexcept { + if (access != PageFaultAccess::Read && access != PageFaultAccess::Write) { + EXIT("TextureCache: unsupported page-fault access %u, addr=0x%016" PRIx64 + " size=0x%016" PRIx64 " phase=%u\n", + static_cast(access), vaddr, size, static_cast(phase)); + } + if (phase == PageFaultPhase::Invalidate) { + if (g_texture_fault_owner != nullptr) { + EXIT("TextureCache: recursive page-fault invalidation, owner=%p\n", + g_texture_fault_owner); + } + m_fault_mutex.lock(); + g_texture_fault_owner = this; + + const bool metadata = + access == PageFaultAccess::Write && + m_metadata_tracker.InvalidateVirtualGpuWrite(access, vaddr, size, phase); + CpuFaultAction action = CpuFaultAction::Untracked; + bool needs_readback = false; + { + FaultSafeTextureLock lock(this, m_lock); + if (access == PageFaultAccess::Write) { + MarkSampledAliasesCpuDirtyLocked(vaddr, size); + } + needs_readback = FindGpuReadbackPageCandidateLocked(vaddr, size) != nullptr; + if (!needs_readback) { + action = m_memory_tracker.BeginCpuFault(vaddr, size, access); + } + } + + if (needs_readback) { + m_readback->Request(access, vaddr, size); + action = m_memory_tracker.BeginCpuFault(vaddr, size, access); + if (action != CpuFaultAction::Download) { + EXIT("TextureCache: downloaded image did not retain the active fault page " + "tracking, addr=0x%016" PRIx64 " size=0x%016" PRIx64 " action=%u\n", + vaddr, size, static_cast(action)); + } + } else if (action == CpuFaultAction::Download) { + EXIT("generic region invalidation cannot download GPU-dirty memory\n"); + } + return metadata || action != CpuFaultAction::Untracked; + } + + if (g_texture_fault_owner != this) { + EXIT("TextureCache: page-fault phase has no matching invalidation, phase=%u owner=%p\n", + static_cast(phase), g_texture_fault_owner); + } + if (phase == PageFaultPhase::Complete) { + const bool metadata = + access == PageFaultAccess::Write && + m_metadata_tracker.InvalidateVirtualGpuWrite(access, vaddr, size, phase); + const bool downloaded = m_readback->IsReady(access, vaddr, size); + const bool image = m_memory_tracker.CompleteCpuFault(vaddr, size, access, downloaded); + const bool readback = m_readback->Complete(access, vaddr, size); + if (readback && !image) { + EXIT("TextureCache: image readback completed without a matching tracker fault\n"); + } + return metadata || image || readback; + } + if (phase != PageFaultPhase::Release) { + EXIT("TextureCache: unsupported page-fault phase %u\n", static_cast(phase)); + } + if (access == PageFaultAccess::Write) { + (void)m_metadata_tracker.InvalidateVirtualGpuWrite(access, vaddr, size, phase); + } + m_readback->Release(access, vaddr, size); + g_texture_fault_owner = nullptr; + m_fault_mutex.unlock(); + return true; +} + +void TextureCache::UnmapMemory(uint64_t vaddr, uint64_t size) { + if (vaddr == 0 || size == 0 || vaddr >= TRACKER_ADDRESS_SIZE || + size > TRACKER_ADDRESS_SIZE - vaddr) { + EXIT("TextureCache: invalid unmap range, addr=0x%016" PRIx64 " size=0x%016" PRIx64 "\n", + vaddr, size); + } + std::lock_guard transaction(m_resource_mutex); + FaultSafeTextureLock lock(this, m_lock); + const auto end = vaddr + size; + for (const auto& [address, meta]: m_surface_metas) { + if (PageOverlaps(vaddr, size, address, meta.size) && + (vaddr > address || end < address + meta.size)) { + EXIT("TextureCache: partial metadata unmap is unsupported, unmap=0x%016" PRIx64 + "+0x%016" PRIx64 " metadata=0x%016" PRIx64 "+0x%016" PRIx64 "\n", + vaddr, size, address, meta.size); + } + } + for (const auto& cached: m_images) { + for (uint32_t i = 0; i < cached->RangeCount(); i++) { + if (PageOverlaps(vaddr, size, cached->Address(i), cached->Size(i)) && + (vaddr > cached->Address(i) || end < cached->Address(i) + cached->Size(i))) { + EXIT("TextureCache: partial image unmap is unsupported, unmap=0x%016" PRIx64 + "+0x%016" PRIx64 " image=0x%016" PRIx64 "+0x%016" PRIx64 "\n", + vaddr, size, cached->Address(i), cached->Size(i)); + } + } + if (cached->OverlapsRange(vaddr, size, false) && + cached->kind == CachedImage::Kind::VideoOut) { + EXIT("TextureCache: registered video-out surface must be unregistered before unmap, " + "unmap=0x%016" PRIx64 "+0x%016" PRIx64 " image=0x%016" PRIx64 "+0x%016" PRIx64 + " buffer_modified=%d\n", + vaddr, size, cached->Address(), cached->Size(), cached->buffer_modified); + } + } + std::vector retire_depth_metadata; + for (const auto& cached: m_images) { + if (cached->kind != CachedImage::Kind::DepthTarget || + !cached->OverlapsRange(vaddr, size, false) || cached->depth.htile_address == 0) { + continue; + } + const bool retained_owner = + std::any_of(m_images.begin(), m_images.end(), [&](const auto& other) { + return other.get() != cached.get() && + other->kind == CachedImage::Kind::DepthTarget && + !other->OverlapsRange(vaddr, size, false) && + other->depth.htile_address == cached->depth.htile_address && + other->depth.htile_size == cached->depth.htile_size; + }); + if (!retained_owner && + std::find(retire_depth_metadata.begin(), retire_depth_metadata.end(), + cached->depth.htile_address) == retire_depth_metadata.end()) { + retire_depth_metadata.push_back(cached->depth.htile_address); + } + } + for (const auto address: retire_depth_metadata) { + const auto meta = m_surface_metas.find(address); + const auto owner = std::find_if(m_images.begin(), m_images.end(), [&](const auto& cached) { + return cached->kind == CachedImage::Kind::DepthTarget && + cached->OverlapsRange(vaddr, size, false) && + cached->depth.htile_address == address; + }); + if (meta == m_surface_metas.end() || owner == m_images.end() || + meta->second.size != (*owner)->depth.htile_size) { + EXIT("TextureCache: retiring depth image has invalid HTile registration, " + "unmap=0x%016" PRIx64 "+0x%016" PRIx64 " metadata=0x%016" PRIx64 + " registered=0x%016" PRIx64 " expected=0x%016" PRIx64 "\n", + vaddr, size, address, meta != m_surface_metas.end() ? meta->second.size : 0, + owner != m_images.end() ? (*owner)->depth.htile_size : 0); + } + } + std::vector metadata_ranges; + metadata_ranges.reserve(m_surface_metas.size()); + for (const auto& [address, meta]: m_surface_metas) { + const bool retiring = Overlaps(vaddr, size, address, meta.size) || + std::find(retire_depth_metadata.begin(), retire_depth_metadata.end(), + address) != retire_depth_metadata.end(); + metadata_ranges.push_back({address, meta.size, retiring}); + } + const auto metadata_conflict = FindImageRetirementConflict(metadata_ranges); + if (metadata_conflict.Exists()) { + const auto& retired = metadata_ranges[metadata_conflict.retired]; + const auto& retained = metadata_ranges[metadata_conflict.retained]; + EXIT("TextureCache: depth-image retirement leaves a tracked metadata alias, " + "unmap=0x%016" PRIx64 "+0x%016" PRIx64 " retired=0x%016" PRIx64 "+0x%016" PRIx64 + " retained=0x%016" PRIx64 "+0x%016" PRIx64 "\n", + vaddr, size, retired.address, retired.size, retained.address, retained.size); + } + GraphicContext* wait_ctx = nullptr; + for (auto& cached: m_images) { + if (cached->OverlapsRange(vaddr, size, false)) { + if (wait_ctx != nullptr && wait_ctx != cached->ctx) { + EXIT("TextureCache: unmap spans multiple graphics contexts, first=%p second=%p\n", + static_cast(wait_ctx), static_cast(cached->ctx)); + } + wait_ctx = cached->ctx; + } + } + if (wait_ctx != nullptr) { + VulkanDeviceWaitIdle(wait_ctx); + } + for (auto& cached: m_images) { + if (!cached->OverlapsRange(vaddr, size, false) || !cached->gpu_modified) { + continue; + } + for (uint32_t i = 0; i < cached->RangeCount(); i++) { + m_memory_tracker.UnmarkRegionAsGpuModified(cached->Address(i), cached->Size(i)); + } + cached->gpu_modified = false; + } + for (auto& cached: m_images) { + if (!cached->OverlapsRange(vaddr, size, false)) { + continue; + } + for (uint32_t i = 0; i < cached->RangeCount(); i++) { + m_memory_tracker.UntrackMemory(cached->Address(i), cached->Size(i)); + } + } + for (auto it = m_surface_metas.begin(); it != m_surface_metas.end();) { + const bool allocation_unmapped = Overlaps(vaddr, size, it->first, it->second.size); + const bool depth_owner_retired = + std::find(retire_depth_metadata.begin(), retire_depth_metadata.end(), it->first) != + retire_depth_metadata.end(); + if (!allocation_unmapped && !depth_owner_retired) { + ++it; + continue; + } + if (depth_owner_retired) { + LOGF("TextureCache: retiring HTile metadata with depth image, " + "unmap=0x%016" PRIx64 "+0x%016" PRIx64 " metadata=0x%016" PRIx64 "+0x%016" PRIx64 + "\n", + vaddr, size, it->first, it->second.size); + } + if (it->second.gpu_modified) { + m_metadata_tracker.ForEachDownloadRange(it->first, it->second.size, + [](uint64_t, uint64_t) noexcept {}); + } + m_metadata_tracker.UntrackMemory(it->first, it->second.size); + it = m_surface_metas.erase(it); + } + m_memory_tracker.UntrackMemory(vaddr, size); + for (auto it = m_images.begin(); it != m_images.end();) { + if (!(**it).OverlapsRange(vaddr, size, false)) { + ++it; + continue; + } + it = m_images.erase(it); + } +} + +static constexpr int DummyTextureIndex(bool uint_format, bool image_3d) { + return (image_3d ? 2 : 0) + (uint_format ? 1 : 0); +} + +static constexpr uint32_t DummyTextureSwizzle() { + return Prospero::GpuEnumValue(Prospero::CompSwizzle::kRed) | (Prospero::GpuEnumValue(Prospero::CompSwizzle::kGreen) << 3u) | + (Prospero::GpuEnumValue(Prospero::CompSwizzle::kBlue) << 6u) | + (Prospero::GpuEnumValue(Prospero::CompSwizzle::kAlpha) << 9u); +} + +static TextureImageCreateParams MakeDummyTextureParams(bool uint_format, bool image_3d, + TextureFormatUsage usage, + const char* owner) { + TextureImageCreateParams params {}; + params.fmt = static_cast( + Prospero::GpuEnumValue(uint_format ? Prospero::BufferFormat::k8_8_8_8UInt : Prospero::BufferFormat::k8_8_8_8UNorm)); + params.width = 1; + params.height = 1; + params.base_level = 0; + params.levels = 1; + params.depth = 1; + params.type = Prospero::GpuEnumValue(image_3d ? Prospero::ImageType::kColor3D : Prospero::ImageType::kColor2D); + params.swizzle = DummyTextureSwizzle(); + params.format_usage = usage; + params.required_format_usage = usage; + params.view_usage = usage; + params.image_layout = TextureUploadDestination::MipLevels; + params.allow_cube_view = true; + params.storage_swizzle_fallback = TextureHasFormatUsage(usage, TextureFormatUsage::Storage); + params.owner = owner; + return params; +} + +static VulkanImage* CreateDummyTexture(GraphicContext* ctx, bool uint_format, bool image_3d, + bool storage, VulkanMemory** memory) { + if (memory == nullptr || *memory != nullptr) { + EXIT("TextureCache: invalid dummy texture memory slot, slot=%p current=%p storage=%d\n", + static_cast(memory), + memory == nullptr ? nullptr : static_cast(*memory), storage); + } + auto* image = (storage ? static_cast(new StorageTextureVulkanImage) + : new TextureVulkanImage); + auto usage = (storage ? TextureFormatUsage::Storage : TextureFormatUsage::Sampled); + auto layout = (storage ? VK_IMAGE_LAYOUT_GENERAL : VK_IMAGE_LAYOUT_SHADER_READ_ONLY_OPTIMAL); + auto owner = (storage ? "DummyStorageTexture" : "DummySampledTexture"); + + *memory = new VulkanMemory {}; + auto params = MakeDummyTextureParams(uint_format, image_3d, usage, owner); + auto components = TextureCreateImage(ctx, image, *memory, params); + + static constexpr uint32_t zero = 0; + UtilFillImage(ctx, image, &zero, sizeof(zero), 1, static_cast(layout)); + TextureCreateImageViews(ctx, image, components, params.type, 0, params.base_level, + params.levels, params.depth, params.allow_cube_view, params.view_usage); + return image; +} + +void TextureCache::DeleteImageViews(GraphicContext* ctx, VulkanImage* image) { + EXIT_IF(ctx == nullptr); + EXIT_IF(image == nullptr); + + for (auto& view: image->image_view) { + if (view != nullptr) { + vkDestroyImageView(ctx->device, view, nullptr); + view = nullptr; + } + } +} + +void TextureCache::DeleteGpuTexture(GraphicContext* ctx, GpuTextureVulkanImage* image, + VulkanMemory* mem) { + KYTY_PROFILER_BLOCK("TextureCache::DeleteGpuTexture"); + + EXIT_IF(ctx == nullptr); + EXIT_IF(image == nullptr); + + if (image->type == VulkanImageType::StorageTexture) { + auto* storage = static_cast(image); + for (auto& cached: storage->sampled_views) { + if (cached.view != nullptr) { + vkDestroyImageView(ctx->device, cached.view, nullptr); + cached.view = nullptr; + } + } + storage->sampled_views.clear(); + } + DeleteImageViews(ctx, image); + VulkanDeleteImage(ctx, image, mem); + + switch (image->type) { + case VulkanImageType::Texture: delete static_cast(image); break; + case VulkanImageType::StorageTexture: + delete static_cast(image); + break; + default: EXIT("unsupported gpu texture image type: %d\n", static_cast(image->type)); + } +} + +void TextureCache::DeleteRenderTexture(GraphicContext* ctx, RenderTextureVulkanImage* image, + VulkanMemory* mem) { + KYTY_PROFILER_BLOCK("TextureCache::DeleteRenderTexture"); + + EXIT_IF(ctx == nullptr); + EXIT_IF(image == nullptr); + + g_render_ctx->GetFramebufferCache()->FreeFramebufferByColor(image); + for (auto& view: image->render_view) { + if (view != nullptr) { + vkDestroyImageView(ctx->device, view, nullptr); + view = nullptr; + } + } + for (auto& cached: image->attachment_views) { + if (cached.view != nullptr) { + vkDestroyImageView(ctx->device, cached.view, nullptr); + cached.view = nullptr; + } + } + image->attachment_views.clear(); + for (auto& cached: image->sampled_views) { + if (cached.view != nullptr) { + vkDestroyImageView(ctx->device, cached.view, nullptr); + cached.view = nullptr; + } + } + image->sampled_views.clear(); + for (auto& cached: image->storage_views) { + if (cached.view != nullptr) { + vkDestroyImageView(ctx->device, cached.view, nullptr); + cached.view = nullptr; + } + } + image->storage_views.clear(); + DeleteImageViews(ctx, image); + VulkanDeleteImage(ctx, image, mem); + delete image; +} + +void TextureCache::DeleteDepthStencil(GraphicContext* ctx, DepthStencilVulkanImage* image, + VulkanMemory* mem) { + KYTY_PROFILER_BLOCK("TextureCache::DeleteDepthStencil"); + + EXIT_IF(ctx == nullptr); + EXIT_IF(image == nullptr); + + g_render_ctx->GetFramebufferCache()->FreeFramebufferByDepth(image); + DeleteImageViews(ctx, image); + VulkanDeleteImage(ctx, image, mem); + delete image; +} + +void TextureCache::DeleteVideoOut(GraphicContext* ctx, VideoOutVulkanImage* image, + VulkanMemory* mem) { + KYTY_PROFILER_BLOCK("TextureCache::DeleteVideoOut"); + + EXIT_IF(ctx == nullptr); + EXIT_IF(image == nullptr); + + g_render_ctx->GetFramebufferCache()->FreeFramebufferByColor(image); + DeleteImageViews(ctx, image); + VulkanDeleteImage(ctx, image, mem); + delete image; +} + +VulkanImage* TextureCache::GetDummySampledTexture(bool uint_format, bool image_3d) { + KYTY_PROFILER_BLOCK("TextureCache::GetDummySampledTexture"); + + Common::LockGuard lock(m_dummy_mutex); + auto* ctx = g_render_ctx->GetGraphicCtx(); + if (m_dummy_ctx != nullptr && m_dummy_ctx != ctx) { + EXIT("TextureCache: sampled dummy texture context changed, previous=%p current=%p\n", + static_cast(m_dummy_ctx), static_cast(ctx)); + } + m_dummy_ctx = ctx; + const auto index = DummyTextureIndex(uint_format, image_3d); + auto*& image = m_dummy_sampled_textures[index]; + if (image == nullptr) { + image = + CreateDummyTexture(ctx, uint_format, image_3d, false, &m_dummy_sampled_memory[index]); + } + return image; +} + +VulkanImage* TextureCache::GetDummyStorageTexture(bool uint_format, bool image_3d) { + KYTY_PROFILER_BLOCK("TextureCache::GetDummyStorageTexture"); + + Common::LockGuard lock(m_dummy_mutex); + auto* ctx = g_render_ctx->GetGraphicCtx(); + if (m_dummy_ctx != nullptr && m_dummy_ctx != ctx) { + EXIT("TextureCache: storage dummy texture context changed, previous=%p current=%p\n", + static_cast(m_dummy_ctx), static_cast(ctx)); + } + m_dummy_ctx = ctx; + const auto index = DummyTextureIndex(uint_format, image_3d); + auto*& image = m_dummy_storage_textures[index]; + if (image == nullptr) { + image = + CreateDummyTexture(ctx, uint_format, image_3d, true, &m_dummy_storage_memory[index]); + } + return image; +} + +} // namespace Libs::Graphics diff --git a/src/graphics/host_gpu/renderer/textureCache.h b/src/graphics/host_gpu/renderer/textureCache.h new file mode 100644 index 000000000..04f4e801d --- /dev/null +++ b/src/graphics/host_gpu/renderer/textureCache.h @@ -0,0 +1,143 @@ +#ifndef EMULATOR_SRC_GRAPHICS_HOST_GPU_RENDERER_TEXTURECACHE_H_ +#define EMULATOR_SRC_GRAPHICS_HOST_GPU_RENDERER_TEXTURECACHE_H_ + +#include "common/abi.h" +#include "common/common.h" +#include "common/threads.h" +#include "graphics/host_gpu/memoryTracker.h" +#include "graphics/host_gpu/renderer/imageInfo.h" +#include "graphics/host_gpu/renderer/tiler.h" + +#include +#include +#include +#include +#include +#include + +namespace Libs::Graphics { + +struct DepthStencilVulkanImage; +struct GpuTextureVulkanImage; +struct GraphicContext; +struct RenderTextureVulkanImage; +struct StorageTextureVulkanImage; +struct VideoOutVulkanImage; +struct VulkanImage; +struct VulkanMemory; +class BufferCache; +class CommandBuffer; +class ResourceMutex; + +class TextureCache { +public: + TextureCache(PageManager& page_manager, BufferCache& buffer_cache, ResourceMutex& resource_mutex); + ~TextureCache(); + KYTY_CLASS_NO_COPY(TextureCache); + + [[nodiscard]] VulkanImage* FindTexture(CommandBuffer* command, GraphicContext* ctx, + const ImageInfo& info, bool metadata_read); + [[nodiscard]] StorageTextureVulkanImage* + FindStorageTexture(CommandBuffer* command, GraphicContext* ctx, const ImageInfo& info); + [[nodiscard]] RenderTextureVulkanImage* + FindRenderTarget(CommandBuffer* command, GraphicContext* ctx, const RenderTargetInfo& info); + [[nodiscard]] DepthStencilVulkanImage* + FindDepthTarget(CommandBuffer* command, GraphicContext* ctx, const DepthTargetInfo& info); + [[nodiscard]] std::vector + RegisterVideoOutSurfaces(GraphicContext* ctx, const std::vector& infos); + void RefreshVideoOut(VideoOutVulkanImage* image, bool render_target = false); + void UnregisterVideoOutSurfaces(const std::vector& images); + [[nodiscard]] bool ClearColorImageFromBuffer(CommandBuffer* command, uint64_t vaddr, + uint64_t size, uint32_t packed_clear); + [[nodiscard]] bool ClearDepthImageFromBuffer(CommandBuffer* command, uint64_t vaddr, + uint64_t size, uint32_t packed_clear); + [[nodiscard]] bool ClearStencilImageFromBuffer(CommandBuffer* command, uint64_t vaddr, + uint64_t size, uint32_t packed_clear); + void MarkGpuWritten(VulkanImage* image); + void PrepareHostWrite(uint64_t vaddr, uint64_t size); + void InvalidateMemoryFromGPU(uint64_t vaddr, uint64_t size, + bool formatted_buffer_write = false); + [[nodiscard]] RenderTextureVulkanImage* FindRenderTargetByRange(CommandBuffer* command, + uint64_t vaddr, uint64_t size); + [[nodiscard]] VkImageView GetRenderTargetAttachmentView(GraphicContext* ctx, + RenderTextureVulkanImage* image, + VkFormat format, uint32_t level); + [[nodiscard]] VkImageView GetRenderTargetSampledView(GraphicContext* ctx, + RenderTextureVulkanImage* image, + VkFormat view_format, int variant, + uint32_t base_level, uint32_t level_count); + [[nodiscard]] VkImageView GetRenderTargetStorageView(GraphicContext* ctx, + RenderTextureVulkanImage* image, + VkFormat view_format, uint32_t base_level, + uint32_t level_count); + [[nodiscard]] VkImageView GetStorageTextureSampledView(GraphicContext* ctx, + StorageTextureVulkanImage* image, + const ImageInfo& info); + [[nodiscard]] DepthStencilVulkanImage* FindDepthTargetByRange(uint64_t vaddr, uint64_t size); + [[nodiscard]] bool HasPageOverlap(uint64_t vaddr, uint64_t size); + [[nodiscard]] bool HasRangeOverlap(uint64_t vaddr, uint64_t size); + [[nodiscard]] bool HasGpuModifiedRangeOverlap(uint64_t vaddr, uint64_t size); + [[nodiscard]] bool HasGpuTargetPageOverlap(uint64_t vaddr, uint64_t size); + void RegisterMeta(uint64_t vaddr, uint64_t size); + [[nodiscard]] bool IsMeta(uint64_t vaddr); + [[nodiscard]] bool IsMetaRange(uint64_t vaddr, uint64_t size); + [[nodiscard]] bool HasMetaOverlap(uint64_t vaddr, uint64_t size); + [[nodiscard]] bool IsMetaGpuModified(uint64_t vaddr, uint64_t size); + [[nodiscard]] bool IsMetaCleared(uint64_t vaddr, uint32_t slice); + [[nodiscard]] bool ClearMeta(uint64_t vaddr); + [[nodiscard]] bool TouchMeta(uint64_t vaddr, uint32_t slice, bool is_clear); + [[nodiscard]] bool InvalidateMemory(PageFaultAccess access, uint64_t vaddr, uint64_t size, + PageFaultPhase phase) noexcept; + void UnmapMemory(uint64_t vaddr, uint64_t size); + + static void DeleteGpuTexture(GraphicContext* ctx, GpuTextureVulkanImage* image, + VulkanMemory* mem); + static void DeleteRenderTexture(GraphicContext* ctx, RenderTextureVulkanImage* image, + VulkanMemory* mem); + static void DeleteDepthStencil(GraphicContext* ctx, DepthStencilVulkanImage* image, + VulkanMemory* mem); + static void DeleteVideoOut(GraphicContext* ctx, VideoOutVulkanImage* image, VulkanMemory* mem); + + VulkanImage* GetDummySampledTexture(bool uint_format, bool image_3d); + VulkanImage* GetDummyStorageTexture(bool uint_format, bool image_3d); + +private: + struct CachedImage; + struct ReadbackWorker; + struct MetaDataInfo { + uint64_t size = 0; + uint32_t clear_mask = 0; + bool gpu_modified = false; + }; + static void DeleteImageViews(GraphicContext* ctx, VulkanImage* image); + [[nodiscard]] bool HasMetaOverlapLocked(uint64_t vaddr, uint64_t size) const; + [[nodiscard]] CachedImage* FindGpuReadbackPageCandidateLocked(uint64_t vaddr, + uint64_t size) const; + void RequireNoMetaOverlapLocked(uint64_t vaddr, uint64_t size) const; + void MarkSampledAliasesCpuDirtyLocked(uint64_t vaddr, uint64_t size); + void RetireImages(const std::vector& retire); + void SynchronizeRenderTargetToBufferLocked(CachedImage& cached); + + Common::Mutex m_dummy_mutex; + std::array m_dummy_sampled_textures {}; + std::array m_dummy_storage_textures {}; + std::array m_dummy_sampled_memory {}; + std::array m_dummy_storage_memory {}; + GraphicContext* m_dummy_ctx = nullptr; + TrackingSpinLock m_lock; + std::mutex m_fault_mutex; + MemoryTracker m_memory_tracker; + MemoryTracker m_metadata_tracker; + Tiler m_tiler; + BufferCache& m_buffer_cache; + ResourceMutex& m_resource_mutex; + std::vector> m_images; + std::map m_surface_metas; + std::unique_ptr m_readback; + std::vector m_buffer_transition_linear; + std::vector m_buffer_transition_guest; +}; + +} // namespace Libs::Graphics + +#endif // EMULATOR_SRC_GRAPHICS_HOST_GPU_RENDERER_TEXTURECACHE_H_ diff --git a/src/graphics/host_gpu/renderer/tiler.cpp b/src/graphics/host_gpu/renderer/tiler.cpp new file mode 100644 index 000000000..b1173ce9a --- /dev/null +++ b/src/graphics/host_gpu/renderer/tiler.cpp @@ -0,0 +1,124 @@ +#include "graphics/host_gpu/renderer/tiler.h" + +#include "common/assert.h" +#include "graphics/guest_gpu/gpu_defs.h" +#include "graphics/guest_gpu/tile.h" +#include "graphics/host_gpu/graphicContext.h" +#include "graphics/host_gpu/objects/textureCommon.h" +#include "graphics/host_gpu/renderer/image.h" +#include "graphics/host_gpu/utils.h" + +namespace Libs::Graphics { + +void Tiler::DetileImage(GraphicContext* ctx, GpuTextureVulkanImage* image, + const ImageInfo& info, const BufferImageCopySource& source, + bool refresh, bool storage) const { + if (ctx == nullptr || image == nullptr || info.address == 0 || info.size == 0 || + info.width == 0 || info.height == 0 || info.depth == 0 || info.levels == 0 || + info.levels >= 16 || !source.cpu_current || source.address != info.address || + source.size != info.size || (source.buffer == nullptr && source.offset != 0)) { + EXIT("Tiler: unsupported sampled-image detile, ctx=%p image=%p source_buffer=%p " + "source=0x%016" PRIx64 "+0x%016" PRIx64 " offset=0x%016" PRIx64 + " current=%d image=0x%016" PRIx64 "+0x%016" PRIx64 + " extent=%ux%ux%u levels=%u tile=%u storage=%d\n", + static_cast(ctx), static_cast(image), + static_cast(source.buffer), source.address, source.size, source.offset, + source.cpu_current, info.address, info.size, info.width, info.height, info.depth, + info.levels, info.tile, storage); + } + if (refresh) { + VulkanDeviceWaitIdle(ctx); + } + const bool array_texture = TextureIsLayeredTexture(info.type); + const bool volume_texture = TextureIs3DTexture(info.type); + auto layout = TextureCalcUploadLayout( + info.format, info.width, info.height, info.levels, info.depth, info.pitch, info.tile, + info.size, true, false, volume_texture, storage ? "StorageTextureCache" : "TextureCache"); + const auto slice_layout = TextureUploadSliceLayout::MipChainPerSlice; + auto regions = TextureBuildUploadRegions(layout, image->format, info.width, info.height, + info.depth, info.levels, array_texture, volume_texture, + TextureUploadDestination::MipLevels, slice_layout); + // Keep the buffer owner explicit at the detiling seam. The current PS5 + // backend consumes the coherent guest publication; a GPU detiler can consume source.buffer and + // source.offset here without changing TextureCache ownership or alias classification. + TextureUploadGuestImage( + ctx, image, reinterpret_cast(source.address), info.size, regions, layout, + info.format, info.width, info.height, info.depth, info.levels, slice_layout, + storage ? "StorageTextureCache" : "TextureCache", + static_cast(storage ? VK_IMAGE_LAYOUT_GENERAL + : VK_IMAGE_LAYOUT_SHADER_READ_ONLY_OPTIMAL)); +} + +void Tiler::DetileImage(GraphicContext* ctx, DepthStencilVulkanImage* image, + const DepthTargetInfo& info, const BufferImageCopySource& source, + bool refresh) const { + const bool d16 = info.guest_format == Prospero::GpuEnumValue(Prospero::BufferFormat::k16UNorm) && + info.format == VK_FORMAT_D16_UNORM && info.bytes_per_element == 2; + const bool d32 = + info.guest_format == Prospero::GpuEnumValue(Prospero::BufferFormat::k32Float) && + (info.format == VK_FORMAT_D32_SFLOAT || info.format == VK_FORMAT_D32_SFLOAT_S8_UINT) && + info.bytes_per_element == 4; + if (ctx == nullptr || image == nullptr || info.address == 0 || info.size == 0 || + info.width == 0 || info.height == 0 || info.pitch < info.width || + info.tile_mode != Prospero::GpuEnumValue(Prospero::TileMode::kDepth) || (!d16 && !d32) || + !source.cpu_current || source.address != info.address || source.size != info.size || + (source.buffer == nullptr && source.offset != 0)) { + EXIT("Tiler: unsupported depth detile, ctx=%p image=%p source_buffer=%p " + "source=0x%016" PRIx64 "+0x%016" PRIx64 " offset=0x%016" PRIx64 + " current=%d depth=0x%016" PRIx64 "+0x%016" PRIx64 + " extent=%ux%u pitch=%u tile=%u format=%d guest_format=%u bpe=%u\n", + static_cast(ctx), static_cast(image), + static_cast(source.buffer), source.address, source.size, source.offset, + source.cpu_current, info.address, info.size, info.width, info.height, info.pitch, + info.tile_mode, static_cast(info.format), info.guest_format, + info.bytes_per_element); + } + if (refresh) { + VulkanDeviceWaitIdle(ctx); + } + UtilScratchBuffer linear(info.size); + TileConvertTiledToLinearDepth(linear.Data(), reinterpret_cast(source.address), + info.guest_format, info.width, info.height, info.pitch, + info.size); + UtilFillDepthImage(ctx, image, linear.Data(), info.size, info.pitch); +} + +void Tiler::TileImage(void* dst, const void* src, const RenderTargetInfo& info) const { + const bool supported_element = info.bytes_per_element == 1 || info.bytes_per_element == 2 || + info.bytes_per_element == 4 || info.bytes_per_element == 8; + if (dst == nullptr || src == nullptr || info.address == 0 || info.size == 0 || + info.width == 0 || info.height == 0 || info.pitch < info.width || + info.tile_mode != Prospero::GpuEnumValue(Prospero::TileMode::kRenderTarget) || info.levels != 1 || + !supported_element) { + EXIT("Tiler: unsupported render-target tile, dst=%p src=%p " + "addr=0x%016" PRIx64 "+0x%016" PRIx64 + " extent=%ux%u pitch=%u levels=%u tile=%u bpe=%u\n", + dst, src, info.address, info.size, info.width, info.height, info.pitch, info.levels, + info.tile_mode, info.bytes_per_element); + } + TileConvertLinearToTiledRenderTarget(dst, src, info.width, info.height, info.pitch, + info.bytes_per_element, info.size); +} + +void Tiler::TileImage(void* dst, const void* src, const DepthTargetInfo& info) const { + const bool supported_format = + (info.guest_format == Prospero::GpuEnumValue(Prospero::BufferFormat::k16UNorm) && + info.format == VK_FORMAT_D16_UNORM && info.bytes_per_element == 2) || + (info.guest_format == Prospero::GpuEnumValue(Prospero::BufferFormat::k32Float) && + info.format == VK_FORMAT_D32_SFLOAT && info.bytes_per_element == 4); + if (dst == nullptr || src == nullptr || info.address == 0 || info.size == 0 || + info.stencil_address != 0 || info.stencil_size != 0 || info.width == 0 || + info.height == 0 || info.pitch < info.width || + info.tile_mode != Prospero::GpuEnumValue(Prospero::TileMode::kDepth) || !supported_format) { + EXIT("Tiler: unsupported depth-target tile, dst=%p src=%p " + "depth=0x%016" PRIx64 "+0x%016" PRIx64 " stencil=0x%016" PRIx64 "+0x%016" PRIx64 + " extent=%ux%u pitch=%u tile=%u format=%d guest_format=%u bpe=%u\n", + dst, src, info.address, info.size, info.stencil_address, info.stencil_size, info.width, + info.height, info.pitch, info.tile_mode, static_cast(info.format), + info.guest_format, info.bytes_per_element); + } + TileConvertLinearToTiledDepth(dst, src, info.guest_format, info.width, info.height, info.pitch, + info.size); +} + +} // namespace Libs::Graphics diff --git a/src/graphics/host_gpu/renderer/tiler.h b/src/graphics/host_gpu/renderer/tiler.h new file mode 100644 index 000000000..c137d407c --- /dev/null +++ b/src/graphics/host_gpu/renderer/tiler.h @@ -0,0 +1,30 @@ +#ifndef EMULATOR_SRC_GRAPHICS_HOST_GPU_RENDERER_TILER_H_ +#define EMULATOR_SRC_GRAPHICS_HOST_GPU_RENDERER_TILER_H_ + +#include "common/common.h" +#include "graphics/host_gpu/renderer/bufferCache.h" +#include "graphics/host_gpu/renderer/imageInfo.h" + +namespace Libs::Graphics { + +struct DepthStencilVulkanImage; +struct GpuTextureVulkanImage; +struct GraphicContext; + +class Tiler final { +public: + Tiler() = default; + KYTY_CLASS_NO_COPY(Tiler); + + void DetileImage(GraphicContext* ctx, GpuTextureVulkanImage* image, const ImageInfo& info, + const BufferImageCopySource& source, bool refresh, bool storage) const; + void DetileImage(GraphicContext* ctx, DepthStencilVulkanImage* image, + const DepthTargetInfo& info, const BufferImageCopySource& source, + bool refresh) const; + void TileImage(void* dst, const void* src, const RenderTargetInfo& info) const; + void TileImage(void* dst, const void* src, const DepthTargetInfo& info) const; +}; + +} // namespace Libs::Graphics + +#endif // EMULATOR_SRC_GRAPHICS_HOST_GPU_RENDERER_TILER_H_ diff --git a/src/graphics/host_gpu/utils.cpp b/src/graphics/host_gpu/utils.cpp new file mode 100644 index 000000000..4239b20ac --- /dev/null +++ b/src/graphics/host_gpu/utils.cpp @@ -0,0 +1,1161 @@ +#include "graphics/host_gpu/utils.h" + +#include "common/assert.h" +#include "common/logging/log.h" +#include "common/profiler.h" +#include "common/threads.h" +#include "graphics/host_gpu/graphicContext.h" +#include "graphics/host_gpu/renderer/render.h" +#include "graphics/host_gpu/renderer/renderContext.h" +#include "graphics/host_gpu/vma.h" +#include "graphics/presentation/window.h" + +#include +#include +#include +#include + +namespace Libs::Graphics { + +static Common::Mutex g_scratch_buffer_mutex; +static std::unique_ptr g_scratch_buffer; +static uint64_t g_scratch_buffer_capacity = 0; + +UtilScratchBuffer::UtilScratchBuffer(uint64_t size) { + EXIT_IF(size == 0); + + g_scratch_buffer_mutex.Lock(); + + if (g_scratch_buffer_capacity < size) { + g_scratch_buffer = std::make_unique(size); + g_scratch_buffer_capacity = size; + } + + m_data = g_scratch_buffer.get(); +} + +UtilScratchBuffer::~UtilScratchBuffer() { + g_scratch_buffer_mutex.Unlock(); +} + +bool UtilBufferIsTiled(uint64_t vaddr, uint64_t size) { + if ((size & 0x7u) == 0) { + if (size == 0) { + return false; + } + + const auto* ptr = reinterpret_cast(vaddr); + const auto* ptr_end = reinterpret_cast(vaddr + size); + const auto element = *ptr; + + for (; ptr < ptr_end; ptr++) { + if (element != *ptr) { + return true; + } + } + return false; + } + return true; +} + +void VulkanDeviceWaitIdle(GraphicContext* ctx) { + EXIT_IF(ctx == nullptr); + EXIT_IF(ctx->device == nullptr); + + Common::Mutex* locked[GraphicContext::QUEUES_NUM] {}; + int locked_num = 0; + + for (int id = 0; id < GraphicContext::QUEUES_NUM; id++) { + auto* mutex = ctx->queues[id].mutex; + if (mutex == nullptr || ctx->queues[id].vk_queue == nullptr) { + continue; + } + + bool already_locked = false; + for (int i = 0; i < locked_num; i++) { + if (locked[i] == mutex) { + already_locked = true; + break; + } + } + + if (!already_locked) { + mutex->Lock(); + locked[locked_num++] = mutex; + } + } + + auto result = vkDeviceWaitIdle(ctx->device); + + for (int i = locked_num - 1; i >= 0; i--) { + locked[i]->Unlock(); + } + + if (result != VK_SUCCESS) { + LOGF("vkDeviceWaitIdle failed: %s (%d)\n", string_VkResult(result), + static_cast(result)); + } + EXIT_NOT_IMPLEMENTED(result != VK_SUCCESS); +} + +void UtilResetImageViews(VulkanImage* image) { + EXIT_IF(image == nullptr); + + for (auto& view: image->image_view) { + view = nullptr; + } +} + +void UtilCreateImageView(GraphicContext* ctx, VulkanImage* image, int view_index, + VkImageViewType view_type, VkImageAspectFlags aspect_mask, + VkComponentMapping components, uint32_t base_array_layer, + uint32_t base_mip_level, uint32_t layer_count, uint32_t level_count, + VkFormat view_format, VkImageUsageFlags view_usage) { + EXIT_IF(ctx == nullptr); + EXIT_IF(image == nullptr); + if (view_index < 0 || view_index >= VulkanImage::VIEW_MAX || image->image == nullptr || + image->image_view[view_index] != nullptr) { + EXIT("invalid image-view creation target: image=%p index=%d current_view=%d\n", + static_cast(image), view_index, + view_index >= 0 && view_index < VulkanImage::VIEW_MAX && + image->image_view[view_index] != nullptr); + } + + VkImageViewUsageCreateInfo usage_info {}; + usage_info.sType = VK_STRUCTURE_TYPE_IMAGE_VIEW_USAGE_CREATE_INFO; + usage_info.pNext = nullptr; + usage_info.usage = view_usage; + + VkImageViewCreateInfo create_info {}; + create_info.sType = VK_STRUCTURE_TYPE_IMAGE_VIEW_CREATE_INFO; + create_info.pNext = (view_usage != 0 ? &usage_info : nullptr); + create_info.flags = 0; + create_info.image = image->image; + create_info.viewType = view_type; + create_info.format = (view_format != VK_FORMAT_UNDEFINED ? view_format : image->format); + create_info.components = components; + create_info.subresourceRange.aspectMask = aspect_mask; + create_info.subresourceRange.baseArrayLayer = base_array_layer; + create_info.subresourceRange.baseMipLevel = base_mip_level; + create_info.subresourceRange.layerCount = layer_count; + create_info.subresourceRange.levelCount = level_count; + + const auto result = + vkCreateImageView(ctx->device, &create_info, nullptr, &image->image_view[view_index]); + if (result != VK_SUCCESS || image->image_view[view_index] == nullptr) { + EXIT("failed to create image view: result=%d image_format=%d view_format=%d index=%d\n", + static_cast(result), static_cast(image->format), + static_cast(create_info.format), view_index); + } +} + +static void SetImageLayout(VkCommandBuffer buffer, VulkanImage* dst_image, uint32_t base_level, + uint32_t levels, VkImageAspectFlags aspect_mask, + VkImageLayout old_image_layout, VkImageLayout new_image_layout); + +class ReusableStagingBuffer { +public: + ReusableStagingBuffer() = default; + ~ReusableStagingBuffer() = default; + KYTY_CLASS_NO_COPY(ReusableStagingBuffer); + + void UploadToBuffer(GraphicContext* ctx, VulkanBuffer* dst_buffer, const void* src_data, + uint64_t size, uint64_t dst_offset = 0) { + Common::LockGuard lock(m_mutex); + + CopyFromHost(ctx, src_data, size, VK_BUFFER_USAGE_TRANSFER_SRC_BIT); + VulkanDeviceWaitIdle(ctx); + + CommandBuffer buffer(GraphicContext::QUEUE_UTIL); + EXIT_NOT_IMPLEMENTED(buffer.IsInvalid()); + auto* vk_buffer = buffer.GetPool()->buffers[buffer.GetIndex()]; + buffer.Begin(); + VkBufferMemoryBarrier pre_barrier {}; + pre_barrier.sType = VK_STRUCTURE_TYPE_BUFFER_MEMORY_BARRIER; + pre_barrier.srcAccessMask = VK_ACCESS_MEMORY_READ_BIT | VK_ACCESS_MEMORY_WRITE_BIT; + pre_barrier.dstAccessMask = VK_ACCESS_TRANSFER_WRITE_BIT; + pre_barrier.srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; + pre_barrier.dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; + pre_barrier.buffer = dst_buffer->buffer; + pre_barrier.offset = dst_offset; + pre_barrier.size = size; + vkCmdPipelineBarrier(vk_buffer, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, + VK_PIPELINE_STAGE_TRANSFER_BIT, 0, 0, nullptr, 1, &pre_barrier, 0, + nullptr); + VkBufferCopy copy_region {}; + copy_region.dstOffset = dst_offset; + copy_region.size = size; + vkCmdCopyBuffer(vk_buffer, m_buffer.buffer, dst_buffer->buffer, 1, ©_region); + VkBufferMemoryBarrier post_barrier {}; + post_barrier.sType = VK_STRUCTURE_TYPE_BUFFER_MEMORY_BARRIER; + post_barrier.srcAccessMask = VK_ACCESS_TRANSFER_WRITE_BIT; + post_barrier.dstAccessMask = VK_ACCESS_VERTEX_ATTRIBUTE_READ_BIT | + VK_ACCESS_INDEX_READ_BIT | VK_ACCESS_SHADER_READ_BIT | + VK_ACCESS_SHADER_WRITE_BIT; + post_barrier.srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; + post_barrier.dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; + post_barrier.buffer = dst_buffer->buffer; + post_barrier.offset = dst_offset; + post_barrier.size = size; + vkCmdPipelineBarrier( + vk_buffer, VK_PIPELINE_STAGE_TRANSFER_BIT, + VK_PIPELINE_STAGE_VERTEX_INPUT_BIT | VK_PIPELINE_STAGE_VERTEX_SHADER_BIT | + VK_PIPELINE_STAGE_FRAGMENT_SHADER_BIT | VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT, + 0, 0, nullptr, 1, &post_barrier, 0, nullptr); + buffer.End(); + buffer.Execute(); + buffer.WaitForFence(); + } + + void UploadToImage(GraphicContext* ctx, VulkanImage* dst_image, const void* src_data, + uint64_t size, uint32_t src_pitch, uint64_t dst_layout) { + Common::LockGuard lock(m_mutex); + + CopyFromHost(ctx, src_data, size, VK_BUFFER_USAGE_TRANSFER_SRC_BIT); + + CommandBuffer buffer(GraphicContext::QUEUE_UTIL); + // buffer.SetQueue(GraphicContext::QUEUE_UTIL); + + EXIT_NOT_IMPLEMENTED(buffer.IsInvalid()); + + buffer.Begin(); + UtilBufferToImage(&buffer, &m_buffer, src_pitch, dst_image, dst_layout); + buffer.End(); + buffer.Execute(); + buffer.WaitForFence(); + } + + void UploadToDepthImage(GraphicContext* ctx, DepthStencilVulkanImage* image, + const void* src_data, uint64_t size, uint32_t src_pitch) { + Common::LockGuard lock(m_mutex); + CopyFromHost(ctx, src_data, size, VK_BUFFER_USAGE_TRANSFER_SRC_BIT); + CommandBuffer buffer(GraphicContext::QUEUE_UTIL); + EXIT_NOT_IMPLEMENTED(buffer.IsInvalid()); + buffer.Begin(); + auto* vk_buffer = buffer.GetPool()->buffers[buffer.GetIndex()]; + VkImageAspectFlags aspects = VK_IMAGE_ASPECT_DEPTH_BIT; + if (image->format == VK_FORMAT_D16_UNORM_S8_UINT || + image->format == VK_FORMAT_D24_UNORM_S8_UINT || + image->format == VK_FORMAT_D32_SFLOAT_S8_UINT) { + aspects |= VK_IMAGE_ASPECT_STENCIL_BIT; + } + SetImageLayout(vk_buffer, image, 0, 1, aspects, VK_IMAGE_LAYOUT_UNDEFINED, + VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL); + VkBufferImageCopy region {}; + region.bufferRowLength = src_pitch == image->extent.width ? 0 : src_pitch; + region.imageSubresource.aspectMask = VK_IMAGE_ASPECT_DEPTH_BIT; + region.imageSubresource.layerCount = 1; + region.imageExtent = {image->extent.width, image->extent.height, 1}; + vkCmdCopyBufferToImage(vk_buffer, m_buffer.buffer, image->image, + VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL, 1, ®ion); + SetImageLayout(vk_buffer, image, 0, 1, aspects, VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL, + VK_IMAGE_LAYOUT_DEPTH_STENCIL_ATTACHMENT_OPTIMAL); + buffer.End(); + buffer.Execute(); + buffer.WaitForFence(); + } + + void UploadToImage(GraphicContext* ctx, VulkanImage* dst_image, const void* src_data, + uint64_t size, const std::vector& regions, + uint64_t dst_layout) { + Common::LockGuard lock(m_mutex); + + CopyFromHost(ctx, src_data, size, VK_BUFFER_USAGE_TRANSFER_SRC_BIT); + + CommandBuffer buffer(GraphicContext::QUEUE_UTIL); + // buffer.SetQueue(GraphicContext::QUEUE_UTIL); + + EXIT_NOT_IMPLEMENTED(buffer.IsInvalid()); + + buffer.Begin(); + UtilBufferToImage(&buffer, &m_buffer, dst_image, regions, dst_layout); + buffer.End(); + buffer.Execute(); + buffer.WaitForFence(); + } + + void DownloadFromImage(GraphicContext* ctx, void* dst_data, uint64_t size, uint32_t dst_pitch, + VulkanImage* src_image, uint64_t src_layout, VkImageAspectFlags aspect) { + Common::LockGuard lock(m_mutex); + + EnsureBuffer(ctx, size, VK_BUFFER_USAGE_TRANSFER_DST_BIT); + + CommandBuffer buffer(GraphicContext::QUEUE_UTIL); + // buffer.SetQueue(GraphicContext::QUEUE_UTIL); + + EXIT_NOT_IMPLEMENTED(buffer.IsInvalid()); + + buffer.Begin(); + UtilImageToBuffer(&buffer, src_image, &m_buffer, dst_pitch, src_layout, aspect); + buffer.End(); + buffer.Execute(); + buffer.WaitForFence(); + + std::memcpy(dst_data, m_mapped_data, size); + } + +private: + void CopyFromHost(GraphicContext* ctx, const void* src_data, uint64_t size, + VkBufferUsageFlags usage) { + EXIT_IF(ctx == nullptr); + EXIT_IF(src_data == nullptr); + EXIT_IF(size == 0); + + EnsureBuffer(ctx, size, usage); + + std::memcpy(m_mapped_data, src_data, size); + } + + void EnsureBuffer(GraphicContext* ctx, uint64_t size, VkBufferUsageFlags usage) { + EXIT_IF(ctx == nullptr); + EXIT_IF(size == 0); + + if (m_capacity < size || m_buffer.usage != usage) { + if (m_buffer.buffer != nullptr) { + VulkanUnmapMemory(ctx, &m_buffer.memory); + m_mapped_data = nullptr; + VulkanDeleteBuffer(ctx, &m_buffer); + } + + m_buffer.usage = usage; + m_buffer.memory.property = + VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT | VK_MEMORY_PROPERTY_HOST_COHERENT_BIT; + VulkanCreateBuffer(ctx, size, &m_buffer); + m_capacity = size; + + VulkanMapMemory(ctx, &m_buffer.memory, &m_mapped_data); + EXIT_IF(m_mapped_data == nullptr); + } + } + + Common::Mutex m_mutex; + VulkanBuffer m_buffer; + uint64_t m_capacity = 0; + void* m_mapped_data = nullptr; +}; + +// Do not replace with one +static ReusableStagingBuffer g_texture_staging_buffer; +static ReusableStagingBuffer g_vertex_staging_buffer; +static ReusableStagingBuffer g_index_staging_buffer; +static ReusableStagingBuffer g_readback_staging_buffer; + +static Common::Mutex g_compressed_image_copy_mutex; +static VulkanBuffer g_compressed_image_copy_buffer; + +bool UtilIsBcFormat(VkFormat format) { + return format == VK_FORMAT_BC1_RGB_UNORM_BLOCK || format == VK_FORMAT_BC1_RGB_SRGB_BLOCK || + format == VK_FORMAT_BC1_RGBA_UNORM_BLOCK || format == VK_FORMAT_BC1_RGBA_SRGB_BLOCK || + format == VK_FORMAT_BC2_UNORM_BLOCK || format == VK_FORMAT_BC2_SRGB_BLOCK || + format == VK_FORMAT_BC3_UNORM_BLOCK || format == VK_FORMAT_BC3_SRGB_BLOCK || + format == VK_FORMAT_BC4_UNORM_BLOCK || format == VK_FORMAT_BC4_SNORM_BLOCK || + format == VK_FORMAT_BC5_UNORM_BLOCK || format == VK_FORMAT_BC5_SNORM_BLOCK || + format == VK_FORMAT_BC6H_UFLOAT_BLOCK || format == VK_FORMAT_BC6H_SFLOAT_BLOCK || + format == VK_FORMAT_BC7_UNORM_BLOCK || format == VK_FORMAT_BC7_SRGB_BLOCK; +} + +uint32_t UtilGetBcBlockSize(VkFormat format) { + switch (format) { + case VK_FORMAT_BC1_RGB_UNORM_BLOCK: + case VK_FORMAT_BC1_RGB_SRGB_BLOCK: + case VK_FORMAT_BC1_RGBA_UNORM_BLOCK: + case VK_FORMAT_BC1_RGBA_SRGB_BLOCK: + case VK_FORMAT_BC4_UNORM_BLOCK: + case VK_FORMAT_BC4_SNORM_BLOCK: return 8; + case VK_FORMAT_BC2_UNORM_BLOCK: + case VK_FORMAT_BC2_SRGB_BLOCK: + case VK_FORMAT_BC3_UNORM_BLOCK: + case VK_FORMAT_BC3_SRGB_BLOCK: + case VK_FORMAT_BC5_UNORM_BLOCK: + case VK_FORMAT_BC5_SNORM_BLOCK: + case VK_FORMAT_BC6H_UFLOAT_BLOCK: + case VK_FORMAT_BC6H_SFLOAT_BLOCK: + case VK_FORMAT_BC7_UNORM_BLOCK: + case VK_FORMAT_BC7_SRGB_BLOCK: return 16; + default: return 0; + } +} + +static uint32_t GetFormatTexelSize(VkFormat format) { + switch (format) { + case VK_FORMAT_R32G32_SFLOAT: + case VK_FORMAT_R32G32_UINT: return 8; + case VK_FORMAT_R32G32B32A32_SFLOAT: + case VK_FORMAT_R32G32B32A32_UINT: return 16; + default: return 0; + } +} + +static VulkanBuffer* GetCompressedImageCopyBuffer(GraphicContext* ctx, uint64_t size) { + EXIT_IF(ctx == nullptr); + EXIT_IF(size == 0); + + constexpr uint64_t buffer_size = 64ull * 1024ull * 1024ull; + EXIT_NOT_IMPLEMENTED(size > buffer_size); + + Common::LockGuard lock(g_compressed_image_copy_mutex); + + if (g_compressed_image_copy_buffer.buffer == nullptr) { + g_compressed_image_copy_buffer.usage = + VK_BUFFER_USAGE_TRANSFER_SRC_BIT | VK_BUFFER_USAGE_TRANSFER_DST_BIT; + g_compressed_image_copy_buffer.memory.property = 0; + VulkanCreateBuffer(ctx, buffer_size, &g_compressed_image_copy_buffer); + } + + return &g_compressed_image_copy_buffer; +} + +static ReusableStagingBuffer* GetStagingBuffer(StagingBufferType type) { + switch (type) { + case StagingBufferType::Texture: return &g_texture_staging_buffer; + case StagingBufferType::Vertex: return &g_vertex_staging_buffer; + case StagingBufferType::Index: return &g_index_staging_buffer; + case StagingBufferType::ReadBack: return &g_readback_staging_buffer; + default: EXIT("unknown staging buffer type\n"); + } + + return nullptr; +} + +static void SetImageLayout(VkCommandBuffer buffer, VulkanImage* dst_image, uint32_t base_level, + uint32_t levels, VkImageAspectFlags aspect_mask, + VkImageLayout old_image_layout, VkImageLayout new_image_layout) { + EXIT_IF(buffer == nullptr); + EXIT_IF(dst_image == nullptr); + + if ((old_image_layout == VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL && + new_image_layout == VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL) || + (old_image_layout == VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL && + new_image_layout == VK_IMAGE_LAYOUT_GENERAL) || + (old_image_layout == VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL && + new_image_layout == VK_IMAGE_LAYOUT_PRESENT_SRC_KHR)) { + static std::atomic log_count {0}; + if (log_count.fetch_add(1, std::memory_order_relaxed) < 64) { + LOGF("set_image_layout: image=%p type=%d tracked=%d requested=%d->%d base=%u " + "levels=%u\n", + reinterpret_cast(dst_image->image), static_cast(dst_image->type), + static_cast(dst_image->layout), static_cast(old_image_layout), + static_cast(new_image_layout), base_level, levels); + } + } + + if (old_image_layout == new_image_layout) { + dst_image->layout = new_image_layout; + return; + } + + VkImageMemoryBarrier image_memory_barrier {}; + image_memory_barrier.sType = VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER; + image_memory_barrier.pNext = nullptr; + image_memory_barrier.srcAccessMask = 0; + image_memory_barrier.dstAccessMask = 0; + image_memory_barrier.oldLayout = old_image_layout; + image_memory_barrier.newLayout = new_image_layout; + image_memory_barrier.srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; + image_memory_barrier.dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; + image_memory_barrier.image = dst_image->image; + image_memory_barrier.subresourceRange.aspectMask = aspect_mask; + image_memory_barrier.subresourceRange.baseMipLevel = base_level; + image_memory_barrier.subresourceRange.levelCount = levels; + image_memory_barrier.subresourceRange.baseArrayLayer = 0; + image_memory_barrier.subresourceRange.layerCount = dst_image->layers; + + switch (old_image_layout) { + case VK_IMAGE_LAYOUT_COLOR_ATTACHMENT_OPTIMAL: + image_memory_barrier.srcAccessMask = VK_ACCESS_COLOR_ATTACHMENT_WRITE_BIT; + break; + case VK_IMAGE_LAYOUT_DEPTH_STENCIL_ATTACHMENT_OPTIMAL: + image_memory_barrier.srcAccessMask = VK_ACCESS_DEPTH_STENCIL_ATTACHMENT_READ_BIT | + VK_ACCESS_DEPTH_STENCIL_ATTACHMENT_WRITE_BIT; + break; + case VK_IMAGE_LAYOUT_GENERAL: + image_memory_barrier.srcAccessMask = + VK_ACCESS_SHADER_READ_BIT | VK_ACCESS_SHADER_WRITE_BIT | + VK_ACCESS_COLOR_ATTACHMENT_READ_BIT | VK_ACCESS_COLOR_ATTACHMENT_WRITE_BIT | + VK_ACCESS_TRANSFER_READ_BIT | VK_ACCESS_TRANSFER_WRITE_BIT; + break; + case VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL: + image_memory_barrier.srcAccessMask = VK_ACCESS_TRANSFER_WRITE_BIT; + break; + case VK_IMAGE_LAYOUT_PREINITIALIZED: + image_memory_barrier.srcAccessMask = VK_ACCESS_HOST_WRITE_BIT; + break; + default: break; + } + + switch (new_image_layout) { + case VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL: + image_memory_barrier.dstAccessMask = VK_ACCESS_TRANSFER_WRITE_BIT; + break; + case VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL: + image_memory_barrier.dstAccessMask = VK_ACCESS_TRANSFER_READ_BIT; + break; + case VK_IMAGE_LAYOUT_SHADER_READ_ONLY_OPTIMAL: + image_memory_barrier.srcAccessMask |= + VK_ACCESS_TRANSFER_WRITE_BIT | VK_ACCESS_HOST_WRITE_BIT; + image_memory_barrier.dstAccessMask = VK_ACCESS_SHADER_READ_BIT; + break; + case VK_IMAGE_LAYOUT_GENERAL: + image_memory_barrier.dstAccessMask = + VK_ACCESS_SHADER_READ_BIT | VK_ACCESS_SHADER_WRITE_BIT; + break; + case VK_IMAGE_LAYOUT_COLOR_ATTACHMENT_OPTIMAL: + image_memory_barrier.dstAccessMask = VK_ACCESS_COLOR_ATTACHMENT_WRITE_BIT; + break; + case VK_IMAGE_LAYOUT_DEPTH_STENCIL_ATTACHMENT_OPTIMAL: + image_memory_barrier.dstAccessMask = VK_ACCESS_DEPTH_STENCIL_ATTACHMENT_READ_BIT | + VK_ACCESS_DEPTH_STENCIL_ATTACHMENT_WRITE_BIT; + break; + default: break; + } + + auto src_stages = static_cast( + VK_PIPELINE_STAGE_TOP_OF_PIPE_BIT | VK_PIPELINE_STAGE_TRANSFER_BIT | + VK_PIPELINE_STAGE_COLOR_ATTACHMENT_OUTPUT_BIT | VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT | + VK_PIPELINE_STAGE_EARLY_FRAGMENT_TESTS_BIT | VK_PIPELINE_STAGE_LATE_FRAGMENT_TESTS_BIT | + VK_PIPELINE_STAGE_HOST_BIT); + auto dest_stages = static_cast( + VK_PIPELINE_STAGE_TRANSFER_BIT | VK_PIPELINE_STAGE_VERTEX_SHADER_BIT | + VK_PIPELINE_STAGE_FRAGMENT_SHADER_BIT | VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT | + VK_PIPELINE_STAGE_COLOR_ATTACHMENT_OUTPUT_BIT | VK_PIPELINE_STAGE_EARLY_FRAGMENT_TESTS_BIT | + VK_PIPELINE_STAGE_LATE_FRAGMENT_TESTS_BIT); + + vkCmdPipelineBarrier(buffer, src_stages, dest_stages, 0, 0, nullptr, 0, nullptr, 1, + &image_memory_barrier); + + dst_image->layout = new_image_layout; +} + +static bool CopyBlockImageToCompressedImage(VkCommandBuffer vk_buffer, const ImageImageCopy& r, + VulkanImage* dst_image) { + if (r.src_image == nullptr || dst_image == nullptr || !UtilIsBcFormat(dst_image->format)) { + return false; + } + + auto src_texel_size = GetFormatTexelSize(r.src_image->format); + auto dst_block_size = UtilGetBcBlockSize(dst_image->format); + if (src_texel_size == 0 || src_texel_size != dst_block_size) { + return false; + } + + auto block_width = (r.width + 3u) / 4u; + auto block_height = (r.height + 3u) / 4u; + if (block_width > r.src_image->extent.width || block_height > r.src_image->extent.height) { + return false; + } + + auto* ctx = WindowGetGraphicContext(); + auto* copy_buffer = GetCompressedImageCopyBuffer(ctx, static_cast(block_width) * + block_height * dst_block_size); + + VkBufferImageCopy src_region {}; + src_region.bufferOffset = 0; + src_region.bufferRowLength = 0; + src_region.bufferImageHeight = 0; + src_region.imageSubresource.aspectMask = VK_IMAGE_ASPECT_COLOR_BIT; + src_region.imageSubresource.mipLevel = r.src_level; + src_region.imageSubresource.baseArrayLayer = r.src_layer; + src_region.imageSubresource.layerCount = 1; + src_region.imageOffset = {r.src_x, r.src_y, 0}; + src_region.imageExtent = {block_width, block_height, 1}; + + auto src_layout = r.src_image->layout; + SetImageLayout(vk_buffer, r.src_image, r.src_level, 1, VK_IMAGE_ASPECT_COLOR_BIT, src_layout, + VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL); + + vkCmdCopyImageToBuffer(vk_buffer, r.src_image->image, VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL, + copy_buffer->buffer, 1, &src_region); + + VkBufferMemoryBarrier barrier {}; + barrier.sType = VK_STRUCTURE_TYPE_BUFFER_MEMORY_BARRIER; + barrier.srcAccessMask = VK_ACCESS_TRANSFER_WRITE_BIT; + barrier.dstAccessMask = VK_ACCESS_TRANSFER_READ_BIT; + barrier.srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; + barrier.dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; + barrier.buffer = copy_buffer->buffer; + barrier.offset = 0; + barrier.size = VK_WHOLE_SIZE; + vkCmdPipelineBarrier(vk_buffer, VK_PIPELINE_STAGE_TRANSFER_BIT, VK_PIPELINE_STAGE_TRANSFER_BIT, + 0, 0, nullptr, 1, &barrier, 0, nullptr); + + VkBufferImageCopy dst_region {}; + dst_region.bufferOffset = 0; + dst_region.bufferRowLength = 0; + dst_region.bufferImageHeight = 0; + dst_region.imageSubresource.aspectMask = VK_IMAGE_ASPECT_COLOR_BIT; + dst_region.imageSubresource.mipLevel = r.dst_level; + dst_region.imageSubresource.baseArrayLayer = r.dst_layer; + dst_region.imageSubresource.layerCount = 1; + dst_region.imageOffset = {r.dst_x, r.dst_y, 0}; + dst_region.imageExtent = {r.width, r.height, 1}; + + vkCmdCopyBufferToImage(vk_buffer, copy_buffer->buffer, dst_image->image, + VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL, 1, &dst_region); + + SetImageLayout(vk_buffer, r.src_image, r.src_level, 1, VK_IMAGE_ASPECT_COLOR_BIT, + VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL, src_layout); + + return true; +} + +void UtilBufferToImage(CommandBuffer* buffer, VulkanBuffer* src_buffer, uint32_t src_pitch, + VulkanImage* dst_image, uint64_t dst_layout) { + EXIT_IF(src_buffer == nullptr); + EXIT_IF(src_buffer->buffer == nullptr); + EXIT_IF(dst_image == nullptr); + EXIT_IF(dst_image->image == nullptr); + + auto* vk_buffer = buffer->GetPool()->buffers[buffer->GetIndex()]; + + SetImageLayout(vk_buffer, dst_image, 0, 1, VK_IMAGE_ASPECT_COLOR_BIT, VK_IMAGE_LAYOUT_UNDEFINED, + VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL); + + VkBufferImageCopy region {}; + region.bufferOffset = 0; + region.bufferRowLength = (src_pitch != dst_image->extent.width ? src_pitch : 0); + region.bufferImageHeight = 0; + + region.imageSubresource.aspectMask = VK_IMAGE_ASPECT_COLOR_BIT; + region.imageSubresource.mipLevel = 0; + region.imageSubresource.baseArrayLayer = 0; + region.imageSubresource.layerCount = 1; + + region.imageOffset = {0, 0, 0}; + region.imageExtent = {dst_image->extent.width, dst_image->extent.height, 1}; + + vkCmdCopyBufferToImage(vk_buffer, src_buffer->buffer, dst_image->image, + VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL, 1, ®ion); + + SetImageLayout(vk_buffer, dst_image, 0, 1, VK_IMAGE_ASPECT_COLOR_BIT, + VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL, static_cast(dst_layout)); +} + +void UtilImageToBuffer(CommandBuffer* buffer, VulkanImage* src_image, VulkanBuffer* dst_buffer, + uint32_t dst_pitch, uint64_t src_layout, VkImageAspectFlags aspect) { + EXIT_IF(dst_buffer == nullptr); + EXIT_IF(dst_buffer->buffer == nullptr); + EXIT_IF(src_image == nullptr); + EXIT_IF(src_image->image == nullptr); + if (aspect != VK_IMAGE_ASPECT_COLOR_BIT && aspect != VK_IMAGE_ASPECT_DEPTH_BIT && + aspect != VK_IMAGE_ASPECT_STENCIL_BIT) { + EXIT("unsupported image readback aspect mask: 0x%x\n", aspect); + } + + auto* vk_buffer = buffer->GetPool()->buffers[buffer->GetIndex()]; + + SetImageLayout(vk_buffer, src_image, 0, 1, aspect, src_image->layout, + VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL); + + VkBufferImageCopy region {}; + region.bufferOffset = 0; + region.bufferRowLength = (dst_pitch != src_image->extent.width ? dst_pitch : 0); + region.bufferImageHeight = 0; + + region.imageSubresource.aspectMask = aspect; + region.imageSubresource.mipLevel = 0; + region.imageSubresource.baseArrayLayer = 0; + region.imageSubresource.layerCount = 1; + + region.imageOffset = {0, 0, 0}; + region.imageExtent = {src_image->extent.width, src_image->extent.height, 1}; + + vkCmdCopyImageToBuffer(vk_buffer, src_image->image, VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL, + dst_buffer->buffer, 1, ®ion); + + SetImageLayout(vk_buffer, src_image, 0, 1, aspect, VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL, + static_cast(src_layout)); +} + +void UtilBufferToImage(CommandBuffer* buffer, VulkanBuffer* src_buffer, VulkanImage* dst_image, + const std::vector& regions, uint64_t dst_layout) { + EXIT_IF(src_buffer == nullptr); + EXIT_IF(src_buffer->buffer == nullptr); + EXIT_IF(dst_image == nullptr); + EXIT_IF(dst_image->image == nullptr); + + auto* vk_buffer = buffer->GetPool()->buffers[buffer->GetIndex()]; + + static thread_local std::vector vk_regions; + vk_regions.resize(regions.size()); + + uint32_t index = 0; + for (const auto& r: regions) { + auto& region = vk_regions[index]; + region.bufferOffset = r.offset; + region.bufferRowLength = (r.width != r.pitch ? r.pitch : 0); + region.bufferImageHeight = 0; + region.imageSubresource.aspectMask = VK_IMAGE_ASPECT_COLOR_BIT; + region.imageSubresource.mipLevel = r.dst_level; + region.imageSubresource.baseArrayLayer = r.dst_layer; + region.imageSubresource.layerCount = 1; + region.imageOffset = {r.dst_x, r.dst_y, r.dst_z}; + region.imageExtent = {r.width, (r.copy_height != 0 ? r.copy_height : r.height), 1}; + index++; + } + + SetImageLayout(vk_buffer, dst_image, 0, VK_REMAINING_MIP_LEVELS, VK_IMAGE_ASPECT_COLOR_BIT, + VK_IMAGE_LAYOUT_UNDEFINED, VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL); + + vkCmdCopyBufferToImage(vk_buffer, src_buffer->buffer, dst_image->image, + VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL, + static_cast(vk_regions.size()), vk_regions.data()); + + SetImageLayout(vk_buffer, dst_image, 0, VK_REMAINING_MIP_LEVELS, VK_IMAGE_ASPECT_COLOR_BIT, + VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL, static_cast(dst_layout)); +} + +void UtilImageToImage(CommandBuffer* buffer, const std::vector& regions, + VulkanImage* dst_image, uint64_t dst_layout) { + EXIT_IF(dst_image == nullptr); + EXIT_IF(dst_image->image == nullptr); + + auto* vk_buffer = buffer->GetPool()->buffers[buffer->GetIndex()]; + + bool same_image_copy = false; + for (const auto& r: regions) { + if (r.src_image == dst_image) { + same_image_copy = true; + break; + } + } + + if (same_image_copy) { + SetImageLayout(vk_buffer, dst_image, 0, VK_REMAINING_MIP_LEVELS, VK_IMAGE_ASPECT_COLOR_BIT, + dst_image->layout, VK_IMAGE_LAYOUT_GENERAL); + + for (const auto& r: regions) { + VkImageCopy region; + + region.srcSubresource.aspectMask = VK_IMAGE_ASPECT_COLOR_BIT; + region.srcSubresource.mipLevel = r.src_level; + region.srcSubresource.baseArrayLayer = r.src_layer; + region.srcSubresource.layerCount = 1; + region.srcOffset = {r.src_x, r.src_y, r.src_z}; + region.dstSubresource.aspectMask = VK_IMAGE_ASPECT_COLOR_BIT; + region.dstSubresource.mipLevel = r.dst_level; + region.dstSubresource.baseArrayLayer = r.dst_layer; + region.dstSubresource.layerCount = 1; + region.dstOffset = {r.dst_x, r.dst_y, r.dst_z}; + region.extent = {r.width, r.height, 1}; + + auto src_layout = VK_IMAGE_LAYOUT_GENERAL; + auto restore_layout = VK_IMAGE_LAYOUT_GENERAL; + if (r.src_image != dst_image) { + restore_layout = r.src_image->layout; + SetImageLayout(vk_buffer, r.src_image, r.src_level, 1, VK_IMAGE_ASPECT_COLOR_BIT, + restore_layout, VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL); + src_layout = VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL; + } + + vkCmdCopyImage(vk_buffer, r.src_image->image, src_layout, dst_image->image, + VK_IMAGE_LAYOUT_GENERAL, 1, ®ion); + + if (r.src_image != dst_image) { + SetImageLayout(vk_buffer, r.src_image, r.src_level, 1, VK_IMAGE_ASPECT_COLOR_BIT, + VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL, restore_layout); + } + } + + SetImageLayout(vk_buffer, dst_image, 0, VK_REMAINING_MIP_LEVELS, VK_IMAGE_ASPECT_COLOR_BIT, + VK_IMAGE_LAYOUT_GENERAL, static_cast(dst_layout)); + return; + } + + SetImageLayout(vk_buffer, dst_image, 0, VK_REMAINING_MIP_LEVELS, VK_IMAGE_ASPECT_COLOR_BIT, + VK_IMAGE_LAYOUT_UNDEFINED, VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL); + + for (const auto& r: regions) { + VkImageCopy region; + + auto src_layout = r.src_image->layout; + + if (CopyBlockImageToCompressedImage(vk_buffer, r, dst_image)) { + continue; + } + + region.srcSubresource.aspectMask = VK_IMAGE_ASPECT_COLOR_BIT; + region.srcSubresource.mipLevel = r.src_level; + region.srcSubresource.baseArrayLayer = r.src_layer; + region.srcSubresource.layerCount = 1; + region.srcOffset = {r.src_x, r.src_y, r.src_z}; + region.dstSubresource.aspectMask = VK_IMAGE_ASPECT_COLOR_BIT; + region.dstSubresource.mipLevel = r.dst_level; + region.dstSubresource.baseArrayLayer = r.dst_layer; + region.dstSubresource.layerCount = 1; + region.dstOffset = {r.dst_x, r.dst_y, r.dst_z}; + region.extent = {r.width, r.height, 1}; + + SetImageLayout(vk_buffer, r.src_image, r.src_level, 1, VK_IMAGE_ASPECT_COLOR_BIT, + src_layout, VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL); + + vkCmdCopyImage(vk_buffer, r.src_image->image, VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL, + dst_image->image, VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL, 1, ®ion); + + SetImageLayout(vk_buffer, r.src_image, r.src_level, 1, VK_IMAGE_ASPECT_COLOR_BIT, + VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL, src_layout); + } + + SetImageLayout(vk_buffer, dst_image, 0, VK_REMAINING_MIP_LEVELS, VK_IMAGE_ASPECT_COLOR_BIT, + VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL, static_cast(dst_layout)); +} + +void UtilBlitImage(CommandBuffer* buffer, VulkanImage* src_image, VulkanSwapchain* dst_swapchain) { + EXIT_IF(src_image == nullptr); + EXIT_IF(src_image->image == nullptr); + EXIT_IF(dst_swapchain == nullptr); + + auto* vk_buffer = buffer->GetPool()->buffers[buffer->GetIndex()]; + + VulkanImage swapchain_image(VulkanImageType::Unknown); + + swapchain_image.image = dst_swapchain->swapchain_images[dst_swapchain->current_index]; + swapchain_image.layout = VK_IMAGE_LAYOUT_UNDEFINED; + + auto src_layout = src_image->layout; + SetImageLayout(vk_buffer, src_image, 0, 1, VK_IMAGE_ASPECT_COLOR_BIT, src_layout, + VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL); + SetImageLayout(vk_buffer, &swapchain_image, 0, 1, VK_IMAGE_ASPECT_COLOR_BIT, + VK_IMAGE_LAYOUT_UNDEFINED, VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL); + + VkImageBlit region {}; + region.srcSubresource.aspectMask = VK_IMAGE_ASPECT_COLOR_BIT; + region.srcSubresource.mipLevel = 0; + region.srcSubresource.baseArrayLayer = 0; + region.srcSubresource.layerCount = 1; + region.srcOffsets[0].x = 0; + region.srcOffsets[0].y = 0; + region.srcOffsets[0].z = 0; + region.srcOffsets[1].x = static_cast(src_image->extent.width); + region.srcOffsets[1].y = static_cast(src_image->extent.height); + region.srcOffsets[1].z = 1; + region.dstSubresource.aspectMask = VK_IMAGE_ASPECT_COLOR_BIT; + region.dstSubresource.mipLevel = 0; + region.dstSubresource.baseArrayLayer = 0; + region.dstSubresource.layerCount = 1; + region.dstOffsets[0].x = 0; + region.dstOffsets[0].y = 0; + region.dstOffsets[0].z = 0; + region.dstOffsets[1].x = static_cast(dst_swapchain->swapchain_extent.width); + region.dstOffsets[1].y = static_cast(dst_swapchain->swapchain_extent.height); + region.dstOffsets[1].z = 1; + + vkCmdBlitImage(vk_buffer, src_image->image, VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL, + swapchain_image.image, VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL, 1, ®ion, + VK_FILTER_LINEAR); + + SetImageLayout(vk_buffer, src_image, 0, 1, VK_IMAGE_ASPECT_COLOR_BIT, + VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL, VK_IMAGE_LAYOUT_GENERAL); +} + +void VulkanCreateBuffer(GraphicContext* gctx, uint64_t size, VulkanBuffer* buffer) { + KYTY_PROFILER_FUNCTION(); + + EXIT_IF(gctx == nullptr); + EXIT_IF(buffer == nullptr); + EXIT_IF(buffer->buffer != nullptr); + EXIT_IF(gctx->allocator == nullptr); + + VkBufferCreateInfo buffer_info {}; + buffer_info.sType = VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO; + buffer_info.size = size; + buffer_info.usage = buffer->usage; + buffer_info.sharingMode = VK_SHARING_MODE_EXCLUSIVE; + + VmaAllocationCreateInfo alloc_info {}; + alloc_info.requiredFlags = buffer->memory.property; + + const auto result = + vmaCreateBuffer(gctx->allocator, &buffer_info, &alloc_info, &buffer->buffer, + &buffer->memory.allocation, &buffer->memory.allocation_info); + if (result != VK_SUCCESS) { + VulkanLogMemoryBudget(gctx); + } + EXIT_NOT_IMPLEMENTED(result != VK_SUCCESS); + + vkGetBufferMemoryRequirements(gctx->device, buffer->buffer, &buffer->memory.requirements); + + buffer->memory.type = buffer->memory.allocation_info.memoryType; + buffer->memory.memory = buffer->memory.allocation_info.deviceMemory; + buffer->memory.offset = buffer->memory.allocation_info.offset; + buffer->memory.unique_id = VulkanNextMemoryUniqueId(); + buffer->buffer_size = size; + + VulkanTrackAllocation(&buffer->memory); +} + +void VulkanDeleteBuffer(GraphicContext* gctx, VulkanBuffer* buffer) { + KYTY_PROFILER_FUNCTION(); + + EXIT_IF(buffer == nullptr); + EXIT_IF(gctx == nullptr); + EXIT_IF(gctx->allocator == nullptr); + EXIT_IF(buffer->memory.allocation == nullptr); + + VulkanUntrackAllocation(&buffer->memory); + vmaDestroyBuffer(gctx->allocator, buffer->buffer, buffer->memory.allocation); + buffer->buffer = nullptr; + buffer->memory.memory = nullptr; + buffer->memory.allocation = nullptr; + buffer->memory.allocation_info = {}; + buffer->memory.offset = 0; +} + +bool VulkanCreateImage(GraphicContext* gctx, const VkImageCreateInfo* image_info, + VulkanImage* image, VulkanMemory* memory) { + KYTY_PROFILER_FUNCTION(); + + EXIT_IF(gctx == nullptr); + EXIT_IF(gctx->allocator == nullptr); + EXIT_IF(image_info == nullptr); + EXIT_IF(image == nullptr); + EXIT_IF(image->image != nullptr); + EXIT_IF(memory == nullptr); + EXIT_IF(memory->allocation != nullptr); + + VmaAllocationCreateInfo alloc_info {}; + alloc_info.requiredFlags = memory->property; + + const auto result = vmaCreateImage(gctx->allocator, image_info, &alloc_info, &image->image, + &memory->allocation, &memory->allocation_info); + + if (result != VK_SUCCESS) { + VulkanLogMemoryBudget(gctx); + return false; + } + + vkGetImageMemoryRequirements(gctx->device, image->image, &memory->requirements); + + memory->type = memory->allocation_info.memoryType; + memory->memory = memory->allocation_info.deviceMemory; + memory->offset = memory->allocation_info.offset; + memory->unique_id = VulkanNextMemoryUniqueId(); + + VulkanTrackAllocation(memory); + + return true; +} + +void VulkanDeleteImage(GraphicContext* gctx, VulkanImage* image, VulkanMemory* memory) { + KYTY_PROFILER_FUNCTION(); + + EXIT_IF(gctx == nullptr); + EXIT_IF(gctx->allocator == nullptr); + EXIT_IF(image == nullptr); + EXIT_IF(memory == nullptr); + EXIT_IF(memory->allocation == nullptr); + + VulkanUntrackAllocation(memory); + vmaDestroyImage(gctx->allocator, image->image, memory->allocation); + image->image = nullptr; + memory->memory = nullptr; + memory->allocation = nullptr; + memory->allocation_info = {}; + memory->offset = 0; +} + +void UtilFillImage(GraphicContext* ctx, VulkanImage* dst_image, const void* src_data, uint64_t size, + uint32_t src_pitch, uint64_t dst_layout) { + KYTY_PROFILER_FUNCTION(); + + EXIT_IF(ctx == nullptr); + EXIT_IF(dst_image == nullptr); + EXIT_IF(src_data == nullptr); + + GetStagingBuffer(StagingBufferType::Texture) + ->UploadToImage(ctx, dst_image, src_data, size, src_pitch, dst_layout); +} + +void UtilFillBuffer(GraphicContext* ctx, void* dst_data, uint64_t size, uint32_t dst_pitch, + VulkanImage* src_image, uint64_t src_layout, VkImageAspectFlags aspect) { + KYTY_PROFILER_FUNCTION(); + + EXIT_IF(ctx == nullptr); + EXIT_IF(src_image == nullptr); + EXIT_IF(dst_data == nullptr); + + GetStagingBuffer(StagingBufferType::ReadBack) + ->DownloadFromImage(ctx, dst_data, size, dst_pitch, src_image, src_layout, aspect); +} + +void UtilSetDepthLayoutOptimal(DepthStencilVulkanImage* image) { + CommandBuffer buffer(GraphicContext::QUEUE_UTIL); + // buffer.SetQueue(GraphicContext::QUEUE_UTIL); + + EXIT_NOT_IMPLEMENTED(buffer.IsInvalid()); + + buffer.Begin(); + + auto* vk_buffer = buffer.GetPool()->buffers[buffer.GetIndex()]; + + VkImageAspectFlags aspect_mask = VK_IMAGE_ASPECT_DEPTH_BIT; + + if (image->format == VK_FORMAT_D24_UNORM_S8_UINT || + image->format == VK_FORMAT_D32_SFLOAT_S8_UINT) { + aspect_mask |= VK_IMAGE_ASPECT_STENCIL_BIT; + } + + SetImageLayout(vk_buffer, image, 0, 1, aspect_mask, VK_IMAGE_LAYOUT_UNDEFINED, + VK_IMAGE_LAYOUT_DEPTH_STENCIL_ATTACHMENT_OPTIMAL); + + buffer.End(); + buffer.Execute(); + buffer.WaitForFence(); +} + +void UtilSetImageLayoutOptimal(VulkanImage* image) { + CommandBuffer buffer(GraphicContext::QUEUE_UTIL); + // buffer.SetQueue(GraphicContext::QUEUE_UTIL); + + EXIT_NOT_IMPLEMENTED(buffer.IsInvalid()); + + buffer.Begin(); + + auto* vk_buffer = buffer.GetPool()->buffers[buffer.GetIndex()]; + + VkImageAspectFlags aspect_mask = VK_IMAGE_ASPECT_COLOR_BIT; + + SetImageLayout(vk_buffer, image, 0, 1, aspect_mask, image->layout, VK_IMAGE_LAYOUT_GENERAL); + + buffer.End(); + buffer.Execute(); + buffer.WaitForFence(); +} + +void UtilFillImage(GraphicContext* ctx, VulkanImage* image, const void* src_data, uint64_t size, + const std::vector& regions, uint64_t dst_layout) { + EXIT_IF(ctx == nullptr); + EXIT_IF(image == nullptr); + EXIT_IF(src_data == nullptr); + + GetStagingBuffer(StagingBufferType::Texture) + ->UploadToImage(ctx, image, src_data, size, regions, dst_layout); +} + +void UtilFillImage(GraphicContext* ctx, const std::vector& regions, + VulkanImage* dst_image, uint64_t dst_layout) { + EXIT_IF(ctx == nullptr); + EXIT_IF(dst_image == nullptr); + + CommandBuffer buffer(GraphicContext::QUEUE_UTIL); + // buffer.SetQueue(GraphicContext::QUEUE_UTIL); + + EXIT_NOT_IMPLEMENTED(buffer.IsInvalid()); + + buffer.Begin(); + UtilImageToImage(&buffer, regions, dst_image, dst_layout); + buffer.End(); + buffer.Execute(); + buffer.WaitForFence(); +} + +void UtilUploadBuffer(GraphicContext* ctx, StagingBufferType type, VulkanBuffer* dst_buffer, + const void* src_data, uint64_t size) { + EXIT_IF(ctx == nullptr); + EXIT_IF(dst_buffer == nullptr); + EXIT_IF(dst_buffer->buffer == nullptr); + EXIT_IF(src_data == nullptr); + + GetStagingBuffer(type)->UploadToBuffer(ctx, dst_buffer, src_data, size); +} + +void UtilFillDepthImage(GraphicContext* ctx, DepthStencilVulkanImage* dst_image, + const void* src_data, uint64_t size, uint32_t src_pitch) { + EXIT_IF(ctx == nullptr || dst_image == nullptr || src_data == nullptr); + GetStagingBuffer(StagingBufferType::Texture) + ->UploadToDepthImage(ctx, dst_image, src_data, size, src_pitch); +} + +void UtilUploadBuffer(GraphicContext* ctx, StagingBufferType type, VulkanBuffer* dst_buffer, + uint64_t dst_offset, const void* src_data, uint64_t size) { + EXIT_IF(ctx == nullptr); + EXIT_IF(dst_buffer == nullptr || dst_buffer->buffer == nullptr || src_data == nullptr); + EXIT_IF(dst_offset > dst_buffer->buffer_size || size > dst_buffer->buffer_size - dst_offset); + GetStagingBuffer(type)->UploadToBuffer(ctx, dst_buffer, src_data, size, dst_offset); +} + +void UtilCopyBuffer(VulkanBuffer* src_buffer, VulkanBuffer* dst_buffer, uint64_t size) { + EXIT_IF(src_buffer == nullptr); + EXIT_IF(src_buffer->buffer == nullptr); + EXIT_IF(dst_buffer == nullptr); + EXIT_IF(dst_buffer->buffer == nullptr); + + CommandBuffer buffer(GraphicContext::QUEUE_UTIL); + // buffer.SetQueue(GraphicContext::QUEUE_UTIL); + + EXIT_NOT_IMPLEMENTED(buffer.IsInvalid()); + + auto* vk_buffer = buffer.GetPool()->buffers[buffer.GetIndex()]; + + buffer.Begin(); + + VkBufferCopy copy_region {}; + copy_region.srcOffset = 0; + copy_region.dstOffset = 0; + copy_region.size = size; + + vkCmdCopyBuffer(vk_buffer, src_buffer->buffer, dst_buffer->buffer, 1, ©_region); + + buffer.End(); + buffer.Execute(); + buffer.WaitForFence(); +} + +void UtilDownloadBuffer(GraphicContext* ctx, VulkanBuffer* src_buffer, uint64_t src_offset, + void* dst_data, uint64_t size) { + EXIT_IF(ctx == nullptr || src_buffer == nullptr || src_buffer->buffer == nullptr || + dst_data == nullptr || size == 0); + EXIT_IF(src_offset > src_buffer->buffer_size || size > src_buffer->buffer_size - src_offset); + const auto family = ctx->queues[GraphicContext::QUEUE_UTIL].family; + EXIT_IF(family == static_cast(-1)); + for (int i = GraphicContext::QUEUE_COMPUTE_START; + i < GraphicContext::QUEUE_COMPUTE_START + GraphicContext::QUEUE_COMPUTE_NUM; i++) { + EXIT_IF(ctx->queues[i].family != family); + } + EXIT_IF(ctx->queues[GraphicContext::QUEUE_GFX].family != family); + + if ((src_buffer->memory.property & VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT) != 0) { + void* mapped = nullptr; + VulkanMapMemory(ctx, &src_buffer->memory, &mapped); + EXIT_IF(mapped == nullptr); + std::memcpy(dst_data, static_cast(mapped) + src_offset, size); + VulkanUnmapMemory(ctx, &src_buffer->memory); + return; + } + + VulkanBuffer readback {}; + readback.usage = VK_BUFFER_USAGE_TRANSFER_DST_BIT; + readback.memory.property = VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT | + VK_MEMORY_PROPERTY_HOST_COHERENT_BIT | + VK_MEMORY_PROPERTY_HOST_CACHED_BIT; + VulkanCreateBuffer(ctx, size, &readback); + + CommandBuffer command(GraphicContext::QUEUE_UTIL); + EXIT_NOT_IMPLEMENTED(command.IsInvalid()); + auto* vk_buffer = command.GetPool()->buffers[command.GetIndex()]; + command.Begin(); + VkBufferMemoryBarrier barrier {}; + barrier.sType = VK_STRUCTURE_TYPE_BUFFER_MEMORY_BARRIER; + barrier.srcAccessMask = VK_ACCESS_MEMORY_WRITE_BIT; + barrier.dstAccessMask = VK_ACCESS_TRANSFER_READ_BIT; + barrier.srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; + barrier.dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; + barrier.buffer = src_buffer->buffer; + barrier.offset = src_offset; + barrier.size = size; + vkCmdPipelineBarrier(vk_buffer, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, + VK_PIPELINE_STAGE_TRANSFER_BIT, 0, 0, nullptr, 1, &barrier, 0, nullptr); + const VkBufferCopy copy {.srcOffset = src_offset, .dstOffset = 0, .size = size}; + vkCmdCopyBuffer(vk_buffer, src_buffer->buffer, readback.buffer, 1, ©); + command.End(); + command.Execute(); + command.WaitForFence(); + + void* mapped = nullptr; + VulkanMapMemory(ctx, &readback.memory, &mapped); + EXIT_IF(mapped == nullptr); + std::memcpy(dst_data, mapped, size); + VulkanUnmapMemory(ctx, &readback.memory); + VulkanDeleteBuffer(ctx, &readback); +} + +} // namespace Libs::Graphics diff --git a/src/graphics/host_gpu/utils.h b/src/graphics/host_gpu/utils.h new file mode 100644 index 000000000..968200cb4 --- /dev/null +++ b/src/graphics/host_gpu/utils.h @@ -0,0 +1,136 @@ +#ifndef EMULATOR_INCLUDE_EMULATOR_GRAPHICS_UTILS_H_ +#define EMULATOR_INCLUDE_EMULATOR_GRAPHICS_UTILS_H_ + +#include "common/abi.h" +#include "common/common.h" + +#include +#include +#include + +namespace Libs::Graphics { + +class CommandBuffer; +struct GraphicContext; +struct VulkanBuffer; +struct VulkanImage; +struct VulkanMemory; +struct DepthStencilVulkanImage; +struct VulkanSwapchain; + +struct BufferImageCopy { + uint32_t offset; + uint32_t pitch; + uint32_t dst_level; + uint32_t width; + uint32_t height; + uint32_t copy_height = 0; + uint32_t dst_layer = 0; + int dst_x; + int dst_y; + int dst_z = 0; +}; + +struct ImageImageCopy { + VulkanImage* src_image; + uint32_t src_level; + uint32_t dst_level; + uint32_t width; + uint32_t height; + uint32_t src_layer = 0; + uint32_t dst_layer = 0; + int src_x; + int src_y; + int src_z = 0; + int dst_x; + int dst_y; + int dst_z = 0; +}; + +enum class StagingBufferType { Texture, Vertex, Index, ReadBack }; + +class UtilScratchBuffer { +public: + explicit UtilScratchBuffer(uint64_t size); + ~UtilScratchBuffer(); + KYTY_CLASS_NO_COPY(UtilScratchBuffer); + + [[nodiscard]] void* Data() const { return m_data; } + +private: + void* m_data = nullptr; +}; + +void UtilBufferToImage(CommandBuffer* buffer, VulkanBuffer* src_buffer, uint32_t src_pitch, + VulkanImage* dst_image, uint64_t dst_layout); +void UtilBufferToImage(CommandBuffer* buffer, VulkanBuffer* src_buffer, VulkanImage* dst_image, + const std::vector& regions, uint64_t dst_layout); +void UtilImageToBuffer(CommandBuffer* buffer, VulkanImage* src_image, VulkanBuffer* dst_buffer, + uint32_t dst_pitch, uint64_t src_layout, + VkImageAspectFlags aspect = VK_IMAGE_ASPECT_COLOR_BIT); +void UtilImageToImage(CommandBuffer* buffer, const std::vector& regions, + VulkanImage* dst_image, uint64_t dst_layout); +void UtilBlitImage(CommandBuffer* buffer, VulkanImage* src_image, VulkanSwapchain* dst_swapchain); +void UtilFillImage(GraphicContext* ctx, VulkanImage* dst_image, const void* src_data, uint64_t size, + uint32_t src_pitch, uint64_t dst_layout); +void UtilFillDepthImage(GraphicContext* ctx, DepthStencilVulkanImage* dst_image, + const void* src_data, uint64_t size, uint32_t src_pitch); +void UtilFillImage(GraphicContext* ctx, VulkanImage* dst_image, const void* src_data, uint64_t size, + const std::vector& regions, uint64_t dst_layout); +void UtilFillImage(GraphicContext* ctx, const std::vector& regions, + VulkanImage* dst_image, uint64_t dst_layout); +void UtilFillBuffer(GraphicContext* ctx, void* dst_data, uint64_t size, uint32_t dst_pitch, + VulkanImage* src_image, uint64_t src_layout, + VkImageAspectFlags aspect = VK_IMAGE_ASPECT_COLOR_BIT); +void UtilUploadBuffer(GraphicContext* ctx, StagingBufferType type, VulkanBuffer* dst_buffer, + const void* src_data, uint64_t size); +void UtilUploadBuffer(GraphicContext* ctx, StagingBufferType type, VulkanBuffer* dst_buffer, + uint64_t dst_offset, const void* src_data, uint64_t size); +void UtilCopyBuffer(VulkanBuffer* src_buffer, VulkanBuffer* dst_buffer, uint64_t size); +void UtilDownloadBuffer(GraphicContext* ctx, VulkanBuffer* src_buffer, uint64_t src_offset, + void* dst_data, uint64_t size); +bool UtilBufferIsTiled(uint64_t vaddr, uint64_t size); +void UtilSetDepthLayoutOptimal(DepthStencilVulkanImage* image); +void UtilSetImageLayoutOptimal(VulkanImage* image); +void UtilResetImageViews(VulkanImage* image); +void UtilCreateImageView(GraphicContext* ctx, VulkanImage* image, int view_index, + VkImageViewType view_type, VkImageAspectFlags aspect_mask, + VkComponentMapping components, uint32_t base_array_layer, + uint32_t base_mip_level, uint32_t layer_count, uint32_t level_count, + VkFormat view_format = VK_FORMAT_UNDEFINED, + VkImageUsageFlags view_usage = 0); +bool UtilIsBcFormat(VkFormat format); +uint32_t UtilGetBcBlockSize(VkFormat format); + +void VulkanCreateBuffer(GraphicContext* gctx, uint64_t size, VulkanBuffer* buffer); +void VulkanDeleteBuffer(GraphicContext* gctx, VulkanBuffer* buffer); +bool VulkanCreateImage(GraphicContext* gctx, const VkImageCreateInfo* image_info, + VulkanImage* image, VulkanMemory* memory); +void VulkanDeleteImage(GraphicContext* gctx, VulkanImage* image, VulkanMemory* memory); +void VulkanDeviceWaitIdle(GraphicContext* ctx); + +inline std::pair UtilCalcMipmapOffset(uint32_t lod, uint32_t width, uint32_t height) { + uint32_t mip_width = width; + uint32_t mip_height = height; + int mip_x = 0; + int mip_y = 0; + + for (uint32_t i = 0; i < 16; i++) { + if (i == lod) { + return {mip_x, mip_y}; + } + + bool odd = ((i & 1u) != 0); + mip_x += static_cast(odd ? mip_width : 0); + mip_y += static_cast(odd ? 0 : mip_height); + + mip_width >>= (mip_width > 1 ? 1u : 0u); + mip_height >>= (mip_height > 1 ? 1u : 0u); + } + + return {mip_x, mip_y}; +} + +} // namespace Libs::Graphics + +#endif /* EMULATOR_INCLUDE_EMULATOR_GRAPHICS_UTILS_H_ */ diff --git a/src/graphics/host_gpu/vma.cpp b/src/graphics/host_gpu/vma.cpp new file mode 100644 index 000000000..8beb67f66 --- /dev/null +++ b/src/graphics/host_gpu/vma.cpp @@ -0,0 +1,197 @@ +#if defined(__clang__) +#pragma clang diagnostic push +#pragma clang diagnostic ignored "-Wnullability-completeness" +#pragma clang diagnostic ignored "-Wunused-private-field" +#pragma clang diagnostic ignored "-Wunused-variable" +#endif + +#define VMA_IMPLEMENTATION +#include + +#if defined(__clang__) +#pragma clang diagnostic pop +#endif + +#include "common/assert.h" +#include "common/logging/log.h" +#include "common/profiler.h" +#include "common/stringUtils.h" +#include "graphics/host_gpu/graphicContext.h" +#include "graphics/host_gpu/vma.h" + +#include +#include +#include +#include +#include +#include + +namespace Libs::Graphics { + +namespace { + +struct MemoryStats { + std::atomic_uint64_t allocated[VK_MAX_MEMORY_TYPES] {}; + std::atomic_uint64_t count[VK_MAX_MEMORY_TYPES] {}; +}; + +MemoryStats g_memory_stats; + +void TrackAllocationImpl(const VulkanMemory& memory) { + g_memory_stats.allocated[memory.type] += memory.requirements.size; + g_memory_stats.count[memory.type]++; +} + +void UntrackAllocationImpl(const VulkanMemory& memory) { + EXIT_IF(g_memory_stats.allocated[memory.type] < memory.requirements.size); + EXIT_IF(g_memory_stats.count[memory.type] == 0); + g_memory_stats.allocated[memory.type] -= memory.requirements.size; + g_memory_stats.count[memory.type]--; +} + +} // namespace + +void VulkanTrackAllocation(const VulkanMemory* memory) { + EXIT_IF(memory == nullptr); + TrackAllocationImpl(*memory); +} + +void VulkanUntrackAllocation(const VulkanMemory* memory) { + EXIT_IF(memory == nullptr); + UntrackAllocationImpl(*memory); +} + +bool VulkanCreateAllocator(GraphicContext* ctx) { + KYTY_PROFILER_FUNCTION(); + EXIT_IF(ctx == nullptr || ctx->instance == nullptr || ctx->physical_device == nullptr || + ctx->device == nullptr || ctx->allocator != nullptr); + + VmaAllocatorCreateInfo info {}; + info.instance = ctx->instance; + info.physicalDevice = ctx->physical_device; + info.device = ctx->device; + info.vulkanApiVersion = VULKAN_TARGET_API_VERSION; + info.flags = ctx->memory_budget_ext_enabled ? VMA_ALLOCATOR_CREATE_EXT_MEMORY_BUDGET_BIT : 0; + + const auto result = vmaCreateAllocator(&info, &ctx->allocator); + if (result != VK_SUCCESS) { + LOGF("vmaCreateAllocator failed: %s\n", string_VkResult(result)); + return false; + } + return true; +} + +void VulkanDestroyAllocator(GraphicContext* ctx) { + if (ctx == nullptr || ctx->allocator == nullptr) { + return; + } + vmaDestroyAllocator(ctx->allocator); + ctx->allocator = nullptr; +} + +uint64_t VulkanNextMemoryUniqueId() { + static std::atomic_uint64_t sequence = 0; + return ++sequence; +} + +void VulkanLogMemoryBudget(GraphicContext* ctx) { + if (ctx == nullptr || ctx->allocator == nullptr || ctx->physical_device == nullptr) { + return; + } + + VkPhysicalDeviceMemoryProperties properties {}; + vkGetPhysicalDeviceMemoryProperties(ctx->physical_device, &properties); + VmaBudget budgets[VK_MAX_MEMORY_HEAPS] {}; + vmaGetHeapBudgets(ctx->allocator, budgets); + for (uint32_t i = 0; i < properties.memoryHeapCount; i++) { + LOGF("VMA heap %u: usage=%" PRIu64 ", budget=%" PRIu64 ", allocation=%" PRIu64 + ", blocks=%" PRIu64 "\n", + i, static_cast(budgets[i].usage), static_cast(budgets[i].budget), + static_cast(budgets[i].statistics.allocationBytes), + static_cast(budgets[i].statistics.blockBytes)); + } +} + +bool VulkanAllocate(GraphicContext* ctx, VulkanMemory* memory) { + KYTY_PROFILER_FUNCTION(); + EXIT_IF(ctx == nullptr || memory == nullptr || memory->memory != nullptr || + memory->allocation != nullptr || ctx->allocator == nullptr || + memory->requirements.size == 0); + + VkPhysicalDeviceMemoryProperties properties {}; + vkGetPhysicalDeviceMemoryProperties(ctx->physical_device, &properties); + uint32_t index = 0; + for (; index < properties.memoryTypeCount; index++) { + if ((memory->requirements.memoryTypeBits & (uint32_t {1} << index)) != 0 && + (properties.memoryTypes[index].propertyFlags & memory->property) == memory->property) { + break; + } + } + + VmaAllocationCreateInfo info {}; + info.requiredFlags = memory->property; + memory->unique_id = VulkanNextMemoryUniqueId(); + const auto result = vmaAllocateMemory(ctx->allocator, &memory->requirements, &info, + &memory->allocation, &memory->allocation_info); + if (result == VK_SUCCESS) { + memory->type = memory->allocation_info.memoryType; + memory->memory = memory->allocation_info.deviceMemory; + memory->offset = memory->allocation_info.offset; + VulkanTrackAllocation(memory); + return true; + } + + VulkanLogMemoryBudget(ctx); + std::vector stats; + for (uint32_t i = 0; i < properties.memoryTypeCount; i++) { + stats.push_back(fmt::format("{}, {}, {}", i, g_memory_stats.count[i].load(), + g_memory_stats.allocated[i].load())); + } + EXIT("size = %" PRIu64 ", index = %u, error: %s:%s\n", memory->requirements.size, index, + string_VkResult(result), Common::Concat(stats, '\n').c_str()); + return false; +} + +void VulkanFree(GraphicContext* ctx, VulkanMemory* memory) { + KYTY_PROFILER_FUNCTION(); + EXIT_IF(ctx == nullptr || memory == nullptr || ctx->allocator == nullptr || + memory->allocation == nullptr); + vmaFreeMemory(ctx->allocator, memory->allocation); + VulkanUntrackAllocation(memory); + memory->memory = nullptr; + memory->allocation = nullptr; + memory->allocation_info = {}; + memory->offset = 0; +} + +void VulkanMapMemory(GraphicContext* ctx, VulkanMemory* memory, void** data) { + KYTY_PROFILER_FUNCTION(); + EXIT_IF(ctx == nullptr || memory == nullptr || data == nullptr || ctx->allocator == nullptr || + memory->allocation == nullptr); + EXIT_NOT_IMPLEMENTED(vmaMapMemory(ctx->allocator, memory->allocation, data) != VK_SUCCESS); +} + +void VulkanUnmapMemory(GraphicContext* ctx, VulkanMemory* memory) { + KYTY_PROFILER_FUNCTION(); + EXIT_IF(ctx == nullptr || memory == nullptr || ctx->allocator == nullptr || + memory->allocation == nullptr); + vmaUnmapMemory(ctx->allocator, memory->allocation); +} + +void VulkanBindImageMemory(GraphicContext* ctx, VulkanImage* image, VulkanMemory* memory) { + KYTY_PROFILER_FUNCTION(); + EXIT_IF(ctx == nullptr || image == nullptr || memory == nullptr || ctx->allocator == nullptr || + memory->allocation == nullptr); + EXIT_NOT_IMPLEMENTED(vmaBindImageMemory(ctx->allocator, memory->allocation, image->image) != + VK_SUCCESS); +} + +void VulkanBindBufferMemory(GraphicContext* ctx, VulkanBuffer* buffer, VulkanMemory* memory) { + KYTY_PROFILER_FUNCTION(); + EXIT_IF(ctx == nullptr || buffer == nullptr || memory == nullptr || ctx->allocator == nullptr || + memory->allocation == nullptr); + EXIT_NOT_IMPLEMENTED(vmaBindBufferMemory(ctx->allocator, memory->allocation, buffer->buffer) != + VK_SUCCESS); +} + +} // namespace Libs::Graphics diff --git a/src/graphics/host_gpu/vma.h b/src/graphics/host_gpu/vma.h new file mode 100644 index 000000000..88a9f0d49 --- /dev/null +++ b/src/graphics/host_gpu/vma.h @@ -0,0 +1,28 @@ +#ifndef EMULATOR_SRC_GRAPHICS_HOST_GPU_VMA_H_ +#define EMULATOR_SRC_GRAPHICS_HOST_GPU_VMA_H_ + +#include "common/common.h" + +namespace Libs::Graphics { + +struct GraphicContext; +struct VulkanBuffer; +struct VulkanImage; +struct VulkanMemory; + +bool VulkanCreateAllocator(GraphicContext* ctx); +void VulkanDestroyAllocator(GraphicContext* ctx); +uint64_t VulkanNextMemoryUniqueId(); +void VulkanTrackAllocation(const VulkanMemory* memory); +void VulkanUntrackAllocation(const VulkanMemory* memory); +void VulkanLogMemoryBudget(GraphicContext* ctx); +bool VulkanAllocate(GraphicContext* ctx, VulkanMemory* memory); +void VulkanFree(GraphicContext* ctx, VulkanMemory* memory); +void VulkanMapMemory(GraphicContext* ctx, VulkanMemory* memory, void** data); +void VulkanUnmapMemory(GraphicContext* ctx, VulkanMemory* memory); +void VulkanBindImageMemory(GraphicContext* ctx, VulkanImage* image, VulkanMemory* memory); +void VulkanBindBufferMemory(GraphicContext* ctx, VulkanBuffer* buffer, VulkanMemory* memory); + +} // namespace Libs::Graphics + +#endif /* EMULATOR_SRC_GRAPHICS_HOST_GPU_VMA_H_ */ diff --git a/src/graphics/host_gpu/vulkanInstance.h b/src/graphics/host_gpu/vulkanInstance.h new file mode 100644 index 000000000..339e19e69 --- /dev/null +++ b/src/graphics/host_gpu/vulkanInstance.h @@ -0,0 +1,59 @@ +#ifndef EMULATOR_SRC_GRAPHICS_HOST_GPU_VULKANINSTANCE_H_ +#define EMULATOR_SRC_GRAPHICS_HOST_GPU_VULKANINSTANCE_H_ + +#include "common/abi.h" +#include "common/common.h" +#include "common/threads.h" + +#include +#include + +namespace Libs::Graphics { + +inline constexpr uint32_t VULKAN_TARGET_API_VERSION = VK_API_VERSION_1_3; + +struct VulkanQueueInfo { + Common::Mutex* mutex = nullptr; + uint32_t family = static_cast(-1); + uint32_t index = static_cast(-1); + VkQueue vk_queue = nullptr; +}; + +struct VulkanInstance { + static constexpr int QUEUES_NUM = 11; + static constexpr int QUEUE_GFX = 8; + static constexpr int QUEUE_GFX_NUM = 1; + static constexpr int QUEUE_UTIL = 9; + static constexpr int QUEUE_UTIL_NUM = 1; + static constexpr int QUEUE_PRESENT = 10; + static constexpr int QUEUE_PRESENT_NUM = 1; + static constexpr int QUEUE_COMPUTE_START = 0; + static constexpr int QUEUE_COMPUTE_NUM = 8; + + VkInstance instance = nullptr; + VkDebugUtilsMessengerEXT debug_messenger = nullptr; + VkPhysicalDevice physical_device = nullptr; + VkPhysicalDeviceProperties physical_device_properties = {}; + VkDevice device = nullptr; + VmaAllocator allocator = nullptr; + bool memory_budget_ext_enabled = false; + bool subgroup_size_control_enabled = false; + uint32_t subgroup_size = 0; + uint32_t min_subgroup_size = 0; + uint32_t max_subgroup_size = 0; + VkShaderStageFlags required_subgroup_size_stages = 0; + VulkanQueueInfo queues[QUEUES_NUM]; + + [[nodiscard]] const VkPhysicalDeviceProperties& GetPhysicalDeviceProperties() const { + return physical_device_properties; + } + + [[nodiscard]] VkDeviceSize StorageMinAlignment() const { + const auto alignment = physical_device_properties.limits.minStorageBufferOffsetAlignment; + return alignment != 0 ? alignment : 1; + } +}; + +} // namespace Libs::Graphics + +#endif // EMULATOR_SRC_GRAPHICS_HOST_GPU_VULKANINSTANCE_H_ diff --git a/src/graphics/presentation/displayBuffer.h b/src/graphics/presentation/displayBuffer.h new file mode 100644 index 000000000..b0c782992 --- /dev/null +++ b/src/graphics/presentation/displayBuffer.h @@ -0,0 +1,25 @@ +#ifndef EMULATOR_INCLUDE_EMULATOR_GRAPHICS_PRESENTATION_DISPLAYBUFFER_H_ +#define EMULATOR_INCLUDE_EMULATOR_GRAPHICS_PRESENTATION_DISPLAYBUFFER_H_ + +#include "common/common.h" + +namespace Libs::Graphics { +struct VideoOutVulkanImage; +} // namespace Libs::Graphics + +namespace Libs::Presentation { + +struct DisplayBufferImage { + Graphics::VideoOutVulkanImage* image = nullptr; + uint32_t index = static_cast(-1); + uint64_t size = 0; + uint64_t pitch = 0; +}; + +DisplayBufferImage DisplayBufferFind(uint64_t addr, bool render_target = false); +int DisplayBufferSubmitFlipFromGpu(int handle, int index, int flip_mode, int64_t flip_arg); +void DisplayBufferCompleteFlipFromGpu(int handle, int index); + +} // namespace Libs::Presentation + +#endif // EMULATOR_INCLUDE_EMULATOR_GRAPHICS_PRESENTATION_DISPLAYBUFFER_H_ diff --git a/src/graphics/presentation/renderDoc.cpp b/src/graphics/presentation/renderDoc.cpp new file mode 100644 index 000000000..d89cd4cc0 --- /dev/null +++ b/src/graphics/presentation/renderDoc.cpp @@ -0,0 +1,293 @@ +#include "graphics/presentation/renderDoc.h" + +#include "SDL_syswm.h" +#include "SDL_version.h" +#include "SDL_video.h" +#include "common/logging/log.h" + +#include +#include +#include +#include +#include +#include + +#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS +#ifndef NOMINMAX +#define NOMINMAX +#endif +#include +#undef min +#undef max +#endif + +namespace Libs::Graphics { + +#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS + +using RenderDocDevicePointer = void*; +using RenderDocWindowHandle = void*; + +enum RenderDocVersion { + eRENDERDOC_API_Version_1_4_2 = 10402, +}; + +enum RenderDocInputButton { + eRENDERDOC_Key_NonPrintable = 0x100, + eRENDERDOC_Key_Divide, + eRENDERDOC_Key_Multiply, + eRENDERDOC_Key_Subtract, + eRENDERDOC_Key_Plus, + eRENDERDOC_Key_F1, +}; + +using pRENDERDOC_SetCaptureKeys = void(__cdecl*)(RenderDocInputButton* keys, int num); +using pRENDERDOC_SetCaptureFilePathTemplate = void(__cdecl*)(const char* pathtemplate); +using pRENDERDOC_GetCaptureFilePathTemplate = const char*(__cdecl*)(); +using pRENDERDOC_GetNumCaptures = uint32_t(__cdecl*)(); +using pRENDERDOC_GetCapture = uint32_t(__cdecl*)(uint32_t idx, char* filename, uint32_t* pathlength, + uint64_t* timestamp); +using pRENDERDOC_UnloadCrashHandler = void(__cdecl*)(); +using pRENDERDOC_SetActiveWindow = void(__cdecl*)(RenderDocDevicePointer device, + RenderDocWindowHandle wndHandle); +using pRENDERDOC_StartFrameCapture = void(__cdecl*)(RenderDocDevicePointer device, + RenderDocWindowHandle wndHandle); +using pRENDERDOC_IsFrameCapturing = uint32_t(__cdecl*)(); +using pRENDERDOC_EndFrameCapture = uint32_t(__cdecl*)(RenderDocDevicePointer device, + RenderDocWindowHandle wndHandle); +using pRENDERDOC_GetAPI = int(__cdecl*)(RenderDocVersion version, void** out_api_pointers); + +struct RenderDocApi { + void* GetAPIVersion; + void* SetCaptureOptionU32; + void* SetCaptureOptionF32; + void* GetCaptureOptionU32; + void* GetCaptureOptionF32; + void* SetFocusToggleKeys; + pRENDERDOC_SetCaptureKeys SetCaptureKeys; + void* GetOverlayBits; + void* MaskOverlayBits; + void* RemoveHooks; + pRENDERDOC_UnloadCrashHandler UnloadCrashHandler; + pRENDERDOC_SetCaptureFilePathTemplate SetCaptureFilePathTemplate; + pRENDERDOC_GetCaptureFilePathTemplate GetCaptureFilePathTemplate; + pRENDERDOC_GetNumCaptures GetNumCaptures; + pRENDERDOC_GetCapture GetCapture; + void* TriggerCapture; + void* IsTargetControlConnected; + void* LaunchReplayUI; + pRENDERDOC_SetActiveWindow SetActiveWindow; + pRENDERDOC_StartFrameCapture StartFrameCapture; + pRENDERDOC_IsFrameCapturing IsFrameCapturing; + pRENDERDOC_EndFrameCapture EndFrameCapture; + void* TriggerMultiFrameCapture; + void* SetCaptureFileComments; + void* DiscardFrameCapture; +}; + +enum class RenderDocState : uint32_t { + Idle, + Requested, + Capturing, +}; + +static RenderDocApi* g_api = nullptr; +static HMODULE g_module = nullptr; +static RenderDocDevicePointer g_device = nullptr; +static RenderDocWindowHandle g_window = nullptr; +static std::atomic g_state = RenderDocState::Idle; +static std::atomic_bool g_init_done = false; +static std::atomic_bool g_unavailable_log = false; + +static RenderDocDevicePointer GetRenderDocDevicePointer(VkInstance instance) { + if (instance == nullptr) { + return nullptr; + } + + return reinterpret_cast(instance); +} + +static bool BindRenderDocApi(HMODULE module) { + auto* get_api = reinterpret_cast(GetProcAddress(module, "RENDERDOC_GetAPI")); + if (get_api == nullptr) { + return false; + } + + void* api = nullptr; + if (get_api(eRENDERDOC_API_Version_1_4_2, &api) == 0 || api == nullptr) { + return false; + } + + g_module = module; + g_api = static_cast(api); + + g_api->SetCaptureFilePathTemplate("_RenderDoc/kyty"); + static RenderDocInputButton capture_keys[] = {eRENDERDOC_Key_F1}; + g_api->SetCaptureKeys(capture_keys, 1); + g_api->UnloadCrashHandler(); + + char module_path[MAX_PATH] = {}; + GetModuleFileNameA(module, module_path, sizeof(module_path)); + LOGF("RenderDoc: bound API from %s\n", module_path); + return true; +} + +static RenderDocWindowHandle GetRenderDocWindowHandle(SDL_Window* window) { + if (window == nullptr) { + return nullptr; + } + + SDL_SysWMinfo info {}; + SDL_VERSION(&info.version); + + if (SDL_GetWindowWMInfo(window, &info) != SDL_TRUE || info.subsystem != SDL_SYSWM_WINDOWS) { + return nullptr; + } + + return info.info.win.window; +} + +static bool IsAvailable() { + return g_api != nullptr && g_device != nullptr && g_window != nullptr; +} + +void RenderDocInit() { + bool expected = false; + if (!g_init_done.compare_exchange_strong(expected, true)) { + return; + } + + HKEY h_reg_key; + LONG result = RegOpenKeyExW(HKEY_LOCAL_MACHINE, + L"SOFTWARE\\Classes\\RenderDoc.RDCCapture.1\\DefaultIcon\\", 0, + KEY_READ, &h_reg_key); + if (result != ERROR_SUCCESS) { + return; + } + std::array key_str {}; + DWORD str_sz_out {key_str.size()}; + result = RegQueryValueExW(h_reg_key, L"", 0, NULL, (LPBYTE)key_str.data(), &str_sz_out); + RegCloseKey(h_reg_key); + if (result != ERROR_SUCCESS) { + return; + } + + std::filesystem::path path {key_str.cbegin(), key_str.cend()}; + path = path.parent_path().append("renderdoc.dll"); + const auto path_to_lib = path.generic_string(); + auto* module = LoadLibraryA(path_to_lib.c_str()); + if (module == nullptr) { + return; + } + + if (!BindRenderDocApi(module)) { + LOGF("RenderDoc: API 1.4.2 is not available; in-app capture disabled\n"); + FreeLibrary(module); + return; + } +} + +bool RenderDocIsLoaded() { + return g_api != nullptr; +} + +void RenderDocSetActiveWindow(VkInstance instance, SDL_Window* window) { + if (g_api == nullptr) { + return; + } + + g_device = GetRenderDocDevicePointer(instance); + g_window = GetRenderDocWindowHandle(window); + + if (g_device == nullptr || g_window == nullptr) { + LOGF("RenderDoc: active Vulkan window was not registered\n"); + return; + } + + g_api->SetActiveWindow(g_device, g_window); + LOGF("RenderDoc: active Vulkan window registered\n"); +} + +void RenderDocRequestCapture() { + if (!IsAvailable()) { + if (!g_unavailable_log.exchange(true)) { + LOGF("RenderDoc: capture requested, but RenderDoc is not available\n"); + } + return; + } + + RenderDocState expected = RenderDocState::Idle; + if (g_state.compare_exchange_strong(expected, RenderDocState::Requested)) { + LOGF("RenderDoc: capture requested; next complete presented frame will be captured\n"); + } else { + LOGF("RenderDoc: capture request ignored because a capture is already pending\n"); + } +} + +static void LogNewestCapture() { + const auto count = g_api->GetNumCaptures(); + if (count == 0) { + return; + } + + char filename[4096] = {}; + uint32_t path_length = sizeof(filename); + uint64_t timestamp = 0; + + if (g_api->GetCapture(count - 1, filename, &path_length, ×tamp) != 0) { + filename[sizeof(filename) - 1] = '\0'; + LOGF("RenderDoc: wrote capture %s\n", filename); + } +} + +void RenderDocOnPresent() { + if (!IsAvailable()) { + return; + } + + switch (g_state.load()) { + case RenderDocState::Idle: return; + case RenderDocState::Requested: + if (g_api->IsFrameCapturing() != 0) { + LOGF("RenderDoc: capture request ignored because RenderDoc is already capturing\n"); + g_state.store(RenderDocState::Idle); + return; + } + + g_api->StartFrameCapture(nullptr, nullptr); + if (g_api->IsFrameCapturing() == 0) { + LOGF("RenderDoc: StartFrameCapture returned, but RenderDoc is not capturing\n"); + g_state.store(RenderDocState::Idle); + return; + } + g_state.store(RenderDocState::Capturing); + LOGF("RenderDoc: capture started\n"); + return; + case RenderDocState::Capturing: break; + } + + const auto ok = g_api->EndFrameCapture(nullptr, nullptr); + g_state.store(RenderDocState::Idle); + + if (ok != 0) { + const auto count = g_api->GetNumCaptures(); + LOGF("RenderDoc: capture finished, count=%u\n", count); + LogNewestCapture(); + } else { + LOGF("RenderDoc: capture failed\n"); + } +} + +#else + +void RenderDocInit() {} +bool RenderDocIsLoaded() { + return false; +} +void RenderDocSetActiveWindow(VkInstance /*instance*/, SDL_Window* /*window*/) {} +void RenderDocRequestCapture() {} +void RenderDocOnPresent() {} + +#endif + +} // namespace Libs::Graphics diff --git a/src/graphics/presentation/renderDoc.h b/src/graphics/presentation/renderDoc.h new file mode 100644 index 000000000..eed135091 --- /dev/null +++ b/src/graphics/presentation/renderDoc.h @@ -0,0 +1,20 @@ +#ifndef EMULATOR_INCLUDE_EMULATOR_GRAPHICS_RENDERDOC_H_ +#define EMULATOR_INCLUDE_EMULATOR_GRAPHICS_RENDERDOC_H_ + +#include "common/common.h" + +#include + +struct SDL_Window; + +namespace Libs::Graphics { + +void RenderDocInit(); +bool RenderDocIsLoaded(); +void RenderDocSetActiveWindow(VkInstance instance, SDL_Window* window); +void RenderDocRequestCapture(); +void RenderDocOnPresent(); + +} // namespace Libs::Graphics + +#endif /* EMULATOR_INCLUDE_EMULATOR_GRAPHICS_RENDERDOC_H_ */ diff --git a/src/graphics/presentation/videoOut.cpp b/src/graphics/presentation/videoOut.cpp new file mode 100644 index 000000000..4fdd5014f --- /dev/null +++ b/src/graphics/presentation/videoOut.cpp @@ -0,0 +1,1891 @@ +#include "graphics/presentation/videoOut.h" + +#include "common/abi.h" +#include "common/assert.h" +#include "common/common.h" +#include "common/emulatorConfig.h" +#include "common/logging/log.h" +#include "common/profiler.h" +#include "common/stringUtils.h" +#include "common/threads.h" +#include "common/timer.h" +#include "graphics/guest_gpu/gpu_defs.h" +#include "graphics/guest_gpu/tile.h" +#include "graphics/host_gpu/renderer/render.h" +#include "graphics/host_gpu/renderer/renderContext.h" +#include "graphics/host_gpu/renderer/textureCache.h" +#include "graphics/presentation/displayBuffer.h" +#include "graphics/presentation/window.h" +#include "kernel/pthread.h" +#include "libs/errno.h" +#include "libs/libs.h" + +#include +#include +#include + +namespace Libs::Graphics { +struct GraphicContext; +} // namespace Libs::Graphics + +namespace Libs::VideoOut { + +LIB_NAME("VideoOut", "VideoOut"); + +namespace EventQueue = LibKernel::EventQueue; + +constexpr int VIDEO_OUT_EVENT_FLIP = 0; +constexpr int VIDEO_OUT_EVENT_VBLANK = 1; +constexpr int VIDEO_OUT_EVENT_PRE_VBLANK_START = 2; +constexpr int VIDEO_OUT_EVENT_SET_MODE = 8; +constexpr int VIDEO_OUT_TRUE = 1; +constexpr int VIDEO_OUT_FALSE = 0; +constexpr int VIDEO_OUT_FLIP_MODE_VSYNC = 1; +constexpr int VIDEO_OUT_FLIP_MODE_VSYNC_MULTI = 4; +constexpr int VIDEO_OUT_BUFFER_INDEX_BLACK = -2; +constexpr int VIDEO_OUT_BUFFER_INDEX_BLANK = -1; +constexpr int VIDEO_OUT_BUFFER_NUM_MAX = 16; +constexpr int VIDEO_OUT_BUFFER_ATTRIBUTE_NUM_MAX = 4; +constexpr uint64_t VIDEO_OUT_OUTPUT_MODE_DEFAULT = 0x0000000000000001ULL; +constexpr uint64_t VIDEO_OUT_OUTPUT_MODE_119_88HZ = 0x000000000000000FULL; +constexpr uint64_t VIDEO_OUT_REFRESH_RATE_59_94HZ = 3; +constexpr uint64_t VIDEO_OUT_REFRESH_RATE_119_88HZ = 13; +constexpr int VIDEO_OUT_BUFFER_ATTRIBUTE_CATEGORY_UNCOMPRESSED = 0; +constexpr int VIDEO_OUT_BUFFER_ATTRIBUTE_CATEGORY_COMPRESSED = 1; +constexpr uint64_t VIDEO_OUT_BUFFER_ATTRIBUTE_OPTION_STRICT_COLORIMETRY = 8; + +struct VideoOutBufferAttribute2 { + uint32_t reserved0; + uint32_t tiling_mode; + uint32_t aspect_ratio; + uint32_t width; + uint32_t height; + uint32_t pitch_in_pixel; + uint64_t option; + uint64_t pixel_format; + uint64_t dcc_cb_register_clear_color; + uint32_t dcc_control; + uint32_t pad0; + uint64_t reserved1[3]; +}; + +// PS5 layout +struct VideoOutFlipStatus { + uint64_t count = 0; + uint64_t processTime = 0; + uint64_t reserved0 = 0; + int64_t flipArg = 0; + uint64_t reserved1 = 0; + uint64_t processTimeCounter = 0; + int32_t gcQueueNum = 0; + int32_t flipPendingNum = 0; + int32_t currentBuffer = 0; + uint32_t reserved2 = 0; + uint64_t submitProcessTimeCounter = 0; + uint64_t reserved3[7] = {}; +}; + +// PS5 layout +struct VideoOutVblankStatus { + uint64_t count = 0; + uint64_t processTime = 0; + uint64_t reserved = 0; + uint64_t processTimeCounter = 0; + uint8_t flags = 0; + uint8_t phase = 0; + uint8_t pad1[6] = {}; +}; + +struct VideoOutOutputStatus { + uint32_t resolution = 0; + uint32_t dynamicRange = 0; + uint64_t refreshRate = 0; + uint64_t flags = 0; + uint64_t reserved[3] = {}; +}; + +struct VideoOutOutputOptions { + uint32_t internalData[16] = {}; +}; + +struct VideoOutColorSettings { + float gamma = 1.0f; + uint32_t reserved[3] = {}; +}; + +struct VideoOutBuffers { + const void* data; + const void* metadata; + const void* reserved[2]; +}; + +struct VideoOutBufferSet { + int start_index = 0; + int num = 0; + int set_id = 0; +}; + +struct VideoOutBufferInfo { + const void* buffer = nullptr; + Graphics::VideoOutVulkanImage* buffer_vulkan = nullptr; + uint64_t buffer_size = 0; + uint64_t buffer_pitch = 0; + int set_id = 0; +}; + +struct VideoOutConfig { + Common::Mutex mutex; + uint32_t width = 0; + uint32_t height = 0; + bool opened = false; + bool closing = false; + bool unregistering[16] = {}; + int flip_rate = 0; + std::vector flip_eqs; + std::vector pre_vblank_eqs; + std::vector vblank_eqs; + std::vector output_mode_eqs; + uint64_t output_mode = VIDEO_OUT_OUTPUT_MODE_DEFAULT; + float gamma = 1.0f; + VideoOutFlipStatus flip_status; + VideoOutVblankStatus pre_vblank_status; + VideoOutVblankStatus vblank_status; + VideoOutBufferInfo buffers[16]; + std::vector buffers_sets; +}; + +class FlipQueue { +public: + FlipQueue() { EXIT_NOT_IMPLEMENTED(!Common::Thread::IsMainThread()); } + virtual ~FlipQueue() { KYTY_NOT_IMPLEMENTED; } + KYTY_CLASS_NO_COPY(FlipQueue); + + bool Submit(VideoOutConfig* cfg, int index, int64_t flip_arg, bool gpu_eop); + bool Flip(uint32_t micros); + bool HasPending(VideoOutConfig* cfg, int start_index, int count); + void GetFlipStatus(VideoOutConfig* cfg, VideoOutFlipStatus* out); + void AllowEopPresent(VideoOutConfig* cfg, int index); + void Wait(VideoOutConfig* cfg, int index); + +private: + struct Request { + VideoOutConfig* cfg; + int index; + int64_t flip_arg; + uint64_t submit_ptc; + bool gpu_eop; + bool prepared; + }; + + Common::Mutex m_mutex; + Common::CondVar m_submit_cond_var; + Common::CondVar m_done_cond_var; + std::list m_requests; + bool m_processing = false; +}; + +class VideoOutContext { +public: + static constexpr int VIDEO_OUT_NUM_MAX = 2; + + VideoOutContext() { EXIT_NOT_IMPLEMENTED(!Common::Thread::IsMainThread()); } + virtual ~VideoOutContext() { KYTY_NOT_IMPLEMENTED; } + + KYTY_CLASS_NO_COPY(VideoOutContext); + + int Open(); + void Close(int handle); + VideoOutConfig* Get(int handle); + bool IsOpened(int handle); + + Presentation::DisplayBufferImage FindImage(const void* buffer, bool render_target); + + void Init(uint32_t width, uint32_t height); + + Graphics::GraphicContext* GetGraphicCtx() { + Common::LockGuard lock(m_mutex); + + if (m_graphic_ctx == nullptr) { + m_graphic_ctx = Graphics::WindowGetGraphicContext(); + } + + return m_graphic_ctx; + } + + FlipQueue& GetFlipQueue() { return m_flip_queue; } + + void VblankBegin(); + void VblankEnd(); + +private: + Common::Mutex m_mutex; + VideoOutConfig m_video_out_ctx[VIDEO_OUT_NUM_MAX]; + Graphics::GraphicContext* m_graphic_ctx = nullptr; + FlipQueue m_flip_queue; +}; + +static VideoOutContext* g_video_out_context = nullptr; + +static void WaitForNextVblank() { + static uint64_t next_vblank_ticks = 0; + + const uint64_t frequency = Common::Timer::QueryPerformanceFrequency(); + const uint64_t period = frequency / Config::GetVblankFrequency(); + uint64_t now = Common::Timer::QueryPerformanceCounter(); + + if (next_vblank_ticks == 0) { + next_vblank_ticks = now; + } + + if (now < next_vblank_ticks) { + const uint64_t wait_ticks = next_vblank_ticks - now; + const uint64_t wait_micros = (wait_ticks * 1000000u) / frequency; + + if (wait_micros > 0) { + Common::Thread::SleepMicro( + static_cast(std::min(wait_micros, UINT32_MAX))); + } + + now = Common::Timer::QueryPerformanceCounter(); + } + + do { + next_vblank_ticks += period; + } while (next_vblank_ticks <= now); +} + +static bool IsFlipDue(VideoOutConfig* cfg) { + EXIT_IF(cfg == nullptr); + + Common::LockGuard lock(cfg->mutex); + + const int interval = cfg->flip_rate + 1; + + return interval <= 1 || (cfg->vblank_status.count % static_cast(interval)) == 0; +} + +static bool IsValidBufferIndex(int index) { + return index >= VIDEO_OUT_BUFFER_INDEX_BLACK && index < VIDEO_OUT_BUFFER_NUM_MAX; +} + +static bool IsSpecialBufferIndex(int index) { + return index == VIDEO_OUT_BUFFER_INDEX_BLANK || index == VIDEO_OUT_BUFFER_INDEX_BLACK; +} + +static Graphics::VideoOutInfo MakeVideoOutInfo(const VideoOutBufferAttribute2& attribute, + uint64_t address, uint64_t metadata_address, + Graphics::VideoOutCompression compression) { + if (attribute.reserved0 != 0 || attribute.aspect_ratio != 0 || attribute.width == 0 || + attribute.height == 0 || attribute.width > 16384 || attribute.height > 16384 || + attribute.pitch_in_pixel != 0 || + (attribute.option != 0 && + attribute.option != VIDEO_OUT_BUFFER_ATTRIBUTE_OPTION_STRICT_COLORIMETRY) || + attribute.tiling_mode != 0 || attribute.pad0 != 0 || attribute.reserved1[0] != 0 || + attribute.reserved1[1] != 0 || attribute.reserved1[2] != 0 || address == 0 || + compression == Graphics::VideoOutCompression::Unsupported) { + EXIT("unsupported or invalid video-out surface attributes\n"); + } + Graphics::VideoOutPixelFormatInfo pixel_format {}; + if (!Graphics::DecodeVideoOutPixelFormat(attribute.pixel_format, &pixel_format)) { + EXIT("unsupported video-out pixel format: 0x%016" PRIx64 "\n", attribute.pixel_format); + } + const auto tile_mode = Graphics::Prospero::GpuEnumValue(Graphics::Prospero::TileMode::kRenderTarget); + const auto pitch = + Graphics::TileGetTexturePitch(pixel_format.guest_format, attribute.width, 1, tile_mode); + Graphics::TileSizeAlign total {}; + Graphics::TileGetTextureTotalSize(pixel_format.guest_format, attribute.width, attribute.height, + 1, pitch, 1, tile_mode, false, &total); + if (total.size == 0 || total.align != 65536 || (address & (total.align - 1u)) != 0) { + EXIT("invalid video-out surface footprint or alignment\n"); + } + Graphics::VideoOutInfo info {}; + info.address = address; + info.size = total.size; + info.metadata_address = metadata_address; + info.format = pixel_format.format; + info.guest_format = pixel_format.guest_format; + info.width = attribute.width; + info.height = attribute.height; + info.pitch = pitch; + info.bytes_per_element = 4; + info.tile_mode = tile_mode; + info.dcc_control = attribute.dcc_control; + info.compression = compression; + return info; +} + +void VideoOutInit(uint32_t width, uint32_t height) { + EXIT_IF(g_video_out_context != nullptr); + + g_video_out_context = new VideoOutContext; + + g_video_out_context->Init(width, height); +} + +void VideoOutContext::Init(uint32_t width, uint32_t height) { + for (auto& ctx: m_video_out_ctx) { + ctx.width = width; + ctx.height = height; + } +} + +int VideoOutContext::Open() { + Common::LockGuard lock(m_mutex); + + int handle = -1; + + for (int i = 1; i < VIDEO_OUT_NUM_MAX; i++) { + if (!m_video_out_ctx[i].opened) { + handle = i; + break; + } + } + + if (handle < 0) { + return -1; + } + auto& config = m_video_out_ctx[handle]; + Common::LockGuard config_lock(config.mutex); + + EXIT_IF(!config.flip_eqs.empty()); + EXIT_IF(!config.pre_vblank_eqs.empty()); + EXIT_IF(!config.vblank_eqs.empty()); + EXIT_IF(!config.output_mode_eqs.empty()); + EXIT_IF(config.flip_rate != 0); + EXIT_IF(!config.buffers_sets.empty()); + for (const auto& buffer: config.buffers) { + EXIT_IF(buffer.buffer != nullptr || buffer.buffer_vulkan != nullptr || + buffer.buffer_size != 0 || buffer.buffer_pitch != 0); + } + for (bool unregistering: config.unregistering) { + EXIT_IF(unregistering); + } + + config.closing = false; + config.opened = true; + config.output_mode = VIDEO_OUT_OUTPUT_MODE_DEFAULT; + config.flip_status = VideoOutFlipStatus(); + config.flip_status.flipArg = -1; + config.flip_status.currentBuffer = -1; + config.flip_status.count = 0; + config.pre_vblank_status = VideoOutVblankStatus(); + config.vblank_status = VideoOutVblankStatus(); + + return handle; +} + +void VideoOutContext::Close(int handle) { + Common::LockGuard lock(m_mutex); + + EXIT_NOT_IMPLEMENTED(handle >= VIDEO_OUT_NUM_MAX); + EXIT_NOT_IMPLEMENTED(!m_video_out_ctx[handle].opened); + + m_video_out_ctx[handle].opened = false; + + m_video_out_ctx[handle].mutex.Lock(); + if (m_video_out_ctx[handle].closing) { + EXIT("video-out handle is already closing\n"); + } + m_video_out_ctx[handle].closing = true; + if (m_flip_queue.HasPending(&m_video_out_ctx[handle], VIDEO_OUT_BUFFER_INDEX_BLACK, + VIDEO_OUT_BUFFER_NUM_MAX - VIDEO_OUT_BUFFER_INDEX_BLACK)) { + EXIT("cannot close video-out handle with pending flips\n"); + } + for (auto& flip_eq: m_video_out_ctx[handle].flip_eqs) { + if (flip_eq != nullptr) { + auto result = EventQueue::KernelDeleteEvent(flip_eq, VIDEO_OUT_EVENT_FLIP, + EventQueue::KERNEL_EVFILT_VIDEO_OUT); + EXIT_NOT_IMPLEMENTED(result != OK); + } + } + m_video_out_ctx[handle].flip_eqs.clear(); + for (auto& vblank_eq: m_video_out_ctx[handle].pre_vblank_eqs) { + if (vblank_eq != nullptr) { + auto result = EventQueue::KernelDeleteEvent(vblank_eq, VIDEO_OUT_EVENT_PRE_VBLANK_START, + EventQueue::KERNEL_EVFILT_VIDEO_OUT); + EXIT_NOT_IMPLEMENTED(result != OK); + } + } + m_video_out_ctx[handle].pre_vblank_eqs.clear(); + for (auto& vblank_eq: m_video_out_ctx[handle].vblank_eqs) { + if (vblank_eq != nullptr) { + auto result = EventQueue::KernelDeleteEvent(vblank_eq, VIDEO_OUT_EVENT_VBLANK, + EventQueue::KERNEL_EVFILT_VIDEO_OUT); + EXIT_NOT_IMPLEMENTED(result != OK); + } + } + m_video_out_ctx[handle].vblank_eqs.clear(); + for (auto& output_mode_eq: m_video_out_ctx[handle].output_mode_eqs) { + if (output_mode_eq != nullptr) { + auto result = EventQueue::KernelDeleteEvent(output_mode_eq, VIDEO_OUT_EVENT_SET_MODE, + EventQueue::KERNEL_EVFILT_VIDEO_OUT); + EXIT_NOT_IMPLEMENTED(result != OK); + } + } + m_video_out_ctx[handle].output_mode_eqs.clear(); + + m_video_out_ctx[handle].flip_rate = 0; + + std::vector images; + for (const auto& buffer: m_video_out_ctx[handle].buffers) { + if ((buffer.buffer == nullptr) != (buffer.buffer_vulkan == nullptr) || + (buffer.buffer_vulkan != nullptr && + (buffer.buffer_size == 0 || buffer.buffer_pitch == 0))) { + EXIT("inconsistent registered video-out buffer state\n"); + } + if (buffer.buffer_vulkan != nullptr) { + images.push_back(buffer.buffer_vulkan); + } + } + if (!images.empty()) { + if (Graphics::g_render_ctx == nullptr) { + EXIT("cannot unregister video-out surfaces without a render context\n"); + } + Graphics::g_render_ctx->GetTextureCache()->UnregisterVideoOutSurfaces(images); + } + for (auto& buffer: m_video_out_ctx[handle].buffers) { + buffer.buffer = nullptr; + buffer.buffer_vulkan = nullptr; + buffer.buffer_size = 0; + buffer.buffer_pitch = 0; + buffer.set_id = 0; + } + + m_video_out_ctx[handle].buffers_sets.clear(); + for (bool unregistering: m_video_out_ctx[handle].unregistering) { + if (unregistering) { + EXIT("video-out close raced with buffer unregistration\n"); + } + } + m_video_out_ctx[handle].mutex.Unlock(); +} + +VideoOutConfig* VideoOutContext::Get(int handle) { + Common::LockGuard lock(m_mutex); + if (handle <= 0 || handle >= VIDEO_OUT_NUM_MAX || !m_video_out_ctx[handle].opened) { + return nullptr; + } + + return m_video_out_ctx + handle; +} + +bool VideoOutContext::IsOpened(int handle) { + Common::LockGuard lock(m_mutex); + + return handle > 0 && handle < VIDEO_OUT_NUM_MAX && m_video_out_ctx[handle].opened; +} + +void VideoOutContext::VblankBegin() { + Common::LockGuard lock(m_mutex); + + for (int i = 1; i < VIDEO_OUT_NUM_MAX; i++) { + auto& ctx = m_video_out_ctx[i]; + if (ctx.opened) { + ctx.mutex.Lock(); + ctx.pre_vblank_status.count++; + ctx.pre_vblank_status.processTime = LibKernel::KernelGetProcessTime(); + ctx.pre_vblank_status.reserved = LibKernel::KernelReadTsc(); + ctx.pre_vblank_status.processTimeCounter = LibKernel::KernelGetProcessTimeCounter(); + + for (auto& vblank_eq: ctx.pre_vblank_eqs) { + if (vblank_eq != nullptr) { + auto result = EventQueue::KernelTriggerEvent( + vblank_eq, VIDEO_OUT_EVENT_PRE_VBLANK_START, + EventQueue::KERNEL_EVFILT_VIDEO_OUT, + reinterpret_cast(ctx.pre_vblank_status.count)); + EXIT_NOT_IMPLEMENTED(result != OK); + } + } + ctx.mutex.Unlock(); + } + } +} + +void VideoOutContext::VblankEnd() { + Common::LockGuard lock(m_mutex); + + for (int i = 1; i < VIDEO_OUT_NUM_MAX; i++) { + auto& ctx = m_video_out_ctx[i]; + if (ctx.opened) { + ctx.mutex.Lock(); + ctx.vblank_status.count++; + ctx.vblank_status.processTime = LibKernel::KernelGetProcessTime(); + ctx.vblank_status.reserved = LibKernel::KernelReadTsc(); + ctx.vblank_status.processTimeCounter = LibKernel::KernelGetProcessTimeCounter(); + + for (auto& vblank_eq: ctx.vblank_eqs) { + if (vblank_eq != nullptr) { + auto result = EventQueue::KernelTriggerEvent( + vblank_eq, VIDEO_OUT_EVENT_VBLANK, EventQueue::KERNEL_EVFILT_VIDEO_OUT, + reinterpret_cast(ctx.vblank_status.count)); + EXIT_NOT_IMPLEMENTED(result != OK); + } + } + ctx.mutex.Unlock(); + } + } +} + +Presentation::DisplayBufferImage VideoOutContext::FindImage(const void* buffer, + bool render_target) { + Presentation::DisplayBufferImage ret; + Common::LockGuard lock(m_mutex); + for (auto& ctx: m_video_out_ctx) { + if (!ctx.opened) { + continue; + } + Common::LockGuard config_lock(ctx.mutex); + if (ctx.closing) { + EXIT("display-buffer lookup raced with video-out close\n"); + } + for (const auto& set: ctx.buffers_sets) { + for (int j = set.start_index; j < set.start_index + set.num; j++) { + if (ctx.buffers[j].buffer == buffer) { + if (ctx.unregistering[j] || ctx.buffers[j].buffer_vulkan == nullptr) { + EXIT("display-buffer lookup found an unavailable video-out surface\n"); + } + ret.image = ctx.buffers[j].buffer_vulkan; + ret.size = ctx.buffers[j].buffer_size; + ret.pitch = ctx.buffers[j].buffer_pitch; + ret.index = j - set.start_index; + Graphics::g_render_ctx->GetTextureCache()->RefreshVideoOut(ret.image, + render_target); + return ret; + } + } + } + } + return ret; +} + +bool FlipQueue::Submit(VideoOutConfig* cfg, int index, int64_t flip_arg, bool gpu_eop) { + Common::LockGuard lock(m_mutex); + + if (m_requests.size() >= 16) { + return false; + } + + Request r {}; + r.cfg = cfg; + r.index = index; + r.flip_arg = flip_arg; + r.submit_ptc = LibKernel::KernelGetProcessTimeCounter(); + r.gpu_eop = gpu_eop; + r.prepared = !gpu_eop; + + m_requests.push_back(r); + + cfg->flip_status.flipPendingNum = static_cast(m_requests.size()); + cfg->flip_status.submitProcessTimeCounter = r.submit_ptc; + if (gpu_eop) { + cfg->flip_status.gcQueueNum++; + } + + m_submit_cond_var.Signal(); + + return true; +} + +void FlipQueue::AllowEopPresent(VideoOutConfig* cfg, int index) { + Common::LockGuard lock(m_mutex); + auto request = std::find_if(m_requests.begin(), m_requests.end(), [cfg, index](const auto& r) { + return r.cfg == cfg && r.index == index && r.gpu_eop && !r.prepared; + }); + if (request == m_requests.end()) { + EXIT("completed GPU flip has no queued request\n"); + } + request->prepared = true; + m_submit_cond_var.Signal(); +} + +void FlipQueue::Wait(VideoOutConfig* cfg, int index) { + Common::LockGuard lock(m_mutex); + + while (std::find_if(m_requests.begin(), m_requests.end(), [cfg, index](const auto& r) { + return r.cfg == cfg && r.index == index; + }) != m_requests.end()) { + m_done_cond_var.Wait(&m_mutex); + } +} + +bool FlipQueue::HasPending(VideoOutConfig* cfg, int start_index, int count) { + if (cfg == nullptr || count <= 0 || start_index > INT_MAX - count) { + EXIT("invalid video-out pending-flip query range\n"); + } + Common::LockGuard lock(m_mutex); + return std::any_of(m_requests.begin(), m_requests.end(), [&](const auto& request) { + return request.cfg == cfg && request.index >= start_index && + request.index < start_index + count; + }); +} + +bool FlipQueue::Flip(uint32_t micros) { + KYTY_PROFILER_BLOCK("FlipQueue::Flip"); + + m_mutex.Lock(); + if (m_requests.empty()) { + m_submit_cond_var.WaitFor(&m_mutex, micros); + + if (m_requests.empty()) { + m_mutex.Unlock(); + return false; + } + } + if (m_processing) { + EXIT("video-out flip queue processing is already active\n"); + } + if (m_requests.front().gpu_eop && !m_requests.front().prepared) { + m_mutex.Unlock(); + return false; + } + m_processing = true; + auto r = m_requests.front(); + m_mutex.Unlock(); + if (!IsFlipDue(r.cfg)) { + Common::LockGuard lock(m_mutex); + m_processing = false; + return false; + } + + Graphics::VideoOutVulkanImage* buffer = nullptr; + r.cfg->mutex.Lock(); + if (!IsSpecialBufferIndex(r.index)) { + buffer = r.cfg->buffers[r.index].buffer_vulkan; + if (buffer == nullptr) { + EXIT("queued video-out flip has no native surface\n"); + } + } + r.cfg->mutex.Unlock(); + if (buffer != nullptr) { + Graphics::g_render_ctx->GetTextureCache()->RefreshVideoOut(buffer); + Graphics::WindowDrawBuffer(buffer); + } + + m_mutex.Lock(); + if (m_requests.empty() || m_requests.front().cfg != r.cfg || + m_requests.front().index != r.index || m_requests.front().submit_ptc != r.submit_ptc) { + EXIT("video-out flip queue changed while processing its front request\n"); + } + m_requests.pop_front(); + + r.cfg->flip_status.count++; + r.cfg->flip_status.processTime = LibKernel::KernelGetProcessTime(); + r.cfg->flip_status.processTimeCounter = LibKernel::KernelGetProcessTimeCounter(); + r.cfg->flip_status.submitProcessTimeCounter = r.submit_ptc; + r.cfg->flip_status.flipArg = r.flip_arg; + r.cfg->flip_status.currentBuffer = r.index; + r.cfg->flip_status.flipPendingNum = static_cast(m_requests.size()); + if (r.gpu_eop && r.cfg->flip_status.gcQueueNum > 0) { + r.cfg->flip_status.gcQueueNum--; + } + + m_processing = false; + m_done_cond_var.Signal(); + m_mutex.Unlock(); + + r.cfg->mutex.Lock(); + for (auto& flip_eq: r.cfg->flip_eqs) { + if (flip_eq != nullptr) { + auto result = EventQueue::KernelTriggerEvent(flip_eq, VIDEO_OUT_EVENT_FLIP, + EventQueue::KERNEL_EVFILT_VIDEO_OUT, + reinterpret_cast(r.flip_arg)); + EXIT_NOT_IMPLEMENTED(result != OK); + } + } + r.cfg->mutex.Unlock(); + + if (Config::GraphicsDebugDumpEnabled() && + Config::GetPrintfDirection() != Config::OutputDirection::Silent) { + LOGF("Flip done: %d\n", r.index); + } + + return true; +} + +void FlipQueue::GetFlipStatus(VideoOutConfig* cfg, VideoOutFlipStatus* out) { + EXIT_IF(cfg == nullptr); + EXIT_IF(out == nullptr); + + Common::LockGuard lock(m_mutex); + + *out = cfg->flip_status; +} + +bool VideoOutFlipWindow(uint32_t micros) { + EXIT_IF(g_video_out_context == nullptr); + + WaitForNextVblank(); + + return g_video_out_context->GetFlipQueue().Flip(micros); +} + +void VideoOutBeginVblank() { + EXIT_IF(g_video_out_context == nullptr); + + g_video_out_context->VblankBegin(); +} + +void VideoOutEndVblank() { + EXIT_IF(g_video_out_context == nullptr); + + g_video_out_context->VblankEnd(); +} + +KYTY_SYSV_ABI int VideoOutOpen(int user_id, int bus_type, int index, const void* param) { + PRINT_NAME(); + + EXIT_IF(g_video_out_context == nullptr); + + EXIT_NOT_IMPLEMENTED(user_id != 255 && user_id != 0); + EXIT_NOT_IMPLEMENTED(bus_type != 0); + EXIT_NOT_IMPLEMENTED(index != 0); + + LOGF("\t param = 0x%016" PRIx64 "\n", reinterpret_cast(param)); + + int handle = g_video_out_context->Open(); + + if (handle < 0) { + return VIDEO_OUT_ERROR_RESOURCE_BUSY; + } + + return handle; +} + +KYTY_SYSV_ABI int VideoOutClose(int handle) { + PRINT_NAME(); + + EXIT_IF(g_video_out_context == nullptr); + + if (!g_video_out_context->IsOpened(handle)) { + return VIDEO_OUT_ERROR_INVALID_HANDLE; + } + + g_video_out_context->Close(handle); + + return OK; +} + +KYTY_SYSV_ABI void VideoOutSetBufferAttribute2(VideoOutBufferAttribute2* attribute, + uint64_t pixel_format, uint32_t tiling_mode, + uint32_t width, uint32_t height, uint64_t option, + uint32_t dcc_control, + uint64_t dcc_cb_register_clear_color) { + PRINT_NAME(); + + EXIT_NOT_IMPLEMENTED(attribute == nullptr); + + LOGF("\t pixel_format = %016" PRIx64 "\n" + "\t tiling_mode = %" PRIu32 "\n" + "\t width = %" PRIu32 "\n" + "\t height = %" PRIu32 "\n" + "\t option = %016" PRIx64 "\n" + "\t dcc_control = %08" PRIx32 "\n" + "\t dcc_cb_register_clear_color = %016" PRIx64 "\n", + pixel_format, tiling_mode, width, height, option, dcc_control, + dcc_cb_register_clear_color); + + memset(attribute, 0, sizeof(VideoOutBufferAttribute2)); + + attribute->tiling_mode = tiling_mode; + attribute->aspect_ratio = 0; + attribute->width = width; + attribute->height = height; + attribute->pitch_in_pixel = 0; + attribute->option = option; + attribute->pixel_format = pixel_format; + attribute->dcc_cb_register_clear_color = dcc_cb_register_clear_color; + attribute->dcc_control = dcc_control; +} + +KYTY_SYSV_ABI int VideoOutSetFlipRate(int handle, int rate) { + PRINT_NAME(); + + EXIT_IF(g_video_out_context == nullptr); + + LOGF("\trate = %d\n", rate); + + auto* ctx = g_video_out_context->Get(handle); + if (ctx == nullptr) { + return VIDEO_OUT_ERROR_INVALID_HANDLE; + } + + if (rate < 0 || rate > 2) { + return VIDEO_OUT_ERROR_INVALID_VALUE; + } + + ctx->flip_rate = rate; + + return OK; +} + +static void FlipEventResetFunc(LibKernel::EventQueue::KernelEqueueEvent* event) { + EXIT_IF(event == nullptr); + event->triggered = false; + event->event.fflags = 0; + event->event.data = 0; +} + +static void FlipEventDeleteFunc(EventQueue::KernelEqueue eq, + LibKernel::EventQueue::KernelEqueueEvent* event) { + EXIT_IF(event == nullptr); + EXIT_IF(event->filter.data == nullptr); + + EXIT_NOT_IMPLEMENTED(event->event.ident != VIDEO_OUT_EVENT_FLIP); + EXIT_NOT_IMPLEMENTED(event->event.filter != EventQueue::KERNEL_EVFILT_VIDEO_OUT); + + if (event->filter.data != nullptr) { + auto* video_out = static_cast(event->filter.data); + video_out->mutex.Lock(); + EXIT_IF(video_out->flip_eqs.empty()); + auto it = std::find(video_out->flip_eqs.begin(), video_out->flip_eqs.end(), eq); + EXIT_NOT_IMPLEMENTED(it == video_out->flip_eqs.end()); + *it = nullptr; + video_out->mutex.Unlock(); + } +} + +static intptr_t MakeVideoOutEventData(intptr_t current_data, void* trigger_data) { + const uint64_t old_data = static_cast(current_data); + uint64_t counter = (old_data >> 12u) & 0xfu; + if (counter != 0xfu) { + counter++; + } + + const uint64_t time = LibKernel::KernelReadTsc() & 0xfffu; + const uint64_t payload = static_cast(reinterpret_cast(trigger_data)); + + return static_cast(time | (counter << 12u) | + ((payload & 0x0000ffffffffffffULL) << 16u)); +} + +static void FlipEventTriggerFunc(LibKernel::EventQueue::KernelEqueueEvent* event, + void* trigger_data) { + EXIT_IF(event == nullptr); + + auto triggered_event = event->event; + triggered_event.fflags = + ((triggered_event.fflags < 0xfu) ? triggered_event.fflags + 1u : triggered_event.fflags); + triggered_event.data = MakeVideoOutEventData(triggered_event.data, trigger_data); + if (event->triggered) { + event->pending_events.push_back(triggered_event); + } else { + event->event = triggered_event; + event->triggered = true; + } +} + +static void VblankEventResetFunc(LibKernel::EventQueue::KernelEqueueEvent* event) { + EXIT_IF(event == nullptr); + event->triggered = false; + event->event.fflags = 0; + event->event.data = 0; +} + +static void VblankEventDeleteFunc(EventQueue::KernelEqueue eq, + LibKernel::EventQueue::KernelEqueueEvent* event) { + EXIT_IF(event == nullptr); + EXIT_IF(event->filter.data == nullptr); + + EXIT_NOT_IMPLEMENTED(event->event.ident != VIDEO_OUT_EVENT_VBLANK); + EXIT_NOT_IMPLEMENTED(event->event.filter != EventQueue::KERNEL_EVFILT_VIDEO_OUT); + + if (event->filter.data != nullptr) { + auto* video_out = static_cast(event->filter.data); + video_out->mutex.Lock(); + EXIT_IF(video_out->vblank_eqs.empty()); + auto it = std::find(video_out->vblank_eqs.begin(), video_out->vblank_eqs.end(), eq); + EXIT_NOT_IMPLEMENTED(it == video_out->vblank_eqs.end()); + *it = nullptr; + video_out->mutex.Unlock(); + } +} + +static void PreVblankEventDeleteFunc(EventQueue::KernelEqueue eq, + LibKernel::EventQueue::KernelEqueueEvent* event) { + EXIT_IF(event == nullptr); + EXIT_IF(event->filter.data == nullptr); + + EXIT_NOT_IMPLEMENTED(event->event.ident != VIDEO_OUT_EVENT_PRE_VBLANK_START); + EXIT_NOT_IMPLEMENTED(event->event.filter != EventQueue::KERNEL_EVFILT_VIDEO_OUT); + + if (event->filter.data != nullptr) { + auto* video_out = static_cast(event->filter.data); + video_out->mutex.Lock(); + EXIT_IF(video_out->pre_vblank_eqs.empty()); + auto it = std::find(video_out->pre_vblank_eqs.begin(), video_out->pre_vblank_eqs.end(), eq); + EXIT_NOT_IMPLEMENTED(it == video_out->pre_vblank_eqs.end()); + *it = nullptr; + video_out->mutex.Unlock(); + } +} + +static void VblankEventTriggerFunc(LibKernel::EventQueue::KernelEqueueEvent* event, + void* trigger_data) { + EXIT_IF(event == nullptr); + + auto triggered_event = event->event; + triggered_event.fflags = + ((triggered_event.fflags < 0xfu) ? triggered_event.fflags + 1u : triggered_event.fflags); + triggered_event.data = MakeVideoOutEventData(triggered_event.data, trigger_data); + if (event->triggered) { + event->pending_events.push_back(triggered_event); + } else { + event->event = triggered_event; + event->triggered = true; + } +} + +static void OutputModeEventResetFunc(LibKernel::EventQueue::KernelEqueueEvent* event) { + EXIT_IF(event == nullptr); + event->triggered = false; + event->event.fflags = 0; + event->event.data = 0; +} + +static void OutputModeEventDeleteFunc(EventQueue::KernelEqueue eq, + LibKernel::EventQueue::KernelEqueueEvent* event) { + EXIT_IF(event == nullptr); + EXIT_IF(event->filter.data == nullptr); + + EXIT_NOT_IMPLEMENTED(event->event.ident != VIDEO_OUT_EVENT_SET_MODE); + EXIT_NOT_IMPLEMENTED(event->event.filter != EventQueue::KERNEL_EVFILT_VIDEO_OUT); + + if (event->filter.data != nullptr) { + auto* video_out = static_cast(event->filter.data); + video_out->mutex.Lock(); + EXIT_IF(video_out->output_mode_eqs.empty()); + auto it = + std::find(video_out->output_mode_eqs.begin(), video_out->output_mode_eqs.end(), eq); + EXIT_NOT_IMPLEMENTED(it == video_out->output_mode_eqs.end()); + *it = nullptr; + video_out->mutex.Unlock(); + } +} + +static void OutputModeEventTriggerFunc(LibKernel::EventQueue::KernelEqueueEvent* event, + void* trigger_data) { + EXIT_IF(event == nullptr); + + auto triggered_event = event->event; + triggered_event.fflags = + ((triggered_event.fflags < 0xfu) ? triggered_event.fflags + 1u : triggered_event.fflags); + triggered_event.data = MakeVideoOutEventData(triggered_event.data, trigger_data); + if (event->triggered) { + event->pending_events.push_back(triggered_event); + } else { + event->event = triggered_event; + event->triggered = true; + } +} + +KYTY_SYSV_ABI int VideoOutDeleteFlipEvent(EventQueue::KernelEqueue eq, int handle) { + PRINT_NAME(); + + EXIT_IF(g_video_out_context == nullptr); + + [[maybe_unused]] auto* ctx = g_video_out_context->Get(handle); + if (ctx == nullptr) { + return VIDEO_OUT_ERROR_INVALID_HANDLE; + } + + if (eq == nullptr) { + return VIDEO_OUT_ERROR_INVALID_EVENT_QUEUE; + } + + return EventQueue::KernelDeleteEvent(eq, VIDEO_OUT_EVENT_FLIP, + EventQueue::KERNEL_EVFILT_VIDEO_OUT); +} + +KYTY_SYSV_ABI int VideoOutAddFlipEvent(EventQueue::KernelEqueue eq, int handle, void* udata) { + PRINT_NAME(); + + EXIT_IF(g_video_out_context == nullptr); + + auto* ctx = g_video_out_context->Get(handle); + if (ctx == nullptr) { + return VIDEO_OUT_ERROR_INVALID_HANDLE; + } + + ctx->mutex.Lock(); + + if (eq == nullptr) { + ctx->mutex.Unlock(); + return VIDEO_OUT_ERROR_INVALID_EVENT_QUEUE; + } + + bool add_eq = + (std::find(ctx->flip_eqs.begin(), ctx->flip_eqs.end(), eq) == ctx->flip_eqs.end()); + + EventQueue::KernelEqueueEvent event; + event.triggered = false; + event.event.ident = VIDEO_OUT_EVENT_FLIP; + event.event.filter = EventQueue::KERNEL_EVFILT_VIDEO_OUT; + event.event.udata = udata; + event.event.fflags = 0; + event.event.data = 0; + event.filter.delete_event_func = FlipEventDeleteFunc; + event.filter.reset_func = FlipEventResetFunc; + event.filter.trigger_func = FlipEventTriggerFunc; + event.filter.data = ctx; + + int result = EventQueue::KernelAddEvent(eq, event); + + if (add_eq) { + ctx->flip_eqs.push_back(eq); + } + + ctx->mutex.Unlock(); + + return result; +} + +KYTY_SYSV_ABI int VideoOutDeleteVblankEvent(EventQueue::KernelEqueue eq, int handle) { + PRINT_NAME(); + + EXIT_IF(g_video_out_context == nullptr); + + [[maybe_unused]] auto* ctx = g_video_out_context->Get(handle); + if (ctx == nullptr) { + return VIDEO_OUT_ERROR_INVALID_HANDLE; + } + + if (eq == nullptr) { + return VIDEO_OUT_ERROR_INVALID_EVENT_QUEUE; + } + + return EventQueue::KernelDeleteEvent(eq, VIDEO_OUT_EVENT_VBLANK, + EventQueue::KERNEL_EVFILT_VIDEO_OUT); +} + +KYTY_SYSV_ABI int VideoOutDeletePreVblankStartEvent(EventQueue::KernelEqueue eq, int handle) { + PRINT_NAME(); + + EXIT_IF(g_video_out_context == nullptr); + + [[maybe_unused]] auto* ctx = g_video_out_context->Get(handle); + if (ctx == nullptr) { + return VIDEO_OUT_ERROR_INVALID_HANDLE; + } + + if (eq == nullptr) { + return VIDEO_OUT_ERROR_INVALID_EVENT_QUEUE; + } + + return EventQueue::KernelDeleteEvent(eq, VIDEO_OUT_EVENT_PRE_VBLANK_START, + EventQueue::KERNEL_EVFILT_VIDEO_OUT); +} + +KYTY_SYSV_ABI int VideoOutAddVblankEvent(LibKernel::EventQueue::KernelEqueue eq, int handle, + void* udata) { + PRINT_NAME(); + + EXIT_IF(g_video_out_context == nullptr); + + auto* ctx = g_video_out_context->Get(handle); + if (ctx == nullptr) { + return VIDEO_OUT_ERROR_INVALID_HANDLE; + } + + ctx->mutex.Lock(); + + if (eq == nullptr) { + ctx->mutex.Unlock(); + return VIDEO_OUT_ERROR_INVALID_EVENT_QUEUE; + } + + bool add_eq = + (std::find(ctx->vblank_eqs.begin(), ctx->vblank_eqs.end(), eq) == ctx->vblank_eqs.end()); + + EventQueue::KernelEqueueEvent event; + event.triggered = false; + event.event.ident = VIDEO_OUT_EVENT_VBLANK; + event.event.filter = EventQueue::KERNEL_EVFILT_VIDEO_OUT; + event.event.udata = udata; + event.event.fflags = 0; + event.event.data = 0; + event.filter.delete_event_func = VblankEventDeleteFunc; + event.filter.reset_func = VblankEventResetFunc; + event.filter.trigger_func = VblankEventTriggerFunc; + event.filter.data = ctx; + + int result = EventQueue::KernelAddEvent(eq, event); + + if (add_eq) { + ctx->vblank_eqs.push_back(eq); + } + + ctx->mutex.Unlock(); + + return result; +} + +KYTY_SYSV_ABI int VideoOutAddPreVblankStartEvent(LibKernel::EventQueue::KernelEqueue eq, int handle, + void* udata) { + PRINT_NAME(); + + EXIT_IF(g_video_out_context == nullptr); + + auto* ctx = g_video_out_context->Get(handle); + if (ctx == nullptr) { + return VIDEO_OUT_ERROR_INVALID_HANDLE; + } + + ctx->mutex.Lock(); + + if (eq == nullptr) { + ctx->mutex.Unlock(); + return VIDEO_OUT_ERROR_INVALID_EVENT_QUEUE; + } + + bool add_eq = (std::find(ctx->pre_vblank_eqs.begin(), ctx->pre_vblank_eqs.end(), eq) == + ctx->pre_vblank_eqs.end()); + + EventQueue::KernelEqueueEvent event; + event.triggered = false; + event.event.ident = VIDEO_OUT_EVENT_PRE_VBLANK_START; + event.event.filter = EventQueue::KERNEL_EVFILT_VIDEO_OUT; + event.event.udata = udata; + event.event.fflags = 0; + event.event.data = 0; + event.filter.delete_event_func = PreVblankEventDeleteFunc; + event.filter.reset_func = VblankEventResetFunc; + event.filter.trigger_func = VblankEventTriggerFunc; + event.filter.data = ctx; + + int result = EventQueue::KernelAddEvent(eq, event); + + if (add_eq) { + ctx->pre_vblank_eqs.push_back(eq); + } + + ctx->mutex.Unlock(); + + return result; +} + +KYTY_SYSV_ABI int VideoOutAddOutputModeEvent(LibKernel::EventQueue::KernelEqueue eq, int handle, + void* udata) { + PRINT_NAME(); + + EXIT_IF(g_video_out_context == nullptr); + + auto* ctx = g_video_out_context->Get(handle); + if (ctx == nullptr) { + return VIDEO_OUT_ERROR_INVALID_HANDLE; + } + + ctx->mutex.Lock(); + + LOGF("\t eq = 0x%016" PRIx64 "\n" + "\t handle = %d\n" + "\t udata = 0x%016" PRIx64 "\n", + reinterpret_cast(eq), handle, reinterpret_cast(udata)); + + if (eq == nullptr) { + ctx->mutex.Unlock(); + return VIDEO_OUT_ERROR_INVALID_EVENT_QUEUE; + } + + bool add_eq = (std::find(ctx->output_mode_eqs.begin(), ctx->output_mode_eqs.end(), eq) == + ctx->output_mode_eqs.end()); + + EventQueue::KernelEqueueEvent event; + event.triggered = true; + event.event.ident = VIDEO_OUT_EVENT_SET_MODE; + event.event.filter = EventQueue::KERNEL_EVFILT_VIDEO_OUT; + event.event.udata = udata; + event.event.fflags = 1; + event.event.data = MakeVideoOutEventData(0, reinterpret_cast(ctx->output_mode)); + event.filter.delete_event_func = OutputModeEventDeleteFunc; + event.filter.reset_func = OutputModeEventResetFunc; + event.filter.trigger_func = OutputModeEventTriggerFunc; + event.filter.data = ctx; + + int result = EventQueue::KernelAddEvent(eq, event); + + if (add_eq) { + ctx->output_mode_eqs.push_back(eq); + } + + ctx->mutex.Unlock(); + + return result; +} + +static int RegisterBuffersInternal(VideoOutConfig* ctx, int set_id, int start_index, + const void* const* addresses, int buffer_num, + const std::vector& infos) { + if (ctx == nullptr || addresses == nullptr || buffer_num <= 0 || + infos.size() != static_cast(buffer_num)) { + EXIT("invalid internal video-out buffer registration arguments\n"); + } + if (set_id < 0 || set_id >= VIDEO_OUT_BUFFER_ATTRIBUTE_NUM_MAX) { + EXIT("internal video-out buffer set identifier is out of range\n"); + } + Graphics::WindowWaitForGraphicInitialized(); + Graphics::GraphicsRenderCreateContext(); + auto* graphic_ctx = g_video_out_context->GetGraphicCtx(); + Common::LockGuard lock(ctx->mutex); + if (ctx->closing) { + EXIT("cannot register buffers on a closing video-out handle\n"); + } + if (std::any_of(ctx->buffers_sets.begin(), ctx->buffers_sets.end(), + [set_id](const auto& set) { return set.set_id == set_id; })) { + return VIDEO_OUT_ERROR_INVALID_INDEX; + } + for (int i = 0; i < buffer_num; i++) { + if (ctx->unregistering[start_index + i]) { + EXIT("video-out buffer registration raced with unregistration\n"); + } + if (ctx->buffers[start_index + i].buffer != nullptr) { + return VIDEO_OUT_ERROR_SLOT_OCCUPIED; + } + } + auto images = + Graphics::g_render_ctx->GetTextureCache()->RegisterVideoOutSurfaces(graphic_ctx, infos); + if (images.size() != infos.size()) { + EXIT("video-out texture cache returned an incomplete surface set\n"); + } + ctx->buffers_sets.push_back({start_index, buffer_num, set_id}); + for (int i = 0; i < buffer_num; i++) { + auto& dst = ctx->buffers[i + start_index]; + dst.set_id = set_id; + dst.buffer = addresses[i]; + dst.buffer_size = infos[i].size; + dst.buffer_pitch = infos[i].pitch; + dst.buffer_vulkan = images[i]; + LOGF("\tbuffers[%d] = %016" PRIx64 " metadata = %016" PRIx64 " dcc = %08" PRIx32 "\n", + i + start_index, reinterpret_cast(addresses[i]), infos[i].metadata_address, + infos[i].dcc_control); + } + + return OK; +} + +KYTY_SYSV_ABI int VideoOutRegisterBuffers2(int handle, int set_index, int buffer_index_start, + const VideoOutBuffers* buffers, int buffer_num, + const VideoOutBufferAttribute2* attribute, int category, + void* option) { + PRINT_NAME(); + + EXIT_IF(g_video_out_context == nullptr); + + auto* ctx = g_video_out_context->Get(handle); + if (ctx == nullptr) { + return VIDEO_OUT_ERROR_INVALID_HANDLE; + } + + if (buffers == nullptr) { + return VIDEO_OUT_ERROR_INVALID_ADDRESS; + } + + if (attribute == nullptr) { + return VIDEO_OUT_ERROR_INVALID_OPTION; + } + + if (set_index < 0 || set_index >= VIDEO_OUT_BUFFER_ATTRIBUTE_NUM_MAX || + buffer_index_start < 0 || buffer_index_start >= VIDEO_OUT_BUFFER_NUM_MAX || + buffer_num < 1 || buffer_num > VIDEO_OUT_BUFFER_NUM_MAX || + buffer_index_start + buffer_num > VIDEO_OUT_BUFFER_NUM_MAX) { + return VIDEO_OUT_ERROR_INVALID_VALUE; + } + + LOGF("\t start_index = %d\n" + "\t buffer_num = %d\n" + "\t set_index = %d\n" + "\t pixel_format = 0x%016" PRIx64 "\n" + "\t tiling_mode = %" PRIu32 "\n" + "\t aspect_ratio = %" PRIu32 "\n" + "\t width = %" PRIu32 "\n" + "\t height = %" PRIu32 "\n" + "\t pitch_in_pixel = %" PRIu32 "\n" + "\t option = %" PRIu64 "\n" + "\t category = %d\n", + buffer_index_start, buffer_num, set_index, attribute->pixel_format, attribute->tiling_mode, + attribute->aspect_ratio, attribute->width, attribute->height, attribute->pitch_in_pixel, + attribute->option, category); + + if (option != nullptr) { + EXIT("video-out buffer registration options are unsupported\n"); + } + if (category != VIDEO_OUT_BUFFER_ATTRIBUTE_CATEGORY_UNCOMPRESSED && + category != VIDEO_OUT_BUFFER_ATTRIBUTE_CATEGORY_COMPRESSED) { + return VIDEO_OUT_ERROR_INVALID_CATEGORY; + } + const bool compressed = category == VIDEO_OUT_BUFFER_ATTRIBUTE_CATEGORY_COMPRESSED; + std::vector addresses(static_cast(buffer_num)); + std::vector infos; + infos.reserve(static_cast(buffer_num)); + + for (int i = 0; i < buffer_num; i++) { + if (buffers[i].reserved[0] != nullptr || buffers[i].reserved[1] != nullptr) { + EXIT("video-out buffer reserved fields are unsupported\n"); + } + const auto data_address = reinterpret_cast(buffers[i].data); + const auto metadata_address = reinterpret_cast(buffers[i].metadata); + const auto compression = Graphics::ClassifyVideoOutCompression( + compressed, metadata_address, attribute->dcc_control, + attribute->dcc_cb_register_clear_color); + if (compression == Graphics::VideoOutCompression::Unsupported) { + EXIT("unsupported video-out compression, category=%d data=0x%016" PRIx64 + " metadata=0x%016" PRIx64 " dcc_control=0x%08" PRIx32 " clear_color=0x%016" PRIx64 + "\n", + category, data_address, metadata_address, attribute->dcc_control, + attribute->dcc_cb_register_clear_color); + } + addresses[static_cast(i)] = buffers[i].data; + infos.push_back(MakeVideoOutInfo(*attribute, data_address, metadata_address, compression)); + } + + return RegisterBuffersInternal(ctx, set_index, buffer_index_start, addresses.data(), buffer_num, + infos); +} + +KYTY_SYSV_ABI int VideoOutSubmitChangeBufferAttribute2(int handle, int set_index, + const VideoOutBufferAttribute2* attribute, + void* option) { + PRINT_NAME(); + + EXIT_IF(g_video_out_context == nullptr); + + auto* ctx = g_video_out_context->Get(handle); + if (ctx == nullptr) { + return VIDEO_OUT_ERROR_INVALID_HANDLE; + } + + if (attribute == nullptr) { + return VIDEO_OUT_ERROR_INVALID_OPTION; + } + if (set_index < 0 || set_index >= VIDEO_OUT_BUFFER_ATTRIBUTE_NUM_MAX) { + return VIDEO_OUT_ERROR_INVALID_INDEX; + } + + if (option != nullptr) { + return VIDEO_OUT_ERROR_INVALID_OPTION; + } + (void)ctx; + (void)set_index; + EXIT("video-out buffer attribute query is unsupported\n"); +} + +KYTY_SYSV_ABI int VideoOutUnregisterBuffers(int handle, int set_index) { + PRINT_NAME(); + + EXIT_IF(g_video_out_context == nullptr); + + auto* ctx = g_video_out_context->Get(handle); + if (ctx == nullptr) { + return VIDEO_OUT_ERROR_INVALID_HANDLE; + } + if (set_index < 0 || set_index >= VIDEO_OUT_BUFFER_ATTRIBUTE_NUM_MAX) { + return VIDEO_OUT_ERROR_INVALID_INDEX; + } + + Common::LockGuard lock(ctx->mutex); + if (ctx->closing) { + EXIT("cannot unregister buffers from a closing video-out handle\n"); + } + auto set_it = std::find_if(ctx->buffers_sets.begin(), ctx->buffers_sets.end(), + [set_index](const auto& set) { return set.set_id == set_index; }); + if (set_it == ctx->buffers_sets.end()) { + return VIDEO_OUT_ERROR_INVALID_INDEX; + } + std::vector images; + images.reserve(static_cast(set_it->num)); + for (int i = set_it->start_index; i < set_it->start_index + set_it->num; i++) { + if (ctx->unregistering[i]) { + EXIT("video-out buffer is already being unregistered\n"); + } + ctx->unregistering[i] = true; + if (ctx->buffers[i].set_id != set_index || ctx->buffers[i].buffer == nullptr || + ctx->buffers[i].buffer_vulkan == nullptr) { + EXIT("video-out buffer set contains inconsistent surface state\n"); + } + images.push_back(ctx->buffers[i].buffer_vulkan); + } + if (g_video_out_context->GetFlipQueue().HasPending(ctx, set_it->start_index, set_it->num)) { + EXIT("cannot unregister video-out buffers with pending flips\n"); + } + Graphics::g_render_ctx->GetTextureCache()->UnregisterVideoOutSurfaces(images); + for (int i = set_it->start_index; i < set_it->start_index + set_it->num; i++) { + ctx->buffers[i] = VideoOutBufferInfo {}; + ctx->unregistering[i] = false; + } + ctx->buffers_sets.erase(set_it); + + return OK; +} + +} // namespace Libs::VideoOut + +namespace Libs::Presentation { + +DisplayBufferImage DisplayBufferFind(uint64_t addr, bool render_target) { + EXIT_IF(VideoOut::g_video_out_context == nullptr); + + return VideoOut::g_video_out_context->FindImage(reinterpret_cast(addr), render_target); +} + +} // namespace Libs::Presentation + +namespace Libs::VideoOut { + +KYTY_SYSV_ABI int VideoOutSubmitFlip(int handle, int index, int flip_mode, int64_t flip_arg) { + PRINT_NAME(); + + EXIT_IF(g_video_out_context == nullptr); + + auto* ctx = g_video_out_context->Get(handle); + if (ctx == nullptr) { + return VIDEO_OUT_ERROR_INVALID_HANDLE; + } + + if (flip_mode < VIDEO_OUT_FLIP_MODE_VSYNC || flip_mode > VIDEO_OUT_FLIP_MODE_VSYNC_MULTI) { + LOGF("\t unsupported flip_mode = %d\n", flip_mode); + return VIDEO_OUT_ERROR_INVALID_VALUE; + } + + if (!IsValidBufferIndex(index)) { + return VIDEO_OUT_ERROR_INVALID_INDEX; + } + Common::LockGuard lock(ctx->mutex); + if (ctx->closing || (!IsSpecialBufferIndex(index) && ctx->unregistering[index])) { + return VIDEO_OUT_ERROR_INVALID_INDEX; + } + if (!IsSpecialBufferIndex(index) && ctx->buffers[index].buffer_vulkan == nullptr) { + return VIDEO_OUT_ERROR_INVALID_INDEX; + } + + if (!g_video_out_context->GetFlipQueue().Submit(ctx, index, flip_arg, false)) { + return VIDEO_OUT_ERROR_FLIP_QUEUE_FULL; + } + + return OK; +} + +} // namespace Libs::VideoOut + +namespace Libs::Presentation { + +int DisplayBufferSubmitFlipFromGpu(int handle, int index, int flip_mode, int64_t flip_arg) { + EXIT_IF(VideoOut::g_video_out_context == nullptr || Graphics::g_render_ctx == nullptr); + + auto* ctx = VideoOut::g_video_out_context->Get(handle); + if (ctx == nullptr) { + return VideoOut::VIDEO_OUT_ERROR_INVALID_HANDLE; + } + + if (flip_mode < VideoOut::VIDEO_OUT_FLIP_MODE_VSYNC || + flip_mode > VideoOut::VIDEO_OUT_FLIP_MODE_VSYNC_MULTI) { + return VideoOut::VIDEO_OUT_ERROR_INVALID_VALUE; + } + + if (!VideoOut::IsValidBufferIndex(index)) { + return VideoOut::VIDEO_OUT_ERROR_INVALID_INDEX; + } + Common::LockGuard lock(ctx->mutex); + if (ctx->closing || (!VideoOut::IsSpecialBufferIndex(index) && ctx->unregistering[index])) { + return VideoOut::VIDEO_OUT_ERROR_INVALID_INDEX; + } + if (!VideoOut::IsSpecialBufferIndex(index) && ctx->buffers[index].buffer_vulkan == nullptr) { + return VideoOut::VIDEO_OUT_ERROR_INVALID_INDEX; + } + if (!VideoOut::g_video_out_context->GetFlipQueue().Submit(ctx, index, flip_arg, true)) { + return VideoOut::VIDEO_OUT_ERROR_FLIP_QUEUE_FULL; + } + + return OK; +} + +void DisplayBufferCompleteFlipFromGpu(int handle, int index) { + EXIT_IF(VideoOut::g_video_out_context == nullptr || Graphics::g_render_ctx == nullptr); + + auto* ctx = VideoOut::g_video_out_context->Get(handle); + if (ctx == nullptr || !VideoOut::IsValidBufferIndex(index)) { + EXIT("GPU flip completed with invalid handle or index, handle=%d index=%d\n", handle, + index); + } + Common::LockGuard lock(ctx->mutex); + if (ctx->closing || (!VideoOut::IsSpecialBufferIndex(index) && ctx->unregistering[index])) { + EXIT("GPU flip completed for an unavailable surface, handle=%d index=%d\n", handle, index); + } + if (!VideoOut::IsSpecialBufferIndex(index)) { + auto* image = ctx->buffers[index].buffer_vulkan; + if (image == nullptr) { + EXIT("GPU flip completed without a native surface, handle=%d index=%d\n", handle, + index); + } + Graphics::g_render_ctx->GetTextureCache()->RefreshVideoOut(image); + } + VideoOut::g_video_out_context->GetFlipQueue().AllowEopPresent(ctx, index); +} + +} // namespace Libs::Presentation + +namespace Libs::VideoOut { + +void VideoOutWaitFlipDone(int handle, int index) { + EXIT_IF(g_video_out_context == nullptr); + + auto* ctx = g_video_out_context->Get(handle); + EXIT_IF(ctx == nullptr); + + EXIT_NOT_IMPLEMENTED(!IsValidBufferIndex(index)); + g_video_out_context->GetFlipQueue().Wait(ctx, index); +} + +KYTY_SYSV_ABI int VideoOutGetFlipStatus(int handle, VideoOutFlipStatus* status) { + PRINT_NAME(); + + EXIT_IF(g_video_out_context == nullptr); + + if (status == nullptr) { + return VIDEO_OUT_ERROR_INVALID_ADDRESS; + } + + auto* ctx = g_video_out_context->Get(handle); + if (ctx == nullptr) { + return VIDEO_OUT_ERROR_INVALID_HANDLE; + } + + g_video_out_context->GetFlipQueue().GetFlipStatus(ctx, status); + + LOGF("\t count = %" PRIu64 "\n" + "\t processTime = %" PRIu64 "\n" + "\t processTimeCounter = %" PRIu64 "\n" + "\t submitProcessTimeCounter = %" PRIu64 "\n" + "\t flipArg = %" PRId64 "\n" + "\t gcQueueNum = %d\n" + "\t flipPendingNum = %d\n" + "\t currentBuffer = %d\n", + status->count, status->processTime, status->processTimeCounter, + status->submitProcessTimeCounter, status->flipArg, status->gcQueueNum, + status->flipPendingNum, status->currentBuffer); + + return OK; +} + +KYTY_SYSV_ABI int VideoOutIsFlipPending(int handle) { + PRINT_NAME(); + + EXIT_IF(g_video_out_context == nullptr); + + auto* ctx = g_video_out_context->Get(handle); + if (ctx == nullptr) { + return VIDEO_OUT_ERROR_INVALID_HANDLE; + } + + VideoOutFlipStatus status {}; + g_video_out_context->GetFlipQueue().GetFlipStatus(ctx, &status); + + LOGF("\t flipPendingNum = %d\n", status.flipPendingNum); + + return status.flipPendingNum; +} + +KYTY_SYSV_ABI int VideoOutGetVblankStatus(int handle, VideoOutVblankStatus* status) { + PRINT_NAME(); + + EXIT_IF(g_video_out_context == nullptr); + + if (status == nullptr) { + return VIDEO_OUT_ERROR_INVALID_ADDRESS; + } + + auto* ctx = g_video_out_context->Get(handle); + if (ctx == nullptr) { + return VIDEO_OUT_ERROR_INVALID_HANDLE; + } + + ctx->mutex.Lock(); + *status = ctx->vblank_status; + ctx->mutex.Unlock(); + + LOGF("\t count = %" PRIu64 "\n" + "\t processTime = %" PRIu64 "\n" + "\t processTimeCounter = %" PRIu64 "\n", + status->count, status->processTime, status->processTimeCounter); + + return OK; +} + +KYTY_SYSV_ABI int VideoOutGetEventId(const EventQueue::KernelEvent* ev) { + PRINT_NAME(); + + if (ev == nullptr) { + return VIDEO_OUT_ERROR_INVALID_ADDRESS; + } + + if (ev->filter != EventQueue::KERNEL_EVFILT_VIDEO_OUT) { + return VIDEO_OUT_ERROR_INVALID_EVENT; + } + + switch (ev->ident) { + case VIDEO_OUT_EVENT_FLIP: + case VIDEO_OUT_EVENT_VBLANK: + case VIDEO_OUT_EVENT_PRE_VBLANK_START: + case VIDEO_OUT_EVENT_SET_MODE: return static_cast(ev->ident); + default: return VIDEO_OUT_ERROR_INVALID_EVENT; + } +} + +KYTY_SYSV_ABI int VideoOutGetEventData(const EventQueue::KernelEvent* ev, int64_t* data) { + PRINT_NAME(); + + if (ev == nullptr || data == nullptr) { + return VIDEO_OUT_ERROR_INVALID_ADDRESS; + } + + if (ev->filter != EventQueue::KERNEL_EVFILT_VIDEO_OUT) { + return VIDEO_OUT_ERROR_INVALID_EVENT; + } + + uint64_t event_data = static_cast(ev->data) >> 16u; + if (ev->ident == VIDEO_OUT_EVENT_FLIP && + (static_cast(ev->data) & 0x8000000000000000ULL) != 0) { + event_data |= 0xffff000000000000ULL; + } + + *data = static_cast(event_data); + + return OK; +} + +KYTY_SYSV_ABI int VideoOutGetEventCount(const EventQueue::KernelEvent* ev) { + PRINT_NAME(); + + if (ev == nullptr) { + return VIDEO_OUT_ERROR_INVALID_ADDRESS; + } + + if (ev->filter != EventQueue::KERNEL_EVFILT_VIDEO_OUT) { + return VIDEO_OUT_ERROR_INVALID_EVENT; + } + + return static_cast((static_cast(ev->data) >> 12u) & 0xfu); +} + +KYTY_SYSV_ABI int VideoOutWaitVblank(int handle) { + PRINT_NAME(); + + EXIT_IF(g_video_out_context == nullptr); + + [[maybe_unused]] auto* ctx = g_video_out_context->Get(handle); + if (ctx == nullptr) { + return VIDEO_OUT_ERROR_INVALID_HANDLE; + } + + WaitForNextVblank(); + g_video_out_context->VblankEnd(); + + return OK; +} + +KYTY_SYSV_ABI int VideoOutGetOutputStatus(int handle, VideoOutOutputStatus* status) { + PRINT_NAME(); + + EXIT_IF(g_video_out_context == nullptr); + + if (status == nullptr) { + return VIDEO_OUT_ERROR_INVALID_ADDRESS; + } + + auto* ctx = g_video_out_context->Get(handle); + if (ctx == nullptr) { + return VIDEO_OUT_ERROR_INVALID_HANDLE; + } + + ctx->mutex.Lock(); + status->resolution = (ctx->width >= 3840 || ctx->height >= 2160 ? 2u : 1u); + status->dynamicRange = 1; + status->refreshRate = + (ctx->output_mode == VIDEO_OUT_OUTPUT_MODE_119_88HZ || Config::GetVblankFrequency() >= 119 + ? VIDEO_OUT_REFRESH_RATE_119_88HZ + : VIDEO_OUT_REFRESH_RATE_59_94HZ); + status->flags = 0; + status->reserved[0] = 0; + status->reserved[1] = 0; + status->reserved[2] = 0; + ctx->mutex.Unlock(); + + return OK; +} + +static int ValidateOutputConfig(int handle, uint64_t mode, const VideoOutOutputOptions* options, + void* reserved_ptr, uint64_t reserved) { + EXIT_IF(g_video_out_context == nullptr); + + if (!g_video_out_context->IsOpened(handle)) { + return VIDEO_OUT_ERROR_INVALID_HANDLE; + } + + if (reserved_ptr != nullptr || reserved != 0) { + return VIDEO_OUT_ERROR_INVALID_VALUE; + } + + if (options != nullptr) { + for (auto v: options->internalData) { + if (v != 0) { + return VIDEO_OUT_ERROR_INVALID_OPTION; + } + } + } + + if (mode != VIDEO_OUT_OUTPUT_MODE_DEFAULT && mode != VIDEO_OUT_OUTPUT_MODE_119_88HZ) { + return VIDEO_OUT_ERROR_UNSUPPORTED_OUTPUT_MODE; + } + + return OK; +} + +KYTY_SYSV_ABI int VideoOutInitializeOutputOptions(VideoOutOutputOptions* options) { + PRINT_NAME(); + + if (options == nullptr) { + return VIDEO_OUT_ERROR_INVALID_ADDRESS; + } + + memset(options, 0, sizeof(VideoOutOutputOptions)); + + return OK; +} + +KYTY_SYSV_ABI int VideoOutIsOutputSupported(int handle, uint64_t mode, + const VideoOutOutputOptions* options, + void* reserved_ptr, uint64_t reserved) { + PRINT_NAME(); + + LOGF("\t mode = 0x%016" PRIx64 "\n", mode); + + int result = ValidateOutputConfig(handle, mode, options, reserved_ptr, reserved); + if (result != OK) { + return result; + } + + if (mode == VIDEO_OUT_OUTPUT_MODE_119_88HZ) { + return (Config::GetVblankFrequency() >= 119 ? VIDEO_OUT_TRUE : VIDEO_OUT_FALSE); + } + + return VIDEO_OUT_TRUE; +} + +KYTY_SYSV_ABI int VideoOutConfigureOutput(int handle, uint64_t mode, + const VideoOutOutputOptions* options, void* reserved_ptr, + uint64_t reserved) { + PRINT_NAME(); + + LOGF("\t mode = 0x%016" PRIx64 "\n", mode); + + int result = VideoOutIsOutputSupported(handle, mode, options, reserved_ptr, reserved); + if (result < 0) { + return result; + } + if (result == VIDEO_OUT_FALSE) { + return VIDEO_OUT_ERROR_UNAVAILABLE_OUTPUT_MODE; + } + + auto* ctx = g_video_out_context->Get(handle); + if (ctx == nullptr) { + return VIDEO_OUT_ERROR_INVALID_HANDLE; + } + + ctx->mutex.Lock(); + ctx->output_mode = mode; + for (auto& output_mode_eq: ctx->output_mode_eqs) { + if (output_mode_eq != nullptr) { + auto trigger_result = EventQueue::KernelTriggerEvent( + output_mode_eq, VIDEO_OUT_EVENT_SET_MODE, EventQueue::KERNEL_EVFILT_VIDEO_OUT, + reinterpret_cast(ctx->output_mode)); + EXIT_NOT_IMPLEMENTED(trigger_result != OK); + } + } + ctx->mutex.Unlock(); + + return OK; +} + +KYTY_SYSV_ABI int VideoOutSetWindowModeMargins(int handle, int top, int bottom) { + PRINT_NAME(); + + EXIT_IF(g_video_out_context == nullptr); + + [[maybe_unused]] auto* ctx = g_video_out_context->Get(handle); + if (ctx == nullptr) { + return VIDEO_OUT_ERROR_INVALID_HANDLE; + } + + LOGF("\t top = %d\n" + "\t bottom = %d\n", + top, bottom); + + return OK; +} + +KYTY_SYSV_ABI int VideoOutLatencyControlWaitBeforeInput(int handle) { + PRINT_NAME(); + + EXIT_IF(g_video_out_context == nullptr); + + [[maybe_unused]] auto* ctx = g_video_out_context->Get(handle); + if (ctx == nullptr) { + return VIDEO_OUT_ERROR_INVALID_HANDLE; + } + + return OK; +} + +KYTY_SYSV_ABI int VideoOutLatencyMeasureSetStartPoint(int handle, uint32_t point) { + PRINT_NAME(); + + EXIT_IF(g_video_out_context == nullptr); + + [[maybe_unused]] auto* ctx = g_video_out_context->Get(handle); + if (ctx == nullptr) { + return VIDEO_OUT_ERROR_INVALID_HANDLE; + } + + LOGF("\t point = %" PRIu32 "\n", point); + + return OK; +} + +KYTY_SYSV_ABI int VideoOutColorSettingsSetGamma(VideoOutColorSettings* settings, float gamma) { + PRINT_NAME(); + + if (settings == nullptr) { + return VIDEO_OUT_ERROR_INVALID_ADDRESS; + } + + if (gamma < 0.1f || gamma > 2.0f) { + return VIDEO_OUT_ERROR_INVALID_VALUE; + } + + settings->gamma = gamma; + return OK; +} + +KYTY_SYSV_ABI int VideoOutAdjustColor(int handle, const VideoOutColorSettings* settings) { + PRINT_NAME(); + + EXIT_IF(g_video_out_context == nullptr); + + if (settings == nullptr) { + return VIDEO_OUT_ERROR_INVALID_ADDRESS; + } + + if (!g_video_out_context->IsOpened(handle)) { + return VIDEO_OUT_ERROR_INVALID_HANDLE; + } + + auto* ctx = g_video_out_context->Get(handle); + if (ctx == nullptr) { + return VIDEO_OUT_ERROR_INVALID_HANDLE; + } + ctx->mutex.Lock(); + ctx->gamma = settings->gamma; + ctx->mutex.Unlock(); + + return OK; +} + +} // namespace Libs::VideoOut diff --git a/src/graphics/presentation/videoOut.h b/src/graphics/presentation/videoOut.h new file mode 100644 index 000000000..04666dc82 --- /dev/null +++ b/src/graphics/presentation/videoOut.h @@ -0,0 +1,84 @@ +#ifndef EMULATOR_INCLUDE_EMULATOR_GRAPHICS_VIDEOOUT_H_ +#define EMULATOR_INCLUDE_EMULATOR_GRAPHICS_VIDEOOUT_H_ + +#include "common/common.h" +// #include "common/subsystems.h" + +#include "common/abi.h" +#include "kernel/eventQueue.h" + +namespace Libs::Graphics { +// struct VulkanSwapchain; +struct VideoOutVulkanImage; +} // namespace Libs::Graphics + +namespace Libs::VideoOut { + +struct VideoOutBufferAttribute2; +struct VideoOutFlipStatus; +struct VideoOutVblankStatus; +struct VideoOutOutputStatus; +struct VideoOutOutputOptions; +struct VideoOutBuffers; +struct VideoOutColorSettings; + +void VideoOutInit(uint32_t width, uint32_t height); +void VideoOutWaitFlipDone(int handle, int index); + +KYTY_SYSV_ABI int VideoOutOpen(int user_id, int bus_type, int index, const void* param); +KYTY_SYSV_ABI int VideoOutClose(int handle); +KYTY_SYSV_ABI void VideoOutSetBufferAttribute2(VideoOutBufferAttribute2* attribute, + uint64_t pixel_format, uint32_t tiling_mode, + uint32_t width, uint32_t height, uint64_t option, + uint32_t dcc_control, + uint64_t dcc_cb_register_clear_color); +KYTY_SYSV_ABI int VideoOutSetFlipRate(int handle, int rate); +KYTY_SYSV_ABI int VideoOutAddFlipEvent(LibKernel::EventQueue::KernelEqueue eq, int handle, + void* udata); +KYTY_SYSV_ABI int VideoOutAddVblankEvent(LibKernel::EventQueue::KernelEqueue eq, int handle, + void* udata); +KYTY_SYSV_ABI int VideoOutAddPreVblankStartEvent(LibKernel::EventQueue::KernelEqueue eq, int handle, + void* udata); +KYTY_SYSV_ABI int VideoOutAddOutputModeEvent(LibKernel::EventQueue::KernelEqueue eq, int handle, + void* udata); +KYTY_SYSV_ABI int VideoOutDeleteFlipEvent(LibKernel::EventQueue::KernelEqueue eq, int handle); +KYTY_SYSV_ABI int VideoOutDeleteVblankEvent(LibKernel::EventQueue::KernelEqueue eq, int handle); +KYTY_SYSV_ABI int VideoOutDeletePreVblankStartEvent(LibKernel::EventQueue::KernelEqueue eq, + int handle); +KYTY_SYSV_ABI int VideoOutRegisterBuffers2(int handle, int set_index, int buffer_index_start, + const VideoOutBuffers* buffers, int buffer_num, + const VideoOutBufferAttribute2* attribute, int category, + void* option); +KYTY_SYSV_ABI int VideoOutSubmitChangeBufferAttribute2(int handle, int set_index, + const VideoOutBufferAttribute2* attribute, + void* option); +KYTY_SYSV_ABI int VideoOutUnregisterBuffers(int handle, int set_index); +KYTY_SYSV_ABI int VideoOutSubmitFlip(int handle, int index, int flip_mode, int64_t flip_arg); +KYTY_SYSV_ABI int VideoOutGetFlipStatus(int handle, VideoOutFlipStatus* status); +KYTY_SYSV_ABI int VideoOutIsFlipPending(int handle); +KYTY_SYSV_ABI int VideoOutGetVblankStatus(int handle, VideoOutVblankStatus* status); +KYTY_SYSV_ABI int VideoOutGetEventId(const LibKernel::EventQueue::KernelEvent* ev); +KYTY_SYSV_ABI int VideoOutGetEventData(const LibKernel::EventQueue::KernelEvent* ev, int64_t* data); +KYTY_SYSV_ABI int VideoOutGetEventCount(const LibKernel::EventQueue::KernelEvent* ev); +KYTY_SYSV_ABI int VideoOutWaitVblank(int handle); +KYTY_SYSV_ABI int VideoOutGetOutputStatus(int handle, VideoOutOutputStatus* status); +KYTY_SYSV_ABI int VideoOutInitializeOutputOptions(VideoOutOutputOptions* options); +KYTY_SYSV_ABI int VideoOutIsOutputSupported(int handle, uint64_t mode, + const VideoOutOutputOptions* options, + void* reserved_ptr, uint64_t reserved); +KYTY_SYSV_ABI int VideoOutConfigureOutput(int handle, uint64_t mode, + const VideoOutOutputOptions* options, void* reserved_ptr, + uint64_t reserved); +KYTY_SYSV_ABI int VideoOutSetWindowModeMargins(int handle, int top, int bottom); +KYTY_SYSV_ABI int VideoOutLatencyControlWaitBeforeInput(int handle); +KYTY_SYSV_ABI int VideoOutLatencyMeasureSetStartPoint(int handle, uint32_t point); +KYTY_SYSV_ABI int VideoOutColorSettingsSetGamma(VideoOutColorSettings* settings, float gamma); +KYTY_SYSV_ABI int VideoOutAdjustColor(int handle, const VideoOutColorSettings* settings); + +void VideoOutBeginVblank(); +void VideoOutEndVblank(); +bool VideoOutFlipWindow(uint32_t micros); + +} // namespace Libs::VideoOut + +#endif /* EMULATOR_INCLUDE_EMULATOR_GRAPHICS_VIDEOOUT_H_ */ diff --git a/src/graphics/presentation/window.h b/src/graphics/presentation/window.h new file mode 100644 index 000000000..fca1d0be4 --- /dev/null +++ b/src/graphics/presentation/window.h @@ -0,0 +1,25 @@ +#ifndef EMULATOR_INCLUDE_EMULATOR_GRAPHICS_WINDOW_H_ +#define EMULATOR_INCLUDE_EMULATOR_GRAPHICS_WINDOW_H_ + +#include "common/abi.h" +#include "common/common.h" + +struct VkSurfaceCapabilitiesKHR; + +namespace Libs::Graphics { + +struct GraphicContext; +struct VideoOutVulkanImage; + +VkSurfaceCapabilitiesKHR* VulkanGetSurfaceCapabilities(); + +GraphicContext* WindowGetGraphicContext(); + +void WindowInit(uint32_t width, uint32_t height); +void WindowRun(); +void WindowWaitForGraphicInitialized(); +void WindowDrawBuffer(VideoOutVulkanImage* image); + +} // namespace Libs::Graphics + +#endif /* EMULATOR_INCLUDE_EMULATOR_GRAPHICS_WINDOW_H_ */ diff --git a/src/graphics/presentation/window/swapchain.cpp b/src/graphics/presentation/window/swapchain.cpp new file mode 100644 index 000000000..3556fb7a1 --- /dev/null +++ b/src/graphics/presentation/window/swapchain.cpp @@ -0,0 +1,389 @@ +#include "SDL.h" +#include "SDL_error.h" +#include "SDL_events.h" +#include "SDL_gamecontroller.h" +#include "SDL_hints.h" +#include "SDL_joystick.h" +#include "SDL_keyboard.h" +#include "SDL_keycode.h" +#include "SDL_mouse.h" +#include "SDL_pixels.h" +#include "SDL_rwops.h" +#include "SDL_stdinc.h" +#include "SDL_surface.h" +#include "SDL_thread.h" +#include "SDL_touch.h" +#include "SDL_video.h" +#include "SDL_vulkan.h" +#include "common/assert.h" +#include "common/common.h" +#include "common/emulatorConfig.h" +#include "common/file.h" +#include "common/logging/log.h" +#include "common/profiler.h" +#include "common/stringUtils.h" +#include "common/subsystems.h" +#include "common/systemInfo.h" +#include "common/threads.h" +#include "common/timer.h" +#include "graphics/host_gpu/graphicContext.h" +#include "graphics/host_gpu/renderer/render.h" +#include "graphics/host_gpu/utils.h" +#include "graphics/host_gpu/vma.h" +#include "graphics/presentation/renderDoc.h" +#include "graphics/presentation/videoOut.h" +#include "graphics/presentation/window.h" +#include "graphics/presentation/window/windowInternal.h" +#include "libs/controller.h" +#include "loader/systemContent.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +// IWYU pragma: no_include + +#define KYTY_ENABLE_DEBUG_PRINTF +#define KYTY_DBG_INPUT + +namespace Libs::Graphics { + +VulkanSwapchain::~VulkanSwapchain() = default; + +[[maybe_unused]] static VkSwapchainKHR VulkanCreateSwapchainInternal( + VkDevice device, VkSurfaceKHR surface, uint32_t width, uint32_t height, uint32_t image_count, + SurfaceCapabilities* r, VkFormat* swapchain_format, VkExtent2D* swapchain_extent, + std::unique_ptr* swapchain_images, + std::unique_ptr* swapchain_image_views, uint32_t* swapchain_images_count) { + EXIT_IF(device == nullptr); + EXIT_IF(surface == nullptr); + EXIT_IF(r == nullptr); + EXIT_IF(swapchain_format == nullptr); + EXIT_IF(swapchain_extent == nullptr); + EXIT_IF(swapchain_images == nullptr); + EXIT_IF(swapchain_image_views == nullptr); + EXIT_IF(swapchain_images_count == nullptr); + + EXIT_NOT_IMPLEMENTED(r->formats.empty()); + + VkExtent2D extent {}; + extent.width = std::clamp(width, r->capabilities.minImageExtent.width, + r->capabilities.maxImageExtent.width); + extent.height = std::clamp(height, r->capabilities.minImageExtent.height, + r->capabilities.maxImageExtent.height); + + image_count = + std::clamp(image_count, r->capabilities.minImageCount, r->capabilities.maxImageCount); + + VkSwapchainCreateInfoKHR create_info {}; + create_info.sType = VK_STRUCTURE_TYPE_SWAPCHAIN_CREATE_INFO_KHR; + create_info.pNext = nullptr; + create_info.flags = 0; + create_info.surface = surface; + create_info.minImageCount = image_count; + + if (r->format_unorm_bgra32) { + create_info.imageFormat = VK_FORMAT_B8G8R8A8_UNORM; + create_info.imageColorSpace = VK_COLOR_SPACE_SRGB_NONLINEAR_KHR; + } else if (r->format_srgb_bgra32) { + create_info.imageFormat = VK_FORMAT_B8G8R8A8_SRGB; + create_info.imageColorSpace = VK_COLOR_SPACE_SRGB_NONLINEAR_KHR; + } else { + const auto& format = r->formats[0]; + create_info.imageFormat = format.format; + create_info.imageColorSpace = format.colorSpace; + } + + create_info.imageExtent = extent; + create_info.imageArrayLayers = 1; + create_info.imageUsage = VK_IMAGE_USAGE_COLOR_ATTACHMENT_BIT | VK_IMAGE_USAGE_TRANSFER_DST_BIT; + create_info.imageSharingMode = VK_SHARING_MODE_EXCLUSIVE; + create_info.queueFamilyIndexCount = 0; + create_info.pQueueFamilyIndices = nullptr; + create_info.preTransform = r->capabilities.currentTransform; + create_info.compositeAlpha = VK_COMPOSITE_ALPHA_OPAQUE_BIT_KHR; + create_info.presentMode = VK_PRESENT_MODE_FIFO_KHR; + create_info.clipped = VK_TRUE; + create_info.oldSwapchain = nullptr; + + *swapchain_format = create_info.imageFormat; + *swapchain_extent = extent; + + VkSwapchainKHR swapchain = nullptr; + + vkCreateSwapchainKHR(device, &create_info, nullptr, &swapchain); + + vkGetSwapchainImagesKHR(device, swapchain, swapchain_images_count, nullptr); + EXIT_NOT_IMPLEMENTED(*swapchain_images_count == 0); + + *swapchain_images = std::make_unique(*swapchain_images_count); + vkGetSwapchainImagesKHR(device, swapchain, swapchain_images_count, swapchain_images->get()); + + *swapchain_image_views = std::make_unique(*swapchain_images_count); + for (uint32_t i = 0; i < *swapchain_images_count; i++) { + VkImageViewCreateInfo create_info {}; + create_info.sType = VK_STRUCTURE_TYPE_IMAGE_VIEW_CREATE_INFO; + create_info.pNext = nullptr; + create_info.flags = 0; + create_info.image = (*swapchain_images)[i]; + create_info.viewType = VK_IMAGE_VIEW_TYPE_2D; + create_info.format = *swapchain_format; + create_info.components.r = VK_COMPONENT_SWIZZLE_IDENTITY; + create_info.components.g = VK_COMPONENT_SWIZZLE_IDENTITY; + create_info.components.b = VK_COMPONENT_SWIZZLE_IDENTITY; + create_info.components.a = VK_COMPONENT_SWIZZLE_IDENTITY; + create_info.subresourceRange.aspectMask = VK_IMAGE_ASPECT_COLOR_BIT; + create_info.subresourceRange.baseArrayLayer = 0; + create_info.subresourceRange.baseMipLevel = 0; + create_info.subresourceRange.layerCount = 1; + create_info.subresourceRange.levelCount = 1; + + vkCreateImageView(device, &create_info, nullptr, &((*swapchain_image_views)[i])); + } + + return swapchain; +} + +VulkanSwapchain* VulkanCreateSwapchain(GraphicContext* ctx, uint32_t image_count) { + EXIT_IF(g_window_ctx == nullptr); + EXIT_IF(ctx == nullptr); + EXIT_IF(ctx->screen_width == 0); + EXIT_IF(ctx->screen_height == 0); + + Common::LockGuard lock(g_window_ctx->mutex); + + auto swapchain_owner = std::make_unique(); + auto* s = swapchain_owner.get(); + + s->swapchain = VulkanCreateSwapchainInternal( + ctx->device, g_window_ctx->surface, ctx->screen_width, ctx->screen_height, image_count, + g_window_ctx->surface_capabilities, &s->swapchain_format, &s->swapchain_extent, + &s->swapchain_images, &s->swapchain_image_views, &s->swapchain_images_count); + if (s->swapchain == nullptr) { + EXIT("Could not create swapchain"); + } + + s->current_index = static_cast(-1); + s->present_frame = 0; + + VkSemaphoreCreateInfo semaphore_info {}; + semaphore_info.sType = VK_STRUCTURE_TYPE_SEMAPHORE_CREATE_INFO; + semaphore_info.pNext = nullptr; + semaphore_info.flags = 0; + + s->image_acquired_semaphores = std::make_unique(s->swapchain_images_count); + s->render_complete_semaphores = std::make_unique(s->swapchain_images_count); + for (uint32_t i = 0; i < s->swapchain_images_count; i++) { + auto result = vkCreateSemaphore(ctx->device, &semaphore_info, nullptr, + &s->image_acquired_semaphores[i]); + EXIT_NOT_IMPLEMENTED(result != VK_SUCCESS); + + result = vkCreateSemaphore(ctx->device, &semaphore_info, nullptr, + &s->render_complete_semaphores[i]); + EXIT_NOT_IMPLEMENTED(result != VK_SUCCESS); + } + + return swapchain_owner.release(); +} + +static void VulkanDeleteSwapchain(GraphicContext* ctx, VulkanSwapchain* s) { + if (ctx == nullptr || s == nullptr) { + return; + } + auto swapchain_owner = std::unique_ptr(s); + + VulkanDeviceWaitIdle(ctx); + + s->present_command_buffers.clear(); + + if (s->image_acquired_semaphores != nullptr) { + for (uint32_t i = 0; i < s->swapchain_images_count; i++) { + if (s->image_acquired_semaphores[i] != nullptr) { + vkDestroySemaphore(ctx->device, s->image_acquired_semaphores[i], nullptr); + } + } + } + if (s->render_complete_semaphores != nullptr) { + for (uint32_t i = 0; i < s->swapchain_images_count; i++) { + if (s->render_complete_semaphores[i] != nullptr) { + vkDestroySemaphore(ctx->device, s->render_complete_semaphores[i], nullptr); + } + } + } + if (s->swapchain_image_views != nullptr) { + for (uint32_t i = 0; i < s->swapchain_images_count; i++) { + if (s->swapchain_image_views[i] != nullptr) { + vkDestroyImageView(ctx->device, s->swapchain_image_views[i], nullptr); + } + } + } + + if (s->swapchain != nullptr) { + vkDestroySwapchainKHR(ctx->device, s->swapchain, nullptr); + } +} + +static void VulkanRefreshSurfaceSize() { + EXIT_IF(g_window_ctx == nullptr); + EXIT_IF(g_window_ctx->window == nullptr); + EXIT_IF(g_window_ctx->surface_capabilities == nullptr); + + int width = 0; + int height = 0; + SDL_Vulkan_GetDrawableSize(g_window_ctx->window, &width, &height); + if (width > 0 && height > 0) { + g_window_ctx->graphic_ctx.screen_width = static_cast(width); + g_window_ctx->graphic_ctx.screen_height = static_cast(height); + } + + VulkanGetSurfaceCapabilities(g_window_ctx->graphic_ctx.physical_device, g_window_ctx->surface, + g_window_ctx->surface_capabilities); +} + +static void VulkanRecreateSwapchain() { + EXIT_IF(g_window_ctx == nullptr); + + LOGF("Recreating Vulkan swapchain\n"); + VulkanRefreshSurfaceSize(); + VulkanDeleteSwapchain(&g_window_ctx->graphic_ctx, g_window_ctx->swapchain); + g_window_ctx->swapchain = VulkanCreateSwapchain(&g_window_ctx->graphic_ctx, 2); +} + +static CommandBuffer* WindowGetPresentCommandBuffer(VulkanSwapchain* swapchain, + uint32_t present_frame) { + EXIT_IF(swapchain == nullptr); + EXIT_IF(swapchain->swapchain_images_count == 0); + EXIT_IF(present_frame >= swapchain->swapchain_images_count); + + if (swapchain->present_command_buffers.empty()) { + swapchain->present_command_buffers.resize(swapchain->swapchain_images_count); + } + EXIT_IF(swapchain->present_command_buffers.size() != swapchain->swapchain_images_count); + + auto& buffer = swapchain->present_command_buffers[present_frame]; + if (buffer == nullptr) { + buffer = std::make_unique(GraphicContext::QUEUE_GFX); + } + + return buffer.get(); +} + +void WindowDrawBuffer(VideoOutVulkanImage* image) { + KYTY_PROFILER_FUNCTION(); + + EXIT_IF(image == nullptr); + EXIT_IF(g_window_ctx == nullptr); + EXIT_IF(g_window_ctx->swapchain == nullptr); + + if (g_window_ctx->window_hidden) { + WindowUpdateIcon(); + + SDL_ShowWindow(g_window_ctx->window); + + g_window_ctx->window_hidden = false; + VulkanRecreateSwapchain(); + } + + auto* swapchain = g_window_ctx->swapchain; + + const auto present_frame = swapchain->present_frame; + EXIT_IF(present_frame >= swapchain->swapchain_images_count); + + auto& buffer = *WindowGetPresentCommandBuffer(swapchain, present_frame); + buffer.WaitForFenceAndReset(); + + swapchain->current_index = static_cast(-1); + + auto result = vkAcquireNextImageKHR( + g_window_ctx->graphic_ctx.device, swapchain->swapchain, UINT64_MAX, + swapchain->image_acquired_semaphores[present_frame], nullptr, &swapchain->current_index); + + switch (result) { + case VK_SUCCESS: break; + case VK_SUBOPTIMAL_KHR: LOGF("vkAcquireNextImageKHR returned VK_SUBOPTIMAL_KHR\n"); break; + case VK_ERROR_OUT_OF_DATE_KHR: + LOGF("vkAcquireNextImageKHR returned VK_ERROR_OUT_OF_DATE_KHR\n"); + VulkanRecreateSwapchain(); + return; + default: EXIT("vkAcquireNextImageKHR failed: %s\n", string_VkResult(result)); + } + EXIT_NOT_IMPLEMENTED(swapchain->current_index == static_cast(-1)); + + EXIT_NOT_IMPLEMENTED(buffer.IsInvalid()); + + auto* vk_buffer = buffer.GetPool()->buffers[buffer.GetIndex()]; + + buffer.Begin(); + + UtilBlitImage(&buffer, image, swapchain); + + VkImageMemoryBarrier pre_present_barrier {}; + pre_present_barrier.sType = VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER; + pre_present_barrier.pNext = nullptr; + pre_present_barrier.srcAccessMask = VK_ACCESS_TRANSFER_WRITE_BIT; + pre_present_barrier.dstAccessMask = VK_ACCESS_MEMORY_READ_BIT; + pre_present_barrier.oldLayout = VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL; + pre_present_barrier.newLayout = VK_IMAGE_LAYOUT_PRESENT_SRC_KHR; + pre_present_barrier.srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; + pre_present_barrier.dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; + pre_present_barrier.subresourceRange.aspectMask = VK_IMAGE_ASPECT_COLOR_BIT; + pre_present_barrier.subresourceRange.baseMipLevel = 0; + pre_present_barrier.subresourceRange.levelCount = 1; + pre_present_barrier.subresourceRange.baseArrayLayer = 0; + pre_present_barrier.subresourceRange.layerCount = 1; + pre_present_barrier.image = swapchain->swapchain_images[swapchain->current_index]; + vkCmdPipelineBarrier(vk_buffer, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, + VK_PIPELINE_STAGE_BOTTOM_OF_PIPE_BIT, 0, 0, nullptr, 0, nullptr, 1, + &pre_present_barrier); + + buffer.End(); + + auto render_complete_semaphore = + swapchain->render_complete_semaphores[swapchain->current_index]; + const VkPipelineStageFlags wait_stage = VK_PIPELINE_STAGE_TRANSFER_BIT; + buffer.ExecuteWithSemaphore(swapchain->image_acquired_semaphores[present_frame], wait_stage, + render_complete_semaphore); + + VkPresentInfoKHR present; + present.sType = VK_STRUCTURE_TYPE_PRESENT_INFO_KHR; + present.pNext = nullptr; + present.swapchainCount = 1; + present.pSwapchains = &swapchain->swapchain; + present.pImageIndices = &swapchain->current_index; + present.pWaitSemaphores = &render_complete_semaphore; + present.waitSemaphoreCount = 1; + present.pResults = nullptr; + + const auto& queue = g_window_ctx->graphic_ctx.queues[GraphicContext::QUEUE_PRESENT]; + + if (queue.mutex != nullptr) { + queue.mutex->Lock(); + } + result = vkQueuePresentKHR(queue.vk_queue, &present); + if (queue.mutex != nullptr) { + queue.mutex->Unlock(); + } + switch (result) { + case VK_SUCCESS: break; + case VK_SUBOPTIMAL_KHR: LOGF("vkQueuePresentKHR returned VK_SUBOPTIMAL_KHR\n"); break; + case VK_ERROR_OUT_OF_DATE_KHR: + LOGF("vkQueuePresentKHR returned VK_ERROR_OUT_OF_DATE_KHR\n"); + VulkanRecreateSwapchain(); + return; + default: EXIT("vkQueuePresentKHR failed: %s\n", string_VkResult(result)); + } + + swapchain->present_frame = (present_frame + 1u) % swapchain->swapchain_images_count; + + RenderDocOnPresent(); + WindowUpdateTitle(); +} + +} // namespace Libs::Graphics diff --git a/src/graphics/presentation/window/vulkanWindow.cpp b/src/graphics/presentation/window/vulkanWindow.cpp new file mode 100644 index 000000000..e0fe64416 --- /dev/null +++ b/src/graphics/presentation/window/vulkanWindow.cpp @@ -0,0 +1,1120 @@ +#include "SDL.h" +#include "SDL_error.h" +#include "SDL_events.h" +#include "SDL_gamecontroller.h" +#include "SDL_hints.h" +#include "SDL_joystick.h" +#include "SDL_keyboard.h" +#include "SDL_keycode.h" +#include "SDL_mouse.h" +#include "SDL_pixels.h" +#include "SDL_rwops.h" +#include "SDL_stdinc.h" +#include "SDL_surface.h" +#include "SDL_thread.h" +#include "SDL_touch.h" +#include "SDL_video.h" +#include "SDL_vulkan.h" +#include "common/assert.h" +#include "common/common.h" +#include "common/emulatorConfig.h" +#include "common/file.h" +#include "common/logging/log.h" +#include "common/profiler.h" +#include "common/stringUtils.h" +#include "common/subsystems.h" +#include "common/systemInfo.h" +#include "common/threads.h" +#include "common/timer.h" +#include "graphics/host_gpu/graphicContext.h" +#include "graphics/host_gpu/renderer/render.h" +#include "graphics/host_gpu/utils.h" +#include "graphics/host_gpu/vma.h" +#include "graphics/presentation/renderDoc.h" +#include "graphics/presentation/videoOut.h" +#include "graphics/presentation/window.h" +#include "graphics/presentation/window/windowInternal.h" +#include "libs/controller.h" +#include "loader/systemContent.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +// IWYU pragma: no_include + +#define KYTY_ENABLE_DEBUG_PRINTF +#define KYTY_DBG_INPUT + +namespace Libs::Graphics { + +struct VulkanExtensions { + bool enable_validation_layers = false; + + std::vector required_extensions; + std::vector available_extensions; + std::vector required_layers; + std::vector available_layers; +}; + +static bool HasExtension(const std::vector& extensions, const char* name) { + return std::any_of(extensions.begin(), extensions.end(), + [name](const auto& ext) { return strcmp(ext.extensionName, name) == 0; }); +} + +static bool HasExtension(const std::vector& extensions, const char* name) { + return std::any_of(extensions.begin(), extensions.end(), + [name](const char* ext) { return strcmp(ext, name) == 0; }); +} + +static bool HasLayer(const std::vector& layers, const char* name) { + return std::any_of(layers.begin(), layers.end(), + [name](const auto& layer) { return strcmp(layer.layerName, name) == 0; }); +} + +void VulkanGetSurfaceCapabilities(VkPhysicalDevice physical_device, VkSurfaceKHR surface, + SurfaceCapabilities* r) { + vkGetPhysicalDeviceSurfaceCapabilitiesKHR(physical_device, surface, &r->capabilities); + + uint32_t formats_count = 0; + vkGetPhysicalDeviceSurfaceFormatsKHR(physical_device, surface, &formats_count, nullptr); + + EXIT_NOT_IMPLEMENTED(formats_count == 0); + + r->formats = std::vector(formats_count); // @suppress("Ambiguous problem") + vkGetPhysicalDeviceSurfaceFormatsKHR(physical_device, surface, &formats_count, + r->formats.data()); + + uint32_t present_modes_count = 0; + vkGetPhysicalDeviceSurfacePresentModesKHR(physical_device, surface, &present_modes_count, + nullptr); + + EXIT_NOT_IMPLEMENTED(present_modes_count == 0); + + r->present_modes = + std::vector(present_modes_count); // @suppress("Ambiguous problem") + vkGetPhysicalDeviceSurfacePresentModesKHR(physical_device, surface, &present_modes_count, + r->present_modes.data()); + + r->format_srgb_bgra32 = false; + for (const auto& f: r->formats) { + if (f.format == VK_FORMAT_B8G8R8A8_SRGB && + f.colorSpace == VK_COLOR_SPACE_SRGB_NONLINEAR_KHR) { + r->format_srgb_bgra32 = true; + break; + } + if (f.format == VK_FORMAT_B8G8R8A8_UNORM && + f.colorSpace == VK_COLOR_SPACE_SRGB_NONLINEAR_KHR) { + r->format_unorm_bgra32 = true; + break; + } + } +} + +static bool CheckFormat(VkPhysicalDevice device, VkFormat format, bool tile, + VkFormatFeatureFlags features) { + VkFormatProperties format_props {}; + vkGetPhysicalDeviceFormatProperties(device, format, &format_props); + + const auto supported_features = + (tile ? format_props.optimalTilingFeatures : format_props.linearTilingFeatures); + return (supported_features & features) == features; +} + +struct QueueInfo { + uint32_t family = 0; + uint32_t index = 0; + bool graphics = false; + bool compute = false; + bool transfer = false; + bool present = false; +}; + +struct VulkanQueues { + uint32_t family_count = 0; + std::vector family_used; + std::vector available; + std::vector graphics; + std::vector compute; + std::vector transfer; + std::vector present; +}; + +static void VulkanDumpQueues(const VulkanQueues& qs) { + LOGF("Queues selected:\n" + "\t family_count = %u\n", + qs.family_count); + std::vector nums; + for (auto u: qs.family_used) { + nums.push_back(fmt::format("{}", u)); + } + LOGF("\t family_used = [%s]\n" + "\t graphics:\n", + Common::Concat(nums, ", ").c_str()); + for (const auto& q: qs.graphics) { + LOGF("\t\t family = %u, index = %u\n", q.family, q.index); + } + LOGF("\t compute:\n"); + for (const auto& q: qs.compute) { + LOGF("\t\t family = %u, index = %u\n", q.family, q.index); + } + LOGF("\t transfer:\n"); + for (const auto& q: qs.transfer) { + LOGF("\t\t family = %u, index = %u\n", q.family, q.index); + } + LOGF("\t present:\n"); + for (const auto& q: qs.present) { + LOGF("\t\t family = %u, index = %u\n", q.family, q.index); + } +} + +static VulkanQueues VulkanFindQueues(VkPhysicalDevice device, VkSurfaceKHR surface, + uint32_t graphics_num, uint32_t compute_num, + uint32_t transfer_num, uint32_t present_num) { + EXIT_IF(device == nullptr); + EXIT_IF(surface == nullptr); + + VulkanQueues qs; + + uint32_t queue_family_count = 0; + vkGetPhysicalDeviceQueueFamilyProperties(device, &queue_family_count, nullptr); + std::vector queue_families(queue_family_count); + vkGetPhysicalDeviceQueueFamilyProperties(device, &queue_family_count, queue_families.data()); + + qs.family_count = queue_family_count; + + uint32_t family = 0; + for (auto& f: queue_families) { + VkBool32 presentation_supported = VK_FALSE; + vkGetPhysicalDeviceSurfaceSupportKHR(device, family, surface, &presentation_supported); + + LOGF("\tqueue family: %s [count = %u], [present = %s]\n", + string_VkQueueFlags(f.queueFlags).c_str(), f.queueCount, + (presentation_supported == VK_TRUE ? "true" : "false")); + + for (uint32_t i = 0; i < f.queueCount; i++) { + QueueInfo info; + info.family = family; + info.index = i; + info.graphics = (f.queueFlags & VK_QUEUE_GRAPHICS_BIT) != 0; + info.compute = (f.queueFlags & VK_QUEUE_COMPUTE_BIT) != 0; + info.transfer = (f.queueFlags & VK_QUEUE_TRANSFER_BIT) != 0; + info.present = (presentation_supported == VK_TRUE); + + qs.available.push_back(info); + } + + qs.family_used.push_back(0); + + family++; + } + + auto select_queues = [&qs](uint32_t count, auto matches, auto& selected) { + for (uint32_t i = 0; i < count; i++) { + auto it = std::find_if(qs.available.begin(), qs.available.end(), matches); + if (it == qs.available.end()) { + continue; + } + + qs.family_used[it->family]++; + selected.push_back(*it); + qs.available.erase(it); + } + }; + + select_queues(graphics_num, [](const auto& q) { return q.graphics; }, qs.graphics); + select_queues(compute_num, [](const auto& q) { return q.compute; }, qs.compute); + select_queues(transfer_num, [](const auto& q) { return q.transfer; }, qs.transfer); + select_queues(present_num, [](const auto& q) { return q.present; }, qs.present); + + return qs; +} + +// NOLINTNEXTLINE(readability-function-cognitive-complexity) +static void VulkanFindPhysicalDevice(VkInstance instance, VkSurfaceKHR surface, + const std::vector& device_extensions, + SurfaceCapabilities* out_capabilities, + VkPhysicalDevice* out_device, VulkanQueues* out_queues) { + EXIT_IF(instance == nullptr); + EXIT_IF(surface == nullptr); + EXIT_IF(out_capabilities == nullptr); + EXIT_IF(out_device == nullptr); + EXIT_IF(out_queues == nullptr); + + uint32_t devices_count = 0; + vkEnumeratePhysicalDevices(instance, &devices_count, nullptr); + + EXIT_NOT_IMPLEMENTED(devices_count == 0); + + std::vector devices(devices_count); + vkEnumeratePhysicalDevices(instance, &devices_count, devices.data()); + + VkPhysicalDevice best_device = nullptr; + VulkanQueues best_queues; + + for (const auto& device: devices) { + bool skip_device = false; + + VkPhysicalDeviceProperties device_properties {}; + vkGetPhysicalDeviceProperties(device, &device_properties); + + LOGF("Vulkan device: %s\n", device_properties.deviceName); + if (device_properties.apiVersion < VULKAN_TARGET_API_VERSION) { + LOGF("Vulkan %u.%u is required, but device supports only %u.%u.%u\n", + VK_VERSION_MAJOR(VULKAN_TARGET_API_VERSION), + VK_VERSION_MINOR(VULKAN_TARGET_API_VERSION), + VK_VERSION_MAJOR(device_properties.apiVersion), + VK_VERSION_MINOR(device_properties.apiVersion), + VK_VERSION_PATCH(device_properties.apiVersion)); + continue; + } + + VkPhysicalDeviceFeatures2 device_features2 {}; + + VkPhysicalDeviceVulkan13Features features13 {}; + features13.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_VULKAN_1_3_FEATURES; + features13.pNext = nullptr; + + VkPhysicalDeviceColorWriteEnableFeaturesEXT color_write_ext {}; + color_write_ext.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_COLOR_WRITE_ENABLE_FEATURES_EXT; + color_write_ext.pNext = nullptr; + + VkPhysicalDeviceDepthClipControlFeaturesEXT depth_clip_control {}; + depth_clip_control.sType = + VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_DEPTH_CLIP_CONTROL_FEATURES_EXT; + depth_clip_control.pNext = &color_write_ext; + + VkPhysicalDeviceVulkan12Features features12 {}; + features12.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_VULKAN_1_2_FEATURES; + features12.pNext = &depth_clip_control; + features13.pNext = &features12; + + device_features2.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_FEATURES_2; + device_features2.pNext = &features13; + + vkGetPhysicalDeviceFeatures2(device, &device_features2); + + auto qs = VulkanFindQueues( + device, surface, GraphicContext::QUEUE_GFX_NUM, GraphicContext::QUEUE_COMPUTE_NUM, + GraphicContext::QUEUE_UTIL_NUM, GraphicContext::QUEUE_PRESENT_NUM); + + VulkanDumpQueues(qs); + + if (qs.graphics.size() != GraphicContext::QUEUE_GFX_NUM || + !(qs.compute.size() >= 1 && qs.compute.size() <= GraphicContext::QUEUE_COMPUTE_NUM) || + qs.transfer.size() != GraphicContext::QUEUE_UTIL_NUM || + qs.present.size() != GraphicContext::QUEUE_PRESENT_NUM) { + LOGF("Not enough queues\n"); + skip_device = true; + } + + if (color_write_ext.colorWriteEnable != VK_TRUE) { + LOGF("colorWriteEnable is not supported\n"); + skip_device = true; + } + + if (depth_clip_control.depthClipControl != VK_TRUE) { + LOGF("depthClipControl is not supported\n"); + skip_device = true; + } + + if (features12.samplerMirrorClampToEdge != VK_TRUE) { + LOGF("samplerMirrorClampToEdge is not supported\n"); + skip_device = true; + } + if (features13.robustImageAccess != VK_TRUE) { + LOGF("robustImageAccess is not supported\n"); + skip_device = true; + } + + if (device_features2.features.fragmentStoresAndAtomics != VK_TRUE) { + LOGF("fragmentStoresAndAtomics is not supported\n"); + skip_device = true; + } + + if (device_features2.features.samplerAnisotropy != VK_TRUE) { + LOGF("samplerAnisotropy is not supported\n"); + skip_device = true; + } + if (device_features2.features.robustBufferAccess != VK_TRUE) { + LOGF("robustBufferAccess is not supported\n"); + skip_device = true; + } + if (device_features2.features.depthBounds != VK_TRUE) { + LOGF("depthBounds is not supported\n"); + skip_device = true; + } + if (device_features2.features.shaderStorageImageWriteWithoutFormat != VK_TRUE) { + LOGF("shaderStorageImageWriteWithoutFormat is not supported\n"); + skip_device = true; + } + if (device_features2.features.shaderStorageImageReadWithoutFormat != VK_TRUE) { + LOGF("shaderStorageImageReadWithoutFormat is not supported\n"); + skip_device = true; + } + + if (device_features2.features.shaderImageGatherExtended != VK_TRUE) { + LOGF("shaderImageGatherExtended is not supported\n"); + skip_device = true; + } + + if (device_features2.features.independentBlend != VK_TRUE) { + LOGF("independentBlend is not supported\n"); + skip_device = true; + } + if (device_features2.features.tessellationShader != VK_TRUE) { + LOGF("tessellationShader is not supported\n"); + skip_device = true; + } + + if (!skip_device) { + uint32_t extensions_count = 0; + vkEnumerateDeviceExtensionProperties(device, nullptr, &extensions_count, nullptr); + + EXIT_NOT_IMPLEMENTED(extensions_count == 0); + + std::vector available_extensions(extensions_count); + vkEnumerateDeviceExtensionProperties(device, nullptr, &extensions_count, + available_extensions.data()); + + for (const char* ext: device_extensions) { + if (!HasExtension(available_extensions, ext)) { + skip_device = true; + break; + } + } + + if (skip_device) { + for (const auto& ext: available_extensions) { + LOGF("Vulkan available extension: %s, version = %u\n", ext.extensionName, + ext.specVersion); + } + } + } + + if (!skip_device) { + VulkanGetSurfaceCapabilities(device, surface, out_capabilities); + + if ((out_capabilities->capabilities.supportedUsageFlags & + VK_IMAGE_USAGE_TRANSFER_DST_BIT) == 0) { + LOGF("Surface cannot be destination of blit\n"); + skip_device = true; + } + } + + if (!skip_device && + !CheckFormat(device, VK_FORMAT_R8G8B8A8_SRGB, false, VK_FORMAT_FEATURE_BLIT_SRC_BIT)) { + LOGF("Format VK_FORMAT_R8G8B8A8_SRGB cannot be used as transfer source\n"); + skip_device = true; + } + + if (!skip_device && !CheckFormat(device, VK_FORMAT_D32_SFLOAT, true, + VK_FORMAT_FEATURE_DEPTH_STENCIL_ATTACHMENT_BIT)) { + LOGF("Format VK_FORMAT_D32_SFLOAT cannot be used as depth buffer\n"); + skip_device = true; + } + + if (!skip_device && !CheckFormat(device, VK_FORMAT_D32_SFLOAT_S8_UINT, true, + VK_FORMAT_FEATURE_DEPTH_STENCIL_ATTACHMENT_BIT)) { + LOGF("Format VK_FORMAT_D32_SFLOAT_S8_UINT cannot be used as depth buffer\n"); + skip_device = true; + } + + if (!skip_device && !CheckFormat(device, VK_FORMAT_D16_UNORM, true, + VK_FORMAT_FEATURE_DEPTH_STENCIL_ATTACHMENT_BIT)) { + LOGF("Format VK_FORMAT_D16_UNORM cannot be used as depth buffer\n"); + skip_device = true; + } + + if (!skip_device && !CheckFormat(device, VK_FORMAT_D24_UNORM_S8_UINT, true, + VK_FORMAT_FEATURE_DEPTH_STENCIL_ATTACHMENT_BIT)) { + LOGF("Format VK_FORMAT_D24_UNORM_S8_UINT cannot be used as depth buffer\n"); + skip_device = true; + } + + if (!skip_device && !CheckFormat(device, VK_FORMAT_BC3_SRGB_BLOCK, true, + VK_FORMAT_FEATURE_SAMPLED_IMAGE_BIT | + VK_FORMAT_FEATURE_TRANSFER_DST_BIT)) { + LOGF("Format VK_FORMAT_BC3_SRGB_BLOCK cannot be used as texture\n"); + skip_device = true; + } + + if (!skip_device && !CheckFormat(device, VK_FORMAT_R8G8B8A8_SRGB, true, + VK_FORMAT_FEATURE_SAMPLED_IMAGE_BIT | + VK_FORMAT_FEATURE_TRANSFER_DST_BIT)) { + LOGF("Format VK_FORMAT_R8G8B8A8_SRGB cannot be used as texture\n"); + skip_device = true; + } + + if (!skip_device && !CheckFormat(device, VK_FORMAT_R8_UNORM, true, + VK_FORMAT_FEATURE_SAMPLED_IMAGE_BIT | + VK_FORMAT_FEATURE_TRANSFER_DST_BIT)) { + LOGF("Format VK_FORMAT_R8_UNORM cannot be used as texture\n"); + skip_device = true; + } + + if (!skip_device && !CheckFormat(device, VK_FORMAT_R8G8_UNORM, true, + VK_FORMAT_FEATURE_SAMPLED_IMAGE_BIT | + VK_FORMAT_FEATURE_TRANSFER_DST_BIT)) { + LOGF("Format VK_FORMAT_R8G8_UNORM cannot be used as texture\n"); + skip_device = true; + } + + if (!skip_device && !CheckFormat(device, VK_FORMAT_R8G8B8A8_SRGB, true, + VK_FORMAT_FEATURE_STORAGE_IMAGE_BIT | + VK_FORMAT_FEATURE_TRANSFER_DST_BIT)) { + LOGF("Format VK_FORMAT_R8G8B8A8_SRGB cannot be used as texture\n"); + + if (!skip_device && !CheckFormat(device, VK_FORMAT_R8G8B8A8_UNORM, true, + VK_FORMAT_FEATURE_STORAGE_IMAGE_BIT | + VK_FORMAT_FEATURE_TRANSFER_DST_BIT)) { + LOGF("Format VK_FORMAT_R8G8B8A8_UNORM cannot be used as texture\n"); + skip_device = true; + } + } + + if (!skip_device && !CheckFormat(device, VK_FORMAT_B8G8R8A8_SRGB, true, + VK_FORMAT_FEATURE_STORAGE_IMAGE_BIT | + VK_FORMAT_FEATURE_TRANSFER_DST_BIT)) { + LOGF("Format VK_FORMAT_B8G8R8A8_SRGB cannot be used as texture\n"); + + if (!skip_device && !CheckFormat(device, VK_FORMAT_B8G8R8A8_UNORM, true, + VK_FORMAT_FEATURE_STORAGE_IMAGE_BIT | + VK_FORMAT_FEATURE_TRANSFER_DST_BIT)) { + LOGF("Format VK_FORMAT_B8G8R8A8_UNORM cannot be used as texture\n"); + skip_device = true; + } + } + + if (!skip_device && device_properties.limits.maxSamplerAnisotropy < 16.0f) { + LOGF("maxSamplerAnisotropy < 16.0f"); + skip_device = true; + } + + if (skip_device) { + continue; + } + + if (best_device == nullptr || + device_properties.deviceType == VK_PHYSICAL_DEVICE_TYPE_DISCRETE_GPU) { + best_device = device; + best_queues = qs; + } + } + + *out_device = best_device; + *out_queues = best_queues; +} + +static void VulkanInitSubgroupSizeControl(VkPhysicalDevice physical_device, GraphicContext* ctx) { + EXIT_IF(physical_device == nullptr); + EXIT_IF(ctx == nullptr); + + VkPhysicalDeviceSubgroupSizeControlProperties subgroup_size_control {}; + subgroup_size_control.sType = + VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_SUBGROUP_SIZE_CONTROL_PROPERTIES; + subgroup_size_control.pNext = nullptr; + + VkPhysicalDeviceVulkan11Properties properties11 {}; + properties11.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_VULKAN_1_1_PROPERTIES; + properties11.pNext = &subgroup_size_control; + + VkPhysicalDeviceProperties2 properties2 {}; + properties2.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_PROPERTIES_2; + properties2.pNext = &properties11; + + vkGetPhysicalDeviceProperties2(physical_device, &properties2); + + VkPhysicalDeviceVulkan13Features features13 {}; + features13.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_VULKAN_1_3_FEATURES; + features13.pNext = nullptr; + + VkPhysicalDeviceFeatures2 features2 {}; + features2.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_FEATURES_2; + features2.pNext = &features13; + + vkGetPhysicalDeviceFeatures2(physical_device, &features2); + + ctx->subgroup_size = properties11.subgroupSize; + ctx->min_subgroup_size = subgroup_size_control.minSubgroupSize; + ctx->max_subgroup_size = subgroup_size_control.maxSubgroupSize; + ctx->required_subgroup_size_stages = subgroup_size_control.requiredSubgroupSizeStages; + ctx->subgroup_size_control_enabled = features13.subgroupSizeControl == VK_TRUE && + subgroup_size_control.minSubgroupSize <= 64 && + subgroup_size_control.maxSubgroupSize >= 64; + + LOGF("Vulkan subgroup: default=%u min=%u max=%u stages=0x%08x size_control=%s\n", + ctx->subgroup_size, ctx->min_subgroup_size, ctx->max_subgroup_size, + ctx->required_subgroup_size_stages, ctx->subgroup_size_control_enabled ? "true" : "false"); +} + +static VkDevice VulkanCreateDevice(VkPhysicalDevice physical_device, VkSurfaceKHR surface, + const VulkanExtensions* r, const VulkanQueues& queues, + const std::vector& device_extensions, + const GraphicContext* ctx) { + EXIT_IF(physical_device == nullptr); + EXIT_IF(r == nullptr); + EXIT_IF(surface == nullptr); + EXIT_IF(ctx == nullptr); + + std::vector queue_create_info(queues.family_count); + std::vector> queue_priority(queues.family_count); + uint32_t queue_create_info_num = 0; + + for (uint32_t i = 0; i < queues.family_count; i++) { + if (queues.family_used[i] != 0) { + for (uint32_t pi = 0; pi < queues.family_used[i]; pi++) { + queue_priority[queue_create_info_num].push_back(1.0f); + } + + queue_create_info[queue_create_info_num].sType = + VK_STRUCTURE_TYPE_DEVICE_QUEUE_CREATE_INFO; + queue_create_info[queue_create_info_num].pNext = nullptr; + queue_create_info[queue_create_info_num].flags = 0; + queue_create_info[queue_create_info_num].queueFamilyIndex = i; + queue_create_info[queue_create_info_num].queueCount = queues.family_used[i]; + queue_create_info[queue_create_info_num].pQueuePriorities = + queue_priority[queue_create_info_num].data(); + + queue_create_info_num++; + } + } + + VkPhysicalDeviceColorWriteEnableFeaturesEXT color_write_ext {}; + color_write_ext.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_COLOR_WRITE_ENABLE_FEATURES_EXT; + color_write_ext.pNext = nullptr; + color_write_ext.colorWriteEnable = VK_TRUE; + + VkPhysicalDeviceDepthClipControlFeaturesEXT depth_clip_control {}; + depth_clip_control.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_DEPTH_CLIP_CONTROL_FEATURES_EXT; + depth_clip_control.pNext = &color_write_ext; + depth_clip_control.depthClipControl = VK_TRUE; + + VkPhysicalDeviceVulkan12Features features12 {}; + features12.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_VULKAN_1_2_FEATURES; + features12.pNext = &depth_clip_control; + features12.samplerMirrorClampToEdge = VK_TRUE; + + VkPhysicalDeviceSubgroupSizeControlFeatures subgroup_size_control {}; + subgroup_size_control.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_SUBGROUP_SIZE_CONTROL_FEATURES; + subgroup_size_control.pNext = &features12; + + VkPhysicalDeviceVulkan13Features supported_features13 {}; + supported_features13.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_VULKAN_1_3_FEATURES; + supported_features13.pNext = nullptr; + + const auto robustness2_ext_enabled = + HasExtension(device_extensions, VK_EXT_ROBUSTNESS_2_EXTENSION_NAME); + + VkPhysicalDeviceRobustness2FeaturesEXT supported_robustness2 {}; + supported_robustness2.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_ROBUSTNESS_2_FEATURES_EXT; + supported_robustness2.pNext = nullptr; + if (robustness2_ext_enabled) { + supported_features13.pNext = &supported_robustness2; + } + + VkPhysicalDeviceFeatures2 supported_features2 {}; + supported_features2.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_FEATURES_2; + supported_features2.pNext = &supported_features13; + vkGetPhysicalDeviceFeatures2(physical_device, &supported_features2); + + VkPhysicalDeviceFeatures device_features {}; + device_features.fragmentStoresAndAtomics = VK_TRUE; + device_features.samplerAnisotropy = VK_TRUE; + device_features.robustBufferAccess = VK_TRUE; + device_features.depthBounds = VK_TRUE; + device_features.shaderStorageImageWriteWithoutFormat = VK_TRUE; + device_features.shaderStorageImageReadWithoutFormat = VK_TRUE; + device_features.shaderImageGatherExtended = VK_TRUE; + device_features.independentBlend = VK_TRUE; + device_features.tessellationShader = VK_TRUE; + device_features.vertexPipelineStoresAndAtomics = + supported_features2.features.vertexPipelineStoresAndAtomics; + + if (ctx->subgroup_size_control_enabled && supported_features13.subgroupSizeControl == VK_TRUE) { + subgroup_size_control.subgroupSizeControl = VK_TRUE; + } + + const auto* base_feature_chain = (subgroup_size_control.subgroupSizeControl == VK_TRUE + ? static_cast(&subgroup_size_control) + : static_cast(&features12)); + + VkPhysicalDeviceRobustness2FeaturesEXT robustness2 {}; + robustness2.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_ROBUSTNESS_2_FEATURES_EXT; + robustness2.pNext = const_cast(base_feature_chain); + if (robustness2_ext_enabled) { + robustness2.robustBufferAccess2 = supported_robustness2.robustBufferAccess2; + robustness2.robustImageAccess2 = supported_robustness2.robustImageAccess2; + robustness2.nullDescriptor = supported_robustness2.nullDescriptor; + } + + VkPhysicalDeviceVulkan13Features features13 {}; + features13.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_VULKAN_1_3_FEATURES; + features13.pNext = + robustness2_ext_enabled ? &robustness2 : const_cast(base_feature_chain); + features13.robustImageAccess = supported_features13.robustImageAccess; + + LOGF("Vulkan robustness: robustImageAccess=%s robustImageAccess2=%s\n", + features13.robustImageAccess == VK_TRUE ? "true" : "false", + robustness2_ext_enabled && robustness2.robustImageAccess2 == VK_TRUE ? "true" : "false"); + + VkDeviceCreateInfo create_info {}; + create_info.sType = VK_STRUCTURE_TYPE_DEVICE_CREATE_INFO; + create_info.pNext = &features13; + create_info.flags = 0; + create_info.pQueueCreateInfos = queue_create_info.data(); + create_info.queueCreateInfoCount = queue_create_info_num; + create_info.enabledLayerCount = + (r->enable_validation_layers ? static_cast(r->required_layers.size()) : 0); + create_info.ppEnabledLayerNames = + (r->enable_validation_layers ? r->required_layers.data() : nullptr); + create_info.enabledExtensionCount = static_cast(device_extensions.size()); + create_info.ppEnabledExtensionNames = device_extensions.data(); + create_info.pEnabledFeatures = &device_features; + + VkDevice device = nullptr; + + auto result = vkCreateDevice(physical_device, &create_info, nullptr, &device); + if (result != VK_SUCCESS) { + LOGF("vkCreateDevice failed: %s\n", string_VkResult(result)); + return nullptr; + } + + return device; +} + +static void VulkanGetExtensions(SDL_Window* window, VulkanExtensions* r) { + EXIT_IF(window == nullptr); + EXIT_IF(r == nullptr); + + uint32_t required_extensions_count = 0; + uint32_t available_extensions_count = 0; + uint32_t available_layers_count = 0; + + auto sdl_result = SDL_Vulkan_GetInstanceExtensions(window, &required_extensions_count, nullptr); + + EXIT_NOT_IMPLEMENTED(sdl_result == SDL_FALSE); + EXIT_NOT_IMPLEMENTED(required_extensions_count == 0); + + r->required_extensions = + std::vector(required_extensions_count); // @suppress("Ambiguous problem") + std::memset(r->required_extensions.data(), 0, + sizeof(const char*) * r->required_extensions.size()); + + sdl_result = SDL_Vulkan_GetInstanceExtensions(window, &required_extensions_count, + r->required_extensions.data()); + + EXIT_NOT_IMPLEMENTED(sdl_result == SDL_FALSE); + EXIT_NOT_IMPLEMENTED(required_extensions_count == 0); + EXIT_NOT_IMPLEMENTED(required_extensions_count != r->required_extensions.size()); + + vkEnumerateInstanceExtensionProperties(nullptr, &available_extensions_count, nullptr); + + r->available_extensions = std::vector( + available_extensions_count); // @suppress("Ambiguous problem") + std::memset(r->available_extensions.data(), 0, + sizeof(VkExtensionProperties) * r->available_extensions.size()); + + vkEnumerateInstanceExtensionProperties(nullptr, &available_extensions_count, + r->available_extensions.data()); + + EXIT_NOT_IMPLEMENTED(available_extensions_count != r->available_extensions.size()); + + r->enable_validation_layers = Config::VulkanValidationEnabled(); + + if (HasExtension(r->available_extensions, VK_EXT_DEBUG_UTILS_EXTENSION_NAME)) { + r->required_extensions.push_back(VK_EXT_DEBUG_UTILS_EXTENSION_NAME); + } else { + r->enable_validation_layers = false; + } + + for (const char* ext: r->required_extensions) { + LOGF("Vulkan required extension: %s\n", ext); + } + + for (const auto& ext: r->available_extensions) { + LOGF("Vulkan available extension: %s, version = %u\n", ext.extensionName, ext.specVersion); + } + + vkEnumerateInstanceLayerProperties(&available_layers_count, nullptr); + + r->available_layers = + std::vector(available_layers_count); // @suppress("Ambiguous problem") + std::memset(r->available_layers.data(), 0, + sizeof(VkLayerProperties) * r->available_layers.size()); + vkEnumerateInstanceLayerProperties(&available_layers_count, r->available_layers.data()); + + EXIT_NOT_IMPLEMENTED(available_layers_count != r->available_layers.size()); + + for (const auto& l: r->available_layers) { + LOGF("Vulkan available layer: %s, specVersion = %u, implVersion = %u, %s\n", l.layerName, + l.specVersion, l.implementationVersion, l.description); + } + + r->required_layers = {"VK_LAYER_KHRONOS_validation"}; + + if (r->enable_validation_layers) { + for (const char* l: r->required_layers) { + if (!HasLayer(r->available_layers, l)) { + LOGF("no validation layer: %s\n", l); + r->enable_validation_layers = false; + break; + } + } + } + + if (r->enable_validation_layers) { + vkEnumerateInstanceExtensionProperties("VK_LAYER_KHRONOS_validation", + &available_extensions_count, nullptr); + + std::vector available_extensions(available_extensions_count); + + vkEnumerateInstanceExtensionProperties("VK_LAYER_KHRONOS_validation", + &available_extensions_count, + available_extensions.data()); + + for (const auto& ext: available_extensions) { + LOGF("VK_LAYER_KHRONOS_validation available extension: %s, version = %u\n", + ext.extensionName, ext.specVersion); + } + + if (HasExtension(available_extensions, VK_EXT_VALIDATION_FEATURES_EXTENSION_NAME)) { + r->required_extensions.push_back(VK_EXT_VALIDATION_FEATURES_EXTENSION_NAME); + } else { + r->enable_validation_layers = false; + } + } +} + +static VKAPI_ATTR VkBool32 VKAPI_CALL VulkanDebugMessengerCallback( + VkDebugUtilsMessageSeverityFlagBitsEXT message_severity, + VkDebugUtilsMessageTypeFlagsEXT message_types, + const VkDebugUtilsMessengerCallbackDataEXT* callback_data, void* /*user_data*/) { + EXIT_IF(callback_data == nullptr); + EXIT_IF(callback_data->pMessage == nullptr); + + const char* severity_str = nullptr; + fmt::text_style severity_style = Log::Color::Default; + bool skip = false; + bool error = false; + bool debug_printf = false; + switch (message_severity) { + case VK_DEBUG_UTILS_MESSAGE_SEVERITY_VERBOSE_BIT_EXT: + severity_str = "V"; + severity_style = Log::Color::BrightWhite; + skip = true; + break; + case VK_DEBUG_UTILS_MESSAGE_SEVERITY_INFO_BIT_EXT: + if ((message_types & VK_DEBUG_UTILS_MESSAGE_TYPE_VALIDATION_BIT_EXT) != 0 && + Config::SpirvDebugPrintfEnabled() && callback_data->pMessageIdName != nullptr && + strcmp(callback_data->pMessageIdName, "UNASSIGNED-DEBUG-PRINTF") == 0) { + debug_printf = true; + severity_style = Log::Color::BrightYellow; + skip = true; + } else { + severity_str = "I"; + severity_style = Log::Color::Default; + skip = true; + } + break; + case VK_DEBUG_UTILS_MESSAGE_SEVERITY_WARNING_BIT_EXT: + severity_str = "W"; + severity_style = Log::Color::Red; + break; + case VK_DEBUG_UTILS_MESSAGE_SEVERITY_ERROR_BIT_EXT: + severity_str = "E"; + severity_style = Log::Color::BrightRed; + error = true; + break; + default: severity_str = "?"; + } + + if (error) { + EXIT_COLOR(severity_style, "[Vulkan][%s][%u]: %s\n", severity_str, + static_cast(message_types), callback_data->pMessage); + } + + if (!skip) { + LOGF_COLOR(severity_style, "[Vulkan][%s][%u]: %s\n", severity_str, + static_cast(message_types), callback_data->pMessage); + } + + if (debug_printf) { + auto strs = Common::Split(std::string(callback_data->pMessage), '|'); + if (!strs.empty()) { + LOGF_COLOR(severity_style, "%s\n", strs[strs.size() - 1].c_str()); + } + } + + return VK_FALSE; +} + +static VKAPI_ATTR VkResult VKAPI_CALL VulkanCreateDebugUtilsMessengerEXT( + VkInstance instance, const VkDebugUtilsMessengerCreateInfoEXT* create_info, + const VkAllocationCallbacks* allocator, VkDebugUtilsMessengerEXT* messenger) { + EXIT_IF(instance == nullptr); + + if (auto func = reinterpret_cast( + vkGetInstanceProcAddr(instance, "vkCreateDebugUtilsMessengerEXT")); + func != nullptr) { + return func(instance, create_info, allocator, messenger); + } + return VK_ERROR_EXTENSION_NOT_PRESENT; +} + +static void VulkanCreateQueues(GraphicContext* ctx, const VulkanQueues& queues) { + EXIT_IF(ctx == nullptr); + EXIT_IF(ctx->device == nullptr); + EXIT_IF(queues.graphics.size() != 1); + EXIT_IF(queues.transfer.size() != 1); + EXIT_IF(queues.present.size() != 1); + EXIT_IF(!(queues.compute.size() >= 1 && + queues.compute.size() <= GraphicContext::QUEUE_COMPUTE_NUM)); + + auto get_queue = [ctx](int id, const QueueInfo& info, bool with_mutex = false) { + ctx->queues[id].family = info.family; + ctx->queues[id].index = info.index; + EXIT_IF(ctx->queues[id].vk_queue != nullptr); + vkGetDeviceQueue(ctx->device, ctx->queues[id].family, ctx->queues[id].index, + &ctx->queues[id].vk_queue); + EXIT_NOT_IMPLEMENTED(ctx->queues[id].vk_queue == nullptr); + if (with_mutex) { + ctx->queues[id].mutex = new Common::Mutex; + } + }; + + get_queue(GraphicContext::QUEUE_GFX, queues.graphics[0], true); + ctx->queues[GraphicContext::QUEUE_UTIL].family = ctx->queues[GraphicContext::QUEUE_GFX].family; + ctx->queues[GraphicContext::QUEUE_UTIL].index = ctx->queues[GraphicContext::QUEUE_GFX].index; + ctx->queues[GraphicContext::QUEUE_UTIL].vk_queue = + ctx->queues[GraphicContext::QUEUE_GFX].vk_queue; + ctx->queues[GraphicContext::QUEUE_UTIL].mutex = ctx->queues[GraphicContext::QUEUE_GFX].mutex; + LOGF("Vulkan queue: using graphics queue for utility submissions to preserve resource " + "ordering\n"); + get_queue(GraphicContext::QUEUE_PRESENT, queues.present[0], true); + + for (int id = 0; id < GraphicContext::QUEUE_COMPUTE_NUM; id++) { + bool with_mutex = (GraphicContext::QUEUE_COMPUTE_NUM == queues.compute.size()); + get_queue(GraphicContext::QUEUE_COMPUTE_START + id, + queues.compute[id % queues.compute.size()], with_mutex); + } + + for (int id = 0; id < GraphicContext::QUEUES_NUM; id++) { + auto& queue = ctx->queues[id]; + if (queue.vk_queue == nullptr) { + continue; + } + + if (queue.mutex == nullptr) { + queue.mutex = new Common::Mutex; + } + + for (int other_id = 0; other_id < id; other_id++) { + auto& other = ctx->queues[other_id]; + if (other.vk_queue != queue.vk_queue) { + continue; + } + + if (other.mutex == nullptr && queue.mutex != nullptr) { + other.mutex = queue.mutex; + } else { + queue.mutex = other.mutex; + } + LOGF("Vulkan queue: sharing mutex for queue ids %d and %d\n", other_id, id); + break; + } + } +} + +static void VulkanCheckInstanceVersion() { + uint32_t version = VK_API_VERSION_1_0; + + auto enumerate_instance_version = reinterpret_cast( + vkGetInstanceProcAddr(nullptr, "vkEnumerateInstanceVersion")); + if (enumerate_instance_version != nullptr) { + auto result = enumerate_instance_version(&version); + if (result != VK_SUCCESS) { + EXIT("Could not query Vulkan loader version: %s\n", string_VkResult(result)); + } + } + + LOGF("Vulkan loader version: %u.%u.%u\n", VK_VERSION_MAJOR(version), VK_VERSION_MINOR(version), + VK_VERSION_PATCH(version)); + if (version < VULKAN_TARGET_API_VERSION) { + EXIT("Vulkan %u.%u is required, but loader supports only %u.%u.%u\n", + VK_VERSION_MAJOR(VULKAN_TARGET_API_VERSION), + VK_VERSION_MINOR(VULKAN_TARGET_API_VERSION), VK_VERSION_MAJOR(version), + VK_VERSION_MINOR(version), VK_VERSION_PATCH(version)); + } +} + +void VulkanCreate(WindowContext* ctx) { + EXIT_IF(ctx->window == nullptr); + EXIT_IF(ctx->graphic_ctx.instance != nullptr); + EXIT_IF(ctx->graphic_ctx.physical_device != nullptr); + EXIT_IF(ctx->graphic_ctx.device != nullptr); + EXIT_IF(ctx->surface_capabilities != nullptr); + + VulkanExtensions r; + VulkanGetExtensions(ctx->window, &r); + VulkanCheckInstanceVersion(); + + VkApplicationInfo app_info {}; + app_info.sType = VK_STRUCTURE_TYPE_APPLICATION_INFO; + app_info.pNext = nullptr; + app_info.pApplicationName = "Kyty"; + app_info.applicationVersion = 1; + app_info.pEngineName = "Kyty"; + app_info.engineVersion = 1; + app_info.apiVersion = VULKAN_TARGET_API_VERSION; // NOLINT + + VkValidationFeatureEnableEXT enabled_features[] = { +#ifdef KYTY_ENABLE_BEST_PRACTICES + VK_VALIDATION_FEATURE_ENABLE_BEST_PRACTICES_EXT, +#endif +#ifdef KYTY_ENABLE_DEBUG_PRINTF + VK_VALIDATION_FEATURE_ENABLE_DEBUG_PRINTF_EXT, +#endif + }; + + uint32_t enabled_features_count = + sizeof(enabled_features) / sizeof(VkValidationFeatureEnableEXT); + +#ifdef KYTY_ENABLE_DEBUG_PRINTF + if (!Config::SpirvDebugPrintfEnabled()) { + enabled_features_count--; + } +#endif + + VkValidationFeaturesEXT validation_features {}; + validation_features.sType = VK_STRUCTURE_TYPE_VALIDATION_FEATURES_EXT; + validation_features.pNext = nullptr; + validation_features.enabledValidationFeatureCount = enabled_features_count; + validation_features.pEnabledValidationFeatures = enabled_features; + validation_features.disabledValidationFeatureCount = 0; + validation_features.pDisabledValidationFeatures = nullptr; + + VkDebugUtilsMessengerCreateInfoEXT dbg_create_info {}; + dbg_create_info.sType = VK_STRUCTURE_TYPE_DEBUG_UTILS_MESSENGER_CREATE_INFO_EXT; + dbg_create_info.pNext = &validation_features; + dbg_create_info.flags = 0; + dbg_create_info.messageSeverity = + static_cast(VK_DEBUG_UTILS_MESSAGE_SEVERITY_VERBOSE_BIT_EXT) | + static_cast(VK_DEBUG_UTILS_MESSAGE_SEVERITY_INFO_BIT_EXT) | + static_cast(VK_DEBUG_UTILS_MESSAGE_SEVERITY_WARNING_BIT_EXT) | + static_cast(VK_DEBUG_UTILS_MESSAGE_SEVERITY_ERROR_BIT_EXT); + dbg_create_info.messageType = + static_cast(VK_DEBUG_UTILS_MESSAGE_TYPE_GENERAL_BIT_EXT) | + static_cast(VK_DEBUG_UTILS_MESSAGE_TYPE_VALIDATION_BIT_EXT) | + static_cast(VK_DEBUG_UTILS_MESSAGE_TYPE_PERFORMANCE_BIT_EXT); + dbg_create_info.pfnUserCallback = VulkanDebugMessengerCallback; + dbg_create_info.pUserData = nullptr; + + VkInstanceCreateInfo inst_info {}; + inst_info.sType = VK_STRUCTURE_TYPE_INSTANCE_CREATE_INFO; + inst_info.pNext = (r.enable_validation_layers ? &dbg_create_info : nullptr); + inst_info.flags = 0; + inst_info.pApplicationInfo = &app_info; + inst_info.enabledExtensionCount = static_cast(r.required_extensions.size()); + inst_info.ppEnabledExtensionNames = r.required_extensions.data(); + inst_info.enabledLayerCount = + (r.enable_validation_layers ? static_cast(r.required_layers.size()) : 0); + inst_info.ppEnabledLayerNames = + (r.enable_validation_layers ? r.required_layers.data() : nullptr); + + const VkResult result = vkCreateInstance(&inst_info, nullptr, &ctx->graphic_ctx.instance); + switch (result) { + case VK_SUCCESS: break; + case VK_ERROR_INCOMPATIBLE_DRIVER: EXIT("Unable to find a compatible Vulkan Driver"); + default: EXIT("Could not create a Vulkan instance (for unknown reasons)"); + } + + if (r.enable_validation_layers) { + dbg_create_info.pNext = nullptr; + if (VulkanCreateDebugUtilsMessengerEXT(ctx->graphic_ctx.instance, &dbg_create_info, nullptr, + &ctx->graphic_ctx.debug_messenger) != VK_SUCCESS) { + EXIT("Could not create debug messenger"); + } + } + + if (SDL_Vulkan_CreateSurface(ctx->window, ctx->graphic_ctx.instance, &ctx->surface) == + SDL_FALSE) { + EXIT("Could not create a Vulkan surface"); + } + + std::vector device_extensions = { + VK_KHR_SWAPCHAIN_EXTENSION_NAME, VK_EXT_DEPTH_CLIP_ENABLE_EXTENSION_NAME, + VK_EXT_DEPTH_CLIP_CONTROL_EXTENSION_NAME, VK_EXT_COLOR_WRITE_ENABLE_EXTENSION_NAME, + "VK_KHR_maintenance1"}; + +#ifdef KYTY_ENABLE_DEBUG_PRINTF + if (Config::SpirvDebugPrintfEnabled()) { + device_extensions.push_back(VK_KHR_SHADER_NON_SEMANTIC_INFO_EXTENSION_NAME); + } +#endif + + ctx->surface_capabilities = new SurfaceCapabilities {}; + + VulkanQueues queues; + + VulkanFindPhysicalDevice(ctx->graphic_ctx.instance, ctx->surface, device_extensions, + ctx->surface_capabilities, &ctx->graphic_ctx.physical_device, &queues); + + if (ctx->graphic_ctx.physical_device == nullptr) { + EXIT("Could not find suitable device"); + } + + vkGetPhysicalDeviceProperties(ctx->graphic_ctx.physical_device, + &ctx->graphic_ctx.physical_device_properties); + const auto& device_properties = ctx->graphic_ctx.GetPhysicalDeviceProperties(); + + LOGF("Select device: %s\n", device_properties.deviceName); + + { + uint32_t extensions_count = 0; + vkEnumerateDeviceExtensionProperties(ctx->graphic_ctx.physical_device, nullptr, + &extensions_count, nullptr); + + std::vector available_extensions(extensions_count); + vkEnumerateDeviceExtensionProperties(ctx->graphic_ctx.physical_device, nullptr, + &extensions_count, available_extensions.data()); + + if (HasExtension(available_extensions, VK_EXT_MEMORY_BUDGET_EXTENSION_NAME)) { + device_extensions.push_back(VK_EXT_MEMORY_BUDGET_EXTENSION_NAME); + ctx->graphic_ctx.memory_budget_ext_enabled = true; + } + if (HasExtension(available_extensions, VK_EXT_ROBUSTNESS_2_EXTENSION_NAME)) { + device_extensions.push_back(VK_EXT_ROBUSTNESS_2_EXTENSION_NAME); + } + } + + memcpy(ctx->device_name, device_properties.deviceName, sizeof(ctx->device_name)); + std::snprintf(ctx->processor_name, sizeof(ctx->processor_name), "%s", + Common::GetSystemInfo().ProcessorName.c_str()); + + VulkanInitSubgroupSizeControl(ctx->graphic_ctx.physical_device, &ctx->graphic_ctx); + + ctx->graphic_ctx.device = VulkanCreateDevice(ctx->graphic_ctx.physical_device, ctx->surface, &r, + queues, device_extensions, &ctx->graphic_ctx); + if (ctx->graphic_ctx.device == nullptr) { + EXIT("Could not create device"); + } + + if (!VulkanCreateAllocator(&ctx->graphic_ctx)) { + EXIT("Could not create Vulkan memory allocator"); + } + + VulkanCreateQueues(&ctx->graphic_ctx, queues); + + ctx->swapchain = VulkanCreateSwapchain(&ctx->graphic_ctx, 2); + RenderDocSetActiveWindow(ctx->graphic_ctx.instance, ctx->window); +} + +} // namespace Libs::Graphics diff --git a/src/graphics/presentation/window/window.cpp b/src/graphics/presentation/window/window.cpp new file mode 100644 index 000000000..ae5285990 --- /dev/null +++ b/src/graphics/presentation/window/window.cpp @@ -0,0 +1,1155 @@ +#include "graphics/presentation/window.h" + +#include "SDL.h" +#include "SDL_error.h" +#include "SDL_events.h" +#include "SDL_gamecontroller.h" +#include "SDL_hints.h" +#include "SDL_joystick.h" +#include "SDL_keyboard.h" +#include "SDL_keycode.h" +#include "SDL_mouse.h" +#include "SDL_pixels.h" +#include "SDL_rwops.h" +#include "SDL_stdinc.h" +#include "SDL_surface.h" +#include "SDL_thread.h" +#include "SDL_touch.h" +#include "SDL_video.h" +#include "SDL_vulkan.h" +#include "common/assert.h" +#include "common/common.h" +#include "common/emulatorConfig.h" +#include "common/file.h" +#include "common/logging/log.h" +#include "common/profiler.h" +#include "common/stringUtils.h" +#include "common/subsystems.h" +#include "common/systemInfo.h" +#include "common/threads.h" +#include "common/timer.h" +#include "graphics/host_gpu/graphicContext.h" +#include "graphics/host_gpu/renderer/render.h" +#include "graphics/host_gpu/utils.h" +#include "graphics/host_gpu/vma.h" +#include "graphics/presentation/renderDoc.h" +#include "graphics/presentation/videoOut.h" +#include "graphics/presentation/window/windowInternal.h" +#include "libs/controller.h" +#include "loader/systemContent.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define STB_IMAGE_IMPLEMENTATION +#define STBI_NO_SIMD +#include "stb_image.h" + +#include + +// IWYU pragma: no_include + +#define KYTY_ENABLE_DEBUG_PRINTF +#define KYTY_DBG_INPUT + +namespace Libs::Graphics { + +constexpr float FPS_UPDATE_TIME = 1.0f; +constexpr int KEYBOARD_CONTROLLER_ID = -1000; + +struct EventKeyboard { + bool down; + bool up; + bool pressed; + bool released; + bool repeat; + int scan_code; + int key_code; + uint16_t mod; + double timestamp_seconds; +}; + +static uint32_t KeyboardKeyToPadButton(int key_code) { + switch (key_code) { + case SDLK_w: return Controller::PAD_BUTTON_UP; + case SDLK_a: return Controller::PAD_BUTTON_LEFT; + case SDLK_s: return Controller::PAD_BUTTON_DOWN; + case SDLK_d: return Controller::PAD_BUTTON_RIGHT; + case SDLK_j: return Controller::PAD_BUTTON_CROSS; + case SDLK_i: return Controller::PAD_BUTTON_TRIANGLE; + case SDLK_k: return Controller::PAD_BUTTON_SQUARE; + case SDLK_l: return Controller::PAD_BUTTON_CIRCLE; + case SDLK_q: return Controller::PAD_BUTTON_L1; + case SDLK_e: return Controller::PAD_BUTTON_R1; + case SDLK_RETURN: + case SDLK_RETURN2: return Controller::PAD_BUTTON_OPTIONS; + case SDLK_BACKSPACE: + case SDLK_TAB: return Controller::PAD_BUTTON_TOUCH_PAD; + default: return 0; + } +} + +static uint32_t ControllerButtonToPadButton(int button) { + switch (button) { + case SDL_CONTROLLER_BUTTON_A: return Controller::PAD_BUTTON_CROSS; + case SDL_CONTROLLER_BUTTON_B: return Controller::PAD_BUTTON_CIRCLE; + case SDL_CONTROLLER_BUTTON_X: return Controller::PAD_BUTTON_SQUARE; + case SDL_CONTROLLER_BUTTON_Y: return Controller::PAD_BUTTON_TRIANGLE; + case SDL_CONTROLLER_BUTTON_BACK: return Controller::PAD_BUTTON_TOUCH_PAD; + case SDL_CONTROLLER_BUTTON_START: return Controller::PAD_BUTTON_OPTIONS; + case SDL_CONTROLLER_BUTTON_LEFTSTICK: return Controller::PAD_BUTTON_L3; + case SDL_CONTROLLER_BUTTON_RIGHTSTICK: return Controller::PAD_BUTTON_R3; + case SDL_CONTROLLER_BUTTON_LEFTSHOULDER: return Controller::PAD_BUTTON_L1; + case SDL_CONTROLLER_BUTTON_RIGHTSHOULDER: return Controller::PAD_BUTTON_R1; + case SDL_CONTROLLER_BUTTON_DPAD_UP: return Controller::PAD_BUTTON_UP; + case SDL_CONTROLLER_BUTTON_DPAD_DOWN: return Controller::PAD_BUTTON_DOWN; + case SDL_CONTROLLER_BUTTON_DPAD_LEFT: return Controller::PAD_BUTTON_LEFT; + case SDL_CONTROLLER_BUTTON_DPAD_RIGHT: return Controller::PAD_BUTTON_RIGHT; + default: return 0; + } +} + +static Controller::Axis ControllerAxisFromSdl(int axis_id) { + switch (axis_id) { + case SDL_CONTROLLER_AXIS_LEFTX: return Controller::Axis::LeftX; + case SDL_CONTROLLER_AXIS_LEFTY: return Controller::Axis::LeftY; + case SDL_CONTROLLER_AXIS_RIGHTX: return Controller::Axis::RightX; + case SDL_CONTROLLER_AXIS_RIGHTY: return Controller::Axis::RightY; + case SDL_CONTROLLER_AXIS_TRIGGERLEFT: return Controller::Axis::TriggerLeft; + case SDL_CONTROLLER_AXIS_TRIGGERRIGHT: return Controller::Axis::TriggerRight; + default: return Controller::Axis::AxisMax; + } +} + +static bool ControllerAxisIsTrigger(int axis_id) { + return axis_id == SDL_CONTROLLER_AXIS_TRIGGERLEFT || + axis_id == SDL_CONTROLLER_AXIS_TRIGGERRIGHT; +} + +static int ControllerAxisValueFromSdl(int axis_id, int axis_value) { + return ControllerAxisIsTrigger(axis_id) + ? Controller::controller_get_axis(0, SDL_JOYSTICK_AXIS_MAX, axis_value) + : Controller::controller_get_axis(SDL_JOYSTICK_AXIS_MIN, SDL_JOYSTICK_AXIS_MAX, + axis_value); +} + +struct EventMouse { + bool down; + bool up; + bool left; + bool middle; + bool right; + bool x1; + bool x2; + bool touch; + bool pressed; + bool released; + int num_of_clicks; + bool wheel; + int x; + int y; + bool motion; + int motion_x; + int motion_y; + double timestamp_seconds; +}; + +struct EventFinger { + bool down; + bool up; + bool motion; + int touch_id; + int finger_id; + float x; + float y; + float dx; + float dy; + float pressure; + double timestamp_seconds; +}; + +struct EventController { + int id; + int button; + int axis_id; + int axis_value; + bool down; + bool up; + bool added; + bool removed; + bool remapped; + bool axis; + bool pressed; + bool released; + double timestamp_seconds; +}; + +enum class DisplayOrientation { + Unknown, /* The display orientation can't be determined */ + Landscape, /* The display is in landscape mode, with the right side up, relative to portrait + mode */ + LandscapeFlipped, /* The display is in landscape mode, with the left side up, relative to + portrait mode */ + Portrait, /* The display is in portrait mode */ + PortraitFlipped, /* The display is in portrait mode, upside down */ + + DisplayEventOrientation = 0xF0 +}; + +struct EventDisplay { + DisplayOrientation orientation; +}; + +constexpr uint32_t KYTY_SDL_BUTTON_LMASK = SDL_BUTTON_LMASK; // NOLINT(hicpp-signed-bitwise) +constexpr uint32_t KYTY_SDL_BUTTON_MMASK = SDL_BUTTON_MMASK; // NOLINT(hicpp-signed-bitwise) +constexpr uint32_t KYTY_SDL_BUTTON_RMASK = SDL_BUTTON_RMASK; // NOLINT(hicpp-signed-bitwise) +constexpr uint32_t KYTY_SDL_BUTTON_X1MASK = SDL_BUTTON_X1MASK; // NOLINT(hicpp-signed-bitwise) +constexpr uint32_t KYTY_SDL_BUTTON_X2MASK = SDL_BUTTON_X2MASK; // NOLINT(hicpp-signed-bitwise) + +struct WindowGame { + void* private_data = nullptr; + void* event = nullptr; + + bool m_game_need_exit = {false}; + bool m_game_is_paused = {false}; + uint32_t m_screen_width = {0}; + uint32_t m_screen_height = {0}; + double m_current_time_seconds = {0.0}; + double m_previous_time_seconds = {0.0}; + int m_update_num = {0}; + int m_frame_num = {0}; + double m_update_time_seconds = {0.0}; + double m_current_fps = {0.0}; + int m_max_updates_per_frame = {4}; + double m_update_fixed_time = 1.0 / 60.0; + int m_fps_frames_num = {0}; + double m_fps_start_time = {0}; +}; + +struct WindowGamePrivate { + WindowGamePrivate() = default; + + Common::Mutex mutex; + int skip_frames = 0; + GraphicContext* ctx = nullptr; +}; + +WindowContext* g_window_ctx = nullptr; + +constexpr const char* KYTY_SDL_WINDOW_CAPTION = "Game"; +constexpr uint32_t KYTY_SDL_WINDOW_FLAGS = + (static_cast(SDL_WINDOW_HIDDEN) | static_cast(SDL_WINDOW_VULKAN)); +constexpr int KYTY_SDL_WINDOWPOS_CENTERED = SDL_WINDOWPOS_CENTERED; /*NOLINT(hicpp-signed-bitwise)*/ + +static void CalcFrameTime(WindowGame* game, double game_time_s) { + game->m_previous_time_seconds = game->m_current_time_seconds; + game->m_current_time_seconds = game_time_s; + + game->m_frame_num++; + game->m_fps_frames_num++; + + const auto fps_time = game->m_current_time_seconds - game->m_fps_start_time; + if (fps_time > FPS_UPDATE_TIME) { + game->m_current_fps = static_cast(game->m_fps_frames_num) / fps_time; + game->m_fps_frames_num = 0; + game->m_fps_start_time = game->m_current_time_seconds; + } +} + +static bool Init(WindowGame* /*game*/) { + return true; +} +static bool Update(WindowGame* /*game*/) { + return true; +} +static bool Render(WindowGame* /*game*/) { + return true; +} +static bool Close(WindowGame* /*game*/) { + return true; +} +static void SetPause(WindowGame* game, bool flag) { + LOGF("Pause: %s\n", flag ? "true" : "false"); + + game->m_game_is_paused = flag; +} + +static bool RenderAndUpdate(WindowGame* game) { + static double lag = 0.0; + + lag += game->m_current_time_seconds - game->m_previous_time_seconds; + + int num = 0; + + bool ok = true; + + while (lag >= game->m_update_fixed_time) { + if (num < game->m_max_updates_per_frame) { + ok = ok && Update(game); + + game->m_update_num++; + num++; + game->m_update_time_seconds = game->m_update_num * game->m_update_fixed_time; + } + + lag -= game->m_update_fixed_time; + } + + ok = ok && Render(game); + + return ok; +} + +bool GameInit(WindowGame* game, const Common::Timer& timer, void* data) { + EXIT_IF(game == nullptr); + EXIT_IF(data == nullptr); + EXIT_IF(game->private_data || game->event); + + auto* ctx = static_cast(data); + + EXIT_IF(ctx->screen_width == 0 || ctx->screen_height == 0); + + auto* pdata = new WindowGamePrivate; + pdata->ctx = ctx; + + game->private_data = pdata; + game->event = new SDL_Event; + + game->m_screen_width = ctx->screen_width; + game->m_screen_height = ctx->screen_height; + + CalcFrameTime(game, timer.GetTimeS()); + + return Init(game); +} + +bool GameClose(WindowGame* game) { + EXIT_IF(!game); + + EXIT_IF(!game->private_data || !game->event); + + delete (static_cast(game->private_data)); + delete (static_cast(game->event)); + + return Close(game); +} + +void GameShowWindow(WindowGame* game, const Common::Timer& timer) { + EXIT_IF(!game); + + auto* p = static_cast(game->private_data); + + EXIT_IF(!p); + + p->mutex.Lock(); + { + if (p->skip_frames > 0) { + p->skip_frames--; + LOGF("skip frame %d\n", p->skip_frames); + } else { + VideoOut::VideoOutBeginVblank(); + if (VideoOut::VideoOutFlipWindow(0)) { + CalcFrameTime(game, timer.GetTimeS()); + } + VideoOut::VideoOutEndVblank(); + } + } + p->mutex.Unlock(); +} + +void GameEventQuit(WindowGame* game) { + LOGF("Event: quit\n"); + + game->m_game_need_exit = true; +} + +void GameEventTerminate(WindowGame* game) { + LOGF("Event: terminate\n"); + + game->m_game_need_exit = true; +} + +void GameEventKeyboard(WindowGame* game, const EventKeyboard* key) { +#ifdef KYTY_DBG_INPUT + LOGF("Key: time = %.04f, %s%s, %s%s, %s, scan = %d, key = %d, mod = %04" PRIx16 "\n", + key->timestamp_seconds, (key->down ? "down" : ""), (key->up ? "up" : ""), + (key->pressed ? "pressed" : ""), (key->released ? "released" : ""), + (key->repeat ? "repeat" : ""), key->scan_code, key->key_code, key->mod); +#endif + +#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS || KYTY_PLATFORM == KYTY_PLATFORM_LINUX + if (key->down) { + switch (key->key_code) { + case SDLK_ESCAPE: game->m_game_need_exit = true; break; + case SDLK_SPACE: SetPause(game, !game->m_game_is_paused); break; + case SDLK_F1: + if (!key->repeat) { + RenderDocRequestCapture(); + } + break; + default: break; + } + } + + const auto button = KeyboardKeyToPadButton(key->key_code); + if (button != 0 && (key->down || key->up) && !key->repeat) { + static bool keyboard_connected = false; + if (!keyboard_connected) { + Controller::ControllerConnect(KEYBOARD_CONTROLLER_ID); + keyboard_connected = true; + } + Controller::ControllerButton(KEYBOARD_CONTROLLER_ID, button, key->down); + } +#endif +} + +void GameEventMouse([[maybe_unused]] WindowGame* game, [[maybe_unused]] const EventMouse* mb) { +#ifdef KYTY_DBG_INPUT + if (mb->wheel) { + LOGF("Mouse wheel: time = %.04f, %s[%d, %d]\n", mb->timestamp_seconds, + (mb->touch ? "touch, " : ""), mb->x, mb->y); + } else if (mb->motion) { + LOGF("Mouse motion: time = %.04f, %s%s%s%s%s%s, [%d, %d], (%d, %d)\n", + mb->timestamp_seconds, (mb->left ? "left" : ""), (mb->middle ? "middle" : ""), + (mb->right ? "right" : ""), (mb->x1 ? "x1" : ""), (mb->x2 ? "x2" : ""), + (mb->touch ? "_touch" : ""), mb->x, mb->y, mb->motion_x, mb->motion_y); + } else { + LOGF("Mouse click: time = %.04f, %d, %s%s%s%s%s%s, %s%s, %s%s, [%d, %d]\n", + mb->timestamp_seconds, mb->num_of_clicks, (mb->left ? "left" : ""), + (mb->middle ? "middle" : ""), (mb->right ? "right" : ""), (mb->x1 ? "x1" : ""), + (mb->x2 ? "x2" : ""), (mb->touch ? "_touch" : ""), (mb->down ? "down" : ""), + (mb->up ? "up" : ""), (mb->pressed ? "pressed" : ""), (mb->released ? "released" : ""), + mb->x, mb->y); + } +#endif +} + +void GameEventFinger([[maybe_unused]] WindowGame* game, [[maybe_unused]] const EventFinger* f) { +#ifdef KYTY_DBG_INPUT + if (f->motion) { + LOGF("Finger motion: time = %.04f, %d, %d, (x,y) = [%f, %f], (dx,dy) = [%f, %f], pressure " + "= %f\n", + f->timestamp_seconds, f->touch_id, f->finger_id, f->x, f->y, f->dx, f->dy, + f->pressure); + } else { + LOGF("Finger press: time = %.04f, %d, %d, %s%s, (x,y) = [%f, %f], (dx,dy) = [%f, %f], " + "pressure = %f\n", + f->timestamp_seconds, f->touch_id, f->finger_id, (f->down ? "down" : ""), + (f->up ? "up" : ""), f->x, f->y, f->dx, f->dy, f->pressure); + } +#endif +} + +void GameEventController([[maybe_unused]] WindowGame* game, + [[maybe_unused]] const EventController* f) { + EXIT_NOT_IMPLEMENTED(f->remapped); + +#ifdef KYTY_DBG_INPUT + if (f->added || f->removed) { + LOGF("Controller %s: %d, time = %.04f\n", (f->added ? "added" : "removed"), f->id, + f->timestamp_seconds); + } else if (f->axis) { + LOGF("Controller axis: %d, axis = %d, value = %d, time = %.04f\n", f->id, f->axis_id, + f->axis_value, f->timestamp_seconds); + } else { + LOGF("Controller button: " + "%d, %s%s, %s%s, button = %d, time = %.04f\n", + f->id, (f->down ? "down" : ""), (f->up ? "up" : ""), (f->pressed ? "pressed" : ""), + (f->released ? "released" : ""), f->button, f->timestamp_seconds); + } +#endif + + if (f->added) { + auto* pad = SDL_GameControllerOpen(f->id); + EXIT_NOT_IMPLEMENTED(pad == nullptr); + int id = SDL_JoystickInstanceID(SDL_GameControllerGetJoystick(pad)); + Controller::ControllerConnect(id); + } + + if (f->removed) { + Controller::ControllerDisconnect(f->id); + SDL_GameControllerClose(SDL_GameControllerFromInstanceID(f->id)); + } + + if (f->down || f->up) { + const auto button = ControllerButtonToPadButton(f->button); + if (button != 0) { + Controller::ControllerButton(f->id, button, f->down); + } + } + + if (f->axis) { + const auto axis = ControllerAxisFromSdl(f->axis_id); + if (axis != Controller::Axis::AxisMax) { + Controller::ControllerAxis(f->id, axis, + ControllerAxisValueFromSdl(f->axis_id, f->axis_value)); + } + } +} + +void GameEventDisplay([[maybe_unused]] WindowGame* game, [[maybe_unused]] const EventDisplay* d) { + auto* p = static_cast(game->private_data); + auto* ctx = static_cast(p->ctx); + + p->mutex.Lock(); + game->m_screen_width = ctx->screen_width; + game->m_screen_height = ctx->screen_height; + p->mutex.Unlock(); +} + +void GameEventLowMemory(WindowGame* /*game*/) { + LOGF("Event: low_memory\n"); +} + +void GameEventWillEnterBackground(WindowGame* game) { + LOGF("Event: will_enter_background\n"); + + SetPause(game, true); +} + +void GameEventDidEnterBackground(WindowGame* /*game*/) { + LOGF("Event: did_enter_background\n"); +} + +void GameEventWillEnterForeground(WindowGame* /*game*/) { + LOGF("Event: will_enter_foreground\n"); +} + +void GameEventDidEnterForeground(WindowGame* game) { + LOGF("Event: did_enter_foreground\n"); + + SetPause(game, false); +} + +void GameEventResize(WindowGame* game, uint32_t new_width, uint32_t new_height) { + EXIT_IF(new_width == 0 || new_height == 0); + EXIT_IF(!game); + + auto* p = static_cast(game->private_data); + EXIT_IF(p == nullptr); + + auto* ctx = static_cast(p->ctx); + EXIT_IF(ctx == nullptr); + + p->mutex.Lock(); + { + p->skip_frames++; + ctx->screen_width = new_width; + ctx->screen_height = new_height; + + game->m_screen_width = ctx->screen_width; + game->m_screen_height = ctx->screen_height; + } + p->mutex.Unlock(); +} + +static void ProcessWindowEvent(WindowGame* game, SDL_WindowEvent window) { + switch (window.event) { + case SDL_WINDOWEVENT_SHOWN: LOGF("Window %" PRIu32 " shown\n", window.windowID); break; + + case SDL_WINDOWEVENT_HIDDEN: LOGF("Window %" PRIu32 " hidden\n", window.windowID); break; + + case SDL_WINDOWEVENT_EXPOSED: LOGF("Window %" PRIu32 " exposed\n", window.windowID); break; + + case SDL_WINDOWEVENT_MOVED: + LOGF("Window %" PRIu32 " moved to %" PRId32 ",%" PRId32 "\n", window.windowID, + window.data1, window.data2); + break; + + case SDL_WINDOWEVENT_RESIZED: + LOGF("Window %" PRIu32 " resized to %" PRId32 "x%" PRId32 "\n", window.windowID, + window.data1, window.data2); + + LOGF("m: %d\n", static_cast(SDL_ThreadID())); + GameEventResize(game, window.data1, window.data2); + + break; + + case SDL_WINDOWEVENT_SIZE_CHANGED: + LOGF("Window %" PRIu32 " size changed to %" PRId32 "x%" PRId32 "\n", window.windowID, + window.data1, window.data2); + + LOGF("m: %d\n", static_cast(SDL_ThreadID())); + GameEventResize(game, window.data1, window.data2); + + break; + + case SDL_WINDOWEVENT_MINIMIZED: + LOGF("Window %" PRIu32 " minimized\n", window.windowID); + break; + case SDL_WINDOWEVENT_MAXIMIZED: + LOGF("Window %" PRIu32 " maximized\n", window.windowID); + break; + case SDL_WINDOWEVENT_RESTORED: + LOGF("Window %" PRIu32 " restored\n", window.windowID); + break; + case SDL_WINDOWEVENT_ENTER: + LOGF("Mouse entered window %" PRIu32 "\n", window.windowID); + break; + case SDL_WINDOWEVENT_LEAVE: LOGF("Mouse left window %" PRIu32 "\n", window.windowID); break; + case SDL_WINDOWEVENT_FOCUS_GAINED: + LOGF("Window %" PRIu32 " gained keyboard focus\n", window.windowID); + break; + case SDL_WINDOWEVENT_FOCUS_LOST: + LOGF("Window %" PRIu32 " lost keyboard focus\n", window.windowID); + break; + case SDL_WINDOWEVENT_CLOSE: LOGF("Window %" PRIu32 " closed\n", window.windowID); break; + default: + LOGF("Window %" PRIu32 " got unknown event %" PRIu8 "\n", window.windowID, + window.event); + break; + } +} + +static void ProcessDisplayEvent(WindowGame* game, SDL_DisplayEvent display) { + bool sdl = false; + + switch (display.event) { + case SDL_DISPLAYEVENT_ORIENTATION: sdl = true; [[fallthrough]]; + case static_cast(DisplayOrientation::DisplayEventOrientation): { + LOGF("Display %" PRIu32 "[%s] changed orientation to %d - ", display.display, + sdl ? "SDL" : "Kyty", static_cast(display.data1)); + + EventDisplay d {}; + d.orientation = DisplayOrientation::Unknown; + + switch (display.data1) { + case SDL_ORIENTATION_UNKNOWN: LOGF("UNKNOWN\n"); break; + case SDL_ORIENTATION_LANDSCAPE: + LOGF("LANDSCAPE\n"); + d.orientation = DisplayOrientation::Landscape; + break; + case SDL_ORIENTATION_LANDSCAPE_FLIPPED: + LOGF("LANDSCAPE_FLIPPED\n"); + d.orientation = DisplayOrientation::LandscapeFlipped; + break; + case SDL_ORIENTATION_PORTRAIT: + LOGF("PORTRAIT\n"); + d.orientation = DisplayOrientation::Portrait; + break; + case SDL_ORIENTATION_PORTRAIT_FLIPPED: + d.orientation = DisplayOrientation::PortraitFlipped; + LOGF("PORTRAIT_FLIPPED\n"); + break; + default: LOGF("???\n"); + } + + if (!sdl) { + GameEventDisplay(game, &d); + } + + break; + } + default: + LOGF("Display %" PRIu32 " got unknown event 0x%" PRIx8 "\n", display.display, + display.event); + break; + } +} + +int GamePollEvent(WindowGame* game) { + EXIT_IF(!game); + + auto* event = static_cast(game->event); + + EXIT_IF(!event); + + return SDL_PollEvent(event); +} + +int GameWaitEvent(WindowGame* game) { + EXIT_IF(!game); + + auto* event = static_cast(game->event); + + EXIT_IF(!event); + + return SDL_WaitEvent(event); +} + +void GameProcessEvent(WindowGame* game, double time_s) { + EXIT_IF(!game); + + auto* event = static_cast(game->event); + + EXIT_IF(!event); + + EXIT_IF(SDL_GetEventState(SDL_DISPLAYEVENT) != SDL_ENABLE); + + switch (event->type) { + case SDL_QUIT: GameEventQuit(game); break; + + case SDL_APP_TERMINATING: GameEventTerminate(game); break; + + case SDL_APP_LOWMEMORY: GameEventLowMemory(game); break; + + case SDL_APP_WILLENTERBACKGROUND: GameEventWillEnterBackground(game); break; + + case SDL_APP_DIDENTERBACKGROUND: GameEventDidEnterBackground(game); break; + + case SDL_APP_WILLENTERFOREGROUND: GameEventWillEnterForeground(game); break; + + case SDL_APP_DIDENTERFOREGROUND: GameEventDidEnterForeground(game); break; + + case SDL_KEYDOWN: + case SDL_KEYUP: { + EventKeyboard key {}; + + key.down = (event->type == SDL_KEYDOWN); + key.up = (event->type == SDL_KEYUP); + key.pressed = (event->key.state == SDL_PRESSED); + key.released = (event->key.state == SDL_RELEASED); + key.repeat = (event->key.repeat != 0u); + key.scan_code = event->key.keysym.scancode; + key.key_code = event->key.keysym.sym; + key.mod = event->key.keysym.mod; + key.timestamp_seconds = time_s; + + GameEventKeyboard(game, &key); + + break; + } + + case SDL_WINDOWEVENT: ProcessWindowEvent(game, event->window); break; + + case SDL_DISPLAYEVENT: ProcessDisplayEvent(game, event->display); break; + + case SDL_MOUSEBUTTONDOWN: + case SDL_MOUSEBUTTONUP: { + EventMouse mb {}; + + mb.down = (event->button.type == SDL_MOUSEBUTTONDOWN); + mb.up = (event->button.type == SDL_MOUSEBUTTONUP); + mb.left = (event->button.button == SDL_BUTTON_LEFT); + mb.middle = (event->button.button == SDL_BUTTON_MIDDLE); + mb.right = (event->button.button == SDL_BUTTON_RIGHT); + mb.x1 = (event->button.button == SDL_BUTTON_X1); + mb.x2 = (event->button.button == SDL_BUTTON_X2); + mb.touch = (event->button.which == SDL_TOUCH_MOUSEID); + mb.pressed = (event->button.state == SDL_PRESSED); + mb.released = (event->button.state == SDL_RELEASED); + mb.num_of_clicks = event->button.clicks; + mb.wheel = false; + mb.x = event->button.x; + mb.y = event->button.y; + mb.motion = false; + mb.motion_x = 0; + mb.motion_y = 0; + mb.timestamp_seconds = time_s; + + GameEventMouse(game, &mb); + + break; + } + + case SDL_MOUSEWHEEL: { + EventMouse mb {}; + + mb.down = false; + mb.up = false; + mb.left = false; + mb.middle = false; + mb.right = false; + mb.x1 = false; + mb.x2 = false; + mb.touch = (event->wheel.which == SDL_TOUCH_MOUSEID); + mb.pressed = false; + mb.released = false; + mb.num_of_clicks = 0; + mb.wheel = true; + mb.x = event->wheel.x; + mb.y = event->wheel.y; + mb.motion = false; + mb.motion_x = 0; + mb.motion_y = 0; + mb.timestamp_seconds = time_s; + + GameEventMouse(game, &mb); + + break; + } + + case SDL_MOUSEMOTION: { + EventMouse mb {}; + + mb.down = false; + mb.up = false; + mb.left = ((event->motion.state & KYTY_SDL_BUTTON_LMASK) != 0u); + mb.middle = ((event->motion.state & KYTY_SDL_BUTTON_MMASK) != 0u); + mb.right = ((event->motion.state & KYTY_SDL_BUTTON_RMASK) != 0u); + mb.x1 = ((event->motion.state & KYTY_SDL_BUTTON_X1MASK) != 0u); + mb.x2 = ((event->motion.state & KYTY_SDL_BUTTON_X2MASK) != 0u); + mb.touch = (event->motion.which == SDL_TOUCH_MOUSEID); + mb.pressed = false; + mb.released = false; + mb.num_of_clicks = 0; + mb.wheel = false; + mb.x = event->motion.x; + mb.y = event->motion.y; + mb.motion = true; + mb.motion_x = event->motion.xrel; + mb.motion_y = event->motion.yrel; + mb.timestamp_seconds = time_s; + + GameEventMouse(game, &mb); + + break; + } + + case SDL_FINGERMOTION: + case SDL_FINGERDOWN: + case SDL_FINGERUP: { + EventFinger f {}; + + f.down = (event->tfinger.type == SDL_FINGERDOWN); + f.up = (event->tfinger.type == SDL_FINGERUP); + f.motion = (event->tfinger.type == SDL_FINGERMOTION); + f.finger_id = static_cast(event->tfinger.fingerId); + f.touch_id = static_cast(event->tfinger.touchId); + f.x = event->tfinger.x; + f.y = event->tfinger.y; + f.dx = event->tfinger.dx; + f.dy = event->tfinger.dy; + f.pressure = event->tfinger.pressure; + f.timestamp_seconds = time_s; + + GameEventFinger(game, &f); + + break; + } + + case SDL_CONTROLLERAXISMOTION: { + EventController c {}; + + c.id = event->caxis.which; + c.button = SDL_CONTROLLER_BUTTON_INVALID; + c.axis_id = event->caxis.axis; + c.axis_value = event->caxis.value; + c.down = false; + c.up = false; + c.added = false; + c.removed = false; + c.remapped = false; + c.axis = true; + c.pressed = false; + c.released = false; + c.timestamp_seconds = time_s; + + GameEventController(game, &c); + + break; + } + + case SDL_CONTROLLERBUTTONDOWN: + case SDL_CONTROLLERBUTTONUP: { + EventController c {}; + + c.id = event->cbutton.which; + c.button = event->cbutton.button; + c.axis_id = SDL_CONTROLLER_AXIS_INVALID; + c.axis_value = 0; + c.down = (event->cbutton.type == SDL_CONTROLLERBUTTONDOWN); + c.up = (event->cbutton.type == SDL_CONTROLLERBUTTONUP); + c.added = false; + c.removed = false; + c.remapped = false; + c.axis = false; + c.pressed = (event->cbutton.state == SDL_PRESSED); + c.released = (event->cbutton.state == SDL_RELEASED); + c.timestamp_seconds = time_s; + + GameEventController(game, &c); + + break; + } + + case SDL_CONTROLLERDEVICEADDED: + case SDL_CONTROLLERDEVICEREMOVED: + case SDL_CONTROLLERDEVICEREMAPPED: { + EventController c {}; + + c.id = event->cdevice.which; + c.button = SDL_CONTROLLER_BUTTON_INVALID; + c.axis_id = SDL_CONTROLLER_AXIS_INVALID; + c.axis_value = 0; + c.down = false; + c.up = false; + c.added = (event->cdevice.type == SDL_CONTROLLERDEVICEADDED); + c.removed = (event->cdevice.type == SDL_CONTROLLERDEVICEREMOVED); + c.remapped = (event->cdevice.type == SDL_CONTROLLERDEVICEREMAPPED); + c.axis = false; + c.pressed = false; + c.released = false; + c.timestamp_seconds = time_s; + + GameEventController(game, &c); + + break; + } + } +} + +void GameMainLoop(WindowGame* game, void* data) { + bool need_exit = false; + + Common::Timer timer; + timer.Start(); + + if (!GameInit(game, timer, data)) { + need_exit = true; + } + + for (;;) { + if (need_exit) { + break; + } + + if (GamePollEvent(game) != 0) { + GameProcessEvent(game, timer.GetTimeS()); + continue; + } + + if (game->m_game_is_paused) { + if (!timer.IsPaused()) { + timer.Pause(); + } + + GameWaitEvent(game); + + GameProcessEvent(game, timer.GetTimeS()); + need_exit = game->m_game_need_exit; + continue; + } + + need_exit = game->m_game_need_exit; + + if (game->m_game_is_paused) { + if (!timer.IsPaused()) { + timer.Pause(); + } + } else { + if (timer.IsPaused()) { + timer.Resume(); + } + + if (!need_exit) { + need_exit = !RenderAndUpdate(game); + } + + if (!need_exit) { + GameShowWindow(game, timer); + } + } + } + + GameClose(game); +} + +static void WindowCreate(WindowContext* ctx) { + EXIT_IF(ctx == nullptr); + EXIT_IF(ctx->window != nullptr); + EXIT_IF(ctx->graphic_ctx.screen_width == 0); + EXIT_IF(ctx->graphic_ctx.screen_height == 0); + + int width = static_cast(ctx->graphic_ctx.screen_width); + int height = static_cast(ctx->graphic_ctx.screen_height); + +#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS + SDL_SetHint(SDL_HINT_WINDOWS_DPI_SCALING, "0"); +#endif + + if (SDL_InitSubSystem(SDL_INIT_VIDEO | SDL_INIT_GAMECONTROLLER) < 0) { + EXIT("%s\n", SDL_GetError()); + } + + LOGF("WindowCreate(): width = %d, height = %d\n", width, height); + + ctx->window = + SDL_CreateWindow(KYTY_SDL_WINDOW_CAPTION, KYTY_SDL_WINDOWPOS_CENTERED, + KYTY_SDL_WINDOWPOS_CENTERED, width, height, KYTY_SDL_WINDOW_FLAGS); + + ctx->window_hidden = true; + + if (ctx->window == nullptr) { + EXIT("%s\n", SDL_GetError()); + } + + SDL_SetWindowResizable(ctx->window, SDL_FALSE); +} + +void WindowInit(uint32_t width, uint32_t height) { + EXIT_NOT_IMPLEMENTED(!Common::Thread::IsMainThread()); + EXIT_IF(g_window_ctx != nullptr); + + g_window_ctx = new WindowContext; + + g_window_ctx->graphic_ctx.screen_width = width; + g_window_ctx->graphic_ctx.screen_height = height; +} + +void WindowWaitForGraphicInitialized() { + EXIT_IF(g_window_ctx == nullptr); + + Common::LockGuard lock(g_window_ctx->mutex); + + while (!g_window_ctx->graphic_initialized) { + g_window_ctx->graphic_initialized_condvar.Wait(&g_window_ctx->mutex); + } +} + +void WindowRun() { + EXIT_IF(g_window_ctx == nullptr); + + KYTY_PROFILER_THREAD("Thread_Window"); + + WindowGame game; + + g_window_ctx->mutex.Lock(); + { + EXIT_IF(g_window_ctx->graphic_initialized); + + WindowCreate(g_window_ctx); + VulkanCreate(g_window_ctx); + + g_window_ctx->game = &game; + + g_window_ctx->graphic_initialized = true; + g_window_ctx->graphic_initialized_condvar.Signal(); + } + g_window_ctx->mutex.Unlock(); + + GraphicsRenderCreateContext(); + + GameMainLoop(&game, &g_window_ctx->graphic_ctx); + + // TODO: replace std::_Exit shutdown with full Vulkan teardown, then destroy + // the VMA allocator immediately before vkDestroyDevice. + Common::SubsystemsListSingleton::Instance()->ShutdownAll(); + std::_Exit(0); +} + +VkSurfaceCapabilitiesKHR* VulkanGetSurfaceCapabilities() { + EXIT_IF(g_window_ctx == nullptr); + + Common::LockGuard lock(g_window_ctx->mutex); + + return &g_window_ctx->surface_capabilities->capabilities; +} + +GraphicContext* WindowGetGraphicContext() { + EXIT_IF(g_window_ctx == nullptr); + + Common::LockGuard lock(g_window_ctx->mutex); + + return &g_window_ctx->graphic_ctx; +} + +static int WindowIconRead(void* user, char* data, int size) { + auto* src = static_cast(user); + uint32_t bytes_read = 0; + src->Read(data, static_cast(size), &bytes_read); + return static_cast(bytes_read); +} + +static void WindowIconSkip(void* user, int n) { + auto* src = static_cast(user); + const uint64_t position = src->Tell(); + + if (n >= 0) { + src->Seek(position + static_cast(n)); + } else { + const uint64_t distance = static_cast(-static_cast(n)); + EXIT_IF(distance > position); + src->Seek(position - distance); + } +} + +static int WindowIconEof(void* user) { + auto* src = static_cast(user); + return src->IsEOF() ? 1 : 0; +} + +struct WindowIcon { + SDL_Surface* surface = nullptr; + void* pixels = nullptr; + + ~WindowIcon() { + SDL_FreeSurface(surface); + stbi_image_free(pixels); + } +}; + +static void WindowLoadPngIcon(const std::string& path, WindowIcon* icon) { + Common::File f; + if (!f.Open(path, Common::File::Mode::Read)) { + EXIT("Can't open icon file %s\n", path.c_str()); + } + + int width = 0; + int height = 0; + + stbi_io_callbacks cb {}; + cb.read = WindowIconRead; + cb.skip = WindowIconSkip; + cb.eof = WindowIconEof; + + icon->pixels = stbi_load_from_callbacks(&cb, &f, &width, &height, nullptr, 4); + f.Close(); + + EXIT_IF(icon->pixels == nullptr); + + icon->surface = SDL_CreateRGBSurfaceWithFormatFrom(icon->pixels, width, height, 32, width * 4, + SDL_PIXELFORMAT_RGBA32); + EXIT_NOT_IMPLEMENTED(icon->surface == nullptr); +} + +void WindowUpdateIcon() { + EXIT_IF(g_window_ctx == nullptr); + + static WindowIcon icon; + static bool icon_loaded = false; + + if (!icon_loaded) { + std::string icon_path; + if (Loader::SystemContentGetIconPath(&icon_path)) { + WindowLoadPngIcon(icon_path, &icon); + } + icon_loaded = true; + } + + if (icon.surface != nullptr) { + SDL_SetWindowIcon(g_window_ctx->window, icon.surface); + } +} + +void WindowUpdateTitle() { + EXIT_IF(g_window_ctx == nullptr); + EXIT_IF(g_window_ctx->game == nullptr); + + static char title[128]; + static char title_id[12]; + static char app_ver[12]; + static bool has_title = Loader::SystemContentParamSfoGetString("TITLE", title, sizeof(title)); + static bool has_title_id = + Loader::SystemContentParamSfoGetString("TITLE_ID", title_id, sizeof(title_id)); + static bool has_app_ver = + Loader::SystemContentParamSfoGetString("APP_VER", app_ver, sizeof(app_ver)); + + auto fps = fmt::format("{}{}{}{}{}{}[{}] [{}], frame: {}, fps: {:f}", (has_title ? title : ""), + (has_title ? ", " : ""), (has_title_id ? title_id : ""), + (has_title_id ? ", " : ""), (has_app_ver ? app_ver : ""), + (has_app_ver ? " " : ""), g_window_ctx->device_name, + g_window_ctx->processor_name, g_window_ctx->game->m_frame_num, + g_window_ctx->game->m_current_fps); + + SDL_SetWindowTitle(g_window_ctx->window, fps.c_str()); +} + +} // namespace Libs::Graphics diff --git a/src/graphics/presentation/window/windowInternal.h b/src/graphics/presentation/window/windowInternal.h new file mode 100644 index 000000000..45f37b0ad --- /dev/null +++ b/src/graphics/presentation/window/windowInternal.h @@ -0,0 +1,53 @@ +#ifndef EMULATOR_SRC_GRAPHICS_PRESENTATION_WINDOW_WINDOWINTERNAL_H_ +#define EMULATOR_SRC_GRAPHICS_PRESENTATION_WINDOW_WINDOWINTERNAL_H_ + +#include "SDL_video.h" +#include "common/threads.h" +#include "graphics/host_gpu/graphicContext.h" + +#include +#include +#include + +namespace Libs::Graphics { + +struct WindowGame; + +struct SurfaceCapabilities { + VkSurfaceCapabilitiesKHR capabilities {}; + std::vector formats; + std::vector present_modes; + bool format_srgb_bgra32 = false; + bool format_unorm_bgra32 = false; +}; + +struct WindowContext { + GraphicContext graphic_ctx; + VulkanSwapchain* swapchain = nullptr; + SDL_Window* window = nullptr; + bool window_hidden = true; + VkSurfaceKHR surface = nullptr; + SurfaceCapabilities* surface_capabilities = nullptr; + WindowGame* game = nullptr; + + char device_name[VK_MAX_PHYSICAL_DEVICE_NAME_SIZE] = {0}; + char processor_name[64] = {0}; + + Common::Mutex mutex; + bool graphic_initialized = false; + Common::CondVar graphic_initialized_condvar; +}; + +extern WindowContext* g_window_ctx; + +void VulkanGetSurfaceCapabilities(VkPhysicalDevice physical_device, VkSurfaceKHR surface, + SurfaceCapabilities* capabilities); +VulkanSwapchain* VulkanCreateSwapchain(GraphicContext* ctx, uint32_t image_count); +void VulkanCreate(WindowContext* ctx); + +void WindowUpdateIcon(); +void WindowUpdateTitle(); + +} // namespace Libs::Graphics + +#endif // EMULATOR_SRC_GRAPHICS_PRESENTATION_WINDOW_WINDOWINTERNAL_H_ diff --git a/src/graphics/rt/hardware.cpp b/src/graphics/rt/hardware.cpp new file mode 100644 index 000000000..80b00a71a --- /dev/null +++ b/src/graphics/rt/hardware.cpp @@ -0,0 +1,98 @@ +#include "graphics/rt/hardware.h" + +#include "common/assert.h" +#include "common/logging/log.h" + +#include +#include + +namespace Libs::Graphics::Rt { +namespace { + +bool ExtensionAvailable(const std::vector& available, const char* name) { + return std::ranges::any_of(available, [name](const auto& extension) { + return std::strcmp(extension.extensionName, name) == 0; + }); +} + +void AppendExtensionIfMissing(std::vector& extensions, const char* name) { + if (std::ranges::none_of(extensions, [name](const char* extension) { + return std::strcmp(extension, name) == 0; + })) { + extensions.push_back(name); + } +} + +} // namespace + +void AppendHardwareRayTracingDeviceExtensions( + const std::vector& available_extensions, + std::vector* device_extensions, GraphicContext* ctx) { + EXIT_IF(device_extensions == nullptr || ctx == nullptr); + + const char* required[] = { + VK_KHR_BUFFER_DEVICE_ADDRESS_EXTENSION_NAME, + VK_KHR_DEFERRED_HOST_OPERATIONS_EXTENSION_NAME, + VK_KHR_ACCELERATION_STRUCTURE_EXTENSION_NAME, + VK_KHR_RAY_QUERY_EXTENSION_NAME, + }; + for (const auto* extension: required) { + if (!ExtensionAvailable(available_extensions, extension)) { + LOGF("Vulkan RT: extension %s is unavailable; hardware ray tracing disabled\n", + extension); + ctx->rt_extensions_enabled = false; + return; + } + } + for (const auto* extension: required) { + AppendExtensionIfMissing(*device_extensions, extension); + } + if (ExtensionAvailable(available_extensions, VK_KHR_SPIRV_1_4_EXTENSION_NAME)) { + AppendExtensionIfMissing(*device_extensions, VK_KHR_SPIRV_1_4_EXTENSION_NAME); + } + if (ExtensionAvailable(available_extensions, VK_KHR_SHADER_FLOAT_CONTROLS_EXTENSION_NAME)) { + AppendExtensionIfMissing(*device_extensions, VK_KHR_SHADER_FLOAT_CONTROLS_EXTENSION_NAME); + } + + ctx->rt_extensions_enabled = true; + LOGF("Vulkan RT: enabling hardware ray query extensions\n"); +} + +void LoadHardwareRayTracingFunctions(GraphicContext* ctx) { + EXIT_IF(ctx == nullptr || ctx->device == nullptr); + if (!ctx->rt_extensions_enabled) { + return; + } + + ctx->vkGetBufferDeviceAddressKHR = reinterpret_cast( + vkGetDeviceProcAddr(ctx->device, "vkGetBufferDeviceAddressKHR")); + if (ctx->vkGetBufferDeviceAddressKHR == nullptr) { + ctx->vkGetBufferDeviceAddressKHR = reinterpret_cast( + vkGetDeviceProcAddr(ctx->device, "vkGetBufferDeviceAddress")); + } + ctx->vkCreateAccelerationStructureKHR = reinterpret_cast( + vkGetDeviceProcAddr(ctx->device, "vkCreateAccelerationStructureKHR")); + ctx->vkDestroyAccelerationStructureKHR = + reinterpret_cast( + vkGetDeviceProcAddr(ctx->device, "vkDestroyAccelerationStructureKHR")); + ctx->vkGetAccelerationStructureBuildSizesKHR = + reinterpret_cast( + vkGetDeviceProcAddr(ctx->device, "vkGetAccelerationStructureBuildSizesKHR")); + ctx->vkCmdBuildAccelerationStructuresKHR = + reinterpret_cast( + vkGetDeviceProcAddr(ctx->device, "vkCmdBuildAccelerationStructuresKHR")); + ctx->vkGetAccelerationStructureDeviceAddressKHR = + reinterpret_cast( + vkGetDeviceProcAddr(ctx->device, "vkGetAccelerationStructureDeviceAddressKHR")); + + if (ctx->vkGetBufferDeviceAddressKHR == nullptr || + ctx->vkCreateAccelerationStructureKHR == nullptr || + ctx->vkDestroyAccelerationStructureKHR == nullptr || + ctx->vkGetAccelerationStructureBuildSizesKHR == nullptr || + ctx->vkCmdBuildAccelerationStructuresKHR == nullptr || + ctx->vkGetAccelerationStructureDeviceAddressKHR == nullptr) { + EXIT("Vulkan RT: failed to load required device functions\n"); + } +} + +} // namespace Libs::Graphics::Rt diff --git a/src/graphics/rt/hardware.h b/src/graphics/rt/hardware.h new file mode 100644 index 000000000..936615f52 --- /dev/null +++ b/src/graphics/rt/hardware.h @@ -0,0 +1,22 @@ +#ifndef EMULATOR_INCLUDE_EMULATOR_GRAPHICS_RT_HARDWARE_H_ +#define EMULATOR_INCLUDE_EMULATOR_GRAPHICS_RT_HARDWARE_H_ + +#include "graphics/host_gpu/graphicContext.h" + +#include + +namespace Libs::Graphics { + +namespace Rt { + +void AppendHardwareRayTracingDeviceExtensions( + const std::vector& available_extensions, + std::vector* device_extensions, GraphicContext* ctx); + +void LoadHardwareRayTracingFunctions(GraphicContext* ctx); + +} // namespace Rt + +} // namespace Libs::Graphics + +#endif /* EMULATOR_INCLUDE_EMULATOR_GRAPHICS_RT_HARDWARE_H_ */ diff --git a/src/graphics/shader/recompiler/BindingLayout.cpp b/src/graphics/shader/recompiler/BindingLayout.cpp new file mode 100644 index 000000000..4bc15ad49 --- /dev/null +++ b/src/graphics/shader/recompiler/BindingLayout.cpp @@ -0,0 +1,293 @@ +#include "graphics/shader/recompiler/BindingLayout.h" + +#include "graphics/shader/recompiler/ScalarProvenance.h" + +#include +#include +#include +#include + +namespace Libs::Graphics::ShaderRecompiler::IR { +namespace { + +constexpr uint32_t MaxPushConstantBytes = 128; + +constexpr std::array ImageBindingKinds = { + DescriptorBindingKind::Sampled2D, DescriptorBindingKind::Sampled2DArray, + DescriptorBindingKind::Sampled3D, DescriptorBindingKind::SampledUint2D, + DescriptorBindingKind::SampledUint2DArray, DescriptorBindingKind::SampledUint3D, + DescriptorBindingKind::Storage2D, DescriptorBindingKind::Storage2DArray, + DescriptorBindingKind::Storage3D, DescriptorBindingKind::StorageUint2D, + DescriptorBindingKind::StorageUint2DArray, DescriptorBindingKind::StorageUint3D, +}; + +bool ImageBinding(const ImageResource& image, DescriptorBindingKind& result) { + using Dimension = Decoder::ImageDimension; + using Kind = DescriptorBindingKind; + + switch (image.kind) { + case ResourceKind::Image: + switch (image.dimension) { + case Dimension::Dim2D: result = Kind::Sampled2D; return true; + case Dimension::Dim2DArray: result = Kind::Sampled2DArray; return true; + case Dimension::Dim3D: result = Kind::Sampled3D; return true; + default: return false; + } + case ResourceKind::ImageUint: + switch (image.dimension) { + case Dimension::Dim2D: result = Kind::SampledUint2D; return true; + case Dimension::Dim2DArray: result = Kind::SampledUint2DArray; return true; + case Dimension::Dim3D: result = Kind::SampledUint3D; return true; + default: return false; + } + case ResourceKind::StorageImage: + switch (image.dimension) { + case Dimension::Dim2D: result = Kind::Storage2D; return true; + case Dimension::Dim2DArray: result = Kind::Storage2DArray; return true; + case Dimension::Dim3D: result = Kind::Storage3D; return true; + default: return false; + } + case ResourceKind::StorageImageUint: + switch (image.dimension) { + case Dimension::Dim2D: result = Kind::StorageUint2D; return true; + case Dimension::Dim2DArray: result = Kind::StorageUint2DArray; return true; + case Dimension::Dim3D: result = Kind::StorageUint3D; return true; + default: return false; + } + default: return false; + } +} + +bool CollectValue(const ScalarProvenance& provenance, uint32_t id, std::vector* visited, + std::set* registers) { + if (id <= ScalarProvenance::Unknown) { + return true; + } + if (id >= provenance.values.size()) { + return false; + } + if ((*visited)[id] != 0) { + return true; + } + (*visited)[id] = 1; + const auto& value = provenance.values[id]; + if (value.op == ScalarValueOp::UserData) { + registers->insert(value.imm); + return true; + } + if (value.op == ScalarValueOp::Phi) { + for (const auto arg: value.phi_args) { + if (!CollectValue(provenance, arg, visited, registers)) { + return false; + } + } + return true; + } + for (uint32_t i = 0; i < ScalarValueArgCount(value.op); i++) { + if (!CollectValue(provenance, value.args[i], visited, registers)) { + return false; + } + } + return true; +} + +bool CollectSource(const Program& program, uint32_t source, bool allow_unknown, + std::vector* visited, std::set* registers) { + if (allow_unknown && source == ScalarProvenance::Unknown) { + return true; + } + if (source < 2u || source - 2u >= program.provenance.descriptors.size()) { + return false; + } + const auto& descriptor = program.provenance.descriptors[source - 2u]; + for (uint32_t i = 0; i < descriptor.dword_count; i++) { + if (!CollectValue(program.provenance, descriptor.dwords[i], visited, registers)) { + return false; + } + } + return true; +} + +bool CollectUserData(const Program& program, std::vector& result) { + std::set registers; + std::vector visited(program.provenance.values.size()); + for (const auto& buffer: program.info.buffers) { + if (!CollectSource(program, buffer.source, false, &visited, ®isters)) { + return false; + } + } + for (const auto& image: program.info.images) { + if (!CollectSource(program, image.source, false, &visited, ®isters)) { + return false; + } + } + for (const auto& sampler: program.info.samplers) { + if (!CollectSource(program, sampler.source, false, &visited, ®isters)) { + return false; + } + } + for (const auto& address: program.info.addresses) { + if (!CollectSource(program, address.source, true, &visited, ®isters)) { + return false; + } + } + for (const auto& read: program.srt.reads) { + if (!CollectValue(program.provenance, read.value, &visited, ®isters)) { + return false; + } + } + for (const auto read: program.srt.dynamic_reads) { + if (!CollectValue(program.provenance, read, &visited, ®isters)) { + return false; + } + } + for (const auto source: program.srt.dynamic_sources) { + if (!CollectSource(program, source, true, &visited, ®isters)) { + return false; + } + } + for (const auto& block: program.blocks) { + for (const auto& inst: block.instructions) { + if (inst.op == Opcode::SLoadDword && inst.memory.kind == ResourceKind::ScalarBuffer && + !CollectValue(program.provenance, inst.scalar_value, &visited, ®isters)) { + return false; + } + for (uint32_t i = 0; i < inst.src_count; i++) { + if (!CollectValue(program.provenance, inst.scalar_sources[i], &visited, + ®isters)) { + return false; + } + } + } + } + result.assign(registers.begin(), registers.end()); + return true; +} + +void AddBinding(BindingLayout* layout, DescriptorBindingKind kind, + std::vector resources = {}) { + layout->descriptors.push_back( + {kind, static_cast(layout->descriptors.size()), std::move(resources)}); +} + +bool UsesGds(const Program& program) { + for (const auto& block: program.blocks) { + for (const auto& inst: block.instructions) { + if (inst.memory.kind == ResourceKind::Gds) { + return true; + } + } + } + return false; +} + +} // namespace + +bool AllocateBindings(Program* program, const BindingLayoutOptions& options, std::string* error) { + if (program == nullptr || !program->shader_info_complete || program->binding_layout_complete) { + if (error != nullptr) { + *error = program == nullptr ? "invalid binding-layout program" + : !program->shader_info_complete ? "shader info is not ready" + : "binding layout already allocated"; + } + return false; + } + if (options.push_constant_offset > MaxPushConstantBytes) { + if (error != nullptr) { + *error = "push-constant offset exceeds the Vulkan minimum guarantee"; + } + return false; + } + if (options.push_constant_offset % 4u != 0) { + if (error != nullptr) { + *error = "push-constant offset is not dword aligned"; + } + return false; + } + + BindingLayout next; + next.descriptor_set = options.descriptor_set; + next.push_constant_offset = options.push_constant_offset; + if (!CollectUserData(*program, next.user_data_registers)) { + if (error != nullptr) { + *error = "shader plan contains an invalid scalar provenance reference"; + } + return false; + } + + if (!program->info.buffers.empty()) { + std::vector resources(program->info.buffers.size()); + for (uint32_t i = 0; i < resources.size(); i++) { + resources[i] = i; + } + AddBinding(&next, DescriptorBindingKind::Buffers, std::move(resources)); + } + + std::array, ImageBindingKinds.size()> image_groups; + for (uint32_t i = 0; i < program->info.images.size(); i++) { + DescriptorBindingKind kind; + if (!ImageBinding(program->info.images[i], kind)) { + if (error != nullptr) { + *error = "shader info contains an invalid image binding class"; + } + return false; + } + const auto group = std::find(ImageBindingKinds.begin(), ImageBindingKinds.end(), kind); + if (group == ImageBindingKinds.end()) { + if (error != nullptr) { + *error = "shader info contains an unmapped image binding class"; + } + return false; + } + image_groups[static_cast(group - ImageBindingKinds.begin())].push_back(i); + } + for (uint32_t i = 0; i < image_groups.size(); i++) { + if (!image_groups[i].empty()) { + AddBinding(&next, ImageBindingKinds[i], std::move(image_groups[i])); + } + } + + if (!program->info.samplers.empty()) { + std::vector resources(program->info.samplers.size()); + for (uint32_t i = 0; i < resources.size(); i++) { + resources[i] = i; + } + AddBinding(&next, DescriptorBindingKind::Samplers, std::move(resources)); + } + if (UsesGds(*program)) { + AddBinding(&next, DescriptorBindingKind::Gds); + } + if (!program->info.addresses.empty()) { + std::vector resources(program->info.addresses.size()); + for (uint32_t i = 0; i < resources.size(); i++) { + resources[i] = i; + } + AddBinding(&next, DescriptorBindingKind::AddressMemory, std::move(resources)); + } + if (!program->srt.reads.empty()) { + AddBinding(&next, DescriptorBindingKind::FlattenedSrt); + } + + const auto available_push_dwords = (MaxPushConstantBytes - options.push_constant_offset) / 4u; + const auto push_limit = std::min(options.max_push_dwords, available_push_dwords); + if (next.user_data_registers.size() <= push_limit) { + next.push_constant_size = static_cast(next.user_data_registers.size() * 4u); + } else { + AddBinding(&next, DescriptorBindingKind::UserData); + } + + program->bindings = std::move(next); + program->binding_layout_complete = true; + return true; +} + +const DescriptorBinding* FindBinding(const BindingLayout& layout, DescriptorBindingKind kind) { + for (const auto& binding: layout.descriptors) { + if (binding.kind == kind) { + return &binding; + } + } + return nullptr; +} + +} // namespace Libs::Graphics::ShaderRecompiler::IR diff --git a/src/graphics/shader/recompiler/BindingLayout.h b/src/graphics/shader/recompiler/BindingLayout.h new file mode 100644 index 000000000..9af42360e --- /dev/null +++ b/src/graphics/shader/recompiler/BindingLayout.h @@ -0,0 +1,20 @@ +#ifndef EMULATOR_INCLUDE_EMULATOR_GRAPHICS_SHADER_RECOMPILER_BINDINGLAYOUT_H_ +#define EMULATOR_INCLUDE_EMULATOR_GRAPHICS_SHADER_RECOMPILER_BINDINGLAYOUT_H_ + +#include "graphics/shader/recompiler/ShaderIR.h" + +namespace Libs::Graphics::ShaderRecompiler::IR { + +struct BindingLayoutOptions { + uint32_t descriptor_set = 0; + uint32_t push_constant_offset = 0; + uint32_t max_push_dwords = 32; +}; + +bool AllocateBindings(Program* program, const BindingLayoutOptions& options, std::string* error); + +const DescriptorBinding* FindBinding(const BindingLayout& layout, DescriptorBindingKind kind); + +} // namespace Libs::Graphics::ShaderRecompiler::IR + +#endif /* EMULATOR_INCLUDE_EMULATOR_GRAPHICS_SHADER_RECOMPILER_BINDINGLAYOUT_H_ */ diff --git a/src/graphics/shader/recompiler/BufferFormat.h b/src/graphics/shader/recompiler/BufferFormat.h new file mode 100644 index 000000000..5694d423c --- /dev/null +++ b/src/graphics/shader/recompiler/BufferFormat.h @@ -0,0 +1,256 @@ +#ifndef EMULATOR_INCLUDE_EMULATOR_GRAPHICS_SHADER_RECOMPILER_BUFFERFORMAT_H_ +#define EMULATOR_INCLUDE_EMULATOR_GRAPHICS_SHADER_RECOMPILER_BUFFERFORMAT_H_ + +#include "graphics/guest_gpu/gpu_defs.h" + +namespace Libs::Graphics::ShaderRecompiler::Format { + +enum class ComponentType { + Unknown, + Uint, + Sint, + Unorm, + Snorm, + Uscaled, + Sscaled, + Float, +}; + +struct BufferFormatInfo { + Prospero::BufferFormat format = Prospero::BufferFormat::kInvalid; + ComponentType type = ComponentType::Unknown; + uint32_t component_count = 0; + uint32_t byte_size = 0; + uint32_t component_bits[4] = {}; + uint32_t component_bit_offset[4] = {}; + bool packed_bitfield = false; +}; + +constexpr Prospero::BufferFormat DecodeTBufferFormat(uint32_t data_format, uint32_t number_format) { + return static_cast(((number_format & 0x7u) << 4u) | (data_format & 0xfu)); +} + +constexpr ComponentType GetFormatComponentType(Prospero::BufferFormat format) { + switch (format) { + case Prospero::BufferFormat::k8UInt: + case Prospero::BufferFormat::k16UInt: + case Prospero::BufferFormat::k8_8UInt: + case Prospero::BufferFormat::k32UInt: + case Prospero::BufferFormat::k16_16UInt: + case Prospero::BufferFormat::k11_11_10UInt: + case Prospero::BufferFormat::k10_11_11UInt: + case Prospero::BufferFormat::k2_10_10_10UInt: + case Prospero::BufferFormat::k10_10_10_2UInt: + case Prospero::BufferFormat::k8_8_8_8UInt: + case Prospero::BufferFormat::k32_32UInt: + case Prospero::BufferFormat::k16_16_16_16UInt: + case Prospero::BufferFormat::k32_32_32UInt: + case Prospero::BufferFormat::k32_32_32_32UInt: return ComponentType::Uint; + case Prospero::BufferFormat::k8SInt: + case Prospero::BufferFormat::k16SInt: + case Prospero::BufferFormat::k8_8SInt: + case Prospero::BufferFormat::k32SInt: + case Prospero::BufferFormat::k16_16SInt: + case Prospero::BufferFormat::k11_11_10SInt: + case Prospero::BufferFormat::k10_11_11SInt: + case Prospero::BufferFormat::k2_10_10_10SInt: + case Prospero::BufferFormat::k10_10_10_2SInt: + case Prospero::BufferFormat::k8_8_8_8SInt: + case Prospero::BufferFormat::k32_32SInt: + case Prospero::BufferFormat::k16_16_16_16SInt: + case Prospero::BufferFormat::k32_32_32SInt: + case Prospero::BufferFormat::k32_32_32_32SInt: return ComponentType::Sint; + case Prospero::BufferFormat::k8UNorm: + case Prospero::BufferFormat::k16UNorm: + case Prospero::BufferFormat::k8_8UNorm: + case Prospero::BufferFormat::k16_16UNorm: + case Prospero::BufferFormat::k11_11_10UNorm: + case Prospero::BufferFormat::k10_11_11UNorm: + case Prospero::BufferFormat::k2_10_10_10UNorm: + case Prospero::BufferFormat::k10_10_10_2UNorm: + case Prospero::BufferFormat::k8_8_8_8UNorm: + case Prospero::BufferFormat::k16_16_16_16UNorm: return ComponentType::Unorm; + case Prospero::BufferFormat::k8SNorm: + case Prospero::BufferFormat::k16SNorm: + case Prospero::BufferFormat::k8_8SNorm: + case Prospero::BufferFormat::k16_16SNorm: + case Prospero::BufferFormat::k11_11_10SNorm: + case Prospero::BufferFormat::k10_11_11SNorm: + case Prospero::BufferFormat::k2_10_10_10SNorm: + case Prospero::BufferFormat::k10_10_10_2SNorm: + case Prospero::BufferFormat::k8_8_8_8SNorm: + case Prospero::BufferFormat::k16_16_16_16SNorm: return ComponentType::Snorm; + case Prospero::BufferFormat::k8UScaled: + case Prospero::BufferFormat::k16UScaled: + case Prospero::BufferFormat::k8_8UScaled: + case Prospero::BufferFormat::k16_16UScaled: + case Prospero::BufferFormat::k11_11_10UScaled: + case Prospero::BufferFormat::k10_11_11UScaled: + case Prospero::BufferFormat::k2_10_10_10UScaled: + case Prospero::BufferFormat::k10_10_10_2UScaled: + case Prospero::BufferFormat::k8_8_8_8UScaled: + case Prospero::BufferFormat::k16_16_16_16UScaled: return ComponentType::Uscaled; + case Prospero::BufferFormat::k8SScaled: + case Prospero::BufferFormat::k16SScaled: + case Prospero::BufferFormat::k8_8SScaled: + case Prospero::BufferFormat::k16_16SScaled: + case Prospero::BufferFormat::k11_11_10SScaled: + case Prospero::BufferFormat::k10_11_11SScaled: + case Prospero::BufferFormat::k2_10_10_10SScaled: + case Prospero::BufferFormat::k10_10_10_2SScaled: + case Prospero::BufferFormat::k8_8_8_8SScaled: + case Prospero::BufferFormat::k16_16_16_16SScaled: return ComponentType::Sscaled; + case Prospero::BufferFormat::k16Float: + case Prospero::BufferFormat::k32Float: + case Prospero::BufferFormat::k16_16Float: + case Prospero::BufferFormat::k11_11_10Float: + case Prospero::BufferFormat::k10_11_11Float: + case Prospero::BufferFormat::k32_32Float: + case Prospero::BufferFormat::k16_16_16_16Float: + case Prospero::BufferFormat::k32_32_32Float: + case Prospero::BufferFormat::k32_32_32_32Float: return ComponentType::Float; + default: return ComponentType::Unknown; + } +} + +constexpr BufferFormatInfo GetFormatInfo(Prospero::BufferFormat format) { + const auto type = GetFormatComponentType(format); + if (type == ComponentType::Unknown) { + return {}; + } + + switch (format) { + case Prospero::BufferFormat::k8UNorm: + case Prospero::BufferFormat::k8SNorm: + case Prospero::BufferFormat::k8UScaled: + case Prospero::BufferFormat::k8SScaled: + case Prospero::BufferFormat::k8UInt: + case Prospero::BufferFormat::k8SInt: + return {format, type, 1, 1, {8, 0, 0, 0}, {0, 0, 0, 0}, false}; + case Prospero::BufferFormat::k16UNorm: + case Prospero::BufferFormat::k16SNorm: + case Prospero::BufferFormat::k16UScaled: + case Prospero::BufferFormat::k16SScaled: + case Prospero::BufferFormat::k16UInt: + case Prospero::BufferFormat::k16SInt: + case Prospero::BufferFormat::k16Float: + return {format, type, 1, 2, {16, 0, 0, 0}, {0, 0, 0, 0}, false}; + case Prospero::BufferFormat::k8_8UNorm: + case Prospero::BufferFormat::k8_8SNorm: + case Prospero::BufferFormat::k8_8UScaled: + case Prospero::BufferFormat::k8_8SScaled: + case Prospero::BufferFormat::k8_8UInt: + case Prospero::BufferFormat::k8_8SInt: + return {format, type, 2, 2, {8, 8, 0, 0}, {0, 8, 0, 0}, false}; + case Prospero::BufferFormat::k32UInt: + case Prospero::BufferFormat::k32SInt: + case Prospero::BufferFormat::k32Float: + return {format, type, 1, 4, {32, 0, 0, 0}, {0, 0, 0, 0}, false}; + case Prospero::BufferFormat::k16_16UNorm: + case Prospero::BufferFormat::k16_16SNorm: + case Prospero::BufferFormat::k16_16UScaled: + case Prospero::BufferFormat::k16_16SScaled: + case Prospero::BufferFormat::k16_16UInt: + case Prospero::BufferFormat::k16_16SInt: + case Prospero::BufferFormat::k16_16Float: + return {format, type, 2, 4, {16, 16, 0, 0}, {0, 16, 0, 0}, false}; + case Prospero::BufferFormat::k11_11_10UNorm: + case Prospero::BufferFormat::k11_11_10SNorm: + case Prospero::BufferFormat::k11_11_10UScaled: + case Prospero::BufferFormat::k11_11_10SScaled: + case Prospero::BufferFormat::k11_11_10UInt: + case Prospero::BufferFormat::k11_11_10SInt: + case Prospero::BufferFormat::k11_11_10Float: + return {format, type, 3, 4, {11, 11, 10, 0}, {0, 11, 22, 0}, true}; + case Prospero::BufferFormat::k10_11_11UNorm: + case Prospero::BufferFormat::k10_11_11SNorm: + case Prospero::BufferFormat::k10_11_11UScaled: + case Prospero::BufferFormat::k10_11_11SScaled: + case Prospero::BufferFormat::k10_11_11UInt: + case Prospero::BufferFormat::k10_11_11SInt: + case Prospero::BufferFormat::k10_11_11Float: + return {format, type, 3, 4, {10, 11, 11, 0}, {0, 10, 21, 0}, true}; + case Prospero::BufferFormat::k2_10_10_10UNorm: + case Prospero::BufferFormat::k2_10_10_10SNorm: + case Prospero::BufferFormat::k2_10_10_10UScaled: + case Prospero::BufferFormat::k2_10_10_10SScaled: + case Prospero::BufferFormat::k2_10_10_10UInt: + case Prospero::BufferFormat::k2_10_10_10SInt: + return {format, type, 4, 4, {2, 10, 10, 10}, {0, 2, 12, 22}, true}; + case Prospero::BufferFormat::k10_10_10_2UNorm: + case Prospero::BufferFormat::k10_10_10_2SNorm: + case Prospero::BufferFormat::k10_10_10_2UScaled: + case Prospero::BufferFormat::k10_10_10_2SScaled: + case Prospero::BufferFormat::k10_10_10_2UInt: + case Prospero::BufferFormat::k10_10_10_2SInt: + return {format, type, 4, 4, {10, 10, 10, 2}, {0, 10, 20, 30}, true}; + case Prospero::BufferFormat::k8_8_8_8UNorm: + case Prospero::BufferFormat::k8_8_8_8SNorm: + case Prospero::BufferFormat::k8_8_8_8UScaled: + case Prospero::BufferFormat::k8_8_8_8SScaled: + case Prospero::BufferFormat::k8_8_8_8UInt: + case Prospero::BufferFormat::k8_8_8_8SInt: + return {format, type, 4, 4, {8, 8, 8, 8}, {0, 8, 16, 24}, false}; + case Prospero::BufferFormat::k32_32UInt: + case Prospero::BufferFormat::k32_32SInt: + case Prospero::BufferFormat::k32_32Float: + return {format, type, 2, 8, {32, 32, 0, 0}, {0, 32, 0, 0}, false}; + case Prospero::BufferFormat::k16_16_16_16UNorm: + case Prospero::BufferFormat::k16_16_16_16SNorm: + case Prospero::BufferFormat::k16_16_16_16UScaled: + case Prospero::BufferFormat::k16_16_16_16SScaled: + case Prospero::BufferFormat::k16_16_16_16UInt: + case Prospero::BufferFormat::k16_16_16_16SInt: + case Prospero::BufferFormat::k16_16_16_16Float: + return {format, type, 4, 8, {16, 16, 16, 16}, {0, 16, 32, 48}, false}; + case Prospero::BufferFormat::k32_32_32UInt: + case Prospero::BufferFormat::k32_32_32SInt: + case Prospero::BufferFormat::k32_32_32Float: + return {format, type, 3, 12, {32, 32, 32, 0}, {0, 32, 64, 0}, false}; + case Prospero::BufferFormat::k32_32_32_32UInt: + case Prospero::BufferFormat::k32_32_32_32SInt: + case Prospero::BufferFormat::k32_32_32_32Float: + return {format, type, 4, 16, {32, 32, 32, 32}, {0, 32, 64, 96}, false}; + default: return {}; + } +} + +constexpr bool IsKnownFormat(Prospero::BufferFormat format) { + return GetFormatInfo(format).type != ComponentType::Unknown; +} + +constexpr uint32_t GetFormatComponentCount(Prospero::BufferFormat format) { + return GetFormatInfo(format).component_count; +} + +constexpr uint32_t GetFormatByteSize(Prospero::BufferFormat format) { + return GetFormatInfo(format).byte_size; +} + +constexpr uint32_t GetFormatComponentBits(Prospero::BufferFormat format, uint32_t component) { + const auto info = GetFormatInfo(format); + return component < info.component_count ? info.component_bits[component] : 0u; +} + +constexpr uint32_t GetFormatComponentBitOffset(Prospero::BufferFormat format, uint32_t component) { + const auto info = GetFormatInfo(format); + return component < info.component_count ? info.component_bit_offset[component] : 0u; +} + +constexpr uint32_t GetFormatComponentByteOffset(Prospero::BufferFormat format, uint32_t component) { + const auto info = GetFormatInfo(format); + if (component >= info.component_count) { + return 0; + } + return info.packed_bitfield ? 0u : info.component_bit_offset[component] / 8u; +} + +constexpr bool CanUseTypedBufferLoad(Prospero::BufferFormat format) { + const auto info = GetFormatInfo(format); + return info.type != ComponentType::Unknown && !info.packed_bitfield && + info.component_bits[0] == 32u; +} + +} // namespace Libs::Graphics::ShaderRecompiler::Format + +#endif /* EMULATOR_INCLUDE_EMULATOR_GRAPHICS_SHADER_RECOMPILER_BUFFERFORMAT_H_ */ diff --git a/src/graphics/shader/recompiler/ExecMask.cpp b/src/graphics/shader/recompiler/ExecMask.cpp new file mode 100644 index 000000000..7859fe7bb --- /dev/null +++ b/src/graphics/shader/recompiler/ExecMask.cpp @@ -0,0 +1,51 @@ +#include "graphics/shader/recompiler/ExecMask.h" + +namespace Libs::Graphics::ShaderRecompiler::Exec { + +Mask FullMask(uint32_t wave_size) { + return {0xffffffffu, wave_size == 64u ? 0xffffffffu : 0u}; +} + +Mask ApplyWaveSize(Mask mask, uint32_t wave_size) { + const auto full = FullMask(wave_size); + mask.low &= full.low; + mask.high &= full.high; + return mask; +} + +Mask ReadExec(const State& state) { + return ApplyWaveSize(state.exec, state.wave_size); +} + +void WriteExec(State& state, Mask mask) { + state.exec = ApplyWaveSize(mask, state.wave_size); +} + +void AndExec(State& state, Mask mask) { + WriteExec(state, {state.exec.low & mask.low, state.exec.high & mask.high}); +} + +void OrExec(State& state, Mask mask) { + WriteExec(state, {state.exec.low | mask.low, state.exec.high | mask.high}); +} + +void XorExec(State& state, Mask mask) { + WriteExec(state, {state.exec.low ^ mask.low, state.exec.high ^ mask.high}); +} + +void InvertExec(State& state) { + const auto full = FullMask(state.wave_size); + WriteExec(state, {~state.exec.low & full.low, ~state.exec.high & full.high}); +} + +bool IsExecZero(const State& state) { + const auto exec = ReadExec(state); + return exec.low == 0u && exec.high == 0u; +} + +bool IsVccZero(const State& state) { + const auto vcc = ApplyWaveSize(state.vcc, state.wave_size); + return vcc.low == 0u && vcc.high == 0u; +} + +} // namespace Libs::Graphics::ShaderRecompiler::Exec diff --git a/src/graphics/shader/recompiler/ExecMask.h b/src/graphics/shader/recompiler/ExecMask.h new file mode 100644 index 000000000..e30b42ce1 --- /dev/null +++ b/src/graphics/shader/recompiler/ExecMask.h @@ -0,0 +1,34 @@ +#ifndef EMULATOR_INCLUDE_EMULATOR_GRAPHICS_SHADER_RECOMPILER_EXECMASK_H_ +#define EMULATOR_INCLUDE_EMULATOR_GRAPHICS_SHADER_RECOMPILER_EXECMASK_H_ + +#include "common/common.h" + +namespace Libs::Graphics::ShaderRecompiler::Exec { + +struct Mask { + uint32_t low = 0; + uint32_t high = 0; +}; + +struct State { + uint32_t wave_size = 64; + Mask exec; + Mask vcc; + bool scc = false; +}; + +Mask FullMask(uint32_t wave_size); +Mask ApplyWaveSize(Mask mask, uint32_t wave_size); + +Mask ReadExec(const State& state); +void WriteExec(State& state, Mask mask); +void AndExec(State& state, Mask mask); +void OrExec(State& state, Mask mask); +void XorExec(State& state, Mask mask); +void InvertExec(State& state); +bool IsExecZero(const State& state); +bool IsVccZero(const State& state); + +} // namespace Libs::Graphics::ShaderRecompiler::Exec + +#endif /* EMULATOR_INCLUDE_EMULATOR_GRAPHICS_SHADER_RECOMPILER_EXECMASK_H_ */ diff --git a/src/graphics/shader/recompiler/ExportOps.cpp b/src/graphics/shader/recompiler/ExportOps.cpp new file mode 100644 index 000000000..a554c5510 --- /dev/null +++ b/src/graphics/shader/recompiler/ExportOps.cpp @@ -0,0 +1,53 @@ +#include "graphics/shader/recompiler/ExportOps.h" + +#include + +namespace Libs::Graphics::ShaderRecompiler::Decoder { + +bool DecodeExp(uint32_t pc, std::span code, uint32_t word_index, Instruction* inst, + std::string* error) { + if (word_index + 1u >= code.size()) { + if (error != nullptr) { + *error = fmt::format("truncated EXP instruction at pc 0x{:08x}", pc); + } + return false; + } + + const uint32_t word0 = code[word_index]; + const uint32_t word1 = code[word_index + 1u]; + const uint32_t target = (word0 >> 4u) & 0x3fu; + const uint32_t en = word0 & 0xfu; + + inst->pc = pc; + inst->word = word0; + inst->word_count = 2; + inst->family = Family::EXP; + inst->opcode_id = target; + inst->opcode = Opcode::Exp; + inst->exp.target = target; + inst->exp.en = en; + inst->exp.compr = ((word0 >> 10u) & 1u) != 0; + inst->exp.done = ((word0 >> 11u) & 1u) != 0; + inst->exp.vm = ((word0 >> 12u) & 1u) != 0; + SetRawWords(inst, code, word_index, 2); + + if (!DecodeVectorGpr(word1 & 0xffu, &inst->src0, error) || + !DecodeVectorGpr((word1 >> 8u) & 0xffu, &inst->src1, error) || + !DecodeVectorGpr((word1 >> 16u) & 0xffu, &inst->src2, error) || + !DecodeVectorGpr((word1 >> 24u) & 0xffu, &inst->src3, error)) { + return false; + } + + if (target == 0x14u && inst->exp.done && en == 0x1u) { + inst->src_count = 1u; + } else { + inst->src_count = inst->exp.compr ? 2u : 4u; + } + if (en == 0u) { + inst->src_count = 0; + } + + return true; +} + +} // namespace Libs::Graphics::ShaderRecompiler::Decoder diff --git a/src/graphics/shader/recompiler/ExportOps.h b/src/graphics/shader/recompiler/ExportOps.h new file mode 100644 index 000000000..2f7fbf498 --- /dev/null +++ b/src/graphics/shader/recompiler/ExportOps.h @@ -0,0 +1,13 @@ +#ifndef EMULATOR_INCLUDE_EMULATOR_GRAPHICS_SHADER_RECOMPILER_EXPORTOPS_H_ +#define EMULATOR_INCLUDE_EMULATOR_GRAPHICS_SHADER_RECOMPILER_EXPORTOPS_H_ + +#include "graphics/shader/recompiler/ShaderDecoder.h" + +namespace Libs::Graphics::ShaderRecompiler::Decoder { + +bool DecodeExp(uint32_t pc, std::span code, uint32_t word_index, Instruction* inst, + std::string* error); + +} // namespace Libs::Graphics::ShaderRecompiler::Decoder + +#endif /* EMULATOR_INCLUDE_EMULATOR_GRAPHICS_SHADER_RECOMPILER_EXPORTOPS_H_ */ diff --git a/src/graphics/shader/recompiler/ImageOps.cpp b/src/graphics/shader/recompiler/ImageOps.cpp new file mode 100644 index 000000000..6f4d30a47 --- /dev/null +++ b/src/graphics/shader/recompiler/ImageOps.cpp @@ -0,0 +1,331 @@ +#include "graphics/shader/recompiler/ImageOps.h" + +#include +#include +#include + +namespace Libs::Graphics::ShaderRecompiler::Decoder { +namespace { + +struct MimgSampleInfo { + uint32_t opcode = 0; + const char* name = nullptr; + uint32_t flags = 0; + uint32_t address_components = 3; +}; + +struct MimgGatherInfo { + uint32_t opcode = 0; + const char* name = nullptr; + Opcode decoded = Opcode::Unsupported; + uint32_t flags = 0; + uint32_t address_components = 3; +}; + +struct MimgAtomicInfo { + uint32_t opcode = 0; + const char* name = nullptr; + Opcode decoded = Opcode::Unsupported; +}; + +constexpr ImageDimension DecodeImageDimension(uint32_t dim) { + switch (dim) { + case 1u: return ImageDimension::Dim2D; + case 2u: return ImageDimension::Dim3D; + case 3u: return ImageDimension::Dim2DArray; + case 5u: + case 7u: return ImageDimension::Dim2DArray; + case 6u: return ImageDimension::Dim2D; + default: return ImageDimension::Unknown; + } +} + +constexpr uint32_t ImageCoordComponents(ImageDimension dimension) { + switch (dimension) { + case ImageDimension::Dim3D: + case ImageDimension::Dim2DArray: return 3u; + default: return 2u; + } +} + +constexpr uint32_t ImageSampleAddressComponents(uint32_t flags, ImageDimension dimension) { + const auto coord_components = ImageCoordComponents(dimension); + uint32_t components = coord_components; + if ((flags & ImageSampleFlagOffset) != 0) { + components++; + } + if ((flags & ImageSampleFlagCompare) != 0) { + components++; + } + if ((flags & ImageSampleFlagBias) != 0) { + components++; + } + if ((flags & ImageSampleFlagLod) != 0) { + components++; + } + if ((flags & ImageSampleFlagDerivative) != 0) { + components += 4u; + } + return components; +} + +constexpr MimgSampleInfo SampleInfo(uint32_t opcode, const char* name, uint32_t flags) { + return {opcode, name, flags, ImageSampleAddressComponents(flags, ImageDimension::Dim2D)}; +} + +constexpr MimgSampleInfo MIMG_SAMPLE_OPS[] = { + SampleInfo(0x20u, "image_sample", 0), + SampleInfo(0x21u, "image_sample_cl", ImageSampleFlagLodClamp), + SampleInfo(0x22u, "image_sample_d", ImageSampleFlagDerivative), + SampleInfo(0x23u, "image_sample_d_cl", ImageSampleFlagDerivative | ImageSampleFlagLodClamp), + SampleInfo(0x24u, "image_sample_l", ImageSampleFlagLod), + SampleInfo(0x25u, "image_sample_b", ImageSampleFlagBias), + SampleInfo(0x26u, "image_sample_b_cl", ImageSampleFlagBias | ImageSampleFlagLodClamp), + SampleInfo(0x27u, "image_sample_lz", ImageSampleFlagLevelZero), + SampleInfo(0x28u, "image_sample_c", ImageSampleFlagCompare), + SampleInfo(0x29u, "image_sample_c_cl", ImageSampleFlagCompare | ImageSampleFlagLodClamp), + SampleInfo(0x2au, "image_sample_c_d", ImageSampleFlagCompare | ImageSampleFlagDerivative), + SampleInfo(0x2bu, "image_sample_c_d_cl", + ImageSampleFlagCompare | ImageSampleFlagDerivative | ImageSampleFlagLodClamp), + SampleInfo(0x2cu, "image_sample_c_l", ImageSampleFlagCompare | ImageSampleFlagLod), + SampleInfo(0x2du, "image_sample_c_b", ImageSampleFlagCompare | ImageSampleFlagBias), + SampleInfo(0x2eu, "image_sample_c_b_cl", + ImageSampleFlagCompare | ImageSampleFlagBias | ImageSampleFlagLodClamp), + SampleInfo(0x2fu, "image_sample_c_lz", ImageSampleFlagCompare | ImageSampleFlagLevelZero), + SampleInfo(0x30u, "image_sample_o", ImageSampleFlagOffset), + SampleInfo(0x31u, "image_sample_cl_o", ImageSampleFlagLodClamp | ImageSampleFlagOffset), + SampleInfo(0x32u, "image_sample_d_o", ImageSampleFlagDerivative | ImageSampleFlagOffset), + SampleInfo(0x33u, "image_sample_d_cl_o", + ImageSampleFlagDerivative | ImageSampleFlagLodClamp | ImageSampleFlagOffset), + SampleInfo(0x34u, "image_sample_l_o", ImageSampleFlagLod | ImageSampleFlagOffset), + SampleInfo(0x35u, "image_sample_b_o", ImageSampleFlagBias | ImageSampleFlagOffset), + SampleInfo(0x36u, "image_sample_b_cl_o", + ImageSampleFlagBias | ImageSampleFlagLodClamp | ImageSampleFlagOffset), + SampleInfo(0x37u, "image_sample_lz_o", ImageSampleFlagLevelZero | ImageSampleFlagOffset), + SampleInfo(0x38u, "image_sample_c_o", ImageSampleFlagCompare | ImageSampleFlagOffset), + SampleInfo(0x39u, "image_sample_c_cl_o", + ImageSampleFlagCompare | ImageSampleFlagLodClamp | ImageSampleFlagOffset), + SampleInfo(0x3au, "image_sample_c_d_o", + ImageSampleFlagCompare | ImageSampleFlagDerivative | ImageSampleFlagOffset), + SampleInfo(0x3bu, "image_sample_c_d_cl_o", + ImageSampleFlagCompare | ImageSampleFlagDerivative | ImageSampleFlagLodClamp | + ImageSampleFlagOffset), + SampleInfo(0x3cu, "image_sample_c_l_o", + ImageSampleFlagCompare | ImageSampleFlagLod | ImageSampleFlagOffset), + SampleInfo(0x3du, "image_sample_c_b_o", + ImageSampleFlagCompare | ImageSampleFlagBias | ImageSampleFlagOffset), + SampleInfo(0x3eu, "image_sample_c_b_cl_o", + ImageSampleFlagCompare | ImageSampleFlagBias | ImageSampleFlagLodClamp | + ImageSampleFlagOffset), + SampleInfo(0x3fu, "image_sample_c_lz_o", + ImageSampleFlagCompare | ImageSampleFlagLevelZero | ImageSampleFlagOffset), + SampleInfo(0x68u, "image_sample_cd", ImageSampleFlagDerivative | ImageSampleFlagCd), + SampleInfo(0x69u, "image_sample_cd_cl", + ImageSampleFlagDerivative | ImageSampleFlagCd | ImageSampleFlagLodClamp), + SampleInfo(0x6au, "image_sample_c_cd", + ImageSampleFlagCompare | ImageSampleFlagDerivative | ImageSampleFlagCd), + SampleInfo(0x6bu, "image_sample_c_cd_cl", + ImageSampleFlagCompare | ImageSampleFlagDerivative | ImageSampleFlagCd | + ImageSampleFlagLodClamp), + SampleInfo(0x6cu, "image_sample_cd_o", + ImageSampleFlagDerivative | ImageSampleFlagCd | ImageSampleFlagOffset), + SampleInfo(0x6du, "image_sample_cd_cl_o", + ImageSampleFlagDerivative | ImageSampleFlagCd | ImageSampleFlagLodClamp | + ImageSampleFlagOffset), + SampleInfo(0x6eu, "image_sample_c_cd_o", + ImageSampleFlagCompare | ImageSampleFlagDerivative | ImageSampleFlagCd | + ImageSampleFlagOffset), + SampleInfo(0x6fu, "image_sample_c_cd_cl_o", + ImageSampleFlagCompare | ImageSampleFlagDerivative | ImageSampleFlagCd | + ImageSampleFlagLodClamp | ImageSampleFlagOffset), + SampleInfo(0xa0u, "image_sample_a", 0), + SampleInfo(0xa1u, "image_sample_cl_a", ImageSampleFlagLodClamp), + SampleInfo(0xa5u, "image_sample_b_a", ImageSampleFlagBias), + SampleInfo(0xa6u, "image_sample_b_cl_a", ImageSampleFlagBias | ImageSampleFlagLodClamp), + SampleInfo(0xa8u, "image_sample_c_a", ImageSampleFlagCompare), + SampleInfo(0xa9u, "image_sample_c_cl_a", ImageSampleFlagCompare | ImageSampleFlagLodClamp), + SampleInfo(0xadu, "image_sample_c_b_a", ImageSampleFlagCompare | ImageSampleFlagBias), + SampleInfo(0xaeu, "image_sample_c_b_cl_a", + ImageSampleFlagCompare | ImageSampleFlagBias | ImageSampleFlagLodClamp), + SampleInfo(0xb0u, "image_sample_o_a", ImageSampleFlagOffset), + SampleInfo(0xb1u, "image_sample_cl_o_a", ImageSampleFlagLodClamp | ImageSampleFlagOffset), + SampleInfo(0xb5u, "image_sample_b_o_a", ImageSampleFlagBias | ImageSampleFlagOffset), + SampleInfo(0xb6u, "image_sample_b_cl_o_a", + ImageSampleFlagBias | ImageSampleFlagLodClamp | ImageSampleFlagOffset), + SampleInfo(0xb8u, "image_sample_c_o_a", ImageSampleFlagCompare | ImageSampleFlagOffset), + SampleInfo(0xb9u, "image_sample_c_cl_o_a", + ImageSampleFlagCompare | ImageSampleFlagLodClamp | ImageSampleFlagOffset), + SampleInfo(0xbdu, "image_sample_c_b_o_a", + ImageSampleFlagCompare | ImageSampleFlagBias | ImageSampleFlagOffset), + SampleInfo(0xbeu, "image_sample_c_b_cl_o_a", + ImageSampleFlagCompare | ImageSampleFlagBias | ImageSampleFlagLodClamp | + ImageSampleFlagOffset), +}; + +constexpr MimgGatherInfo MIMG_GATHER_OPS[] = { + {0x47u, "image_gather4_lz", Opcode::ImageGather4Lz, ImageSampleFlagLevelZero, 2u}, + {0x48u, "image_gather4_c", Opcode::ImageGather4C, ImageSampleFlagCompare, 3u}, + {0x4fu, "image_gather4_c_lz", Opcode::ImageGather4CLz, + ImageSampleFlagCompare | ImageSampleFlagLevelZero, 3u}, + {0x57u, "image_gather4_lz_o", Opcode::ImageGather4LzO, + ImageSampleFlagLevelZero | ImageSampleFlagOffset, 3u}, + {0x58u, "image_gather4_c_o", Opcode::ImageGather4CO, + ImageSampleFlagCompare | ImageSampleFlagOffset, 4u}, + {0x5fu, "image_gather4_c_lz_o", Opcode::ImageGather4CLzO, + ImageSampleFlagCompare | ImageSampleFlagLevelZero | ImageSampleFlagOffset, 4u}, + {0x61u, "image_gather4h", Opcode::ImageGather4H, ImageSampleFlagGatherHorizontal, 2u}, +}; + +constexpr MimgAtomicInfo MIMG_ATOMIC_OPS[] = { + {0x11u, "image_atomic_add", Opcode::ImageAtomicAdd}, + {0x15u, "image_atomic_umin", Opcode::ImageAtomicUMin}, + {0x17u, "image_atomic_umax", Opcode::ImageAtomicUMax}, + {0x18u, "image_atomic_and", Opcode::ImageAtomicAnd}, + {0x19u, "image_atomic_or", Opcode::ImageAtomicOr}, + {0x1au, "image_atomic_xor", Opcode::ImageAtomicXor}, +}; + +const MimgSampleInfo* LookupSample(uint32_t opcode) { + for (const auto& info: MIMG_SAMPLE_OPS) { + if (info.opcode == opcode) { + return &info; + } + } + return nullptr; +} + +const MimgGatherInfo* LookupGather(uint32_t opcode) { + for (const auto& info: MIMG_GATHER_OPS) { + if (info.opcode == opcode) { + return &info; + } + } + return nullptr; +} + +const MimgAtomicInfo* LookupAtomic(uint32_t opcode) { + for (const auto& info: MIMG_ATOMIC_OPS) { + if (info.opcode == opcode) { + return &info; + } + } + return nullptr; +} + +uint32_t CountDmaskComponents(uint32_t dmask) { + uint32_t count = 0; + for (uint32_t i = 0; i < 4u; i++) { + count += (dmask >> i) & 1u; + } + return count != 0 ? count : 1u; +} + +bool IsSingleDmaskBit(uint32_t dmask) { + return dmask != 0u && (dmask & (dmask - 1u)) == 0u; +} + +} // namespace + +bool DecodeMimg(uint32_t pc, std::span code, uint32_t word_index, Instruction* inst, + std::string* error) { + if (word_index + 1u >= code.size()) { + if (error != nullptr) { + *error = fmt::format("truncated MIMG instruction at pc 0x{:08x}", pc); + } + return false; + } + + const uint32_t word0 = code[word_index]; + const uint32_t word1 = code[word_index + 1u]; + const uint32_t opcode = ((word0 >> 18u) & 0x7fu) | ((word0 & 1u) << 7u); + const uint32_t nsa_dwords = (word0 >> 1u) & 0x3u; + const auto dimension = DecodeImageDimension((word0 >> 3u) & 0x7u); + const uint32_t word_count = 2u + nsa_dwords; + if (word_index + word_count > code.size()) { + if (error != nullptr) { + *error = fmt::format("truncated MIMG NSA instruction at pc 0x{:08x}", pc); + } + return false; + } + + const uint32_t vdata = (word1 >> 8u) & 0xffu; + const uint32_t vaddr = word1 & 0xffu; + const uint32_t srsrc = (word1 >> 16u) & 0x1fu; + const uint32_t ssamp = (word1 >> 21u) & 0x1fu; + const bool a16 = ((word1 >> 30u) & 0x1u) != 0u; + const auto* sample = LookupSample(opcode); + const auto* gather = LookupGather(opcode); + const auto* atomic = LookupAtomic(opcode); + + inst->pc = pc; + inst->word = word0; + inst->word_count = word_count; + inst->family = Family::MIMG; + inst->opcode_id = opcode; + inst->opcode = (sample != nullptr ? Opcode::ImageSample + : gather != nullptr ? gather->decoded + : atomic != nullptr ? atomic->decoded + : opcode == 0x00u ? Opcode::ImageLoad + : opcode == 0x01u ? Opcode::ImageLoadMip + : opcode == 0x08u ? Opcode::ImageStore + : opcode == 0x09u ? Opcode::ImageStoreMip + : opcode == 0x0eu ? Opcode::ImageGetResinfo + : opcode == 0x60u ? Opcode::ImageGetLod + : Opcode::Unsupported); + inst->dmask = (word0 >> 8u) & 0xfu; + inst->data_dwords = gather != nullptr ? 4u : CountDmaskComponents(inst->dmask); + inst->glc = ((word0 >> 13u) & 1u) != 0; + inst->slc = ((word0 >> 25u) & 1u) != 0; + inst->image_sample_flags = (sample != nullptr ? sample->flags + : gather != nullptr ? gather->flags + : 0u) | + (a16 ? ImageSampleFlagA16 : 0u); + inst->image_dimension = dimension; + inst->image_nsa_dwords = nsa_dwords; + for (uint32_t i = 0; i < nsa_dwords * 4u && i < MaxImageNsaAddressComponents; i++) { + inst->image_nsa_addr[i] = (code[word_index + 2u + i / 4u] >> ((i % 4u) * 8u)) & 0xffu; + } + inst->image_address_components = + sample != nullptr ? ImageSampleAddressComponents(sample->flags, dimension) + : gather != nullptr ? ImageSampleAddressComponents(gather->flags, dimension) + : atomic != nullptr ? 3u + : opcode == 0x60u ? ImageCoordComponents(dimension) + : opcode == 0x01u ? ImageCoordComponents(dimension) + 1u + : opcode == 0x00u ? ImageCoordComponents(dimension) + : (opcode == 0x09u ? ImageCoordComponents(dimension) + 1u + : opcode == 0x08u ? ImageCoordComponents(dimension) + : 0u); + SetRawWords(inst, code, word_index, word_count); + + if (inst->opcode == Opcode::Unsupported) { + SetUnsupported(inst, Family::MIMG, opcode, "MIMG opcode is not implemented"); + } + if (gather != nullptr && !IsSingleDmaskBit(inst->dmask)) { + SetUnsupported(inst, Family::MIMG, opcode, + "MIMG image gather requires exactly one dmask bit"); + } + + DecodeVectorGpr(vdata, &inst->dst, nullptr); + DecodeVectorGpr(vaddr, &inst->src0, nullptr); + DecodeScalarSource(srsrc * 4u, pc, &inst->src1, nullptr); + DecodeScalarSource(ssamp * 4u, pc, &inst->src2, nullptr); + inst->src_count = 3; + return true; +} + +const char* MimgSampleOpcodeName(uint32_t opcode) { + const auto* sample = LookupSample(opcode); + return sample != nullptr ? sample->name : nullptr; +} + +uint32_t MimgSampleOpcodeFlags(uint32_t opcode) { + const auto* sample = LookupSample(opcode); + return sample != nullptr ? sample->flags : 0u; +} + +uint32_t MimgSampleAddressComponents(uint32_t opcode) { + const auto* sample = LookupSample(opcode); + return sample != nullptr ? sample->address_components : 0u; +} + +} // namespace Libs::Graphics::ShaderRecompiler::Decoder diff --git a/src/graphics/shader/recompiler/ImageOps.h b/src/graphics/shader/recompiler/ImageOps.h new file mode 100644 index 000000000..0994e8f15 --- /dev/null +++ b/src/graphics/shader/recompiler/ImageOps.h @@ -0,0 +1,17 @@ +#ifndef EMULATOR_INCLUDE_EMULATOR_GRAPHICS_SHADER_RECOMPILER_IMAGEOPS_H_ +#define EMULATOR_INCLUDE_EMULATOR_GRAPHICS_SHADER_RECOMPILER_IMAGEOPS_H_ + +#include "graphics/shader/recompiler/ShaderDecoder.h" + +namespace Libs::Graphics::ShaderRecompiler::Decoder { + +bool DecodeMimg(uint32_t pc, std::span code, uint32_t word_index, Instruction* inst, + std::string* error); + +const char* MimgSampleOpcodeName(uint32_t opcode); +uint32_t MimgSampleOpcodeFlags(uint32_t opcode); +uint32_t MimgSampleAddressComponents(uint32_t opcode); + +} // namespace Libs::Graphics::ShaderRecompiler::Decoder + +#endif /* EMULATOR_INCLUDE_EMULATOR_GRAPHICS_SHADER_RECOMPILER_IMAGEOPS_H_ */ diff --git a/src/graphics/shader/recompiler/MemoryOps.cpp b/src/graphics/shader/recompiler/MemoryOps.cpp new file mode 100644 index 000000000..c01a83f02 --- /dev/null +++ b/src/graphics/shader/recompiler/MemoryOps.cpp @@ -0,0 +1,480 @@ +#include "graphics/shader/recompiler/MemoryOps.h" + +#include +#include + +namespace Libs::Graphics::ShaderRecompiler::Decoder { +namespace { + +struct MemoryOpcodeInfo { + uint32_t opcode = 0; + Opcode decoded = Opcode::Unsupported; + uint32_t data_dwords = 1; + uint32_t data_bits = 32; + bool data_signed = false; + bool typed = false; + bool formatted = false; +}; + +constexpr MemoryOpcodeInfo SMEM_OPS[] = { + {0x00u, Opcode::SLoadDword, 1, 32}, {0x01u, Opcode::SLoadDwordx2, 2, 32}, + {0x02u, Opcode::SLoadDwordx4, 4, 32}, {0x03u, Opcode::SLoadDwordx8, 8, 32}, + {0x04u, Opcode::SLoadDwordx16, 16, 32}, {0x08u, Opcode::SBufferLoadDword, 1, 32}, + {0x09u, Opcode::SBufferLoadDwordx2, 2, 32}, {0x0au, Opcode::SBufferLoadDwordx4, 4, 32}, + {0x0bu, Opcode::SBufferLoadDwordx8, 8, 32}, {0x0cu, Opcode::SBufferLoadDwordx16, 16, 32}, +}; + +constexpr MemoryOpcodeInfo MUBUF_OPS[] = { + {0x00u, Opcode::BufferLoadFormatX, 1, 32, false, false, true}, + {0x01u, Opcode::BufferLoadFormatXy, 2, 32, false, false, true}, + {0x02u, Opcode::BufferLoadFormatXyz, 3, 32, false, false, true}, + {0x03u, Opcode::BufferLoadFormatXyzw, 4, 32, false, false, true}, + {0x04u, Opcode::BufferStoreFormatX, 1, 32, false, false, true}, + {0x05u, Opcode::BufferStoreFormatXy, 2, 32, false, false, true}, + {0x06u, Opcode::BufferStoreFormatXyz, 3, 32, false, false, true}, + {0x07u, Opcode::BufferStoreFormatXyzw, 4, 32, false, false, true}, + {0x08u, Opcode::BufferLoadUbyte, 1, 8}, + {0x09u, Opcode::BufferLoadSbyte, 1, 8, true}, + {0x0au, Opcode::BufferLoadUshort, 1, 16}, + {0x0bu, Opcode::BufferLoadSshort, 1, 16, true}, + {0x0cu, Opcode::BufferLoadDword, 1, 32}, + {0x0du, Opcode::BufferLoadDwordx2, 2, 32}, + {0x0eu, Opcode::BufferLoadDwordx4, 4, 32}, + {0x0fu, Opcode::BufferLoadDwordx3, 3, 32}, + {0x18u, Opcode::BufferStoreByte, 1, 8}, + {0x1au, Opcode::BufferStoreShort, 1, 16}, + {0x1cu, Opcode::BufferStoreDword, 1, 32}, + {0x1du, Opcode::BufferStoreDwordx2, 2, 32}, + {0x1eu, Opcode::BufferStoreDwordx4, 4, 32}, + {0x1fu, Opcode::BufferStoreDwordx3, 3, 32}, + {0x30u, Opcode::BufferAtomicSwap, 1, 32}, + {0x32u, Opcode::BufferAtomicAdd, 1, 32}, + {0x33u, Opcode::BufferAtomicSub, 1, 32}, + {0x35u, Opcode::BufferAtomicSMin, 1, 32}, + {0x36u, Opcode::BufferAtomicUMin, 1, 32}, + {0x37u, Opcode::BufferAtomicSMax, 1, 32}, + {0x38u, Opcode::BufferAtomicUMax, 1, 32}, + {0x39u, Opcode::BufferAtomicAnd, 1, 32}, + {0x3au, Opcode::BufferAtomicOr, 1, 32}, + {0x3bu, Opcode::BufferAtomicXor, 1, 32}, +}; + +constexpr MemoryOpcodeInfo MTBUF_OPS[] = { + {0x00u, Opcode::TBufferLoadFormatX, 1, 32, false, true, true}, + {0x01u, Opcode::TBufferLoadFormatXy, 2, 32, false, true, true}, + {0x02u, Opcode::TBufferLoadFormatXyz, 3, 32, false, true, true}, + {0x03u, Opcode::TBufferLoadFormatXyzw, 4, 32, false, true, true}, + {0x04u, Opcode::TBufferStoreFormatX, 1, 32, false, true, true}, + {0x05u, Opcode::TBufferStoreFormatXy, 2, 32, false, true, true}, + {0x06u, Opcode::TBufferStoreFormatXyz, 3, 32, false, true, true}, + {0x07u, Opcode::TBufferStoreFormatXyzw, 4, 32, false, true, true}, +}; + +constexpr MemoryOpcodeInfo FLAT_OPS[] = { + {0x08u, Opcode::FlatLoadUbyte, 1, 8}, {0x09u, Opcode::FlatLoadSbyte, 1, 8, true}, + {0x0au, Opcode::FlatLoadUshort, 1, 16}, {0x0bu, Opcode::FlatLoadSshort, 1, 16, true}, + {0x0cu, Opcode::FlatLoadDword, 1, 32}, {0x0du, Opcode::FlatLoadDwordx2, 2, 32}, + {0x0eu, Opcode::FlatLoadDwordx4, 4, 32}, {0x0fu, Opcode::FlatLoadDwordx3, 3, 32}, + {0x18u, Opcode::FlatStoreByte, 1, 8}, {0x1au, Opcode::FlatStoreShort, 1, 16}, + {0x1cu, Opcode::FlatStoreDword, 1, 32}, {0x1du, Opcode::FlatStoreDwordx2, 2, 32}, + {0x1eu, Opcode::FlatStoreDwordx4, 4, 32}, {0x1fu, Opcode::FlatStoreDwordx3, 3, 32}, +}; + +constexpr MemoryOpcodeInfo DS_OPS[] = { + {0x00u, Opcode::DsAddU32, 1, 32}, {0x01u, Opcode::DsSubU32, 1, 32}, + {0x05u, Opcode::DsMinI32, 1, 32}, {0x06u, Opcode::DsMaxI32, 1, 32}, + {0x07u, Opcode::DsMinU32, 1, 32}, {0x08u, Opcode::DsMaxU32, 1, 32}, + {0x09u, Opcode::DsAndB32, 1, 32}, {0x0au, Opcode::DsOrB32, 1, 32}, + {0x0bu, Opcode::DsXorB32, 1, 32}, {0x0du, Opcode::DsWriteB32, 1, 32}, + {0x0eu, Opcode::DsWrite2B32, 2, 32}, {0x0fu, Opcode::DsWrite2St64B32, 2, 32}, + {0x12u, Opcode::DsMinF32, 1, 32}, {0x13u, Opcode::DsMaxF32, 1, 32}, + {0x1eu, Opcode::DsWriteByte, 1, 8}, {0x1fu, Opcode::DsWriteShort, 1, 16}, + {0x20u, Opcode::DsAddRtnU32, 1, 32}, {0x21u, Opcode::DsSubRtnU32, 1, 32}, + {0x25u, Opcode::DsMinRtnI32, 1, 32}, {0x26u, Opcode::DsMaxRtnI32, 1, 32}, + {0x27u, Opcode::DsMinRtnU32, 1, 32}, {0x28u, Opcode::DsMaxRtnU32, 1, 32}, + {0x29u, Opcode::DsAndRtnB32, 1, 32}, {0x2au, Opcode::DsOrRtnB32, 1, 32}, + {0x2bu, Opcode::DsXorRtnB32, 1, 32}, {0x2du, Opcode::DsWrxchgRtnB32, 1, 32}, + {0x35u, Opcode::DsSwizzleB32, 1, 32}, {0x36u, Opcode::DsReadB32, 1, 32}, + {0x37u, Opcode::DsRead2B32, 2, 32}, {0x38u, Opcode::DsRead2B32, 2, 32}, + {0x39u, Opcode::DsReadSbyte, 1, 8, true}, {0x3au, Opcode::DsReadUbyte, 1, 8}, + {0x3bu, Opcode::DsReadSshort, 1, 16, true}, {0x3cu, Opcode::DsReadUshort, 1, 16}, + {0x3du, Opcode::DsConsume, 1, 32}, {0x3eu, Opcode::DsAppend, 1, 32}, + {0x4du, Opcode::DsWriteB64, 2, 32}, {0x4eu, Opcode::DsWrite2B64, 4, 32}, + {0x4fu, Opcode::DsWrite2St64B64, 4, 32}, {0x76u, Opcode::DsReadB64, 2, 32}, + {0x77u, Opcode::DsRead2B64, 4, 32}, {0x78u, Opcode::DsRead2St64B64, 4, 32}, + {0xb0u, Opcode::DsWriteAddtidB32, 1, 32}, {0xb1u, Opcode::DsReadAddtidB32, 1, 32}, + {0xdeu, Opcode::DsWriteB96, 3, 32}, {0xdfu, Opcode::DsWriteB128, 4, 32}, + {0xfeu, Opcode::DsReadB96, 3, 32}, {0xffu, Opcode::DsReadB128, 4, 32}, +}; + +const MemoryOpcodeInfo* LookupMemoryOpcode(const MemoryOpcodeInfo* ops, uint32_t count, + uint32_t opcode) { + for (uint32_t i = 0; i < count; i++) { + if (ops[i].opcode == opcode) { + return &ops[i]; + } + } + return nullptr; +} + +uint32_t SignExtendU32(uint32_t value, uint32_t bits) { + if (bits == 0u || bits >= 32u) { + return value; + } + const uint32_t sign = 1u << (bits - 1u); + return (value ^ sign) - sign; +} + +void MarkMemoryUnsupported(Instruction* inst, Family family, uint32_t opcode, const char* reason) { + inst->family = family; + inst->opcode_id = opcode; + inst->opcode = Opcode::Unsupported; + SetUnsupported(inst, family, opcode, reason); +} + +void ApplyMemoryInfo(Instruction* inst, const MemoryOpcodeInfo* info) { + if (info == nullptr) { + inst->opcode = Opcode::Unsupported; + return; + } + inst->opcode = info->decoded; + inst->data_dwords = info->data_dwords; + inst->data_bits = info->data_bits; + inst->data_signed = info->data_signed; + inst->typed = info->typed; + inst->formatted = info->formatted; +} + +bool IsDsWriteOpcode(Opcode opcode) { + switch (opcode) { + case Opcode::DsWriteByte: + case Opcode::DsWriteShort: + case Opcode::DsWrite2B32: + case Opcode::DsWrite2St64B32: + case Opcode::DsWrite2B64: + case Opcode::DsWrite2St64B64: + case Opcode::DsWriteB32: + case Opcode::DsWriteB64: + case Opcode::DsWriteB96: + case Opcode::DsWriteB128: return true; + default: return false; + } +} + +bool IsDsAtomicOpcode(Opcode opcode) { + switch (opcode) { + case Opcode::DsAddU32: + case Opcode::DsAddRtnU32: + case Opcode::DsSubU32: + case Opcode::DsSubRtnU32: + case Opcode::DsMinI32: + case Opcode::DsMinRtnI32: + case Opcode::DsMaxI32: + case Opcode::DsMaxRtnI32: + case Opcode::DsMinU32: + case Opcode::DsMinRtnU32: + case Opcode::DsMaxU32: + case Opcode::DsMaxRtnU32: + case Opcode::DsAndB32: + case Opcode::DsAndRtnB32: + case Opcode::DsOrB32: + case Opcode::DsOrRtnB32: + case Opcode::DsXorB32: + case Opcode::DsXorRtnB32: + case Opcode::DsWrxchgRtnB32: return true; + default: return false; + } +} + +uint32_t DsSourceCount(Opcode opcode) { + switch (opcode) { + case Opcode::DsWrite2B32: + case Opcode::DsWrite2St64B32: + case Opcode::DsWrite2B64: + case Opcode::DsWrite2St64B64: + case Opcode::DsMinF32: + case Opcode::DsMaxF32: return 3u; + case Opcode::DsReadAddtidB32: + case Opcode::DsConsume: + case Opcode::DsAppend: return 0u; + default: return IsDsWriteOpcode(opcode) || IsDsAtomicOpcode(opcode) ? 2u : 1u; + } +} + +bool IsFlatStoreOpcode(Opcode opcode) { + switch (opcode) { + case Opcode::FlatStoreByte: + case Opcode::FlatStoreShort: + case Opcode::FlatStoreDword: + case Opcode::FlatStoreDwordx2: + case Opcode::FlatStoreDwordx3: + case Opcode::FlatStoreDwordx4: return true; + default: return false; + } +} + +} // namespace + +bool DecodeSmem(uint32_t pc, std::span code, uint32_t word_index, Instruction* inst, + std::string* error) { + if (word_index + 1u >= code.size()) { + if (error != nullptr) { + *error = fmt::format("truncated SMEM instruction at pc 0x{:08x}", pc); + } + return false; + } + + const uint32_t word0 = code[word_index]; + const uint32_t word1 = code[word_index + 1u]; + const uint32_t opcode = (word0 >> 18u) & 0xffu; + const uint32_t sdst = (word0 >> 6u) & 0x7fu; + const uint32_t sbase = word0 & 0x3fu; + const uint32_t soffset = (word1 >> 25u) & 0x7fu; + + inst->pc = pc; + inst->word = word0; + inst->word_count = 2; + inst->offset = SignExtendU32(word1 & 0x1fffffu, 21u); + inst->glc = ((word0 >> 16u) & 1u) != 0; + inst->family = Family::SMEM; + inst->opcode_id = opcode; + const auto* info = + LookupMemoryOpcode(SMEM_OPS, static_cast(std::size(SMEM_OPS)), opcode); + ApplyMemoryInfo(inst, info); + SetRawWords(inst, code, word_index, 2); + if (inst->opcode == Opcode::Unsupported) { + MarkMemoryUnsupported(inst, Family::SMEM, opcode, "SMEM opcode is not implemented"); + } + + DecodeScalarDestination(sdst, pc, &inst->dst, nullptr); + // SMEM encodes SBASE in SGPR pairs. Scalar-buffer loads still use the same + // pair index; their descriptor operand consumes four SGPRs from that base. + DecodeScalarSource(sbase * 2u, pc, &inst->src0, nullptr); + DecodeScalarSource(soffset, pc, &inst->src1, nullptr); + inst->src_count = 2; + return true; +} + +bool DecodeMubuf(uint32_t pc, std::span code, uint32_t word_index, + Instruction* inst, std::string* error) { + if (word_index + 1u >= code.size()) { + if (error != nullptr) { + *error = fmt::format("truncated MUBUF instruction at pc 0x{:08x}", pc); + } + return false; + } + + const uint32_t word0 = code[word_index]; + const uint32_t word1 = code[word_index + 1u]; + const uint32_t opcode = ((word0 >> 18u) & 0x7fu) | (((word0 >> 25u) & 1u) << 7u); + const uint32_t vdata = (word1 >> 8u) & 0xffu; + const uint32_t vaddr = word1 & 0xffu; + const uint32_t srsrc = (word1 >> 16u) & 0x1fu; + const uint32_t soffset = (word1 >> 24u) & 0xffu; + + inst->pc = pc; + inst->word = word0; + inst->word_count = 2; + inst->offset = word0 & 0xfffu; + inst->idxen = ((word0 >> 13u) & 1u) != 0; + inst->offen = ((word0 >> 12u) & 1u) != 0; + inst->glc = ((word0 >> 14u) & 1u) != 0; + inst->slc = ((word1 >> 22u) & 1u) != 0; + inst->family = Family::MUBUF; + inst->opcode_id = opcode; + const auto* info = + LookupMemoryOpcode(MUBUF_OPS, static_cast(std::size(MUBUF_OPS)), opcode); + ApplyMemoryInfo(inst, info); + SetRawWords(inst, code, word_index, 2); + if (inst->opcode == Opcode::Unsupported) { + MarkMemoryUnsupported(inst, Family::MUBUF, opcode, "MUBUF opcode is not implemented"); + } + + DecodeVectorGpr(vdata, &inst->dst, nullptr); + DecodeVectorGpr(vaddr, &inst->src0, nullptr); + DecodeScalarSource(srsrc * 4u, pc, &inst->src1, nullptr); + DecodeScalarSource(soffset, pc, &inst->src2, nullptr); + inst->src_count = 3; + return true; +} + +bool DecodeMtbuf(uint32_t pc, std::span code, uint32_t word_index, + Instruction* inst, std::string* error) { + if (word_index + 1u >= code.size()) { + if (error != nullptr) { + *error = fmt::format("truncated MTBUF instruction at pc 0x{:08x}", pc); + } + return false; + } + + const uint32_t word0 = code[word_index]; + const uint32_t word1 = code[word_index + 1u]; + const uint32_t opcode = ((word0 >> 16u) & 0x7u) | (((word1 >> 21u) & 1u) << 3u); + const uint32_t dfmt = (word0 >> 19u) & 0xfu; + const uint32_t nfmt = (word0 >> 23u) & 0x7u; + const uint32_t vdata = (word1 >> 8u) & 0xffu; + const uint32_t vaddr = word1 & 0xffu; + const uint32_t srsrc = (word1 >> 16u) & 0x1fu; + const uint32_t soffset = (word1 >> 24u) & 0xffu; + + inst->pc = pc; + inst->word = word0; + inst->word_count = 2; + inst->offset = word0 & 0xfffu; + inst->idxen = ((word0 >> 13u) & 1u) != 0; + inst->offen = ((word0 >> 12u) & 1u) != 0; + inst->glc = ((word0 >> 14u) & 1u) != 0; + inst->slc = ((word1 >> 22u) & 1u) != 0; + inst->family = Family::MTBUF; + inst->opcode_id = opcode; + inst->data_format = dfmt; + inst->number_format = nfmt; + const auto* info = + LookupMemoryOpcode(MTBUF_OPS, static_cast(std::size(MTBUF_OPS)), opcode); + ApplyMemoryInfo(inst, info); + SetRawWords(inst, code, word_index, 2); + if (inst->opcode == Opcode::Unsupported) { + MarkMemoryUnsupported(inst, Family::MTBUF, opcode, "MTBUF opcode is not implemented"); + } + + DecodeVectorGpr(vdata, &inst->dst, nullptr); + DecodeVectorGpr(vaddr, &inst->src0, nullptr); + DecodeScalarSource(srsrc * 4u, pc, &inst->src1, nullptr); + DecodeScalarSource(soffset, pc, &inst->src2, nullptr); + inst->src_count = 3; + return true; +} + +bool DecodeFlat(uint32_t pc, std::span code, uint32_t word_index, Instruction* inst, + std::string* error) { + if (word_index + 1u >= code.size()) { + if (error != nullptr) { + *error = fmt::format("truncated FLAT instruction at pc 0x{:08x}", pc); + } + return false; + } + + const uint32_t word0 = code[word_index]; + const uint32_t word1 = code[word_index + 1u]; + const uint32_t offset = word0 & 0xfffu; + const uint32_t dlc = (word0 >> 12u) & 1u; + const uint32_t lds = (word0 >> 13u) & 1u; + const uint32_t seg = (word0 >> 14u) & 0x3u; + const uint32_t opcode = (word0 >> 18u) & 0x7fu; + const uint32_t vdst = (word1 >> 24u) & 0xffu; + const uint32_t saddr = (word1 >> 16u) & 0x7fu; + const uint32_t data = (word1 >> 8u) & 0xffu; + const uint32_t addr = word1 & 0xffu; + + inst->pc = pc; + inst->word = word0; + inst->word_count = 2; + inst->offset = seg == 0u ? (offset & 0x7ffu) : SignExtendU32(offset, 12u); + inst->glc = ((word0 >> 16u) & 1u) != 0; + inst->slc = ((word0 >> 17u) & 1u) != 0; + inst->family = Family::FLAT; + inst->opcode_id = opcode; + inst->memory_segment = seg; + const auto* info = + LookupMemoryOpcode(FLAT_OPS, static_cast(std::size(FLAT_OPS)), opcode); + ApplyMemoryInfo(inst, info); + SetRawWords(inst, code, word_index, 2); + + if (dlc != 0 || lds != 0 || inst->glc || inst->slc || seg == 3u) { + SetUnsupported(inst, Family::FLAT, opcode, "FLAT modifiers or segment are not implemented"); + return true; + } + if (inst->opcode == Opcode::Unsupported) { + MarkMemoryUnsupported(inst, Family::FLAT, opcode, "FLAT opcode is not implemented"); + return true; + } + + DecodeVectorGpr(IsFlatStoreOpcode(inst->opcode) ? data : vdst, &inst->dst, nullptr); + DecodeVectorGpr(addr, &inst->src0, nullptr); + inst->src_count = 1; + if (seg == 0u || saddr == 0x7du || saddr == 0x7fu) { + DecodeVectorGpr(addr + 1u, &inst->src1, nullptr); + inst->src_count = 2; + } else { + DecodeScalarSource(saddr, pc, &inst->src1, nullptr); + inst->src_count = 2; + } + return true; +} + +bool DecodeDs(uint32_t pc, std::span code, uint32_t word_index, Instruction* inst, + std::string* error) { + if (word_index + 1u >= code.size()) { + if (error != nullptr) { + *error = fmt::format("truncated DS instruction at pc 0x{:08x}", pc); + } + return false; + } + + const uint32_t word0 = code[word_index]; + const uint32_t word1 = code[word_index + 1u]; + const uint32_t opcode = (word0 >> 18u) & 0xffu; + const uint32_t offset0 = word0 & 0xffu; + const uint32_t offset1 = (word0 >> 8u) & 0xffu; + const uint32_t vdst = (word1 >> 24u) & 0xffu; + const uint32_t data1 = (word1 >> 16u) & 0xffu; + const uint32_t data0 = (word1 >> 8u) & 0xffu; + const uint32_t addr = word1 & 0xffu; + + inst->pc = pc; + inst->word = word0; + inst->word_count = 2; + inst->offset = offset0 | (offset1 << 8u); + inst->gds = ((word0 >> 17u) & 1u) != 0u; + inst->family = Family::DS; + inst->opcode_id = opcode; + const auto* info = LookupMemoryOpcode(DS_OPS, static_cast(std::size(DS_OPS)), opcode); + ApplyMemoryInfo(inst, info); + SetRawWords(inst, code, word_index, 2); + if (inst->opcode == Opcode::Unsupported) { + MarkMemoryUnsupported(inst, Family::DS, opcode, "DS opcode is not implemented"); + } + if (inst->opcode == Opcode::DsSwizzleB32 && inst->offset >= 0xe000u) { + SetUnsupported(inst, Family::DS, opcode, "DS swizzle FFT mode is not implemented"); + } + if (inst->gds && + (inst->opcode == Opcode::DsSwizzleB32 || inst->opcode == Opcode::DsWriteAddtidB32 || + inst->opcode == Opcode::DsReadAddtidB32)) { + SetUnsupported(inst, Family::DS, opcode, "DS swizzle/addtid is available only for LDS"); + } + if (inst->gds && (inst->opcode == Opcode::DsAppend || inst->opcode == Opcode::DsConsume) && + inst->offset != 0u) { + SetUnsupported(inst, Family::DS, opcode, + "GDS append/consume requires a zero instruction offset"); + } + if (inst->opcode == Opcode::DsWriteAddtidB32 && data1 != 0u) { + SetUnsupported(inst, Family::DS, opcode, + "DS write addtid data1 operand is not implemented"); + } + if (inst->opcode == Opcode::DsReadAddtidB32 && (data0 != 0u || data1 != 0u)) { + SetUnsupported(inst, Family::DS, opcode, + "DS read addtid data operands are not implemented"); + } + if (inst->opcode == Opcode::DsWrite2B32 || opcode == 0x37u) { + inst->offset = offset0 * 4u; + inst->secondary_offset = offset1 * 4u; + } else if (inst->opcode == Opcode::DsWrite2St64B32 || opcode == 0x38u) { + inst->offset = offset0 * 256u; + inst->secondary_offset = offset1 * 256u; + } else if (inst->opcode == Opcode::DsWrite2B64 || inst->opcode == Opcode::DsRead2B64) { + inst->offset = offset0 * 8u; + inst->secondary_offset = offset1 * 8u; + } else if (inst->opcode == Opcode::DsWrite2St64B64 || inst->opcode == Opcode::DsRead2St64B64) { + inst->offset = offset0 * 512u; + inst->secondary_offset = offset1 * 512u; + } + + DecodeVectorGpr(vdst, &inst->dst, nullptr); + DecodeVectorGpr(addr, &inst->src0, nullptr); + DecodeVectorGpr(data0, &inst->src1, nullptr); + DecodeVectorGpr(data1, &inst->src2, nullptr); + inst->src_count = DsSourceCount(inst->opcode); + return true; +} + +} // namespace Libs::Graphics::ShaderRecompiler::Decoder diff --git a/src/graphics/shader/recompiler/MemoryOps.h b/src/graphics/shader/recompiler/MemoryOps.h new file mode 100644 index 000000000..ce0369430 --- /dev/null +++ b/src/graphics/shader/recompiler/MemoryOps.h @@ -0,0 +1,21 @@ +#ifndef EMULATOR_INCLUDE_EMULATOR_GRAPHICS_SHADER_RECOMPILER_MEMORYOPS_H_ +#define EMULATOR_INCLUDE_EMULATOR_GRAPHICS_SHADER_RECOMPILER_MEMORYOPS_H_ + +#include "graphics/shader/recompiler/ShaderDecoder.h" + +namespace Libs::Graphics::ShaderRecompiler::Decoder { + +bool DecodeSmem(uint32_t pc, std::span code, uint32_t word_index, Instruction* inst, + std::string* error); +bool DecodeMubuf(uint32_t pc, std::span code, uint32_t word_index, + Instruction* inst, std::string* error); +bool DecodeMtbuf(uint32_t pc, std::span code, uint32_t word_index, + Instruction* inst, std::string* error); +bool DecodeFlat(uint32_t pc, std::span code, uint32_t word_index, Instruction* inst, + std::string* error); +bool DecodeDs(uint32_t pc, std::span code, uint32_t word_index, Instruction* inst, + std::string* error); + +} // namespace Libs::Graphics::ShaderRecompiler::Decoder + +#endif /* EMULATOR_INCLUDE_EMULATOR_GRAPHICS_SHADER_RECOMPILER_MEMORYOPS_H_ */ diff --git a/src/graphics/shader/recompiler/ResourceMaterialization.cpp b/src/graphics/shader/recompiler/ResourceMaterialization.cpp new file mode 100644 index 000000000..85c21fe1e --- /dev/null +++ b/src/graphics/shader/recompiler/ResourceMaterialization.cpp @@ -0,0 +1,354 @@ +#include "graphics/shader/recompiler/ResourceMaterialization.h" + +#include "graphics/guest_gpu/gpu_format.h" +#include "graphics/shader/recompiler/BufferFormat.h" + +#include + +namespace Libs::Graphics::ShaderRecompiler::IR { +namespace { + +constexpr uint64_t AddressMask = 0x0000ffffffffffffull; + +Decoder::ImageDimension DescriptorDimension(const DescriptorValue& descriptor) { + switch (static_cast((descriptor.dwords[3] >> 28u) & 0xfu)) { + case Prospero::ImageType::kColor3D: return Decoder::ImageDimension::Dim3D; + case Prospero::ImageType::kCube: + case Prospero::ImageType::kColor1DArray: + case Prospero::ImageType::kColor2DArray: + case Prospero::ImageType::kColor2DMsaaArray: return Decoder::ImageDimension::Dim2DArray; + case Prospero::ImageType::kColor1D: + case Prospero::ImageType::kColor2D: + case Prospero::ImageType::kColor2DMsaa: return Decoder::ImageDimension::Dim2D; + default: return Decoder::ImageDimension::Unknown; + } +} + +bool NullImageDescriptor(const DescriptorValue& descriptor) { + return descriptor.dwords[0] == 0 && (descriptor.dwords[1] & 0xffu) == 0; +} + +uint32_t DescriptorImageSwizzle(const DescriptorValue& descriptor) { + return descriptor.dwords[3] & 0xfffu; +} + +uint32_t DescriptorPackedStride(const DescriptorValue& descriptor) { + const auto stride = (descriptor.dwords[1] >> 16u) & 0x3fffu; + const auto swizzle = (descriptor.dwords[1] >> 31u) & 1u; + const auto index_stride = (descriptor.dwords[3] >> 21u) & 3u; + const auto add_tid = (descriptor.dwords[3] >> 23u) & 1u; + return stride | (swizzle << 14u) | (index_stride << 16u) | (add_tid << 20u); +} + +uint32_t DescriptorBufferFormat(const DescriptorValue& descriptor) { + return (descriptor.dwords[3] >> 12u) & 0x7fu; +} + +uint64_t AddressSpecialization(const AddressResource& resource, + const ResourceSnapshot::Address& snapshot) { + return resource.kind == ResourceKind::Flat || resource.source == ScalarProvenance::Unknown + ? snapshot.binding_base + : snapshot.guest_base - snapshot.binding_base; +} + +} // namespace + +bool ValidateResourceSnapshot(const Program& program, const ResourceSnapshot& snapshot, + std::string* error) { + if (!program.resource_tracking_complete) { + if (error != nullptr) { + *error = "shader resources were not tracked"; + } + return false; + } + if (snapshot.buffers.size() != program.info.buffers.size() || + snapshot.images.size() != program.info.images.size() || + snapshot.samplers.size() != program.info.samplers.size() || + snapshot.addresses.size() != program.info.addresses.size()) { + if (error != nullptr) { + *error = "resource snapshot does not match dense shader topology"; + } + return false; + } + if (snapshot.flattened_srt.size() != program.srt.reads.size()) { + if (error != nullptr) { + *error = "flattened SRT snapshot does not match the shader plan"; + } + return false; + } + if (program.binding_layout_complete) { + for (const auto reg: program.bindings.user_data_registers) { + if (reg < program.user_data_base || + reg - program.user_data_base >= snapshot.user_data.size()) { + if (error != nullptr) { + *error = fmt::format("runtime snapshot is missing user SGPR {}", reg); + } + return false; + } + } + } + const auto CheckWidth = [&](const auto& values, uint32_t width, const char* kind) { + for (uint32_t i = 0; i < values.size(); i++) { + if (values[i].dword_count != width) { + if (error != nullptr) { + *error = fmt::format("{} descriptor {} has {} dwords", kind, i, + values[i].dword_count); + } + return false; + } + } + return true; + }; + for (uint32_t i = 0; i < snapshot.addresses.size(); i++) { + if (snapshot.addresses[i].binding_base > snapshot.addresses[i].guest_base) { + if (error != nullptr) { + *error = fmt::format("address resource {} binds above its guest base", i); + } + return false; + } + } + return CheckWidth(snapshot.buffers, 4, "buffer") && CheckWidth(snapshot.images, 8, "image") && + CheckWidth(snapshot.samplers, 4, "sampler"); +} + +bool ValidateResourceSpecialization(const Program& program, const ResourceSnapshot& snapshot, + std::string* error) { + if (!ValidateResourceSnapshot(program, snapshot, error)) { + return false; + } + for (uint32_t i = 0; i < program.info.buffers.size(); i++) { + const auto& buffer = program.info.buffers[i]; + const auto& descriptor = snapshot.buffers[i]; + if (buffer.packed_stride != DescriptorPackedStride(descriptor) || + buffer.descriptor_format != DescriptorBufferFormat(descriptor)) { + if (error != nullptr) { + *error = fmt::format("buffer descriptor {} no longer matches specialization", i); + } + return false; + } + } + for (uint32_t i = 0; i < program.info.images.size(); i++) { + const auto& image = program.info.images[i]; + const auto& descriptor = snapshot.images[i]; + if (NullImageDescriptor(descriptor)) { + continue; + } + const auto dimension = DescriptorDimension(descriptor); + if (dimension == Decoder::ImageDimension::Unknown || dimension != image.dimension) { + if (error != nullptr) { + *error = + fmt::format("image descriptor {} no longer matches specialized dimension", i); + } + return false; + } + if (image.kind == ResourceKind::Image || image.kind == ResourceKind::ImageUint || + image.kind == ResourceKind::StorageImage || + image.kind == ResourceKind::StorageImageUint) { + const bool storage = image.kind == ResourceKind::StorageImage || + image.kind == ResourceKind::StorageImageUint; + if (storage && image.storage_swizzle != DescriptorImageSwizzle(descriptor)) { + if (error != nullptr) { + *error = fmt::format("storage image descriptor {} changed swizzle", i); + } + return false; + } + const auto uint_descriptor = + Prospero::IsUintTextureFormat((descriptor.dwords[1] >> 20u) & 0x1ffu); + const auto uint_program = image.kind == ResourceKind::ImageUint || + image.kind == ResourceKind::StorageImageUint; + if (uint_descriptor != uint_program && !(image.atomic && uint_program)) { + if (error != nullptr) { + *error = + fmt::format("image descriptor {} no longer matches specialized format", i); + } + return false; + } + if (image.depth_compare && uint_program) { + if (error != nullptr) { + *error = fmt::format("integer image descriptor {} uses depth comparison", i); + } + return false; + } + } + } + for (uint32_t i = 0; i < program.info.addresses.size(); i++) { + if (program.info.addresses[i].specialized_base != + AddressSpecialization(program.info.addresses[i], snapshot.addresses[i])) { + if (error != nullptr) { + *error = fmt::format("address resource {} no longer matches specialization", i); + } + return false; + } + } + return true; +} + +bool MaterializeResources(const Program& program, const SrtRuntime& runtime, + ResourceSnapshot* snapshot, std::string* error) { + if (snapshot == nullptr || !program.resource_tracking_complete) { + if (error != nullptr) { + *error = snapshot == nullptr ? "invalid resource snapshot" + : "shader resources were not tracked"; + } + return false; + } + + std::vector requests; + requests.reserve(program.info.buffers.size() + program.info.images.size() + + program.info.samplers.size() + program.info.addresses.size()); + for (const auto& buffer: program.info.buffers) { + requests.push_back({buffer.source, buffer.first_use_pc}); + } + for (const auto& image: program.info.images) { + requests.push_back({image.source, image.first_use_pc}); + } + for (const auto& sampler: program.info.samplers) { + requests.push_back({sampler.source, sampler.first_use_pc}); + } + for (const auto& address: program.info.addresses) { + if (address.source != ScalarProvenance::Unknown) { + requests.push_back({address.source, address.first_use_pc}); + } + } + + std::vector values; + std::vector flattened_srt; + if (!EvaluateRuntimeSources(program, requests, runtime, &values, &flattened_srt, error)) { + return false; + } + + ResourceSnapshot next; + auto cursor = values.begin(); + next.buffers.assign(cursor, cursor + program.info.buffers.size()); + cursor += program.info.buffers.size(); + next.images.assign(cursor, cursor + program.info.images.size()); + cursor += program.info.images.size(); + next.samplers.assign(cursor, cursor + program.info.samplers.size()); + cursor += program.info.samplers.size(); + for (const auto& address: program.info.addresses) { + if (address.source != ScalarProvenance::Unknown) { + const auto value = *cursor++; + auto base = (static_cast(value.dwords[0]) | + static_cast(value.dwords[1]) << 32u) & + AddressMask; + if (address.kind == ResourceKind::ScalarBuffer) { + base &= ~uint64_t {3}; + } + const auto before = static_cast(-static_cast(address.min_offset)); + const auto binding_base = address.kind == ResourceKind::Flat + ? base & ~(FlatAddressWindowSize - 1u) + : base >= before ? base - before + : 0; + next.addresses.push_back({base, binding_base}); + } else { + if (!runtime.flat_memory_base.has_value()) { + if (error != nullptr) { + *error = + fmt::format("unbased {} address at pc 0x{:08x} requires runtime " + "guest-address translation", + address.kind == ResourceKind::Flat ? "FLAT" : "global/scratch", + address.first_use_pc); + } + return false; + } + next.addresses.push_back({*runtime.flat_memory_base, *runtime.flat_memory_base}); + } + } + next.flattened_srt = std::move(flattened_srt); + next.user_data.assign(runtime.user_data.begin(), runtime.user_data.end()); + if (!ValidateResourceSnapshot(program, next, error)) { + return false; + } + *snapshot = std::move(next); + return true; +} + +bool SpecializeResources(Program* program, const ResourceSnapshot& snapshot, std::string* error) { + if (program == nullptr || !program->resource_tracking_complete || + program->shader_info_complete || program->binding_layout_complete) { + if (error != nullptr) { + *error = program == nullptr ? "invalid resource specialization program" + : !program->resource_tracking_complete ? "shader resources were not tracked" + : "resource specialization is too late"; + } + return false; + } + if (!ValidateResourceSnapshot(*program, snapshot, error)) { + return false; + } + + auto next = program->info; + for (uint32_t i = 0; i < next.buffers.size(); i++) { + next.buffers[i].packed_stride = DescriptorPackedStride(snapshot.buffers[i]); + next.buffers[i].descriptor_format = DescriptorBufferFormat(snapshot.buffers[i]); + } + for (uint32_t i = 0; i < next.addresses.size(); i++) { + next.addresses[i].specialized_base = + AddressSpecialization(next.addresses[i], snapshot.addresses[i]); + } + for (uint32_t i = 0; i < next.images.size(); i++) { + const auto& descriptor = snapshot.images[i]; + auto& image = next.images[i]; + if (NullImageDescriptor(descriptor)) { + continue; + } + const auto descriptor_dimension = DescriptorDimension(descriptor); + if (descriptor_dimension == Decoder::ImageDimension::Unknown) { + if (error != nullptr) { + *error = fmt::format( + "image descriptor {} has unsupported type {}: {:08x},{:08x},{:08x},{:08x}," + "{:08x},{:08x},{:08x},{:08x}", + i, (descriptor.dwords[3] >> 28u) & 0xfu, descriptor.dwords[0], + descriptor.dwords[1], descriptor.dwords[2], descriptor.dwords[3], + descriptor.dwords[4], descriptor.dwords[5], descriptor.dwords[6], + descriptor.dwords[7]); + } + return false; + } + image.dimension = descriptor_dimension; + if (image.kind == ResourceKind::StorageImage || + image.kind == ResourceKind::StorageImageUint) { + image.storage_swizzle = DescriptorImageSwizzle(descriptor); + } + if (Prospero::IsUintTextureFormat((descriptor.dwords[1] >> 20u) & 0x1ffu)) { + switch (image.kind) { + case ResourceKind::Image: image.kind = ResourceKind::ImageUint; break; + case ResourceKind::StorageImage: image.kind = ResourceKind::StorageImageUint; break; + default: break; + } + } + } + struct ImagePatch { + Instruction* inst; + ResourceKind kind; + Decoder::ImageDimension dimension; + }; + std::vector patches; + for (auto& block: program->blocks) { + for (auto& inst: block.instructions) { + if (inst.memory.kind != ResourceKind::Image && + inst.memory.kind != ResourceKind::ImageUint && + inst.memory.kind != ResourceKind::StorageImage && + inst.memory.kind != ResourceKind::StorageImageUint) { + continue; + } + if (inst.memory.resource >= next.images.size()) { + if (error != nullptr) { + *error = fmt::format("image instruction at pc 0x{:08x} has invalid resource {}", + inst.pc, inst.memory.resource); + } + return false; + } + const auto& image = next.images[inst.memory.resource]; + patches.push_back({&inst, image.kind, image.dimension}); + } + } + program->info = std::move(next); + for (const auto& patch: patches) { + patch.inst->memory.kind = patch.kind; + patch.inst->memory.image_dimension = patch.dimension; + } + return true; +} + +} // namespace Libs::Graphics::ShaderRecompiler::IR diff --git a/src/graphics/shader/recompiler/ResourceMaterialization.h b/src/graphics/shader/recompiler/ResourceMaterialization.h new file mode 100644 index 000000000..714f858ad --- /dev/null +++ b/src/graphics/shader/recompiler/ResourceMaterialization.h @@ -0,0 +1,42 @@ +#ifndef EMULATOR_INCLUDE_EMULATOR_GRAPHICS_SHADER_RECOMPILER_RESOURCEMATERIALIZATION_H_ +#define EMULATOR_INCLUDE_EMULATOR_GRAPHICS_SHADER_RECOMPILER_RESOURCEMATERIALIZATION_H_ + +#include "graphics/shader/recompiler/SrtWalker.h" + +namespace Libs::Graphics::ShaderRecompiler::IR { + +constexpr uint64_t FlatAddressWindowSize = 0x10000; + +struct ResourceSnapshot { + struct Address { + uint64_t guest_base = 0; + uint64_t binding_base = 0; + + bool operator==(const Address& other) const = default; + }; + + std::vector buffers; + std::vector images; + std::vector samplers; + std::vector
addresses; + std::vector flattened_srt; + std::vector user_data; +}; + +bool ValidateResourceSnapshot(const Program& program, const ResourceSnapshot& snapshot, + std::string* error); +bool ValidateResourceSpecialization(const Program& program, const ResourceSnapshot& snapshot, + std::string* error); + +// Resolves the immutable dense resource topology against one runtime user-data/SRT snapshot. +// On failure the destination is unchanged. +bool MaterializeResources(const Program& program, const SrtRuntime& runtime, + ResourceSnapshot* snapshot, std::string* error); + +// Applies runtime descriptor shape/format facts to a copied dense topology before layout and +// emission. On failure the program is unchanged. +bool SpecializeResources(Program* program, const ResourceSnapshot& snapshot, std::string* error); + +} // namespace Libs::Graphics::ShaderRecompiler::IR + +#endif /* EMULATOR_INCLUDE_EMULATOR_GRAPHICS_SHADER_RECOMPILER_RESOURCEMATERIALIZATION_H_ */ diff --git a/src/graphics/shader/recompiler/ResourceTracking.cpp b/src/graphics/shader/recompiler/ResourceTracking.cpp new file mode 100644 index 000000000..57d8b180b --- /dev/null +++ b/src/graphics/shader/recompiler/ResourceTracking.cpp @@ -0,0 +1,449 @@ +#include "graphics/shader/recompiler/ResourceTracking.h" + +#include "graphics/shader/recompiler/ScalarProvenance.h" + +#include +#include + +namespace Libs::Graphics::ShaderRecompiler::IR { +namespace { + +const char* StageName(ShaderType stage) { + switch (stage) { + case ShaderType::Vertex: return "vertex"; + case ShaderType::Pixel: return "pixel"; + case ShaderType::Fetch: return "fetch"; + case ShaderType::Compute: return "compute"; + default: return "unknown"; + } +} + +bool IsAtomic(Opcode op) { + switch (op) { + case Opcode::AtomicSwapU32: + case Opcode::AtomicAddU32: + case Opcode::AtomicSubU32: + case Opcode::AtomicSMinI32: + case Opcode::AtomicUMinU32: + case Opcode::AtomicSMaxI32: + case Opcode::AtomicUMaxU32: + case Opcode::AtomicAndU32: + case Opcode::AtomicOrU32: + case Opcode::AtomicXorU32: return true; + default: return false; + } +} + +bool IsWrite(Opcode op) { + switch (op) { + case Opcode::BufferStoreByte: + case Opcode::BufferStoreShort: + case Opcode::BufferStoreDword: + case Opcode::ImageStore: return true; + default: return IsAtomic(op); + } +} + +bool NeedsSampler(Opcode op) { + return op == Opcode::ImageSample || op == Opcode::ImageGather4 || op == Opcode::ImageGetLod; +} + +bool IsDepthCompare(const Instruction& inst) { + return (inst.memory.image_sample_flags & Decoder::ImageSampleFlagCompare) != 0; +} + +ImageMipMode MipMode(const Instruction& inst) { + const bool storage = inst.memory.kind == ResourceKind::StorageImage || + inst.memory.kind == ResourceKind::StorageImageUint; + return storage && inst.memory.image_has_mip ? ImageMipMode::DynamicStorage : ImageMipMode::None; +} + +bool IsBuffer(const Instruction& inst) { + return inst.memory.kind == ResourceKind::Buffer || + (inst.memory.kind == ResourceKind::ScalarBuffer && inst.op == Opcode::SBufferLoadDword); +} + +bool IsImage(const Instruction& inst) { + return inst.memory.kind == ResourceKind::Image || inst.memory.kind == ResourceKind::ImageUint || + inst.memory.kind == ResourceKind::StorageImage || + inst.memory.kind == ResourceKind::StorageImageUint; +} + +bool IsAddress(const Instruction& inst) { + return inst.op == Opcode::SLoadDword || inst.memory.kind == ResourceKind::Flat || + inst.memory.kind == ResourceKind::Global || inst.memory.kind == ResourceKind::Scratch; +} + +uint32_t ByteExtent(const Instruction& inst) { + const auto bytes = std::max((inst.memory.data_bits + 7u) / 8u, 1u); + const auto count = std::max(inst.memory.data_dwords, 1u); + const auto end = + static_cast(inst.memory.offset) + static_cast(bytes) * count; + return end > UINT32_MAX ? UINT32_MAX : static_cast(end); +} + +bool ContainsUnknown(const ScalarProvenance& provenance, uint32_t id, std::vector* visited, + std::vector* path) { + path->push_back(id); + if (id <= ScalarProvenance::Unknown || id >= provenance.values.size()) { + return true; + } + if ((*visited)[id] != 0) { + path->pop_back(); + return false; + } + (*visited)[id] = 1; + const auto& value = provenance.values[id]; + if (value.op == ScalarValueOp::Phi) { + for (const auto arg: value.phi_args) { + if (ContainsUnknown(provenance, arg, visited, path)) { + return true; + } + } + path->pop_back(); + return false; + } + const auto args = ScalarValueArgCount(value.op); + for (uint32_t i = 0; i < args; i++) { + if (ContainsUnknown(provenance, value.args[i], visited, path)) { + return true; + } + } + path->pop_back(); + return false; +} + +bool IsLoopInvariantValue(const ScalarProvenance& provenance, uint32_t id, + std::vector* visiting) { + if (id <= ScalarProvenance::Unknown || id >= provenance.values.size()) { + return false; + } + if ((*visiting)[id] != 0) { + return true; + } + (*visiting)[id] = 1; + const auto& value = provenance.values[id]; + if (value.op == ScalarValueOp::Phi) { + uint32_t invariant = ScalarProvenance::Undefined; + for (const auto arg: value.phi_args) { + if (arg == id) { + continue; + } + if (invariant == ScalarProvenance::Undefined) { + invariant = arg; + } else if (arg != invariant) { + return false; + } + } + return invariant != ScalarProvenance::Undefined && + IsLoopInvariantValue(provenance, invariant, visiting); + } + for (uint32_t i = 0; i < ScalarValueArgCount(value.op); i++) { + if (!IsLoopInvariantValue(provenance, value.args[i], visiting)) { + return false; + } + } + return true; +} + +bool IsLoopInvariantDescriptor(const ScalarProvenance& provenance, + const DescriptorValue& descriptor) { + std::vector visiting(provenance.values.size()); + for (uint32_t i = 0; i < descriptor.dword_count; i++) { + if (!IsLoopInvariantValue(provenance, descriptor.dwords[i], &visiting)) { + return false; + } + } + return true; +} + +class Tracker { +public: + explicit Tracker(Program& program): m_program(program) {} + + bool Run(std::string* error) { + if (m_program.resource_tracking_complete) { + return Fail(0, error, "resources already tracked"); + } + if (!m_program.srt_plan_complete) { + return Fail(0, error, "SRT plan is not ready"); + } + if (!m_program.srt_patching_complete) { + return Fail(0, error, "SRT reads were not patched"); + } + for (auto& block: m_program.blocks) { + for (auto& inst: block.instructions) { + if (!Collect(&inst, error)) { + return false; + } + } + } + m_program.info = std::move(m_info); + for (const auto& patch: m_patches) { + patch.inst->memory.resource = patch.resource; + patch.inst->memory.sampler = patch.sampler; + patch.inst->memory.resource_source = ScalarProvenance::Undefined; + patch.inst->memory.sampler_source = ScalarProvenance::Undefined; + } + m_program.resource_tracking_complete = true; + return true; + } + +private: + struct Patch { + Instruction* inst = nullptr; + uint32_t resource = 0; + uint32_t sampler = 0; + }; + + bool Fail(uint32_t pc, std::string* error, const std::string& reason) const { + if (error != nullptr) { + *error = fmt::format("shader resource tracking: hash=0x{:016x} stage={} pc=0x{:08x} {}", + m_program.shader_hash, StageName(m_program.stage), pc, reason); + } + return false; + } + + bool ValidateSource(uint32_t source, uint32_t dwords, uint32_t pc, std::string* error) const { + const auto* descriptor = GetDescriptorSource(m_program, source); + if (descriptor == nullptr || descriptor->dword_count != dwords) { + return Fail(pc, error, + fmt::format("descriptor source {} is missing or has wrong width", source)); + } + std::vector visited(m_program.provenance.values.size()); + for (uint32_t i = 0; i < descriptor->dword_count; i++) { + std::vector path; + if (ContainsUnknown(m_program.provenance, descriptor->dwords[i], &visited, &path)) { + const auto value = descriptor->dwords[i]; + std::string chain; + for (const auto id: path) { + const auto op = id < m_program.provenance.values.size() + ? static_cast(m_program.provenance.values[id].op) + : UINT32_MAX; + chain += fmt::format("{}{}:{}({})", chain.empty() ? "" : " -> ", id, op, + ScalarValueToString(m_program.provenance, id)); + } + return Fail( + pc, error, + fmt::format( + "descriptor source {} dword {} contains an unknown value {} ({}) path {}", + source, i, value, ScalarValueToString(m_program.provenance, value), chain)); + } + } + const auto dynamic = + std::find(m_program.srt.dynamic_sources.begin(), m_program.srt.dynamic_sources.end(), + source) != m_program.srt.dynamic_sources.end(); + if (dynamic && !IsLoopInvariantDescriptor(m_program.provenance, *descriptor)) { + std::string detail; + for (uint32_t i = 0; i < descriptor->dword_count; i++) { + const auto id = descriptor->dwords[i]; + detail += + fmt::format(" d{}={}({})", i, id, + id < m_program.provenance.values.size() + ? static_cast(m_program.provenance.values[id].op) + : UINT32_MAX); + if (id < m_program.provenance.values.size()) { + for (const auto arg: m_program.provenance.values[id].phi_args) { + detail += fmt::format( + "/{}({})", arg, + arg < m_program.provenance.values.size() + ? static_cast(m_program.provenance.values[arg].op) + : UINT32_MAX); + } + } + } + return Fail(pc, error, + fmt::format("descriptor source {} requires unsupported GPU selection{}", + source, detail)); + } + if (!DescriptorSourceResolved(m_program, source) && !dynamic) { + return Fail(pc, error, fmt::format("descriptor source {} is unresolved", source)); + } + return true; + } + + uint32_t AddBuffer(const Instruction& inst) { + for (uint32_t i = 0; i < m_info.buffers.size(); i++) { + if (m_info.buffers[i].source == inst.memory.resource_source) { + Merge(&m_info.buffers[i], inst); + return i; + } + } + if (m_info.buffers.size() >= ShaderInfo::MaxBuffers) { + return UINT32_MAX; + } + BufferResource resource; + resource.source = inst.memory.resource_source; + resource.first_use_pc = inst.pc; + Merge(&resource, inst); + m_info.buffers.push_back(resource); + return static_cast(m_info.buffers.size() - 1); + } + + void Merge(BufferResource* resource, const Instruction& inst) const { + resource->first_use_pc = std::min(resource->first_use_pc, inst.pc); + resource->max_byte_extent = std::max(resource->max_byte_extent, ByteExtent(inst)); + resource->read = resource->read || !IsWrite(inst.op) || IsAtomic(inst.op); + resource->written = resource->written || IsWrite(inst.op); + resource->atomic = resource->atomic || IsAtomic(inst.op); + resource->formatted = resource->formatted || inst.memory.formatted; + resource->scalar = resource->scalar || inst.memory.kind == ResourceKind::ScalarBuffer; + } + + uint32_t AddImage(const Instruction& inst) { + for (uint32_t i = 0; i < m_info.images.size(); i++) { + auto& image = m_info.images[i]; + if (image.source == inst.memory.resource_source && image.kind == inst.memory.kind && + image.dimension == inst.memory.image_dimension && image.mip_mode == MipMode(inst) && + image.depth_compare == IsDepthCompare(inst)) { + Merge(&image, inst); + return i; + } + } + if (m_info.images.size() >= ShaderInfo::MaxImages) { + return UINT32_MAX; + } + ImageResource image; + image.source = inst.memory.resource_source; + image.first_use_pc = inst.pc; + image.kind = inst.memory.kind; + image.dimension = inst.memory.image_dimension; + image.mip_mode = MipMode(inst); + image.depth_compare = IsDepthCompare(inst); + Merge(&image, inst); + m_info.images.push_back(image); + return static_cast(m_info.images.size() - 1); + } + + uint32_t AddAddress(const Instruction& inst) { + auto immediate = static_cast(inst.memory.offset); + if (inst.memory.kind == ResourceKind::ScalarBuffer) { + immediate = static_cast(static_cast(immediate) & ~3u); + } + const auto min_offset = + inst.memory.resource_source == ScalarProvenance::Unknown ? 0 : std::min(immediate, 0); + for (uint32_t i = 0; i < m_info.addresses.size(); i++) { + auto& address = m_info.addresses[i]; + if (address.source == inst.memory.resource_source && address.kind == inst.memory.kind) { + address.first_use_pc = std::min(address.first_use_pc, inst.pc); + address.min_offset = std::min(address.min_offset, min_offset); + address.read = address.read || !IsWrite(inst.op) || IsAtomic(inst.op); + address.written = address.written || IsWrite(inst.op); + address.atomic = address.atomic || IsAtomic(inst.op); + return i; + } + } + if (m_info.addresses.size() >= ShaderInfo::MaxAddresses) { + return UINT32_MAX; + } + AddressResource address {inst.memory.resource_source, inst.pc, inst.memory.kind, + min_offset}; + address.read = !IsWrite(inst.op) || IsAtomic(inst.op); + address.written = IsWrite(inst.op); + address.atomic = IsAtomic(inst.op); + m_info.addresses.push_back(address); + return static_cast(m_info.addresses.size() - 1); + } + + void Merge(ImageResource* resource, const Instruction& inst) const { + resource->first_use_pc = std::min(resource->first_use_pc, inst.pc); + resource->read = resource->read || !IsWrite(inst.op) || IsAtomic(inst.op); + resource->written = resource->written || IsWrite(inst.op); + resource->atomic = resource->atomic || IsAtomic(inst.op); + } + + uint32_t AddSampler(const Instruction& inst) { + for (uint32_t i = 0; i < m_info.samplers.size(); i++) { + if (m_info.samplers[i].source == inst.memory.sampler_source) { + m_info.samplers[i].first_use_pc = + std::min(m_info.samplers[i].first_use_pc, inst.pc); + return i; + } + } + if (m_info.samplers.size() >= ShaderInfo::MaxSamplers) { + return UINT32_MAX; + } + m_info.samplers.push_back({inst.memory.sampler_source, inst.pc}); + return static_cast(m_info.samplers.size() - 1); + } + + bool AddSampledPair(uint32_t image, uint32_t sampler, uint32_t pc, std::string* error) { + for (auto& pair: m_info.sampled_pairs) { + if (pair.image == image && pair.sampler == sampler) { + pair.first_use_pc = std::min(pair.first_use_pc, pc); + return true; + } + } + if (m_info.sampled_pairs.size() >= ShaderInfo::MaxSampledPairs) { + return Fail(pc, error, "sampled image/sampler pair limit exceeded"); + } + m_info.sampled_pairs.push_back({image, sampler, pc}); + return true; + } + + bool Collect(Instruction* inst, std::string* error) { + if (IsAddress(*inst)) { + const bool unbased = inst->memory.resource_source == ScalarProvenance::Unknown; + if ((!unbased && !ValidateSource(inst->memory.resource_source, 2, inst->pc, error)) || + (unbased && inst->memory.kind != ResourceKind::Flat && + inst->memory.kind != ResourceKind::Global && + inst->memory.kind != ResourceKind::Scratch)) { + return unbased ? Fail(inst->pc, error, "scalar memory base is unresolved") : false; + } + const auto resource = AddAddress(*inst); + if (resource == UINT32_MAX) { + return Fail(inst->pc, error, "address resource limit exceeded"); + } + m_patches.push_back({inst, resource, 0}); + return true; + } + if (!IsBuffer(*inst) && !IsImage(*inst)) { + return true; + } + if (!ValidateSource(inst->memory.resource_source, IsBuffer(*inst) ? 4u : 8u, inst->pc, + error)) { + return false; + } + Patch patch; + patch.inst = inst; + patch.resource = IsBuffer(*inst) ? AddBuffer(*inst) : AddImage(*inst); + if (patch.resource == UINT32_MAX) { + return Fail(inst->pc, error, + IsBuffer(*inst) ? "buffer resource limit exceeded" + : "image resource limit exceeded"); + } + if (NeedsSampler(inst->op)) { + if (!ValidateSource(inst->memory.sampler_source, 4, inst->pc, error)) { + return false; + } + patch.sampler = AddSampler(*inst); + if (patch.sampler == UINT32_MAX) { + return Fail(inst->pc, error, "sampler resource limit exceeded"); + } + if (!AddSampledPair(patch.resource, patch.sampler, inst->pc, error)) { + return false; + } + } + m_patches.push_back(patch); + return true; + } + + Program& m_program; + ShaderInfo m_info; + std::vector m_patches; +}; + +} // namespace + +bool TrackResources(Program* program, std::string* error) { + if (program == nullptr) { + if (error != nullptr) { + *error = "shader resource tracking: hash=0x0000000000000000 stage=unknown " + "pc=0x00000000 invalid program"; + } + return false; + } + return Tracker(*program).Run(error); +} + +} // namespace Libs::Graphics::ShaderRecompiler::IR diff --git a/src/graphics/shader/recompiler/ResourceTracking.h b/src/graphics/shader/recompiler/ResourceTracking.h new file mode 100644 index 000000000..23245a332 --- /dev/null +++ b/src/graphics/shader/recompiler/ResourceTracking.h @@ -0,0 +1,14 @@ +#ifndef EMULATOR_INCLUDE_EMULATOR_GRAPHICS_SHADER_RECOMPILER_RESOURCETRACKING_H_ +#define EMULATOR_INCLUDE_EMULATOR_GRAPHICS_SHADER_RECOMPILER_RESOURCETRACKING_H_ + +#include "graphics/shader/recompiler/ShaderIR.h" + +namespace Libs::Graphics::ShaderRecompiler::IR { + +// Collects immutable resource topology, then replaces descriptor operands with dense indices. +// On failure the program is left unpatched. +bool TrackResources(Program* program, std::string* error); + +} // namespace Libs::Graphics::ShaderRecompiler::IR + +#endif /* EMULATOR_INCLUDE_EMULATOR_GRAPHICS_SHADER_RECOMPILER_RESOURCETRACKING_H_ */ diff --git a/src/graphics/shader/recompiler/ScalarAluOps.cpp b/src/graphics/shader/recompiler/ScalarAluOps.cpp new file mode 100644 index 000000000..2597e20f6 --- /dev/null +++ b/src/graphics/shader/recompiler/ScalarAluOps.cpp @@ -0,0 +1,283 @@ +#include "graphics/shader/recompiler/ScalarAluOps.h" + +#include + +namespace Libs::Graphics::ShaderRecompiler::Decoder { +namespace { + +struct OpcodeMap { + uint32_t opcode = 0; + Opcode ir = Opcode::Unknown; +}; + +constexpr OpcodeMap SOP2_OPS[] = { + {0x00u, Opcode::SAddU32}, {0x01u, Opcode::SSubU32}, {0x02u, Opcode::SAddI32}, + {0x03u, Opcode::SSubI32}, {0x04u, Opcode::SAddcU32}, {0x05u, Opcode::SSubbU32}, + {0x06u, Opcode::SMinI32}, {0x07u, Opcode::SMinU32}, {0x08u, Opcode::SMaxI32}, + {0x09u, Opcode::SMaxU32}, {0x0au, Opcode::SCselectB32}, {0x0bu, Opcode::SCselectB64}, + {0x0eu, Opcode::SAndB32}, {0x0fu, Opcode::SAndB64}, {0x10u, Opcode::SOrB32}, + {0x11u, Opcode::SOrB64}, {0x12u, Opcode::SXorB32}, {0x13u, Opcode::SXorB64}, + {0x14u, Opcode::SAndn2B32}, {0x15u, Opcode::SAndn2B64}, {0x16u, Opcode::SOrn2B32}, + {0x17u, Opcode::SOrn2B64}, {0x18u, Opcode::SNandB32}, {0x19u, Opcode::SNandB64}, + {0x1au, Opcode::SNorB32}, {0x1bu, Opcode::SNorB64}, {0x1cu, Opcode::SXnorB32}, + {0x1du, Opcode::SXnorB64}, {0x1eu, Opcode::SLshlB32}, {0x1fu, Opcode::SLshlB64}, + {0x20u, Opcode::SLshrB32}, {0x21u, Opcode::SLshrB64}, {0x22u, Opcode::SAshrI32}, + {0x24u, Opcode::SBfmB32}, {0x25u, Opcode::SBfmB64}, {0x26u, Opcode::SMulI32}, + {0x27u, Opcode::SBfeU32}, {0x29u, Opcode::SBfeU64}, {0x2eu, Opcode::SLshl1AddU32}, + {0x2fu, Opcode::SLshl2AddU32}, {0x30u, Opcode::SLshl3AddU32}, {0x31u, Opcode::SLshl4AddU32}, + {0x32u, Opcode::SPackLlB32B16}, {0x33u, Opcode::SPackLhB32B16}, {0x34u, Opcode::SPackHhB32B16}, + {0x35u, Opcode::SMulHiU32}, +}; + +constexpr OpcodeMap SOP1_OPS[] = { + {0x03u, Opcode::SMovB32}, + {0x04u, Opcode::SMovB64}, + {0x07u, Opcode::SNotB32}, + {0x08u, Opcode::SNotB64}, + {0x0au, Opcode::SWqmB64}, + {0x0bu, Opcode::SBrevB32}, + {0x0fu, Opcode::SBcnt1I32B32}, + {0x10u, Opcode::SBcnt1I32B64}, + {0x13u, Opcode::SFf1I32B32}, + {0x16u, Opcode::SFlbitI32B64}, + {0x1bu, Opcode::SBitset0B32}, + {0x1du, Opcode::SBitset1B32}, + {0x1fu, Opcode::SGetpcB64}, + {0x20u, Opcode::SSetpcB64}, + {0x24u, Opcode::SAndSaveexecB64}, + {0x28u, Opcode::SOrn2SaveexecB64}, + {0x34u, Opcode::SAbsI32}, + {0x37u, Opcode::SAndn1SaveexecB64}, + {0x3bu, Opcode::SBitreplicateB64B32}, + {0x3cu, Opcode::SAndSaveexecB32}, + {0x44u, Opcode::SAndn1SaveexecB32}, +}; + +constexpr OpcodeMap SOPC_OPS[] = { + {0x00u, Opcode::SCmpEqI32}, {0x01u, Opcode::SCmpLgI32}, {0x02u, Opcode::SCmpGtI32}, + {0x03u, Opcode::SCmpGeI32}, {0x04u, Opcode::SCmpLtI32}, {0x05u, Opcode::SCmpLeI32}, + {0x06u, Opcode::SCmpEqU32}, {0x07u, Opcode::SCmpLgU32}, {0x08u, Opcode::SCmpGtU32}, + {0x09u, Opcode::SCmpGeU32}, {0x0au, Opcode::SCmpLtU32}, {0x0bu, Opcode::SCmpLeU32}, + {0x0cu, Opcode::SBitcmp0B32}, {0x0du, Opcode::SBitcmp1B32}, {0x13u, Opcode::SCmpLgU64}, +}; + +constexpr OpcodeMap SOPK_OPS[] = { + {0x00u, Opcode::SMovkI32}, {0x03u, Opcode::SCmpEqI32}, {0x04u, Opcode::SCmpLgI32}, + {0x05u, Opcode::SCmpGtI32}, {0x06u, Opcode::SCmpGeI32}, {0x07u, Opcode::SCmpLtI32}, + {0x08u, Opcode::SCmpLeI32}, {0x09u, Opcode::SCmpEqU32}, {0x0au, Opcode::SCmpLgU32}, + {0x0bu, Opcode::SCmpGtU32}, {0x0cu, Opcode::SCmpGeU32}, {0x0du, Opcode::SCmpLtU32}, + {0x0eu, Opcode::SCmpLeU32}, {0x0fu, Opcode::SAddI32}, {0x10u, Opcode::SMulkI32}, + {0x13u, Opcode::SSetregB32}, {0x17u, Opcode::SWaitcnt}, {0x18u, Opcode::SWaitcnt}, + {0x19u, Opcode::SWaitcnt}, {0x1au, Opcode::SWaitcnt}, +}; + +constexpr OpcodeMap SOPP_OPS[] = { + {0x00u, Opcode::SNop}, {0x01u, Opcode::SEndpgm}, {0x02u, Opcode::SBranch}, + {0x04u, Opcode::SCbranchScc0}, {0x05u, Opcode::SCbranchScc1}, {0x06u, Opcode::SCbranchVccz}, + {0x07u, Opcode::SCbranchVccnz}, {0x08u, Opcode::SCbranchExecz}, {0x09u, Opcode::SCbranchExecnz}, + {0x0au, Opcode::SBarrier}, {0x0cu, Opcode::SWaitcnt}, {0x0eu, Opcode::SSleep}, + {0x10u, Opcode::SSendmsg}, {0x16u, Opcode::STtraceData}, {0x20u, Opcode::SInstPrefetch}, +}; + +Opcode Lookup(const OpcodeMap* ops, uint32_t count, uint32_t opcode) { + for (uint32_t i = 0; i < count; i++) { + if (ops[i].opcode == opcode) { + return ops[i].ir; + } + } + return Opcode::Unsupported; +} + +bool DecodeBinarySources(uint32_t pc, std::span code, uint32_t word_index, + Instruction* inst, uint32_t ssrc0, uint32_t ssrc1, std::string* error) { + if (!DecodeScalarSource(ssrc0, pc, &inst->src0, error) || + !DecodeScalarSource(ssrc1, pc, &inst->src1, error)) { + return false; + } + inst->src_count = 2; + return ReadLiteralOperands(code, word_index, inst, error); +} + +} // namespace + +bool DecodeSop1(uint32_t pc, std::span code, uint32_t word_index, Instruction* inst, + std::string* error) { + const uint32_t word = code[word_index]; + const uint32_t opcode = (word >> 8u) & 0xffu; + const uint32_t ssrc0 = word & 0xffu; + const uint32_t sdst = (word >> 16u) & 0x7fu; + + inst->pc = pc; + inst->word = word; + inst->family = Family::SOP1; + inst->opcode_id = opcode; + inst->opcode = Lookup(SOP1_OPS, static_cast(std::size(SOP1_OPS)), opcode); + SetRawWords(inst, code, word_index, 1); + + if (inst->opcode == Opcode::Unsupported) { + SetUnsupported(inst, Family::SOP1, opcode, "SOP1 opcode is not implemented"); + return true; + } + + switch (inst->opcode) { + case Opcode::SGetpcB64: + inst->src_count = 0; + return DecodeScalarDestination(sdst, pc, &inst->dst, error); + case Opcode::SSetpcB64: + inst->src_count = 1; + inst->dst.kind = OperandKind::Null; + if (!DecodeScalarSource(ssrc0, pc, &inst->src0, error)) { + return false; + } + return ReadLiteralOperands(code, word_index, inst, error); + default: break; + } + + if (!DecodeScalarSource(ssrc0, pc, &inst->src0, error) || + !DecodeScalarDestination(sdst, pc, &inst->dst, error)) { + return false; + } + inst->src_count = 1; + return ReadLiteralOperands(code, word_index, inst, error); +} + +bool DecodeSop2(uint32_t pc, std::span code, uint32_t word_index, Instruction* inst, + std::string* error) { + const uint32_t word = code[word_index]; + const uint32_t opcode = (word >> 23u) & 0x7fu; + const uint32_t ssrc1 = (word >> 8u) & 0xffu; + const uint32_t ssrc0 = word & 0xffu; + const uint32_t sdst = (word >> 16u) & 0x7fu; + + inst->pc = pc; + inst->word = word; + inst->family = Family::SOP2; + inst->opcode_id = opcode; + inst->opcode = Lookup(SOP2_OPS, static_cast(std::size(SOP2_OPS)), opcode); + SetRawWords(inst, code, word_index, 1); + + if (inst->opcode == Opcode::Unsupported) { + SetUnsupported(inst, Family::SOP2, opcode, "SOP2 opcode is not implemented"); + return true; + } + + if (!DecodeScalarDestination(sdst, pc, &inst->dst, error)) { + return false; + } + return DecodeBinarySources(pc, code, word_index, inst, ssrc0, ssrc1, error); +} + +bool DecodeSopk(uint32_t pc, std::span code, uint32_t word_index, Instruction* inst, + std::string* error) { + const uint32_t word = code[word_index]; + const uint32_t opcode = (word >> 23u) & 0x1fu; + const uint32_t sdst = (word >> 16u) & 0x7fu; + const auto imm = static_cast(word & 0xffffu); + + (void)error; + + inst->pc = pc; + inst->word = word; + inst->family = Family::SOPK; + inst->opcode_id = opcode; + inst->opcode = Lookup(SOPK_OPS, static_cast(std::size(SOPK_OPS)), opcode); + inst->src0.kind = OperandKind::IntegerInlineConstant; + inst->src0.signed_val = imm; + inst->src0.value = static_cast(imm); + inst->src_count = 1; + SetRawWords(inst, code, word_index, 1); + + if (inst->opcode == Opcode::Unsupported) { + SetUnsupported(inst, Family::SOPK, opcode, "SOPK opcode is not implemented"); + return true; + } + + switch (inst->opcode) { + case Opcode::SMovkI32: return DecodeScalarDestination(sdst, pc, &inst->dst, error); + case Opcode::SWaitcnt: { + const uint32_t waitcnt = word & 0xffffu; + inst->dst.kind = OperandKind::Null; + inst->src0.signed_val = static_cast(waitcnt); + inst->src0.value = waitcnt; + inst->src_count = 1; + return true; + } + case Opcode::SSetregB32: + inst->dst.kind = OperandKind::Null; + inst->src1.kind = OperandKind::LiteralConstant; + inst->src1.value = word & 0xffffu; + inst->src1.signed_val = static_cast(imm); + inst->src_count = 2; + return DecodeScalarSource(sdst, pc, &inst->src0, error); + default: break; + } + + inst->src1 = inst->src0; + if (!DecodeScalarSource(sdst, pc, &inst->src0, error)) { + return false; + } + if (inst->opcode == Opcode::SAddI32 || inst->opcode == Opcode::SMulkI32) { + inst->src_count = 2; + return DecodeScalarDestination(sdst, pc, &inst->dst, error); + } + + inst->dst.kind = OperandKind::Scc; + inst->src_count = 2; + return true; +} + +bool DecodeSopc(uint32_t pc, std::span code, uint32_t word_index, Instruction* inst, + std::string* error) { + const uint32_t word = code[word_index]; + const uint32_t ssrc1 = (word >> 8u) & 0xffu; + const uint32_t ssrc0 = word & 0xffu; + const uint32_t opcode = (word >> 16u) & 0x7fu; + + inst->pc = pc; + inst->word = word; + inst->family = Family::SOPC; + inst->opcode_id = opcode; + inst->opcode = Lookup(SOPC_OPS, static_cast(std::size(SOPC_OPS)), opcode); + inst->dst.kind = OperandKind::Scc; + SetRawWords(inst, code, word_index, 1); + + if (inst->opcode == Opcode::Unsupported) { + SetUnsupported(inst, Family::SOPC, opcode, "SOPC opcode is not implemented"); + return true; + } + + return DecodeBinarySources(pc, code, word_index, inst, ssrc0, ssrc1, error); +} + +bool DecodeSopp(uint32_t pc, std::span code, uint32_t word_index, Instruction* inst, + std::string* error) { + const uint32_t word = code[word_index]; + const uint32_t opcode = (word >> 16u) & 0x7fu; + const uint32_t simm = word & 0xffffu; + + (void)error; + + inst->pc = pc; + inst->word = word; + inst->family = Family::SOPP; + inst->opcode_id = opcode; + inst->opcode = Lookup(SOPP_OPS, static_cast(std::size(SOPP_OPS)), opcode); + inst->src0.kind = OperandKind::LiteralConstant; + inst->src0.value = simm; + inst->src0.signed_val = static_cast(simm); + inst->src_count = (inst->opcode == Opcode::SNop || inst->opcode == Opcode::SWaitcnt || + inst->opcode == Opcode::SSleep || inst->opcode == Opcode::SSendmsg || + inst->opcode == Opcode::STtraceData || inst->opcode == Opcode::SInstPrefetch) + ? 1 + : 0; + inst->branch_offset = static_cast(static_cast(simm)) * 4; + inst->branch_target = pc + 4u + static_cast(inst->branch_offset); + SetRawWords(inst, code, word_index, 1); + + if (inst->opcode == Opcode::Unsupported) { + SetUnsupported(inst, Family::SOPP, opcode, "SOPP control-flow opcode is not implemented"); + } + return true; +} + +} // namespace Libs::Graphics::ShaderRecompiler::Decoder diff --git a/src/graphics/shader/recompiler/ScalarAluOps.h b/src/graphics/shader/recompiler/ScalarAluOps.h new file mode 100644 index 000000000..2e7658231 --- /dev/null +++ b/src/graphics/shader/recompiler/ScalarAluOps.h @@ -0,0 +1,21 @@ +#ifndef EMULATOR_INCLUDE_EMULATOR_GRAPHICS_SHADER_RECOMPILER_SCALARALUOPS_H_ +#define EMULATOR_INCLUDE_EMULATOR_GRAPHICS_SHADER_RECOMPILER_SCALARALUOPS_H_ + +#include "graphics/shader/recompiler/ShaderDecoder.h" + +namespace Libs::Graphics::ShaderRecompiler::Decoder { + +bool DecodeSop1(uint32_t pc, std::span code, uint32_t word_index, Instruction* inst, + std::string* error); +bool DecodeSop2(uint32_t pc, std::span code, uint32_t word_index, Instruction* inst, + std::string* error); +bool DecodeSopk(uint32_t pc, std::span code, uint32_t word_index, Instruction* inst, + std::string* error); +bool DecodeSopc(uint32_t pc, std::span code, uint32_t word_index, Instruction* inst, + std::string* error); +bool DecodeSopp(uint32_t pc, std::span code, uint32_t word_index, Instruction* inst, + std::string* error); + +} // namespace Libs::Graphics::ShaderRecompiler::Decoder + +#endif /* EMULATOR_INCLUDE_EMULATOR_GRAPHICS_SHADER_RECOMPILER_SCALARALUOPS_H_ */ diff --git a/src/graphics/shader/recompiler/ScalarProvenance.cpp b/src/graphics/shader/recompiler/ScalarProvenance.cpp new file mode 100644 index 000000000..1266a6be2 --- /dev/null +++ b/src/graphics/shader/recompiler/ScalarProvenance.cpp @@ -0,0 +1,946 @@ +#include "graphics/shader/recompiler/ScalarProvenance.h" + +#include +#include +#include +#include +#include +#include + +namespace Libs::Graphics::ShaderRecompiler::IR { +uint32_t ScalarValueArgCount(ScalarValueOp op) { + switch (op) { + case ScalarValueOp::Not: return 1; + case ScalarValueOp::Add: + case ScalarValueOp::Sub: + case ScalarValueOp::Mul: + case ScalarValueOp::And: + case ScalarValueOp::AndNot: + case ScalarValueOp::Or: + case ScalarValueOp::OrNot: + case ScalarValueOp::Xor: + case ScalarValueOp::ShiftLeft: + case ScalarValueOp::ShiftRight: + case ScalarValueOp::ShiftRightArithmetic: + case ScalarValueOp::BitFieldMaskU32: + case ScalarValueOp::BitFieldMaskU64Low: + case ScalarValueOp::BitFieldMaskU64High: return 2; + case ScalarValueOp::AddCarry: + case ScalarValueOp::Carry: + case ScalarValueOp::SubBorrow: + case ScalarValueOp::Borrow: + case ScalarValueOp::Add3: + case ScalarValueOp::ShiftLeftAdd: + case ScalarValueOp::ShiftLeftAddCarry: + case ScalarValueOp::AddShiftLeft: + case ScalarValueOp::XorAdd: + case ScalarValueOp::ShiftLeftOr: + case ScalarValueOp::ReadConst: return 3; + case ScalarValueOp::ReadConstBuffer: return 5; + default: return 0; + } +} + +namespace { + +constexpr uint32_t ScalarRegisters = 128; +constexpr uint32_t VectorRegisters = 256; + +struct ScalarState { + std::array regs = {}; + std::array address_bases = {}; + uint32_t scc = ScalarProvenance::Undefined; + uint32_t m0 = ScalarProvenance::Undefined; + std::map vector_lanes; + + void Fill(uint32_t value) { + regs.fill(value); + address_bases.fill(value); + scc = value; + m0 = value; + vector_lanes.clear(); + } + + bool operator==(const ScalarState&) const = default; +}; + +struct ValueKey { + ScalarValueOp op = ScalarValueOp::Undefined; + uint32_t imm = 0; + std::array args = {}; + + auto operator<=>(const ValueKey&) const = default; +}; + +struct DescriptorKey { + std::array dwords = {}; + uint32_t dword_count = 0; + + auto operator<=>(const DescriptorKey&) const = default; +}; + +bool ScalarRegister(const Operand& operand, uint32_t& reg) { + if (operand.kind != OperandKind::Register) { + return false; + } + if (operand.reg.file == RegisterFile::Scalar && operand.reg.index < ScalarRegisters) { + reg = operand.reg.index; + return true; + } + if (operand.reg.file == RegisterFile::Vcc && operand.reg.index < 2) { + reg = 106u + operand.reg.index; + return true; + } + return false; +} + +bool VectorRegister(const Operand& operand, uint32_t& reg) { + if (operand.kind != OperandKind::Register || operand.reg.file != RegisterFile::Vector || + operand.reg.index >= VectorRegisters) { + return false; + } + reg = operand.reg.index; + return true; +} + +bool FlatStore(Opcode op) { + switch (op) { + case Opcode::FlatStoreByte: + case Opcode::FlatStoreShort: + case Opcode::FlatStoreDword: return true; + default: return false; + } +} + +bool PairDwordOpcode(Opcode op) { + switch (op) { + case Opcode::MoveU64: + case Opcode::WqmB64: + case Opcode::SaveexecB64: + case Opcode::BitwiseAndU64: + case Opcode::BitwiseAndNotU64: + case Opcode::BitwiseOrU64: + case Opcode::BitwiseOrNotU64: + case Opcode::BitwiseXorU64: + case Opcode::BitwiseNandU64: + case Opcode::BitwiseNorU64: + case Opcode::BitwiseXnorU64: + case Opcode::BitwiseNotU64: + case Opcode::BitFieldMaskU64: + case Opcode::BitFieldExtractU64: + case Opcode::BitReplicateB64B32: + case Opcode::ShiftLeftLogicalU64: + case Opcode::ShiftRightLogicalU64: + case Opcode::SelectU64: return true; + default: return false; + } +} + +bool ScalarMemoryLoad(Opcode op) { + return op == Opcode::SLoadDword || op == Opcode::SBufferLoadDword; +} + +bool ValueResolved(const ScalarProvenance& provenance, uint32_t id, std::vector* visited) { + if (id <= ScalarProvenance::Unknown || id >= provenance.values.size()) { + return false; + } + if ((*visited)[id] == 1) { + return false; + } + if ((*visited)[id] == 2) { + return true; + } + if ((*visited)[id] == 3) { + return false; + } + (*visited)[id] = 1; + const auto& value = provenance.values[id]; + bool resolved = true; + if (value.op == ScalarValueOp::Phi) { + resolved = !value.phi_args.empty() && + std::all_of(value.phi_args.begin(), value.phi_args.end(), [&](uint32_t arg) { + return ValueResolved(provenance, arg, visited); + }); + } else { + const auto args = ScalarValueArgCount(value.op); + for (uint32_t i = 0; i < args && resolved; i++) { + resolved = ValueResolved(provenance, value.args[i], visited); + } + } + (*visited)[id] = resolved ? 2 : 3; + return resolved; +} + +class Builder { +public: + explicit Builder(Program* program): m_program(program), m_graph(program->provenance) {} + + bool Run(std::string* error) { + m_graph = {}; + m_graph.values.resize(2); + m_graph.values[ScalarProvenance::Undefined].op = ScalarValueOp::Undefined; + m_graph.values[ScalarProvenance::Unknown].op = ScalarValueOp::Unknown; + + const auto block_count = m_program->blocks.size(); + m_entry.resize(block_count); + m_exit.resize(block_count); + m_phi.resize(block_count); + m_exit_ready.assign(block_count, false); + m_queued.assign(block_count, false); + for (auto& state: m_entry) { + state.Fill(ScalarProvenance::Undefined); + } + for (auto& state: m_exit) { + state.Fill(ScalarProvenance::Undefined); + } + for (auto& phis: m_phi) { + phis.Fill(ScalarProvenance::Undefined); + } + + m_user_data.Fill(ScalarProvenance::Undefined); + for (uint32_t reg = 0; reg < ScalarRegisters; reg++) { + m_user_data.regs[reg] = + reg >= m_program->user_data_base && + reg - m_program->user_data_base < m_program->user_data_count + ? InternValue({ScalarValueOp::UserData, 0, reg}) + : ScalarProvenance::Unknown; + } + m_user_data.scc = ScalarProvenance::Unknown; + m_user_data.m0 = ScalarProvenance::Unknown; + + if (block_count == 0) { + return true; + } + for (const auto& block: m_program->blocks) { + for (const auto predecessor: block.predecessors) { + if (predecessor >= block_count) { + return Fail(error, fmt::format("scalar provenance has invalid predecessor {}", + predecessor)); + } + } + for (const auto successor: block.successors) { + if (successor >= block_count) { + return Fail(error, fmt::format("scalar provenance has invalid successor {}", + successor)); + } + } + if (!ValidateScalarMemoryGroups(block, error)) { + return false; + } + } + Queue(0); + while (!m_work.empty()) { + const auto block_index = m_work.front(); + m_work.pop_front(); + m_queued[block_index] = false; + + auto next_entry = MergeEntry(block_index); + const bool entry_changed = next_entry != m_entry[block_index]; + m_entry[block_index] = next_entry; + + auto next_exit = Execute(m_program->blocks[block_index], next_entry, false); + const bool was_ready = m_exit_ready[block_index]; + if (was_ready && !entry_changed && next_exit == m_exit[block_index]) { + continue; + } + m_exit[block_index] = next_exit; + m_exit_ready[block_index] = true; + for (const auto successor: m_program->blocks[block_index].successors) { + Queue(successor); + } + } + + for (size_t block = 0; block < block_count; block++) { + Execute(m_program->blocks[block], m_entry[block], true); + } + return true; + } + +private: + bool Fail(std::string* error, std::string text) const { + if (error != nullptr) { + *error = std::move(text); + } + return false; + } + + uint32_t AddValue(ScalarValue value) { + m_graph.values.push_back(std::move(value)); + return static_cast(m_graph.values.size() - 1); + } + + bool ValidateScalarMemoryGroups(const BasicBlock& block, std::string* error) const { + for (size_t index = 0; index < block.instructions.size();) { + const auto& first = block.instructions[index]; + if (!ScalarMemoryLoad(first.op) || first.memory.component_count == 1) { + index++; + continue; + } + const auto count = first.memory.component_count; + uint32_t first_dst = 0; + if (count == 0 || first.memory.component_index != 0 || + index + count > block.instructions.size() || + !ScalarRegister(first.dst, first_dst)) { + return Fail(error, + fmt::format("malformed scalar-memory group at pc=0x{:x}", first.pc)); + } + for (uint32_t component = 0; component < count; component++) { + const auto& inst = block.instructions[index + component]; + uint32_t dst = 0; + if (inst.op != first.op || inst.pc != first.pc || + inst.memory.component_count != count || + inst.memory.component_index != component || + inst.memory.resource != first.memory.resource || + inst.memory.offset != first.memory.offset + component * 4u || + inst.src_count != first.src_count || + !std::equal(std::begin(inst.src), std::end(inst.src), std::begin(first.src)) || + !ScalarRegister(inst.dst, dst) || dst != first_dst + component) { + return Fail(error, + fmt::format("malformed scalar-memory component {} at pc=0x{:x}", + component, first.pc)); + } + } + index += count; + } + return true; + } + + uint32_t InternValue(ScalarValue value) { + const ValueKey key {value.op, value.imm, value.args}; + if (const auto found = m_values.find(key); found != m_values.end()) { + return found->second; + } + const auto id = AddValue(std::move(value)); + m_values.emplace(key, id); + return id; + } + + uint32_t Constant(uint32_t value) { return InternValue({ScalarValueOp::Constant, 0, value}); } + + static uint64_t VectorLaneKey(uint32_t reg, uint32_t lane) { + return (static_cast(reg) << 32u) | lane; + } + + uint32_t OperandValue(const Operand& operand, const ScalarState& state) { + uint32_t reg = 0; + if (ScalarRegister(operand, reg)) { + return state.regs[reg]; + } + if (operand.kind == OperandKind::Register && operand.reg.file == RegisterFile::Scc) { + return state.scc; + } + if (operand.kind == OperandKind::Register && operand.reg.file == RegisterFile::M0) { + return state.m0; + } + if (operand.kind == OperandKind::ImmediateU32 || + operand.kind == OperandKind::PcRelativeU32) { + if (operand.kind == OperandKind::PcRelativeU32) { + return InternValue({ScalarValueOp::PcRelativeLow, 0, operand.imm}); + } + return Constant(operand.imm); + } + return ScalarProvenance::Unknown; + } + + uint32_t Define(const Instruction& inst, ScalarValueOp op, const ScalarState& state, + uint32_t imm = 0) { + ScalarValue node; + node.op = op; + node.pc = inst.pc; + node.imm = imm; + for (uint32_t i = 0; i < inst.src_count && i < node.args.size(); i++) { + node.args[i] = OperandValue(inst.src[i], state); + } + return InternValue(std::move(node)); + } + + uint32_t DefineBorrow(const Instruction& inst, const ScalarState& state) { + ScalarValue node; + node.op = ScalarValueOp::Borrow; + node.pc = inst.pc; + node.args[0] = + inst.src_count > 0 ? OperandValue(inst.src[0], state) : ScalarProvenance::Unknown; + node.args[1] = + inst.src_count > 1 ? OperandValue(inst.src[1], state) : ScalarProvenance::Unknown; + node.args[2] = inst.src_count > 2 ? OperandValue(inst.src[2], state) : Constant(0); + return InternValue(std::move(node)); + } + + bool ConstantOperand(const Operand& operand, const ScalarState& state, uint32_t* value) { + if (value == nullptr) { + return false; + } + const auto id = OperandValue(operand, state); + if (id >= m_graph.values.size() || m_graph.values[id].op != ScalarValueOp::Constant) { + return false; + } + *value = m_graph.values[id].imm; + return true; + } + + uint32_t ReadVectorLane(const Instruction& inst, const ScalarState& state) { + if (inst.src_count < 2 || inst.src[0].kind != OperandKind::Register || + inst.src[0].reg.file != RegisterFile::Vector || + (m_program->wave_size != 32 && m_program->wave_size != 64)) { + return ScalarProvenance::Unknown; + } + uint32_t lane = 0; + if (!ConstantOperand(inst.src[1], state, &lane)) { + return ScalarProvenance::Unknown; + } + const auto found = state.vector_lanes.find( + VectorLaneKey(inst.src[0].reg.index, lane % m_program->wave_size)); + return found != state.vector_lanes.end() ? found->second : ScalarProvenance::Unknown; + } + + void ClearVectorLanes(uint32_t reg, ScalarState* state) { + const auto first = state->vector_lanes.lower_bound(VectorLaneKey(reg, 0)); + const auto last = state->vector_lanes.lower_bound((static_cast(reg) + 1u) << 32u); + state->vector_lanes.erase(first, last); + } + + void WriteVectorDestination(const Instruction& inst, ScalarState* state) { + if (inst.dst.kind != OperandKind::Register || inst.dst.reg.file != RegisterFile::Vector) { + return; + } + const uint32_t reg = inst.dst.reg.index; + uint32_t address_base = ScalarProvenance::Unknown; + if (inst.op == Opcode::MoveU32 || inst.op == Opcode::IAddU32 || + inst.op == Opcode::IAddCarryU32 || inst.op == Opcode::ISubU32) { + for (uint32_t i = 0; i < inst.src_count; i++) { + uint32_t src_reg = 0; + if (VectorRegister(inst.src[i], src_reg) && + state->address_bases[src_reg] > ScalarProvenance::Unknown) { + address_base = state->address_bases[src_reg]; + break; + } + } + if (address_base == ScalarProvenance::Unknown) { + for (uint32_t i = 0; i < inst.src_count; i++) { + uint32_t src_reg = 0; + if (ScalarRegister(inst.src[i], src_reg) && + state->regs[src_reg] > ScalarProvenance::Unknown) { + address_base = state->regs[src_reg]; + break; + } + } + } + } + state->address_bases[reg] = ScalarProvenance::Unknown; + if (inst.op == Opcode::MoveRelDestU32) { + state->vector_lanes.clear(); + return; + } + if (inst.op != Opcode::WriteLaneU32) { + uint32_t dwords = std::max(inst.memory.data_dwords, 1u); + if (PairDwordOpcode(inst.op) || inst.op == Opcode::UMadU64U32) { + dwords = std::max(dwords, 2u); + } + for (uint32_t i = 0; i < dwords && reg <= UINT32_MAX - i; i++) { + ClearVectorLanes(reg + i, state); + if (reg + i < state->address_bases.size()) { + state->address_bases[reg + i] = ScalarProvenance::Unknown; + } + } + state->address_bases[reg] = address_base; + return; + } + if (inst.src_count < 2 || (m_program->wave_size != 32 && m_program->wave_size != 64)) { + ClearVectorLanes(reg, state); + return; + } + uint32_t lane = 0; + if (!ConstantOperand(inst.src[1], *state, &lane)) { + ClearVectorLanes(reg, state); + return; + } + state->vector_lanes[VectorLaneKey(reg, lane % m_program->wave_size)] = + OperandValue(inst.src[0], *state); + } + + uint32_t ReadConst(const Instruction& inst, const ScalarState& state, bool buffer) { + ScalarValue node; + node.op = buffer ? ScalarValueOp::ReadConstBuffer : ScalarValueOp::ReadConst; + node.pc = inst.pc; + const auto base = buffer ? inst.memory.resource * 4u : inst.memory.resource; + if (base + 1u >= ScalarRegisters) { + return ScalarProvenance::Unknown; + } + node.imm = inst.memory.offset; + if (buffer) { + if (base + 3u >= ScalarRegisters) { + return ScalarProvenance::Unknown; + } + for (uint32_t i = 0; i < 4; i++) { + node.args[i] = state.regs[base + i]; + } + node.args[4] = inst.src_count != 0 ? OperandValue(inst.src[0], state) : Constant(0); + } else { + node.args[0] = state.regs[base]; + node.args[1] = state.regs[base + 1u]; + node.args[2] = inst.src_count != 0 ? OperandValue(inst.src[0], state) : Constant(0); + } + return InternValue(std::move(node)); + } + + ScalarValueOp Operation(Opcode op) const { + switch (op) { + case Opcode::IAddU32: + case Opcode::ScalarSignedAddOverflowI32: return ScalarValueOp::Add; + case Opcode::IAddCarryU32: + case Opcode::ScalarAddCarryU32: return ScalarValueOp::AddCarry; + case Opcode::ISubU32: + case Opcode::ScalarSubBorrowU32: + case Opcode::ScalarSignedSubOverflowI32: return ScalarValueOp::Sub; + case Opcode::ScalarSubBorrowCarryU32: return ScalarValueOp::SubBorrow; + case Opcode::IMulU32: return ScalarValueOp::Mul; + case Opcode::BitwiseAndU32: return ScalarValueOp::And; + case Opcode::BitwiseAndNotU32: return ScalarValueOp::AndNot; + case Opcode::BitwiseOrU32: return ScalarValueOp::Or; + case Opcode::BitwiseOrNotU32: return ScalarValueOp::OrNot; + case Opcode::BitwiseXorU32: return ScalarValueOp::Xor; + case Opcode::BitwiseNotU32: return ScalarValueOp::Not; + case Opcode::ShiftLeftLogicalU32: return ScalarValueOp::ShiftLeft; + case Opcode::ShiftRightLogicalU32: return ScalarValueOp::ShiftRight; + case Opcode::ShiftRightArithmeticI32: return ScalarValueOp::ShiftRightArithmetic; + case Opcode::BitFieldMaskU32: return ScalarValueOp::BitFieldMaskU32; + case Opcode::IAdd3U32: return ScalarValueOp::Add3; + case Opcode::ScalarShiftLeftAddCarryU32: + case Opcode::ShiftLeftAddU32: return ScalarValueOp::ShiftLeftAdd; + case Opcode::AddShiftLeftU32: return ScalarValueOp::AddShiftLeft; + case Opcode::XorAddU32: return ScalarValueOp::XorAdd; + case Opcode::ShiftLeftOrU32: return ScalarValueOp::ShiftLeftOr; + default: return ScalarValueOp::Unknown; + } + } + + void UpdateScc(const Instruction& inst, const ScalarState& before, ScalarState* state) { + switch (inst.op) { + case Opcode::ScalarAddCarryU32: + state->scc = Define(inst, ScalarValueOp::Carry, before); + break; + case Opcode::ScalarSubBorrowU32: + case Opcode::ScalarSubBorrowCarryU32: state->scc = DefineBorrow(inst, before); break; + case Opcode::ScalarShiftLeftAddCarryU32: + state->scc = Define(inst, ScalarValueOp::ShiftLeftAddCarry, before); + break; + default: break; + } + } + + void WriteDestination(const Instruction& inst, ScalarState* state) { + if (inst.dst.kind == OperandKind::Register && inst.dst.reg.file == RegisterFile::Scc) { + state->scc = inst.op == Opcode::MoveU32 && inst.src_count != 0 + ? OperandValue(inst.src[0], *state) + : ScalarProvenance::Unknown; + return; + } + if (inst.dst.kind == OperandKind::Register && inst.dst.reg.file == RegisterFile::M0) { + const auto before = *state; + const auto op = Operation(inst.op); + state->m0 = inst.op == Opcode::MoveU32 && inst.src_count != 0 + ? OperandValue(inst.src[0], before) + : op != ScalarValueOp::Unknown ? Define(inst, op, before) + : ScalarProvenance::Unknown; + UpdateScc(inst, before, state); + return; + } + + uint32_t dst = 0; + if (!ScalarRegister(inst.dst, dst)) { + return; + } + + const auto before = *state; + state->regs[dst] = ScalarProvenance::Unknown; + if (PairDwordOpcode(inst.op) && dst + 1 < ScalarRegisters) { + state->regs[dst + 1] = ScalarProvenance::Unknown; + } + + uint32_t value = ScalarProvenance::Unknown; + switch (inst.op) { + case Opcode::MoveU32: + case Opcode::MoveF32Bits: + value = inst.src_count == 0 ? ScalarProvenance::Unknown + : inst.src[0].kind == OperandKind::PcRelativeU32 + ? InternValue({ScalarValueOp::PcRelativeLow, inst.pc, inst.src[0].imm}) + : OperandValue(inst.src[0], before); + if (inst.op == Opcode::MoveU32 && inst.src_count != 0 && + inst.src[0].kind == OperandKind::ImmediateU32 && inst.src[0].imm == 0 && + dst != 0) { + const auto low = before.regs[dst - 1]; + if (low < m_graph.values.size() && + m_graph.values[low].op == ScalarValueOp::PcRelativeLow && + m_graph.values[low].pc == inst.pc) { + value = InternValue( + {ScalarValueOp::PcRelativeHigh, inst.pc, m_graph.values[low].imm}); + } + } + break; + case Opcode::MoveU64: { + uint32_t src = 0; + if (inst.src_count != 0 && dst + 1 < ScalarRegisters) { + if (ScalarRegister(inst.src[0], src) && src + 1 < ScalarRegisters) { + value = before.regs[src]; + state->regs[dst + 1] = before.regs[src + 1]; + } else { + value = OperandValue(inst.src[0], before); + state->regs[dst + 1] = Constant( + inst.src[0].kind == OperandKind::ImmediateU32 && inst.src[0].sext_64 + ? UINT32_MAX + : 0u); + } + } + break; + } + case Opcode::BitFieldMaskU64: + value = Define(inst, ScalarValueOp::BitFieldMaskU64Low, before); + if (dst + 1 < ScalarRegisters) { + state->regs[dst + 1] = Define(inst, ScalarValueOp::BitFieldMaskU64High, before); + } + break; + case Opcode::SLoadDword: value = ReadConst(inst, before, false); break; + case Opcode::SBufferLoadDword: value = ReadConst(inst, before, true); break; + case Opcode::ReadLaneU32: value = ReadVectorLane(inst, before); break; + default: { + const auto op = Operation(inst.op); + if (op != ScalarValueOp::Unknown) { + value = Define(inst, op, before); + } + break; + } + } + state->regs[dst] = value; + UpdateScc(inst, before, state); + + uint32_t dst2 = 0; + if (ScalarRegister(inst.dst2, dst2)) { + state->regs[dst2] = + inst.src_count > 1 ? OperandValue(inst.src[1], before) : ScalarProvenance::Unknown; + } + } + + uint32_t AddDescriptor(DescriptorValue descriptor) { + const DescriptorKey key {descriptor.dwords, descriptor.dword_count}; + if (const auto found = m_descriptors.find(key); found != m_descriptors.end()) { + return found->second; + } + m_graph.descriptors.push_back(descriptor); + const auto id = static_cast(m_graph.descriptors.size() - 1) + 2u; + m_descriptors.emplace(key, id); + return id; + } + + uint32_t AddDescriptor(const ScalarState& state, uint32_t base, uint32_t dwords) { + if (base >= ScalarRegisters || dwords == 0 || dwords > 8 || + base + dwords > ScalarRegisters) { + return ScalarProvenance::Unknown; + } + DescriptorValue descriptor; + descriptor.dword_count = dwords; + for (uint32_t i = 0; i < dwords; i++) { + descriptor.dwords[i] = state.regs[base + i]; + } + return AddDescriptor(descriptor); + } + + uint32_t AddFlatAddressDescriptor(const Instruction& inst, const ScalarState& state) { + const uint32_t first = FlatStore(inst.op) ? 1u : 0u; + if (inst.src_count < first + 2u) { + return ScalarProvenance::Unknown; + } + uint32_t low = 0; + uint32_t high = 0; + if (!VectorRegister(inst.src[first], low) || !VectorRegister(inst.src[first + 1u], high) || + state.address_bases[low] <= ScalarProvenance::Unknown || + state.address_bases[high] <= ScalarProvenance::Unknown) { + return ScalarProvenance::Unknown; + } + DescriptorValue descriptor; + descriptor.dword_count = 2; + descriptor.dwords[0] = state.address_bases[low]; + descriptor.dwords[1] = state.address_bases[high]; + return AddDescriptor(descriptor); + } + + void AttachSources(Instruction* inst, const ScalarState& state) { + inst->memory.resource_source = 0; + inst->memory.sampler_source = 0; + if (inst->op == Opcode::SLoadDword) { + inst->memory.resource_source = AddDescriptor(state, inst->memory.resource, 2); + return; + } + if (inst->memory.kind == ResourceKind::Flat || inst->memory.kind == ResourceKind::Global || + inst->memory.kind == ResourceKind::Scratch) { + inst->memory.resource_source = ScalarProvenance::Unknown; + if (inst->memory.kind == ResourceKind::Flat) { + inst->memory.resource_source = AddFlatAddressDescriptor(*inst, state); + } + for (uint32_t i = 0; i < inst->src_count; i++) { + const auto& source = inst->src[i]; + if (inst->memory.resource_source == ScalarProvenance::Unknown && + source.kind == OperandKind::Register && + source.reg.file == RegisterFile::Scalar) { + inst->memory.resource_source = AddDescriptor(state, source.reg.index, 2); + break; + } + } + return; + } + if (inst->memory.kind == ResourceKind::Buffer || + (inst->memory.kind == ResourceKind::ScalarBuffer && + inst->op == Opcode::SBufferLoadDword)) { + const auto base = inst->memory.resource * 4u; + inst->memory.resource_source = AddDescriptor(state, base, 4); + return; + } + if (inst->memory.kind == ResourceKind::Image || + inst->memory.kind == ResourceKind::ImageUint || + inst->memory.kind == ResourceKind::StorageImage || + inst->memory.kind == ResourceKind::StorageImageUint) { + inst->memory.resource_source = AddDescriptor(state, inst->memory.resource * 4u, 8); + if (inst->op == Opcode::ImageSample || inst->op == Opcode::ImageGather4 || + inst->op == Opcode::ImageGetLod) { + inst->memory.sampler_source = AddDescriptor(state, inst->memory.sampler * 4u, 4); + } + } + } + + void AttachScalarSources(Instruction* inst, const ScalarState& state) { + std::fill(std::begin(inst->scalar_sources), std::end(inst->scalar_sources), + ScalarProvenance::Undefined); + for (uint32_t i = 0; i < inst->src_count; i++) { + const auto& source = inst->src[i]; + if (source.kind == OperandKind::Register && source.reg.file != RegisterFile::Vector && + source.reg.file != RegisterFile::Exec) { + inst->scalar_sources[i] = OperandValue(source, state); + } + } + } + + ScalarState Execute(BasicBlock& block, ScalarState state, bool attach_sources) { + for (size_t index = 0; index < block.instructions.size();) { + auto& inst = block.instructions[index]; + uint32_t first_dst = 0; + if (ScalarMemoryLoad(inst.op) && inst.memory.component_count > 1 && + ScalarRegister(inst.dst, first_dst)) { + const auto before = state; + for (uint32_t component = 0; component < inst.memory.component_count; component++) { + auto& component_inst = block.instructions[index + component]; + if (attach_sources) { + component_inst.scalar_value = ScalarProvenance::Undefined; + AttachScalarSources(&component_inst, before); + AttachSources(&component_inst, before); + } + auto component_state = before; + WriteDestination(component_inst, &component_state); + uint32_t dst = 0; + ScalarRegister(component_inst.dst, dst); + state.regs[dst] = component_state.regs[dst]; + if (attach_sources) { + component_inst.scalar_value = component_state.regs[dst]; + } + } + index += inst.memory.component_count; + continue; + } + if (attach_sources) { + inst.scalar_value = ScalarProvenance::Undefined; + AttachScalarSources(&inst, state); + AttachSources(&inst, state); + } + WriteVectorDestination(inst, &state); + WriteDestination(inst, &state); + if (attach_sources) { + uint32_t dst = 0; + if (ScalarRegister(inst.dst, dst)) { + inst.scalar_value = state.regs[dst]; + } + } + index++; + } + return state; + } + + ScalarState MergeEntry(size_t block_index) { + ScalarState result; + result.Fill(ScalarProvenance::Undefined); + const auto& block = m_program->blocks[block_index]; + const auto merge = [&](uint32_t initial, auto predecessor_value, uint32_t* phi) { + std::vector incoming; + if (initial != ScalarProvenance::Undefined) { + incoming.push_back(initial); + } + for (const auto predecessor: block.predecessors) { + const auto value = predecessor_value(predecessor, m_exit[predecessor]); + if (value != ScalarProvenance::Undefined && + std::find(incoming.begin(), incoming.end(), value) == incoming.end()) { + incoming.push_back(value); + } + } + if (incoming.empty()) { + return ScalarProvenance::Undefined; + } + if (incoming.size() == 1) { + return incoming[0]; + } + if (*phi == ScalarProvenance::Undefined) { + *phi = AddValue({ScalarValueOp::Phi, block.start_pc}); + } + m_graph.values[*phi].phi_args = std::move(incoming); + return *phi; + }; + + for (uint32_t reg = 0; reg < ScalarRegisters; reg++) { + const auto initial = + block_index == 0 ? m_user_data.regs[reg] : ScalarProvenance::Undefined; + result.regs[reg] = merge( + initial, [reg](uint32_t, const ScalarState& state) { return state.regs[reg]; }, + &m_phi[block_index].regs[reg]); + } + for (uint32_t reg = 0; reg < VectorRegisters; reg++) { + result.address_bases[reg] = merge( + block_index == 0 ? ScalarProvenance::Unknown : ScalarProvenance::Undefined, + [reg](uint32_t, const ScalarState& state) { return state.address_bases[reg]; }, + &m_phi[block_index].address_bases[reg]); + } + result.scc = merge( + block_index == 0 ? m_user_data.scc : ScalarProvenance::Undefined, + [](uint32_t, const ScalarState& state) { return state.scc; }, &m_phi[block_index].scc); + result.m0 = merge( + block_index == 0 ? m_user_data.m0 : ScalarProvenance::Undefined, + [](uint32_t, const ScalarState& state) { return state.m0; }, &m_phi[block_index].m0); + + std::vector vector_lanes; + for (const auto predecessor: block.predecessors) { + for (const auto& [lane, value]: m_exit[predecessor].vector_lanes) { + (void)value; + vector_lanes.push_back(lane); + } + } + std::sort(vector_lanes.begin(), vector_lanes.end()); + vector_lanes.erase(std::unique(vector_lanes.begin(), vector_lanes.end()), + vector_lanes.end()); + for (const auto lane: vector_lanes) { + auto& phi = m_phi[block_index].vector_lanes[lane]; + result.vector_lanes[lane] = merge( + block_index == 0 ? ScalarProvenance::Unknown : ScalarProvenance::Undefined, + [this, lane](uint32_t predecessor, const ScalarState& state) { + if (!m_exit_ready[predecessor]) { + return ScalarProvenance::Undefined; + } + const auto found = state.vector_lanes.find(lane); + return found != state.vector_lanes.end() ? found->second + : ScalarProvenance::Unknown; + }, + &phi); + } + return result; + } + + void Queue(uint32_t block) { + if (!m_queued[block]) { + m_queued[block] = true; + m_work.push_back(block); + } + } + + Program* m_program; + ScalarProvenance& m_graph; + ScalarState m_user_data; + std::vector m_entry; + std::vector m_exit; + std::vector m_phi; + std::vector m_exit_ready; + std::vector m_queued; + std::deque m_work; + std::map m_values; + std::map m_descriptors; +}; + +} // namespace + +bool BuildScalarProvenance(Program* program, std::string* error) { + if (program == nullptr) { + if (error != nullptr) { + *error = "invalid scalar provenance program"; + } + return false; + } + if (program->resource_tracking_complete) { + if (error != nullptr) { + *error = "cannot rebuild scalar provenance after resource tracking"; + } + return false; + } + if (program->user_data_count > 64) { + if (error != nullptr) { + *error = "scalar provenance user-data count exceeds 64 SGPRs"; + } + return false; + } + if (program->user_data_base > ScalarRegisters || + program->user_data_count > ScalarRegisters - program->user_data_base) { + if (error != nullptr) { + *error = "scalar provenance user-data window is out of range"; + } + return false; + } + if (program->srt_patching_complete) { + if (error != nullptr) { + *error = "cannot rebuild scalar provenance after SRT patching"; + } + return false; + } + program->srt = {}; + program->srt_plan_complete = false; + return Builder(program).Run(error); +} + +const DescriptorValue* GetDescriptorSource(const Program& program, uint32_t source) { + if (source < 2 || source - 2 >= program.provenance.descriptors.size()) { + return nullptr; + } + return &program.provenance.descriptors[source - 2]; +} + +bool DescriptorSourceResolved(const Program& program, uint32_t source) { + const auto* descriptor = GetDescriptorSource(program, source); + if (descriptor == nullptr) { + return false; + } + std::vector visited(program.provenance.values.size()); + for (uint32_t i = 0; i < descriptor->dword_count; i++) { + if (!ValueResolved(program.provenance, descriptor->dwords[i], &visited)) { + return false; + } + } + return true; +} + +std::string ScalarValueToString(const ScalarProvenance& provenance, uint32_t value) { + if (value >= provenance.values.size()) { + return "invalid"; + } + const auto& node = provenance.values[value]; + switch (node.op) { + case ScalarValueOp::Undefined: return "undefined"; + case ScalarValueOp::Unknown: return "unknown"; + case ScalarValueOp::UserData: return fmt::format("ud{}", node.imm); + case ScalarValueOp::Constant: return fmt::format("0x{:08x}", node.imm); + case ScalarValueOp::PcRelativeLow: return fmt::format("pc_lo+0x{:x}", node.imm); + case ScalarValueOp::PcRelativeHigh: return fmt::format("pc_hi+0x{:x}", node.imm); + case ScalarValueOp::ReadConst: + return fmt::format("read_const({}, {}, {}, +{})", node.args[0], node.args[1], + node.args[2], node.imm); + case ScalarValueOp::ReadConstBuffer: + return fmt::format("read_const_buffer({}, {}, {}, {}, {}, +{})", node.args[0], + node.args[1], node.args[2], node.args[3], node.args[4], node.imm); + case ScalarValueOp::Phi: return fmt::format("phi{}", value); + case ScalarValueOp::BitFieldMaskU32: + return fmt::format("bfm_u32({}, {})", node.args[0], node.args[1]); + default: return fmt::format("value{}", value); + } +} + +} // namespace Libs::Graphics::ShaderRecompiler::IR diff --git a/src/graphics/shader/recompiler/ScalarProvenance.h b/src/graphics/shader/recompiler/ScalarProvenance.h new file mode 100644 index 000000000..5657fb364 --- /dev/null +++ b/src/graphics/shader/recompiler/ScalarProvenance.h @@ -0,0 +1,20 @@ +#ifndef EMULATOR_INCLUDE_EMULATOR_GRAPHICS_SHADER_RECOMPILER_SCALARPROVENANCE_H_ +#define EMULATOR_INCLUDE_EMULATOR_GRAPHICS_SHADER_RECOMPILER_SCALARPROVENANCE_H_ + +#include "graphics/shader/recompiler/ShaderIR.h" + +namespace Libs::Graphics::ShaderRecompiler::IR { + +// Builds scalar reaching definitions and attaches descriptor sources to individual memory uses. +bool BuildScalarProvenance(Program* program, std::string* error); + +uint32_t ScalarValueArgCount(ScalarValueOp op); + +const DescriptorValue* GetDescriptorSource(const Program& program, uint32_t source); +bool DescriptorSourceResolved(const Program& program, uint32_t source); + +std::string ScalarValueToString(const ScalarProvenance& provenance, uint32_t value); + +} // namespace Libs::Graphics::ShaderRecompiler::IR + +#endif /* EMULATOR_INCLUDE_EMULATOR_GRAPHICS_SHADER_RECOMPILER_SCALARPROVENANCE_H_ */ diff --git a/src/graphics/shader/recompiler/ShaderCFG.cpp b/src/graphics/shader/recompiler/ShaderCFG.cpp new file mode 100644 index 000000000..4fe5ab4d8 --- /dev/null +++ b/src/graphics/shader/recompiler/ShaderCFG.cpp @@ -0,0 +1,1539 @@ +#include "graphics/shader/recompiler/ShaderCFG.h" + +#include +#include +#include +#include +#include +#include + +namespace Libs::Graphics::ShaderRecompiler::CFG { +namespace { + +using Decoder::Instruction; +using Decoder::Opcode; + +struct SetpcTargetInfo { + uint32_t target = 0; + bool indirect = false; + uint32_t pc_sgpr = UINT32_MAX; + uint32_t selector_code = UINT32_MAX; + uint32_t table_load_pc = UINT32_MAX; + std::vector target_pcs; + std::vector selector_values; + std::vector selector_target_pcs; +}; + +void SetError(std::string* error, const std::string& message) { + if (error != nullptr) { + *error = message; + } +} + +void SetFailure(Graph* graph, FailureKind kind, uint32_t block_id, const std::string& message, + std::string* error) { + if (graph != nullptr) { + graph->unsupported = true; + graph->failure_kind = kind; + graph->failure_block = block_id; + graph->unsupported_reason = message; + } + SetError(error, message); +} + +uint32_t InstructionEndPc(const Instruction& inst) { + return inst.pc + inst.word_count * 4u; +} + +uint32_t ProgramEndPc(const Decoder::Program& program) { + if (program.instructions.empty()) { + return 0; + } + return InstructionEndPc(program.instructions.back()); +} + +bool IsUnconditionalBranch(Opcode opcode) { + return opcode == Opcode::SBranch; +} + +bool IsConditionalBranch(Opcode opcode) { + switch (opcode) { + case Opcode::SCbranchScc0: + case Opcode::SCbranchScc1: + case Opcode::SCbranchVccz: + case Opcode::SCbranchVccnz: + case Opcode::SCbranchExecz: + case Opcode::SCbranchExecnz: return true; + default: return false; + } +} + +bool IsBranch(Opcode opcode) { + return IsUnconditionalBranch(opcode) || IsConditionalBranch(opcode); +} + +BranchCondition ConditionForOpcode(Opcode opcode) { + switch (opcode) { + case Opcode::SBranch: return BranchCondition::Always; + case Opcode::SCbranchScc0: return BranchCondition::SccZero; + case Opcode::SCbranchScc1: return BranchCondition::SccNonZero; + case Opcode::SCbranchVccz: return BranchCondition::VccZero; + case Opcode::SCbranchVccnz: return BranchCondition::VccNonZero; + case Opcode::SCbranchExecz: return BranchCondition::ExecZero; + case Opcode::SCbranchExecnz: return BranchCondition::ExecNonZero; + default: return BranchCondition::Unknown; + } +} + +bool IsRegister(const Decoder::Operand& operand, Decoder::OperandKind kind, uint32_t reg) { + return operand.kind == kind && operand.reg == reg; +} + +bool ScalarOperandCode(const Decoder::Operand& operand, uint32_t* code) { + if (code == nullptr) { + return false; + } + switch (operand.kind) { + case Decoder::OperandKind::Sgpr: *code = operand.reg; return true; + case Decoder::OperandKind::VccLo: *code = 106u; return true; + case Decoder::OperandKind::VccHi: *code = 107u; return true; + case Decoder::OperandKind::M0: *code = 124u; return true; + case Decoder::OperandKind::ExecLo: *code = 126u; return true; + case Decoder::OperandKind::ExecHi: *code = 127u; return true; + default: return false; + } +} + +bool IsScalarCode(const Decoder::Operand& operand, uint32_t code) { + uint32_t operand_code = 0; + return ScalarOperandCode(operand, &operand_code) && operand_code == code; +} + +bool IsImmediate(const Decoder::Operand& operand, uint32_t* value) { + if (operand.kind == Decoder::OperandKind::IntegerInlineConstant || + operand.kind == Decoder::OperandKind::LiteralConstant || + operand.kind == Decoder::OperandKind::FloatInlineConstant) { + if (value != nullptr) { + *value = operand.value; + } + return true; + } + return false; +} + +bool IsImmediateSigned(const Decoder::Operand& operand, int32_t* value) { + if (operand.kind == Decoder::OperandKind::IntegerInlineConstant || + operand.kind == Decoder::OperandKind::LiteralConstant) { + if (value != nullptr) { + *value = operand.signed_val; + } + return true; + } + if (operand.kind == Decoder::OperandKind::FloatInlineConstant) { + if (value != nullptr) { + *value = static_cast(operand.value); + } + return true; + } + return false; +} + +bool OtherScalarSource(const Instruction& inst, uint32_t known_code, + const Decoder::Operand** other) { + if (other == nullptr) { + return false; + } + if (IsScalarCode(inst.src0, known_code)) { + *other = &inst.src1; + return true; + } + if (IsScalarCode(inst.src1, known_code)) { + *other = &inst.src0; + return true; + } + return false; +} + +bool InstructionWritesScalarCode(const Instruction& inst, uint32_t code) { + uint32_t dst_code = 0; + if (!ScalarOperandCode(inst.dst, &dst_code)) { + return false; + } + const uint32_t count = std::max(1u, inst.data_dwords); + return code >= dst_code && code < dst_code + count; +} + +bool FindPreviousGetpc(const Decoder::Program& program, uint32_t before_index, uint32_t dst_code, + uint32_t* index) { + for (uint32_t i = before_index; i > 0; i--) { + const uint32_t candidate_index = i - 1u; + const auto& candidate = program.instructions[candidate_index]; + if (candidate.opcode == Opcode::SGetpcB64 && IsScalarCode(candidate.dst, dst_code)) { + if (index != nullptr) { + *index = candidate_index; + } + return true; + } + if (InstructionWritesScalarCode(candidate, dst_code)) { + return false; + } + } + return false; +} + +bool ResolvePcRelativeBase(const Decoder::Program& program, uint32_t before_index, + uint32_t base_code, uint32_t* pc) { + if (pc == nullptr) { + return false; + } + for (uint32_t i = before_index; i > 0; i--) { + const uint32_t candidate_index = i - 1u; + const auto& candidate = program.instructions[candidate_index]; + if (!InstructionWritesScalarCode(candidate, base_code)) { + continue; + } + const bool adds = + candidate.opcode == Opcode::SAddU32 || candidate.opcode == Opcode::SAddI32; + const bool subs = + candidate.opcode == Opcode::SSubU32 || candidate.opcode == Opcode::SSubI32; + if (!adds && !subs) { + return false; + } + + const Decoder::Operand* offset = nullptr; + int32_t imm = 0; + uint32_t getpc_index = 0; + if (!OtherScalarSource(candidate, base_code, &offset) || + !IsImmediateSigned(*offset, &imm) || + !FindPreviousGetpc(program, candidate_index, base_code, &getpc_index)) { + return false; + } + + const auto base = InstructionEndPc(program.instructions[getpc_index]); + *pc = adds ? base + static_cast(imm) : base - static_cast(imm); + *pc &= ~3u; + return true; + } + return false; +} + +bool FindPreviousScalarLoadPair(const Decoder::Program& program, uint32_t before_index, + uint32_t dst_code, uint32_t* index) { + for (uint32_t i = before_index; i > 0; i--) { + const uint32_t candidate_index = i - 1u; + const auto& candidate = program.instructions[candidate_index]; + if (InstructionWritesScalarCode(candidate, dst_code)) { + if (candidate.opcode == Opcode::SLoadDwordx2 && IsScalarCode(candidate.dst, dst_code)) { + if (index != nullptr) { + *index = candidate_index; + } + return true; + } + return false; + } + } + return false; +} + +bool ResolveJumpTableEntryCount(const Decoder::Program& program, uint32_t before_index, + const Decoder::Operand& byte_offset_operand, + uint32_t* entry_count) { + if (entry_count == nullptr) { + return false; + } + uint32_t byte_offset_code = 0; + if (!ScalarOperandCode(byte_offset_operand, &byte_offset_code)) { + return false; + } + + for (uint32_t i = before_index; i > 0; i--) { + const uint32_t shift_index = i - 1u; + const auto& shift = program.instructions[shift_index]; + if (!InstructionWritesScalarCode(shift, byte_offset_code)) { + continue; + } + if (shift.opcode != Opcode::SLshlB32 || !IsScalarCode(shift.dst, byte_offset_code)) { + return false; + } + + const Decoder::Operand* shift_amount_operand = nullptr; + uint32_t shift_amount = 0; + if (!OtherScalarSource(shift, byte_offset_code, &shift_amount_operand) || + !IsImmediate(*shift_amount_operand, &shift_amount) || shift_amount != 3u) { + return false; + } + + uint32_t index_code = byte_offset_code; + + for (uint32_t j = shift_index; j > 0; j--) { + const uint32_t clamp_index = j - 1u; + const auto& clamp = program.instructions[clamp_index]; + if (!InstructionWritesScalarCode(clamp, index_code)) { + continue; + } + if (clamp.opcode != Opcode::SMinU32 || !IsScalarCode(clamp.dst, index_code)) { + return false; + } + + const Decoder::Operand* clamp_other = nullptr; + uint32_t max_index = 0; + if (!OtherScalarSource(clamp, index_code, &clamp_other) || + !IsImmediate(*clamp_other, &max_index)) { + return false; + } + *entry_count = max_index + 1u; + return *entry_count != 0; + } + return false; + } + return false; +} + +bool AddUniqueTargetPc(std::vector* targets, uint32_t target) { + if (targets == nullptr) { + return false; + } + if (std::find(targets->begin(), targets->end(), target) == targets->end()) { + targets->push_back(target); + } + return true; +} + +bool ScalarCodeWrittenInRange(const Decoder::Program& program, uint32_t begin_index, + uint32_t end_index, uint32_t code) { + const auto end = + std::min(end_index, static_cast(program.instructions.size())); + for (uint32_t i = begin_index; i < end; i++) { + if (InstructionWritesScalarCode(program.instructions[i], code)) { + return true; + } + } + return false; +} + +bool ResolveSetpcJumpTable(const Decoder::Program& program, uint32_t setpc_index, + SetpcTargetInfo* info) { + if (info == nullptr || setpc_index < 4u || setpc_index >= program.instructions.size()) { + return false; + } + + const auto& setpc = program.instructions[setpc_index]; + uint32_t pc_reg = 0; + if (setpc.opcode != Opcode::SSetpcB64 || !ScalarOperandCode(setpc.src0, &pc_reg) || + setpc.src0.kind != Decoder::OperandKind::Sgpr) { + return false; + } + + const auto& getpc = program.instructions[setpc_index - 3u]; + const auto& add = program.instructions[setpc_index - 2u]; + const auto& addc = program.instructions[setpc_index - 1u]; + if (getpc.opcode != Opcode::SGetpcB64 || !IsScalarCode(getpc.dst, pc_reg) || + add.opcode != Opcode::SAddU32 || !IsScalarCode(add.dst, pc_reg) || + addc.opcode != Opcode::SAddcU32 || !IsScalarCode(addc.dst, pc_reg + 1u)) { + return false; + } + + const Decoder::Operand* offset_low_operand = nullptr; + const Decoder::Operand* offset_high_operand = nullptr; + uint32_t offset_low_code = 0; + uint32_t offset_high_code = 0; + if (!OtherScalarSource(add, pc_reg, &offset_low_operand) || + !OtherScalarSource(addc, pc_reg + 1u, &offset_high_operand) || + !ScalarOperandCode(*offset_low_operand, &offset_low_code) || + !ScalarOperandCode(*offset_high_operand, &offset_high_code) || + offset_high_code != offset_low_code + 1u) { + return false; + } + + uint32_t load_index = 0; + if (!FindPreviousScalarLoadPair(program, setpc_index - 3u, offset_low_code, &load_index)) { + return false; + } + const auto& load = program.instructions[load_index]; + uint32_t table_base_code = 0; + if (!ScalarOperandCode(load.src0, &table_base_code)) { + return false; + } + + uint32_t table_pc = 0; + if (!ResolvePcRelativeBase(program, load_index, table_base_code, &table_pc)) { + return false; + } + + uint32_t entry_count = 0; + if (!ResolveJumpTableEntryCount(program, load_index, load.src1, &entry_count)) { + return false; + } + uint32_t selector_code = UINT32_MAX; + if (!ScalarOperandCode(load.src1, &selector_code) || + ScalarCodeWrittenInRange(program, load_index + 1u, setpc_index, selector_code)) { + return false; + } + + const uint32_t target_base = InstructionEndPc(getpc); + const uint32_t table_word = table_pc / 4u; + if ((table_pc & 3u) != 0 || table_word + entry_count * 2u > program.code.size()) { + return false; + } + + std::vector targets; + std::vector selector_values; + std::vector selector_target_pcs; + for (uint32_t i = 0; i < entry_count; i++) { + const uint32_t low = program.code[table_word + i * 2u]; + const uint32_t high = program.code[table_word + i * 2u + 1u]; + if (high != 0u && high != UINT32_MAX) { + return false; + } + const auto target_pc = (target_base + low) & ~3u; + AddUniqueTargetPc(&targets, target_pc); + selector_values.push_back(i * 8u); + selector_target_pcs.push_back(target_pc); + } + if (targets.empty()) { + return false; + } + + info->indirect = true; + info->pc_sgpr = pc_reg; + info->selector_code = selector_code; + info->table_load_pc = load.pc; + info->target_pcs = std::move(targets); + info->selector_values = std::move(selector_values); + info->selector_target_pcs = std::move(selector_target_pcs); + return true; +} + +bool ResolveSetpcTarget(const Decoder::Program& program, uint32_t setpc_index, uint32_t* target) { + if (target == nullptr || setpc_index >= program.instructions.size()) { + return false; + } + + const auto& setpc = program.instructions[setpc_index]; + if (setpc.opcode != Opcode::SSetpcB64 || setpc.src0.kind != Decoder::OperandKind::Sgpr) { + return false; + } + + const auto pc_reg = setpc.src0.reg; + if (setpc_index >= 2u) { + const auto& arith = program.instructions[setpc_index - 1u]; + const auto& getpc = program.instructions[setpc_index - 2u]; + if (getpc.opcode == Opcode::SGetpcB64 && getpc.dst.kind == Decoder::OperandKind::Sgpr && + getpc.dst.reg == pc_reg && arith.dst.kind == Decoder::OperandKind::Sgpr && + arith.dst.reg == pc_reg) { + uint32_t imm = 0; + bool adds = arith.opcode == Opcode::SAddU32 || arith.opcode == Opcode::SAddI32; + bool subs = arith.opcode == Opcode::SSubU32 || arith.opcode == Opcode::SSubI32; + if ((adds || subs) && + (IsRegister(arith.src0, Decoder::OperandKind::Sgpr, pc_reg) || + IsRegister(arith.src1, Decoder::OperandKind::Sgpr, pc_reg)) && + (IsImmediate(arith.src0, &imm) || IsImmediate(arith.src1, &imm))) { + const auto base = InstructionEndPc(getpc); + *target = adds ? base + imm : base - imm; + *target &= ~3u; + return true; + } + } + } + + if (setpc_index >= 1u) { + const auto& getpc = program.instructions[setpc_index - 1u]; + if (getpc.opcode == Opcode::SGetpcB64 && getpc.dst.kind == Decoder::OperandKind::Sgpr && + getpc.dst.reg == pc_reg) { + *target = InstructionEndPc(getpc); + return true; + } + } + + return false; +} + +bool ResolveSetpcTargets(const Decoder::Program& program, uint32_t setpc_index, + SetpcTargetInfo* info) { + if (info == nullptr) { + return false; + } + *info = {}; + if (ResolveSetpcJumpTable(program, setpc_index, info)) { + return true; + } + uint32_t target = 0; + if (ResolveSetpcTarget(program, setpc_index, &target)) { + info->target = target; + return true; + } + return false; +} + +bool IsValidTarget(uint32_t target, const std::set& instruction_pcs, uint32_t first_pc, + uint32_t end_pc) { + return target == end_pc || (target >= first_pc && instruction_pcs.contains(target)); +} + +void AddUnique(std::vector* values, uint32_t value) { + if (std::find(values->begin(), values->end(), value) == values->end()) { + values->push_back(value); + } +} + +std::vector AllBlockIds(uint32_t count) { + std::vector ids; + ids.reserve(count); + for (uint32_t i = 0; i < count; i++) { + ids.push_back(i); + } + return ids; +} + +std::vector IntersectSorted(const std::vector& a, + const std::vector& b) { + std::vector ret; + ret.reserve(std::min(a.size(), b.size())); + std::set_intersection(a.begin(), a.end(), b.begin(), b.end(), std::back_inserter(ret)); + return ret; +} + +void SortUnique(std::vector* values) { + std::sort(values->begin(), values->end()); + values->erase(std::unique(values->begin(), values->end()), values->end()); +} + +bool Contains(const std::vector& values, uint32_t value) { + return std::find(values.begin(), values.end(), value) != values.end(); +} + +bool ReplaceValue(std::vector* values, uint32_t old_value, uint32_t new_value) { + bool changed = false; + for (auto& value: *values) { + if (value == old_value) { + value = new_value; + changed = true; + } + } + if (changed) { + SortUnique(values); + } + return changed; +} + +bool RemoveValue(std::vector* values, uint32_t value) { + const auto old_size = values->size(); + values->erase(std::remove(values->begin(), values->end(), value), values->end()); + return values->size() != old_size; +} + +bool ReplaceTerminatorTarget(Terminator* terminator, uint32_t old_value, uint32_t new_value) { + bool changed = false; + if (terminator->true_block == old_value) { + terminator->true_block = new_value; + changed = true; + } + if (terminator->false_block == old_value) { + terminator->false_block = new_value; + changed = true; + } + return changed; +} + +uint32_t RemapId(uint32_t id, const std::vector& id_map) { + return id != UINT32_MAX && id < id_map.size() ? id_map[id] : id; +} + +void RemapIds(std::vector* values, const std::vector& id_map) { + for (auto& value: *values) { + value = RemapId(value, id_map); + } + SortUnique(values); +} + +void RebuildPredecessors(Graph* graph) { + for (auto& block: graph->blocks) { + block.predecessors.clear(); + SortUnique(&block.successors); + } + for (const auto& block: graph->blocks) { + for (auto succ: block.successors) { + if (succ < graph->blocks.size()) { + AddUnique(&graph->blocks[succ].predecessors, block.id); + } + } + } + for (auto& block: graph->blocks) { + SortUnique(&block.predecessors); + } +} + +void ComputeDominators(Graph* graph) { + const auto count = static_cast(graph->blocks.size()); + const auto all = AllBlockIds(count); + + for (auto& block: graph->blocks) { + block.dominators = + (block.id == graph->entry_block ? std::vector {block.id} : all); + } + + bool changed = true; + while (changed) { + changed = false; + for (auto& block: graph->blocks) { + if (block.id == graph->entry_block) { + continue; + } + std::vector next; + if (block.predecessors.empty()) { + next = {block.id}; + } else { + next = graph->blocks[block.predecessors.front()].dominators; + for (uint32_t i = 1; i < block.predecessors.size(); i++) { + next = IntersectSorted(next, graph->blocks[block.predecessors[i]].dominators); + } + AddUnique(&next, block.id); + SortUnique(&next); + } + if (next != block.dominators) { + block.dominators = std::move(next); + changed = true; + } + } + } +} + +void ComputePostDominators(Graph* graph) { + const auto count = static_cast(graph->blocks.size()); + const auto all = AllBlockIds(count); + + for (auto& block: graph->blocks) { + block.post_dominators = block.successors.empty() ? std::vector {block.id} : all; + } + + bool changed = true; + while (changed) { + changed = false; + for (auto& block: graph->blocks) { + std::vector next; + if (block.successors.empty()) { + next = {block.id}; + } else { + next = graph->blocks[block.successors.front()].post_dominators; + for (uint32_t i = 1; i < block.successors.size(); i++) { + next = + IntersectSorted(next, graph->blocks[block.successors[i]].post_dominators); + } + AddUnique(&next, block.id); + SortUnique(&next); + } + if (next != block.post_dominators) { + block.post_dominators = std::move(next); + changed = true; + } + } + } +} + +void ComputeBackEdges(Graph* graph) { + graph->back_edges.clear(); + for (const auto& block: graph->blocks) { + for (auto succ: block.successors) { + if (graph->Dominates(succ, block.id)) { + graph->back_edges.push_back({block.id, succ, true}); + } + } + } +} + +std::vector NaturalLoopBody(const Graph& graph, uint32_t header, uint32_t latch, + bool* natural) { + std::vector body; + std::vector stack; + body.reserve(graph.blocks.size()); + stack.reserve(graph.blocks.size()); + AddUnique(&body, header); + AddUnique(&body, latch); + if (latch != header) { + stack.push_back(latch); + } + if (natural != nullptr) { + *natural = true; + } + + while (!stack.empty()) { + const auto block_id = stack.back(); + stack.pop_back(); + const auto* block = graph.FindBlock(block_id); + if (block == nullptr) { + continue; + } + for (auto pred: block->predecessors) { + if (!graph.Dominates(header, pred) && natural != nullptr) { + *natural = false; + } + if (!Contains(body, pred)) { + body.push_back(pred); + if (pred != header) { + stack.push_back(pred); + } + } + } + } + SortUnique(&body); + return body; +} + +void ComputeNaturalLoops(Graph* graph) { + graph->natural_loops.clear(); + for (auto& edge: graph->back_edges) { + bool natural = true; + auto body = NaturalLoopBody(*graph, edge.to, edge.from, &natural); + edge.natural = natural; + + NaturalLoop loop; + loop.header = edge.to; + loop.latch = edge.from; + loop.continue_block = edge.from; + loop.body_blocks = body; + + for (auto block_id: body) { + const auto* block = graph->FindBlock(block_id); + if (block == nullptr) { + continue; + } + for (auto succ: block->successors) { + if (!Contains(body, succ)) { + AddUnique(&loop.exit_blocks, succ); + } + } + } + SortUnique(&loop.exit_blocks); + + if (!loop.exit_blocks.empty()) { + uint32_t merge = loop.exit_blocks.front(); + for (uint32_t i = 1; i < loop.exit_blocks.size(); i++) { + merge = graph->FindNearestCommonPostDominator(merge, loop.exit_blocks[i]); + } + loop.merge = merge; + } + graph->natural_loops.push_back(std::move(loop)); + } +} + +struct TarjanState { + const Graph* graph = nullptr; + uint32_t next_index = 0; + std::vector index; + std::vector lowlink; + std::vector on_stack; + std::vector stack; + std::vector components; +}; + +void TarjanVisit(TarjanState* state, uint32_t block_id) { + state->index[block_id] = state->next_index; + state->lowlink[block_id] = state->next_index; + state->next_index++; + state->stack.push_back(block_id); + state->on_stack[block_id] = true; + + const auto* block = state->graph->FindBlock(block_id); + if (block != nullptr) { + for (auto succ: block->successors) { + if (state->index[succ] == UINT32_MAX) { + TarjanVisit(state, succ); + state->lowlink[block_id] = std::min(state->lowlink[block_id], state->lowlink[succ]); + } else if (state->on_stack[succ]) { + state->lowlink[block_id] = std::min(state->lowlink[block_id], state->index[succ]); + } + } + } + + if (state->lowlink[block_id] != state->index[block_id]) { + return; + } + + StronglyConnectedComponent component; + for (;;) { + const auto member = state->stack.back(); + state->stack.pop_back(); + state->on_stack[member] = false; + component.blocks.push_back(member); + if (member == block_id) { + break; + } + } + SortUnique(&component.blocks); + + bool cyclic = component.blocks.size() > 1u; + for (auto member: component.blocks) { + const auto* member_block = state->graph->FindBlock(member); + if (member_block != nullptr && Contains(member_block->successors, member)) { + cyclic = true; + } + if (member_block != nullptr) { + for (auto pred: member_block->predecessors) { + if (!Contains(component.blocks, pred)) { + AddUnique(&component.entry_blocks, member); + } + } + } + } + SortUnique(&component.entry_blocks); + component.irreducible = cyclic && component.entry_blocks.size() > 1u; + state->components.push_back(std::move(component)); +} + +void ComputeComponents(Graph* graph) { + TarjanState state; + state.graph = graph; + state.index.assign(graph->blocks.size(), UINT32_MAX); + state.lowlink.assign(graph->blocks.size(), UINT32_MAX); + state.on_stack.assign(graph->blocks.size(), false); + state.stack.reserve(graph->blocks.size()); + state.components.reserve(graph->blocks.size()); + + for (const auto& block: graph->blocks) { + if (state.index[block.id] == UINT32_MAX) { + TarjanVisit(&state, block.id); + } + } + + graph->components = std::move(state.components); + graph->irreducible = false; + for (const auto& component: graph->components) { + if (component.irreducible) { + graph->irreducible = true; + graph->failure_kind = FailureKind::IrreducibleControlFlow; + graph->failure_block = component.entry_blocks.empty() ? component.blocks.front() + : component.entry_blocks.front(); + graph->unsupported_reason = "irreducible CFG: cyclic component has multiple entries"; + break; + } + } +} + +void RecomputeAnalyses(Graph* graph) { + ComputeDominators(graph); + ComputePostDominators(graph); + ComputeBackEdges(graph); + ComputeNaturalLoops(graph); + ComputeComponents(graph); +} + +uint32_t MoveBlockBefore(Graph* graph, uint32_t block_id, uint32_t before_id) { + if (block_id == before_id || block_id >= graph->blocks.size() || + before_id >= graph->blocks.size()) { + return block_id; + } + + const auto block_pos = block_id; + const auto before_pos = before_id; + std::vector old_blocks = std::move(graph->blocks); + std::vector new_blocks; + new_blocks.reserve(old_blocks.size()); + + for (uint32_t i = 0; i < old_blocks.size(); i++) { + if (i == before_pos) { + new_blocks.push_back(std::move(old_blocks[block_pos])); + } + if (i != block_pos) { + new_blocks.push_back(std::move(old_blocks[i])); + } + } + + std::vector id_map(new_blocks.size(), UINT32_MAX); + for (uint32_t i = 0; i < new_blocks.size(); i++) { + id_map[new_blocks[i].id] = i; + } + + graph->blocks = std::move(new_blocks); + graph->entry_block = RemapId(graph->entry_block, id_map); + for (auto& block: graph->blocks) { + block.id = RemapId(block.id, id_map); + RemapIds(&block.predecessors, id_map); + RemapIds(&block.successors, id_map); + RemapIds(&block.dominators, id_map); + RemapIds(&block.post_dominators, id_map); + block.terminator.true_block = RemapId(block.terminator.true_block, id_map); + block.terminator.false_block = RemapId(block.terminator.false_block, id_map); + block.terminator.merge_block = RemapId(block.terminator.merge_block, id_map); + block.terminator.continue_block = RemapId(block.terminator.continue_block, id_map); + } + + return RemapId(block_id, id_map); +} + +std::vector DominatedBlocks(const Graph& graph, uint32_t header, + uint32_t stop_block = UINT32_MAX) { + std::vector blocks; + std::vector stack = {header}; + blocks.reserve(graph.blocks.size()); + stack.reserve(graph.blocks.size()); + + while (!stack.empty()) { + const auto block_id = stack.back(); + stack.pop_back(); + if (block_id == stop_block || Contains(blocks, block_id) || + !graph.Dominates(header, block_id)) { + continue; + } + + const auto* block = graph.FindBlock(block_id); + if (block == nullptr) { + continue; + } + + AddUnique(&blocks, block_id); + for (auto succ: block->successors) { + if (succ != stop_block && graph.Dominates(header, succ)) { + stack.push_back(succ); + } + } + } + + SortUnique(&blocks); + return blocks; +} + +uint32_t AppendSyntheticMergeBlock(Graph* graph, uint32_t old_merge) { + const auto* merge = graph->FindBlock(old_merge); + + BasicBlock block; + block.id = static_cast(graph->blocks.size()); + block.start_pc = merge != nullptr ? merge->start_pc : 0u; + block.end_pc = block.start_pc; + block.inst_begin = merge != nullptr ? merge->inst_begin : 0u; + block.inst_end = block.inst_begin; + block.successors = {old_merge}; + block.terminator.kind = TerminatorKind::Branch; + block.terminator.condition = BranchCondition::Always; + block.terminator.true_block = old_merge; + graph->blocks.push_back(std::move(block)); + return graph->blocks.back().id; +} + +bool IsSyntheticMergeForwarder(const Graph& graph, uint32_t block_id, uint32_t merge) { + const auto* block = graph.FindBlock(block_id); + return block != nullptr && block->inst_begin == block->inst_end && + block->successors.size() == 1u && block->successors.front() == merge && + block->terminator.kind == TerminatorKind::Branch && + block->terminator.condition == BranchCondition::Always && + block->terminator.true_block == merge; +} + +bool IsInsideLoopConstruct(const Graph& graph, const NaturalLoop& loop, uint32_t block_id) { + return block_id != UINT32_MAX && block_id != loop.merge && block_id != loop.continue_block && + graph.Dominates(loop.header, block_id) && + (loop.merge == UINT32_MAX || !graph.Dominates(loop.merge, block_id)); +} + +bool SelectionMergeLeavesContainingLoop(const Graph& graph, uint32_t header, uint32_t merge) { + for (const auto& loop: graph.natural_loops) { + if (IsInsideLoopConstruct(graph, loop, header) && + !IsInsideLoopConstruct(graph, loop, merge)) { + return true; + } + } + return false; +} + +bool SplitSharedMergeBlock(Graph* graph, uint32_t merge, + const std::vector& construct_blocks, + bool force_split = false) { + if (graph == nullptr || merge == UINT32_MAX || merge >= graph->blocks.size() || + Contains(construct_blocks, merge)) { + return false; + } + + const auto* merge_block = graph->FindBlock(merge); + if (merge_block == nullptr) { + return false; + } + + std::vector construct_predecessors; + std::vector external_predecessors; + for (auto pred: merge_block->predecessors) { + if (Contains(construct_blocks, pred)) { + AddUnique(&construct_predecessors, pred); + } else { + AddUnique(&external_predecessors, pred); + } + } + + if (construct_predecessors.empty() || (!force_split && external_predecessors.empty())) { + return false; + } + std::vector predecessors_to_split; + for (auto pred: construct_predecessors) { + if (!IsSyntheticMergeForwarder(*graph, pred, merge)) { + AddUnique(&predecessors_to_split, pred); + } + } + if (predecessors_to_split.empty()) { + return false; + } + + const auto synthetic_merge = AppendSyntheticMergeBlock(graph, merge); + auto* synthetic_block = graph->FindBlock(synthetic_merge); + if (synthetic_block != nullptr) { + synthetic_block->predecessors = predecessors_to_split; + SortUnique(&synthetic_block->predecessors); + } + + for (auto pred: predecessors_to_split) { + auto* block = graph->FindBlock(pred); + if (block == nullptr) { + continue; + } + ReplaceValue(&block->successors, merge, synthetic_merge); + ReplaceTerminatorTarget(&block->terminator, merge, synthetic_merge); + } + + auto* old_merge = graph->FindBlock(merge); + if (old_merge != nullptr) { + for (auto pred: predecessors_to_split) { + RemoveValue(&old_merge->predecessors, pred); + } + AddUnique(&old_merge->predecessors, synthetic_merge); + SortUnique(&old_merge->predecessors); + } + + MoveBlockBefore(graph, synthetic_merge, merge); + return true; +} + +bool SplitOneLoopMerge(Graph* graph) { + const auto& loops = graph->natural_loops; + for (const auto& loop: loops) { + if (SplitSharedMergeBlock(graph, loop.merge, loop.body_blocks)) { + return true; + } + } + return false; +} + +bool SplitOneSelectionMerge(Graph* graph) { + std::vector loop_headers; + loop_headers.reserve(graph->natural_loops.size()); + for (const auto& loop: graph->natural_loops) { + AddUnique(&loop_headers, loop.header); + } + + const auto original_block_count = static_cast(graph->blocks.size()); + for (uint32_t block_id = 0; block_id < original_block_count; block_id++) { + const auto* block = graph->FindBlock(block_id); + if (block == nullptr || block->terminator.kind != TerminatorKind::ConditionalBranch || + Contains(loop_headers, block_id)) { + continue; + } + + const auto merge = graph->FindNearestCommonPostDominator(block->terminator.true_block, + block->terminator.false_block); + const auto construct_blocks = DominatedBlocks(*graph, block_id, merge); + const auto force_split = SelectionMergeLeavesContainingLoop(*graph, block_id, merge); + if (SplitSharedMergeBlock(graph, merge, construct_blocks, force_split)) { + return true; + } + } + return false; +} + +bool SplitSharedMergeBlocks(Graph* graph, std::string* error) { + const auto original_block_count = static_cast(graph->blocks.size()); + const auto split_budget = + std::max(16u, std::min(128u, original_block_count)); + for (uint32_t splits = 0; splits < split_budget; splits++) { + if (!SplitOneLoopMerge(graph) && !SplitOneSelectionMerge(graph)) { + return true; + } + RebuildPredecessors(graph); + RecomputeAnalyses(graph); + } + SetFailure(graph, FailureKind::StructuredControlFlow, graph->entry_block, + fmt::format("CFG shared merge splitting exceeded budget: original_blocks={} " + "current_blocks={} split_budget={}", + original_block_count, static_cast(graph->blocks.size()), + split_budget), + error); + return false; +} + +void ClearStructuredTerminators(Graph* graph) { + for (auto& block: graph->blocks) { + block.terminator.merge_block = UINT32_MAX; + block.terminator.continue_block = UINT32_MAX; + block.terminator.loop_header = false; + } +} + +std::string VectorToString(const std::vector& values) { + std::string text; + for (uint32_t i = 0; i < values.size(); i++) { + if (i != 0) { + text += ","; + } + text += fmt::format("{}", values[i]); + } + return text; +} + +} // namespace + +const BasicBlock* Graph::FindBlock(uint32_t id) const { + if (id < blocks.size() && blocks[id].id == id) { + return &blocks[id]; + } + for (const auto& block: blocks) { + if (block.id == id) { + return █ + } + } + return nullptr; +} + +BasicBlock* Graph::FindBlock(uint32_t id) { + return const_cast(static_cast(this)->FindBlock(id)); +} + +const BasicBlock* Graph::FindBlockByPc(uint32_t pc) const { + for (const auto& block: blocks) { + if (block.start_pc == pc) { + return █ + } + } + return nullptr; +} + +BasicBlock* Graph::FindBlockByPc(uint32_t pc) { + return const_cast(static_cast(this)->FindBlockByPc(pc)); +} + +bool Graph::Dominates(uint32_t dominator, uint32_t block) const { + const auto* target = FindBlock(block); + return target != nullptr && Contains(target->dominators, dominator); +} + +bool Graph::PostDominates(uint32_t post_dominator, uint32_t block) const { + const auto* target = FindBlock(block); + return target != nullptr && Contains(target->post_dominators, post_dominator); +} + +uint32_t Graph::FindNearestCommonPostDominator(uint32_t block_a, uint32_t block_b) const { + const auto* a = FindBlock(block_a); + const auto* b = FindBlock(block_b); + if (a == nullptr || b == nullptr) { + return UINT32_MAX; + } + + const auto common = IntersectSorted(a->post_dominators, b->post_dominators); + for (auto candidate: common) { + bool nearest = true; + for (auto other: common) { + if (other != candidate && !PostDominates(other, candidate)) { + nearest = false; + break; + } + } + if (nearest) { + return candidate; + } + } + return common.empty() ? UINT32_MAX : common.front(); +} + +bool BuildGraph(const Decoder::Program& program, Graph* graph, std::string* error) { + if (graph == nullptr) { + SetError(error, "invalid CFG output"); + return false; + } + *graph = {}; + + if (program.instructions.empty()) { + SetError(error, "cannot build CFG for empty shader"); + return false; + } + + const auto first_pc = program.instructions.front().pc; + const auto end_pc = ProgramEndPc(program); + + std::set instruction_pcs; + for (const auto& inst: program.instructions) { + instruction_pcs.insert(inst.pc); + if (inst.opcode == Opcode::Unsupported) { + SetFailure(graph, FailureKind::UnsupportedInstruction, UINT32_MAX, + fmt::format("unsupported decoded instruction in CFG at pc 0x{:08x}: {}", + inst.pc, Decoder::InstructionToString(inst).c_str()), + error); + return false; + } + } + + std::set labels; + labels.insert(first_pc); + labels.insert(end_pc); + + std::map setpc_targets; + bool indirect_setpc = false; + for (uint32_t i = 0; i < program.instructions.size(); i++) { + const auto& inst = program.instructions[i]; + const auto next_pc = InstructionEndPc(inst); + if (IsBranch(inst.opcode)) { + if (!IsValidTarget(inst.branch_target, instruction_pcs, first_pc, end_pc)) { + SetFailure(graph, FailureKind::InvalidBranchTarget, UINT32_MAX, + fmt::format("branch at pc 0x{:08x} targets invalid pc 0x{:08x}", inst.pc, + inst.branch_target), + error); + return false; + } + labels.insert(inst.branch_target); + if (next_pc <= end_pc) { + labels.insert(next_pc); + } + } else if (inst.opcode == Opcode::SSetpcB64) { + SetpcTargetInfo target_info; + if (!ResolveSetpcTargets(program, i, &target_info)) { + SetFailure(graph, FailureKind::InvalidBranchTarget, UINT32_MAX, + fmt::format("unsupported dynamic S_SETPC_B64 at pc 0x{:08x}", inst.pc), + error); + return false; + } + const auto& target_pcs = target_info.indirect + ? target_info.target_pcs + : std::vector {target_info.target}; + for (const auto target: target_pcs) { + if (!IsValidTarget(target, instruction_pcs, first_pc, end_pc)) { + SetFailure(graph, FailureKind::InvalidBranchTarget, UINT32_MAX, + fmt::format("S_SETPC_B64 at pc 0x{:08x} targets invalid pc 0x{:08x}", + inst.pc, target), + error); + return false; + } + labels.insert(target); + } + indirect_setpc = indirect_setpc || target_info.indirect; + setpc_targets.emplace(inst.pc, std::move(target_info)); + if (next_pc <= end_pc) { + labels.insert(next_pc); + } + } else if (inst.opcode == Opcode::SEndpgm) { + labels.insert(next_pc); + } + } + + std::vector sorted_labels(labels.begin(), labels.end()); + std::sort(sorted_labels.begin(), sorted_labels.end()); + for (uint32_t i = 0; i < sorted_labels.size(); i++) { + const auto start = sorted_labels[i]; + if (start > end_pc) { + continue; + } + if (start != end_pc && !instruction_pcs.contains(start)) { + SetFailure(graph, FailureKind::InvalidLabel, UINT32_MAX, + fmt::format("CFG label does not start on an instruction: 0x{:08x}", start), + error); + return false; + } + + BasicBlock block; + block.id = static_cast(graph->blocks.size()); + block.start_pc = start; + block.end_pc = i + 1u < sorted_labels.size() ? sorted_labels[i + 1u] : end_pc; + block.inst_begin = static_cast( + std::lower_bound(program.instructions.begin(), program.instructions.end(), start, + [](const Instruction& inst, uint32_t pc) { return inst.pc < pc; }) - + program.instructions.begin()); + block.inst_end = static_cast( + std::lower_bound(program.instructions.begin(), program.instructions.end(), block.end_pc, + [](const Instruction& inst, uint32_t pc) { return inst.pc < pc; }) - + program.instructions.begin()); + graph->blocks.push_back(std::move(block)); + } + + graph->entry_block = 0; + + std::map pc_to_block; + for (const auto& block: graph->blocks) { + pc_to_block.emplace(block.start_pc, block.id); + } + + for (auto& block: graph->blocks) { + block.terminator = {}; + if (block.inst_begin == block.inst_end) { + block.terminator.kind = TerminatorKind::Return; + continue; + } + + const auto& last = program.instructions[block.inst_end - 1u]; + const auto next_pc = InstructionEndPc(last); + if (last.opcode == Opcode::SEndpgm) { + block.terminator.kind = TerminatorKind::Branch; + block.terminator.condition = BranchCondition::Always; + block.terminator.true_block = pc_to_block.at(end_pc); + } else if (last.opcode == Opcode::SSetpcB64) { + const auto& target_info = setpc_targets.at(last.pc); + if (target_info.indirect) { + block.terminator.kind = TerminatorKind::IndirectBranch; + block.terminator.condition = BranchCondition::Always; + block.terminator.indirect_pc_sgpr = target_info.pc_sgpr; + block.terminator.indirect_selector_code = target_info.selector_code; + for (const auto target_pc: target_info.target_pcs) { + block.terminator.indirect_target_pcs.push_back(target_pc); + block.terminator.indirect_targets.push_back(pc_to_block.at(target_pc)); + } + const auto selector_count = std::min(target_info.selector_values.size(), + target_info.selector_target_pcs.size()); + for (uint32_t i = 0; i < selector_count; i++) { + block.terminator.indirect_selector_values.push_back( + target_info.selector_values[i]); + block.terminator.indirect_selector_targets.push_back( + pc_to_block.at(target_info.selector_target_pcs[i])); + } + if (target_info.table_load_pc != UINT32_MAX) { + AddUnique(&graph->code_table_load_pcs, target_info.table_load_pc); + } + } else { + block.terminator.kind = TerminatorKind::Branch; + block.terminator.condition = BranchCondition::Always; + block.terminator.true_block = pc_to_block.at(target_info.target); + } + } else if (IsUnconditionalBranch(last.opcode)) { + block.terminator.kind = TerminatorKind::Branch; + block.terminator.condition = BranchCondition::Always; + block.terminator.true_block = pc_to_block.at(last.branch_target); + } else if (IsConditionalBranch(last.opcode)) { + block.terminator.kind = TerminatorKind::ConditionalBranch; + block.terminator.condition = ConditionForOpcode(last.opcode); + block.terminator.true_block = pc_to_block.at(last.branch_target); + const auto fallthrough = pc_to_block.find(next_pc); + if (fallthrough == pc_to_block.end()) { + SetFailure(graph, FailureKind::MissingFallthrough, block.id, + fmt::format("conditional branch at pc 0x{:08x} has no fallthrough block", + last.pc), + error); + return false; + } + block.terminator.false_block = fallthrough->second; + } else { + auto next = pc_to_block.find(block.end_pc); + if (next != pc_to_block.end() && block.end_pc != block.start_pc) { + block.terminator.kind = TerminatorKind::Branch; + block.terminator.condition = BranchCondition::Always; + block.terminator.true_block = next->second; + } else { + block.terminator.kind = TerminatorKind::Return; + } + } + + switch (block.terminator.kind) { + case TerminatorKind::Branch: + AddUnique(&block.successors, block.terminator.true_block); + break; + case TerminatorKind::ConditionalBranch: + AddUnique(&block.successors, block.terminator.true_block); + AddUnique(&block.successors, block.terminator.false_block); + break; + case TerminatorKind::IndirectBranch: + for (const auto target: block.terminator.indirect_targets) { + AddUnique(&block.successors, target); + } + break; + case TerminatorKind::Return: + case TerminatorKind::Unsupported: break; + } + } + + for (auto& block: graph->blocks) { + SortUnique(&block.successors); + for (auto succ: block.successors) { + AddUnique(&graph->blocks[succ].predecessors, block.id); + } + } + for (auto& block: graph->blocks) { + SortUnique(&block.predecessors); + } + SortUnique(&graph->code_table_load_pcs); + + ComputeDominators(graph); + ComputePostDominators(graph); + ComputeBackEdges(graph); + ComputeNaturalLoops(graph); + ComputeComponents(graph); + + if (indirect_setpc) { + graph->irreducible = true; + graph->unsupported = false; + graph->failure_kind = FailureKind::IrreducibleControlFlow; + graph->unsupported_reason = "indirect S_SETPC_B64 jump table requires dispatcher fallback"; + } + + if (graph->irreducible) { + graph->unsupported = false; + } + + return true; +} + +bool Structurize(Graph* graph, std::string* error) { + if (graph == nullptr) { + SetError(error, "invalid CFG input"); + return false; + } + if (graph->unsupported || graph->irreducible) { + if (graph->unsupported_reason.empty()) { + graph->unsupported_reason = "unsupported CFG"; + } + SetError(error, graph->unsupported_reason); + return false; + } + + if (!SplitSharedMergeBlocks(graph, error)) { + return false; + } + ClearStructuredTerminators(graph); + + std::map merge_headers; + auto reserve_merge_block = [&](uint32_t header, uint32_t merge) { + const auto [it, inserted] = merge_headers.emplace(merge, header); + if (!inserted && it->second != header) { + SetFailure( + graph, FailureKind::StructuredControlFlow, header, + fmt::format( + "duplicate structured merge block {} for header {} (already used by header {})", + merge, header, it->second), + error); + return false; + } + return true; + }; + + for (const auto& loop: graph->natural_loops) { + auto* header = graph->FindBlock(loop.header); + if (header == nullptr || loop.merge == UINT32_MAX || loop.continue_block == UINT32_MAX) { + SetFailure( + graph, FailureKind::StructuredControlFlow, loop.header, + fmt::format("loop at block {} has no structured merge/continue", loop.header), + error); + return false; + } + if (!reserve_merge_block(loop.header, loop.merge)) { + return false; + } + header->terminator.loop_header = true; + header->terminator.merge_block = loop.merge; + header->terminator.continue_block = loop.continue_block; + } + + for (auto& block: graph->blocks) { + if (block.terminator.kind != TerminatorKind::ConditionalBranch || + block.terminator.loop_header) { + continue; + } + + const auto merge = graph->FindNearestCommonPostDominator(block.terminator.true_block, + block.terminator.false_block); + if (merge == UINT32_MAX) { + SetFailure(graph, FailureKind::StructuredControlFlow, block.id, + fmt::format("conditional block {} has no structured merge", block.id), + error); + return false; + } + + if (!reserve_merge_block(block.id, merge)) { + return false; + } + block.terminator.merge_block = merge; + } + + return true; +} + +std::string BranchConditionToString(BranchCondition condition) { + switch (condition) { + case BranchCondition::Always: return "always"; + case BranchCondition::SccZero: return "scc0"; + case BranchCondition::SccNonZero: return "scc1"; + case BranchCondition::VccZero: return "vccz"; + case BranchCondition::VccNonZero: return "vccnz"; + case BranchCondition::ExecZero: return "execz"; + case BranchCondition::ExecNonZero: return "execnz"; + default: return "unknown"; + } +} + +std::string FailureKindToString(FailureKind kind) { + switch (kind) { + case FailureKind::None: return "None"; + case FailureKind::InvalidInput: return "InvalidInput"; + case FailureKind::UnsupportedInstruction: return "UnsupportedInstruction"; + case FailureKind::InvalidBranchTarget: return "InvalidBranchTarget"; + case FailureKind::MissingFallthrough: return "MissingFallthrough"; + case FailureKind::InvalidLabel: return "InvalidLabel"; + case FailureKind::IrreducibleControlFlow: return "IrreducibleControlFlow"; + case FailureKind::StructuredControlFlow: return "StructuredControlFlow"; + default: return "Unknown"; + } +} + +std::string FormatBlockDiagnostic(const Graph& graph, uint32_t block_id, const char* stage, + const std::string& reason) { + const auto* block = graph.FindBlock(block_id); + const char* stage_text = stage != nullptr ? stage : "unknown"; + if (block == nullptr) { + return fmt::format("stage={} block={} reason={}", stage_text, block_id, reason.c_str()); + } + return fmt::format("stage={} block={} pc=0x{:08x}..0x{:08x} preds={} succs={} reason={}", + stage_text, block->id, block->start_pc, block->end_pc, + static_cast(block->predecessors.size()), + static_cast(block->successors.size()), reason.c_str()); +} + +std::string GraphToString(const Graph& graph) { + std::string text; + text += + fmt::format("entry_block={} irreducible={} unsupported={} failure={} failure_block={}\n", + graph.entry_block, graph.irreducible ? 1u : 0u, graph.unsupported ? 1u : 0u, + FailureKindToString(graph.failure_kind).c_str(), graph.failure_block); + if (!graph.unsupported_reason.empty()) { + text += "unsupported_reason="; + text += graph.unsupported_reason; + text += "\n"; + } + + for (const auto& block: graph.blocks) { + text += fmt::format("block_{} pc=0x{:08x} end=0x{:08x} inst=[{},{})\n", block.id, + block.start_pc, block.end_pc, block.inst_begin, block.inst_end); + text += fmt::format(" predecessors=[{}] successors=[{}]\n", + VectorToString(block.predecessors).c_str(), + VectorToString(block.successors).c_str()); + text += fmt::format(" dominators=[{}] post_dominators=[{}]\n", + VectorToString(block.dominators).c_str(), + VectorToString(block.post_dominators).c_str()); + text += fmt::format( + " terminator={} condition={} true={} false={} merge={} continue={} loop_header={} " + "indirect_sgpr={} indirect_selector={} indirect_targets=[{}] selector_values=[{}]\n", + static_cast(block.terminator.kind), + BranchConditionToString(block.terminator.condition).c_str(), + block.terminator.true_block, block.terminator.false_block, block.terminator.merge_block, + block.terminator.continue_block, block.terminator.loop_header ? 1u : 0u, + block.terminator.indirect_pc_sgpr, block.terminator.indirect_selector_code, + VectorToString(block.terminator.indirect_targets).c_str(), + VectorToString(block.terminator.indirect_selector_values).c_str()); + } + + for (const auto& edge: graph.back_edges) { + text += fmt::format("backedge {} -> {} natural={}\n", edge.from, edge.to, + edge.natural ? 1u : 0u); + } + for (const auto& loop: graph.natural_loops) { + text += fmt::format("loop header={} latch={} merge={} continue={} body=[{}] exits=[{}]\n", + loop.header, loop.latch, loop.merge, loop.continue_block, + VectorToString(loop.body_blocks).c_str(), + VectorToString(loop.exit_blocks).c_str()); + } + for (const auto& component: graph.components) { + text += fmt::format("scc blocks=[{}] entries=[{}] irreducible={}\n", + VectorToString(component.blocks).c_str(), + VectorToString(component.entry_blocks).c_str(), + component.irreducible ? 1u : 0u); + } + return text; +} + +} // namespace Libs::Graphics::ShaderRecompiler::CFG diff --git a/src/graphics/shader/recompiler/ShaderCFG.h b/src/graphics/shader/recompiler/ShaderCFG.h new file mode 100644 index 000000000..43659297d --- /dev/null +++ b/src/graphics/shader/recompiler/ShaderCFG.h @@ -0,0 +1,118 @@ +#ifndef EMULATOR_INCLUDE_EMULATOR_GRAPHICS_SHADER_RECOMPILER_SHADERCFG_H_ +#define EMULATOR_INCLUDE_EMULATOR_GRAPHICS_SHADER_RECOMPILER_SHADERCFG_H_ + +#include "common/common.h" +#include "common/stringUtils.h" +#include "graphics/shader/recompiler/ShaderDecoder.h" + +#include + +namespace Libs::Graphics::ShaderRecompiler::CFG { + +enum class BranchCondition { + Always, + SccZero, + SccNonZero, + VccZero, + VccNonZero, + ExecZero, + ExecNonZero, + Unknown +}; + +enum class TerminatorKind { Branch, ConditionalBranch, IndirectBranch, Return, Unsupported }; + +enum class FailureKind { + None, + InvalidInput, + UnsupportedInstruction, + InvalidBranchTarget, + MissingFallthrough, + InvalidLabel, + IrreducibleControlFlow, + StructuredControlFlow +}; + +struct Terminator { + TerminatorKind kind = TerminatorKind::Return; + BranchCondition condition = BranchCondition::Always; + uint32_t true_block = UINT32_MAX; + uint32_t false_block = UINT32_MAX; + uint32_t merge_block = UINT32_MAX; + uint32_t continue_block = UINT32_MAX; + uint32_t indirect_pc_sgpr = UINT32_MAX; + uint32_t indirect_selector_code = UINT32_MAX; + std::vector indirect_target_pcs; + std::vector indirect_targets; + std::vector indirect_selector_values; + std::vector indirect_selector_targets; + bool loop_header = false; +}; + +struct BasicBlock { + uint32_t id = 0; + uint32_t start_pc = 0; + uint32_t end_pc = 0; + uint32_t inst_begin = 0; + uint32_t inst_end = 0; + std::vector predecessors; + std::vector successors; + std::vector dominators; + std::vector post_dominators; + Terminator terminator; +}; + +struct BackEdge { + uint32_t from = UINT32_MAX; + uint32_t to = UINT32_MAX; + bool natural = false; +}; + +struct NaturalLoop { + uint32_t header = UINT32_MAX; + uint32_t latch = UINT32_MAX; + uint32_t merge = UINT32_MAX; + uint32_t continue_block = UINT32_MAX; + std::vector body_blocks; + std::vector exit_blocks; +}; + +struct StronglyConnectedComponent { + std::vector blocks; + std::vector entry_blocks; + bool irreducible = false; +}; + +struct Graph { + std::vector blocks; + std::vector back_edges; + std::vector natural_loops; + std::vector components; + std::vector code_table_load_pcs; + uint32_t entry_block = UINT32_MAX; + bool irreducible = false; + bool unsupported = false; + FailureKind failure_kind = FailureKind::None; + uint32_t failure_block = UINT32_MAX; + std::string unsupported_reason; + + const BasicBlock* FindBlock(uint32_t id) const; + BasicBlock* FindBlock(uint32_t id); + const BasicBlock* FindBlockByPc(uint32_t pc) const; + BasicBlock* FindBlockByPc(uint32_t pc); + bool Dominates(uint32_t dominator, uint32_t block) const; + bool PostDominates(uint32_t post_dominator, uint32_t block) const; + uint32_t FindNearestCommonPostDominator(uint32_t block_a, uint32_t block_b) const; +}; + +bool BuildGraph(const Decoder::Program& program, Graph* graph, std::string* error); +bool Structurize(Graph* graph, std::string* error); +std::string BranchConditionToString(BranchCondition condition); +std::string FailureKindToString(FailureKind kind); +std::string FormatBlockDiagnostic(const Graph& graph, uint32_t block_id, const char* stage, + const std::string& reason); +std::string GraphToString(const Graph& graph); + +} // namespace Libs::Graphics::ShaderRecompiler::CFG + +#endif /* EMULATOR_INCLUDE_EMULATOR_GRAPHICS_SHADER_RECOMPILER_SHADERCFG_H_ */ diff --git a/src/graphics/shader/recompiler/ShaderDecoder.cpp b/src/graphics/shader/recompiler/ShaderDecoder.cpp new file mode 100644 index 000000000..4b1e4003d --- /dev/null +++ b/src/graphics/shader/recompiler/ShaderDecoder.cpp @@ -0,0 +1,1216 @@ +#include "graphics/shader/recompiler/ShaderDecoder.h" + +#include "graphics/shader/recompiler/ExportOps.h" +#include "graphics/shader/recompiler/ImageOps.h" +#include "graphics/shader/recompiler/MemoryOps.h" +#include "graphics/shader/recompiler/ScalarAluOps.h" +#include "graphics/shader/recompiler/VectorAluOps.h" + +#include +#include +#include +#include + +namespace Libs::Graphics::ShaderRecompiler::Decoder { +namespace { + +uint32_t FloatBits(float value) { + return std::bit_cast(value); +} + +void SetError(std::string* error, const char* message) { + if (error != nullptr) { + *error = message; + } +} + +bool HasLiteral(const Instruction& inst) { + return inst.src0.kind == OperandKind::LiteralConstant || + inst.src1.kind == OperandKind::LiteralConstant || + inst.src2.kind == OperandKind::LiteralConstant || + inst.src3.kind == OperandKind::LiteralConstant; +} + +bool IsControlFlowBranch(Opcode opcode) { + switch (opcode) { + case Opcode::SBranch: + case Opcode::SCbranchScc0: + case Opcode::SCbranchScc1: + case Opcode::SCbranchVccz: + case Opcode::SCbranchVccnz: + case Opcode::SCbranchExecz: + case Opcode::SCbranchExecnz: return true; + default: return false; + } +} + +void ApplyLiteral(Operand* operand, uint32_t literal) { + if (operand->kind == OperandKind::LiteralConstant) { + operand->value = literal; + operand->signed_val = static_cast(literal); + } +} + +std::string RawWordsToString(const Instruction& inst) { + std::string text; + for (uint32_t i = 0; i < inst.raw_count; i++) { + if (i != 0) { + text += " "; + } + text += fmt::format("0x{:08x}", inst.raw[i]); + } + return text; +} + +std::string FormatBinary(const Instruction& inst) { + std::string text = fmt::format("0x{:08x}: {} {}", inst.pc, OpcodeToString(inst.opcode).c_str(), + OperandToString(inst.dst).c_str()); + if (inst.dst2.kind != OperandKind::Unknown) { + text += ", "; + text += OperandToString(inst.dst2); + } + const Operand* sources[] = {&inst.src0, &inst.src1, &inst.src2, &inst.src3}; + for (uint32_t i = 0; i < inst.src_count && i < 4u; i++) { + text += ", "; + text += OperandToString(*sources[i]); + } + return text; +} + +std::string FormatMemory(const Instruction& inst) { + std::string text = fmt::format("0x{:08x}: {} {}", inst.pc, OpcodeToString(inst.opcode).c_str(), + OperandToString(inst.dst).c_str()); + const Operand* sources[] = {&inst.src0, &inst.src1, &inst.src2, &inst.src3}; + for (uint32_t i = 0; i < inst.src_count && i < 4u; i++) { + text += ", "; + text += OperandToString(*sources[i]); + } + text += fmt::format(" ; offset={} offset2={} dwords={} bits={} dfmt={} nfmt={} signed={} " + "typed={} formatted={} segment={} glc={} slc={} idxen={} offen={}", + inst.offset, inst.secondary_offset, inst.data_dwords, inst.data_bits, + inst.data_format, inst.number_format, inst.data_signed ? 1u : 0u, + inst.typed ? 1u : 0u, inst.formatted ? 1u : 0u, inst.memory_segment, + inst.glc ? 1u : 0u, inst.slc ? 1u : 0u, inst.idxen ? 1u : 0u, + inst.offen ? 1u : 0u); + return text; +} + +std::string WithUnsupportedReason(const Instruction& inst, const std::string& text) { + if (inst.unsupported_reason.empty()) { + return text; + } + return text + fmt::format(" ; family={} opcode=0x{:02x} raw=[{}] reason={}", + FamilyToString(inst.family).c_str(), inst.opcode_id, + RawWordsToString(inst).c_str(), inst.unsupported_reason.c_str()); +} + +void AppendFlag(std::string* text, bool* first, uint32_t flags, uint32_t flag, const char* name) { + if ((flags & flag) == 0) { + return; + } + if (!*first) { + *text += "|"; + } + *text += name; + *first = false; +} + +std::string ImageSampleFlagsToString(uint32_t flags) { + if (flags == 0) { + return "none"; + } + std::string text; + bool first = true; + AppendFlag(&text, &first, flags, ImageSampleFlagLod, "lod"); + AppendFlag(&text, &first, flags, ImageSampleFlagBias, "bias"); + AppendFlag(&text, &first, flags, ImageSampleFlagDerivative, "derivative"); + AppendFlag(&text, &first, flags, ImageSampleFlagCompare, "compare"); + AppendFlag(&text, &first, flags, ImageSampleFlagOffset, "offset"); + AppendFlag(&text, &first, flags, ImageSampleFlagLevelZero, "level_zero"); + AppendFlag(&text, &first, flags, ImageSampleFlagLodClamp, "lod_clamp"); + AppendFlag(&text, &first, flags, ImageSampleFlagA16, "a16"); + AppendFlag(&text, &first, flags, ImageSampleFlagCd, "cd"); + AppendFlag(&text, &first, flags, ImageSampleFlagGatherHorizontal, "gather_horizontal"); + return text; +} + +const char* ImageDimensionToString(ImageDimension dimension) { + switch (dimension) { + case ImageDimension::Dim2D: return "2d"; + case ImageDimension::Dim3D: return "3d"; + case ImageDimension::Dim2DArray: return "2d_array"; + default: return "unknown"; + } +} + +std::string FormatMimg(const Instruction& inst) { + const char* sample_name = + inst.opcode == Opcode::ImageSample ? MimgSampleOpcodeName(inst.opcode_id) : nullptr; + const std::string name = + sample_name != nullptr ? std::string(sample_name) : OpcodeToString(inst.opcode); + std::string text = fmt::format( + "0x{:08x}: {} {}, {}, {}, {} ; dmask=0x{:x} image_dim={}", inst.pc, name.c_str(), + OperandToString(inst.dst).c_str(), OperandToString(inst.src0).c_str(), + OperandToString(inst.src1).c_str(), OperandToString(inst.src2).c_str(), inst.dmask, + ImageDimensionToString(inst.image_dimension)); + switch (inst.opcode) { + case Opcode::ImageSample: + case Opcode::ImageGather4Lz: + case Opcode::ImageGather4C: + case Opcode::ImageGather4CLz: + case Opcode::ImageGather4LzO: + case Opcode::ImageGather4CO: + case Opcode::ImageGather4CLzO: + case Opcode::ImageGather4H: + text += fmt::format(" sample_flags={} addr_components={}", + ImageSampleFlagsToString(inst.image_sample_flags).c_str(), + inst.image_address_components); + break; + default: break; + } + if (inst.image_nsa_dwords != 0) { + text += fmt::format(" nsa_dwords={} nsa_addr=", inst.image_nsa_dwords); + const auto count = + std::min(inst.image_nsa_dwords * 4u, MaxImageNsaAddressComponents); + for (uint32_t i = 0; i < count; i++) { + if (i != 0) { + text += ","; + } + text += fmt::format("{}", inst.image_nsa_addr[i]); + } + } + return text; +} + +std::string FormatExp(const Instruction& inst) { + std::string text = fmt::format("0x{:08x}: exp target=0x{:02x} en=0x{:x} done={} compr={} vm={}", + inst.pc, inst.exp.target, inst.exp.en, inst.exp.done ? 1u : 0u, + inst.exp.compr ? 1u : 0u, inst.exp.vm ? 1u : 0u); + const Operand* sources[] = {&inst.src0, &inst.src1, &inst.src2, &inst.src3}; + for (uint32_t i = 0; i < inst.src_count && i < 4u; i++) { + text += i == 0 ? " " : ", "; + text += OperandToString(*sources[i]); + } + return text; +} + +} // namespace + +bool DecodeScalarSource(uint32_t code, uint32_t pc, Operand* operand, std::string* error) { + if (operand == nullptr) { + SetError(error, "internal decoder error: null source operand"); + return false; + } + + *operand = {}; + + if (code <= 105u) { + operand->kind = OperandKind::Sgpr; + operand->reg = code; + return true; + } + if (code >= 128u && code <= 192u) { + operand->kind = OperandKind::IntegerInlineConstant; + operand->signed_val = static_cast(code - 128u); + operand->value = static_cast(operand->signed_val); + return true; + } + if (code >= 193u && code <= 208u) { + operand->kind = OperandKind::IntegerInlineConstant; + operand->signed_val = 192 - static_cast(code); + operand->value = static_cast(operand->signed_val); + return true; + } + if (code >= 240u && code <= 247u) { + constexpr float values[] = {0.5f, -0.5f, 1.0f, -1.0f, 2.0f, -2.0f, 4.0f, -4.0f}; + operand->kind = OperandKind::FloatInlineConstant; + operand->float_val = values[code - 240u]; + operand->value = FloatBits(operand->float_val); + return true; + } + if (code >= 256u && code <= 511u) { + return DecodeVectorGpr(code - 256u, operand, error); + } + + switch (code) { + case 106u: operand->kind = OperandKind::VccLo; return true; + case 107u: operand->kind = OperandKind::VccHi; return true; + case 124u: operand->kind = OperandKind::M0; return true; + case 125u: operand->kind = OperandKind::Null; return true; + case 126u: operand->kind = OperandKind::ExecLo; return true; + case 127u: operand->kind = OperandKind::ExecHi; return true; + case 239u: operand->kind = OperandKind::PopsExitingWaveId; return true; + case 248u: + operand->kind = OperandKind::FloatInlineConstant; + operand->float_val = 0.15915494309189535f; + operand->value = FloatBits(operand->float_val); + return true; + case 251u: operand->kind = OperandKind::VccZ; return true; + case 252u: operand->kind = OperandKind::ExecZ; return true; + case 253u: operand->kind = OperandKind::Scc; return true; + case 255u: operand->kind = OperandKind::LiteralConstant; return true; + default: + if (error != nullptr) { + *error = fmt::format("unsupported scalar source operand 0x{:08x} at pc 0x{:08x}", + code, pc); + } + return false; + } +} + +bool DecodeScalarDestination(uint32_t code, uint32_t pc, Operand* operand, std::string* error) { + if (operand == nullptr) { + SetError(error, "internal decoder error: null destination operand"); + return false; + } + + *operand = {}; + + if (code <= 105u) { + operand->kind = OperandKind::Sgpr; + operand->reg = code; + return true; + } + + switch (code) { + case 106u: operand->kind = OperandKind::VccLo; return true; + case 107u: operand->kind = OperandKind::VccHi; return true; + case 124u: operand->kind = OperandKind::M0; return true; + case 125u: operand->kind = OperandKind::Null; return true; + case 126u: operand->kind = OperandKind::ExecLo; return true; + case 127u: operand->kind = OperandKind::ExecHi; return true; + default: + if (error != nullptr) { + *error = fmt::format( + "unsupported scalar destination operand 0x{:08x} at pc 0x{:08x}", code, pc); + } + return false; + } +} + +bool DecodeVectorGpr(uint32_t reg, Operand* operand, std::string* error) { + if (operand == nullptr) { + SetError(error, "internal decoder error: null VGPR operand"); + return false; + } + if (reg > 255u) { + SetError(error, "VGPR index is out of range"); + return false; + } + *operand = {}; + operand->kind = OperandKind::Vgpr; + operand->reg = reg; + return true; +} + +bool ReadLiteralOperands(std::span code, uint32_t word_index, Instruction* inst, + std::string* error) { + if (!HasLiteral(*inst)) { + return true; + } + if (word_index + inst->word_count >= code.size()) { + if (error != nullptr) { + *error = fmt::format("missing literal constant at pc 0x{:08x}", inst->pc); + } + return false; + } + + const auto literal = code[word_index + inst->word_count]; + ApplyLiteral(&inst->src0, literal); + ApplyLiteral(&inst->src1, literal); + ApplyLiteral(&inst->src2, literal); + ApplyLiteral(&inst->src3, literal); + inst->word_count++; + SetRawWords(inst, code, word_index, inst->word_count); + return true; +} + +void SetRawWords(Instruction* inst, std::span code, uint32_t word_index, + uint32_t word_count) { + inst->word_count = word_count; + inst->raw_count = std::min(word_count, MaxInstructionRawWords); + for (uint32_t i = 0; i < inst->raw_count; i++) { + inst->raw[i] = code[word_index + i]; + } +} + +void SetUnsupported(Instruction* inst, Family family, uint32_t opcode_id, const char* reason) { + inst->opcode = Opcode::Unsupported; + inst->family = family; + inst->opcode_id = opcode_id; + inst->unsupported_reason = reason; +} + +bool DecodeProgram(std::span code, Program* program, std::string* error) { + if (code.empty() || code.size() > UINT32_MAX / sizeof(uint32_t) || program == nullptr) { + SetError(error, "invalid shader decoder input"); + return false; + } + + program->instructions.clear(); + program->code = code; + + std::set branch_targets; + for (uint32_t word_index = 0; word_index < code.size();) { + const uint32_t pc = word_index * 4u; + const uint32_t word = code[word_index]; + + Instruction inst; + bool ok = false; + if ((word & 0x80000000u) == 0u) { + ok = DecodeVop2(pc, code, word_index, &inst, error); + } else if ((word & 0xc0000000u) == 0x80000000u) { + const auto opcode = (word >> 23u) & 0x7fu; + switch (opcode) { + case 0x7du: ok = DecodeSop1(pc, code, word_index, &inst, error); break; + case 0x7eu: ok = DecodeSopc(pc, code, word_index, &inst, error); break; + case 0x7fu: ok = DecodeSopp(pc, code, word_index, &inst, error); break; + default: + ok = opcode >= 0x60u ? DecodeSopk(pc, code, word_index, &inst, error) + : DecodeSop2(pc, code, word_index, &inst, error); + break; + } + } else { + switch (word >> 26u) { + case 0x32u: ok = DecodeVintrp(pc, code, word_index, &inst, error); break; + case 0x33u: ok = DecodeVop3p(pc, code, word_index, &inst, error); break; + case 0x35u: ok = DecodeVop3(pc, code, word_index, &inst, error); break; + case 0x36u: ok = DecodeDs(pc, code, word_index, &inst, error); break; + case 0x37u: ok = DecodeFlat(pc, code, word_index, &inst, error); break; + case 0x38u: ok = DecodeMubuf(pc, code, word_index, &inst, error); break; + case 0x3au: ok = DecodeMtbuf(pc, code, word_index, &inst, error); break; + case 0x3cu: ok = DecodeMimg(pc, code, word_index, &inst, error); break; + case 0x3du: ok = DecodeSmem(pc, code, word_index, &inst, error); break; + case 0x3eu: ok = DecodeExp(pc, code, word_index, &inst, error); break; + default: + if (error != nullptr) { + *error = fmt::format( + "unknown RDNA2 instruction family at pc 0x{:08x}, raw=0x{:08x}", pc, + word); + } + return false; + } + } + + if (!ok) { + return false; + } + + program->instructions.push_back(inst); + word_index += inst.word_count; + + if (IsControlFlowBranch(inst.opcode)) { + branch_targets.insert(inst.branch_target); + } + if (inst.opcode == Opcode::SEndpgm && + (word_index >= code.size() || !branch_targets.contains(word_index * 4u))) { + return true; + } + } + + SetError(error, "shader decode reached the code boundary before S_ENDPGM"); + return false; +} + +std::string FamilyToString(Family family) { + switch (family) { + case Family::SOP1: return "SOP1"; + case Family::SOP2: return "SOP2"; + case Family::SOPK: return "SOPK"; + case Family::SOPC: return "SOPC"; + case Family::SOPP: return "SOPP"; + case Family::VOP1: return "VOP1"; + case Family::VOP2: return "VOP2"; + case Family::VOP3: return "VOP3"; + case Family::VOP3P: return "VOP3P"; + case Family::VOPC: return "VOPC"; + case Family::VINTRP: return "VINTRP"; + case Family::SMEM: return "SMEM"; + case Family::MUBUF: return "MUBUF"; + case Family::MTBUF: return "MTBUF"; + case Family::FLAT: return "FLAT"; + case Family::DS: return "DS"; + case Family::MIMG: return "MIMG"; + case Family::EXP: return "EXP"; + default: return "Unknown"; + } +} + +std::string OpcodeToString(Opcode opcode) { + switch (opcode) { + case Opcode::SMovB32: return "s_mov_b32"; + case Opcode::SMovB64: return "s_mov_b64"; + case Opcode::SMovkI32: return "s_movk_i32"; + case Opcode::SAbsI32: return "s_abs_i32"; + case Opcode::SBrevB32: return "s_brev_b32"; + case Opcode::SBcnt1I32B32: return "s_bcnt1_i32_b32"; + case Opcode::SBcnt1I32B64: return "s_bcnt1_i32_b64"; + case Opcode::SFf1I32B32: return "s_ff1_i32_b32"; + case Opcode::SFlbitI32B64: return "s_flbit_i32_b64"; + case Opcode::SBitreplicateB64B32: return "s_bitreplicate_b64_b32"; + case Opcode::SGetpcB64: return "s_getpc_b64"; + case Opcode::SSetpcB64: return "s_setpc_b64"; + case Opcode::SAndSaveexecB32: return "s_and_saveexec_b32"; + case Opcode::SAndn1SaveexecB32: return "s_andn1_saveexec_b32"; + case Opcode::SAndSaveexecB64: return "s_and_saveexec_b64"; + case Opcode::SOrn2SaveexecB64: return "s_orn2_saveexec_b64"; + case Opcode::SAndn1SaveexecB64: return "s_andn1_saveexec_b64"; + case Opcode::SNotB32: return "s_not_b32"; + case Opcode::SNotB64: return "s_not_b64"; + case Opcode::SWqmB64: return "s_wqm_b64"; + case Opcode::SAddU32: return "s_add_u32"; + case Opcode::SAddcU32: return "s_addc_u32"; + case Opcode::SSubbU32: return "s_subb_u32"; + case Opcode::SSubU32: return "s_sub_u32"; + case Opcode::SAddI32: return "s_add_i32"; + case Opcode::SSubI32: return "s_sub_i32"; + case Opcode::SBitcmp0B32: return "s_bitcmp0_b32"; + case Opcode::SBitcmp1B32: return "s_bitcmp1_b32"; + case Opcode::SBitset0B32: return "s_bitset0_b32"; + case Opcode::SBitset1B32: return "s_bitset1_b32"; + case Opcode::SMinI32: return "s_min_i32"; + case Opcode::SMaxI32: return "s_max_i32"; + case Opcode::SMinU32: return "s_min_u32"; + case Opcode::SMaxU32: return "s_max_u32"; + case Opcode::SAndB32: return "s_and_b32"; + case Opcode::SAndB64: return "s_and_b64"; + case Opcode::SAndn2B32: return "s_andn2_b32"; + case Opcode::SAndn2B64: return "s_andn2_b64"; + case Opcode::SOrB32: return "s_or_b32"; + case Opcode::SOrB64: return "s_or_b64"; + case Opcode::SOrn2B32: return "s_orn2_b32"; + case Opcode::SOrn2B64: return "s_orn2_b64"; + case Opcode::SXorB32: return "s_xor_b32"; + case Opcode::SXorB64: return "s_xor_b64"; + case Opcode::SNandB32: return "s_nand_b32"; + case Opcode::SNandB64: return "s_nand_b64"; + case Opcode::SNorB32: return "s_nor_b32"; + case Opcode::SNorB64: return "s_nor_b64"; + case Opcode::SXnorB32: return "s_xnor_b32"; + case Opcode::SXnorB64: return "s_xnor_b64"; + case Opcode::SLshlB32: return "s_lshl_b32"; + case Opcode::SLshlB64: return "s_lshl_b64"; + case Opcode::SLshl1AddU32: return "s_lshl1_add_u32"; + case Opcode::SLshl2AddU32: return "s_lshl2_add_u32"; + case Opcode::SLshl3AddU32: return "s_lshl3_add_u32"; + case Opcode::SLshl4AddU32: return "s_lshl4_add_u32"; + case Opcode::SLshrB32: return "s_lshr_b32"; + case Opcode::SLshrB64: return "s_lshr_b64"; + case Opcode::SAshrI32: return "s_ashr_i32"; + case Opcode::SMulI32: return "s_mul_i32"; + case Opcode::SMulHiU32: return "s_mul_hi_u32"; + case Opcode::SMulkI32: return "s_mulk_i32"; + case Opcode::SBfeU32: return "s_bfe_u32"; + case Opcode::SBfeU64: return "s_bfe_u64"; + case Opcode::SBfmB32: return "s_bfm_b32"; + case Opcode::SBfmB64: return "s_bfm_b64"; + case Opcode::SCselectB32: return "s_cselect_b32"; + case Opcode::SCselectB64: return "s_cselect_b64"; + case Opcode::SPackLlB32B16: return "s_pack_ll_b32_b16"; + case Opcode::SPackLhB32B16: return "s_pack_lh_b32_b16"; + case Opcode::SPackHhB32B16: return "s_pack_hh_b32_b16"; + case Opcode::SCmpEqI32: return "s_cmp_eq_i32"; + case Opcode::SCmpLgI32: return "s_cmp_lg_i32"; + case Opcode::SCmpGtI32: return "s_cmp_gt_i32"; + case Opcode::SCmpGeI32: return "s_cmp_ge_i32"; + case Opcode::SCmpLtI32: return "s_cmp_lt_i32"; + case Opcode::SCmpLeI32: return "s_cmp_le_i32"; + case Opcode::SCmpEqU32: return "s_cmp_eq_u32"; + case Opcode::SCmpLgU32: return "s_cmp_lg_u32"; + case Opcode::SCmpGtU32: return "s_cmp_gt_u32"; + case Opcode::SCmpGeU32: return "s_cmp_ge_u32"; + case Opcode::SCmpLtU32: return "s_cmp_lt_u32"; + case Opcode::SCmpLeU32: return "s_cmp_le_u32"; + case Opcode::SCmpLgU64: return "s_cmp_lg_u64"; + case Opcode::VNop: return "v_nop"; + case Opcode::VMovB32: return "v_mov_b32"; + case Opcode::VMovreldB32: return "v_movreld_b32"; + case Opcode::VMovrelsB32: return "v_movrels_b32"; + case Opcode::VReadfirstlaneB32: return "v_readfirstlane_b32"; + case Opcode::VReadlaneB32: return "v_readlane_b32"; + case Opcode::VWritelaneB32: return "v_writelane_b32"; + case Opcode::VPermlane16B32: return "v_permlane16_b32"; + case Opcode::VPermlanex16B32: return "v_permlanex16_b32"; + case Opcode::VCubeidF32: return "v_cubeid_f32"; + case Opcode::VCubescF32: return "v_cubesc_f32"; + case Opcode::VCubetcF32: return "v_cubetc_f32"; + case Opcode::VCubemaF32: return "v_cubema_f32"; + case Opcode::VCndmaskB32: return "v_cndmask_b32"; + case Opcode::VDot2cF32F16: return "v_dot2c_f32_f16"; + case Opcode::VCvtF32I32: return "v_cvt_f32_i32"; + case Opcode::VCvtF32U32: return "v_cvt_f32_u32"; + case Opcode::VCvtU32F32: return "v_cvt_u32_f32"; + case Opcode::VCvtI32F32: return "v_cvt_i32_f32"; + case Opcode::VCvtF16F32: return "v_cvt_f16_f32"; + case Opcode::VCvtF32F16: return "v_cvt_f32_f16"; + case Opcode::VCvtF16U16: return "v_cvt_f16_u16"; + case Opcode::VCvtU16F16: return "v_cvt_u16_f16"; + case Opcode::VCvtF16I16: return "v_cvt_f16_i16"; + case Opcode::VCvtI16F16: return "v_cvt_i16_f16"; + case Opcode::VCvtRpiI32F32: return "v_cvt_rpi_i32_f32"; + case Opcode::VCvtFlrI32F32: return "v_cvt_flr_i32_f32"; + case Opcode::VCvtOffF32I4: return "v_cvt_off_f32_i4"; + case Opcode::VCvtF32Ubyte0: return "v_cvt_f32_ubyte0"; + case Opcode::VCvtF32Ubyte1: return "v_cvt_f32_ubyte1"; + case Opcode::VCvtF32Ubyte2: return "v_cvt_f32_ubyte2"; + case Opcode::VCvtF32Ubyte3: return "v_cvt_f32_ubyte3"; + case Opcode::VRcpF32: return "v_rcp_f32"; + case Opcode::VFractF32: return "v_fract_f32"; + case Opcode::VTruncF32: return "v_trunc_f32"; + case Opcode::VCeilF32: return "v_ceil_f32"; + case Opcode::VRndneF32: return "v_rndne_f32"; + case Opcode::VFloorF32: return "v_floor_f32"; + case Opcode::VExpF32: return "v_exp_f32"; + case Opcode::VLogF32: return "v_log_f32"; + case Opcode::VRsqF32: return "v_rsq_f32"; + case Opcode::VSqrtF32: return "v_sqrt_f32"; + case Opcode::VRcpF16: return "v_rcp_f16"; + case Opcode::VSqrtF16: return "v_sqrt_f16"; + case Opcode::VRsqF16: return "v_rsq_f16"; + case Opcode::VLogF16: return "v_log_f16"; + case Opcode::VExpF16: return "v_exp_f16"; + case Opcode::VFloorF16: return "v_floor_f16"; + case Opcode::VCeilF16: return "v_ceil_f16"; + case Opcode::VTruncF16: return "v_trunc_f16"; + case Opcode::VRndneF16: return "v_rndne_f16"; + case Opcode::VSinF32: return "v_sin_f32"; + case Opcode::VCosF32: return "v_cos_f32"; + case Opcode::VNotB32: return "v_not_b32"; + case Opcode::VBfrevB32: return "v_bfrev_b32"; + case Opcode::VFfbhU32: return "v_ffbh_u32"; + case Opcode::VFfblB32: return "v_ffbl_b32"; + case Opcode::VAddF32: return "v_add_f32"; + case Opcode::VSubF32: return "v_sub_f32"; + case Opcode::VSubrevF32: return "v_subrev_f32"; + case Opcode::VMulF32: return "v_mul_f32"; + case Opcode::VMacF32: return "v_mac_f32"; + case Opcode::VMadmkF32: return "v_madmk_f32"; + case Opcode::VMadakF32: return "v_madak_f32"; + case Opcode::VMinF32: return "v_min_f32"; + case Opcode::VMaxF32: return "v_max_f32"; + case Opcode::VAddF16: return "v_add_f16"; + case Opcode::VSubF16: return "v_sub_f16"; + case Opcode::VSubrevF16: return "v_subrev_f16"; + case Opcode::VMulF16: return "v_mul_f16"; + case Opcode::VFmacF16: return "v_fmac_f16"; + case Opcode::VFmamkF16: return "v_fmamk_f16"; + case Opcode::VFmaakF16: return "v_fmaak_f16"; + case Opcode::VMinI32: return "v_min_i32"; + case Opcode::VMaxI32: return "v_max_i32"; + case Opcode::VMulI32I24: return "v_mul_i32_i24"; + case Opcode::VMulU32U24: return "v_mul_u32_u24"; + case Opcode::VMulLoU32: return "v_mul_lo_u32"; + case Opcode::VMulHiU32: return "v_mul_hi_u32"; + case Opcode::VMulLoI32: return "v_mul_lo_i32"; + case Opcode::VMulHiI32: return "v_mul_hi_i32"; + case Opcode::VCvtPkrtzF16F32: return "v_cvt_pkrtz_f16_f32"; + case Opcode::VCvtPkU8F32: return "v_cvt_pk_u8_f32"; + case Opcode::VMadF32: return "v_mad_f32"; + case Opcode::VMadI32I24: return "v_mad_i32_i24"; + case Opcode::VMadU32U24: return "v_mad_u32_u24"; + case Opcode::VMadU64U32: return "v_mad_u64_u32"; + case Opcode::VFmaF32: return "v_fma_f32"; + case Opcode::VFmaF16: return "v_fma_f16"; + case Opcode::VPackB32F16: return "v_pack_b32_f16"; + case Opcode::VBfeU32: return "v_bfe_u32"; + case Opcode::VBfeI32: return "v_bfe_i32"; + case Opcode::VBfiB32: return "v_bfi_b32"; + case Opcode::VAlignbitB32: return "v_alignbit_b32"; + case Opcode::VMin3F32: return "v_min3_f32"; + case Opcode::VMin3I32: return "v_min3_i32"; + case Opcode::VMin3U32: return "v_min3_u32"; + case Opcode::VMin3F16: return "v_min3_f16"; + case Opcode::VMax3F32: return "v_max3_f32"; + case Opcode::VMax3I32: return "v_max3_i32"; + case Opcode::VMax3U32: return "v_max3_u32"; + case Opcode::VMax3F16: return "v_max3_f16"; + case Opcode::VMed3F32: return "v_med3_f32"; + case Opcode::VMed3I32: return "v_med3_i32"; + case Opcode::VMed3U32: return "v_med3_u32"; + case Opcode::VMed3F16: return "v_med3_f16"; + case Opcode::VSadU32: return "v_sad_u32"; + case Opcode::VAdd3U32: return "v_add3_u32"; + case Opcode::VLshlAddU32: return "v_lshl_add_u32"; + case Opcode::VAddLshlU32: return "v_add_lshl_u32"; + case Opcode::VXadU32: return "v_xad_u32"; + case Opcode::VLshlOrB32: return "v_lshl_or_b32"; + case Opcode::VAndOrB32: return "v_and_or_b32"; + case Opcode::VOr3B32: return "v_or3_b32"; + case Opcode::VXor3B32: return "v_xor3_b32"; + case Opcode::VAddI32: return "v_add_i32"; + case Opcode::VSubI32: return "v_sub_i32"; + case Opcode::VSubrevI32: return "v_subrev_i32"; + case Opcode::VBfmB32: return "v_bfm_b32"; + case Opcode::VLdexpF32: return "v_ldexp_f32"; + case Opcode::VCvtPknormI16F32: return "v_cvt_pknorm_i16_f32"; + case Opcode::VCvtPknormU16F32: return "v_cvt_pknorm_u16_f32"; + case Opcode::VCvtPkU16U32: return "v_cvt_pk_u16_u32"; + case Opcode::VPkMadI16: return "v_pk_mad_i16"; + case Opcode::VPkMulLoU16: return "v_pk_mul_lo_u16"; + case Opcode::VPkAddI16: return "v_pk_add_i16"; + case Opcode::VPkSubI16: return "v_pk_sub_i16"; + case Opcode::VPkLshlrevB16: return "v_pk_lshlrev_b16"; + case Opcode::VPkLshrrevB16: return "v_pk_lshrrev_b16"; + case Opcode::VPkAshrrevI16: return "v_pk_ashrrev_i16"; + case Opcode::VPkMaxI16: return "v_pk_max_i16"; + case Opcode::VPkMinI16: return "v_pk_min_i16"; + case Opcode::VPkMadU16: return "v_pk_mad_u16"; + case Opcode::VPkAddF16: return "v_pk_add_f16"; + case Opcode::VPkMulF16: return "v_pk_mul_f16"; + case Opcode::VPkMinF16: return "v_pk_min_f16"; + case Opcode::VPkMaxF16: return "v_pk_max_f16"; + case Opcode::VPkFmaF16: return "v_pk_fma_f16"; + case Opcode::VPkFmacF16: return "v_pk_fmac_f16"; + case Opcode::VPkAddU16: return "v_pk_add_u16"; + case Opcode::VPkSubU16: return "v_pk_sub_u16"; + case Opcode::VPkMaxU16: return "v_pk_max_u16"; + case Opcode::VPkMinU16: return "v_pk_min_u16"; + case Opcode::VMadMixloF16: return "v_mad_mixlo_f16"; + case Opcode::VMadMixhiF16: return "v_mad_mixhi_f16"; + case Opcode::VAddNcU32: return "v_add_nc_u32"; + case Opcode::VAddcU32: return "v_addc_u32"; + case Opcode::VSubNcU32: return "v_sub_nc_u32"; + case Opcode::VSubrevNcU32: return "v_subrev_nc_u32"; + case Opcode::VAddNcU16: return "v_add_nc_u16"; + case Opcode::VSubNcU16: return "v_sub_nc_u16"; + case Opcode::VMaxU16: return "v_max_u16"; + case Opcode::VMaxI16: return "v_max_i16"; + case Opcode::VMinU16: return "v_min_u16"; + case Opcode::VMinI16: return "v_min_i16"; + case Opcode::VAddNcI16: return "v_add_nc_i16"; + case Opcode::VSubNcI16: return "v_sub_nc_i16"; + case Opcode::VAndB32: return "v_and_b32"; + case Opcode::VOrB32: return "v_or_b32"; + case Opcode::VXorB32: return "v_xor_b32"; + case Opcode::VXnorB32: return "v_xnor_b32"; + case Opcode::VLshlB32: return "v_lshl_b32"; + case Opcode::VLshlrevB32: return "v_lshlrev_b32"; + case Opcode::VLshrB32: return "v_lshr_b32"; + case Opcode::VLshrrevB32: return "v_lshrrev_b32"; + case Opcode::VAshrI32: return "v_ashr_i32"; + case Opcode::VAshrrevI32: return "v_ashrrev_i32"; + case Opcode::VLshlrevB16: return "v_lshlrev_b16"; + case Opcode::VLshrrevB16: return "v_lshrrev_b16"; + case Opcode::VAshrrevI16: return "v_ashrrev_i16"; + case Opcode::VBcntU32B32: return "v_bcnt_u32_b32"; + case Opcode::VMbcntLoU32B32: return "v_mbcnt_lo_u32_b32"; + case Opcode::VMbcntHiU32B32: return "v_mbcnt_hi_u32_b32"; + case Opcode::VMinU32: return "v_min_u32"; + case Opcode::VMaxU32: return "v_max_u32"; + case Opcode::VMaxF16: return "v_max_f16"; + case Opcode::VMinF16: return "v_min_f16"; + case Opcode::VCmpFF32: return "v_cmp_f_f32"; + case Opcode::VCmpLtF32: return "v_cmp_lt_f32"; + case Opcode::VCmpEqF32: return "v_cmp_eq_f32"; + case Opcode::VCmpLeF32: return "v_cmp_le_f32"; + case Opcode::VCmpGtF32: return "v_cmp_gt_f32"; + case Opcode::VCmpLgF32: return "v_cmp_lg_f32"; + case Opcode::VCmpGeF32: return "v_cmp_ge_f32"; + case Opcode::VCmpOF32: return "v_cmp_o_f32"; + case Opcode::VCmpUF32: return "v_cmp_u_f32"; + case Opcode::VCmpNgeF32: return "v_cmp_nge_f32"; + case Opcode::VCmpNlgF32: return "v_cmp_nlg_f32"; + case Opcode::VCmpNgtF32: return "v_cmp_ngt_f32"; + case Opcode::VCmpNleF32: return "v_cmp_nle_f32"; + case Opcode::VCmpNeqF32: return "v_cmp_neq_f32"; + case Opcode::VCmpNltF32: return "v_cmp_nlt_f32"; + case Opcode::VCmpTruF32: return "v_cmp_tru_f32"; + case Opcode::VCmpxLtF32: return "v_cmpx_lt_f32"; + case Opcode::VCmpxEqF32: return "v_cmpx_eq_f32"; + case Opcode::VCmpxLeF32: return "v_cmpx_le_f32"; + case Opcode::VCmpxGtF32: return "v_cmpx_gt_f32"; + case Opcode::VCmpxLgF32: return "v_cmpx_lg_f32"; + case Opcode::VCmpxGeF32: return "v_cmpx_ge_f32"; + case Opcode::VCmpxNgeF32: return "v_cmpx_nge_f32"; + case Opcode::VCmpxNlgF32: return "v_cmpx_nlg_f32"; + case Opcode::VCmpxNgtF32: return "v_cmpx_ngt_f32"; + case Opcode::VCmpxNleF32: return "v_cmpx_nle_f32"; + case Opcode::VCmpxNeqF32: return "v_cmpx_neq_f32"; + case Opcode::VCmpxNltF32: return "v_cmpx_nlt_f32"; + case Opcode::VCmpFI32: return "v_cmp_f_i32"; + case Opcode::VCmpLtI32: return "v_cmp_lt_i32"; + case Opcode::VCmpEqI32: return "v_cmp_eq_i32"; + case Opcode::VCmpLeI32: return "v_cmp_le_i32"; + case Opcode::VCmpGtI32: return "v_cmp_gt_i32"; + case Opcode::VCmpNeI32: return "v_cmp_ne_i32"; + case Opcode::VCmpGeI32: return "v_cmp_ge_i32"; + case Opcode::VCmpTI32: return "v_cmp_t_i32"; + case Opcode::VCmpClassF32: return "v_cmp_class_f32"; + case Opcode::VCmpLtI16: return "v_cmp_lt_i16"; + case Opcode::VCmpEqI16: return "v_cmp_eq_i16"; + case Opcode::VCmpLeI16: return "v_cmp_le_i16"; + case Opcode::VCmpGtI16: return "v_cmp_gt_i16"; + case Opcode::VCmpNeI16: return "v_cmp_ne_i16"; + case Opcode::VCmpGeI16: return "v_cmp_ge_i16"; + case Opcode::VCmpLtF16: return "v_cmp_lt_f16"; + case Opcode::VCmpEqF16: return "v_cmp_eq_f16"; + case Opcode::VCmpLeF16: return "v_cmp_le_f16"; + case Opcode::VCmpGtF16: return "v_cmp_gt_f16"; + case Opcode::VCmpLgF16: return "v_cmp_lg_f16"; + case Opcode::VCmpGeF16: return "v_cmp_ge_f16"; + case Opcode::VCmpNeqF16: return "v_cmp_neq_f16"; + case Opcode::VCmpxLtF16: return "v_cmpx_lt_f16"; + case Opcode::VCmpxEqF16: return "v_cmpx_eq_f16"; + case Opcode::VCmpxLeF16: return "v_cmpx_le_f16"; + case Opcode::VCmpxGtF16: return "v_cmpx_gt_f16"; + case Opcode::VCmpxGeF16: return "v_cmpx_ge_f16"; + case Opcode::VCmpxNeqF16: return "v_cmpx_neq_f16"; + case Opcode::VCmpxNltF16: return "v_cmpx_nlt_f16"; + case Opcode::VCmpxLtI32: return "v_cmpx_lt_i32"; + case Opcode::VCmpxEqI32: return "v_cmpx_eq_i32"; + case Opcode::VCmpxLeI32: return "v_cmpx_le_i32"; + case Opcode::VCmpxGtI32: return "v_cmpx_gt_i32"; + case Opcode::VCmpxNeI32: return "v_cmpx_ne_i32"; + case Opcode::VCmpxGeI32: return "v_cmpx_ge_i32"; + case Opcode::VCmpLtU16: return "v_cmp_lt_u16"; + case Opcode::VCmpEqU16: return "v_cmp_eq_u16"; + case Opcode::VCmpLeU16: return "v_cmp_le_u16"; + case Opcode::VCmpGtU16: return "v_cmp_gt_u16"; + case Opcode::VCmpNeU16: return "v_cmp_ne_u16"; + case Opcode::VCmpGeU16: return "v_cmp_ge_u16"; + case Opcode::VCmpFU32: return "v_cmp_f_u32"; + case Opcode::VCmpLtU32: return "v_cmp_lt_u32"; + case Opcode::VCmpEqU32: return "v_cmp_eq_u32"; + case Opcode::VCmpLeU32: return "v_cmp_le_u32"; + case Opcode::VCmpGtU32: return "v_cmp_gt_u32"; + case Opcode::VCmpNeU32: return "v_cmp_ne_u32"; + case Opcode::VCmpGeU32: return "v_cmp_ge_u32"; + case Opcode::VCmpTU32: return "v_cmp_t_u32"; + case Opcode::VCmpNeU64: return "v_cmp_ne_u64"; + case Opcode::VCmpxLtU32: return "v_cmpx_lt_u32"; + case Opcode::VCmpxEqU32: return "v_cmpx_eq_u32"; + case Opcode::VCmpxLeU32: return "v_cmpx_le_u32"; + case Opcode::VCmpxGtU32: return "v_cmpx_gt_u32"; + case Opcode::VCmpxNeU32: return "v_cmpx_ne_u32"; + case Opcode::VCmpxGeU32: return "v_cmpx_ge_u32"; + case Opcode::SLoadDword: return "s_load_dword"; + case Opcode::SLoadDwordx2: return "s_load_dwordx2"; + case Opcode::SLoadDwordx4: return "s_load_dwordx4"; + case Opcode::SLoadDwordx8: return "s_load_dwordx8"; + case Opcode::SLoadDwordx16: return "s_load_dwordx16"; + case Opcode::SBufferLoadDword: return "s_buffer_load_dword"; + case Opcode::SBufferLoadDwordx2: return "s_buffer_load_dwordx2"; + case Opcode::SBufferLoadDwordx4: return "s_buffer_load_dwordx4"; + case Opcode::SBufferLoadDwordx8: return "s_buffer_load_dwordx8"; + case Opcode::SBufferLoadDwordx16: return "s_buffer_load_dwordx16"; + case Opcode::BufferLoadFormatX: return "buffer_load_format_x"; + case Opcode::BufferLoadFormatXy: return "buffer_load_format_xy"; + case Opcode::BufferLoadFormatXyz: return "buffer_load_format_xyz"; + case Opcode::BufferLoadFormatXyzw: return "buffer_load_format_xyzw"; + case Opcode::BufferStoreFormatX: return "buffer_store_format_x"; + case Opcode::BufferStoreFormatXy: return "buffer_store_format_xy"; + case Opcode::BufferStoreFormatXyz: return "buffer_store_format_xyz"; + case Opcode::BufferStoreFormatXyzw: return "buffer_store_format_xyzw"; + case Opcode::BufferLoadUbyte: return "buffer_load_ubyte"; + case Opcode::BufferLoadSbyte: return "buffer_load_sbyte"; + case Opcode::BufferLoadUshort: return "buffer_load_ushort"; + case Opcode::BufferLoadSshort: return "buffer_load_sshort"; + case Opcode::BufferLoadDword: return "buffer_load_dword"; + case Opcode::BufferLoadDwordx2: return "buffer_load_dwordx2"; + case Opcode::BufferLoadDwordx3: return "buffer_load_dwordx3"; + case Opcode::BufferLoadDwordx4: return "buffer_load_dwordx4"; + case Opcode::BufferStoreByte: return "buffer_store_byte"; + case Opcode::BufferStoreShort: return "buffer_store_short"; + case Opcode::BufferStoreDword: return "buffer_store_dword"; + case Opcode::BufferStoreDwordx2: return "buffer_store_dwordx2"; + case Opcode::BufferStoreDwordx3: return "buffer_store_dwordx3"; + case Opcode::BufferStoreDwordx4: return "buffer_store_dwordx4"; + case Opcode::TBufferLoadFormatX: return "tbuffer_load_format_x"; + case Opcode::TBufferLoadFormatXy: return "tbuffer_load_format_xy"; + case Opcode::TBufferLoadFormatXyz: return "tbuffer_load_format_xyz"; + case Opcode::TBufferLoadFormatXyzw: return "tbuffer_load_format_xyzw"; + case Opcode::TBufferStoreFormatX: return "tbuffer_store_format_x"; + case Opcode::TBufferStoreFormatXy: return "tbuffer_store_format_xy"; + case Opcode::TBufferStoreFormatXyz: return "tbuffer_store_format_xyz"; + case Opcode::TBufferStoreFormatXyzw: return "tbuffer_store_format_xyzw"; + case Opcode::BufferAtomicSwap: return "buffer_atomic_swap"; + case Opcode::BufferAtomicAdd: return "buffer_atomic_add"; + case Opcode::BufferAtomicSub: return "buffer_atomic_sub"; + case Opcode::BufferAtomicSMin: return "buffer_atomic_smin"; + case Opcode::BufferAtomicUMin: return "buffer_atomic_umin"; + case Opcode::BufferAtomicSMax: return "buffer_atomic_smax"; + case Opcode::BufferAtomicUMax: return "buffer_atomic_umax"; + case Opcode::BufferAtomicAnd: return "buffer_atomic_and"; + case Opcode::BufferAtomicOr: return "buffer_atomic_or"; + case Opcode::BufferAtomicXor: return "buffer_atomic_xor"; + case Opcode::FlatLoadUbyte: return "flat_load_ubyte"; + case Opcode::FlatLoadSbyte: return "flat_load_sbyte"; + case Opcode::FlatLoadUshort: return "flat_load_ushort"; + case Opcode::FlatLoadSshort: return "flat_load_sshort"; + case Opcode::FlatLoadDword: return "flat_load_dword"; + case Opcode::FlatLoadDwordx2: return "flat_load_dwordx2"; + case Opcode::FlatLoadDwordx3: return "flat_load_dwordx3"; + case Opcode::FlatLoadDwordx4: return "flat_load_dwordx4"; + case Opcode::FlatStoreByte: return "flat_store_byte"; + case Opcode::FlatStoreShort: return "flat_store_short"; + case Opcode::FlatStoreDword: return "flat_store_dword"; + case Opcode::FlatStoreDwordx2: return "flat_store_dwordx2"; + case Opcode::FlatStoreDwordx3: return "flat_store_dwordx3"; + case Opcode::FlatStoreDwordx4: return "flat_store_dwordx4"; + case Opcode::DsAddU32: return "ds_add_u32"; + case Opcode::DsAddRtnU32: return "ds_add_rtn_u32"; + case Opcode::DsSubU32: return "ds_sub_u32"; + case Opcode::DsSubRtnU32: return "ds_sub_rtn_u32"; + case Opcode::DsMinI32: return "ds_min_i32"; + case Opcode::DsMinRtnI32: return "ds_min_rtn_i32"; + case Opcode::DsMaxI32: return "ds_max_i32"; + case Opcode::DsMaxRtnI32: return "ds_max_rtn_i32"; + case Opcode::DsMinU32: return "ds_min_u32"; + case Opcode::DsMinRtnU32: return "ds_min_rtn_u32"; + case Opcode::DsMaxU32: return "ds_max_u32"; + case Opcode::DsMaxRtnU32: return "ds_max_rtn_u32"; + case Opcode::DsAndB32: return "ds_and_b32"; + case Opcode::DsAndRtnB32: return "ds_and_rtn_b32"; + case Opcode::DsOrB32: return "ds_or_b32"; + case Opcode::DsOrRtnB32: return "ds_or_rtn_b32"; + case Opcode::DsXorB32: return "ds_xor_b32"; + case Opcode::DsXorRtnB32: return "ds_xor_rtn_b32"; + case Opcode::DsWrxchgRtnB32: return "ds_wrxchg_rtn_b32"; + case Opcode::DsMinF32: return "ds_min_f32"; + case Opcode::DsMaxF32: return "ds_max_f32"; + case Opcode::DsSwizzleB32: return "ds_swizzle_b32"; + case Opcode::DsConsume: return "ds_consume"; + case Opcode::DsAppend: return "ds_append"; + case Opcode::DsReadSbyte: return "ds_read_i8"; + case Opcode::DsReadUbyte: return "ds_read_u8"; + case Opcode::DsReadSshort: return "ds_read_i16"; + case Opcode::DsReadUshort: return "ds_read_u16"; + case Opcode::DsRead2B32: return "ds_read2_b32"; + case Opcode::DsReadB32: return "ds_read_b32"; + case Opcode::DsReadB64: return "ds_read_b64"; + case Opcode::DsRead2B64: return "ds_read2_b64"; + case Opcode::DsRead2St64B64: return "ds_read2st64_b64"; + case Opcode::DsReadB96: return "ds_read_b96"; + case Opcode::DsReadB128: return "ds_read_b128"; + case Opcode::DsWriteByte: return "ds_write_b8"; + case Opcode::DsWriteShort: return "ds_write_b16"; + case Opcode::DsWrite2B32: return "ds_write2_b32"; + case Opcode::DsWrite2St64B32: return "ds_write2st64_b32"; + case Opcode::DsWrite2B64: return "ds_write2_b64"; + case Opcode::DsWrite2St64B64: return "ds_write2st64_b64"; + case Opcode::DsWriteB32: return "ds_write_b32"; + case Opcode::DsWriteB64: return "ds_write_b64"; + case Opcode::DsWriteB96: return "ds_write_b96"; + case Opcode::DsWriteB128: return "ds_write_b128"; + case Opcode::DsWriteAddtidB32: return "ds_write_addtid_b32"; + case Opcode::DsReadAddtidB32: return "ds_read_addtid_b32"; + case Opcode::ImageGetResinfo: return "image_get_resinfo"; + case Opcode::ImageGetLod: return "image_get_lod"; + case Opcode::ImageLoad: return "image_load"; + case Opcode::ImageLoadMip: return "image_load_mip"; + case Opcode::ImageStore: return "image_store"; + case Opcode::ImageStoreMip: return "image_store_mip"; + case Opcode::ImageAtomicAdd: return "image_atomic_add"; + case Opcode::ImageAtomicUMin: return "image_atomic_umin"; + case Opcode::ImageAtomicUMax: return "image_atomic_umax"; + case Opcode::ImageAtomicAnd: return "image_atomic_and"; + case Opcode::ImageAtomicOr: return "image_atomic_or"; + case Opcode::ImageAtomicXor: return "image_atomic_xor"; + case Opcode::ImageSample: return "image_sample"; + case Opcode::ImageGather4Lz: return "image_gather4_lz"; + case Opcode::ImageGather4C: return "image_gather4_c"; + case Opcode::ImageGather4CLz: return "image_gather4_c_lz"; + case Opcode::ImageGather4LzO: return "image_gather4_lz_o"; + case Opcode::ImageGather4CO: return "image_gather4_c_o"; + case Opcode::ImageGather4CLzO: return "image_gather4_c_lz_o"; + case Opcode::ImageGather4H: return "image_gather4h"; + case Opcode::VInterpP1F32: return "v_interp_p1_f32"; + case Opcode::VInterpP2F32: return "v_interp_p2_f32"; + case Opcode::VInterpMovF32: return "v_interp_mov_f32"; + case Opcode::SNop: return "s_nop"; + case Opcode::SWaitcnt: return "s_waitcnt"; + case Opcode::SBarrier: return "s_barrier"; + case Opcode::SBranch: return "s_branch"; + case Opcode::SCbranchScc0: return "s_cbranch_scc0"; + case Opcode::SCbranchScc1: return "s_cbranch_scc1"; + case Opcode::SCbranchVccz: return "s_cbranch_vccz"; + case Opcode::SCbranchVccnz: return "s_cbranch_vccnz"; + case Opcode::SCbranchExecz: return "s_cbranch_execz"; + case Opcode::SCbranchExecnz: return "s_cbranch_execnz"; + case Opcode::SSendmsg: return "s_sendmsg"; + case Opcode::SSetregB32: return "s_setreg_b32"; + case Opcode::SSleep: return "s_sleep"; + case Opcode::STtraceData: return "s_ttracedata"; + case Opcode::SInstPrefetch: return "s_inst_prefetch"; + case Opcode::SEndpgm: return "s_endpgm"; + case Opcode::Exp: return "exp"; + case Opcode::Unsupported: return "unsupported"; + default: return "unknown"; + } +} + +std::string OperandToString(const Operand& operand) { + std::string text; + switch (operand.kind) { + case OperandKind::LiteralConstant: text = fmt::format("0x{:08x}", operand.value); break; + case OperandKind::IntegerInlineConstant: + text = fmt::format("{}", operand.signed_val); + break; + case OperandKind::FloatInlineConstant: text = fmt::format("{:f}", operand.float_val); break; + case OperandKind::Sgpr: text = fmt::format("s{}", operand.reg); break; + case OperandKind::Vgpr: text = fmt::format("v{}", operand.reg); break; + case OperandKind::VccLo: text = "vcc_lo"; break; + case OperandKind::VccHi: text = "vcc_hi"; break; + case OperandKind::VccZ: text = "vccz"; break; + case OperandKind::ExecLo: text = "exec_lo"; break; + case OperandKind::ExecHi: text = "exec_hi"; break; + case OperandKind::ExecZ: text = "execz"; break; + case OperandKind::Scc: text = "scc"; break; + case OperandKind::M0: text = "m0"; break; + case OperandKind::PopsExitingWaveId: text = "pops_exiting_wave_id"; break; + case OperandKind::Null: text = "null"; break; + default: text = "unknown"; break; + } + if (operand.sdwa_sel != 6 || operand.sdwa_sext) { + text += fmt::format(".sdwa(sel={},sext={})", operand.sdwa_sel, operand.sdwa_sext ? 1u : 0u); + } + if (operand.absolute) { + text += ".abs"; + } + if (operand.negate) { + text += ".neg"; + } + if (operand.op_sel || operand.op_sel_hi || operand.negate_hi) { + text += fmt::format(".opsel(lo={},hi={},neghi={})", operand.op_sel ? 1u : 0u, + operand.op_sel_hi ? 1u : 0u, operand.negate_hi ? 1u : 0u); + } + if (operand.omod != 0) { + text += fmt::format(".omod({})", operand.omod); + } + if (operand.clamp) { + text += ".clamp"; + } + if (operand.dpp) { + text += fmt::format(".dpp(ctrl=0x{:x},fi={},bc={})", operand.dpp_ctrl, + operand.dpp_fetch_inactive ? 1u : 0u, operand.dpp_bound_ctrl ? 1u : 0u); + } + return text; +} + +std::string InstructionToString(const Instruction& inst) { + if (inst.opcode == Opcode::Unsupported) { + return fmt::format("0x{:08x}: unsupported family={} opcode=0x{:02x} raw=[{}] reason={}", + inst.pc, FamilyToString(inst.family).c_str(), inst.opcode_id, + RawWordsToString(inst).c_str(), inst.unsupported_reason.c_str()); + } + + switch (inst.opcode) { + case Opcode::SMovB32: + case Opcode::SMovB64: + return WithUnsupportedReason(inst, fmt::format("0x{:08x}: {} {}, {}", inst.pc, + OpcodeToString(inst.opcode).c_str(), + OperandToString(inst.dst).c_str(), + OperandToString(inst.src0).c_str())); + case Opcode::SAbsI32: + case Opcode::SBrevB32: + case Opcode::SBcnt1I32B32: + case Opcode::SFf1I32B32: + case Opcode::SNotB64: + case Opcode::SWqmB64: + case Opcode::SAndSaveexecB32: + case Opcode::SAndn1SaveexecB32: + case Opcode::SAndSaveexecB64: + case Opcode::SOrn2SaveexecB64: + case Opcode::SAndn1SaveexecB64: + return WithUnsupportedReason(inst, fmt::format("0x{:08x}: {} {}, {}", inst.pc, + OpcodeToString(inst.opcode).c_str(), + OperandToString(inst.dst).c_str(), + OperandToString(inst.src0).c_str())); + case Opcode::SGetpcB64: + return WithUnsupportedReason(inst, fmt::format("0x{:08x}: s_getpc_b64 {}", inst.pc, + OperandToString(inst.dst).c_str())); + case Opcode::SSetpcB64: + return WithUnsupportedReason(inst, fmt::format("0x{:08x}: s_setpc_b64 {}", inst.pc, + OperandToString(inst.src0).c_str())); + case Opcode::SSetregB32: + return WithUnsupportedReason(inst, fmt::format("0x{:08x}: s_setreg_b32 {}, {}", inst.pc, + OperandToString(inst.src0).c_str(), + OperandToString(inst.src1).c_str())); + case Opcode::SNop: + case Opcode::SWaitcnt: + case Opcode::SSleep: + case Opcode::SSendmsg: + case Opcode::STtraceData: + case Opcode::SInstPrefetch: + return WithUnsupportedReason(inst, fmt::format("0x{:08x}: {} {}", inst.pc, + OpcodeToString(inst.opcode).c_str(), + OperandToString(inst.src0).c_str())); + case Opcode::SBarrier: + return WithUnsupportedReason(inst, fmt::format("0x{:08x}: s_barrier", inst.pc)); + case Opcode::VNop: + return WithUnsupportedReason(inst, fmt::format("0x{:08x}: v_nop", inst.pc)); + case Opcode::SEndpgm: + return WithUnsupportedReason(inst, fmt::format("0x{:08x}: s_endpgm", inst.pc)); + case Opcode::SBranch: + case Opcode::SCbranchScc0: + case Opcode::SCbranchScc1: + case Opcode::SCbranchVccz: + case Opcode::SCbranchVccnz: + case Opcode::SCbranchExecz: + case Opcode::SCbranchExecnz: + return WithUnsupportedReason(inst, fmt::format("0x{:08x}: {} 0x{:08x}", inst.pc, + OpcodeToString(inst.opcode).c_str(), + inst.branch_target)); + case Opcode::Exp: return WithUnsupportedReason(inst, FormatExp(inst)); + case Opcode::ImageSample: + case Opcode::ImageStore: + case Opcode::ImageStoreMip: + case Opcode::ImageAtomicAdd: + case Opcode::ImageAtomicUMin: + case Opcode::ImageAtomicUMax: + case Opcode::ImageAtomicAnd: + case Opcode::ImageAtomicOr: + case Opcode::ImageAtomicXor: + case Opcode::ImageLoad: + case Opcode::ImageLoadMip: + case Opcode::ImageGetResinfo: + case Opcode::ImageGetLod: + case Opcode::ImageGather4Lz: + case Opcode::ImageGather4C: + case Opcode::ImageGather4CLz: + case Opcode::ImageGather4LzO: + case Opcode::ImageGather4CO: + case Opcode::ImageGather4CLzO: + case Opcode::ImageGather4H: return WithUnsupportedReason(inst, FormatMimg(inst)); + case Opcode::SLoadDword: + case Opcode::SLoadDwordx2: + case Opcode::SLoadDwordx4: + case Opcode::SLoadDwordx8: + case Opcode::SLoadDwordx16: + case Opcode::SBufferLoadDword: + case Opcode::SBufferLoadDwordx2: + case Opcode::SBufferLoadDwordx4: + case Opcode::SBufferLoadDwordx8: + case Opcode::SBufferLoadDwordx16: + case Opcode::BufferLoadFormatX: + case Opcode::BufferLoadFormatXy: + case Opcode::BufferLoadFormatXyz: + case Opcode::BufferLoadFormatXyzw: + case Opcode::BufferStoreFormatX: + case Opcode::BufferStoreFormatXy: + case Opcode::BufferStoreFormatXyz: + case Opcode::BufferStoreFormatXyzw: + case Opcode::BufferLoadUbyte: + case Opcode::BufferLoadUshort: + case Opcode::BufferLoadDword: + case Opcode::BufferLoadDwordx2: + case Opcode::BufferLoadDwordx3: + case Opcode::BufferLoadDwordx4: + case Opcode::BufferStoreByte: + case Opcode::BufferStoreShort: + case Opcode::BufferStoreDword: + case Opcode::BufferStoreDwordx2: + case Opcode::BufferStoreDwordx3: + case Opcode::BufferStoreDwordx4: + case Opcode::TBufferLoadFormatX: + case Opcode::TBufferLoadFormatXy: + case Opcode::TBufferLoadFormatXyz: + case Opcode::TBufferLoadFormatXyzw: + case Opcode::TBufferStoreFormatX: + case Opcode::TBufferStoreFormatXy: + case Opcode::TBufferStoreFormatXyz: + case Opcode::TBufferStoreFormatXyzw: + case Opcode::BufferAtomicSwap: + case Opcode::BufferAtomicAdd: + case Opcode::BufferAtomicSub: + case Opcode::BufferAtomicSMin: + case Opcode::BufferAtomicUMin: + case Opcode::BufferAtomicSMax: + case Opcode::BufferAtomicUMax: + case Opcode::BufferAtomicAnd: + case Opcode::BufferAtomicOr: + case Opcode::BufferAtomicXor: + case Opcode::BufferLoadSbyte: + case Opcode::BufferLoadSshort: + case Opcode::FlatLoadUbyte: + case Opcode::FlatLoadSbyte: + case Opcode::FlatLoadUshort: + case Opcode::FlatLoadSshort: + case Opcode::FlatLoadDword: + case Opcode::FlatLoadDwordx2: + case Opcode::FlatLoadDwordx3: + case Opcode::FlatLoadDwordx4: + case Opcode::FlatStoreByte: + case Opcode::FlatStoreShort: + case Opcode::FlatStoreDword: + case Opcode::FlatStoreDwordx2: + case Opcode::FlatStoreDwordx3: + case Opcode::FlatStoreDwordx4: + case Opcode::DsAddU32: + case Opcode::DsAddRtnU32: + case Opcode::DsSubU32: + case Opcode::DsSubRtnU32: + case Opcode::DsMinI32: + case Opcode::DsMinRtnI32: + case Opcode::DsMaxI32: + case Opcode::DsMaxRtnI32: + case Opcode::DsMinU32: + case Opcode::DsMinRtnU32: + case Opcode::DsMaxU32: + case Opcode::DsMaxRtnU32: + case Opcode::DsAndB32: + case Opcode::DsAndRtnB32: + case Opcode::DsOrB32: + case Opcode::DsOrRtnB32: + case Opcode::DsXorB32: + case Opcode::DsXorRtnB32: + case Opcode::DsWrxchgRtnB32: + case Opcode::DsMinF32: + case Opcode::DsMaxF32: + case Opcode::DsSwizzleB32: + case Opcode::DsReadSbyte: + case Opcode::DsReadUbyte: + case Opcode::DsReadSshort: + case Opcode::DsReadUshort: + case Opcode::DsRead2B32: + case Opcode::DsReadB32: + case Opcode::DsReadB64: + case Opcode::DsRead2B64: + case Opcode::DsRead2St64B64: + case Opcode::DsReadB96: + case Opcode::DsReadB128: + case Opcode::DsWriteByte: + case Opcode::DsWriteShort: + case Opcode::DsWrite2B32: + case Opcode::DsWrite2St64B32: + case Opcode::DsWrite2B64: + case Opcode::DsWrite2St64B64: + case Opcode::DsWriteB32: + case Opcode::DsWriteB64: + case Opcode::DsWriteB96: + case Opcode::DsWriteB128: + case Opcode::DsWriteAddtidB32: + case Opcode::DsReadAddtidB32: return WithUnsupportedReason(inst, FormatMemory(inst)); + case Opcode::SCmpEqU32: + case Opcode::SCmpEqI32: + case Opcode::SCmpLgU32: + case Opcode::SCmpLgI32: + case Opcode::SCmpGtU32: + case Opcode::SCmpGtI32: + case Opcode::SCmpGeU32: + case Opcode::SCmpGeI32: + case Opcode::SCmpLtU32: + case Opcode::SCmpLtI32: + case Opcode::SCmpLeU32: + case Opcode::SCmpLeI32: + case Opcode::SBitcmp0B32: + case Opcode::SBitcmp1B32: + case Opcode::SCmpLgU64: + return WithUnsupportedReason(inst, fmt::format("0x{:08x}: {} {}, {}", inst.pc, + OpcodeToString(inst.opcode).c_str(), + OperandToString(inst.src0).c_str(), + OperandToString(inst.src1).c_str())); + default: return WithUnsupportedReason(inst, FormatBinary(inst)); + } +} + +std::string ProgramToString(const Program& program) { + std::string text; + for (const auto& inst: program.instructions) { + text += InstructionToString(inst); + text += "\n"; + } + return text; +} + +} // namespace Libs::Graphics::ShaderRecompiler::Decoder diff --git a/src/graphics/shader/recompiler/ShaderDecoder.h b/src/graphics/shader/recompiler/ShaderDecoder.h new file mode 100644 index 000000000..a726aeeba --- /dev/null +++ b/src/graphics/shader/recompiler/ShaderDecoder.h @@ -0,0 +1,677 @@ +#ifndef EMULATOR_INCLUDE_EMULATOR_GRAPHICS_SHADER_RECOMPILER_SHADERDECODER_H_ +#define EMULATOR_INCLUDE_EMULATOR_GRAPHICS_SHADER_RECOMPILER_SHADERDECODER_H_ + +#include "common/common.h" +#include "common/stringUtils.h" + +#include +#include + +namespace Libs::Graphics::ShaderRecompiler::Decoder { + +enum class Family { + Unknown, + SOP1, + SOP2, + SOPK, + SOPC, + SOPP, + VOP1, + VOP2, + VOP3, + VOP3P, + VOPC, + VINTRP, + SMEM, + MUBUF, + MTBUF, + FLAT, + DS, + MIMG, + EXP +}; + +enum class Opcode { + Unknown, + Unsupported, + + SMovB32, + SMovB64, + SMovkI32, + SAbsI32, + SBrevB32, + SBcnt1I32B32, + SBcnt1I32B64, + SFf1I32B32, + SFlbitI32B64, + SBitreplicateB64B32, + SGetpcB64, + SSetpcB64, + SAndSaveexecB32, + SAndn1SaveexecB32, + SAndSaveexecB64, + SOrn2SaveexecB64, + SAndn1SaveexecB64, + SNotB32, + SNotB64, + SWqmB64, + SAddU32, + SAddcU32, + SSubbU32, + SSubU32, + SAddI32, + SSubI32, + SBitcmp0B32, + SBitcmp1B32, + SBitset0B32, + SBitset1B32, + SMinI32, + SMaxI32, + SMinU32, + SMaxU32, + SAndB32, + SAndB64, + SAndn2B32, + SAndn2B64, + SOrB32, + SOrB64, + SOrn2B32, + SOrn2B64, + SXorB32, + SXorB64, + SNandB32, + SNandB64, + SNorB32, + SNorB64, + SXnorB32, + SXnorB64, + SLshlB32, + SLshlB64, + SLshl1AddU32, + SLshl2AddU32, + SLshl3AddU32, + SLshl4AddU32, + SLshrB32, + SLshrB64, + SAshrI32, + SMulI32, + SMulHiU32, + SMulkI32, + SBfeU32, + SBfeU64, + SBfmB32, + SBfmB64, + SCselectB32, + SCselectB64, + SPackLlB32B16, + SPackLhB32B16, + SPackHhB32B16, + SCmpEqI32, + SCmpLgI32, + SCmpGtI32, + SCmpGeI32, + SCmpLtI32, + SCmpLeI32, + SCmpEqU32, + SCmpLgU32, + SCmpGtU32, + SCmpGeU32, + SCmpLtU32, + SCmpLeU32, + SCmpLgU64, + + VNop, + VMovB32, + VMovreldB32, + VMovrelsB32, + VReadfirstlaneB32, + VReadlaneB32, + VWritelaneB32, + VPermlane16B32, + VPermlanex16B32, + VCubeidF32, + VCubescF32, + VCubetcF32, + VCubemaF32, + VCndmaskB32, + VDot2cF32F16, + VCvtF32I32, + VCvtF32U32, + VCvtU32F32, + VCvtI32F32, + VCvtF16F32, + VCvtF32F16, + VCvtF16U16, + VCvtU16F16, + VCvtF16I16, + VCvtI16F16, + VCvtRpiI32F32, + VCvtFlrI32F32, + VCvtOffF32I4, + VCvtF32Ubyte0, + VCvtF32Ubyte1, + VCvtF32Ubyte2, + VCvtF32Ubyte3, + VRcpF32, + VFractF32, + VTruncF32, + VCeilF32, + VRndneF32, + VFloorF32, + VExpF32, + VLogF32, + VRsqF32, + VSqrtF32, + VRcpF16, + VSqrtF16, + VRsqF16, + VLogF16, + VExpF16, + VFloorF16, + VCeilF16, + VTruncF16, + VRndneF16, + VSinF32, + VCosF32, + VNotB32, + VBfrevB32, + VFfbhU32, + VFfblB32, + VAddF32, + VSubF32, + VSubrevF32, + VMulF32, + VMacF32, + VMadmkF32, + VMadakF32, + VMinF32, + VMaxF32, + VAddF16, + VSubF16, + VSubrevF16, + VMulF16, + VFmacF16, + VFmamkF16, + VFmaakF16, + VMinI32, + VMaxI32, + VMulI32I24, + VMulU32U24, + VMulLoU32, + VMulHiU32, + VMulLoI32, + VMulHiI32, + VCvtPkrtzF16F32, + VCvtPkU8F32, + VMadF32, + VMadI32I24, + VMadU32U24, + VMadU64U32, + VFmaF32, + VFmaF16, + VPackB32F16, + VBfeU32, + VBfeI32, + VBfiB32, + VAlignbitB32, + VMin3F32, + VMin3I32, + VMin3U32, + VMin3F16, + VMax3F32, + VMax3I32, + VMax3U32, + VMax3F16, + VMed3F32, + VMed3I32, + VMed3U32, + VMed3F16, + VSadU32, + VAdd3U32, + VLshlAddU32, + VAddLshlU32, + VXadU32, + VLshlOrB32, + VAndOrB32, + VOr3B32, + VXor3B32, + VAddI32, + VSubI32, + VSubrevI32, + VBfmB32, + VLdexpF32, + VCvtPknormI16F32, + VCvtPknormU16F32, + VCvtPkU16U32, + VPkMadI16, + VPkMulLoU16, + VPkAddI16, + VPkSubI16, + VPkLshlrevB16, + VPkLshrrevB16, + VPkAshrrevI16, + VPkMaxI16, + VPkMinI16, + VPkMadU16, + VPkAddF16, + VPkMulF16, + VPkMinF16, + VPkMaxF16, + VPkFmaF16, + VPkFmacF16, + VPkAddU16, + VPkSubU16, + VPkMaxU16, + VPkMinU16, + VMadMixloF16, + VMadMixhiF16, + VAddNcU32, + VAddcU32, + VSubNcU32, + VSubrevNcU32, + VAddNcU16, + VSubNcU16, + VMaxU16, + VMaxI16, + VMinU16, + VMinI16, + VAddNcI16, + VSubNcI16, + VAndB32, + VOrB32, + VXorB32, + VXnorB32, + VLshlB32, + VLshlrevB32, + VLshrB32, + VLshrrevB32, + VAshrI32, + VAshrrevI32, + VLshlrevB16, + VLshrrevB16, + VAshrrevI16, + VBcntU32B32, + VMbcntLoU32B32, + VMbcntHiU32B32, + VMinU32, + VMaxU32, + VMaxF16, + VMinF16, + VCmpFF32, + VCmpLtF32, + VCmpEqF32, + VCmpLeF32, + VCmpGtF32, + VCmpLgF32, + VCmpGeF32, + VCmpOF32, + VCmpUF32, + VCmpNgeF32, + VCmpNlgF32, + VCmpNgtF32, + VCmpNleF32, + VCmpNeqF32, + VCmpNltF32, + VCmpTruF32, + VCmpxLtF32, + VCmpxEqF32, + VCmpxLeF32, + VCmpxGtF32, + VCmpxLgF32, + VCmpxGeF32, + VCmpxNgeF32, + VCmpxNlgF32, + VCmpxNgtF32, + VCmpxNleF32, + VCmpxNeqF32, + VCmpxNltF32, + VCmpFI32, + VCmpLtI32, + VCmpEqI32, + VCmpLeI32, + VCmpGtI32, + VCmpNeI32, + VCmpGeI32, + VCmpTI32, + VCmpClassF32, + VCmpLtI16, + VCmpEqI16, + VCmpLeI16, + VCmpGtI16, + VCmpNeI16, + VCmpGeI16, + VCmpLtF16, + VCmpEqF16, + VCmpLeF16, + VCmpGtF16, + VCmpLgF16, + VCmpGeF16, + VCmpNeqF16, + VCmpxLtF16, + VCmpxEqF16, + VCmpxLeF16, + VCmpxGtF16, + VCmpxGeF16, + VCmpxNeqF16, + VCmpxNltF16, + VCmpxLtI32, + VCmpxEqI32, + VCmpxLeI32, + VCmpxGtI32, + VCmpxNeI32, + VCmpxGeI32, + VCmpLtU16, + VCmpEqU16, + VCmpLeU16, + VCmpGtU16, + VCmpNeU16, + VCmpGeU16, + VCmpFU32, + VCmpLtU32, + VCmpEqU32, + VCmpLeU32, + VCmpGtU32, + VCmpNeU32, + VCmpGeU32, + VCmpTU32, + VCmpNeU64, + VCmpxLtU32, + VCmpxEqU32, + VCmpxLeU32, + VCmpxGtU32, + VCmpxNeU32, + VCmpxGeU32, + + SLoadDword, + SLoadDwordx2, + SLoadDwordx4, + SLoadDwordx8, + SLoadDwordx16, + SBufferLoadDword, + SBufferLoadDwordx2, + SBufferLoadDwordx4, + SBufferLoadDwordx8, + SBufferLoadDwordx16, + BufferLoadFormatX, + BufferLoadFormatXy, + BufferLoadFormatXyz, + BufferLoadFormatXyzw, + BufferStoreFormatX, + BufferStoreFormatXy, + BufferStoreFormatXyz, + BufferStoreFormatXyzw, + BufferLoadUbyte, + BufferLoadSbyte, + BufferLoadUshort, + BufferLoadSshort, + BufferLoadDword, + BufferLoadDwordx2, + BufferLoadDwordx3, + BufferLoadDwordx4, + BufferStoreByte, + BufferStoreShort, + BufferStoreDword, + BufferStoreDwordx2, + BufferStoreDwordx3, + BufferStoreDwordx4, + TBufferLoadFormatX, + TBufferLoadFormatXy, + TBufferLoadFormatXyz, + TBufferLoadFormatXyzw, + TBufferStoreFormatX, + TBufferStoreFormatXy, + TBufferStoreFormatXyz, + TBufferStoreFormatXyzw, + BufferAtomicSwap, + BufferAtomicAdd, + BufferAtomicSub, + BufferAtomicSMin, + BufferAtomicUMin, + BufferAtomicSMax, + BufferAtomicUMax, + BufferAtomicAnd, + BufferAtomicOr, + BufferAtomicXor, + FlatLoadUbyte, + FlatLoadSbyte, + FlatLoadUshort, + FlatLoadSshort, + FlatLoadDword, + FlatLoadDwordx2, + FlatLoadDwordx3, + FlatLoadDwordx4, + FlatStoreByte, + FlatStoreShort, + FlatStoreDword, + FlatStoreDwordx2, + FlatStoreDwordx3, + FlatStoreDwordx4, + DsAddU32, + DsAddRtnU32, + DsSubU32, + DsSubRtnU32, + DsMinI32, + DsMinRtnI32, + DsMaxI32, + DsMaxRtnI32, + DsMinU32, + DsMinRtnU32, + DsMaxU32, + DsMaxRtnU32, + DsAndB32, + DsAndRtnB32, + DsOrB32, + DsOrRtnB32, + DsXorB32, + DsXorRtnB32, + DsWrxchgRtnB32, + DsMinF32, + DsMaxF32, + DsSwizzleB32, + DsConsume, + DsAppend, + DsReadSbyte, + DsReadUbyte, + DsReadSshort, + DsReadUshort, + DsRead2B32, + DsReadB32, + DsReadB64, + DsRead2B64, + DsRead2St64B64, + DsReadB96, + DsReadB128, + DsWriteByte, + DsWriteShort, + DsWrite2B32, + DsWrite2St64B32, + DsWrite2B64, + DsWrite2St64B64, + DsWriteB32, + DsWriteB64, + DsWriteB96, + DsWriteB128, + DsWriteAddtidB32, + DsReadAddtidB32, + ImageGetResinfo, + ImageGetLod, + ImageLoad, + ImageLoadMip, + ImageStore, + ImageStoreMip, + ImageAtomicAdd, + ImageAtomicUMin, + ImageAtomicUMax, + ImageAtomicAnd, + ImageAtomicOr, + ImageAtomicXor, + ImageSample, + ImageGather4Lz, + ImageGather4C, + ImageGather4CLz, + ImageGather4LzO, + ImageGather4CO, + ImageGather4CLzO, + ImageGather4H, + VInterpP1F32, + VInterpP2F32, + VInterpMovF32, + + SNop, + SWaitcnt, + SBarrier, + SBranch, + SCbranchScc0, + SCbranchScc1, + SCbranchVccz, + SCbranchVccnz, + SCbranchExecz, + SCbranchExecnz, + SSendmsg, + SSetregB32, + SSleep, + STtraceData, + SInstPrefetch, + SEndpgm, + Exp +}; + +enum class OperandKind { + Unknown, + LiteralConstant, + IntegerInlineConstant, + FloatInlineConstant, + Sgpr, + VccLo, + VccHi, + VccZ, + ExecLo, + ExecHi, + ExecZ, + Scc, + M0, + PopsExitingWaveId, + Null, + Vgpr, +}; + +enum ImageSampleFlag : uint32_t { + ImageSampleFlagLod = 1u << 0u, + ImageSampleFlagBias = 1u << 1u, + ImageSampleFlagDerivative = 1u << 2u, + ImageSampleFlagCompare = 1u << 3u, + ImageSampleFlagOffset = 1u << 4u, + ImageSampleFlagLevelZero = 1u << 5u, + ImageSampleFlagLodClamp = 1u << 6u, + ImageSampleFlagA16 = 1u << 7u, + ImageSampleFlagCd = 1u << 8u, + ImageSampleFlagGatherHorizontal = 1u << 9u, +}; + +enum class ImageDimension : uint32_t { + Unknown, + Dim2D, + Dim3D, + Dim2DArray, +}; + +constexpr uint32_t MaxInstructionRawWords = 5u; +constexpr uint32_t MaxImageNsaAddressComponents = 12u; + +struct Operand { + OperandKind kind = OperandKind::Unknown; + uint32_t value = 0; + int32_t signed_val = 0; + float float_val = 0.0f; + uint32_t reg = 0; + uint32_t sdwa_sel = 6; + uint32_t sdwa_dst_unused = 0; + uint32_t omod = 0; + uint32_t dpp_ctrl = 0; + uint32_t dpp_row_mask = 0xf; + uint32_t dpp_bank_mask = 0xf; + bool sdwa_sext = false; + bool dpp_fetch_inactive = false; + bool dpp_bound_ctrl = false; + bool op_sel = false; + bool op_sel_hi = false; + bool negate = false; + bool negate_hi = false; + bool absolute = false; + bool clamp = false; + bool dpp = false; +}; + +struct Instruction { + uint32_t pc = 0; + uint32_t word = 0; + uint32_t word_count = 1; + uint32_t raw[MaxInstructionRawWords] = {}; + uint32_t raw_count = 1; + Family family = Family::Unknown; + uint32_t opcode_id = 0; + Opcode opcode = Opcode::Unknown; + Operand dst; + Operand dst2; + Operand src0; + Operand src1; + Operand src2; + Operand src3; + uint32_t src_count = 0; + uint32_t offset = 0; + uint32_t secondary_offset = 0; + uint32_t dmask = 0; + uint32_t data_dwords = 1; + uint32_t data_bits = 32; + uint32_t data_format = 0; + uint32_t number_format = 0; + uint32_t image_sample_flags = 0; + ImageDimension image_dimension = ImageDimension::Unknown; + uint32_t image_address_components = 0; + uint32_t image_nsa_dwords = 0; + uint32_t image_nsa_addr[MaxImageNsaAddressComponents] = {}; + uint32_t memory_segment = 0; + bool data_signed = false; + bool typed = false; + bool formatted = false; + bool gds = false; + bool glc = false; + bool slc = false; + bool idxen = false; + bool offen = false; + int32_t branch_offset = 0; + uint32_t branch_target = 0; + struct { + uint32_t target = 0; + uint32_t en = 0; + bool done = false; + bool compr = false; + bool vm = false; + } exp; + std::string unsupported_reason; +}; + +struct Program { + std::span code; + std::vector instructions; +}; + +bool DecodeProgram(std::span code, Program* program, std::string* error); + +bool DecodeScalarSource(uint32_t code, uint32_t pc, Operand* operand, std::string* error); +bool DecodeScalarDestination(uint32_t code, uint32_t pc, Operand* operand, std::string* error); +bool DecodeVectorGpr(uint32_t reg, Operand* operand, std::string* error); +bool ReadLiteralOperands(std::span code, uint32_t word_index, Instruction* inst, + std::string* error); +void SetRawWords(Instruction* inst, std::span code, uint32_t word_index, + uint32_t word_count); +void SetUnsupported(Instruction* inst, Family family, uint32_t opcode_id, const char* reason); +std::string FamilyToString(Family family); +std::string OpcodeToString(Opcode opcode); +std::string OperandToString(const Operand& operand); +std::string InstructionToString(const Instruction& inst); +std::string ProgramToString(const Program& program); + +} // namespace Libs::Graphics::ShaderRecompiler::Decoder + +#endif /* EMULATOR_INCLUDE_EMULATOR_GRAPHICS_SHADER_RECOMPILER_SHADERDECODER_H_ */ diff --git a/src/graphics/shader/recompiler/ShaderIR.h b/src/graphics/shader/recompiler/ShaderIR.h new file mode 100644 index 000000000..e44415ea3 --- /dev/null +++ b/src/graphics/shader/recompiler/ShaderIR.h @@ -0,0 +1,763 @@ +#ifndef EMULATOR_INCLUDE_EMULATOR_GRAPHICS_SHADER_RECOMPILER_SHADERIR_H_ +#define EMULATOR_INCLUDE_EMULATOR_GRAPHICS_SHADER_RECOMPILER_SHADERIR_H_ + +#include "common/common.h" +#include "common/stringUtils.h" +#include "graphics/shader/recompiler/ShaderCFG.h" +#include "graphics/shader/recompiler/ShaderDecoder.h" +#include "graphics/shader/shader.h" + +#include +#include + +namespace Libs::Graphics::ShaderRecompiler::IR { + +enum class Opcode { + ControlNop, + Waitcnt, + Barrier, + Sendmsg, + TtraceData, + InstPrefetch, + MoveU32, + MoveF32Bits, + MoveRelDestU32, + MoveRelSourceU32, + MoveU64, + WqmB64, + SaveexecB32, + SaveexecB64, + ReadFirstLaneU32, + ReadLaneU32, + WriteLaneU32, + Permlane16B32, + Permlanex16B32, + AbsI32, + IAddU32, + IAddCarryU32, + ISubBorrowU32, + ScalarAddCarryU32, + ScalarSubBorrowU32, + ScalarSubBorrowCarryU32, + ScalarSignedAddOverflowI32, + ScalarSignedSubOverflowI32, + ScalarShiftLeftAddCarryU32, + ISubU32, + IMulU32, + UMulHighU32, + SMulHighI32, + IMadI24U32, + UMadU24U32, + UMadU64U32, + SadU32, + IAdd3U32, + IMulI24U32, + UMulU24U32, + IMinI32, + IMaxI32, + IMin3I32, + IMax3I32, + IMed3I32, + UMinU32, + UMaxU32, + UMin3U32, + UMax3U32, + UMed3U32, + BitwiseAndU32, + BitwiseAndU64, + BitwiseAndNotU32, + BitwiseAndNotU64, + BitwiseOrU32, + BitwiseOrU64, + BitwiseOrNotU32, + BitwiseOrNotU64, + BitwiseAndOrU32, + BitwiseOr3U32, + BitwiseXorU32, + BitwiseXorU64, + BitwiseXor3U32, + BitwiseNandU32, + BitwiseNandU64, + BitwiseNorU32, + BitwiseNorU64, + BitwiseXnorU32, + BitwiseXnorU64, + BitwiseNotU32, + BitwiseNotU64, + BitClearU32, + BitSetU32, + BitReverseU32, + BitCountU32, + BitCountU64, + BitReplicateB64B32, + BitCountAddU32, + MaskedBitCountLowU32, + MaskedBitCountHighU32, + FindLsbU32, + FindMsbFromHighU32, + FindMsbFromHighU64, + BitFieldMaskU32, + BitFieldMaskU64, + BitFieldExtractU32, + BitFieldExtractU64, + BitFieldExtract3U32, + BitFieldExtract3I32, + BitFieldInsertSelectU32, + BitCompare0B32, + BitCompare1B32, + AlignBitU32, + ShiftLeftAddU32, + AddShiftLeftU32, + XorAddU32, + ShiftLeftOrU32, + ShiftLeftLogicalU32, + ShiftLeftLogicalU64, + ShiftLeftLogicalU16, + ShiftRightLogicalU32, + ShiftRightLogicalU64, + ShiftRightLogicalU16, + ShiftRightArithmeticI32, + ShiftRightArithmeticI16, + SelectU32, + SelectMaskU32, + SelectF32Bits, + SelectMaskF32Bits, + SelectU64, + PackLowLowU16, + PackLowHighU16, + PackHighHighU16, + CompareFalse, + CompareTrue, + CompareEqU32, + CompareNeU32, + CompareGtU32, + CompareGeU32, + CompareLtU32, + CompareLeU32, + CompareNeU64, + CompareMaskEqU32, + CompareMaskNeU32, + CompareMaskGtU32, + CompareMaskGeU32, + CompareMaskLtU32, + CompareMaskLeU32, + CompareEqI32, + CompareNeI32, + CompareGtI32, + CompareGeI32, + CompareLtI32, + CompareLeI32, + CompareEqI16, + CompareNeI16, + CompareGtI16, + CompareGeI16, + CompareLtI16, + CompareLeI16, + CompareMaskEqI32, + CompareMaskNeI32, + CompareMaskGtI32, + CompareMaskGeI32, + CompareMaskLtI32, + CompareMaskLeI32, + CompareEqU16, + CompareNeU16, + CompareGtU16, + CompareGeU16, + CompareLtU16, + CompareLeU16, + CompareEqF32, + CompareNeF32, + CompareGtF32, + CompareGeF32, + CompareLtF32, + CompareLeF32, + CompareOrderedF32, + CompareUnorderedF32, + CompareUnordEqF32, + CompareUnordNeF32, + CompareUnordGtF32, + CompareUnordGeF32, + CompareUnordLtF32, + CompareUnordLeF32, + CompareClassF32, + CompareEqF16, + CompareNeF16, + CompareGtF16, + CompareGeF16, + CompareLtF16, + CompareLeF16, + CompareUnordNeF16, + CompareMaskEqF16, + CompareMaskNeF16, + CompareMaskGtF16, + CompareMaskGeF16, + CompareMaskLtF16, + CompareMaskLeF16, + CompareMaskUnordNeF16, + CompareMaskUnordGeF16, + CompareMaskEqF32, + CompareMaskNeF32, + CompareMaskGtF32, + CompareMaskGeF32, + CompareMaskLtF32, + CompareMaskLeF32, + CompareMaskUnordEqF32, + CompareMaskUnordNeF32, + CompareMaskUnordGtF32, + CompareMaskUnordGeF32, + CompareMaskUnordLtF32, + CompareMaskUnordLeF32, + ConvertByteU32ToF32, + ConvertU32ToF32, + ConvertI32ToF32, + ConvertF32ToU32, + ConvertF32ToI32, + ConvertF32ToF16, + ConvertF16ToF32, + ConvertU16ToF16, + ConvertF16ToU16, + ConvertI16ToF16, + ConvertF16ToI16, + ConvertRoundPlusInfF32ToI32, + ConvertFloorF32ToI32, + ConvertI4ToOffsetF32, + LdexpF32, + PackF32ToF16Rtz, + PackSnorm2x16F32, + PackUnorm2x16F32, + PackU16U32, + PackU8F32, + PackB32F16, + PackedMadI16, + PackedMulLoU16, + PackedAddI16, + PackedSubI16, + PackedLshlrevB16, + PackedLshrrevB16, + PackedAshrrevI16, + PackedMaxI16, + PackedMinI16, + PackedMadU16, + PackedAddU16, + PackedSubU16, + PackedMaxU16, + PackedMinU16, + PackedAddF16, + PackedMulF16, + PackedMinF16, + PackedMaxF16, + PackedFmaF16, + AddF16, + SubF16, + MulF16, + MinF16, + MaxF16, + FmaF16, + MadMixF16, + IAddU16, + ISubI16, + IMinI16, + IMaxI16, + UMinU16, + UMaxU16, + RcpF32, + FractF32, + TruncF32, + CeilF32, + RoundEvenF32, + FloorF32, + Exp2F32, + Log2F32, + InverseSqrtF32, + SqrtF32, + RcpF16, + SqrtF16, + InverseSqrtF16, + Log2F16, + Exp2F16, + FloorF16, + CeilF16, + TruncF16, + RoundEvenF16, + SinF32, + CosF32, + CubeIdF32, + CubeScF32, + CubeTcF32, + CubeMaF32, + FAddF32, + FSubF32, + FMulF32, + FMinF32, + FMaxF32, + FMadF32, + Dot2AccF32F16, + FMin3F32, + FMax3F32, + FMed3F32, + Min3F16, + Max3F16, + Med3F16, + LoadSrtDword, + SLoadDword, + SBufferLoadDword, + BufferLoadUbyte, + BufferLoadSbyte, + BufferLoadUshort, + BufferLoadSshort, + BufferLoadDword, + BufferStoreByte, + BufferStoreShort, + BufferStoreDword, + AtomicSwapU32, + AtomicAddU32, + AtomicSubU32, + AtomicSMinI32, + AtomicUMinU32, + AtomicSMaxI32, + AtomicUMaxU32, + AtomicAndU32, + AtomicOrU32, + AtomicXorU32, + FlatLoadUbyte, + FlatLoadSbyte, + FlatLoadUshort, + FlatLoadSshort, + FlatLoadDword, + FlatStoreByte, + FlatStoreShort, + FlatStoreDword, + DsReadUbyte, + DsReadSbyte, + DsReadUshort, + DsReadSshort, + DsReadB32, + DsWriteByte, + DsWriteShort, + DsWriteB32, + DsMinF32, + DsMaxF32, + DsSwizzleB32, + DsConsume, + DsAppend, + DsWriteAddtidB32, + DsReadAddtidB32, + ImageGetResinfo, + ImageGetLod, + ImageLoad, + ImageStore, + ImageSample, + ImageGather4, + LoadInputF32, + Export, +}; + +enum class RegisterFile { Scalar, Vector, Vcc, Exec, Scc, M0 }; + +struct Register { + RegisterFile file = RegisterFile::Scalar; + uint32_t index = 0; + + bool operator==(const Register& other) const { + return file == other.file && index == other.index; + } +}; + +enum class OperandKind { Register, ImmediateU32, PcRelativeU32, Null }; + +struct Operand { + OperandKind kind = OperandKind::Null; + Register reg; + uint32_t imm = 0; + bool sext_64 = false; + uint32_t sdwa_sel = 6; + uint32_t omod = 0; + bool sdwa_sext = false; + bool op_sel = false; + bool op_sel_hi = false; + bool negate = false; + bool negate_hi = false; + bool absolute = false; + bool clamp = false; + uint32_t dpp_ctrl = 0; + uint32_t dpp_row_mask = 0xf; + uint32_t dpp_bank_mask = 0xf; + bool dpp_fetch_inactive = false; + bool dpp_bound_ctrl = false; + bool dpp = false; + + bool operator==(const Operand& other) const = default; +}; + +enum class ResourceKind { + None, + ScalarBuffer, + Buffer, + Flat, + Global, + Scratch, + Lds, + Gds, + Image, + ImageUint, + StorageImage, + StorageImageUint, + Sampler +}; + +struct MemoryInfo { + ResourceKind kind = ResourceKind::None; + uint32_t resource = 0; + uint32_t sampler = 0; + uint32_t offset = 0; + uint32_t secondary_offset = 0; + uint32_t dmask = 0; + uint32_t data_dwords = 1; + uint32_t data_bits = 32; + uint32_t component_index = 0; + uint32_t component_count = 1; + uint32_t data_format = 0; + uint32_t number_format = 0; + uint32_t image_sample_flags = 0; + Decoder::ImageDimension image_dimension = Decoder::ImageDimension::Unknown; + uint32_t image_address_components = 0; + uint32_t image_nsa_dwords = 0; + uint32_t image_nsa_addr[Decoder::MaxImageNsaAddressComponents] = {}; + uint32_t memory_segment = 0; + // Dense resource indices are assigned by resource tracking. These source IDs refer to the + // exact scalar definitions reaching this instruction before that patching step. + uint32_t resource_source = 0; + uint32_t sampler_source = 0; + bool data_signed = false; + bool typed = false; + bool formatted = false; + bool image_has_mip = false; + bool glc = false; + bool slc = false; + bool idxen = false; + bool offen = false; + + bool operator==(const MemoryInfo& other) const = default; +}; + +enum class ExportTargetKind { Unknown, Null, Position, Primitive, Parameter, Mrt, MrtZ }; + +struct ExportInfo { + ExportTargetKind kind = ExportTargetKind::Unknown; + uint32_t target = 0; + uint32_t index = 0; + uint32_t en = 0; + bool done = false; + bool compr = false; + bool vm = false; + + bool operator==(const ExportInfo& other) const = default; +}; + +struct InputInfo { + uint32_t attr = 0; + uint32_t chan = 0; + + bool operator==(const InputInfo& other) const = default; +}; + +enum class SaveexecMode { And, Orn2, Andn1 }; + +struct Instruction { + uint32_t pc = 0; + Opcode op = Opcode::MoveU32; + Operand dst; + Operand dst2; + Operand src[4]; + uint32_t src_count = 0; + uint32_t scalar_sources[4] = {}; + uint32_t scalar_value = 0; + MemoryInfo memory; + ExportInfo export_info; + InputInfo input_info; + SaveexecMode saveexec_mode = SaveexecMode::And; + + bool operator==(const Instruction& other) const = default; +}; + +struct BasicBlock { + uint32_t id = 0; + uint32_t start_pc = 0; + uint32_t end_pc = 0; + uint32_t inst_begin = 0; + uint32_t inst_end = 0; + std::vector predecessors; + std::vector successors; + CFG::Terminator terminator; + std::vector instructions; +}; + +enum class ScalarValueOp { + Undefined, + Unknown, + UserData, + Constant, + PcRelativeLow, + PcRelativeHigh, + Add, + AddCarry, + Carry, + Sub, + SubBorrow, + Borrow, + Mul, + And, + AndNot, + Or, + OrNot, + Xor, + Not, + ShiftLeft, + ShiftRight, + ShiftRightArithmetic, + BitFieldMaskU32, + BitFieldMaskU64Low, + BitFieldMaskU64High, + Add3, + ShiftLeftAdd, + ShiftLeftAddCarry, + AddShiftLeft, + XorAdd, + ShiftLeftOr, + ReadConst, + ReadConstBuffer, + Phi, +}; + +struct ScalarValue { + ScalarValueOp op = ScalarValueOp::Undefined; + uint32_t pc = 0; + uint32_t imm = 0; + std::array args = {}; + std::vector phi_args; + + bool operator==(const ScalarValue& other) const = default; +}; + +struct DescriptorValue { + std::array dwords = {}; + uint32_t dword_count = 0; + + bool operator==(const DescriptorValue& other) const { + return dword_count == other.dword_count && dwords == other.dwords; + } +}; + +struct ScalarProvenance { + static constexpr uint32_t Undefined = 0; + static constexpr uint32_t Unknown = 1; + + std::vector values; + std::vector descriptors; + + bool operator==(const ScalarProvenance& other) const = default; +}; + +struct SrtRead { + uint32_t value = 0; + uint32_t flat_offset = 0; + uint32_t use_pc = 0; + + bool operator==(const SrtRead& other) const = default; +}; + +struct SrtPlan { + std::vector reads; + std::vector dynamic_reads; + std::vector dynamic_sources; + + bool operator==(const SrtPlan& other) const = default; +}; + +struct BufferResource { + uint32_t source = 0; + uint32_t first_use_pc = 0; + uint32_t max_byte_extent = 0; + uint32_t packed_stride = 0; + uint32_t descriptor_format = 0; + bool read = false; + bool written = false; + bool atomic = false; + bool formatted = false; + bool scalar = false; + + bool operator==(const BufferResource& other) const = default; +}; + +enum class ImageMipMode { None, DynamicStorage }; + +constexpr uint32_t StorageImageIdentitySwizzle = 0x00000facu; + +struct ImageResource { + uint32_t source = 0; + uint32_t first_use_pc = 0; + ResourceKind kind = ResourceKind::None; + Decoder::ImageDimension dimension = Decoder::ImageDimension::Unknown; + ImageMipMode mip_mode = ImageMipMode::None; + uint32_t storage_swizzle = StorageImageIdentitySwizzle; + bool read = false; + bool written = false; + bool atomic = false; + bool depth_compare = false; + + bool operator==(const ImageResource& other) const = default; +}; + +struct SamplerResource { + uint32_t source = 0; + uint32_t first_use_pc = 0; + + bool operator==(const SamplerResource& other) const = default; +}; + +struct SampledResourcePair { + uint32_t image = 0; + uint32_t sampler = 0; + uint32_t first_use_pc = 0; + + bool operator==(const SampledResourcePair& other) const = default; +}; + +struct AddressResource { + uint32_t source = 0; + uint32_t first_use_pc = 0; + ResourceKind kind = ResourceKind::Flat; + int32_t min_offset = 0; + uint64_t specialized_base = 0; + bool read = false; + bool written = false; + bool atomic = false; + + bool operator==(const AddressResource& other) const = default; +}; + +enum class StageInputKind { + VertexIndex, + InstanceIndex, + FragCoord, + FrontFacing, + WorkgroupId, + LocalInvocationId, + LocalInvocationIndex, + GlobalInvocationId, + Parameter, +}; + +enum class StageOutputKind { Position, Parameter, Mrt, Depth, SampleMask }; + +struct StageInput { + StageInputKind kind = StageInputKind::VertexIndex; + uint32_t location = 0; + uint32_t component_count = 1; + std::string debug_name; + + bool operator==(const StageInput& other) const = default; +}; + +struct StageOutput { + StageOutputKind kind = StageOutputKind::Parameter; + uint32_t index = 0; + uint32_t location = 0; + std::string debug_name; + + bool operator==(const StageOutput& other) const = default; +}; + +enum class DescriptorBindingKind { + Buffers, + Sampled2D, + Sampled2DArray, + Sampled3D, + SampledUint2D, + SampledUint2DArray, + SampledUint3D, + Storage2D, + Storage2DArray, + Storage3D, + StorageUint2D, + StorageUint2DArray, + StorageUint3D, + Samplers, + Gds, + AddressMemory, + FlattenedSrt, + UserData, + Count, +}; + +struct DescriptorBinding { + DescriptorBindingKind kind = DescriptorBindingKind::Buffers; + uint32_t binding = 0; + std::vector resources; + + bool operator==(const DescriptorBinding& other) const = default; +}; + +struct BindingLayout { + uint32_t descriptor_set = 0; + uint32_t push_constant_offset = 0; + uint32_t push_constant_size = 0; + std::vector user_data_registers; + std::vector descriptors; + + bool operator==(const BindingLayout& other) const = default; +}; + +struct ShaderInfo { + static constexpr uint32_t MaxBuffers = 32; + static constexpr uint32_t MaxAddresses = 32; + static constexpr uint32_t MaxImages = 32; + static constexpr uint32_t MaxSamplers = 32; + static constexpr uint32_t MaxSampledPairs = 64; + + std::vector buffers; + std::vector addresses; + std::vector images; + std::vector samplers; + std::vector sampled_pairs; + std::vector inputs; + std::vector outputs; + int32_t vertex_offset_sgpr = -1; + + bool operator==(const ShaderInfo& other) const = default; +}; + +struct Program { + ShaderType stage = ShaderType::Unknown; + ShaderLaneMaskMode lane_mask_mode = ShaderLaneMaskMode::NativeWave; + uint64_t shader_hash = 0; + uint32_t wave_size = 64; + uint32_t user_data_base = 0; + uint32_t user_data_count = 64; + bool dispatcher_fallback = false; + CFG::FailureKind cfg_failure_kind = CFG::FailureKind::None; + std::string fallback_reason; + std::vector blocks; + ScalarProvenance provenance; + SrtPlan srt; + bool srt_plan_complete = false; + bool srt_patching_complete = false; + ShaderInfo info; + bool resource_tracking_complete = false; + bool shader_info_complete = false; + BindingLayout bindings; + bool binding_layout_complete = false; +}; + +bool LowerProgram(const Decoder::Program& decoded, const CFG::Graph& cfg, ShaderType stage, + uint32_t wave_size, Program* program, std::string* error); + +std::string RegisterToString(Register reg); +std::string OperandToString(const Operand& operand); +std::string ExportTargetKindToString(ExportTargetKind kind); +std::string InstructionToString(const Instruction& inst); +std::string ProgramToString(const Program& program); + +} // namespace Libs::Graphics::ShaderRecompiler::IR + +#endif /* EMULATOR_INCLUDE_EMULATOR_GRAPHICS_SHADER_RECOMPILER_SHADERIR_H_ */ diff --git a/src/graphics/shader/recompiler/ShaderInfoCollection.cpp b/src/graphics/shader/recompiler/ShaderInfoCollection.cpp new file mode 100644 index 000000000..50aaaa7ba --- /dev/null +++ b/src/graphics/shader/recompiler/ShaderInfoCollection.cpp @@ -0,0 +1,225 @@ +#include "graphics/shader/recompiler/ShaderInfoCollection.h" + +#include +#include + +namespace Libs::Graphics::ShaderRecompiler::IR { +namespace { + +bool HasInput(const ShaderInfo& info, StageInputKind kind, uint32_t location) { + return std::any_of(info.inputs.begin(), info.inputs.end(), [=](const auto& input) { + return input.kind == kind && input.location == location; + }); +} + +void AddInput(ShaderInfo* info, StageInputKind kind, uint32_t location, uint32_t components, + std::string name) { + if (!HasInput(*info, kind, location)) { + info->inputs.push_back({kind, location, components, std::move(name)}); + } +} + +bool HasOutput(const ShaderInfo& info, StageOutputKind kind, uint32_t index) { + return std::any_of(info.outputs.begin(), info.outputs.end(), [=](const auto& output) { + return output.kind == kind && output.index == index; + }); +} + +void AddOutput(ShaderInfo* info, StageOutputKind kind, uint32_t index, uint32_t location, + std::string name) { + if (!HasOutput(*info, kind, index)) { + info->outputs.push_back({kind, index, location, std::move(name)}); + } +} + +bool NeedsLocalInvocationIndex(const Program& program) { + for (const auto& block: program.blocks) { + for (const auto& inst: block.instructions) { + if (inst.op == Opcode::DsWriteAddtidB32 || inst.op == Opcode::DsReadAddtidB32) { + return true; + } + } + } + return false; +} + +bool ValidateOptions(const Program& program, const ShaderInfoOptions& options, std::string* error) { + auto Fail = [&](const char* message) { + if (error != nullptr) { + *error = message; + } + return false; + }; + switch (program.stage) { + case ShaderType::Vertex: + if (options.vertex == nullptr) { + return Fail("vertex shader info requires vertex metadata"); + } + if (options.vertex->resources_num < 0 || + options.vertex->resources_num > ShaderVertexInputInfo::RES_MAX) { + return Fail("vertex resource count is out of range"); + } + for (const auto& block: program.blocks) { + for (const auto& inst: block.instructions) { + if (inst.op == Opcode::LoadInputF32 && + (inst.input_info.chan >= 4 || + inst.input_info.attr >= + static_cast(options.vertex->resources_num))) { + return Fail("vertex input reference is out of range"); + } + } + } + return true; + case ShaderType::Pixel: + if (options.pixel == nullptr) { + return Fail("pixel shader info requires pixel metadata"); + } + return options.pixel->input_num <= std::size(options.pixel->interpolator_settings) || + Fail("pixel input count is out of range"); + case ShaderType::Compute: + if (options.compute == nullptr) { + return Fail("compute shader info requires compute metadata"); + } + return (options.compute->thread_ids_num >= 0 && options.compute->thread_ids_num <= 3) || + Fail("compute thread ID count is out of range"); + default: return Fail("unsupported shader stage for info collection"); + } +} + +void CollectVertexInputs(const Program& program, const ShaderVertexInputInfo* vertex, + ShaderInfo* info) { + AddInput(info, StageInputKind::VertexIndex, 0, 1, "gl_VertexIndex"); + AddInput(info, StageInputKind::InstanceIndex, 0, 1, "gl_InstanceIndex"); + if (vertex == nullptr) { + return; + } + uint32_t used_components[ShaderVertexInputInfo::RES_MAX] = {}; + for (const auto& block: program.blocks) { + for (const auto& inst: block.instructions) { + if (inst.op == Opcode::LoadInputF32 && + inst.input_info.attr < static_cast(vertex->resources_num) && + inst.input_info.attr < ShaderVertexInputInfo::RES_MAX) { + used_components[inst.input_info.attr] = + std::max(used_components[inst.input_info.attr], inst.input_info.chan + 1u); + } + } + } + for (uint32_t attr = 0; attr < static_cast(vertex->resources_num) && + attr < ShaderVertexInputInfo::RES_MAX; + attr++) { + if (used_components[attr] != 0) { + AddInput(info, StageInputKind::Parameter, attr, used_components[attr], + fmt::format("in_attr_{}", attr)); + } + } +} + +void CollectPixelInputs(const ShaderPixelInputInfo* pixel, ShaderInfo* info) { + if (pixel == nullptr) { + return; + } + if (pixel->HasPositionInput()) { + AddInput(info, StageInputKind::FragCoord, 0, 4, "gl_FragCoord"); + } + if (pixel->ps_front_face) { + AddInput(info, StageInputKind::FrontFacing, 0, 1, "gl_FrontFacing"); + } + for (uint32_t input = 0; input < pixel->input_num; input++) { + AddInput(info, StageInputKind::Parameter, input, 4, fmt::format("in_param_{}", input)); + } +} + +void CollectComputeInputs(const Program& program, const ShaderComputeInputInfo* compute, + ShaderInfo* info) { + if (compute != nullptr) { + if (compute->group_id[0] || compute->group_id[1] || compute->group_id[2]) { + AddInput(info, StageInputKind::WorkgroupId, 0, 3, "gl_WorkGroupID"); + } + if (compute->thread_ids_num > 0) { + AddInput(info, StageInputKind::LocalInvocationId, 0, 3, "gl_LocalInvocationID"); + } + if (compute->thread_ids_num > 0 || compute->tg_size_en) { + AddInput(info, StageInputKind::LocalInvocationIndex, 0, 1, "gl_LocalInvocationIndex"); + } + if (compute->dispatch_thread_dimensions) { + AddInput(info, StageInputKind::GlobalInvocationId, 0, 3, "gl_GlobalInvocationID"); + } + } + if (NeedsLocalInvocationIndex(program)) { + AddInput(info, StageInputKind::LocalInvocationIndex, 0, 1, "gl_LocalInvocationIndex"); + } +} + +void CollectOutputs(const Program& program, const ShaderPixelInputInfo* pixel, ShaderInfo* info) { + for (const auto& block: program.blocks) { + for (const auto& inst: block.instructions) { + if (inst.op != Opcode::Export) { + continue; + } + if (inst.export_info.kind == ExportTargetKind::MrtZ) { + if (pixel != nullptr && (inst.export_info.en & 0x1u) != 0 && + pixel->ps_depth_export_enable) { + AddOutput(info, StageOutputKind::Depth, 0, 0, "gl_FragDepth"); + } + if (pixel != nullptr && (inst.export_info.en & 0x4u) != 0 && + pixel->ps_sample_mask_export_enable) { + AddOutput(info, StageOutputKind::SampleMask, 0, 0, "gl_SampleMask"); + } + continue; + } + if (inst.export_info.en == 0) { + continue; + } + switch (inst.export_info.kind) { + case ExportTargetKind::Position: + AddOutput(info, StageOutputKind::Position, inst.export_info.index, 0, + "out_position"); + break; + case ExportTargetKind::Parameter: + AddOutput(info, StageOutputKind::Parameter, inst.export_info.index, + inst.export_info.index, + fmt::format("out_param_{}", inst.export_info.index)); + break; + case ExportTargetKind::Mrt: + AddOutput(info, StageOutputKind::Mrt, inst.export_info.index, + inst.export_info.index, + fmt::format("out_mrt_{}", inst.export_info.index)); + break; + default: break; + } + } + } +} + +} // namespace + +bool CollectShaderInfo(Program* program, const ShaderInfoOptions& options, std::string* error) { + if (program == nullptr || !program->resource_tracking_complete || + program->shader_info_complete) { + if (error != nullptr) { + *error = program == nullptr ? "invalid shader info program" + : !program->resource_tracking_complete ? "shader resources were not tracked" + : "shader info already collected"; + } + return false; + } + if (!ValidateOptions(*program, options, error)) { + return false; + } + + auto next = program->info; + next.inputs.clear(); + next.outputs.clear(); + switch (program->stage) { + case ShaderType::Vertex: CollectVertexInputs(*program, options.vertex, &next); break; + case ShaderType::Pixel: CollectPixelInputs(options.pixel, &next); break; + case ShaderType::Compute: CollectComputeInputs(*program, options.compute, &next); break; + default: return false; + } + CollectOutputs(*program, options.pixel, &next); + program->info = std::move(next); + program->shader_info_complete = true; + return true; +} + +} // namespace Libs::Graphics::ShaderRecompiler::IR diff --git a/src/graphics/shader/recompiler/ShaderInfoCollection.h b/src/graphics/shader/recompiler/ShaderInfoCollection.h new file mode 100644 index 000000000..3694f135c --- /dev/null +++ b/src/graphics/shader/recompiler/ShaderInfoCollection.h @@ -0,0 +1,20 @@ +#ifndef EMULATOR_INCLUDE_EMULATOR_GRAPHICS_SHADER_RECOMPILER_SHADERINFOCOLLECTION_H_ +#define EMULATOR_INCLUDE_EMULATOR_GRAPHICS_SHADER_RECOMPILER_SHADERINFOCOLLECTION_H_ + +#include "graphics/shader/recompiler/ShaderIR.h" + +namespace Libs::Graphics::ShaderRecompiler::IR { + +struct ShaderInfoOptions { + const ShaderVertexInputInfo* vertex = nullptr; + const ShaderPixelInputInfo* pixel = nullptr; + const ShaderComputeInputInfo* compute = nullptr; +}; + +// Completes the immutable shader interface after resource tracking. On failure Program::info and +// all completion state remain unchanged. +bool CollectShaderInfo(Program* program, const ShaderInfoOptions& options, std::string* error); + +} // namespace Libs::Graphics::ShaderRecompiler::IR + +#endif /* EMULATOR_INCLUDE_EMULATOR_GRAPHICS_SHADER_RECOMPILER_SHADERINFOCOLLECTION_H_ */ diff --git a/src/graphics/shader/recompiler/ShaderRecompiler.cpp b/src/graphics/shader/recompiler/ShaderRecompiler.cpp new file mode 100644 index 000000000..d22d9b9ce --- /dev/null +++ b/src/graphics/shader/recompiler/ShaderRecompiler.cpp @@ -0,0 +1,872 @@ +#include "graphics/shader/recompiler/ShaderRecompiler.h" + +#include "common/assert.h" +#include "common/logging/log.h" +#include "graphics/shader/recompiler/BindingLayout.h" +#include "graphics/shader/recompiler/ResourceMaterialization.h" +#include "graphics/shader/recompiler/ResourceTracking.h" +#include "graphics/shader/recompiler/ScalarProvenance.h" +#include "graphics/shader/recompiler/ShaderCFG.h" +#include "graphics/shader/recompiler/ShaderDecoder.h" +#include "graphics/shader/recompiler/ShaderIR.h" +#include "graphics/shader/recompiler/ShaderInfoCollection.h" +#include "graphics/shader/recompiler/SpirvEmitter.h" +#include "graphics/shader/recompiler/SrtPatcher.h" +#include "graphics/shader/recompiler/SrtWalker.h" + +#include +#include +#include +#include +#include +#include +#include + +namespace Libs::Graphics::ShaderRecompiler { + +namespace { + +bool ReadZeroMemory(void*, uint64_t, uint32_t* value) { + if (value == nullptr) { + return false; + } + *value = 0; + return true; +} + +const char* GetDumpLabel(const CompileOptions& options) { + return options.dump_label != nullptr ? options.dump_label : "ShaderRecompiler"; +} + +std::string MakeIrDump(const CFG::Graph& cfg, const IR::Program& ir) { + std::string dump = "CFG:\n"; + dump += CFG::GraphToString(cfg); + dump += "\nIR:\n"; + dump += IR::ProgramToString(ir); + return dump; +} + +const char* StageName(ShaderType stage) { + switch (stage) { + case ShaderType::Compute: return "CS"; + case ShaderType::Vertex: return "VS"; + case ShaderType::Pixel: return "PS"; + default: return "unknown"; + } +} + +std::string FormatCfgFailure(const CFG::Graph& cfg, const CompileOptions& options, + const std::string& reason) { + const auto block_id = cfg.failure_block != UINT32_MAX ? cfg.failure_block : cfg.entry_block; + return CFG::FormatBlockDiagnostic(cfg, block_id, StageName(options.stage), reason); +} + +bool InstructionMaySplitSpirvBlock(const IR::Instruction& inst) { + switch (inst.op) { + case IR::Opcode::SLoadDword: + case IR::Opcode::SBufferLoadDword: + case IR::Opcode::BufferLoadUbyte: + case IR::Opcode::BufferLoadSbyte: + case IR::Opcode::BufferLoadUshort: + case IR::Opcode::BufferLoadSshort: + case IR::Opcode::BufferLoadDword: + case IR::Opcode::BufferStoreByte: + case IR::Opcode::BufferStoreShort: + case IR::Opcode::BufferStoreDword: + case IR::Opcode::AtomicSwapU32: + case IR::Opcode::AtomicAddU32: + case IR::Opcode::AtomicSubU32: + case IR::Opcode::AtomicSMinI32: + case IR::Opcode::AtomicUMinU32: + case IR::Opcode::AtomicSMaxI32: + case IR::Opcode::AtomicUMaxU32: + case IR::Opcode::AtomicAndU32: + case IR::Opcode::AtomicOrU32: + case IR::Opcode::AtomicXorU32: + case IR::Opcode::FlatLoadUbyte: + case IR::Opcode::FlatLoadSbyte: + case IR::Opcode::FlatLoadUshort: + case IR::Opcode::FlatLoadSshort: + case IR::Opcode::FlatLoadDword: + case IR::Opcode::FlatStoreByte: + case IR::Opcode::FlatStoreShort: + case IR::Opcode::FlatStoreDword: + case IR::Opcode::DsMinF32: + case IR::Opcode::DsMaxF32: + case IR::Opcode::DsWriteByte: + case IR::Opcode::DsWriteShort: + case IR::Opcode::DsWriteB32: + case IR::Opcode::DsWriteAddtidB32: + case IR::Opcode::DsAppend: + case IR::Opcode::DsConsume: + case IR::Opcode::ImageStore: + case IR::Opcode::Export: return true; + default: return false; + } +} + +bool NeedsDispatcherForStructuredLoopHeader(const IR::Program& ir, std::string* reason) { + for (const auto& block: ir.blocks) { + if (!block.terminator.loop_header) { + continue; + } + for (const auto& inst: block.instructions) { + if (!InstructionMaySplitSpirvBlock(inst)) { + continue; + } + if (reason != nullptr) { + *reason = fmt::format("loop header block {} contains an instruction whose SPIR-V " + "lowering emits internal control flow: {}", + block.id, IR::InstructionToString(inst).c_str()); + } + return true; + } + } + return false; +} + +enum class EmbeddedFetchValueType { + Unknown, + Constant, + AttribTable, + Attrib, + BufferTable, + Buffer, + Index +}; + +struct EmbeddedFetchSgprInfo { + EmbeddedFetchValueType type = EmbeddedFetchValueType::Unknown; + int attrib_id = 0; + uint32_t value = 0; + std::vector prolog_loads; +}; + +struct EmbeddedFetchVgprInfo { + EmbeddedFetchValueType type = EmbeddedFetchValueType::Unknown; +}; + +using EmbeddedFetchVectorLanes = std::map; + +uint64_t EmbeddedFetchVectorLaneKey(uint32_t reg, uint32_t lane) { + return (static_cast(reg) << 32u) | lane; +} + +uint32_t EmbeddedFetchLane(uint32_t lane, uint32_t wave_size) { + return wave_size == 32 || wave_size == 64 ? lane % wave_size : lane; +} + +void ClearEmbeddedFetchVectorLanes(EmbeddedFetchVectorLanes* lanes, uint32_t reg) { + const auto first = lanes->lower_bound(EmbeddedFetchVectorLaneKey(reg, 0)); + const auto last = lanes->lower_bound(EmbeddedFetchVectorLaneKey(reg + 1u, 0)); + lanes->erase(first, last); +} + +struct EmbeddedFetchLoad { + uint32_t pc = 0; + int attrib_id = -1; + uint32_t components = 0; + std::vector prolog_loads; +}; + +struct EmbeddedFetchData { + std::vector loads; + int32_t vertex_offset_sgpr = -1; +}; + +bool IsDecodedSgpr(const Decoder::Operand& op) { + return op.kind == Decoder::OperandKind::Sgpr || op.kind == Decoder::OperandKind::VccLo || + op.kind == Decoder::OperandKind::VccHi; +} + +uint32_t DecodedSgprReg(const Decoder::Operand& op) { + return op.kind == Decoder::OperandKind::VccLo + ? 106u + : (op.kind == Decoder::OperandKind::VccHi ? 107u : op.reg); +} + +bool IsDecodedVgpr(const Decoder::Operand& op) { + return op.kind == Decoder::OperandKind::Vgpr; +} + +uint32_t DecodedDstSize(const Decoder::Instruction& inst) { + return std::max(inst.data_dwords, 1u); +} + +uint32_t EmbeddedFetchDstSize(const Decoder::Instruction& inst) { + return inst.opcode == Decoder::Opcode::VMadU64U32 ? 2u : DecodedDstSize(inst); +} + +bool EmbeddedFetchHasBranch(Decoder::Opcode opcode) { + switch (opcode) { + case Decoder::Opcode::SSetpcB64: + case Decoder::Opcode::SBranch: + case Decoder::Opcode::SCbranchScc0: + case Decoder::Opcode::SCbranchScc1: + case Decoder::Opcode::SCbranchVccz: + case Decoder::Opcode::SCbranchVccnz: + case Decoder::Opcode::SCbranchExecz: + case Decoder::Opcode::SCbranchExecnz: return true; + default: return false; + } +} + +void ClearEmbeddedFetchSgprs(std::array& sgprs, + const Decoder::Operand& dst, uint32_t size) { + if (!IsDecodedSgpr(dst)) { + return; + } + const auto register_id = DecodedSgprReg(dst); + for (uint32_t i = 0; i < size && register_id + i < sgprs.size(); i++) { + sgprs[register_id + i] = {}; + } +} + +bool TryDecodedOperandConstant(const std::array& sgprs, + const Decoder::Operand& op, uint32_t& value) { + switch (op.kind) { + case Decoder::OperandKind::LiteralConstant: + case Decoder::OperandKind::IntegerInlineConstant: + case Decoder::OperandKind::FloatInlineConstant: value = op.value; return true; + case Decoder::OperandKind::Null: value = 0; return true; + default: break; + } + if (IsDecodedSgpr(op) && DecodedSgprReg(op) < sgprs.size() && + sgprs[DecodedSgprReg(op)].type == EmbeddedFetchValueType::Constant) { + value = sgprs[DecodedSgprReg(op)].value; + return true; + } + return false; +} + +bool TryDecodedSmemOffset(const std::array& sgprs, + const Decoder::Instruction& inst, uint32_t& raw_offset) { + uint32_t base = 0; + if (!TryDecodedOperandConstant(sgprs, inst.src1, base)) { + return false; + } + const auto value = static_cast(base) + inst.offset; + if (value > 0xffffffffull) { + return false; + } + raw_offset = static_cast(value); + return true; +} + +bool IsEmbeddedFetchSLoad(const Decoder::Instruction& inst) { + switch (inst.opcode) { + case Decoder::Opcode::SLoadDword: + case Decoder::Opcode::SLoadDwordx2: + case Decoder::Opcode::SLoadDwordx4: + case Decoder::Opcode::SLoadDwordx8: + case Decoder::Opcode::SLoadDwordx16: return true; + default: return false; + } +} + +bool IsEmbeddedFetchBufferLoad(const Decoder::Instruction& inst) { + switch (inst.opcode) { + case Decoder::Opcode::BufferLoadFormatX: + case Decoder::Opcode::BufferLoadFormatXy: + case Decoder::Opcode::BufferLoadFormatXyz: + case Decoder::Opcode::BufferLoadFormatXyzw: return true; + default: return false; + } +} + +bool IsEmbeddedFetchAttribPropagationAlu(const Decoder::Instruction& inst) { + switch (inst.opcode) { + case Decoder::Opcode::SBfeU32: + case Decoder::Opcode::SAndB32: + case Decoder::Opcode::SAddI32: + case Decoder::Opcode::SAddU32: + case Decoder::Opcode::SLshlB32: return true; + default: return false; + } +} + +int BufferTableAttribFromOffset(uint32_t raw_offset, int dword) { + return static_cast((raw_offset + static_cast(dword) * 4u) / 16u); +} + +void AppendUniquePcs(std::vector* dst, const std::vector& src) { + EXIT_IF(dst == nullptr); + dst->reserve(dst->size() + src.size()); + for (auto pc: src) { + if (std::find(dst->begin(), dst->end(), pc) == dst->end()) { + dst->push_back(pc); + } + } +} + +EmbeddedFetchData DetectEmbeddedVertexFetch(const Decoder::Program& decoded, + const ShaderVertexInputInfo* input_info, + uint32_t user_data_base, uint32_t user_data_count, + uint32_t wave_size) { + EmbeddedFetchData data; + if (input_info == nullptr || !input_info->fetch_embedded) { + return data; + } + data.loads.reserve(input_info->resources_num); + int32_t offset_candidate = -1; + bool offset_conflict = false; + + const int shift_regs = 8; + const int attrib_reg = input_info->fetch_attrib_reg + shift_regs; + const int buffer_reg = input_info->fetch_buffer_reg + shift_regs; + + std::array sgprs {}; + std::array vgprs {}; + EmbeddedFetchVectorLanes vector_lanes; + const bool track_vector_lanes = + std::none_of(decoded.instructions.begin(), decoded.instructions.end(), + [](const auto& inst) { return EmbeddedFetchHasBranch(inst.opcode); }); + + if (attrib_reg >= 0 && attrib_reg < static_cast(sgprs.size())) { + sgprs[attrib_reg].type = EmbeddedFetchValueType::AttribTable; + } + if (attrib_reg + 1 >= 0 && attrib_reg + 1 < static_cast(sgprs.size())) { + sgprs[attrib_reg + 1].type = EmbeddedFetchValueType::AttribTable; + } + if (buffer_reg >= 0 && buffer_reg < static_cast(sgprs.size())) { + sgprs[buffer_reg].type = EmbeddedFetchValueType::BufferTable; + } + if (buffer_reg + 1 >= 0 && buffer_reg + 1 < static_cast(sgprs.size())) { + sgprs[buffer_reg + 1].type = EmbeddedFetchValueType::BufferTable; + } + + for (const auto& inst: decoded.instructions) { + if (data.loads.empty() && inst.opcode == Decoder::Opcode::VAddI32 && + IsDecodedVgpr(inst.dst) && inst.dst.reg == 0 && IsDecodedSgpr(inst.src0) && + IsDecodedVgpr(inst.src1) && inst.src1.reg == 0) { + const auto reg = DecodedSgprReg(inst.src0); + if (reg >= user_data_base && reg - user_data_base < user_data_count) { + if (offset_candidate >= 0 && offset_candidate != static_cast(reg)) { + offset_conflict = true; + } else { + offset_candidate = static_cast(reg); + } + } + } + switch (inst.opcode) { + case Decoder::Opcode::VWritelaneB32: { + uint32_t lane = 0; + if (IsDecodedVgpr(inst.dst) && inst.dst.reg < vgprs.size()) { + vgprs[inst.dst.reg] = {}; + } + if (track_vector_lanes && IsDecodedVgpr(inst.dst) && IsDecodedSgpr(inst.src0) && + DecodedSgprReg(inst.src0) < sgprs.size() && + TryDecodedOperandConstant(sgprs, inst.src1, lane)) { + vector_lanes[EmbeddedFetchVectorLaneKey(inst.dst.reg, + EmbeddedFetchLane(lane, wave_size))] = + sgprs[DecodedSgprReg(inst.src0)]; + } else if (IsDecodedVgpr(inst.dst)) { + ClearEmbeddedFetchVectorLanes(&vector_lanes, inst.dst.reg); + } + break; + } + case Decoder::Opcode::VReadlaneB32: { + uint32_t lane = 0; + if (track_vector_lanes && IsDecodedSgpr(inst.dst) && + DecodedSgprReg(inst.dst) < sgprs.size() && IsDecodedVgpr(inst.src0) && + TryDecodedOperandConstant(sgprs, inst.src1, lane)) { + const auto found = vector_lanes.find(EmbeddedFetchVectorLaneKey( + inst.src0.reg, EmbeddedFetchLane(lane, wave_size))); + sgprs[DecodedSgprReg(inst.dst)] = + found != vector_lanes.end() ? found->second : EmbeddedFetchSgprInfo {}; + } else if (IsDecodedSgpr(inst.dst)) { + ClearEmbeddedFetchSgprs(sgprs, inst.dst, 1); + } + break; + } + case Decoder::Opcode::SMovB32: + if (IsDecodedSgpr(inst.dst) && IsDecodedSgpr(inst.src0) && + DecodedSgprReg(inst.src0) < sgprs.size()) { + sgprs[DecodedSgprReg(inst.dst)] = sgprs[DecodedSgprReg(inst.src0)]; + } else if (IsDecodedSgpr(inst.dst)) { + uint32_t value = 0; + if (TryDecodedOperandConstant(sgprs, inst.src0, value)) { + auto& dst = sgprs[DecodedSgprReg(inst.dst)]; + dst.type = EmbeddedFetchValueType::Constant; + dst.value = value; + dst.prolog_loads.clear(); + } else { + ClearEmbeddedFetchSgprs(sgprs, inst.dst, 1); + } + } + break; + case Decoder::Opcode::SMovkI32: + if (IsDecodedSgpr(inst.dst)) { + auto& dst = sgprs[DecodedSgprReg(inst.dst)]; + dst.type = EmbeddedFetchValueType::Constant; + dst.value = inst.src0.value; + dst.prolog_loads.clear(); + } + break; + default: + if (IsEmbeddedFetchSLoad(inst)) { + if (IsDecodedSgpr(inst.src0) && DecodedSgprReg(inst.src0) < sgprs.size() && + sgprs[DecodedSgprReg(inst.src0)].type == + EmbeddedFetchValueType::AttribTable) { + uint32_t raw_offset = 0; + if (TryDecodedSmemOffset(sgprs, inst, raw_offset)) { + const auto register_id = DecodedSgprReg(inst.dst); + const int index = static_cast(raw_offset / 4u); + for (uint32_t i = 0; + i < DecodedDstSize(inst) && register_id + i < sgprs.size(); i++) { + auto& dst = sgprs[register_id + i]; + dst.type = EmbeddedFetchValueType::Attrib; + dst.attrib_id = index + static_cast(i); + dst.prolog_loads = {inst.pc}; + } + } else { + ClearEmbeddedFetchSgprs(sgprs, inst.dst, DecodedDstSize(inst)); + } + } else if (IsDecodedSgpr(inst.src0) && + DecodedSgprReg(inst.src0) < sgprs.size() && + sgprs[DecodedSgprReg(inst.src0)].type == + EmbeddedFetchValueType::BufferTable) { + const auto register_id = DecodedSgprReg(inst.dst); + uint32_t raw_offset = 0; + if (TryDecodedSmemOffset(sgprs, inst, raw_offset)) { + for (uint32_t i = 0; + i < DecodedDstSize(inst) && register_id + i < sgprs.size(); i++) { + auto& dst = sgprs[register_id + i]; + dst.type = EmbeddedFetchValueType::Buffer; + dst.attrib_id = + BufferTableAttribFromOffset(raw_offset, static_cast(i)); + dst.prolog_loads = {inst.pc}; + } + } else if (IsDecodedSgpr(inst.src1) && + DecodedSgprReg(inst.src1) < sgprs.size() && + sgprs[DecodedSgprReg(inst.src1)].type == + EmbeddedFetchValueType::Attrib && + (inst.offset & 0x3u) == 0) { + for (uint32_t i = 0; + i < DecodedDstSize(inst) && register_id + i < sgprs.size(); i++) { + auto& dst = sgprs[register_id + i]; + dst.type = EmbeddedFetchValueType::Buffer; + dst.attrib_id = sgprs[DecodedSgprReg(inst.src1)].attrib_id; + dst.prolog_loads = sgprs[DecodedSgprReg(inst.src1)].prolog_loads; + dst.prolog_loads.push_back(inst.pc); + } + } else { + ClearEmbeddedFetchSgprs(sgprs, inst.dst, DecodedDstSize(inst)); + } + } else { + ClearEmbeddedFetchSgprs(sgprs, inst.dst, DecodedDstSize(inst)); + } + } else if (inst.opcode == Decoder::Opcode::VCndmaskB32) { + if (IsDecodedVgpr(inst.dst) && inst.dst.reg < vgprs.size()) { + ClearEmbeddedFetchVectorLanes(&vector_lanes, inst.dst.reg); + } + if (IsDecodedVgpr(inst.dst) && inst.dst.reg < vgprs.size() && + IsDecodedVgpr(inst.src0) && inst.src0.reg == 8 && + IsDecodedVgpr(inst.src1) && inst.src1.reg == 5) { + vgprs[inst.dst.reg].type = EmbeddedFetchValueType::Index; + } + } else if (IsEmbeddedFetchAttribPropagationAlu(inst)) { + if (IsDecodedSgpr(inst.dst) && IsDecodedSgpr(inst.src0) && + DecodedSgprReg(inst.src0) < sgprs.size() && + sgprs[DecodedSgprReg(inst.src0)].type == EmbeddedFetchValueType::Attrib) { + sgprs[DecodedSgprReg(inst.dst)] = sgprs[DecodedSgprReg(inst.src0)]; + } else if (IsDecodedSgpr(inst.dst)) { + uint32_t src0 = 0; + uint32_t src1 = 0; + if (TryDecodedOperandConstant(sgprs, inst.src0, src0) && + TryDecodedOperandConstant(sgprs, inst.src1, src1)) { + auto& dst = sgprs[DecodedSgprReg(inst.dst)]; + dst.type = EmbeddedFetchValueType::Constant; + switch (inst.opcode) { + case Decoder::Opcode::SAndB32: dst.value = src0 & src1; break; + case Decoder::Opcode::SLshlB32: + dst.value = src0 << (src1 & 31u); + break; + case Decoder::Opcode::SBfeU32: + dst.value = src0 >> (src1 & 31u); + break; + default: dst.value = src0 + src1; break; + } + dst.prolog_loads.clear(); + } else { + ClearEmbeddedFetchSgprs(sgprs, inst.dst, 1); + } + } + } else if (IsEmbeddedFetchBufferLoad(inst)) { + if (IsDecodedVgpr(inst.src0) && inst.src0.reg < vgprs.size() && + vgprs[inst.src0.reg].type == EmbeddedFetchValueType::Index && + IsDecodedSgpr(inst.src1) && DecodedSgprReg(inst.src1) < sgprs.size() && + sgprs[DecodedSgprReg(inst.src1)].type == EmbeddedFetchValueType::Buffer) { + const auto& buffer = sgprs[DecodedSgprReg(inst.src1)]; + EmbeddedFetchLoad load; + load.pc = inst.pc; + load.attrib_id = buffer.attrib_id; + load.components = DecodedDstSize(inst); + load.prolog_loads = buffer.prolog_loads; + if (data.loads.empty() && !offset_conflict) { + data.vertex_offset_sgpr = offset_candidate; + } + data.loads.push_back(load); + } + } + break; + } + if (inst.opcode == Decoder::Opcode::VMovreldB32) { + vector_lanes.clear(); + } else if (inst.opcode != Decoder::Opcode::VWritelaneB32 && IsDecodedVgpr(inst.dst)) { + for (uint32_t i = 0; i < EmbeddedFetchDstSize(inst) && inst.dst.reg + i < vgprs.size(); + i++) { + ClearEmbeddedFetchVectorLanes(&vector_lanes, inst.dst.reg + i); + } + } + } + + return data; +} + +const EmbeddedFetchLoad* FindEmbeddedFetchLoad(const std::vector& loads, + uint32_t pc) { + for (const auto& load: loads) { + if (load.pc == pc) { + return &load; + } + } + return nullptr; +} + +bool EmbeddedFetchPcInList(const std::vector& pcs, uint32_t pc) { + return std::find(pcs.begin(), pcs.end(), pc) != pcs.end(); +} + +bool IsIrFetchPrologLoad(const IR::Instruction& inst) { + return inst.op == IR::Opcode::SLoadDword || inst.op == IR::Opcode::SBufferLoadDword; +} + +int ResolveEmbeddedFetchResource(const ShaderVertexInputInfo* input_info, + const EmbeddedFetchLoad& load) { + if (input_info == nullptr) { + return -1; + } + if (load.attrib_id >= 0 && load.attrib_id < input_info->resources_num && + input_info->resources_dst[load.attrib_id].attr_id == load.attrib_id) { + return load.attrib_id; + } + for (int i = 0; i < input_info->resources_num; i++) { + const auto& dst = input_info->resources_dst[i]; + if (dst.attr_id == load.attrib_id && + load.components <= static_cast(std::max(dst.registers_num, 1))) { + return i; + } + } + for (int i = 0; i < input_info->resources_num; i++) { + if (input_info->resources_dst[i].attr_id == load.attrib_id) { + return i; + } + } + return -1; +} + +uint32_t RewriteEmbeddedVertexFetches(IR::Program* ir, const ShaderVertexInputInfo* input_info, + const std::vector& loads) { + if (ir == nullptr || input_info == nullptr || loads.empty()) { + return 0; + } + + std::vector prolog_pcs; + prolog_pcs.reserve(loads.size()); + for (const auto& load: loads) { + AppendUniquePcs(&prolog_pcs, load.prolog_loads); + } + + auto* mutable_input_info = const_cast(input_info); + uint32_t rewritten = 0; + for (auto& block: ir->blocks) { + for (auto& inst: block.instructions) { + if (IsIrFetchPrologLoad(inst) && EmbeddedFetchPcInList(prolog_pcs, inst.pc)) { + auto pc = inst.pc; + inst = {}; + inst.pc = pc; + inst.op = IR::Opcode::ControlNop; + continue; + } + + if (inst.op != IR::Opcode::BufferLoadDword) { + continue; + } + const auto* load = FindEmbeddedFetchLoad(loads, inst.pc); + if (load == nullptr || inst.memory.component_index >= load->components) { + continue; + } + + const auto resource_id = ResolveEmbeddedFetchResource(input_info, *load); + if (resource_id < 0 || resource_id >= input_info->resources_num) { + LOGF("ShaderRecompiler VS embedded fetch remap failed: pc=0x%08" PRIx32 + " attrib=%d resources=%d\n", + inst.pc, load->attrib_id, input_info->resources_num); + continue; + } + + inst.op = IR::Opcode::LoadInputF32; + inst.input_info.attr = static_cast(resource_id); + inst.input_info.chan = inst.memory.component_index; + inst.memory = {}; + inst.src_count = 0; + + mutable_input_info->resource_fetch_components[resource_id] = + std::max(mutable_input_info->resource_fetch_components[resource_id], + static_cast(inst.input_info.chan + 1u)); + rewritten++; + } + } + + return rewritten; +} + +} // namespace + +bool TryRecompile(std::span code, const CompileOptions& options, + CompileResult* result, std::string* error) { + if (code.empty() || result == nullptr) { + if (error != nullptr) { + *error = "invalid shader recompiler input"; + } + return false; + } + if (options.stage != ShaderType::Compute && options.stage != ShaderType::Vertex && + options.stage != ShaderType::Pixel) { + if (error != nullptr) { + *error = "shader recompiler supports compute, vertex, and pixel stages"; + } + return false; + } + + const auto compile_begin = std::chrono::steady_clock::now(); + const auto phase_ms = [&compile_begin]() { + return static_cast(std::chrono::duration_cast( + std::chrono::steady_clock::now() - compile_begin) + .count()); + }; + + LOGF("%s phase begin: stage=%s hash=0x%016" PRIx64 " code_words=%" PRIu64 " decode\n", + GetDumpLabel(options), StageName(options.stage), options.shader_hash, + static_cast(code.size())); + + Decoder::Program decoded; + if (!Decoder::DecodeProgram(code, &decoded, error)) { + return false; + } + LOGF("%s phase end: stage=%s hash=0x%016" PRIx64 " decode instructions=%" PRIu64 + " elapsed_ms=%" PRIu64 "\n", + GetDumpLabel(options), StageName(options.stage), options.shader_hash, + static_cast(decoded.instructions.size()), phase_ms()); + + std::string decoded_dump; + if (options.dump_ir) { + decoded_dump = Decoder::ProgramToString(decoded); + if (options.early_dump) { + LOGF("%s decoded RDNA2 (early):\n%s", GetDumpLabel(options), decoded_dump.c_str()); + } + } + + CFG::Graph cfg; + LOGF("%s phase begin: stage=%s hash=0x%016" PRIx64 " CFG BuildGraph\n", GetDumpLabel(options), + StageName(options.stage), options.shader_hash); + if (!CFG::BuildGraph(decoded, &cfg, error)) { + return false; + } + LOGF("%s phase end: stage=%s hash=0x%016" PRIx64 " CFG BuildGraph blocks=%" PRIu64 + " loops=%" PRIu64 " back_edges=%" PRIu64 " elapsed_ms=%" PRIu64 "\n", + GetDumpLabel(options), StageName(options.stage), options.shader_hash, + static_cast(cfg.blocks.size()), static_cast(cfg.natural_loops.size()), + static_cast(cfg.back_edges.size()), phase_ms()); + bool dispatcher_fallback = false; + std::string dispatcher_reason; + if (cfg.irreducible) { + dispatcher_fallback = true; + dispatcher_reason = cfg.unsupported_reason; + const auto diagnostic = FormatCfgFailure(cfg, options, cfg.unsupported_reason); + LOGF("%s irreducible CFG detected: %s\n", GetDumpLabel(options), diagnostic.c_str()); + } else { + std::string structure_error; + const auto unstructured_cfg = cfg; + LOGF("%s phase begin: stage=%s hash=0x%016" PRIx64 " CFG Structurize\n", + GetDumpLabel(options), StageName(options.stage), options.shader_hash); + if (!CFG::Structurize(&cfg, &structure_error)) { + const auto diagnostic = FormatCfgFailure(cfg, options, structure_error); + LOGF("%s structured CFG bug/failure: %s\n", GetDumpLabel(options), diagnostic.c_str()); + dispatcher_fallback = true; + dispatcher_reason = structure_error; + const auto failure_kind = cfg.failure_kind; + const auto failure_block = cfg.failure_block; + cfg = unstructured_cfg; + cfg.unsupported = true; + cfg.failure_kind = failure_kind; + cfg.failure_block = failure_block; + cfg.unsupported_reason = structure_error; + } else { + LOGF("%s structured CFG success: blocks=%" PRIu64 "\n", GetDumpLabel(options), + static_cast(cfg.blocks.size())); + } + LOGF("%s phase end: stage=%s hash=0x%016" PRIx64 " CFG Structurize blocks=%" PRIu64 + " loops=%" PRIu64 " elapsed_ms=%" PRIu64 "\n", + GetDumpLabel(options), StageName(options.stage), options.shader_hash, + static_cast(cfg.blocks.size()), + static_cast(cfg.natural_loops.size()), phase_ms()); + } + + IR::Program ir; + LOGF("%s phase begin: stage=%s hash=0x%016" PRIx64 " IR LowerProgram\n", GetDumpLabel(options), + StageName(options.stage), options.shader_hash); + if (!IR::LowerProgram(decoded, cfg, options.stage, options.wave_size, &ir, error)) { + return false; + } + ir.lane_mask_mode = options.lane_mask_mode; + ir.shader_hash = options.shader_hash; + ir.user_data_base = options.user_data_base; + ir.user_data_count = options.user_data_count; + LOGF("%s phase end: stage=%s hash=0x%016" PRIx64 " IR LowerProgram blocks=%" PRIu64 + " elapsed_ms=%" PRIu64 "\n", + GetDumpLabel(options), StageName(options.stage), options.shader_hash, + static_cast(ir.blocks.size()), phase_ms()); + EmbeddedFetchData embedded_fetch; + if (options.stage == ShaderType::Vertex && options.vertex_input_info != nullptr && + options.vertex_input_info->fetch_embedded) { + embedded_fetch = + DetectEmbeddedVertexFetch(decoded, options.vertex_input_info, ir.user_data_base, + ir.user_data_count, options.wave_size); + auto rewritten = + RewriteEmbeddedVertexFetches(&ir, options.vertex_input_info, embedded_fetch.loads); + if (rewritten > 0 || !embedded_fetch.loads.empty()) { + LOGF("%s embedded vertex fetch rewrite: detected=%" PRIu64 " rewritten=%" PRIu32 "\n", + GetDumpLabel(options), static_cast(embedded_fetch.loads.size()), + rewritten); + } + } + if (!IR::BuildScalarProvenance(&ir, error)) { + return false; + } + std::string srt_error; + if (!IR::BuildSrtPlan(&ir, &srt_error)) { + LOGF("%s SRT planning failed: %s\n", GetDumpLabel(options), srt_error.c_str()); + if (error != nullptr) { + *error = std::move(srt_error); + } + return false; + } + ir.dispatcher_fallback = dispatcher_fallback; + if (!dispatcher_reason.empty()) { + ir.fallback_reason = dispatcher_reason; + } + + if (!IR::PatchSrtReads(&ir, error) || !IR::TrackResources(&ir, error)) { + return false; + } + if (options.stage == ShaderType::Vertex) { + ir.info.vertex_offset_sgpr = embedded_fetch.vertex_offset_sgpr; + } + if (!dispatcher_fallback) { + std::string emitter_reason; + if (NeedsDispatcherForStructuredLoopHeader(ir, &emitter_reason)) { + dispatcher_fallback = true; + dispatcher_reason = emitter_reason; + ir.dispatcher_fallback = true; + ir.cfg_failure_kind = CFG::FailureKind::StructuredControlFlow; + ir.fallback_reason = emitter_reason; + LOGF("%s structured CFG emitter fallback: %s\n", GetDumpLabel(options), + emitter_reason.c_str()); + } + } + + IR::ResourceSnapshot resources; + if (options.resource_snapshot != nullptr) { + resources = *options.resource_snapshot; + } else { + IR::SrtRuntime runtime; + std::array zero_user_data {}; + if (options.user_data == nullptr) { + for (uint32_t i = 2; i < zero_user_data.size(); i += 4) { + zero_user_data[i] = UINT32_MAX; + } + } + const auto* user_data = + options.user_data != nullptr ? options.user_data : zero_user_data.data(); + runtime.user_data = std::span(user_data, options.user_data_count); + runtime.shader_base = options.shader_base != 0 ? options.shader_base + : reinterpret_cast(code.data()); + runtime.read_memory = options.read_memory != nullptr ? options.read_memory + : options.user_data == nullptr ? ReadZeroMemory + : nullptr; + runtime.userdata = options.read_memory_data; + runtime.flat_memory_base = options.flat_memory_base; + if (!IR::MaterializeResources(ir, runtime, &resources, error)) { + return false; + } + } + if (!IR::SpecializeResources(&ir, resources, error)) { + return false; + } + + ShaderVertexInputInfo default_vertex {}; + ShaderPixelInputInfo default_pixel {}; + ShaderComputeInputInfo default_compute {}; + IR::ShaderInfoOptions info_options; + info_options.vertex = + options.vertex_input_info != nullptr ? options.vertex_input_info : &default_vertex; + info_options.pixel = + options.pixel_input_info != nullptr ? options.pixel_input_info : &default_pixel; + info_options.compute = + options.compute_input_info != nullptr ? options.compute_input_info : &default_compute; + if (!IR::CollectShaderInfo(&ir, info_options, error)) { + return false; + } + IR::BindingLayoutOptions layout_options; + layout_options.descriptor_set = options.descriptor_set; + layout_options.push_constant_offset = options.push_constant_offset; + if (!IR::AllocateBindings(&ir, layout_options, error)) { + return false; + } + std::string ir_dump; + if (options.dump_ir) { + ir_dump = MakeIrDump(cfg, ir); + if (options.early_dump) { + LOGF("%s native IR and bindings (early):\n%s", GetDumpLabel(options), ir_dump.c_str()); + } + } + + std::vector spirv; + std::string emit_error; + LOGF("%s phase begin: stage=%s hash=0x%016" PRIx64 " SPIR-V EmitProgram\n", + GetDumpLabel(options), StageName(options.stage), options.shader_hash); + if (!Spirv::EmitProgram(ir, resources, options.vertex_input_info, options.pixel_input_info, + options.compute_input_info, &spirv, &emit_error)) { + if (dispatcher_fallback && error != nullptr) { + *error = fmt::format("dispatcher fallback failed after {}: {}", + dispatcher_reason.c_str(), emit_error.c_str()); + LOGF("%s dispatcher fallback emit failed: %s\n", GetDumpLabel(options), error->c_str()); + } else if (error != nullptr) { + *error = emit_error; + } + return false; + } + LOGF("%s phase end: stage=%s hash=0x%016" PRIx64 " SPIR-V EmitProgram words=%" PRIu64 + " elapsed_ms=%" PRIu64 "\n", + GetDumpLabel(options), StageName(options.stage), options.shader_hash, + static_cast(spirv.size()), phase_ms()); + if (dispatcher_fallback) { + LOGF("%s dispatcher fallback used: %s\n", GetDumpLabel(options), dispatcher_reason.c_str()); + } + + result->spirv = std::move(spirv); + result->program = std::move(ir); + result->resources = std::move(resources); + if (options.dump_ir) { + result->decoded_dump = std::move(decoded_dump); + result->ir_dump = std::move(ir_dump); + } else { + result->decoded_dump.clear(); + result->ir_dump.clear(); + } + return true; +} + +} // namespace Libs::Graphics::ShaderRecompiler diff --git a/src/graphics/shader/recompiler/ShaderRecompiler.h b/src/graphics/shader/recompiler/ShaderRecompiler.h new file mode 100644 index 000000000..9819de561 --- /dev/null +++ b/src/graphics/shader/recompiler/ShaderRecompiler.h @@ -0,0 +1,51 @@ +#ifndef EMULATOR_INCLUDE_EMULATOR_GRAPHICS_SHADER_RECOMPILER_SHADERRECOMPILER_H_ +#define EMULATOR_INCLUDE_EMULATOR_GRAPHICS_SHADER_RECOMPILER_SHADERRECOMPILER_H_ + +#include "common/common.h" +#include "common/stringUtils.h" +#include "graphics/shader/recompiler/ResourceMaterialization.h" +#include "graphics/shader/shader.h" + +#include +#include +#include + +namespace Libs::Graphics::ShaderRecompiler { + +struct CompileOptions { + ShaderType stage = ShaderType::Compute; + ShaderLaneMaskMode lane_mask_mode = ShaderLaneMaskMode::NativeWave; + uint32_t wave_size = 64; + uint32_t user_data_base = 0; + uint32_t user_data_count = 64; + uint64_t shader_hash = 0; + uint64_t shader_base = 0; + std::optional flat_memory_base; + uint32_t descriptor_set = 0; + uint32_t push_constant_offset = 0; + bool dump_ir = true; + bool early_dump = false; + const char* dump_label = nullptr; + const uint32_t* user_data = nullptr; + IR::SrtMemoryReader read_memory = nullptr; + void* read_memory_data = nullptr; + const IR::ResourceSnapshot* resource_snapshot = nullptr; + const ShaderVertexInputInfo* vertex_input_info = nullptr; + const ShaderPixelInputInfo* pixel_input_info = nullptr; + const ShaderComputeInputInfo* compute_input_info = nullptr; +}; + +struct CompileResult { + std::vector spirv; + std::string decoded_dump; + std::string ir_dump; + IR::Program program; + IR::ResourceSnapshot resources; +}; + +bool TryRecompile(std::span code, const CompileOptions& options, + CompileResult* result, std::string* error); + +} // namespace Libs::Graphics::ShaderRecompiler + +#endif /* EMULATOR_INCLUDE_EMULATOR_GRAPHICS_SHADER_RECOMPILER_SHADERRECOMPILER_H_ */ diff --git a/src/graphics/shader/recompiler/SpirvBuilder.cpp b/src/graphics/shader/recompiler/SpirvBuilder.cpp new file mode 100644 index 000000000..4261842c7 --- /dev/null +++ b/src/graphics/shader/recompiler/SpirvBuilder.cpp @@ -0,0 +1,160 @@ +#include "graphics/shader/recompiler/SpirvBuilder.h" + +#include +#include + +namespace Libs::Graphics::ShaderRecompiler::Spirv { + +static constexpr size_t InitialSpirvSectionReserve = 4096; +static constexpr size_t InitialSpirvFunctionSectionReserve = 450000; + +static void AppendInstructionWords(std::vector* section, const uint32_t* words, + size_t words_num) { + if (words_num == 0) { + return; + } + const auto opcode = words[0]; + const auto word_count = static_cast(words_num); + section->push_back((word_count << 16u) | opcode); + section->insert(section->end(), words + 1, words + words_num); +} + +Builder::Builder() { + m_debug.reserve(InitialSpirvSectionReserve); + m_annotations.reserve(InitialSpirvSectionReserve); + m_types.reserve(InitialSpirvSectionReserve); + m_functions.reserve(InitialSpirvFunctionSectionReserve); +} + +uint32_t Builder::AllocateId() { + return m_next_id++; +} + +void Builder::AppendString(std::vector* words, const char* text) { + const auto len = text != nullptr ? std::strlen(text) : 0; + const auto word_count = (len + 1u + 3u) / 4u; + for (size_t i = 0; i < word_count; i++) { + uint32_t word = 0; + for (size_t byte = 0; byte < 4; byte++) { + const auto index = i * 4u + byte; + if (index < len) { + word |= static_cast(static_cast(text[index])) + << (byte * 8u); + } + } + words->push_back(word); + } +} + +void Builder::AppendInstruction(std::vector* section, uint32_t opcode, + const std::vector& operands) { + const uint32_t word_count = static_cast(operands.size() + 1u); + section->push_back((word_count << 16u) | opcode); + section->insert(section->end(), operands.begin(), operands.end()); +} + +void Builder::AppendInstruction(std::vector* section, uint32_t opcode, + std::initializer_list operands) { + const uint32_t word_count = static_cast(operands.size() + 1u); + section->push_back((word_count << 16u) | opcode); + section->insert(section->end(), operands.begin(), operands.end()); +} + +void Builder::AddCapability(std::initializer_list operands) { + AppendInstruction(&m_capabilities, 17u, operands); +} + +void Builder::AddExtension(const char* name) { + std::vector operands; + AppendString(&operands, name); + AppendInstruction(&m_extensions, 10u, operands); +} + +void Builder::AddExtInstImport(uint32_t id, const char* name) { + std::vector operands = {id}; + AppendString(&operands, name); + AppendInstruction(&m_ext_inst_imports, 11u, operands); +} + +void Builder::AddMemoryModel(std::initializer_list operands) { + AppendInstruction(&m_memory_model, 14u, operands); +} + +void Builder::AddEntryPoint(uint32_t execution_model, uint32_t entry_point, const char* name, + const std::vector& interfaces) { + std::vector operands = {execution_model, entry_point}; + AppendString(&operands, name); + operands.insert(operands.end(), interfaces.begin(), interfaces.end()); + AppendInstruction(&m_entry_points, 15u, operands); +} + +void Builder::AddExecutionMode(std::initializer_list operands) { + AppendInstruction(&m_execution_modes, 16u, operands); +} + +void Builder::AddName(uint32_t target, const char* name) { + std::vector operands = {target}; + AppendString(&operands, name); + AppendInstruction(&m_debug, 5u, operands); +} + +void Builder::AddAnnotation(std::initializer_list words) { + AppendInstructionWords(&m_annotations, words.begin(), words.size()); +} + +void Builder::AddType(std::initializer_list words) { + AppendInstructionWords(&m_types, words.begin(), words.size()); +} + +void Builder::AddFunction(std::initializer_list words) { + AppendInstructionWords(&m_functions, words.begin(), words.size()); +} + +void Builder::AddFunction(const std::vector& words) { + AppendInstructionWords(&m_functions, words.data(), words.size()); +} + +std::vector Builder::Build() const { + if (m_debug.size() > InitialSpirvSectionReserve) { + std::printf("SPIR-V builder reserve exceeded: section=debug size=%zu reserve=%zu\n", + m_debug.size(), InitialSpirvSectionReserve); + } + if (m_annotations.size() > InitialSpirvSectionReserve) { + std::printf("SPIR-V builder reserve exceeded: section=annotations size=%zu reserve=%zu\n", + m_annotations.size(), InitialSpirvSectionReserve); + } + if (m_types.size() > InitialSpirvSectionReserve) { + std::printf("SPIR-V builder reserve exceeded: section=types size=%zu reserve=%zu\n", + m_types.size(), InitialSpirvSectionReserve); + } + if (m_functions.size() > InitialSpirvFunctionSectionReserve) { + std::printf("SPIR-V builder reserve exceeded: section=functions size=%zu reserve=%zu\n", + m_functions.size(), InitialSpirvFunctionSectionReserve); + } + + std::vector module; + module.reserve(5u + m_capabilities.size() + m_extensions.size() + m_ext_inst_imports.size() + + m_memory_model.size() + m_entry_points.size() + m_execution_modes.size() + + m_debug.size() + m_annotations.size() + m_types.size() + m_functions.size()); + + module.push_back(0x07230203u); + module.push_back(0x00010300u); + module.push_back(0u); + module.push_back(m_next_id); + module.push_back(0u); + + module.insert(module.end(), m_capabilities.begin(), m_capabilities.end()); + module.insert(module.end(), m_extensions.begin(), m_extensions.end()); + module.insert(module.end(), m_ext_inst_imports.begin(), m_ext_inst_imports.end()); + module.insert(module.end(), m_memory_model.begin(), m_memory_model.end()); + module.insert(module.end(), m_entry_points.begin(), m_entry_points.end()); + module.insert(module.end(), m_execution_modes.begin(), m_execution_modes.end()); + module.insert(module.end(), m_debug.begin(), m_debug.end()); + module.insert(module.end(), m_annotations.begin(), m_annotations.end()); + module.insert(module.end(), m_types.begin(), m_types.end()); + module.insert(module.end(), m_functions.begin(), m_functions.end()); + + return module; +} + +} // namespace Libs::Graphics::ShaderRecompiler::Spirv diff --git a/src/graphics/shader/recompiler/SpirvBuilder.h b/src/graphics/shader/recompiler/SpirvBuilder.h new file mode 100644 index 000000000..ccd184b0a --- /dev/null +++ b/src/graphics/shader/recompiler/SpirvBuilder.h @@ -0,0 +1,56 @@ +#ifndef EMULATOR_INCLUDE_EMULATOR_GRAPHICS_SHADER_RECOMPILER_SPIRVBUILDER_H_ +#define EMULATOR_INCLUDE_EMULATOR_GRAPHICS_SHADER_RECOMPILER_SPIRVBUILDER_H_ + +#include "common/common.h" + +#include +#include + +namespace Libs::Graphics::ShaderRecompiler::Spirv { + +class Builder { +public: + Builder(); + ~Builder() = default; + KYTY_CLASS_DEFAULT_COPY(Builder); + + uint32_t AllocateId(); + + void AddCapability(std::initializer_list operands); + void AddExtension(const char* name); + void AddExtInstImport(uint32_t id, const char* name); + void AddMemoryModel(std::initializer_list operands); + void AddEntryPoint(uint32_t execution_model, uint32_t entry_point, const char* name, + const std::vector& interfaces); + void AddExecutionMode(std::initializer_list operands); + void AddName(uint32_t target, const char* name); + void AddAnnotation(std::initializer_list words); + void AddType(std::initializer_list words); + void AddFunction(std::initializer_list words); + void AddFunction(const std::vector& words); + + [[nodiscard]] std::vector Build() const; + +private: + static void AppendInstruction(std::vector* section, uint32_t opcode, + const std::vector& operands); + static void AppendInstruction(std::vector* section, uint32_t opcode, + std::initializer_list operands); + static void AppendString(std::vector* words, const char* text); + + uint32_t m_next_id = 1; + std::vector m_capabilities; + std::vector m_extensions; + std::vector m_ext_inst_imports; + std::vector m_memory_model; + std::vector m_entry_points; + std::vector m_execution_modes; + std::vector m_debug; + std::vector m_annotations; + std::vector m_types; + std::vector m_functions; +}; + +} // namespace Libs::Graphics::ShaderRecompiler::Spirv + +#endif /* EMULATOR_INCLUDE_EMULATOR_GRAPHICS_SHADER_RECOMPILER_SPIRVBUILDER_H_ */ diff --git a/src/graphics/shader/recompiler/SpirvEmitter.cpp b/src/graphics/shader/recompiler/SpirvEmitter.cpp new file mode 100644 index 000000000..aac056f7a --- /dev/null +++ b/src/graphics/shader/recompiler/SpirvEmitter.cpp @@ -0,0 +1,491 @@ +#include "graphics/shader/recompiler/SpirvEmitter.h" + +#include "graphics/shader/recompiler/SrtWalker.h" +#include "graphics/shader/recompiler/spirvEmitter/spirvEmitterInternal.h" + +#include +#include +#include +#include + +namespace Libs::Graphics::ShaderRecompiler::Spirv { + +namespace { + +bool Fail(std::string* error, const std::string& message) { + if (error != nullptr) { + *error = message; + } + return false; +} + +bool ImageBinding(const IR::ImageResource& image, IR::DescriptorBindingKind& kind) { + using Kind = IR::DescriptorBindingKind; + using Dim = Decoder::ImageDimension; + if (image.kind == IR::ResourceKind::Image || image.kind == IR::ResourceKind::ImageUint) { + const bool integer = image.kind == IR::ResourceKind::ImageUint; + switch (image.dimension) { + case Dim::Dim2D: kind = integer ? Kind::SampledUint2D : Kind::Sampled2D; return true; + case Dim::Dim3D: kind = integer ? Kind::SampledUint3D : Kind::Sampled3D; return true; + case Dim::Dim2DArray: + kind = integer ? Kind::SampledUint2DArray : Kind::Sampled2DArray; + return true; + case Dim::Unknown: return false; + } + } + const bool uint_image = image.kind == IR::ResourceKind::StorageImageUint; + if (image.kind != IR::ResourceKind::StorageImage && !uint_image) { + return false; + } + switch (image.dimension) { + case Dim::Dim2D: kind = uint_image ? Kind::StorageUint2D : Kind::Storage2D; return true; + case Dim::Dim3D: kind = uint_image ? Kind::StorageUint3D : Kind::Storage3D; return true; + case Dim::Dim2DArray: + kind = uint_image ? Kind::StorageUint2DArray : Kind::Storage2DArray; + return true; + case Dim::Unknown: return false; + } + return false; +} + +bool NeedsSampler(IR::Opcode op) { + return op == IR::Opcode::ImageSample || op == IR::Opcode::ImageGather4 || + op == IR::Opcode::ImageGetLod; +} + +bool IsBufferLoad(IR::Opcode op) { + switch (op) { + case IR::Opcode::BufferLoadUbyte: + case IR::Opcode::BufferLoadSbyte: + case IR::Opcode::BufferLoadUshort: + case IR::Opcode::BufferLoadSshort: + case IR::Opcode::BufferLoadDword: return true; + default: return false; + } +} + +bool IsBufferStore(IR::Opcode op) { + return op == IR::Opcode::BufferStoreByte || op == IR::Opcode::BufferStoreShort || + op == IR::Opcode::BufferStoreDword; +} + +bool IsFlatLoad(IR::Opcode op) { + return op == IR::Opcode::FlatLoadUbyte || op == IR::Opcode::FlatLoadSbyte || + op == IR::Opcode::FlatLoadUshort || op == IR::Opcode::FlatLoadSshort || + op == IR::Opcode::FlatLoadDword; +} + +bool IsFlatStore(IR::Opcode op) { + return op == IR::Opcode::FlatStoreByte || op == IR::Opcode::FlatStoreShort || + op == IR::Opcode::FlatStoreDword; +} + +bool IsDsKind(IR::ResourceKind kind) { + return kind == IR::ResourceKind::Lds || kind == IR::ResourceKind::Gds; +} + +bool IsDsRead(IR::Opcode op) { + switch (op) { + case IR::Opcode::DsReadUbyte: + case IR::Opcode::DsReadSbyte: + case IR::Opcode::DsReadUshort: + case IR::Opcode::DsReadSshort: + case IR::Opcode::DsReadB32: return true; + default: return false; + } +} + +bool IsDsWrite(IR::Opcode op) { + switch (op) { + case IR::Opcode::DsWriteByte: + case IR::Opcode::DsWriteShort: + case IR::Opcode::DsWriteB32: return true; + default: return false; + } +} + +bool IsAtomic(IR::Opcode op) { + switch (op) { + case IR::Opcode::AtomicSwapU32: + case IR::Opcode::AtomicAddU32: + case IR::Opcode::AtomicSubU32: + case IR::Opcode::AtomicSMinI32: + case IR::Opcode::AtomicUMinU32: + case IR::Opcode::AtomicSMaxI32: + case IR::Opcode::AtomicUMaxU32: + case IR::Opcode::AtomicAndU32: + case IR::Opcode::AtomicOrU32: + case IR::Opcode::AtomicXorU32: return true; + default: return false; + } +} + +uint32_t BufferAddressOperandCount(const IR::Instruction& inst) { + return 1u + (inst.memory.idxen ? 1u : 0u) + (inst.memory.offen ? 1u : 0u); +} + +bool ValidateInstructionContract(const IR::Instruction& inst, std::string* error) { + if (inst.src_count > std::size(inst.src)) { + return Fail(error, "instruction source count exceeds fixed IR storage"); + } + const auto kind = inst.memory.kind; + const auto flat_kind = kind == IR::ResourceKind::Flat || kind == IR::ResourceKind::Global || + kind == IR::ResourceKind::Scratch; + const auto buffer_address_count = BufferAddressOperandCount(inst); + if ((IsBufferLoad(inst.op) && + (kind != IR::ResourceKind::Buffer || inst.src_count != buffer_address_count)) || + (IsBufferStore(inst.op) && + (kind != IR::ResourceKind::Buffer || inst.src_count != buffer_address_count + 1u)) || + (inst.op == IR::Opcode::SLoadDword && + (kind != IR::ResourceKind::ScalarBuffer || inst.src_count != 1)) || + (inst.op == IR::Opcode::SBufferLoadDword && + (kind != IR::ResourceKind::ScalarBuffer || inst.src_count != 1)) || + (IsFlatLoad(inst.op) && (!flat_kind || inst.src_count != 2)) || + (IsFlatStore(inst.op) && (!flat_kind || inst.src_count != 3))) { + return Fail(error, "memory instruction opcode/kind/operand contract is invalid"); + } + const auto ds_kind = IsDsKind(kind); + const auto ds_resource = !ds_kind || inst.memory.resource == 0; + if ((IsDsRead(inst.op) && (!ds_kind || !ds_resource || inst.src_count != 1 || + inst.dst.kind != IR::OperandKind::Register)) || + (IsDsWrite(inst.op) && (!ds_kind || !ds_resource || inst.src_count != 2 || + inst.dst.kind != IR::OperandKind::Null)) || + ((inst.op == IR::Opcode::DsAppend || inst.op == IR::Opcode::DsConsume) && + (!ds_kind || !ds_resource || inst.src_count != 1 || + inst.dst.kind != IR::OperandKind::Register || + (kind == IR::ResourceKind::Gds && inst.memory.offset != 0))) || + ((inst.op == IR::Opcode::DsMinF32 || inst.op == IR::Opcode::DsMaxF32) && + (!ds_kind || !ds_resource || inst.src_count != 3 || + inst.dst.kind != IR::OperandKind::Null)) || + (inst.op == IR::Opcode::DsWriteAddtidB32 && + (kind != IR::ResourceKind::Lds || !ds_resource || inst.src_count != 2 || + inst.dst.kind != IR::OperandKind::Null)) || + (inst.op == IR::Opcode::DsReadAddtidB32 && + (kind != IR::ResourceKind::Lds || !ds_resource || inst.src_count != 1 || + inst.dst.kind != IR::OperandKind::Register)) || + (inst.op == IR::Opcode::DsSwizzleB32 && + (kind != IR::ResourceKind::None || inst.src_count != 2 || + inst.dst.kind != IR::OperandKind::Register))) { + return Fail(error, "DS instruction has an invalid resource/operand contract"); + } + if ((inst.op == IR::Opcode::ImageSample || inst.op == IR::Opcode::ImageGather4 || + inst.op == IR::Opcode::ImageGetLod || inst.op == IR::Opcode::ImageLoad || + inst.op == IR::Opcode::ImageGetResinfo) && + ((kind != IR::ResourceKind::Image && kind != IR::ResourceKind::ImageUint) || + inst.src_count != 1)) { + return Fail(error, "sampled image instruction has an invalid resource class"); + } + if (inst.op == IR::Opcode::ImageStore && + ((kind != IR::ResourceKind::StorageImage && kind != IR::ResourceKind::StorageImageUint) || + inst.src_count != 2)) { + return Fail(error, "storage image instruction has an invalid resource class"); + } + if (IsAtomic(inst.op) && + !((kind == IR::ResourceKind::Buffer && inst.src_count == buffer_address_count + 1u) || + ((kind == IR::ResourceKind::Lds || kind == IR::ResourceKind::Gds) && + inst.memory.resource == 0 && inst.src_count == 2 && + (inst.dst.kind == IR::OperandKind::Null || + inst.dst.kind == IR::OperandKind::Register)) || + (kind == IR::ResourceKind::StorageImageUint && inst.src_count == 2))) { + return Fail(error, "atomic instruction has an invalid resource/operand contract"); + } + return true; +} + +bool ValidateNativeProgram(const IR::Program& program, std::string* error) { + using Kind = IR::DescriptorBindingKind; + constexpr auto KindCount = static_cast(Kind::Count); + std::array, KindCount> expected; + std::array present {}; + const auto Dense = [](size_t size) { + std::vector values(size); + for (uint32_t i = 0; i < values.size(); i++) { + values[i] = i; + } + return values; + }; + auto Expect = [&](Kind kind, std::vector resources = {}) { + const auto index = static_cast(kind); + present[index] = true; + expected[index] = std::move(resources); + }; + if (!program.info.buffers.empty()) { + Expect(Kind::Buffers, Dense(program.info.buffers.size())); + } + for (uint32_t i = 0; i < program.info.images.size(); i++) { + Kind kind; + if (!ImageBinding(program.info.images[i], kind)) { + return Fail(error, "native shader plan has an invalid image class"); + } + present[static_cast(kind)] = true; + expected[static_cast(kind)].push_back(i); + } + if (!program.info.samplers.empty()) { + Expect(Kind::Samplers, Dense(program.info.samplers.size())); + } + const auto uses_gds = + std::any_of(program.blocks.begin(), program.blocks.end(), [](const auto& block) { + return std::any_of( + block.instructions.begin(), block.instructions.end(), + [](const auto& inst) { return inst.memory.kind == IR::ResourceKind::Gds; }); + }); + if (uses_gds) { + Expect(Kind::Gds); + } + if (!program.info.addresses.empty()) { + Expect(Kind::AddressMemory, Dense(program.info.addresses.size())); + } + if (!program.srt.reads.empty()) { + Expect(Kind::FlattenedSrt); + } + if (!program.bindings.user_data_registers.empty() && program.bindings.push_constant_size == 0) { + Expect(Kind::UserData); + } + + std::array seen {}; + for (uint32_t i = 0; i < program.bindings.descriptors.size(); i++) { + const auto& binding = program.bindings.descriptors[i]; + const auto kind = static_cast(binding.kind); + if (kind >= KindCount || seen[kind] || !present[kind] || + binding.resources != expected[kind]) { + return Fail(error, "native descriptor groups do not match shader topology"); + } + for (uint32_t j = 0; j < i; j++) { + if (program.bindings.descriptors[j].binding == binding.binding) { + return Fail(error, "native descriptor binding numbers are not unique"); + } + } + seen[kind] = true; + } + for (size_t i = 0; i < KindCount; i++) { + if (present[i] != seen[i]) { + return Fail(error, "native shader plan is missing a required descriptor group"); + } + } + const auto has_user_storage = present[static_cast(Kind::UserData)]; + if ((!has_user_storage && + program.bindings.push_constant_size != program.bindings.user_data_registers.size() * 4u) || + (has_user_storage && program.bindings.push_constant_size != 0) || + !std::is_sorted(program.bindings.user_data_registers.begin(), + program.bindings.user_data_registers.end()) || + std::adjacent_find(program.bindings.user_data_registers.begin(), + program.bindings.user_data_registers.end()) != + program.bindings.user_data_registers.end()) { + return Fail(error, "native user-data layout is inconsistent"); + } + + for (const auto& block: program.blocks) { + for (const auto& inst: block.instructions) { + if (!ValidateInstructionContract(inst, error)) { + return false; + } + const auto memory = inst.memory.kind; + if ((memory == IR::ResourceKind::Buffer || (memory == IR::ResourceKind::ScalarBuffer && + inst.op == IR::Opcode::SBufferLoadDword)) && + inst.memory.resource >= program.info.buffers.size()) { + return Fail(error, "buffer instruction has an invalid dense resource"); + } + if ((memory == IR::ResourceKind::Image || memory == IR::ResourceKind::ImageUint || + memory == IR::ResourceKind::StorageImage || + memory == IR::ResourceKind::StorageImageUint) && + (inst.memory.resource >= program.info.images.size() || + program.info.images[inst.memory.resource].kind != memory || + program.info.images[inst.memory.resource].dimension != + inst.memory.image_dimension)) { + return Fail(error, "image instruction has an invalid dense resource"); + } + const bool address = + inst.op == IR::Opcode::SLoadDword || memory == IR::ResourceKind::Flat || + memory == IR::ResourceKind::Global || memory == IR::ResourceKind::Scratch; + if (address && (inst.memory.resource >= program.info.addresses.size() || + program.info.addresses[inst.memory.resource].kind != memory)) { + return Fail(error, "address instruction has an invalid dense resource"); + } + if (NeedsSampler(inst.op) && inst.memory.sampler >= program.info.samplers.size()) { + return Fail(error, "sample instruction has an invalid dense sampler"); + } + if (inst.op == IR::Opcode::LoadSrtDword && + (inst.src_count != 1 || inst.src[0].kind != IR::OperandKind::ImmediateU32 || + inst.src[0].imm >= program.srt.reads.size())) { + return Fail(error, "flattened SRT instruction has an invalid dense slot"); + } + } + } + return true; +} + +} // namespace + +static constexpr size_t InitialEmitterVectorReserve = 4096; + +static void ReportReserveExceeded(const IR::Program& program, const char* vector_name, + size_t size) { + if (size <= InitialEmitterVectorReserve) { + return; + } + std::printf("SPIR-V emitter reserve exceeded: hash=0x%016" PRIx64 + " stage=%u vector=%s size=%zu reserve=%zu\n", + program.shader_hash, static_cast(program.stage), vector_name, size, + InitialEmitterVectorReserve); +} + +static bool IsExecDestination(const IR::Operand& operand) { + return operand.kind == IR::OperandKind::Register && operand.reg.file == IR::RegisterFile::Exec; +} + +static bool IsUniformZeroExecWrite(const IR::Program& program, const IR::Instruction& inst) { + if (inst.op != IR::Opcode::BitFieldMaskU64 || !IsExecDestination(inst.dst) || + inst.src_count < 1u) { + return false; + } + uint32_t count = 0; + return IR::FoldScalarConstant(program.provenance, inst.scalar_sources[0], &count) && + (count & 63u) == 0u; +} + +static bool ProducesLaneMask(const IR::Instruction& inst) { + switch (inst.op) { + case IR::Opcode::IAddCarryU32: + case IR::Opcode::ISubBorrowU32: + case IR::Opcode::UMadU64U32: return inst.dst2.kind != IR::OperandKind::Null; + default: return false; + } +} + +bool ProgramRequiresExactSubgroupSize(const IR::Program& program) { + for (const auto& block: program.blocks) { + if (block.terminator.kind == CFG::TerminatorKind::ConditionalBranch) { + switch (block.terminator.condition) { + case CFG::BranchCondition::VccZero: + case CFG::BranchCondition::VccNonZero: + case CFG::BranchCondition::ExecZero: + case CFG::BranchCondition::ExecNonZero: return true; + default: break; + } + } + for (const auto& inst: block.instructions) { + if (Emitter::InstructionHasDppSource(inst)) { + return true; + } + if (Emitter::IsCompareOpcode(inst.op) && !Emitter::IsSccOperand(inst.dst)) { + return true; + } + if ((IsExecDestination(inst.dst) || IsExecDestination(inst.dst2)) && + !IsUniformZeroExecWrite(program, inst)) { + return true; + } + if (ProducesLaneMask(inst)) { + return true; + } + if (inst.op == IR::Opcode::SelectMaskU32 || inst.op == IR::Opcode::SelectMaskF32Bits) { + return true; + } + if (inst.op == IR::Opcode::SaveexecB32 || inst.op == IR::Opcode::SaveexecB64) { + return true; + } + switch (inst.op) { + case IR::Opcode::WqmB64: + if (inst.dst.kind == IR::OperandKind::Register && + inst.dst.reg.file != IR::RegisterFile::Scalar) { + return true; + } + break; + case IR::Opcode::MaskedBitCountLowU32: + case IR::Opcode::MaskedBitCountHighU32: + case IR::Opcode::ReadFirstLaneU32: + case IR::Opcode::ReadLaneU32: + case IR::Opcode::WriteLaneU32: + case IR::Opcode::Permlane16B32: + case IR::Opcode::Permlanex16B32: + case IR::Opcode::DsSwizzleB32: + case IR::Opcode::DsConsume: + case IR::Opcode::DsAppend: return true; + default: break; + } + } + } + return false; +} + +bool EmitProgram(const IR::Program& program, const IR::ResourceSnapshot& resources, + const ShaderVertexInputInfo* vertex_input_info, + const ShaderPixelInputInfo* pixel_input_info, + const ShaderComputeInputInfo* compute_input_info, std::vector* spirv, + std::string* error) { + using namespace Emitter; + + if (spirv == nullptr) { + SetError(error, "invalid SPIR-V output"); + return false; + } + if (program.stage != ShaderType::Compute && program.stage != ShaderType::Vertex && + program.stage != ShaderType::Pixel) { + SetError(error, "binary SPIR-V emitter supports compute, vertex, and pixel shaders"); + return false; + } + if (!program.srt_patching_complete || !program.resource_tracking_complete || + !program.shader_info_complete || !program.binding_layout_complete) { + SetError(error, "SPIR-V emitter requires a fully planned native shader program"); + return false; + } + if (!IR::ValidateResourceSnapshot(program, resources, error)) { + return false; + } + if (!IR::ValidateResourceSpecialization(program, resources, error)) { + return false; + } + if (!ValidateNativeProgram(program, error)) { + return false; + } + + EmitterState state; + state.program = &program; + state.resources = &resources; + state.vertex_input_info = vertex_input_info; + state.pixel_input_info = pixel_input_info; + state.compute_input_info = compute_input_info; + state.stage = program.stage; + state.wave_size = program.wave_size; + state.per_invocation_masks = program.lane_mask_mode == ShaderLaneMaskMode::PerInvocation; + state.exact_subgroup_operations = + !state.per_invocation_masks && ProgramRequiresExactSubgroupSize(program); + state.registers.reserve(InitialEmitterVectorReserve); + state.inputs.reserve(InitialEmitterVectorReserve); + state.outputs.reserve(InitialEmitterVectorReserve); + state.interface_variables.reserve(InitialEmitterVectorReserve); + state.reachable_blocks.reserve(InitialEmitterVectorReserve); + CollectRegisters(program, &state.registers); + CopyProgramInputsAndOutputs(&state, program); + state.needs_subgroup_ballot = ProgramNeedsSubgroupBallot(program); + state.needs_subgroup_shuffle = ProgramNeedsSubgroupShuffle(program); + state.needs_subgroup_local_invocation_id = ProgramNeedsSubgroupLocalInvocationId(program); + state.needs_compute_derivatives = ProgramNeedsComputeDerivatives(program); + state.needs_image_gather_extended = ProgramNeedsImageGatherExtended(program); + state.needs_function_lds = ProgramNeedsFunctionLds(program); + state.needs_pixel_valid_mask = ProgramNeedsPixelValidMask(program); + ComputeReachableBlocks(&state, program); + AllocateInputVariables(&state); + AllocateOutputVariables(&state); + AllocateDescriptorVariables(&state); + EmitHeaderAndTypes(&state); + AllocateRegisterVariables(&state); + AllocateBlockLabels(&state, program); + AllocateDispatcherState(&state, program); + EmitFunction(&state, program); + + ReportReserveExceeded(program, "registers", state.registers.size()); + ReportReserveExceeded(program, "inputs", state.inputs.size()); + ReportReserveExceeded(program, "outputs", state.outputs.size()); + ReportReserveExceeded(program, "interface_variables", state.interface_variables.size()); + ReportReserveExceeded(program, "reachable_blocks", state.reachable_blocks.size()); + + auto binary = state.builder.Build(); + if (binary.empty()) { + SetError(error, "SPIR-V builder returned an empty module"); + return false; + } + *spirv = std::move(binary); + return true; +} + +} // namespace Libs::Graphics::ShaderRecompiler::Spirv diff --git a/src/graphics/shader/recompiler/SpirvEmitter.h b/src/graphics/shader/recompiler/SpirvEmitter.h new file mode 100644 index 000000000..ddc798ae4 --- /dev/null +++ b/src/graphics/shader/recompiler/SpirvEmitter.h @@ -0,0 +1,22 @@ +#ifndef EMULATOR_INCLUDE_EMULATOR_GRAPHICS_SHADER_RECOMPILER_SPIRVEMITTER_H_ +#define EMULATOR_INCLUDE_EMULATOR_GRAPHICS_SHADER_RECOMPILER_SPIRVEMITTER_H_ + +#include "common/common.h" +#include "common/stringUtils.h" +#include "graphics/shader/recompiler/ResourceMaterialization.h" + +#include + +namespace Libs::Graphics::ShaderRecompiler::Spirv { + +bool ProgramRequiresExactSubgroupSize(const IR::Program& program); + +bool EmitProgram(const IR::Program& program, const IR::ResourceSnapshot& resources, + const ShaderVertexInputInfo* vertex_input_info, + const ShaderPixelInputInfo* pixel_input_info, + const ShaderComputeInputInfo* compute_input_info, std::vector* spirv, + std::string* error); + +} // namespace Libs::Graphics::ShaderRecompiler::Spirv + +#endif /* EMULATOR_INCLUDE_EMULATOR_GRAPHICS_SHADER_RECOMPILER_SPIRVEMITTER_H_ */ diff --git a/src/graphics/shader/recompiler/SrtPatcher.cpp b/src/graphics/shader/recompiler/SrtPatcher.cpp new file mode 100644 index 000000000..734c8e51d --- /dev/null +++ b/src/graphics/shader/recompiler/SrtPatcher.cpp @@ -0,0 +1,133 @@ +#include "graphics/shader/recompiler/SrtPatcher.h" + +#include +#include +#include +#include + +namespace Libs::Graphics::ShaderRecompiler::IR { + +bool PatchSrtReads(Program* program, std::string* error) { + if (program == nullptr || !program->srt_plan_complete || program->srt_patching_complete || + program->resource_tracking_complete) { + if (error != nullptr) { + *error = program == nullptr ? "invalid SRT patch program" + : !program->srt_plan_complete ? "SRT plan is not ready" + : program->srt_patching_complete ? "SRT reads already patched" + : "resources already tracked"; + } + return false; + } + + std::map offsets; + std::map values; + for (const auto& read: program->srt.reads) { + if (read.value >= program->provenance.values.size() || + read.flat_offset >= program->srt.reads.size() || + (program->provenance.values[read.value].op != ScalarValueOp::ReadConst && + program->provenance.values[read.value].op != ScalarValueOp::ReadConstBuffer)) { + if (error != nullptr) { + *error = fmt::format("invalid SRT read value={} flat_offset={}", read.value, + read.flat_offset); + } + return false; + } + const auto [it, inserted] = offsets.emplace(read.value, read.flat_offset); + if (!inserted && it->second != read.flat_offset) { + if (error != nullptr) { + *error = fmt::format("SRT read value {} has conflicting flat offsets", read.value); + } + return false; + } + const auto [value, unique] = values.emplace(read.flat_offset, read.value); + if (!unique && value->second != read.value) { + if (error != nullptr) { + *error = fmt::format("SRT flat offset {} has conflicting values", read.flat_offset); + } + return false; + } + } + if (offsets.size() != program->srt.reads.size() || values.size() != program->srt.reads.size()) { + if (error != nullptr) { + *error = "SRT reads do not form a dense value-to-offset bijection"; + } + return false; + } + + std::set dynamic_reads; + for (const auto value: program->srt.dynamic_reads) { + if (value >= program->provenance.values.size() || offsets.contains(value) || + (program->provenance.values[value].op != ScalarValueOp::ReadConst && + program->provenance.values[value].op != ScalarValueOp::ReadConstBuffer) || + !dynamic_reads.insert(value).second) { + if (error != nullptr) { + *error = fmt::format("invalid dynamic SRT read value={}", value); + } + return false; + } + } + std::set dynamic_sources; + for (const auto source: program->srt.dynamic_sources) { + if (source == ScalarProvenance::Undefined || + (source > ScalarProvenance::Unknown && + source - 2u >= program->provenance.descriptors.size()) || + !dynamic_sources.insert(source).second) { + if (error != nullptr) { + *error = fmt::format("invalid dynamic descriptor source={}", source); + } + return false; + } + } + + struct Patch { + Instruction* inst = nullptr; + uint32_t flat_offset = 0; + }; + std::vector patches; + std::map matched; + for (const auto& [value, offset]: offsets) { + matched.emplace(value, false); + } + for (auto& block: program->blocks) { + for (auto& inst: block.instructions) { + if (inst.op != Opcode::SLoadDword && inst.op != Opcode::SBufferLoadDword) { + continue; + } + if (const auto found = offsets.find(inst.scalar_value); found != offsets.end()) { + const auto value_op = program->provenance.values[found->first].op; + if ((inst.op == Opcode::SLoadDword) != (value_op == ScalarValueOp::ReadConst)) { + if (error != nullptr) { + *error = fmt::format("SRT read value {} has the wrong scalar-load producer", + found->first); + } + return false; + } + patches.push_back({&inst, found->second}); + matched[found->first] = true; + } + } + } + for (const auto& [value, found]: matched) { + if (!found) { + if (error != nullptr) { + *error = fmt::format("SRT read value {} has no scalar-load producer", value); + } + return false; + } + } + + for (const auto& patch: patches) { + patch.inst->op = Opcode::LoadSrtDword; + patch.inst->src_count = 1; + for (auto& src: patch.inst->src) { + src = {}; + } + patch.inst->src[0].kind = OperandKind::ImmediateU32; + patch.inst->src[0].imm = patch.flat_offset; + patch.inst->memory = {}; + } + program->srt_patching_complete = true; + return true; +} + +} // namespace Libs::Graphics::ShaderRecompiler::IR diff --git a/src/graphics/shader/recompiler/SrtPatcher.h b/src/graphics/shader/recompiler/SrtPatcher.h new file mode 100644 index 000000000..b6c16c5b6 --- /dev/null +++ b/src/graphics/shader/recompiler/SrtPatcher.h @@ -0,0 +1,14 @@ +#ifndef EMULATOR_INCLUDE_EMULATOR_GRAPHICS_SHADER_RECOMPILER_SRTPATCHER_H_ +#define EMULATOR_INCLUDE_EMULATOR_GRAPHICS_SHADER_RECOMPILER_SRTPATCHER_H_ + +#include "graphics/shader/recompiler/ShaderIR.h" + +namespace Libs::Graphics::ShaderRecompiler::IR { + +// Replaces each immediate ReadConst producer selected by the SRT plan with a dense flat-buffer +// dword load. Dynamic-offset reads remain explicit. On failure no instruction is changed. +bool PatchSrtReads(Program* program, std::string* error); + +} // namespace Libs::Graphics::ShaderRecompiler::IR + +#endif /* EMULATOR_INCLUDE_EMULATOR_GRAPHICS_SHADER_RECOMPILER_SRTPATCHER_H_ */ diff --git a/src/graphics/shader/recompiler/SrtWalker.cpp b/src/graphics/shader/recompiler/SrtWalker.cpp new file mode 100644 index 000000000..1f41bbeb9 --- /dev/null +++ b/src/graphics/shader/recompiler/SrtWalker.cpp @@ -0,0 +1,655 @@ +#include "graphics/shader/recompiler/SrtWalker.h" + +#include "graphics/host_gpu/hostMemory.h" +#include "graphics/shader/recompiler/ScalarProvenance.h" + +#include +#include + +namespace Libs::Graphics::ShaderRecompiler::IR { +namespace { + +constexpr uint64_t AddressMask = 0x0000ffffffffffffull; + +const char* StageName(ShaderType stage) { + switch (stage) { + case ShaderType::Vertex: return "vertex"; + case ShaderType::Pixel: return "pixel"; + case ShaderType::Fetch: return "fetch"; + case ShaderType::Compute: return "compute"; + default: return "unknown"; + } +} + +std::string Diagnostic(const Program& program, uint32_t pc, const std::string& message) { + return fmt::format("shader SRT: hash=0x{:016x} stage={} pc=0x{:08x} {}", program.shader_hash, + StageName(program.stage), pc, message); +} + +bool AddSignedAddress(uint64_t base, int64_t offset, uint64_t* result) { + if (result == nullptr || base > AddressMask) { + return false; + } + if (offset < 0) { + const auto magnitude = uint64_t {0} - static_cast(offset); + if (magnitude > base) { + return false; + } + *result = base - magnitude; + return true; + } + const auto magnitude = static_cast(offset); + if (magnitude > AddressMask - base) { + return false; + } + *result = base + magnitude; + return true; +} + +bool ApplyOperation(ScalarValueOp op, const uint32_t args[3], uint32_t* result) { + if (result == nullptr) { + return false; + } + const auto shift = args[1] & 31u; + switch (op) { + case ScalarValueOp::Add: *result = args[0] + args[1]; break; + case ScalarValueOp::AddCarry: *result = args[0] + args[1] + (args[2] & 1u); break; + case ScalarValueOp::Carry: + *result = static_cast( + (static_cast(args[0]) + args[1] + (args[2] & 1u)) >> 32u); + break; + case ScalarValueOp::Sub: *result = args[0] - args[1]; break; + case ScalarValueOp::SubBorrow: *result = args[0] - args[1] - (args[2] & 1u); break; + case ScalarValueOp::Borrow: + *result = static_cast(args[1]) + (args[2] & 1u) > args[0] ? 1u : 0u; + break; + case ScalarValueOp::Mul: *result = args[0] * args[1]; break; + case ScalarValueOp::And: *result = args[0] & args[1]; break; + case ScalarValueOp::AndNot: *result = args[0] & ~args[1]; break; + case ScalarValueOp::Or: *result = args[0] | args[1]; break; + case ScalarValueOp::OrNot: *result = args[0] | ~args[1]; break; + case ScalarValueOp::Xor: *result = args[0] ^ args[1]; break; + case ScalarValueOp::Not: *result = ~args[0]; break; + case ScalarValueOp::ShiftLeft: *result = args[0] << shift; break; + case ScalarValueOp::ShiftRight: *result = args[0] >> shift; break; + case ScalarValueOp::ShiftRightArithmetic: + *result = static_cast(static_cast(args[0]) >> shift); + break; + case ScalarValueOp::BitFieldMaskU32: { + const auto count = args[0] & 31u; + const auto offset = args[1] & 31u; + *result = ((uint32_t {1} << count) - 1u) << offset; + break; + } + case ScalarValueOp::BitFieldMaskU64Low: + case ScalarValueOp::BitFieldMaskU64High: { + const auto count = args[0] & 63u; + const auto value = ((uint64_t {1} << count) - 1u) << (args[1] & 63u); + *result = op == ScalarValueOp::BitFieldMaskU64Low ? static_cast(value) + : static_cast(value >> 32u); + break; + } + case ScalarValueOp::Add3: *result = args[0] + args[1] + args[2]; break; + case ScalarValueOp::ShiftLeftAdd: *result = (args[0] << shift) + args[2]; break; + case ScalarValueOp::ShiftLeftAddCarry: + *result = + static_cast(((static_cast(args[0]) << shift) + args[2]) >> 32u); + break; + case ScalarValueOp::AddShiftLeft: *result = (args[0] + args[1]) << (args[2] & 31u); break; + case ScalarValueOp::XorAdd: *result = (args[0] ^ args[1]) + args[2]; break; + case ScalarValueOp::ShiftLeftOr: *result = (args[0] << shift) | args[2]; break; + default: return false; + } + return true; +} + +class ConstantFolder { +public: + explicit ConstantFolder(const ScalarProvenance& provenance) + : m_provenance(provenance), m_values(provenance.values.size()), + m_state(provenance.values.size()) {} + + bool Fold(uint32_t id, uint32_t& result) { + if (id >= m_provenance.values.size() || m_state[id] == 1 || m_state[id] == 3) { + return false; + } + if (m_state[id] == 2) { + result = m_values[id]; + return true; + } + m_state[id] = 1; + const auto& value = m_provenance.values[id]; + uint32_t out = 0; + switch (value.op) { + case ScalarValueOp::Constant: out = value.imm; break; + case ScalarValueOp::Phi: + if (value.phi_args.empty() || !Fold(value.phi_args[0], out)) { + return Dynamic(id); + } + for (size_t i = 1; i < value.phi_args.size(); i++) { + uint32_t other = 0; + if (!Fold(value.phi_args[i], other) || other != out) { + return Dynamic(id); + } + } + break; + default: { + const auto count = ScalarValueArgCount(value.op); + if (count == 0 || count > 3 || value.op == ScalarValueOp::ReadConst) { + return Dynamic(id); + } + uint32_t args[3] = {}; + for (uint32_t i = 0; i < count; i++) { + if (!Fold(value.args[i], args[i])) { + return Dynamic(id); + } + } + if (!ApplyOperation(value.op, args, &out)) { + return Dynamic(id); + } + break; + } + } + m_state[id] = 2; + m_values[id] = out; + result = out; + return true; + } + +private: + bool Dynamic(uint32_t id) { + m_state[id] = 3; + return false; + } + + const ScalarProvenance& m_provenance; + std::vector m_values; + std::vector m_state; +}; + +class PlanBuilder { +public: + explicit PlanBuilder(Program* program): m_program(program), m_folder(program->provenance) { + m_state.resize(program->provenance.values.size()); + } + + bool Run(std::string* error) { + m_program->srt = {}; + for (const auto& block: m_program->blocks) { + for (const auto& inst: block.instructions) { + if (!CollectDescriptor(inst.memory.resource_source, inst.pc, error) || + !CollectDescriptor(inst.memory.sampler_source, inst.pc, error)) { + return false; + } + } + } + for (const auto& block: m_program->blocks) { + for (const auto& inst: block.instructions) { + if (inst.op == Opcode::SLoadDword && inst.scalar_value < m_state.size() && + m_state[inst.scalar_value] == 0) { + const auto& value = m_program->provenance.values[inst.scalar_value]; + uint32_t ignored = 0; + if (value.op == ScalarValueOp::ReadConst && + m_folder.Fold(value.args[2], ignored) && + !CollectValue(inst.scalar_value, inst.pc, error)) { + return false; + } + } + } + } + return true; + } + +private: + bool Fail(uint32_t pc, std::string* error, const std::string& message) const { + if (error != nullptr) { + *error = Diagnostic(*m_program, pc, message); + } + return false; + } + + bool CollectDescriptor(uint32_t source, uint32_t use_pc, std::string* error) { + const auto* descriptor = GetDescriptorSource(*m_program, source); + if (descriptor == nullptr) { + if (source <= ScalarProvenance::Unknown) { + return source == ScalarProvenance::Undefined || AddDynamicSource(source); + } + return Fail(use_pc, error, fmt::format("invalid descriptor source {}", source)); + } + if (!DescriptorSourceResolved(*m_program, source) || + DescriptorNeedsControlFlow(*descriptor)) { + MarkDescriptor(*descriptor); + return AddDynamicSource(source); + } + for (uint32_t i = 0; i < descriptor->dword_count; i++) { + if (!CollectValue(descriptor->dwords[i], use_pc, error)) { + return false; + } + } + return true; + } + + void MarkDescriptor(const DescriptorValue& descriptor) { + for (uint32_t i = 0; i < descriptor.dword_count; i++) { + MarkValue(descriptor.dwords[i]); + } + } + + void MarkValue(uint32_t id) { + if (id >= m_state.size() || m_state[id] != 0) { + return; + } + m_state[id] = 2; + const auto& value = m_program->provenance.values[id]; + if (value.op == ScalarValueOp::Phi) { + for (const auto arg: value.phi_args) { + MarkValue(arg); + } + return; + } + for (uint32_t i = 0; i < ScalarValueArgCount(value.op); i++) { + MarkValue(value.args[i]); + } + } + + bool DescriptorNeedsControlFlow(const DescriptorValue& descriptor) { + std::vector visited(m_program->provenance.values.size()); + for (uint32_t i = 0; i < descriptor.dword_count; i++) { + if (ValueNeedsControlFlow(descriptor.dwords[i], &visited)) { + return true; + } + } + return false; + } + + bool ValueNeedsControlFlow(uint32_t id, std::vector* visited) { + if (id >= m_program->provenance.values.size() || (*visited)[id] != 0) { + return false; + } + (*visited)[id] = 1; + const auto& value = m_program->provenance.values[id]; + if (value.op == ScalarValueOp::Phi) { + uint32_t ignored = 0; + return !m_folder.Fold(id, ignored); + } + for (uint32_t i = 0; i < ScalarValueArgCount(value.op); i++) { + if (ValueNeedsControlFlow(value.args[i], visited)) { + return true; + } + } + return false; + } + + bool AddDynamicSource(uint32_t source) { + if (std::find(m_program->srt.dynamic_sources.begin(), m_program->srt.dynamic_sources.end(), + source) == m_program->srt.dynamic_sources.end()) { + m_program->srt.dynamic_sources.push_back(source); + } + return true; + } + + bool CollectValue(uint32_t id, uint32_t use_pc, std::string* error) { + if (id >= m_program->provenance.values.size()) { + return Fail(use_pc, error, fmt::format("invalid scalar value {}", id)); + } + if (m_state[id] == 2) { + return true; + } + if (m_state[id] == 1) { + return Fail(use_pc, error, fmt::format("cyclic scalar value {}", id)); + } + m_state[id] = 1; + const auto& value = m_program->provenance.values[id]; + if (value.op == ScalarValueOp::Phi) { + for (const auto arg: value.phi_args) { + if (!CollectValue(arg, use_pc, error)) { + return false; + } + } + } else { + for (uint32_t i = 0; i < ScalarValueArgCount(value.op); i++) { + if (!CollectValue(value.args[i], use_pc, error)) { + return false; + } + } + } + if (value.op == ScalarValueOp::ReadConst || value.op == ScalarValueOp::ReadConstBuffer) { + const auto offset_arg = value.op == ScalarValueOp::ReadConst ? 2u : 4u; + uint32_t ignored = 0; + if (m_folder.Fold(value.args[offset_arg], ignored)) { + m_program->srt.reads.push_back( + {id, static_cast(m_program->srt.reads.size()), use_pc}); + } else { + m_program->srt.dynamic_reads.push_back(id); + } + } + m_state[id] = 2; + return true; + } + + Program* m_program; + ConstantFolder m_folder; + std::vector m_state; +}; + +class Evaluator { +public: + Evaluator(const Program& program, const SrtRuntime& runtime, uint32_t use_pc) + : m_program(program), m_runtime(runtime), m_use_pc(use_pc), + m_values(program.provenance.values.size()), m_state(program.provenance.values.size()) {} + + void SetUsePc(uint32_t use_pc) { m_use_pc = use_pc; } + + bool Evaluate(uint32_t id, uint32_t& result, std::string* error) { + if (id >= m_program.provenance.values.size()) { + return Fail(error, fmt::format("invalid scalar value {}", id)); + } + if (m_state[id] == 2) { + result = m_values[id]; + return true; + } + if (m_state[id] == 1) { + return Fail(error, fmt::format("cyclic scalar value {}", id)); + } + m_state[id] = 1; + const auto& value = m_program.provenance.values[id]; + uint32_t out = 0; + switch (value.op) { + case ScalarValueOp::UserData: + if (value.imm < m_program.user_data_base || + value.imm - m_program.user_data_base >= m_runtime.user_data.size()) { + return Fail(error, fmt::format("user SGPR {} is unavailable", value.imm)); + } + out = m_runtime.user_data[value.imm - m_program.user_data_base]; + break; + case ScalarValueOp::Constant: out = value.imm; break; + case ScalarValueOp::PcRelativeLow: + out = static_cast(m_runtime.shader_base + value.imm); + break; + case ScalarValueOp::PcRelativeHigh: + out = static_cast((m_runtime.shader_base + value.imm) >> 32u); + break; + case ScalarValueOp::Phi: { + const auto first = std::find_if(value.phi_args.begin(), value.phi_args.end(), + [id](uint32_t arg) { return arg != id; }); + if (first == value.phi_args.end()) { + return Fail(error, fmt::format("empty scalar phi {}", id)); + } + if (!Evaluate(*first, out, error)) { + return false; + } + for (const auto arg: value.phi_args) { + if (arg == id || arg == *first) { + continue; + } + uint32_t other = 0; + if (!Evaluate(arg, other, error)) { + return false; + } + if (other != out) { + return Fail(error, + fmt::format("scalar phi {} has runtime-dependent values", id)); + } + } + break; + } + case ScalarValueOp::ReadConst: + case ScalarValueOp::ReadConstBuffer: + if (!Read(value, &out, error)) { + return false; + } + break; + case ScalarValueOp::Undefined: + case ScalarValueOp::Unknown: + return Fail(error, fmt::format("scalar value {} is unresolved", id)); + default: + if (!EvaluateOperation(value, &out, error)) { + return false; + } + break; + } + m_state[id] = 2; + m_values[id] = out; + result = out; + return true; + } + +private: + bool Fail(std::string* error, const std::string& message) const { + if (error != nullptr) { + *error = Diagnostic(m_program, m_use_pc, message); + } + return false; + } + + bool EvaluateOperation(const ScalarValue& value, uint32_t* result, std::string* error) { + const auto count = ScalarValueArgCount(value.op); + if (count == 0 || count > 3) { + return Fail(error, fmt::format("unsupported scalar operation {}", + static_cast(value.op))); + } + uint32_t args[3] = {}; + for (uint32_t i = 0; i < count; i++) { + if (!Evaluate(value.args[i], args[i], error)) { + return false; + } + } + return ApplyOperation(value.op, args, result) || + Fail(error, fmt::format("unsupported scalar operation {}", + static_cast(value.op))); + } + + bool Read(const ScalarValue& value, uint32_t* result, std::string* error) { + const bool buffer = value.op == ScalarValueOp::ReadConstBuffer; + uint32_t lo = 0; + uint32_t hi = 0; + uint32_t offset = 0; + if (!Evaluate(value.args[0], lo, error) || !Evaluate(value.args[1], hi, error) || + !Evaluate(value.args[buffer ? 4u : 2u], offset, error)) { + return false; + } + const auto base = (static_cast(hi) << 32u | lo) & AddressMask; + const auto immediate = static_cast(static_cast(value.imm)); + uint64_t address = 0; + if (buffer) { + uint32_t num_records = 0; + uint32_t ignored = 0; + if (!Evaluate(value.args[2], num_records, error) || + !Evaluate(value.args[3], ignored, error)) { + return false; + } + if (immediate < 0) { + return Fail(error, + fmt::format("ReadConstBuffer pc=0x{:08x} has negative immediate {}", + value.pc, immediate)); + } + const auto byte_offset = static_cast(immediate) + offset; + const auto aligned_offset = byte_offset & ~uint64_t {3}; + const auto stride = (hi >> 16u) & 0x3fffu; + const auto size = stride == 0 ? static_cast(num_records) + : static_cast(stride) * num_records; + if (aligned_offset > size || size - aligned_offset < sizeof(uint32_t)) { + return Fail(error, + fmt::format("ReadConstBuffer pc=0x{:08x} offset={} exceeds size={}", + value.pc, aligned_offset, size)); + } + address = ((base & ~uint64_t {3}) + byte_offset) & ~uint64_t {3}; + } else { + const auto base_aligned = base & ~uint64_t {3}; + const auto immediate_aligned = immediate & ~int64_t {3}; + const auto relative = immediate_aligned + static_cast(offset & ~uint32_t {3}); + if (!AddSignedAddress(base_aligned, relative, &address)) { + return Fail(error, + fmt::format("ReadConst pc=0x{:08x} address base=0x{:016x} offset={} " + "is outside the 48-bit address space", + value.pc, base_aligned, relative)); + } + } + const auto read_memory = + m_runtime.read_memory != nullptr ? m_runtime.read_memory : HostMemoryReadDword; + if (!read_memory(m_runtime.userdata, address, result)) { + return Fail( + error, fmt::format("ReadConst pc=0x{:08x} failed at 0x{:016x}", value.pc, address)); + } + return true; + } + + const Program& m_program; + const SrtRuntime& m_runtime; + uint32_t m_use_pc; + std::vector m_values; + std::vector m_state; +}; + +} // namespace + +bool FoldScalarConstant(const ScalarProvenance& provenance, uint32_t value, uint32_t* result) { + if (result == nullptr) { + return false; + } + ConstantFolder folder(provenance); + return folder.Fold(value, *result); +} + +bool BuildSrtPlan(Program* program, std::string* error) { + if (program == nullptr) { + if (error != nullptr) { + *error = "invalid SRT program"; + } + return false; + } + if (program->resource_tracking_complete) { + if (error != nullptr) { + *error = "cannot rebuild SRT plan after resource tracking"; + } + return false; + } + if (program->srt_patching_complete) { + if (error != nullptr) { + *error = "cannot rebuild SRT plan after SRT patching"; + } + return false; + } + program->srt_plan_complete = false; + if (!PlanBuilder(program).Run(error)) { + return false; + } + program->srt_plan_complete = true; + return true; +} + +bool EvaluateDescriptorSource(const Program& program, uint32_t source, uint32_t use_pc, + const SrtRuntime& runtime, DescriptorValue* result, + std::string* error) { + if (result == nullptr) { + if (error != nullptr) { + *error = Diagnostic(program, use_pc, "invalid descriptor result"); + } + return false; + } + const DescriptorSourceRequest request {source, use_pc}; + std::vector results; + if (!EvaluateDescriptorSources(program, std::span {&request, 1}, runtime, &results, error)) { + return false; + } + *result = results[0]; + return true; +} + +static bool EvaluateRuntimeSourcesImpl(const Program& program, + std::span requests, + const SrtRuntime& runtime, + std::vector* results, + std::vector* flat, bool evaluate_flat, + std::string* error) { + if (results == nullptr || flat == nullptr || !program.srt_plan_complete) { + if (error != nullptr) { + *error = Diagnostic(program, 0, + results == nullptr || flat == nullptr + ? "invalid runtime evaluation output" + : "SRT plan is not ready"); + } + return false; + } + std::vector evaluated; + evaluated.reserve(requests.size()); + std::vector flattened(evaluate_flat ? program.srt.reads.size() : 0u); + Evaluator evaluator(program, runtime, 0); + for (const auto& request: requests) { + const auto* descriptor = GetDescriptorSource(program, request.source); + const auto dynamic = + std::find(program.srt.dynamic_sources.begin(), program.srt.dynamic_sources.end(), + request.source) != program.srt.dynamic_sources.end(); + if (descriptor == nullptr || dynamic || + !DescriptorSourceResolved(program, request.source)) { + if (error != nullptr) { + *error = Diagnostic(program, request.use_pc, + fmt::format("descriptor source {} is {}", request.source, + dynamic ? "GPU-dynamic" : "unresolved")); + } + return false; + } + evaluator.SetUsePc(request.use_pc); + auto value = *descriptor; + for (uint32_t i = 0; i < descriptor->dword_count; i++) { + if (!evaluator.Evaluate(descriptor->dwords[i], value.dwords[i], error)) { + return false; + } + } + evaluated.push_back(value); + } + if (evaluate_flat) { + for (const auto& read: program.srt.reads) { + evaluator.SetUsePc(read.use_pc); + if (read.flat_offset >= flattened.size()) { + if (error != nullptr) { + *error = Diagnostic(program, read.use_pc, "invalid flat SRT offset"); + } + return false; + } + if (!evaluator.Evaluate(read.value, flattened[read.flat_offset], error)) { + return false; + } + } + } + *results = std::move(evaluated); + if (evaluate_flat) { + *flat = std::move(flattened); + } + return true; +} + +bool EvaluateDescriptorSources(const Program& program, + std::span requests, + const SrtRuntime& runtime, std::vector* results, + std::string* error) { + if (results == nullptr) { + if (error != nullptr) { + *error = Diagnostic(program, 0, "invalid descriptor results"); + } + return false; + } + std::vector ignored; + return EvaluateRuntimeSourcesImpl(program, requests, runtime, results, &ignored, false, error); +} + +bool EvaluateRuntimeSources(const Program& program, + std::span requests, + const SrtRuntime& runtime, std::vector* results, + std::vector* flat, std::string* error) { + return EvaluateRuntimeSourcesImpl(program, requests, runtime, results, flat, true, error); +} + +bool WalkSrt(const Program& program, const SrtRuntime& runtime, std::vector* flat, + std::string* error) { + if (flat == nullptr) { + if (error != nullptr) { + *error = Diagnostic(program, 0, "invalid SRT walker output"); + } + return false; + } + std::vector ignored; + return EvaluateRuntimeSources(program, {}, runtime, &ignored, flat, error); +} + +} // namespace Libs::Graphics::ShaderRecompiler::IR diff --git a/src/graphics/shader/recompiler/SrtWalker.h b/src/graphics/shader/recompiler/SrtWalker.h new file mode 100644 index 000000000..1bf0fe2c8 --- /dev/null +++ b/src/graphics/shader/recompiler/SrtWalker.h @@ -0,0 +1,55 @@ +#ifndef EMULATOR_INCLUDE_EMULATOR_GRAPHICS_SHADER_RECOMPILER_SRTWALKER_H_ +#define EMULATOR_INCLUDE_EMULATOR_GRAPHICS_SHADER_RECOMPILER_SRTWALKER_H_ + +#include "graphics/shader/recompiler/ShaderIR.h" + +#include +#include + +namespace Libs::Graphics::ShaderRecompiler::IR { + +using SrtMemoryReader = bool (*)(void* userdata, uint64_t address, uint32_t* value); + +struct SrtRuntime { + std::span user_data; + uint64_t shader_base = 0; + SrtMemoryReader read_memory = nullptr; + void* userdata = nullptr; + std::optional flat_memory_base; +}; + +struct DescriptorSourceRequest { + uint32_t source = 0; + uint32_t use_pc = 0; +}; + +bool FoldScalarConstant(const ScalarProvenance& provenance, uint32_t value, uint32_t* result); + +// Collects reachable ReadConst values. Immediate offsets receive compact flat-buffer slots; +// dynamic offsets remain explicit and are never assigned a fake slot. +bool BuildSrtPlan(Program* program, std::string* error); + +bool EvaluateDescriptorSource(const Program& program, uint32_t source, uint32_t use_pc, + const SrtRuntime& runtime, DescriptorValue* result, + std::string* error); + +// Evaluates one runtime snapshot transactionally. Scalar values and ReadConst results shared by +// several descriptors are memoized once across the batch. +bool EvaluateDescriptorSources(const Program& program, + std::span requests, + const SrtRuntime& runtime, std::vector* results, + std::string* error); + +// Evaluates descriptor sources and the flattened immediate SRT with one memoized scalar walk. +// On failure neither destination is changed. +bool EvaluateRuntimeSources(const Program& program, + std::span requests, + const SrtRuntime& runtime, std::vector* results, + std::vector* flat, std::string* error); + +bool WalkSrt(const Program& program, const SrtRuntime& runtime, std::vector* flat, + std::string* error); + +} // namespace Libs::Graphics::ShaderRecompiler::IR + +#endif /* EMULATOR_INCLUDE_EMULATOR_GRAPHICS_SHADER_RECOMPILER_SRTWALKER_H_ */ diff --git a/src/graphics/shader/recompiler/VectorAluOps.cpp b/src/graphics/shader/recompiler/VectorAluOps.cpp new file mode 100644 index 000000000..f11165661 --- /dev/null +++ b/src/graphics/shader/recompiler/VectorAluOps.cpp @@ -0,0 +1,1833 @@ +#include "graphics/shader/recompiler/VectorAluOps.h" + +#include +#include + +namespace Libs::Graphics::ShaderRecompiler::Decoder { +namespace { + +struct OpcodeMap { + uint32_t opcode = 0; + Opcode ir = Opcode::Unknown; +}; + +constexpr OpcodeMap VOP2_OPS[] = { + {0x01u, Opcode::VCndmaskB32}, {0x02u, Opcode::VDot2cF32F16}, + {0x03u, Opcode::VAddF32}, {0x04u, Opcode::VSubF32}, + {0x05u, Opcode::VSubrevF32}, {0x08u, Opcode::VMulF32}, + {0x09u, Opcode::VMulI32I24}, {0x0bu, Opcode::VMulU32U24}, + {0x0fu, Opcode::VMinF32}, {0x10u, Opcode::VMaxF32}, + {0x11u, Opcode::VMinI32}, {0x12u, Opcode::VMaxI32}, + {0x13u, Opcode::VMinU32}, {0x14u, Opcode::VMaxU32}, + {0x15u, Opcode::VLshrB32}, {0x16u, Opcode::VLshrrevB32}, + {0x17u, Opcode::VAshrI32}, {0x18u, Opcode::VAshrrevI32}, + {0x19u, Opcode::VLshlB32}, {0x1au, Opcode::VLshlrevB32}, + {0x1bu, Opcode::VAndB32}, {0x1cu, Opcode::VOrB32}, + {0x1du, Opcode::VXorB32}, {0x1eu, Opcode::VXnorB32}, + {0x1fu, Opcode::VMacF32}, {0x20u, Opcode::VMadmkF32}, + {0x21u, Opcode::VMadakF32}, {0x22u, Opcode::VBcntU32B32}, + {0x23u, Opcode::VMbcntLoU32B32}, {0x24u, Opcode::VMbcntHiU32B32}, + {0x25u, Opcode::VAddNcU32}, {0x28u, Opcode::VAddcU32}, + {0x26u, Opcode::VSubNcU32}, {0x27u, Opcode::VSubrevNcU32}, + {0x2bu, Opcode::VMacF32}, {0x2cu, Opcode::VMadmkF32}, + {0x2du, Opcode::VMadakF32}, {0x2fu, Opcode::VCvtPkrtzF16F32}, + {0x32u, Opcode::VAddF16}, {0x33u, Opcode::VSubF16}, + {0x34u, Opcode::VSubrevF16}, {0x35u, Opcode::VMulF16}, + {0x36u, Opcode::VFmacF16}, {0x37u, Opcode::VFmamkF16}, + {0x38u, Opcode::VFmaakF16}, {0x39u, Opcode::VMaxF16}, + {0x3au, Opcode::VMinF16}, {0x3cu, Opcode::VPkFmacF16}, +}; + +constexpr OpcodeMap VOP1_OPS[] = { + {0x00u, Opcode::VNop}, + {0x01u, Opcode::VMovB32}, + {0x02u, Opcode::VReadfirstlaneB32}, + {0x05u, Opcode::VCvtF32I32}, + {0x06u, Opcode::VCvtF32U32}, + {0x07u, Opcode::VCvtU32F32}, + {0x08u, Opcode::VCvtI32F32}, + {0x0au, Opcode::VCvtF16F32}, + {0x0bu, Opcode::VCvtF32F16}, + {0x0cu, Opcode::VCvtRpiI32F32}, + {0x0du, Opcode::VCvtFlrI32F32}, + {0x0eu, Opcode::VCvtOffF32I4}, + {0x11u, Opcode::VCvtF32Ubyte0}, + {0x12u, Opcode::VCvtF32Ubyte1}, + {0x13u, Opcode::VCvtF32Ubyte2}, + {0x14u, Opcode::VCvtF32Ubyte3}, + {0x2au, Opcode::VRcpF32}, + {0x20u, Opcode::VFractF32}, + {0x21u, Opcode::VTruncF32}, + {0x22u, Opcode::VCeilF32}, + {0x23u, Opcode::VRndneF32}, + {0x24u, Opcode::VFloorF32}, + {0x25u, Opcode::VExpF32}, + {0x27u, Opcode::VLogF32}, + {0x2bu, Opcode::VRcpF32}, + {0x2eu, Opcode::VRsqF32}, + {0x33u, Opcode::VSqrtF32}, + {0x35u, Opcode::VSinF32}, + {0x36u, Opcode::VCosF32}, + {0x37u, Opcode::VNotB32}, + {0x38u, Opcode::VBfrevB32}, + {0x39u, Opcode::VFfbhU32}, + {0x3au, Opcode::VFfblB32}, + {0x42u, Opcode::VMovreldB32}, + {0x43u, Opcode::VMovrelsB32}, + {0x50u, Opcode::VCvtF16U16}, + {0x51u, Opcode::VCvtF16I16}, + {0x52u, Opcode::VCvtU16F16}, + {0x53u, Opcode::VCvtI16F16}, + {0x54u, Opcode::VRcpF16}, + {0x55u, Opcode::VSqrtF16}, + {0x56u, Opcode::VRsqF16}, + {0x57u, Opcode::VLogF16}, + {0x58u, Opcode::VExpF16}, + {0x5bu, Opcode::VFloorF16}, + {0x5cu, Opcode::VCeilF16}, + {0x5du, Opcode::VTruncF16}, + {0x5eu, Opcode::VRndneF16}, +}; + +constexpr OpcodeMap VOP3_ENCODED_VOP1_OPS[] = { + {0x00u, Opcode::VNop}, + {0x01u, Opcode::VMovB32}, + {0x02u, Opcode::VReadfirstlaneB32}, + {0x05u, Opcode::VCvtF32I32}, + {0x06u, Opcode::VCvtF32U32}, + {0x07u, Opcode::VCvtU32F32}, + {0x08u, Opcode::VCvtI32F32}, + {0x0au, Opcode::VCvtF16F32}, + {0x0cu, Opcode::VCvtRpiI32F32}, + {0x0du, Opcode::VCvtFlrI32F32}, + {0x0eu, Opcode::VCvtOffF32I4}, + {0x2au, Opcode::VRcpF32}, + {0x20u, Opcode::VFractF32}, + {0x21u, Opcode::VTruncF32}, + {0x22u, Opcode::VCeilF32}, + {0x23u, Opcode::VRndneF32}, + {0x24u, Opcode::VFloorF32}, + {0x25u, Opcode::VExpF32}, + {0x27u, Opcode::VLogF32}, + {0x2bu, Opcode::VRcpF32}, + {0x2eu, Opcode::VRsqF32}, + {0x33u, Opcode::VSqrtF32}, + {0x35u, Opcode::VSinF32}, + {0x36u, Opcode::VCosF32}, + {0x37u, Opcode::VNotB32}, + {0x38u, Opcode::VBfrevB32}, + {0x39u, Opcode::VFfbhU32}, + {0x3au, Opcode::VFfblB32}, + {0x42u, Opcode::VMovreldB32}, + {0x43u, Opcode::VMovrelsB32}, + {0x50u, Opcode::VCvtF16U16}, + {0x51u, Opcode::VCvtF16I16}, + {0x52u, Opcode::VCvtU16F16}, + {0x53u, Opcode::VCvtI16F16}, + {0x54u, Opcode::VRcpF16}, + {0x55u, Opcode::VSqrtF16}, + {0x56u, Opcode::VRsqF16}, + {0x57u, Opcode::VLogF16}, + {0x58u, Opcode::VExpF16}, + {0x5bu, Opcode::VFloorF16}, + {0x5cu, Opcode::VCeilF16}, + {0x5du, Opcode::VTruncF16}, + {0x5eu, Opcode::VRndneF16}, +}; + +constexpr OpcodeMap VOPC_OPS[] = { + {0x00u, Opcode::VCmpFF32}, {0x01u, Opcode::VCmpLtF32}, {0x02u, Opcode::VCmpEqF32}, + {0x03u, Opcode::VCmpLeF32}, {0x04u, Opcode::VCmpGtF32}, {0x05u, Opcode::VCmpLgF32}, + {0x06u, Opcode::VCmpGeF32}, {0x07u, Opcode::VCmpOF32}, {0x08u, Opcode::VCmpUF32}, + {0x09u, Opcode::VCmpNgeF32}, {0x0au, Opcode::VCmpNlgF32}, {0x0bu, Opcode::VCmpNgtF32}, + {0x0cu, Opcode::VCmpNleF32}, {0x0du, Opcode::VCmpNeqF32}, {0x0eu, Opcode::VCmpNltF32}, + {0x0fu, Opcode::VCmpTruF32}, {0x11u, Opcode::VCmpxLtF32}, {0x12u, Opcode::VCmpxEqF32}, + {0x13u, Opcode::VCmpxLeF32}, {0x14u, Opcode::VCmpxGtF32}, {0x15u, Opcode::VCmpxLgF32}, + {0x16u, Opcode::VCmpxGeF32}, {0x19u, Opcode::VCmpxNgeF32}, {0x1au, Opcode::VCmpxNlgF32}, + {0x1bu, Opcode::VCmpxNgtF32}, {0x1cu, Opcode::VCmpxNleF32}, {0x1du, Opcode::VCmpxNeqF32}, + {0x1eu, Opcode::VCmpxNltF32}, {0x80u, Opcode::VCmpFI32}, {0x81u, Opcode::VCmpLtI32}, + {0x82u, Opcode::VCmpEqI32}, {0x83u, Opcode::VCmpLeI32}, {0x84u, Opcode::VCmpGtI32}, + {0x85u, Opcode::VCmpNeI32}, {0x86u, Opcode::VCmpGeI32}, {0x87u, Opcode::VCmpTI32}, + {0x88u, Opcode::VCmpClassF32}, {0x89u, Opcode::VCmpLtI16}, {0x8au, Opcode::VCmpEqI16}, + {0x8bu, Opcode::VCmpLeI16}, {0x8cu, Opcode::VCmpGtI16}, {0x8du, Opcode::VCmpNeI16}, + {0x8eu, Opcode::VCmpGeI16}, {0x91u, Opcode::VCmpxLtI32}, {0x92u, Opcode::VCmpxEqI32}, + {0x93u, Opcode::VCmpxLeI32}, {0x94u, Opcode::VCmpxGtI32}, {0x95u, Opcode::VCmpxNeI32}, + {0x96u, Opcode::VCmpxGeI32}, {0xa9u, Opcode::VCmpLtU16}, {0xaau, Opcode::VCmpEqU16}, + {0xabu, Opcode::VCmpLeU16}, {0xacu, Opcode::VCmpGtU16}, {0xadu, Opcode::VCmpNeU16}, + {0xaeu, Opcode::VCmpGeU16}, {0xc0u, Opcode::VCmpFU32}, {0xc1u, Opcode::VCmpLtU32}, + {0xc2u, Opcode::VCmpEqU32}, {0xc3u, Opcode::VCmpLeU32}, {0xc4u, Opcode::VCmpGtU32}, + {0xc5u, Opcode::VCmpNeU32}, {0xc6u, Opcode::VCmpGeU32}, {0xc7u, Opcode::VCmpTU32}, + {0xd1u, Opcode::VCmpxLtU32}, {0xd2u, Opcode::VCmpxEqU32}, {0xd3u, Opcode::VCmpxLeU32}, + {0xd4u, Opcode::VCmpxGtU32}, {0xd5u, Opcode::VCmpxNeU32}, {0xd6u, Opcode::VCmpxGeU32}, + {0xe5u, Opcode::VCmpNeU64}, {0xc9u, Opcode::VCmpLtF16}, {0xcau, Opcode::VCmpEqF16}, + {0xcbu, Opcode::VCmpLeF16}, {0xccu, Opcode::VCmpGtF16}, {0xcdu, Opcode::VCmpLgF16}, + {0xceu, Opcode::VCmpGeF16}, {0xedu, Opcode::VCmpNeqF16}, {0xd9u, Opcode::VCmpxLtF16}, + {0xdau, Opcode::VCmpxEqF16}, {0xdbu, Opcode::VCmpxLeF16}, {0xdcu, Opcode::VCmpxGtF16}, + {0xdeu, Opcode::VCmpxGeF16}, {0xfdu, Opcode::VCmpxNeqF16}, {0xfeu, Opcode::VCmpxNltF16}, +}; + +constexpr OpcodeMap VOP3_OPS[] = { + {0x141u, Opcode::VMadF32}, {0x142u, Opcode::VMadI32I24}, + {0x143u, Opcode::VMadU32U24}, {0x176u, Opcode::VMadU64U32}, + {0x144u, Opcode::VCubeidF32}, {0x145u, Opcode::VCubescF32}, + {0x146u, Opcode::VCubetcF32}, {0x147u, Opcode::VCubemaF32}, + {0x14bu, Opcode::VFmaF32}, {0x148u, Opcode::VBfeU32}, + {0x149u, Opcode::VBfeI32}, {0x14au, Opcode::VBfiB32}, + {0x14eu, Opcode::VAlignbitB32}, {0x151u, Opcode::VMin3F32}, + {0x152u, Opcode::VMin3I32}, {0x153u, Opcode::VMin3U32}, + {0x351u, Opcode::VMin3F16}, {0x154u, Opcode::VMax3F32}, + {0x155u, Opcode::VMax3I32}, {0x156u, Opcode::VMax3U32}, + {0x354u, Opcode::VMax3F16}, {0x157u, Opcode::VMed3F32}, + {0x158u, Opcode::VMed3I32}, {0x159u, Opcode::VMed3U32}, + {0x357u, Opcode::VMed3F16}, {0x15du, Opcode::VSadU32}, + {0x15eu, Opcode::VCvtPkU8F32}, {0x178u, Opcode::VXor3B32}, + {0x12fu, Opcode::VCvtPkrtzF16F32}, {0x169u, Opcode::VMulLoU32}, + {0x16au, Opcode::VMulHiU32}, {0x16bu, Opcode::VMulLoI32}, + {0x16cu, Opcode::VMulHiI32}, {0x303u, Opcode::VAddNcU16}, + {0x304u, Opcode::VSubNcU16}, {0x307u, Opcode::VLshrrevB16}, + {0x308u, Opcode::VAshrrevI16}, {0x309u, Opcode::VMaxU16}, + {0x30au, Opcode::VMaxI16}, {0x30bu, Opcode::VMinU16}, + {0x30cu, Opcode::VMinI16}, {0x30du, Opcode::VAddNcI16}, + {0x30eu, Opcode::VSubNcI16}, {0x30fu, Opcode::VAddI32}, + {0x310u, Opcode::VSubI32}, {0x311u, Opcode::VPackB32F16}, + {0x314u, Opcode::VLshlrevB16}, {0x319u, Opcode::VSubrevI32}, + {0x345u, Opcode::VXadU32}, {0x346u, Opcode::VLshlAddU32}, + {0x347u, Opcode::VAddLshlU32}, {0x360u, Opcode::VReadlaneB32}, + {0x361u, Opcode::VWritelaneB32}, {0x362u, Opcode::VLdexpF32}, + {0x363u, Opcode::VBfmB32}, {0x364u, Opcode::VBcntU32B32}, + {0x365u, Opcode::VMbcntLoU32B32}, {0x366u, Opcode::VMbcntHiU32B32}, + {0x368u, Opcode::VCvtPknormI16F32}, {0x369u, Opcode::VCvtPknormU16F32}, + {0x36au, Opcode::VCvtPkU16U32}, {0x36fu, Opcode::VLshlOrB32}, + {0x371u, Opcode::VAndOrB32}, {0x372u, Opcode::VOr3B32}, + {0x377u, Opcode::VPermlane16B32}, {0x378u, Opcode::VPermlanex16B32}, + {0x34bu, Opcode::VFmaF16}, {0x36du, Opcode::VAdd3U32}, +}; + +constexpr OpcodeMap VOP3P_OPS[] = { + {0x00u, Opcode::VPkMadI16}, {0x01u, Opcode::VPkMulLoU16}, {0x02u, Opcode::VPkAddI16}, + {0x03u, Opcode::VPkSubI16}, {0x04u, Opcode::VPkLshlrevB16}, {0x05u, Opcode::VPkLshrrevB16}, + {0x06u, Opcode::VPkAshrrevI16}, {0x07u, Opcode::VPkMaxI16}, {0x08u, Opcode::VPkMinI16}, + {0x09u, Opcode::VPkMadU16}, {0x0au, Opcode::VPkAddU16}, {0x0bu, Opcode::VPkSubU16}, + {0x0cu, Opcode::VPkMaxU16}, {0x0du, Opcode::VPkMinU16}, {0x0eu, Opcode::VPkFmaF16}, + {0x0fu, Opcode::VPkAddF16}, {0x10u, Opcode::VPkMulF16}, {0x11u, Opcode::VPkMinF16}, + {0x12u, Opcode::VPkMaxF16}, {0x20u, Opcode::VFmaF32}, {0x21u, Opcode::VMadMixloF16}, + {0x22u, Opcode::VMadMixhiF16}, +}; + +Opcode Lookup(const OpcodeMap* ops, uint32_t count, uint32_t opcode) { + for (uint32_t i = 0; i < count; i++) { + const auto& op = ops[i]; + if (op.opcode == opcode) { + return op.ir; + } + } + return Opcode::Unsupported; +} + +bool IsVop2LiteralMadOpcode(uint32_t opcode) { + return opcode == 0x20u || opcode == 0x21u || opcode == 0x2cu || opcode == 0x2du; +} + +bool IsUnsupportedVop3EncodedVop2Alias(uint32_t opcode) { + return IsVop2LiteralMadOpcode(opcode) || opcode == 0x02u || opcode == 0x2bu || + opcode == 0x39u || opcode == 0x3au; +} + +Opcode LookupVop3Opcode(uint32_t opcode) { + const auto direct = Lookup(VOP3_OPS, static_cast(std::size(VOP3_OPS)), opcode); + if (direct != Opcode::Unsupported) { + return direct; + } + if (opcode <= 0xffu) { + return Lookup(VOPC_OPS, static_cast(std::size(VOPC_OPS)), opcode); + } + if (opcode >= 0x100u && opcode <= 0x13fu) { + if (IsUnsupportedVop3EncodedVop2Alias(opcode - 0x100u)) { + return Opcode::Unsupported; + } + return Lookup(VOP2_OPS, static_cast(std::size(VOP2_OPS)), opcode - 0x100u); + } + if (opcode >= 0x180u && opcode <= 0x1ffu) { + return Lookup(VOP3_ENCODED_VOP1_OPS, + static_cast(std::size(VOP3_ENCODED_VOP1_OPS)), opcode - 0x180u); + } + return Lookup(VOP3_OPS, static_cast(std::size(VOP3_OPS)), opcode); +} + +bool IsVop3EncodedVopc(uint32_t opcode) { + return opcode <= 0xffu; +} + +bool IsVop3EncodedVop2(uint32_t opcode) { + return opcode >= 0x100u && opcode <= 0x13fu; +} + +bool IsVop3EncodedVop1(uint32_t opcode) { + return opcode >= 0x180u && opcode <= 0x1ffu; +} + +Opcode LookupVintrpOpcode(uint32_t opcode) { + switch (opcode) { + case 0x00u: return Opcode::VInterpP1F32; + case 0x01u: return Opcode::VInterpP2F32; + case 0x02u: return Opcode::VInterpMovF32; + default: return Opcode::Unsupported; + } +} + +bool UsesScalarDestination(Opcode opcode) { + return opcode == Opcode::VReadfirstlaneB32 || opcode == Opcode::VReadlaneB32; +} + +bool IsVop3BCarryOutOpcode(Opcode opcode) { + return opcode == Opcode::VAddI32 || opcode == Opcode::VSubI32 || opcode == Opcode::VSubrevI32; +} + +bool IsVop3BMadU64Opcode(Opcode opcode) { + return opcode == Opcode::VMadU64U32; +} + +bool IsPermlaneOpcode(Opcode opcode) { + return opcode == Opcode::VPermlane16B32 || opcode == Opcode::VPermlanex16B32; +} + +bool IsVop2LowHalfF16Opcode(Opcode opcode) { + return opcode == Opcode::VAddF16 || opcode == Opcode::VSubF16 || opcode == Opcode::VSubrevF16 || + opcode == Opcode::VMulF16 || opcode == Opcode::VFmacF16 || opcode == Opcode::VFmamkF16 || + opcode == Opcode::VFmaakF16 || opcode == Opcode::VMaxF16 || opcode == Opcode::VMinF16; +} + +bool IsNativeVop3F16TernaryOpcode(Opcode opcode) { + return opcode == Opcode::VMin3F16 || opcode == Opcode::VMax3F16 || opcode == Opcode::VMed3F16 || + opcode == Opcode::VFmaF16; +} + +bool IsNativeVop3B16BinaryOpcode(Opcode opcode) { + switch (opcode) { + case Opcode::VAddNcU16: + case Opcode::VSubNcU16: + case Opcode::VMaxU16: + case Opcode::VMaxI16: + case Opcode::VMinU16: + case Opcode::VMinI16: + case Opcode::VAddNcI16: + case Opcode::VSubNcI16: + case Opcode::VLshlrevB16: + case Opcode::VLshrrevB16: + case Opcode::VAshrrevI16: return true; + default: return false; + } +} + +void ApplyDefaultVop2F16Destination(Instruction* inst) { + if (inst != nullptr && IsVop2LowHalfF16Opcode(inst->opcode)) { + inst->dst.sdwa_sel = 4; + } +} + +bool IsVop1FloatResultOpcode(Opcode opcode); +bool IsVopcCompareExec(Opcode opcode); + +bool IsVop1FloatSourceOpcode(Opcode opcode) { + switch (opcode) { + case Opcode::VMovB32: + case Opcode::VCvtF32F16: + case Opcode::VCvtU32F32: + case Opcode::VCvtI32F32: + case Opcode::VCvtRpiI32F32: + case Opcode::VCvtFlrI32F32: + case Opcode::VCvtF16F32: + case Opcode::VRcpF32: + case Opcode::VFractF32: + case Opcode::VTruncF32: + case Opcode::VCeilF32: + case Opcode::VRndneF32: + case Opcode::VFloorF32: + case Opcode::VExpF32: + case Opcode::VLogF32: + case Opcode::VRsqF32: + case Opcode::VSqrtF32: + case Opcode::VSqrtF16: + case Opcode::VFloorF16: + case Opcode::VCeilF16: + case Opcode::VTruncF16: + case Opcode::VRndneF16: + case Opcode::VSinF32: + case Opcode::VCosF32: return true; + default: return false; + } +} + +constexpr uint32_t SdwaSel(uint32_t sel) { + return 1u << sel; +} + +constexpr uint32_t SdwaSelBytes() { + return SdwaSel(0) | SdwaSel(1) | SdwaSel(2) | SdwaSel(3); +} + +constexpr uint32_t SdwaSelWords() { + return SdwaSel(4) | SdwaSel(5); +} + +constexpr uint32_t SdwaSelFull() { + return SdwaSel(6); +} + +struct Vop1SdwaRule { + Opcode opcode = Opcode::Unknown; + uint32_t source_selectors = SdwaSelFull(); + uint32_t partial_dst_selectors = 0; + uint32_t partial_dst_source_selectors = 0; + bool source_modifiers = false; +}; + +constexpr Vop1SdwaRule VOP1_SDWA_RULES[] = { + {Opcode::VMovB32, SdwaSelBytes() | SdwaSelWords() | SdwaSelFull(), SdwaSelWords(), + SdwaSelWords() | SdwaSelFull(), false}, + {Opcode::VCvtF32U32, SdwaSelBytes() | SdwaSelWords() | SdwaSelFull(), 0, 0, false}, + {Opcode::VCvtF32I32, SdwaSelBytes() | SdwaSelWords() | SdwaSelFull(), 0, 0, false}, + {Opcode::VCvtF32F16, SdwaSelWords() | SdwaSelFull(), 0, 0, true}, + {Opcode::VCvtF16F32, SdwaSelFull(), SdwaSelWords(), SdwaSelFull(), false}, + {Opcode::VCvtF16U16, SdwaSelWords() | SdwaSelFull(), SdwaSelWords(), + SdwaSelWords() | SdwaSelFull(), false}, + {Opcode::VCvtU16F16, SdwaSelWords() | SdwaSelFull(), SdwaSelWords(), + SdwaSelWords() | SdwaSelFull(), false}, + {Opcode::VCvtF16I16, SdwaSelWords() | SdwaSelFull(), SdwaSelWords(), + SdwaSelWords() | SdwaSelFull(), false}, + {Opcode::VCvtI16F16, SdwaSelWords() | SdwaSelFull(), SdwaSelWords(), + SdwaSelWords() | SdwaSelFull(), false}, + {Opcode::VRcpF16, SdwaSelWords() | SdwaSelFull(), SdwaSelWords(), + SdwaSelWords() | SdwaSelFull(), true}, + {Opcode::VSqrtF16, SdwaSelWords() | SdwaSelFull(), SdwaSelWords(), + SdwaSelWords() | SdwaSelFull(), false}, + {Opcode::VRsqF16, SdwaSelWords() | SdwaSelFull(), SdwaSelWords(), + SdwaSelWords() | SdwaSelFull(), false}, + {Opcode::VLogF16, SdwaSelWords() | SdwaSelFull(), SdwaSelWords(), + SdwaSelWords() | SdwaSelFull(), true}, + {Opcode::VExpF16, SdwaSelWords() | SdwaSelFull(), SdwaSelWords(), + SdwaSelWords() | SdwaSelFull(), true}, + {Opcode::VFloorF16, SdwaSelWords() | SdwaSelFull(), SdwaSelWords(), + SdwaSelWords() | SdwaSelFull(), false}, + {Opcode::VCeilF16, SdwaSelWords() | SdwaSelFull(), SdwaSelWords(), + SdwaSelWords() | SdwaSelFull(), false}, + {Opcode::VTruncF16, SdwaSelWords() | SdwaSelFull(), SdwaSelWords(), + SdwaSelWords() | SdwaSelFull(), false}, + {Opcode::VRndneF16, SdwaSelWords() | SdwaSelFull(), SdwaSelWords(), + SdwaSelWords() | SdwaSelFull(), false}, + {Opcode::VCvtU32F32, SdwaSelFull(), SdwaSelBytes() | SdwaSelWords(), SdwaSelFull(), false}, + {Opcode::VCvtI32F32, SdwaSelFull(), SdwaSelBytes() | SdwaSelWords(), SdwaSelFull(), false}, + {Opcode::VCvtRpiI32F32, SdwaSelFull(), SdwaSelBytes() | SdwaSelWords(), SdwaSelFull(), false}, + {Opcode::VCvtFlrI32F32, SdwaSelFull(), SdwaSelBytes() | SdwaSelWords(), SdwaSelFull(), false}, +}; + +const Vop1SdwaRule* FindVop1SdwaRule(Opcode opcode) { + for (const auto& rule: VOP1_SDWA_RULES) { + if (rule.opcode == opcode) { + return &rule; + } + } + return nullptr; +} + +bool HasSdwaSelector(uint32_t mask, uint32_t selector) { + return selector <= 6u && (mask & SdwaSel(selector)) != 0; +} + +bool IsValidFullSdwaDestinationUnused(uint32_t dst_u) { + return dst_u != 3u; +} + +bool IsVop1SdwaSourceSupported(Opcode opcode, uint32_t src_sel, bool src_neg, bool src_abs) { + if (src_sel == 6u) { + return true; + } + const auto* rule = FindVop1SdwaRule(opcode); + if (rule == nullptr || !HasSdwaSelector(rule->source_selectors, src_sel)) { + return false; + } + return rule->source_modifiers || (!src_neg && !src_abs); +} + +bool IsVop1SdwaDestinationSupported(Opcode opcode, uint32_t dst_sel, uint32_t dst_u, + uint32_t src_sel) { + if ((opcode == Opcode::VCvtU16F16 || opcode == Opcode::VCvtI16F16) && dst_sel == 6u) { + return false; + } + if (dst_sel == 6u) { + return IsValidFullSdwaDestinationUnused(dst_u); + } + const auto* rule = FindVop1SdwaRule(opcode); + if (rule == nullptr || dst_u != 2u) { + return false; + } + return HasSdwaSelector(rule->partial_dst_selectors, dst_sel) && + HasSdwaSelector(rule->partial_dst_source_selectors, src_sel); +} + +bool HasUnsupportedVop1SdwaSourceModifiers(Opcode opcode, bool src_neg, bool src_abs) { + switch (opcode) { + case Opcode::VCvtU16F16: + case Opcode::VCvtI16F16: + case Opcode::VSqrtF16: + case Opcode::VRsqF16: return src_neg || src_abs; + case Opcode::VLogF16: return src_neg; + default: return false; + } +} + +bool ValidateVop1Sdwa(Instruction* inst, uint32_t opcode, uint32_t modifier) { + const auto dst_sel = (modifier >> 8u) & 0x7u; + const auto dst_u = (modifier >> 11u) & 0x3u; + const auto clamp = (modifier >> 13u) & 0x1u; + const auto omod = (modifier >> 14u) & 0x3u; + const auto src0_sel = (modifier >> 16u) & 0x7u; + const auto src0_neg = (modifier >> 20u) & 0x1u; + const auto src0_abs = (modifier >> 21u) & 0x1u; + + if (src0_sel > 6u || dst_sel > 6u) { + SetUnsupported(inst, Family::VOP1, opcode, "VOP1 SDWA selector is invalid"); + return false; + } + if ((clamp != 0u || omod != 0u) && !IsVop1FloatResultOpcode(inst->opcode)) { + SetUnsupported(inst, Family::VOP1, opcode, "VOP1 SDWA output modifiers are not supported"); + return false; + } + if (!IsVop1SdwaDestinationSupported(inst->opcode, dst_sel, dst_u, src0_sel)) { + SetUnsupported(inst, Family::VOP1, opcode, + "VOP1 SDWA destination selector is not supported"); + return false; + } + if (HasUnsupportedVop1SdwaSourceModifiers(inst->opcode, src0_neg != 0u, src0_abs != 0u)) { + SetUnsupported(inst, Family::VOP1, opcode, "VOP1 SDWA source modifiers are not supported"); + return false; + } + if (!IsVop1SdwaSourceSupported(inst->opcode, src0_sel, src0_neg != 0u, src0_abs != 0u)) { + SetUnsupported(inst, Family::VOP1, opcode, "VOP1 SDWA source selector is not supported"); + return false; + } + return true; +} + +bool DecodeVop1Sdwa(uint32_t pc, std::span code, uint32_t word_index, + uint32_t opcode, uint32_t vdst, Instruction* inst, std::string* error) { + if (word_index + 1u >= code.size()) { + if (error != nullptr) { + *error = fmt::format("truncated VOP1 SDWA instruction at pc 0x{:08x}", pc); + } + return false; + } + + const auto modifier = code[word_index + 1u]; + const auto src0 = modifier & 0xffu; + const auto dst_sel = (modifier >> 8u) & 0x7u; + const auto dst_u = (modifier >> 11u) & 0x3u; + const auto clamp = (modifier >> 13u) & 0x1u; + const auto omod = (modifier >> 14u) & 0x3u; + const auto src0_sel = (modifier >> 16u) & 0x7u; + const auto src0_sext = (modifier >> 19u) & 0x1u; + const auto src0_neg = (modifier >> 20u) & 0x1u; + const auto src0_abs = (modifier >> 21u) & 0x1u; + const auto s0 = (modifier >> 23u) & 0x1u; + + SetRawWords(inst, code, word_index, 2); + if (!ValidateVop1Sdwa(inst, opcode, modifier)) { + return true; + } + + const bool scalar_dst = UsesScalarDestination(inst->opcode); + if (!(scalar_dst ? DecodeScalarDestination(vdst, pc, &inst->dst, error) + : DecodeVectorGpr(vdst, &inst->dst, error)) || + !DecodeScalarSource(src0 + (s0 == 0u ? 256u : 0u), pc, &inst->src0, error)) { + return false; + } + inst->dst.sdwa_sel = dst_sel; + inst->dst.sdwa_dst_unused = dst_u; + inst->dst.clamp = clamp != 0u; + inst->dst.omod = omod; + inst->src0.sdwa_sel = src0_sel; + inst->src0.sdwa_sext = src0_sext != 0u; + inst->src0.negate = src0_neg != 0u; + inst->src0.absolute = src0_abs != 0u; + inst->src_count = 1; + return ReadLiteralOperands(code, word_index, inst, error); +} + +bool DecodeVop1Dpp(uint32_t pc, std::span code, uint32_t word_index, + uint32_t opcode, uint32_t vdst, Instruction* inst, std::string* error) { + if (word_index + 1u >= code.size()) { + if (error != nullptr) { + *error = fmt::format("truncated VOP1 DPP instruction at pc 0x{:08x}", pc); + } + return false; + } + + const auto modifier = code[word_index + 1u]; + const auto src0 = modifier & 0xffu; + SetRawWords(inst, code, word_index, 2); + + const bool scalar_dst = UsesScalarDestination(inst->opcode); + if (!(scalar_dst ? DecodeScalarDestination(vdst, pc, &inst->dst, error) + : DecodeVectorGpr(vdst, &inst->dst, error)) || + !DecodeScalarSource(src0 + 256u, pc, &inst->src0, error)) { + return false; + } + inst->src0.negate = ((modifier >> 20u) & 0x1u) != 0u; + inst->src0.absolute = ((modifier >> 21u) & 0x1u) != 0u; + inst->src0.dpp = true; + inst->src0.dpp_ctrl = (modifier >> 8u) & 0x1ffu; + inst->src0.dpp_fetch_inactive = ((modifier >> 18u) & 0x1u) != 0u; + inst->src0.dpp_bound_ctrl = ((modifier >> 19u) & 0x1u) != 0u; + inst->src0.dpp_bank_mask = (modifier >> 24u) & 0xfu; + inst->src0.dpp_row_mask = (modifier >> 28u) & 0xfu; + inst->src_count = 1; + + if (!IsVop1FloatSourceOpcode(inst->opcode) && (inst->src0.negate || inst->src0.absolute)) { + SetUnsupported(inst, Family::VOP1, opcode, + "VOP1 DPP integer source modifiers are not supported"); + return true; + } + return ReadLiteralOperands(code, word_index, inst, error); +} + +using Vop1ModifierDecodeFn = bool (*)(uint32_t pc, std::span code, + uint32_t word_index, uint32_t opcode, uint32_t vdst, + Instruction* inst, std::string* error); + +struct Vop1ModifierDecoder { + uint32_t escape = 0; + Vop1ModifierDecodeFn decode = nullptr; +}; + +constexpr Vop1ModifierDecoder VOP1_MODIFIER_DECODERS[] = { + {249u, DecodeVop1Sdwa}, + {250u, DecodeVop1Dpp}, +}; + +bool TryDecodeVop1Modifier(uint32_t pc, std::span code, uint32_t word_index, + uint32_t src0, uint32_t opcode, uint32_t vdst, Instruction* inst, + std::string* error, bool* handled) { + if (handled != nullptr) { + *handled = false; + } + for (const auto& decoder: VOP1_MODIFIER_DECODERS) { + if (decoder.escape == src0) { + if (handled != nullptr) { + *handled = true; + } + return decoder.decode(pc, code, word_index, opcode, vdst, inst, error); + } + } + return true; +} + +bool IsVop2FloatSourceOpcode(Opcode opcode) { + switch (opcode) { + case Opcode::VAddF32: + case Opcode::VSubF32: + case Opcode::VSubrevF32: + case Opcode::VMulF32: + case Opcode::VMinF32: + case Opcode::VMaxF32: + case Opcode::VMacF32: + case Opcode::VMadmkF32: + case Opcode::VMadakF32: + case Opcode::VAddF16: + case Opcode::VSubF16: + case Opcode::VSubrevF16: + case Opcode::VMulF16: + case Opcode::VFmacF16: + case Opcode::VFmamkF16: + case Opcode::VFmaakF16: + case Opcode::VMaxF16: + case Opcode::VMinF16: return true; + default: return false; + } +} + +bool IsVop2FloatResultOpcode(Opcode opcode) { + switch (opcode) { + case Opcode::VAddF32: + case Opcode::VSubF32: + case Opcode::VSubrevF32: + case Opcode::VMulF32: + case Opcode::VMinF32: + case Opcode::VMaxF32: + case Opcode::VMacF32: + case Opcode::VMadmkF32: + case Opcode::VMadakF32: + case Opcode::VAddF16: + case Opcode::VSubF16: + case Opcode::VSubrevF16: + case Opcode::VMulF16: + case Opcode::VFmacF16: + case Opcode::VFmamkF16: + case Opcode::VFmaakF16: + case Opcode::VMaxF16: + case Opcode::VMinF16: return true; + default: return false; + } +} + +bool IsVop1FloatResultOpcode(Opcode opcode) { + switch (opcode) { + case Opcode::VCvtF32I32: + case Opcode::VCvtF32U32: + case Opcode::VCvtF32F16: + case Opcode::VCvtOffF32I4: + case Opcode::VCvtF32Ubyte0: + case Opcode::VCvtF32Ubyte1: + case Opcode::VCvtF32Ubyte2: + case Opcode::VCvtF32Ubyte3: + case Opcode::VRcpF32: + case Opcode::VFractF32: + case Opcode::VTruncF32: + case Opcode::VCeilF32: + case Opcode::VRndneF32: + case Opcode::VFloorF32: + case Opcode::VExpF32: + case Opcode::VLogF32: + case Opcode::VRsqF32: + case Opcode::VSqrtF32: + case Opcode::VRcpF16: + case Opcode::VRsqF16: + case Opcode::VLogF16: + case Opcode::VExpF16: + case Opcode::VSinF32: + case Opcode::VCosF32: return true; + default: return false; + } +} + +bool IsVopcFloatCompareOpcode(Opcode opcode) { + switch (opcode) { + case Opcode::VCmpFF32: + case Opcode::VCmpLtF32: + case Opcode::VCmpEqF32: + case Opcode::VCmpLeF32: + case Opcode::VCmpGtF32: + case Opcode::VCmpLgF32: + case Opcode::VCmpGeF32: + case Opcode::VCmpOF32: + case Opcode::VCmpUF32: + case Opcode::VCmpNgeF32: + case Opcode::VCmpNlgF32: + case Opcode::VCmpNgtF32: + case Opcode::VCmpNleF32: + case Opcode::VCmpNeqF32: + case Opcode::VCmpNltF32: + case Opcode::VCmpTruF32: + case Opcode::VCmpxLtF32: + case Opcode::VCmpxEqF32: + case Opcode::VCmpxLeF32: + case Opcode::VCmpxGtF32: + case Opcode::VCmpxLgF32: + case Opcode::VCmpxGeF32: + case Opcode::VCmpxNgeF32: + case Opcode::VCmpxNlgF32: + case Opcode::VCmpxNgtF32: + case Opcode::VCmpxNleF32: + case Opcode::VCmpxNeqF32: + case Opcode::VCmpxNltF32: + case Opcode::VCmpLtF16: + case Opcode::VCmpEqF16: + case Opcode::VCmpLeF16: + case Opcode::VCmpGtF16: + case Opcode::VCmpLgF16: + case Opcode::VCmpGeF16: + case Opcode::VCmpNeqF16: + case Opcode::VCmpxLtF16: + case Opcode::VCmpxEqF16: + case Opcode::VCmpxLeF16: + case Opcode::VCmpxGtF16: + case Opcode::VCmpxGeF16: + case Opcode::VCmpxNeqF16: + case Opcode::VCmpxNltF16: + case Opcode::VCmpClassF32: return true; + default: return false; + } +} + +struct Vop2SdwaFields { + uint32_t src0 = 0; + uint32_t dst_sel = 6; + uint32_t dst_u = 0; + uint32_t clamp = 0; + uint32_t omod = 0; + uint32_t src0_sel = 6; + uint32_t src0_sext = 0; + uint32_t src0_neg = 0; + uint32_t src0_abs = 0; + uint32_t s0 = 0; + uint32_t src1_sel = 6; + uint32_t src1_sext = 0; + uint32_t src1_neg = 0; + uint32_t src1_abs = 0; + uint32_t s1 = 0; +}; + +Vop2SdwaFields DecodeVop2SdwaFields(uint32_t modifier) { + Vop2SdwaFields fields; + fields.src0 = modifier & 0xffu; + fields.dst_sel = (modifier >> 8u) & 0x7u; + fields.dst_u = (modifier >> 11u) & 0x3u; + fields.clamp = (modifier >> 13u) & 0x1u; + fields.omod = (modifier >> 14u) & 0x3u; + fields.src0_sel = (modifier >> 16u) & 0x7u; + fields.src0_sext = (modifier >> 19u) & 0x1u; + fields.src0_neg = (modifier >> 20u) & 0x1u; + fields.src0_abs = (modifier >> 21u) & 0x1u; + fields.s0 = (modifier >> 23u) & 0x1u; + fields.src1_sel = (modifier >> 24u) & 0x7u; + fields.src1_sext = (modifier >> 27u) & 0x1u; + fields.src1_neg = (modifier >> 28u) & 0x1u; + fields.src1_abs = (modifier >> 29u) & 0x1u; + fields.s1 = (modifier >> 31u) & 0x1u; + return fields; +} + +struct Vop2SdwaRule { + Opcode opcode = Opcode::Unknown; + uint32_t dst_selectors = SdwaSelFull(); + uint32_t src0_selectors = SdwaSelFull(); + uint32_t src1_selectors = SdwaSelFull(); + bool partial_dst = false; + bool source_modifiers = false; +}; + +constexpr uint32_t SdwaSelAll() { + return SdwaSelBytes() | SdwaSelWords() | SdwaSelFull(); +} + +constexpr Vop2SdwaRule VOP2_SDWA_RULES[] = { + {Opcode::VCndmaskB32, SdwaSelWords(), SdwaSelWords() | SdwaSelFull(), + SdwaSelWords() | SdwaSelFull(), true, false}, + {Opcode::VAddF32, SdwaSelFull(), SdwaSelFull(), SdwaSelFull(), false, true}, + {Opcode::VSubF32, SdwaSelFull(), SdwaSelFull(), SdwaSelFull(), false, true}, + {Opcode::VMulF32, SdwaSelFull(), SdwaSelFull(), SdwaSelFull(), false, true}, + {Opcode::VAddF16, SdwaSelWords() | SdwaSelFull(), SdwaSelWords() | SdwaSelFull(), + SdwaSelWords() | SdwaSelFull(), true, true}, + {Opcode::VSubF16, SdwaSelWords() | SdwaSelFull(), SdwaSelWords() | SdwaSelFull(), + SdwaSelWords() | SdwaSelFull(), true, true}, + {Opcode::VSubrevF16, SdwaSelWords() | SdwaSelFull(), SdwaSelWords() | SdwaSelFull(), + SdwaSelWords() | SdwaSelFull(), true, true}, + {Opcode::VMulF16, SdwaSelWords() | SdwaSelFull(), SdwaSelWords() | SdwaSelFull(), + SdwaSelWords() | SdwaSelFull(), true, true}, + {Opcode::VMaxF16, SdwaSelWords() | SdwaSelFull(), SdwaSelWords() | SdwaSelFull(), + SdwaSelWords() | SdwaSelFull(), true, true}, + {Opcode::VMinF16, SdwaSelWords() | SdwaSelFull(), SdwaSelWords() | SdwaSelFull(), + SdwaSelWords() | SdwaSelFull(), true, true}, + {Opcode::VMulI32I24, SdwaSelFull(), SdwaSelAll(), SdwaSelAll(), false, false}, + {Opcode::VMulU32U24, SdwaSelFull(), SdwaSelAll(), SdwaSelAll(), false, false}, + {Opcode::VMinU32, SdwaSelFull(), SdwaSelAll(), SdwaSelAll(), false, false}, + {Opcode::VMaxU32, SdwaSelFull(), SdwaSelAll(), SdwaSelAll(), false, false}, + {Opcode::VLshrrevB32, SdwaSelFull(), SdwaSelFull(), SdwaSelWords() | SdwaSelFull(), false, + false}, + {Opcode::VLshlrevB32, SdwaSelFull(), SdwaSelFull(), SdwaSelAll(), false, false}, + {Opcode::VAndB32, SdwaSelWords() | SdwaSelFull(), SdwaSelAll(), SdwaSelAll(), true, false}, + {Opcode::VOrB32, SdwaSelWords() | SdwaSelFull(), SdwaSelAll(), SdwaSelAll(), true, false}, + {Opcode::VXorB32, SdwaSelWords() | SdwaSelFull(), SdwaSelAll(), SdwaSelAll(), true, false}, + {Opcode::VXnorB32, SdwaSelWords() | SdwaSelFull(), SdwaSelAll(), SdwaSelAll(), true, false}, + {Opcode::VAddNcU32, SdwaSelFull(), SdwaSelAll(), SdwaSelAll(), false, false}, + {Opcode::VSubNcU32, SdwaSelFull(), SdwaSelAll(), SdwaSelAll(), false, false}, + {Opcode::VSubrevNcU32, SdwaSelFull(), SdwaSelAll(), SdwaSelAll(), false, false}, +}; + +const Vop2SdwaRule* FindVop2SdwaRule(Opcode opcode) { + for (const auto& rule: VOP2_SDWA_RULES) { + if (rule.opcode == opcode) { + return &rule; + } + } + return nullptr; +} + +bool IsVop2SdwaDestinationSupported(const Vop2SdwaRule& rule, const Vop2SdwaFields& fields) { + if (!HasSdwaSelector(rule.dst_selectors, fields.dst_sel)) { + return false; + } + if (fields.dst_sel == 6u) { + return IsValidFullSdwaDestinationUnused(fields.dst_u); + } + return fields.dst_u == 2u && rule.partial_dst; +} + +bool IsFullWidthVop2Sdwa(const Vop2SdwaFields& fields) { + return fields.dst_sel == 6u && fields.dst_u == 0u && fields.src0_sel == 6u && + fields.src1_sel == 6u; +} + +bool ValidateVop2Sdwa(Instruction* inst, uint32_t opcode, const Vop2SdwaFields& fields) { + if (fields.src0_sel > 6u || fields.src1_sel > 6u || fields.dst_sel > 6u) { + SetUnsupported(inst, Family::VOP2, opcode, "VOP2 SDWA selector is invalid"); + return false; + } + if ((fields.clamp != 0u || fields.omod != 0u) && !IsVop2FloatResultOpcode(inst->opcode)) { + SetUnsupported(inst, Family::VOP2, opcode, "VOP2 SDWA output modifiers are not supported"); + return false; + } + if (IsFullWidthVop2Sdwa(fields)) { + if (!IsVop2FloatSourceOpcode(inst->opcode) && inst->opcode != Opcode::VCndmaskB32 && + (fields.src0_neg != 0u || fields.src0_abs != 0u || fields.src1_neg != 0u || + fields.src1_abs != 0u)) { + SetUnsupported(inst, Family::VOP2, opcode, + "VOP2 SDWA source modifiers are not supported"); + return false; + } + return true; + } + + const auto* rule = FindVop2SdwaRule(inst->opcode); + if (rule == nullptr) { + SetUnsupported(inst, Family::VOP2, opcode, + "VOP2 SDWA modifier is not supported for opcode"); + return false; + } + if (!IsVop2SdwaDestinationSupported(*rule, fields)) { + SetUnsupported(inst, Family::VOP2, opcode, + "VOP2 SDWA destination selector is not supported"); + return false; + } + if (!HasSdwaSelector(rule->src0_selectors, fields.src0_sel) || + !HasSdwaSelector(rule->src1_selectors, fields.src1_sel)) { + SetUnsupported(inst, Family::VOP2, opcode, "VOP2 SDWA source selector is not supported"); + return false; + } + const bool has_source_modifiers = (fields.src0_neg != 0u || fields.src0_abs != 0u || + fields.src1_neg != 0u || fields.src1_abs != 0u); + const bool cndmask_float_modifiers = + inst->opcode == Opcode::VCndmaskB32 && fields.src0_sel == 6u && fields.src1_sel == 6u; + if (!rule->source_modifiers && has_source_modifiers && !cndmask_float_modifiers) { + SetUnsupported(inst, Family::VOP2, opcode, "VOP2 SDWA source modifiers are not supported"); + return false; + } + return true; +} + +bool FinalizeVop2Instruction(uint32_t pc, std::span code, uint32_t word_index, + Instruction* inst, std::string* error) { + (void)pc; + switch (inst->opcode) { + case Opcode::VMadmkF32: + case Opcode::VFmamkF16: + inst->src2 = inst->src1; + inst->src1 = {}; + inst->src1.kind = OperandKind::LiteralConstant; + inst->src_count = 3; + break; + case Opcode::VMadakF32: + case Opcode::VFmaakF16: + inst->src2 = {}; + inst->src2.kind = OperandKind::LiteralConstant; + inst->src_count = 3; + break; + case Opcode::VAddcU32: + inst->dst2.kind = OperandKind::VccLo; + inst->src2.kind = OperandKind::VccLo; + inst->src_count = 3; + break; + default: inst->src_count = 2; break; + } + return ReadLiteralOperands(code, word_index, inst, error); +} + +bool DecodeVop2Sdwa(uint32_t pc, std::span code, uint32_t word_index, + uint32_t opcode, uint32_t vdst, uint32_t vsrc1, Instruction* inst, + std::string* error) { + if (word_index + 1u >= code.size()) { + if (error != nullptr) { + *error = fmt::format("truncated VOP2 SDWA instruction at pc 0x{:08x}", pc); + } + return false; + } + + const auto modifier = code[word_index + 1u]; + const auto fields = DecodeVop2SdwaFields(modifier); + SetRawWords(inst, code, word_index, 2); + if (!ValidateVop2Sdwa(inst, opcode, fields)) { + return true; + } + + if (!DecodeVectorGpr(vdst, &inst->dst, error) || + !DecodeScalarSource(fields.src0 + (fields.s0 == 0u ? 256u : 0u), pc, &inst->src0, error) || + !DecodeScalarSource(vsrc1 + (fields.s1 == 0u ? 256u : 0u), pc, &inst->src1, error)) { + return false; + } + inst->dst.sdwa_sel = fields.dst_sel; + inst->dst.sdwa_dst_unused = fields.dst_u; + inst->dst.clamp = fields.clamp != 0u; + inst->dst.omod = fields.omod; + inst->src0.sdwa_sel = fields.src0_sel; + inst->src0.sdwa_sext = fields.src0_sext != 0u; + inst->src0.negate = fields.src0_neg != 0u; + inst->src0.absolute = fields.src0_abs != 0u; + inst->src1.sdwa_sel = fields.src1_sel; + inst->src1.sdwa_sext = fields.src1_sext != 0u; + inst->src1.negate = fields.src1_neg != 0u; + inst->src1.absolute = fields.src1_abs != 0u; + return FinalizeVop2Instruction(pc, code, word_index, inst, error); +} + +bool DecodeVop2Dpp(uint32_t pc, std::span code, uint32_t word_index, + uint32_t opcode, uint32_t vdst, uint32_t vsrc1, Instruction* inst, + std::string* error) { + if (word_index + 1u >= code.size()) { + if (error != nullptr) { + *error = fmt::format("truncated VOP2 DPP instruction at pc 0x{:08x}", pc); + } + return false; + } + + const auto modifier = code[word_index + 1u]; + const auto src0 = modifier & 0xffu; + SetRawWords(inst, code, word_index, 2); + + if (!DecodeVectorGpr(vdst, &inst->dst, error) || + !DecodeScalarSource(src0 + 256u, pc, &inst->src0, error) || + !DecodeVectorGpr(vsrc1, &inst->src1, error)) { + return false; + } + ApplyDefaultVop2F16Destination(inst); + inst->src0.negate = ((modifier >> 20u) & 0x1u) != 0u; + inst->src0.absolute = ((modifier >> 21u) & 0x1u) != 0u; + inst->src0.dpp = true; + inst->src0.dpp_ctrl = (modifier >> 8u) & 0x1ffu; + inst->src0.dpp_fetch_inactive = ((modifier >> 18u) & 0x1u) != 0u; + inst->src0.dpp_bound_ctrl = ((modifier >> 19u) & 0x1u) != 0u; + inst->src0.dpp_bank_mask = (modifier >> 24u) & 0xfu; + inst->src0.dpp_row_mask = (modifier >> 28u) & 0xfu; + inst->src1.negate = ((modifier >> 22u) & 0x1u) != 0u; + inst->src1.absolute = ((modifier >> 23u) & 0x1u) != 0u; + + if (!IsVop2FloatSourceOpcode(inst->opcode) && + (inst->src0.negate || inst->src0.absolute || inst->src1.negate || inst->src1.absolute)) { + SetUnsupported(inst, Family::VOP2, opcode, + "VOP2 DPP integer source modifiers are not supported"); + return true; + } + return FinalizeVop2Instruction(pc, code, word_index, inst, error); +} + +using Vop2ModifierDecodeFn = bool (*)(uint32_t pc, std::span code, + uint32_t word_index, uint32_t opcode, uint32_t vdst, + uint32_t vsrc1, Instruction* inst, std::string* error); + +struct Vop2ModifierDecoder { + uint32_t escape = 0; + Vop2ModifierDecodeFn decode = nullptr; +}; + +constexpr Vop2ModifierDecoder VOP2_MODIFIER_DECODERS[] = { + {249u, DecodeVop2Sdwa}, + {250u, DecodeVop2Dpp}, +}; + +bool TryDecodeVop2Modifier(uint32_t pc, std::span code, uint32_t word_index, + uint32_t src0, uint32_t opcode, uint32_t vdst, uint32_t vsrc1, + Instruction* inst, std::string* error, bool* handled) { + if (handled != nullptr) { + *handled = false; + } + for (const auto& decoder: VOP2_MODIFIER_DECODERS) { + if (decoder.escape == src0) { + if (handled != nullptr) { + *handled = true; + } + return decoder.decode(pc, code, word_index, opcode, vdst, vsrc1, inst, error); + } + } + return true; +} + +struct VopcSdwaFields { + uint32_t src0 = 0; + uint32_t sdst = 0; + uint32_t sd = 0; + uint32_t src0_sel = 6; + uint32_t src0_sext = 0; + uint32_t src0_neg = 0; + uint32_t src0_abs = 0; + uint32_t s0 = 0; + uint32_t src1_sel = 6; + uint32_t src1_sext = 0; + uint32_t src1_neg = 0; + uint32_t src1_abs = 0; + uint32_t s1 = 0; +}; + +VopcSdwaFields DecodeVopcSdwaFields(uint32_t modifier) { + VopcSdwaFields fields; + fields.src0 = modifier & 0xffu; + fields.sdst = (modifier >> 8u) & 0x7fu; + fields.sd = (modifier >> 15u) & 0x1u; + fields.src0_sel = (modifier >> 16u) & 0x7u; + fields.src0_sext = (modifier >> 19u) & 0x1u; + fields.src0_neg = (modifier >> 20u) & 0x1u; + fields.src0_abs = (modifier >> 21u) & 0x1u; + fields.s0 = (modifier >> 23u) & 0x1u; + fields.src1_sel = (modifier >> 24u) & 0x7u; + fields.src1_sext = (modifier >> 27u) & 0x1u; + fields.src1_neg = (modifier >> 28u) & 0x1u; + fields.src1_abs = (modifier >> 29u) & 0x1u; + fields.s1 = (modifier >> 31u) & 0x1u; + return fields; +} + +bool SupportsVopcSdwa(Opcode opcode) { + return opcode != Opcode::Unsupported; +} + +bool DecodeVopcSdwa(uint32_t pc, std::span code, uint32_t word_index, + uint32_t opcode, uint32_t vsrc1, Instruction* inst, std::string* error) { + if (word_index + 1u >= code.size()) { + if (error != nullptr) { + *error = fmt::format("truncated VOPC SDWA instruction at pc 0x{:08x}", pc); + } + return false; + } + + const auto modifier = code[word_index + 1u]; + const auto fields = DecodeVopcSdwaFields(modifier); + SetRawWords(inst, code, word_index, 2); + if (fields.src0_sel > 6u || fields.src1_sel > 6u) { + SetUnsupported(inst, Family::VOPC, opcode, "VOPC SDWA selector is invalid"); + return true; + } + if (!SupportsVopcSdwa(inst->opcode)) { + SetUnsupported(inst, Family::VOPC, opcode, + "VOPC SDWA modifier is not supported for opcode"); + return true; + } + + if (!DecodeScalarSource(fields.src0 + (fields.s0 == 0u ? 256u : 0u), pc, &inst->src0, error) || + !DecodeScalarSource(vsrc1 + (fields.s1 == 0u ? 256u : 0u), pc, &inst->src1, error)) { + return false; + } + if (IsVopcCompareExec(inst->opcode)) { + inst->dst.kind = OperandKind::ExecLo; + } else if (fields.sd == 0u) { + inst->dst.kind = OperandKind::VccLo; + } else if (!DecodeScalarDestination(fields.sdst, pc, &inst->dst, error)) { + return false; + } + inst->src0.sdwa_sel = fields.src0_sel; + inst->src0.sdwa_sext = fields.src0_sext != 0u; + inst->src0.negate = fields.src0_neg != 0u; + inst->src0.absolute = fields.src0_abs != 0u; + inst->src1.sdwa_sel = fields.src1_sel; + inst->src1.sdwa_sext = fields.src1_sext != 0u; + inst->src1.negate = fields.src1_neg != 0u; + inst->src1.absolute = fields.src1_abs != 0u; + inst->src_count = 2; + return ReadLiteralOperands(code, word_index, inst, error); +} + +bool DecodeVopcDpp(uint32_t pc, std::span code, uint32_t word_index, + uint32_t opcode, uint32_t vsrc1, Instruction* inst, std::string* error) { + if (word_index + 1u >= code.size()) { + if (error != nullptr) { + *error = fmt::format("truncated VOPC DPP instruction at pc 0x{:08x}", pc); + } + return false; + } + + const auto modifier = code[word_index + 1u]; + const auto src0 = modifier & 0xffu; + SetRawWords(inst, code, word_index, 2); + if (inst->opcode == Opcode::Unsupported) { + SetUnsupported(inst, Family::VOPC, opcode, "VOPC opcode is not implemented"); + return true; + } + if (!DecodeScalarSource(src0 + 256u, pc, &inst->src0, error) || + !DecodeVectorGpr(vsrc1, &inst->src1, error)) { + return false; + } + inst->dst.kind = IsVopcCompareExec(inst->opcode) ? OperandKind::ExecLo : OperandKind::VccLo; + inst->src0.negate = ((modifier >> 20u) & 0x1u) != 0u; + inst->src0.absolute = ((modifier >> 21u) & 0x1u) != 0u; + inst->src0.dpp = true; + inst->src0.dpp_ctrl = (modifier >> 8u) & 0x1ffu; + inst->src0.dpp_fetch_inactive = ((modifier >> 18u) & 0x1u) != 0u; + inst->src0.dpp_bound_ctrl = ((modifier >> 19u) & 0x1u) != 0u; + inst->src0.dpp_bank_mask = (modifier >> 24u) & 0xfu; + inst->src0.dpp_row_mask = (modifier >> 28u) & 0xfu; + inst->src1.negate = ((modifier >> 22u) & 0x1u) != 0u; + inst->src1.absolute = ((modifier >> 23u) & 0x1u) != 0u; + inst->src_count = 2; + return ReadLiteralOperands(code, word_index, inst, error); +} + +uint32_t NativeVop3SourceCount(Opcode opcode) { + switch (opcode) { + case Opcode::VMulLoU32: + case Opcode::VMulHiU32: + case Opcode::VMulLoI32: + case Opcode::VMulHiI32: + case Opcode::VMulI32I24: + case Opcode::VAddNcU16: + case Opcode::VSubNcU16: + case Opcode::VMaxU16: + case Opcode::VMaxI16: + case Opcode::VMinU16: + case Opcode::VMinI16: + case Opcode::VAddNcI16: + case Opcode::VSubNcI16: + case Opcode::VLshlrevB16: + case Opcode::VLshrrevB16: + case Opcode::VAshrrevI16: + case Opcode::VAddI32: + case Opcode::VSubI32: + case Opcode::VSubrevI32: + case Opcode::VPackB32F16: + case Opcode::VReadlaneB32: + case Opcode::VWritelaneB32: + case Opcode::VCvtPkrtzF16F32: + case Opcode::VLdexpF32: + case Opcode::VBfmB32: + case Opcode::VBcntU32B32: + case Opcode::VCvtPknormI16F32: + case Opcode::VCvtPknormU16F32: + case Opcode::VCvtPkU16U32: return 2; + default: return 3; + } +} + +uint32_t Vop3pSourceCount(Opcode opcode) { + switch (opcode) { + case Opcode::VPkMulLoU16: + case Opcode::VPkAddI16: + case Opcode::VPkSubI16: + case Opcode::VPkLshlrevB16: + case Opcode::VPkLshrrevB16: + case Opcode::VPkAshrrevI16: + case Opcode::VPkMaxI16: + case Opcode::VPkMinI16: + case Opcode::VPkAddU16: + case Opcode::VPkSubU16: + case Opcode::VPkMaxU16: + case Opcode::VPkMinU16: + case Opcode::VPkAddF16: + case Opcode::VPkMulF16: + case Opcode::VPkMinF16: + case Opcode::VPkMaxF16: return 2; + default: return 3; + } +} + +bool IsPackedVop3p(Opcode opcode) { + switch (opcode) { + case Opcode::VPkAddF16: + case Opcode::VPkMulF16: + case Opcode::VPkMinF16: + case Opcode::VPkMaxF16: + case Opcode::VPkFmaF16: return true; + default: return false; + } +} + +bool IsMadMixF16(Opcode opcode) { + return opcode == Opcode::VMadMixloF16 || opcode == Opcode::VMadMixhiF16; +} + +void ApplyVop3pSourceModifiers(Instruction* inst, uint32_t op_sel, uint32_t op_sel_hi, uint32_t neg, + uint32_t neg_hi) { + Operand* sources[] = {&inst->src0, &inst->src1, &inst->src2}; + for (uint32_t i = 0; i < 3u; i++) { + sources[i]->op_sel = ((op_sel >> i) & 1u) != 0; + sources[i]->op_sel_hi = ((op_sel_hi >> i) & 1u) != 0; + sources[i]->negate = ((neg >> i) & 1u) != 0; + sources[i]->negate_hi = ((neg_hi >> i) & 1u) != 0; + } +} + +void ApplyVop3pMixAbsModifiers(Instruction* inst) { + Operand* sources[] = {&inst->src0, &inst->src1, &inst->src2}; + for (auto* source: sources) { + // RDNA2 MIX opcodes reuse the VOP3P NEG_HI bits as source absolute modifiers. + source->absolute = source->negate_hi; + source->negate_hi = false; + } +} + +void ApplyNativeVop3MadMixModifiers(Instruction* inst, uint32_t op_sel, uint32_t abs, + uint32_t neg) { + Operand* sources[] = {&inst->src0, &inst->src1, &inst->src2}; + for (uint32_t i = 0; i < 3u; i++) { + sources[i]->op_sel = ((op_sel >> i) & 1u) != 0; + sources[i]->op_sel_hi = true; + sources[i]->negate = ((neg >> i) & 1u) != 0; + sources[i]->absolute = ((abs >> i) & 1u) != 0; + } + inst->dst.sdwa_sel = ((op_sel & 0x8u) != 0) ? 5u : 4u; +} + +void ApplyNativeVop3F16TernaryModifiers(Instruction* inst, uint32_t op_sel, uint32_t abs, + uint32_t neg) { + Operand* sources[] = {&inst->src0, &inst->src1, &inst->src2}; + for (uint32_t i = 0; i < 3u; i++) { + sources[i]->op_sel = ((op_sel >> i) & 1u) != 0; + sources[i]->op_sel_hi = true; + sources[i]->negate = ((neg >> i) & 1u) != 0; + sources[i]->absolute = ((abs >> i) & 1u) != 0; + } + inst->dst.sdwa_sel = ((op_sel & 0x8u) != 0) ? 5u : 4u; +} + +void ApplyNativeVop3B16BinaryModifiers(Instruction* inst, uint32_t op_sel) { + inst->src0.op_sel = (op_sel & 0x1u) != 0; + inst->src1.op_sel = (op_sel & 0x2u) != 0; + inst->dst.sdwa_sel = (op_sel & 0x8u) != 0 ? 5u : 4u; +} + +void ApplyNativeVop3PackB32F16Modifiers(Instruction* inst, uint32_t op_sel, uint32_t abs, + uint32_t neg) { + inst->src0.op_sel = (op_sel & 0x1u) != 0; + inst->src1.op_sel = (op_sel & 0x2u) != 0; + inst->src0.negate = (neg & 0x1u) != 0; + inst->src1.negate = (neg & 0x2u) != 0; + inst->src0.absolute = (abs & 0x1u) != 0; + inst->src1.absolute = (abs & 0x2u) != 0; +} + +bool SupportsNativeVop3SourceModifiers(Opcode opcode) { + if (IsVop1FloatSourceOpcode(opcode)) { + return true; + } + if (IsVopcFloatCompareOpcode(opcode)) { + return true; + } + if (IsNativeVop3F16TernaryOpcode(opcode)) { + return true; + } + switch (opcode) { + case Opcode::VCndmaskB32: + case Opcode::VAddF32: + case Opcode::VSubF32: + case Opcode::VSubrevF32: + case Opcode::VMulF32: + case Opcode::VMinF32: + case Opcode::VMaxF32: + case Opcode::VMacF32: + case Opcode::VMadF32: + case Opcode::VFmaF32: + case Opcode::VPackB32F16: + case Opcode::VCubeidF32: + case Opcode::VCubescF32: + case Opcode::VCubetcF32: + case Opcode::VCubemaF32: + case Opcode::VCvtPkrtzF16F32: + case Opcode::VCvtPknormI16F32: + case Opcode::VCvtPknormU16F32: + case Opcode::VMin3F32: + case Opcode::VMax3F32: + case Opcode::VMed3F32: + case Opcode::VLdexpF32: return true; + default: return false; + } +} + +bool SupportsNativeVop3ResultModifiers(Opcode opcode) { + if (IsVop1FloatResultOpcode(opcode)) { + return true; + } + switch (opcode) { + case Opcode::VAddF32: + case Opcode::VSubF32: + case Opcode::VSubrevF32: + case Opcode::VMulF32: + case Opcode::VMinF32: + case Opcode::VMaxF32: + case Opcode::VMacF32: + case Opcode::VMadF32: + case Opcode::VFmaF32: + case Opcode::VFmaF16: + case Opcode::VLdexpF32: + case Opcode::VMin3F32: + case Opcode::VMax3F32: + case Opcode::VMed3F32: return true; + default: return false; + } +} + +bool HasUnsupportedNativeVop3Modifiers(Opcode opcode, bool permlane, bool mad_mix, bool addc, + bool scalar_dst, uint32_t abs, uint32_t op_sel, + uint32_t clamp, uint32_t omod, uint32_t neg) { + const bool source_modifiers = SupportsNativeVop3SourceModifiers(opcode); + const bool result_modifiers = SupportsNativeVop3ResultModifiers(opcode); + + if (permlane) { + return abs != 0u || (op_sel & ~0x3u) != 0u || clamp != 0u || omod != 0u || neg != 0u; + } + if (mad_mix) { + return clamp != 0u || omod != 0u; + } + if (IsNativeVop3F16TernaryOpcode(opcode)) { + return opcode != Opcode::VFmaF16 && (clamp != 0u || omod != 0u); + } + if (IsNativeVop3B16BinaryOpcode(opcode)) { + return abs != 0u || clamp != 0u || omod != 0u || neg != 0u; + } + if (addc || scalar_dst) { + return clamp != 0u || omod != 0u || neg != 0u; + } + switch (opcode) { + case Opcode::VLdexpF32: return (abs & ~1u) != 0u || op_sel != 0u || (neg & ~1u) != 0u; + case Opcode::VCndmaskB32: + return (abs & ~0x3u) != 0u || op_sel != 0u || clamp != 0u || omod != 0u || + (neg & ~0x3u) != 0u; + case Opcode::VPackB32F16: + return (abs & ~0x3u) != 0u || (op_sel & ~0x3u) != 0u || clamp != 0u || omod != 0u || + (neg & ~0x3u) != 0u; + default: break; + } + if (source_modifiers) { + return op_sel != 0u || (omod != 0u && !result_modifiers) || + (clamp != 0u && !result_modifiers); + } + if (result_modifiers) { + return abs != 0u || op_sel != 0u || neg != 0u; + } + return abs != 0u || op_sel != 0u || clamp != 0u || omod != 0u || neg != 0u; +} + +void ApplyNativeVop3SourceModifiers(Instruction* inst, uint32_t abs, uint32_t neg) { + Operand* sources[] = {&inst->src0, &inst->src1, &inst->src2}; + for (uint32_t i = 0; i < inst->src_count && i < 3u; i++) { + sources[i]->absolute = ((abs >> i) & 1u) != 0; + sources[i]->negate = ((neg >> i) & 1u) != 0; + } +} + +bool IsVopcCompareExec(Opcode opcode) { + switch (opcode) { + case Opcode::VCmpxLtF32: + case Opcode::VCmpxEqF32: + case Opcode::VCmpxLeF32: + case Opcode::VCmpxGtF32: + case Opcode::VCmpxLgF32: + case Opcode::VCmpxGeF32: + case Opcode::VCmpxNgeF32: + case Opcode::VCmpxNlgF32: + case Opcode::VCmpxNgtF32: + case Opcode::VCmpxNleF32: + case Opcode::VCmpxNeqF32: + case Opcode::VCmpxNltF32: + case Opcode::VCmpxLtI32: + case Opcode::VCmpxEqI32: + case Opcode::VCmpxLeI32: + case Opcode::VCmpxGtI32: + case Opcode::VCmpxNeI32: + case Opcode::VCmpxGeI32: + case Opcode::VCmpxLtU32: + case Opcode::VCmpxEqU32: + case Opcode::VCmpxLeU32: + case Opcode::VCmpxGtU32: + case Opcode::VCmpxNeU32: + case Opcode::VCmpxGeU32: + case Opcode::VCmpxLtF16: + case Opcode::VCmpxEqF16: + case Opcode::VCmpxLeF16: + case Opcode::VCmpxGtF16: + case Opcode::VCmpxGeF16: + case Opcode::VCmpxNeqF16: + case Opcode::VCmpxNltF16: return true; + default: return false; + } +} + +} // namespace + +bool DecodeVop2(uint32_t pc, std::span code, uint32_t word_index, Instruction* inst, + std::string* error) { + const uint32_t word = code[word_index]; + const uint32_t opcode = (word >> 25u) & 0x3fu; + const uint32_t vdst = (word >> 17u) & 0xffu; + const uint32_t src0 = word & 0x1ffu; + const uint32_t vsrc1 = (word >> 9u) & 0xffu; + + inst->pc = pc; + inst->word = word; + inst->family = Family::VOP2; + inst->opcode_id = opcode; + inst->opcode = Lookup(VOP2_OPS, static_cast(std::size(VOP2_OPS)), opcode); + SetRawWords(inst, code, word_index, 1); + + switch (opcode) { + case 0x3eu: return DecodeVopc(pc, code, word_index, inst, error); + case 0x3fu: return DecodeVop1(pc, code, word_index, inst, error); + default: break; + } + if (inst->opcode == Opcode::Unsupported) { + SetUnsupported(inst, Family::VOP2, opcode, "VOP2 opcode is not implemented"); + return true; + } + bool modifier_handled = false; + if (!TryDecodeVop2Modifier(pc, code, word_index, src0, opcode, vdst, vsrc1, inst, error, + &modifier_handled)) { + return false; + } + if (modifier_handled) { + return true; + } + + if (!DecodeVectorGpr(vdst, &inst->dst, error) || + !DecodeScalarSource(src0, pc, &inst->src0, error) || + !DecodeVectorGpr(vsrc1, &inst->src1, error)) { + return false; + } + ApplyDefaultVop2F16Destination(inst); + return FinalizeVop2Instruction(pc, code, word_index, inst, error); +} + +bool DecodeVop1(uint32_t pc, std::span code, uint32_t word_index, Instruction* inst, + std::string* error) { + const uint32_t word = code[word_index]; + const uint32_t opcode = (word >> 9u) & 0xffu; + const uint32_t src0 = word & 0x1ffu; + const uint32_t vdst = (word >> 17u) & 0xffu; + + inst->pc = pc; + inst->word = word; + inst->family = Family::VOP1; + inst->opcode_id = opcode; + inst->opcode = Lookup(VOP1_OPS, static_cast(std::size(VOP1_OPS)), opcode); + SetRawWords(inst, code, word_index, 1); + + if (inst->opcode == Opcode::Unsupported) { + SetUnsupported(inst, Family::VOP1, opcode, "VOP1 opcode is not implemented"); + return true; + } + if (inst->opcode == Opcode::VNop) { + inst->dst.kind = OperandKind::Null; + inst->src_count = 0; + return true; + } + bool modifier_handled = false; + if (!TryDecodeVop1Modifier(pc, code, word_index, src0, opcode, vdst, inst, error, + &modifier_handled)) { + return false; + } + if (modifier_handled) { + return true; + } + const bool scalar_dst = UsesScalarDestination(inst->opcode); + if (!(scalar_dst ? DecodeScalarDestination(vdst, pc, &inst->dst, error) + : DecodeVectorGpr(vdst, &inst->dst, error)) || + !DecodeScalarSource(src0, pc, &inst->src0, error)) { + return false; + } + inst->src_count = 1; + return ReadLiteralOperands(code, word_index, inst, error); +} + +bool DecodeVopc(uint32_t pc, std::span code, uint32_t word_index, Instruction* inst, + std::string* error) { + const uint32_t word = code[word_index]; + const uint32_t opcode = (word >> 17u) & 0xffu; + const uint32_t src0 = word & 0x1ffu; + const uint32_t vsrc1 = (word >> 9u) & 0xffu; + + inst->pc = pc; + inst->word = word; + inst->family = Family::VOPC; + inst->opcode_id = opcode; + inst->opcode = Lookup(VOPC_OPS, static_cast(std::size(VOPC_OPS)), opcode); + inst->dst.kind = IsVopcCompareExec(inst->opcode) ? OperandKind::ExecLo : OperandKind::VccLo; + SetRawWords(inst, code, word_index, 1); + + switch (src0) { + case 249u: return DecodeVopcSdwa(pc, code, word_index, opcode, vsrc1, inst, error); + case 250u: return DecodeVopcDpp(pc, code, word_index, opcode, vsrc1, inst, error); + default: break; + } + if (inst->opcode == Opcode::Unsupported) { + SetUnsupported(inst, Family::VOPC, opcode, "VOPC opcode is not implemented"); + return true; + } + if (!DecodeScalarSource(src0, pc, &inst->src0, error) || + !DecodeVectorGpr(vsrc1, &inst->src1, error)) { + return false; + } + inst->src_count = 2; + return ReadLiteralOperands(code, word_index, inst, error); +} + +bool DecodeVop3(uint32_t pc, std::span code, uint32_t word_index, Instruction* inst, + std::string* error) { + if (word_index + 1u >= code.size()) { + if (error != nullptr) { + *error = fmt::format("truncated VOP3 instruction at pc 0x{:08x}", pc); + } + return false; + } + + const uint32_t word0 = code[word_index]; + const uint32_t word1 = code[word_index + 1u]; + const uint32_t opcode = (word0 >> 16u) & 0x3ffu; + const uint32_t vdst = word0 & 0xffu; + const uint32_t sdst = (word0 >> 8u) & 0x7fu; + const uint32_t src0 = word1 & 0x1ffu; + const uint32_t src1 = (word1 >> 9u) & 0x1ffu; + const uint32_t src2 = (word1 >> 18u) & 0x1ffu; + const uint32_t abs = (word0 >> 8u) & 0x7u; + const uint32_t op_sel = (word0 >> 11u) & 0xfu; + const uint32_t clamp = (word0 >> 15u) & 0x1u; + const uint32_t omod = (word1 >> 27u) & 0x3u; + const uint32_t neg = (word1 >> 29u) & 0x7u; + + inst->pc = pc; + inst->word = word0; + inst->family = Family::VOP3; + inst->opcode_id = opcode; + inst->opcode = LookupVop3Opcode(opcode); + SetRawWords(inst, code, word_index, 2); + + const bool addc = inst->opcode == Opcode::VAddcU32 && opcode == 0x128u; + const bool vop3b_carry_out = IsVop3BCarryOutOpcode(inst->opcode); + const bool vop3b_mad_u64 = IsVop3BMadU64Opcode(inst->opcode); + const bool vop3b_uses_sdst = addc || vop3b_carry_out || vop3b_mad_u64; + const bool mad_mix = false; + const bool f16_ternary = IsNativeVop3F16TernaryOpcode(inst->opcode); + const bool b16_binary = IsNativeVop3B16BinaryOpcode(inst->opcode); + const bool pack_b32_f16 = inst->opcode == Opcode::VPackB32F16; + const bool permlane = IsPermlaneOpcode(inst->opcode); + const bool vop3_vopc = IsVop3EncodedVopc(opcode); + const bool compare_exec = vop3_vopc && IsVopcCompareExec(inst->opcode); + const bool scalar_dst = vop3_vopc || UsesScalarDestination(inst->opcode); + const bool scalar_modifier_limits = UsesScalarDestination(inst->opcode); + const bool native_source_modifiers = SupportsNativeVop3SourceModifiers(inst->opcode); + const bool native_result_modifiers = SupportsNativeVop3ResultModifiers(inst->opcode); + const bool modifiers = + HasUnsupportedNativeVop3Modifiers(inst->opcode, permlane, mad_mix, vop3b_uses_sdst, + scalar_modifier_limits, abs, op_sel, clamp, omod, neg); + if (modifiers) { + SetUnsupported(inst, Family::VOP3, opcode, + permlane && (op_sel & ~0x3u) != 0 + ? "VOP3 perm-lane op_sel bits are not implemented" + : mad_mix ? "VOP3 mad-mix clamp/omod is not implemented" + : "VOP3 source modifiers are not implemented"); + return true; + } + if (inst->opcode == Opcode::Unsupported) { + SetUnsupported(inst, Family::VOP3, opcode, "VOP3 opcode is not implemented"); + return true; + } + if (inst->opcode == Opcode::VNop) { + inst->dst.kind = OperandKind::Null; + inst->src_count = 0; + return true; + } + bool dst_ok = true; + if (compare_exec) { + inst->dst.kind = OperandKind::ExecLo; + } else if (scalar_dst) { + // VOP3A uses VDST for VOPC and the scalar-destination lane-read opcodes. + dst_ok = DecodeScalarDestination(vdst, pc, &inst->dst, error); + } else { + dst_ok = DecodeVectorGpr(vdst, &inst->dst, error); + } + if (!dst_ok || !DecodeScalarSource(src0, pc, &inst->src0, error)) { + return false; + } + inst->dst.clamp = native_result_modifiers && clamp != 0u; + inst->dst.omod = native_result_modifiers ? omod : 0u; + if (permlane) { + inst->dst.op_sel = (op_sel & 0x1u) != 0u; + inst->dst.op_sel_hi = (op_sel & 0x2u) != 0u; + } + if (vop3_vopc) { + if (!DecodeScalarSource(src1, pc, &inst->src1, error)) { + return false; + } + inst->src_count = 2; + if (native_source_modifiers) { + ApplyNativeVop3SourceModifiers(inst, abs, neg); + } + return ReadLiteralOperands(code, word_index, inst, error); + } + if (IsVop3EncodedVop1(opcode)) { + inst->src_count = 1; + if (native_source_modifiers) { + ApplyNativeVop3SourceModifiers(inst, abs, neg); + } + return ReadLiteralOperands(code, word_index, inst, error); + } + if (addc) { + if (!DecodeScalarDestination(sdst, pc, &inst->dst2, error) || + !DecodeScalarSource(src1, pc, &inst->src1, error) || + !DecodeScalarSource(src2, pc, &inst->src2, error)) { + return false; + } + inst->src_count = 3; + return ReadLiteralOperands(code, word_index, inst, error); + } + if (vop3b_carry_out) { + if (!DecodeScalarDestination(sdst, pc, &inst->dst2, error) || + !DecodeScalarSource(src1, pc, &inst->src1, error)) { + return false; + } + inst->src_count = 2; + return ReadLiteralOperands(code, word_index, inst, error); + } + if (vop3b_mad_u64) { + if (!DecodeScalarDestination(sdst, pc, &inst->dst2, error) || + !DecodeScalarSource(src1, pc, &inst->src1, error) || + !DecodeScalarSource(src2, pc, &inst->src2, error)) { + return false; + } + inst->src_count = 3; + return ReadLiteralOperands(code, word_index, inst, error); + } + if (IsVop3EncodedVop2(opcode)) { + if (!DecodeScalarSource(src1, pc, &inst->src1, error)) { + return false; + } + if (inst->opcode == Opcode::VCndmaskB32) { + if (!DecodeScalarSource(src2, pc, &inst->src2, error)) { + return false; + } + inst->src_count = 3; + } else { + inst->src_count = 2; + } + if (native_source_modifiers) { + ApplyNativeVop3SourceModifiers(inst, abs, neg); + } + return ReadLiteralOperands(code, word_index, inst, error); + } + if (!DecodeScalarSource(src1, pc, &inst->src1, error)) { + return false; + } + inst->src_count = NativeVop3SourceCount(inst->opcode); + if (inst->src_count > 2u && !DecodeScalarSource(src2, pc, &inst->src2, error)) { + return false; + } + if (mad_mix) { + ApplyNativeVop3MadMixModifiers(inst, op_sel, abs, neg); + } else if (f16_ternary) { + ApplyNativeVop3F16TernaryModifiers(inst, op_sel, abs, neg); + } else if (b16_binary) { + ApplyNativeVop3B16BinaryModifiers(inst, op_sel); + } else if (pack_b32_f16) { + ApplyNativeVop3PackB32F16Modifiers(inst, op_sel, abs, neg); + } else if (native_source_modifiers) { + ApplyNativeVop3SourceModifiers(inst, abs, neg); + } + return ReadLiteralOperands(code, word_index, inst, error); +} + +bool DecodeVop3p(uint32_t pc, std::span code, uint32_t word_index, + Instruction* inst, std::string* error) { + if (word_index + 1u >= code.size()) { + if (error != nullptr) { + *error = fmt::format("truncated VOP3P instruction at pc 0x{:08x}", pc); + } + return false; + } + + const uint32_t word0 = code[word_index]; + const uint32_t word1 = code[word_index + 1u]; + const uint32_t opcode = (word0 >> 16u) & 0x7fu; + const uint32_t vdst = word0 & 0xffu; + const uint32_t neg_hi = (word0 >> 8u) & 0x7u; + const uint32_t op_sel = (word0 >> 11u) & 0x7u; + const uint32_t op_sel_hi_2 = (word0 >> 14u) & 0x1u; + const uint32_t clamp = (word0 >> 15u) & 0x1u; + const uint32_t src0 = word1 & 0x1ffu; + const uint32_t src1 = (word1 >> 9u) & 0x1ffu; + const uint32_t src2 = (word1 >> 18u) & 0x1ffu; + const uint32_t op_sel_hi = ((word1 >> 27u) & 0x3u) | (op_sel_hi_2 << 2u); + const uint32_t neg = (word1 >> 29u) & 0x7u; + + inst->pc = pc; + inst->word = word0; + inst->family = Family::VOP3P; + inst->opcode_id = opcode; + inst->opcode = Lookup(VOP3P_OPS, static_cast(std::size(VOP3P_OPS)), opcode); + SetRawWords(inst, code, word_index, 2); + + if (inst->opcode == Opcode::Unsupported) { + SetUnsupported(inst, Family::VOP3P, opcode, "VOP3P opcode is not implemented"); + return true; + } + inst->src_count = Vop3pSourceCount(inst->opcode); + if (!DecodeVectorGpr(vdst, &inst->dst, error) || + !DecodeScalarSource(src0, pc, &inst->src0, error) || + !DecodeScalarSource(src1, pc, &inst->src1, error)) { + return false; + } + if (inst->src_count > 2u && !DecodeScalarSource(src2, pc, &inst->src2, error)) { + return false; + } + if (inst->opcode == Opcode::VFmaF32) { + inst->dst.clamp = clamp != 0u; + } else if (IsMadMixF16(inst->opcode)) { + inst->dst.clamp = clamp != 0u; + } else if (IsPackedVop3p(inst->opcode)) { + inst->dst.clamp = clamp != 0u; + } else if (clamp != 0u) { + SetUnsupported(inst, Family::VOP3P, opcode, "VOP3P integer clamp is not implemented"); + return true; + } + ApplyVop3pSourceModifiers(inst, op_sel, op_sel_hi, neg, neg_hi); + if (inst->opcode == Opcode::VMadMixloF16) { + ApplyVop3pMixAbsModifiers(inst); + inst->dst.sdwa_sel = 4; + } else if (inst->opcode == Opcode::VMadMixhiF16) { + ApplyVop3pMixAbsModifiers(inst); + inst->dst.sdwa_sel = 5; + } else if (inst->opcode == Opcode::VFmaF32) { + ApplyVop3pMixAbsModifiers(inst); + } + return ReadLiteralOperands(code, word_index, inst, error); +} + +bool DecodeVintrp(uint32_t pc, std::span code, uint32_t word_index, + Instruction* inst, std::string* error) { + if (word_index >= code.size()) { + if (error != nullptr) { + *error = fmt::format("truncated VINTRP instruction at pc 0x{:08x}", pc); + } + return false; + } + + const uint32_t word = code[word_index]; + const uint32_t opcode = (word >> 16u) & 0x3u; + const uint32_t vdst = (word >> 18u) & 0xffu; + const uint32_t attr = (word >> 10u) & 0x3fu; + const uint32_t chan = (word >> 8u) & 0x3u; + const uint32_t vsrc = word & 0xffu; + + inst->pc = pc; + inst->word = word; + inst->word_count = 1; + inst->family = Family::VINTRP; + inst->opcode_id = opcode; + inst->opcode = LookupVintrpOpcode(opcode); + SetRawWords(inst, code, word_index, 1); + if (inst->opcode == Opcode::Unsupported) { + SetUnsupported(inst, Family::VINTRP, opcode, "VINTRP opcode is not implemented"); + return true; + } + + if (!DecodeVectorGpr(vdst, &inst->dst, error)) { + return false; + } + if (inst->opcode == Opcode::VInterpMovF32) { + inst->src0.kind = OperandKind::IntegerInlineConstant; + inst->src0.value = vsrc & 0x3u; + inst->src0.signed_val = static_cast(inst->src0.value); + } else if (!DecodeVectorGpr(vsrc, &inst->src0, error)) { + return false; + } + inst->src1.kind = OperandKind::IntegerInlineConstant; + inst->src1.value = attr; + inst->src1.signed_val = static_cast(attr); + inst->src2.kind = OperandKind::IntegerInlineConstant; + inst->src2.value = chan; + inst->src2.signed_val = static_cast(chan); + inst->src_count = 3; + return true; +} + +} // namespace Libs::Graphics::ShaderRecompiler::Decoder diff --git a/src/graphics/shader/recompiler/VectorAluOps.h b/src/graphics/shader/recompiler/VectorAluOps.h new file mode 100644 index 000000000..c7009fa23 --- /dev/null +++ b/src/graphics/shader/recompiler/VectorAluOps.h @@ -0,0 +1,23 @@ +#ifndef EMULATOR_INCLUDE_EMULATOR_GRAPHICS_SHADER_RECOMPILER_VECTORALUOPS_H_ +#define EMULATOR_INCLUDE_EMULATOR_GRAPHICS_SHADER_RECOMPILER_VECTORALUOPS_H_ + +#include "graphics/shader/recompiler/ShaderDecoder.h" + +namespace Libs::Graphics::ShaderRecompiler::Decoder { + +bool DecodeVop2(uint32_t pc, std::span code, uint32_t word_index, Instruction* inst, + std::string* error); +bool DecodeVop1(uint32_t pc, std::span code, uint32_t word_index, Instruction* inst, + std::string* error); +bool DecodeVopc(uint32_t pc, std::span code, uint32_t word_index, Instruction* inst, + std::string* error); +bool DecodeVop3(uint32_t pc, std::span code, uint32_t word_index, Instruction* inst, + std::string* error); +bool DecodeVop3p(uint32_t pc, std::span code, uint32_t word_index, + Instruction* inst, std::string* error); +bool DecodeVintrp(uint32_t pc, std::span code, uint32_t word_index, + Instruction* inst, std::string* error); + +} // namespace Libs::Graphics::ShaderRecompiler::Decoder + +#endif /* EMULATOR_INCLUDE_EMULATOR_GRAPHICS_SHADER_RECOMPILER_VECTORALUOPS_H_ */ diff --git a/src/graphics/shader/recompiler/shaderIR/ShaderIR.cpp b/src/graphics/shader/recompiler/shaderIR/ShaderIR.cpp new file mode 100644 index 000000000..c062f2972 --- /dev/null +++ b/src/graphics/shader/recompiler/shaderIR/ShaderIR.cpp @@ -0,0 +1,1227 @@ +#include "graphics/shader/recompiler/ShaderIR.h" + +#include "graphics/shader/recompiler/shaderIR/ShaderIRInternal.h" +#include "common/assert.h" +#include "graphics/shader/recompiler/BufferFormat.h" + +#include +#include +#include + +namespace Libs::Graphics::ShaderRecompiler::IR { + +Operand MakeSccOperand() { + Operand operand; + operand.kind = OperandKind::Register; + operand.reg.file = RegisterFile::Scc; + operand.reg.index = static_cast(Decoder::OperandKind::Scc); + return operand; +} + +Operand MakeImmediateU32(uint32_t value) { + Operand operand; + operand.kind = OperandKind::ImmediateU32; + operand.imm = value; + return operand; +} + +Operand MakePcRelativeU32(uint32_t value) { + Operand operand; + operand.kind = OperandKind::PcRelativeU32; + operand.imm = value; + return operand; +} + +bool AppendScalarResultSccNonZero(const Decoder::Instruction& decoded, BasicBlock* block, + std::string* error) { + if (!ScalarResultWritesSccNonZero(decoded.opcode)) { + return true; + } + if (block == nullptr || block->instructions.empty()) { + SetError(error, "internal IR error: missing scalar result for SCC update"); + return false; + } + + const auto result = block->instructions.back().dst; + if (result.kind != OperandKind::Register) { + SetError(error, "internal IR error: scalar SCC update without register result"); + return false; + } + + // Scalar SOP result instructions update SCC from result != 0. Text pixel shaders commonly + // branch on SCC immediately after masking EXEC with VCC via s_andn2_b64. + Instruction scc_update; + scc_update.pc = decoded.pc; + scc_update.op = + ScalarResultIs64Bit(decoded.opcode) ? Opcode::CompareNeU64 : Opcode::CompareNeU32; + scc_update.dst = MakeSccOperand(); + scc_update.src[0] = result; + scc_update.src[1] = MakeImmediateU32(0); + scc_update.src_count = 2; + block->instructions.push_back(scc_update); + return true; +} + +uint32_t VectorByteConvertIndex(Decoder::Opcode opcode) { + switch (opcode) { + case Decoder::Opcode::VCvtF32Ubyte0: return 0; + case Decoder::Opcode::VCvtF32Ubyte1: return 1; + case Decoder::Opcode::VCvtF32Ubyte2: return 2; + case Decoder::Opcode::VCvtF32Ubyte3: return 3; + default: return 0xffffffffu; + } +} + +void CopyOperandModifiers(const Decoder::Operand& decoded, Operand& operand) { + operand.sdwa_sel = decoded.sdwa_sel; + operand.omod = decoded.omod; + operand.sdwa_sext = decoded.sdwa_sext; + operand.op_sel = decoded.op_sel; + operand.op_sel_hi = decoded.op_sel_hi; + operand.negate = decoded.negate; + operand.negate_hi = decoded.negate_hi; + operand.absolute = decoded.absolute; + operand.clamp = decoded.clamp; + operand.dpp_ctrl = decoded.dpp_ctrl; + operand.dpp_row_mask = decoded.dpp_row_mask; + operand.dpp_bank_mask = decoded.dpp_bank_mask; + operand.dpp_fetch_inactive = decoded.dpp_fetch_inactive; + operand.dpp_bound_ctrl = decoded.dpp_bound_ctrl; + operand.dpp = decoded.dpp; +} + +bool LowerRegisterOperand(const Decoder::Operand& decoded, Operand* operand, std::string* error) { + if (operand == nullptr) { + SetError(error, "internal IR error: null operand"); + return false; + } + + *operand = {}; + CopyOperandModifiers(decoded, *operand); + + switch (decoded.kind) { + case Decoder::OperandKind::Sgpr: + operand->kind = OperandKind::Register; + operand->reg.file = RegisterFile::Scalar; + operand->reg.index = decoded.reg; + return true; + case Decoder::OperandKind::Vgpr: + operand->kind = OperandKind::Register; + operand->reg.file = RegisterFile::Vector; + operand->reg.index = decoded.reg; + return true; + case Decoder::OperandKind::VccLo: + operand->kind = OperandKind::Register; + operand->reg.file = RegisterFile::Vcc; + operand->reg.index = 0; + return true; + case Decoder::OperandKind::VccHi: + operand->kind = OperandKind::Register; + operand->reg.file = RegisterFile::Vcc; + operand->reg.index = 1; + return true; + case Decoder::OperandKind::ExecLo: + operand->kind = OperandKind::Register; + operand->reg.file = RegisterFile::Exec; + operand->reg.index = 0; + return true; + case Decoder::OperandKind::ExecHi: + operand->kind = OperandKind::Register; + operand->reg.file = RegisterFile::Exec; + operand->reg.index = 1; + return true; + case Decoder::OperandKind::Scc: + case Decoder::OperandKind::VccZ: + case Decoder::OperandKind::ExecZ: + operand->kind = OperandKind::Register; + operand->reg.file = RegisterFile::Scc; + operand->reg.index = static_cast(decoded.kind); + return true; + case Decoder::OperandKind::M0: + operand->kind = OperandKind::Register; + operand->reg.file = RegisterFile::M0; + operand->reg.index = 0; + return true; + case Decoder::OperandKind::Null: operand->kind = OperandKind::Null; return true; + default: SetError(error, "decoded operand cannot be used as an IR register"); return false; + } +} + +bool LowerSourceOperand(const Decoder::Operand& decoded, Operand* operand, std::string* error) { + switch (decoded.kind) { + case Decoder::OperandKind::LiteralConstant: + case Decoder::OperandKind::IntegerInlineConstant: + case Decoder::OperandKind::FloatInlineConstant: + *operand = {}; + CopyOperandModifiers(decoded, *operand); + operand->kind = OperandKind::ImmediateU32; + operand->imm = decoded.value; + operand->sext_64 = decoded.kind == Decoder::OperandKind::IntegerInlineConstant && + decoded.signed_val < 0; + return true; + case Decoder::OperandKind::Null: + case Decoder::OperandKind::PopsExitingWaveId: + *operand = {}; + CopyOperandModifiers(decoded, *operand); + operand->kind = OperandKind::ImmediateU32; + operand->imm = 0; + return true; + default: return LowerRegisterOperand(decoded, operand, error); + } +} + +void ApplyDppDestinationMask(const Decoder::Instruction& decoded, Operand& dst) { + if (dst.kind != OperandKind::Register || dst.reg.file != RegisterFile::Vector) { + return; + } + + const Decoder::Operand* sources[] = {&decoded.src0, &decoded.src1, &decoded.src2}; + for (uint32_t i = 0; i < decoded.src_count && i < 3u; i++) { + if (!sources[i]->dpp) { + continue; + } + dst.dpp = true; + dst.dpp_ctrl = sources[i]->dpp_ctrl; + dst.dpp_row_mask = sources[i]->dpp_row_mask; + dst.dpp_bank_mask = sources[i]->dpp_bank_mask; + dst.dpp_fetch_inactive = sources[i]->dpp_fetch_inactive; + dst.dpp_bound_ctrl = sources[i]->dpp_bound_ctrl; + return; + } +} + +uint32_t ResourceIndexFromOperand(const Decoder::Operand& operand) { + switch (operand.kind) { + case Decoder::OperandKind::Sgpr: return operand.reg / 4u; + case Decoder::OperandKind::Vgpr: return operand.reg; + case Decoder::OperandKind::IntegerInlineConstant: + case Decoder::OperandKind::LiteralConstant: return operand.value; + default: return 0; + } +} + +MemoryInfo MemoryInfoFromDecoded(const Decoder::Instruction& decoded, ResourceKind kind) { + MemoryInfo mem; + mem.kind = kind; + mem.offset = decoded.offset; + mem.secondary_offset = decoded.secondary_offset; + mem.dmask = decoded.dmask; + mem.data_dwords = decoded.data_dwords; + mem.data_bits = decoded.data_bits; + mem.component_count = decoded.data_dwords; + mem.data_format = decoded.data_format; + mem.number_format = decoded.number_format; + mem.image_sample_flags = decoded.image_sample_flags; + mem.image_dimension = decoded.image_dimension; + mem.image_address_components = decoded.image_address_components; + mem.image_nsa_dwords = decoded.image_nsa_dwords; + for (uint32_t i = 0; i < Decoder::MaxImageNsaAddressComponents; i++) { + mem.image_nsa_addr[i] = decoded.image_nsa_addr[i]; + } + mem.memory_segment = decoded.memory_segment; + mem.data_signed = decoded.data_signed; + mem.typed = decoded.typed; + mem.formatted = decoded.formatted; + mem.image_has_mip = decoded.opcode == Decoder::Opcode::ImageLoadMip || + decoded.opcode == Decoder::Opcode::ImageStoreMip; + mem.glc = decoded.glc; + mem.slc = decoded.slc; + mem.idxen = decoded.idxen; + mem.offen = decoded.offen; + mem.resource = ResourceIndexFromOperand(decoded.src1); + mem.sampler = ResourceIndexFromOperand(decoded.src2); + if (kind == ResourceKind::Lds || kind == ResourceKind::Gds) { + mem.resource = 0; + mem.sampler = 0; + } + return mem; +} + +constexpr uint32_t SCALAR_DST_VCC_LO = 106u; +constexpr uint32_t SCALAR_DST_VCC_HI = 107u; +constexpr uint32_t SCALAR_DST_EXEC_LO = 126u; +constexpr uint32_t SCALAR_DST_EXEC_HI = 127u; + +uint32_t RawScalarLoadBase(const Decoder::Operand& operand) { + if (operand.kind == Decoder::OperandKind::Sgpr) { + return operand.reg; + } + return operand.kind == Decoder::OperandKind::VccLo ? SCALAR_DST_VCC_LO : 0u; +} + +void ClearRegisterOffsetModifiers(Decoder::Operand& operand) { + operand.sdwa_sel = 6; + operand.omod = 0; + operand.sdwa_sext = false; + operand.op_sel = false; + operand.op_sel_hi = false; + operand.negate = false; + operand.negate_hi = false; + operand.absolute = false; + operand.dpp_ctrl = 0; + operand.dpp_row_mask = 0xf; + operand.dpp_bank_mask = 0xf; + operand.dpp_fetch_inactive = false; + operand.dpp_bound_ctrl = false; + operand.dpp = false; +} + +bool TryGetScalarDestinationCode(const Decoder::Operand& operand, uint32_t& code) { + switch (operand.kind) { + case Decoder::OperandKind::Sgpr: code = operand.reg; return true; + case Decoder::OperandKind::VccLo: code = SCALAR_DST_VCC_LO; return true; + case Decoder::OperandKind::VccHi: code = SCALAR_DST_VCC_HI; return true; + case Decoder::OperandKind::ExecLo: code = SCALAR_DST_EXEC_LO; return true; + case Decoder::OperandKind::ExecHi: code = SCALAR_DST_EXEC_HI; return true; + default: return false; + } +} + +bool TryOffsetScalarDestination(const Decoder::Operand& operand, uint32_t index, + Decoder::Operand& decoded) { + // Wide scalar destinations advance through the encoded SGPR destination space, where + // aliases such as VCC and EXEC sit beside ordinary SGPRs. Re-decode after adding the + // dword index so s_buffer_load_dwordx2 vcc_lo writes vcc_lo, then vcc_hi. + uint32_t code = 0; + return TryGetScalarDestinationCode(operand, code) && + Decoder::DecodeScalarDestination(code + index, 0, &decoded, nullptr); +} + +Decoder::Operand OffsetDecodedRegister(const Decoder::Operand& operand, uint32_t index) { + if (index == 0) { + return operand; + } + auto ret = operand; + ClearRegisterOffsetModifiers(ret); + switch (ret.kind) { + case Decoder::OperandKind::Vgpr: ret.reg += index; break; + case Decoder::OperandKind::Sgpr: + case Decoder::OperandKind::VccLo: + case Decoder::OperandKind::VccHi: + case Decoder::OperandKind::ExecLo: + case Decoder::OperandKind::ExecHi: { + Decoder::Operand decoded; + if (TryOffsetScalarDestination(ret, index, decoded)) { + ret = decoded; + } + break; + } + default: break; + } + return ret; +} + +MemoryInfo OffsetMemoryInfo(const Decoder::Instruction& decoded, ResourceKind kind, + uint32_t dword_index) { + auto mem = MemoryInfoFromDecoded(decoded, kind); + mem.offset += dword_index * 4u; + mem.data_dwords = 1; + mem.component_index = dword_index; + mem.component_count = decoded.data_dwords; + return mem; +} + +uint32_t TypedBufferComponentOffsetBytes(const Decoder::Instruction& decoded, + uint32_t component_index) { + if (!decoded.typed || !decoded.formatted) { + return component_index * 4u; + } + + const auto format = Format::DecodeTBufferFormat(decoded.data_format, decoded.number_format); + if (!Format::IsKnownFormat(format)) { + return component_index * 4u; + } + return Format::GetFormatComponentByteOffset(format, component_index); +} + +uint32_t TypedBufferFormatComponentCount(const Decoder::Instruction& decoded) { + if (!decoded.typed || !decoded.formatted) { + return decoded.data_dwords; + } + + const auto format = Format::DecodeTBufferFormat(decoded.data_format, decoded.number_format); + if (!Format::IsKnownFormat(format)) { + return decoded.data_dwords; + } + return Format::GetFormatComponentCount(format); +} + +bool LowerMoveImmediateU32(uint32_t pc, const Decoder::Operand& dst, uint32_t value, + BasicBlock* block, std::string* error) { + Instruction inst; + inst.pc = pc; + inst.op = Opcode::MoveU32; + inst.src[0] = MakeImmediateU32(value); + inst.src_count = 1; + if (!LowerRegisterOperand(dst, &inst.dst, error)) { + return false; + } + block->instructions.push_back(inst); + return true; +} + +MemoryInfo OffsetBufferMemoryInfo(const Decoder::Instruction& decoded, uint32_t component_index) { + auto mem = MemoryInfoFromDecoded(decoded, ResourceKind::Buffer); + mem.offset += TypedBufferComponentOffsetBytes(decoded, component_index); + mem.data_dwords = 1; + mem.component_index = component_index; + mem.component_count = decoded.data_dwords; + return mem; +} + +MemoryInfo ByteOffsetMemoryInfo(const Decoder::Instruction& decoded, ResourceKind kind, + uint32_t byte_offset) { + auto mem = MemoryInfoFromDecoded(decoded, kind); + mem.offset = byte_offset; + mem.secondary_offset = 0; + mem.data_dwords = 1; + mem.component_index = 0; + mem.component_count = 1; + return mem; +} + +ResourceKind FlatSegmentResourceKind(uint32_t segment) { + switch (segment) { + case 1u: return ResourceKind::Scratch; + case 2u: return ResourceKind::Global; + default: return ResourceKind::Flat; + } +} + +const Decoder::Operand& DecodedSourceAt(const Decoder::Instruction& decoded, uint32_t index) { + switch (index) { + case 0: return decoded.src0; + case 1: return decoded.src1; + case 2: return decoded.src2; + default: return decoded.src3; + } +} + +Decoder::Operand M0Operand() { + Decoder::Operand operand; + operand.kind = Decoder::OperandKind::M0; + return operand; +} + + +namespace { + +bool LowerScalarSelect(const Decoder::Instruction& decoded, BasicBlock* block, std::string* error) { + Decoder::Operand scc; + scc.kind = Decoder::OperandKind::Scc; + + Instruction inst; + inst.pc = decoded.pc; + inst.op = Opcode::SelectU32; + inst.src_count = 3; + if (!LowerRegisterOperand(decoded.dst, &inst.dst, error) || + !LowerRegisterOperand(scc, &inst.src[0], error) || + !LowerSourceOperand(decoded.src0, &inst.src[1], error) || + !LowerSourceOperand(decoded.src1, &inst.src[2], error)) { + return false; + } + block->instructions.push_back(inst); + return true; +} + +bool LowerScalarSelect64(const Decoder::Instruction& decoded, BasicBlock* block, + std::string* error) { + Decoder::Operand scc; + scc.kind = Decoder::OperandKind::Scc; + + Instruction inst; + inst.pc = decoded.pc; + inst.op = Opcode::SelectU64; + inst.src_count = 3; + if (!LowerRegisterOperand(decoded.dst, &inst.dst, error) || + !LowerRegisterOperand(scc, &inst.src[0], error) || + !LowerSourceOperand(decoded.src0, &inst.src[1], error) || + !LowerSourceOperand(decoded.src1, &inst.src[2], error)) { + return false; + } + block->instructions.push_back(inst); + return true; +} + +uint32_t ScalarShiftLeftAddAmount(Decoder::Opcode opcode) { + switch (opcode) { + case Decoder::Opcode::SLshl1AddU32: return 1; + case Decoder::Opcode::SLshl2AddU32: return 2; + case Decoder::Opcode::SLshl3AddU32: return 3; + case Decoder::Opcode::SLshl4AddU32: return 4; + default: return 0; + } +} + +bool LowerScalarShiftLeftAdd(const Decoder::Instruction& decoded, BasicBlock* block, + std::string* error) { + const auto shift = ScalarShiftLeftAddAmount(decoded.opcode); + if (shift == 0) { + SetError(error, "internal IR error: unknown scalar shift-left-add opcode"); + return false; + } + + Instruction inst; + inst.pc = decoded.pc; + inst.op = Opcode::ScalarShiftLeftAddCarryU32; + inst.src_count = 3; + inst.src[1].kind = OperandKind::ImmediateU32; + inst.src[1].imm = shift; + if (!LowerRegisterOperand(decoded.dst, &inst.dst, error) || + !LowerSourceOperand(decoded.src0, &inst.src[0], error) || + !LowerSourceOperand(decoded.src1, &inst.src[2], error)) { + return false; + } + block->instructions.push_back(inst); + return true; +} + +Opcode ScalarMinMaxSccCompareOpcode(Decoder::Opcode opcode) { + switch (opcode) { + case Decoder::Opcode::SMinI32: return Opcode::CompareLtI32; + case Decoder::Opcode::SMaxI32: return Opcode::CompareGtI32; + case Decoder::Opcode::SMinU32: return Opcode::CompareLtU32; + case Decoder::Opcode::SMaxU32: return Opcode::CompareGtU32; + default: return Opcode::CompareFalse; + } +} + +bool IsScalarMinMaxOpcode(Decoder::Opcode opcode) { + switch (opcode) { + case Decoder::Opcode::SMinI32: + case Decoder::Opcode::SMaxI32: + case Decoder::Opcode::SMinU32: + case Decoder::Opcode::SMaxU32: return true; + default: return false; + } +} + +bool LowerScalarMinMaxWithScc(const Decoder::Instruction& decoded, BasicBlock* block, + std::string* error) { + Instruction result; + result.pc = decoded.pc; + result.op = LookupIrOpcode(decoded.opcode); + result.src_count = 2; + if (!LowerRegisterOperand(decoded.dst, &result.dst, error) || + !LowerSourceOperand(decoded.src0, &result.src[0], error) || + !LowerSourceOperand(decoded.src1, &result.src[1], error)) { + return false; + } + block->instructions.push_back(result); + + Instruction scc; + scc.pc = decoded.pc; + scc.op = ScalarMinMaxSccCompareOpcode(decoded.opcode); + scc.dst = MakeSccOperand(); + scc.src_count = 2; + if (!LowerSourceOperand(decoded.src0, &scc.src[0], error) || + !LowerSourceOperand(decoded.src1, &scc.src[1], error)) { + return false; + } + block->instructions.push_back(scc); + return true; +} + +bool LowerScalarAddCarry(const Decoder::Instruction& decoded, BasicBlock* block, bool carry_in, + std::string* error) { + Decoder::Operand scc; + scc.kind = Decoder::OperandKind::Scc; + + Instruction inst; + inst.pc = decoded.pc; + inst.op = Opcode::ScalarAddCarryU32; + inst.src_count = 3; + if (!LowerRegisterOperand(decoded.dst, &inst.dst, error) || + !LowerSourceOperand(decoded.src0, &inst.src[0], error) || + !LowerSourceOperand(decoded.src1, &inst.src[1], error)) { + return false; + } + if (carry_in) { + if (!LowerRegisterOperand(scc, &inst.src[2], error)) { + return false; + } + } else { + inst.src[2].kind = OperandKind::ImmediateU32; + inst.src[2].imm = 0; + } + block->instructions.push_back(inst); + return true; +} + +bool LowerScalarSubBorrowCarry(const Decoder::Instruction& decoded, BasicBlock* block, + std::string* error) { + Decoder::Operand scc; + scc.kind = Decoder::OperandKind::Scc; + + Instruction inst; + inst.pc = decoded.pc; + inst.op = Opcode::ScalarSubBorrowCarryU32; + inst.src_count = 3; + if (!LowerRegisterOperand(decoded.dst, &inst.dst, error) || + !LowerSourceOperand(decoded.src0, &inst.src[0], error) || + !LowerSourceOperand(decoded.src1, &inst.src[1], error) || + !LowerRegisterOperand(scc, &inst.src[2], error)) { + return false; + } + block->instructions.push_back(inst); + return true; +} + +bool LowerScalarBinaryScc(const Decoder::Instruction& decoded, BasicBlock* block, Opcode op, + std::string* error) { + Instruction inst; + inst.pc = decoded.pc; + inst.op = op; + inst.src_count = 2; + if (!LowerRegisterOperand(decoded.dst, &inst.dst, error) || + !LowerSourceOperand(decoded.src0, &inst.src[0], error) || + !LowerSourceOperand(decoded.src1, &inst.src[1], error)) { + return false; + } + block->instructions.push_back(inst); + return true; +} + +bool LowerScalarBitset0B32(const Decoder::Instruction& decoded, BasicBlock* block, + std::string* error) { + Instruction inst; + inst.pc = decoded.pc; + inst.op = Opcode::BitClearU32; + inst.src_count = 2; + if (!LowerRegisterOperand(decoded.dst, &inst.dst, error) || + !LowerSourceOperand(decoded.dst, &inst.src[0], error) || + !LowerSourceOperand(decoded.src0, &inst.src[1], error)) { + return false; + } + block->instructions.push_back(inst); + return true; +} + +bool LowerScalarBitset1B32(const Decoder::Instruction& decoded, BasicBlock* block, + std::string* error) { + Instruction inst; + inst.pc = decoded.pc; + inst.op = Opcode::BitSetU32; + inst.src_count = 2; + if (!LowerRegisterOperand(decoded.dst, &inst.dst, error) || + !LowerSourceOperand(decoded.dst, &inst.src[0], error) || + !LowerSourceOperand(decoded.src0, &inst.src[1], error)) { + return false; + } + block->instructions.push_back(inst); + return true; +} + +bool LowerVectorCndmask(const Decoder::Instruction& decoded, BasicBlock* block, + std::string* error) { + Decoder::Operand vcc; + vcc.kind = Decoder::OperandKind::VccLo; + const auto& mask = decoded.src_count >= 3 ? decoded.src2 : vcc; + + Instruction inst; + inst.pc = decoded.pc; + inst.op = + decoded.src0.negate || decoded.src0.absolute || decoded.src1.negate || decoded.src1.absolute + ? Opcode::SelectMaskF32Bits + : Opcode::SelectMaskU32; + inst.src_count = 3; + if (!LowerRegisterOperand(decoded.dst, &inst.dst, error) || + !LowerSourceOperand(mask, &inst.src[0], error) || + !LowerSourceOperand(decoded.src1, &inst.src[1], error) || + !LowerSourceOperand(decoded.src0, &inst.src[2], error)) { + return false; + } + block->instructions.push_back(inst); + return true; +} + +bool LowerVectorMoveB32(const Decoder::Instruction& decoded, BasicBlock* block, + std::string* error) { + Instruction inst; + inst.pc = decoded.pc; + inst.op = decoded.src0.negate || decoded.src0.absolute ? Opcode::MoveF32Bits : Opcode::MoveU32; + inst.src_count = 1; + if (!LowerRegisterOperand(decoded.dst, &inst.dst, error) || + !LowerSourceOperand(decoded.src0, &inst.src[0], error)) { + return false; + } + ApplyDppDestinationMask(decoded, inst.dst); + block->instructions.push_back(inst); + return true; +} + +bool LowerVectorMoveRelSource(const Decoder::Instruction& decoded, BasicBlock* block, + std::string* error) { + if (decoded.dst.kind != Decoder::OperandKind::Vgpr || + decoded.src0.kind != Decoder::OperandKind::Vgpr) { + SetError(error, "V_MOVRELS_B32 requires VGPR source and destination"); + return false; + } + if (decoded.dst.sdwa_sel != 6u || decoded.dst.omod != 0u || decoded.dst.clamp || + decoded.src0.sdwa_sel != 6u || decoded.src0.sdwa_sext || decoded.src0.negate || + decoded.src0.absolute || decoded.src0.dpp) { + SetError(error, "V_MOVRELS_B32 modifiers are not implemented"); + return false; + } + + Instruction inst; + inst.pc = decoded.pc; + inst.op = Opcode::MoveRelSourceU32; + inst.src_count = 2; + const auto m0 = M0Operand(); + if (!LowerRegisterOperand(decoded.dst, &inst.dst, error) || + !LowerSourceOperand(decoded.src0, &inst.src[0], error) || + !LowerSourceOperand(m0, &inst.src[1], error)) { + return false; + } + block->instructions.push_back(inst); + return true; +} + +bool LowerVectorMoveRelDestination(const Decoder::Instruction& decoded, BasicBlock* block, + std::string* error) { + if (decoded.dst.kind != Decoder::OperandKind::Vgpr) { + SetError(error, "V_MOVRELD_B32 requires VGPR destination"); + return false; + } + if (decoded.dst.sdwa_sel != 6u || decoded.dst.omod != 0u || decoded.dst.clamp || + decoded.src0.sdwa_sel != 6u || decoded.src0.sdwa_sext || decoded.src0.negate || + decoded.src0.absolute || decoded.src0.dpp) { + SetError(error, "V_MOVRELD_B32 modifiers are not implemented"); + return false; + } + + Instruction inst; + inst.pc = decoded.pc; + inst.op = Opcode::MoveRelDestU32; + inst.src_count = 2; + const auto m0 = M0Operand(); + if (!LowerRegisterOperand(decoded.dst, &inst.dst, error) || + !LowerSourceOperand(decoded.src0, &inst.src[0], error) || + !LowerSourceOperand(m0, &inst.src[1], error)) { + return false; + } + block->instructions.push_back(inst); + return true; +} + +bool LowerVectorAddCarry(const Decoder::Instruction& decoded, BasicBlock* block, + std::string* error) { + Instruction inst; + inst.pc = decoded.pc; + inst.op = Opcode::IAddCarryU32; + inst.src_count = 3; + if (!LowerRegisterOperand(decoded.dst, &inst.dst, error) || + !LowerRegisterOperand(decoded.dst2, &inst.dst2, error) || + !LowerSourceOperand(decoded.src0, &inst.src[0], error) || + !LowerSourceOperand(decoded.src1, &inst.src[1], error) || + !LowerSourceOperand(decoded.src2, &inst.src[2], error)) { + return false; + } + block->instructions.push_back(inst); + return true; +} + +bool LowerVectorCarryOut(const Decoder::Instruction& decoded, BasicBlock* block, + std::string* error) { + Instruction inst; + inst.pc = decoded.pc; + inst.op = + decoded.opcode == Decoder::Opcode::VAddI32 ? Opcode::IAddCarryU32 : Opcode::ISubBorrowU32; + inst.src_count = decoded.opcode == Decoder::Opcode::VAddI32 ? 3u : 2u; + const auto& src0 = decoded.opcode == Decoder::Opcode::VSubrevI32 ? decoded.src1 : decoded.src0; + const auto& src1 = decoded.opcode == Decoder::Opcode::VSubrevI32 ? decoded.src0 : decoded.src1; + if (!LowerRegisterOperand(decoded.dst, &inst.dst, error) || + !LowerRegisterOperand(decoded.dst2, &inst.dst2, error) || + !LowerSourceOperand(src0, &inst.src[0], error) || + !LowerSourceOperand(src1, &inst.src[1], error)) { + return false; + } + if (decoded.opcode == Decoder::Opcode::VAddI32) { + inst.src[2] = MakeImmediateU32(0); + } + block->instructions.push_back(inst); + return true; +} + +bool LowerVectorMadU64U32(const Decoder::Instruction& decoded, BasicBlock* block, + std::string* error) { + Instruction inst; + inst.pc = decoded.pc; + inst.op = Opcode::UMadU64U32; + inst.src_count = 3; + if (!LowerRegisterOperand(decoded.dst, &inst.dst, error) || + !LowerRegisterOperand(decoded.dst2, &inst.dst2, error) || + !LowerSourceOperand(decoded.src0, &inst.src[0], error) || + !LowerSourceOperand(decoded.src1, &inst.src[1], error) || + !LowerSourceOperand(decoded.src2, &inst.src[2], error)) { + return false; + } + block->instructions.push_back(inst); + return true; +} + +bool LowerVectorMacF32(const Decoder::Instruction& decoded, BasicBlock* block, std::string* error) { + Instruction inst; + inst.pc = decoded.pc; + inst.op = Opcode::FMadF32; + inst.src_count = 3; + if (!LowerRegisterOperand(decoded.dst, &inst.dst, error) || + !LowerSourceOperand(decoded.src0, &inst.src[0], error) || + !LowerSourceOperand(decoded.src1, &inst.src[1], error) || + !LowerSourceOperand(decoded.dst, &inst.src[2], error)) { + return false; + } + block->instructions.push_back(inst); + return true; +} + +bool LowerVectorPkFmacF16(const Decoder::Instruction& decoded, BasicBlock* block, + std::string* error) { + Instruction inst; + inst.pc = decoded.pc; + inst.op = Opcode::PackedFmaF16; + inst.src_count = 3; + if (!LowerRegisterOperand(decoded.dst, &inst.dst, error) || + !LowerSourceOperand(decoded.src0, &inst.src[0], error) || + !LowerSourceOperand(decoded.src1, &inst.src[1], error) || + !LowerSourceOperand(decoded.dst, &inst.src[2], error)) { + return false; + } + block->instructions.push_back(inst); + return true; +} + +bool LowerVectorFmacF16(const Decoder::Instruction& decoded, BasicBlock* block, + std::string* error) { + Instruction inst; + inst.pc = decoded.pc; + inst.op = Opcode::FmaF16; + inst.src_count = 3; + if (!LowerRegisterOperand(decoded.dst, &inst.dst, error) || + !LowerSourceOperand(decoded.src0, &inst.src[0], error) || + !LowerSourceOperand(decoded.src1, &inst.src[1], error) || + !LowerSourceOperand(decoded.dst, &inst.src[2], error)) { + return false; + } + block->instructions.push_back(inst); + return true; +} + +bool LowerVectorDot2cF32F16(const Decoder::Instruction& decoded, BasicBlock* block, + std::string* error) { + Instruction inst; + inst.pc = decoded.pc; + inst.op = Opcode::Dot2AccF32F16; + inst.src_count = 3; + if (!LowerRegisterOperand(decoded.dst, &inst.dst, error) || + !LowerSourceOperand(decoded.src0, &inst.src[0], error) || + !LowerSourceOperand(decoded.src1, &inst.src[1], error)) { + return false; + } + inst.src[2] = inst.dst; + block->instructions.push_back(inst); + return true; +} + +bool LowerVectorByteConvert(const Decoder::Instruction& decoded, BasicBlock* block, + std::string* error) { + const auto byte_index = VectorByteConvertIndex(decoded.opcode); + if (byte_index > 3u) { + SetError(error, "internal IR error: unknown vector byte convert opcode"); + return false; + } + + Instruction inst; + inst.pc = decoded.pc; + inst.op = Opcode::ConvertByteU32ToF32; + inst.src_count = 2; + inst.src[1].kind = OperandKind::ImmediateU32; + inst.src[1].imm = byte_index; + if (!LowerRegisterOperand(decoded.dst, &inst.dst, error) || + !LowerSourceOperand(decoded.src0, &inst.src[0], error)) { + return false; + } + block->instructions.push_back(inst); + return true; +} + +bool LowerVInterpP1F32(const Decoder::Instruction& decoded, BasicBlock* block) { + Instruction inst; + inst.pc = decoded.pc; + inst.op = Opcode::ControlNop; + block->instructions.push_back(inst); + return true; +} + +bool LowerVInterpLoadF32(const Decoder::Instruction& decoded, BasicBlock* block, + std::string* error) { + if (decoded.opcode == Decoder::Opcode::VInterpMovF32 && decoded.src0.value != 2u) { + if (error != nullptr) { + *error = fmt::format("v_interp_mov_f32 mode {} is not implemented at pc 0x{:08x}", + decoded.src0.value, decoded.pc); + } + return false; + } + + Instruction inst; + inst.pc = decoded.pc; + inst.op = Opcode::LoadInputF32; + inst.input_info.attr = decoded.src1.value; + inst.input_info.chan = decoded.src2.value; + if (!LowerRegisterOperand(decoded.dst, &inst.dst, error)) { + return false; + } + block->instructions.push_back(inst); + return true; +} + +ExportTargetKind ExportTargetKindFromTarget(uint32_t target, uint32_t* index) { + if (index != nullptr) { + *index = 0; + } + switch (target) { + case 0x08u: return ExportTargetKind::MrtZ; + case 0x09u: return ExportTargetKind::Null; + case 0x14u: return ExportTargetKind::Primitive; + default: break; + } + if (target <= 0x07u) { + if (index != nullptr) { + *index = target; + } + return ExportTargetKind::Mrt; + } + if (target >= 0x0cu && target <= 0x0fu) { + if (index != nullptr) { + *index = target - 0x0cu; + } + return ExportTargetKind::Position; + } + if (target >= 0x20u && target <= 0x3fu) { + if (index != nullptr) { + *index = target - 0x20u; + } + return ExportTargetKind::Parameter; + } + return ExportTargetKind::Unknown; +} + +bool LowerExportInstruction(const Decoder::Instruction& decoded, BasicBlock* block, + std::string* error) { + uint32_t index = 0; + const auto kind = ExportTargetKindFromTarget(decoded.exp.target, &index); + if (kind == ExportTargetKind::Unknown) { + if (error != nullptr) { + *error = fmt::format("unsupported EXP target 0x{:02x} at pc 0x{:08x}", + decoded.exp.target, decoded.pc); + } + return false; + } + + Instruction inst; + inst.pc = decoded.pc; + inst.op = Opcode::Export; + inst.dst.kind = OperandKind::Null; + inst.src_count = decoded.src_count; + inst.export_info.kind = kind; + inst.export_info.target = decoded.exp.target; + inst.export_info.index = index; + inst.export_info.en = decoded.exp.en; + inst.export_info.done = decoded.exp.done; + inst.export_info.compr = decoded.exp.compr; + inst.export_info.vm = decoded.exp.vm; + + const Decoder::Operand* sources[] = {&decoded.src0, &decoded.src1, &decoded.src2, + &decoded.src3}; + for (uint32_t i = 0; i < inst.src_count && i < 4u; i++) { + if (!LowerSourceOperand(*sources[i], &inst.src[i], error)) { + return false; + } + } + + block->instructions.push_back(inst); + return true; +} + +bool LowerControlMarker(const Decoder::Instruction& decoded, BasicBlock* block, Opcode op, + bool has_imm, std::string* error) { + Instruction inst; + inst.pc = decoded.pc; + inst.op = op; + inst.dst.kind = OperandKind::Null; + inst.src_count = has_imm ? 1u : 0u; + if (has_imm && !LowerSourceOperand(decoded.src0, &inst.src[0], error)) { + return false; + } + block->instructions.push_back(inst); + return true; +} + +bool LowerControlInstruction(const Decoder::Instruction& decoded, BasicBlock* block, + std::string* error) { + switch (decoded.opcode) { + case Decoder::Opcode::SNop: + return LowerControlMarker(decoded, block, Opcode::ControlNop, true, error); + case Decoder::Opcode::SWaitcnt: + return LowerControlMarker(decoded, block, Opcode::Waitcnt, true, error); + case Decoder::Opcode::SBarrier: + return LowerControlMarker(decoded, block, Opcode::Barrier, false, error); + case Decoder::Opcode::SSendmsg: + return LowerControlMarker(decoded, block, Opcode::Sendmsg, true, error); + case Decoder::Opcode::SSetregB32: + case Decoder::Opcode::SSleep: + return LowerControlMarker(decoded, block, Opcode::ControlNop, true, error); + case Decoder::Opcode::STtraceData: + return LowerControlMarker(decoded, block, Opcode::TtraceData, true, error); + case Decoder::Opcode::SInstPrefetch: + return LowerControlMarker(decoded, block, Opcode::InstPrefetch, true, error); + default: return false; + } +} + +bool LowerScalarGetpcB64(const Decoder::Instruction& decoded, BasicBlock* block, + std::string* error) { + if (block == nullptr) { + SetError(error, "internal IR error: missing block for s_getpc_b64"); + return false; + } + + for (uint32_t i = 0; i < 2u; i++) { + Instruction inst; + inst.pc = decoded.pc; + inst.op = Opcode::MoveU32; + inst.src[0] = i == 0u ? MakePcRelativeU32(decoded.pc + 4u) : MakeImmediateU32(0u); + inst.src_count = 1; + if (!LowerRegisterOperand(OffsetDecodedRegister(decoded.dst, i), &inst.dst, error)) { + return false; + } + block->instructions.push_back(inst); + } + return true; +} + +bool IsScalarSaveexecOpcode(Decoder::Opcode opcode) { + switch (opcode) { + case Decoder::Opcode::SAndSaveexecB32: + case Decoder::Opcode::SAndn1SaveexecB32: + case Decoder::Opcode::SAndSaveexecB64: + case Decoder::Opcode::SOrn2SaveexecB64: + case Decoder::Opcode::SAndn1SaveexecB64: return true; + default: return false; + } +} + +SaveexecMode ScalarSaveexecMode(Decoder::Opcode opcode) { + switch (opcode) { + case Decoder::Opcode::SOrn2SaveexecB64: return SaveexecMode::Orn2; + case Decoder::Opcode::SAndn1SaveexecB32: + case Decoder::Opcode::SAndn1SaveexecB64: return SaveexecMode::Andn1; + default: return SaveexecMode::And; + } +} + +bool LowerScalarSaveexec(const Decoder::Instruction& decoded, BasicBlock* block, + std::string* error) { + Instruction inst; + inst.pc = decoded.pc; + inst.op = (decoded.opcode == Decoder::Opcode::SAndSaveexecB32 || + decoded.opcode == Decoder::Opcode::SAndn1SaveexecB32) + ? Opcode::SaveexecB32 + : Opcode::SaveexecB64; + inst.saveexec_mode = ScalarSaveexecMode(decoded.opcode); + inst.src_count = 1; + if (!LowerRegisterOperand(decoded.dst, &inst.dst, error) || + !LowerSourceOperand(decoded.src0, &inst.src[0], error)) { + return false; + } + block->instructions.push_back(inst); + return true; +} + +bool IsControlOpcode(Decoder::Opcode opcode) { + switch (opcode) { + case Decoder::Opcode::SNop: + case Decoder::Opcode::SWaitcnt: + case Decoder::Opcode::SBarrier: + case Decoder::Opcode::SSendmsg: + case Decoder::Opcode::SSetregB32: + case Decoder::Opcode::SSleep: + case Decoder::Opcode::STtraceData: + case Decoder::Opcode::SInstPrefetch: return true; + default: return false; + } +} + +bool IsTerminatorOpcode(Decoder::Opcode opcode) { + switch (opcode) { + case Decoder::Opcode::SEndpgm: + case Decoder::Opcode::SBranch: + case Decoder::Opcode::SCbranchScc0: + case Decoder::Opcode::SCbranchScc1: + case Decoder::Opcode::SCbranchVccz: + case Decoder::Opcode::SCbranchVccnz: + case Decoder::Opcode::SCbranchExecz: + case Decoder::Opcode::SCbranchExecnz: + case Decoder::Opcode::SSetpcB64: return true; + default: return false; + } +} + +bool LowerImplemented(const Decoder::Instruction& decoded, BasicBlock* block, std::string* error); + +bool LowerDecodedInstruction(const Decoder::Instruction& inst, BasicBlock* block, + std::string* error) { + switch (inst.opcode) { + case Decoder::Opcode::Unsupported: + if (error != nullptr) { + *error = fmt::format("unsupported decoded instruction: {}", + Decoder::InstructionToString(inst).c_str()); + } + return false; + case Decoder::Opcode::SGetpcB64: return LowerScalarGetpcB64(inst, block, error); + case Decoder::Opcode::VNop: return true; + case Decoder::Opcode::Exp: return LowerExportInstruction(inst, block, error); + case Decoder::Opcode::SCselectB32: return LowerScalarSelect(inst, block, error); + case Decoder::Opcode::SCselectB64: return LowerScalarSelect64(inst, block, error); + case Decoder::Opcode::SAddU32: return LowerScalarAddCarry(inst, block, false, error); + case Decoder::Opcode::SAddcU32: return LowerScalarAddCarry(inst, block, true, error); + case Decoder::Opcode::SSubbU32: return LowerScalarSubBorrowCarry(inst, block, error); + case Decoder::Opcode::SSubU32: + return LowerScalarBinaryScc(inst, block, Opcode::ScalarSubBorrowU32, error); + case Decoder::Opcode::SBitset0B32: return LowerScalarBitset0B32(inst, block, error); + case Decoder::Opcode::SBitset1B32: return LowerScalarBitset1B32(inst, block, error); + case Decoder::Opcode::SAddI32: + return LowerScalarBinaryScc(inst, block, Opcode::ScalarSignedAddOverflowI32, error); + case Decoder::Opcode::SSubI32: + return LowerScalarBinaryScc(inst, block, Opcode::ScalarSignedSubOverflowI32, error); + case Decoder::Opcode::VCndmaskB32: return LowerVectorCndmask(inst, block, error); + case Decoder::Opcode::VMovB32: return LowerVectorMoveB32(inst, block, error); + case Decoder::Opcode::VMovreldB32: return LowerVectorMoveRelDestination(inst, block, error); + case Decoder::Opcode::VMovrelsB32: return LowerVectorMoveRelSource(inst, block, error); + case Decoder::Opcode::VAddcU32: return LowerVectorAddCarry(inst, block, error); + case Decoder::Opcode::VMadU64U32: return LowerVectorMadU64U32(inst, block, error); + case Decoder::Opcode::VMacF32: return LowerVectorMacF32(inst, block, error); + case Decoder::Opcode::VPkFmacF16: return LowerVectorPkFmacF16(inst, block, error); + case Decoder::Opcode::VFmacF16: return LowerVectorFmacF16(inst, block, error); + case Decoder::Opcode::VDot2cF32F16: return LowerVectorDot2cF32F16(inst, block, error); + case Decoder::Opcode::VInterpP1F32: return LowerVInterpP1F32(inst, block); + case Decoder::Opcode::VInterpP2F32: + case Decoder::Opcode::VInterpMovF32: return LowerVInterpLoadF32(inst, block, error); + default: break; + } + + if (IsTerminatorOpcode(inst.opcode)) { + return true; + } + if (IsControlOpcode(inst.opcode)) { + return LowerControlInstruction(inst, block, error); + } + if (IsScalarSaveexecOpcode(inst.opcode)) { + return LowerScalarSaveexec(inst, block, error); + } + if (IsMemoryOpcode(inst.opcode)) { + return LowerMemoryInstruction(inst, block, error); + } + if (ScalarShiftLeftAddAmount(inst.opcode) != 0) { + return LowerScalarShiftLeftAdd(inst, block, error); + } + if (IsScalarMinMaxOpcode(inst.opcode)) { + return LowerScalarMinMaxWithScc(inst, block, error); + } + if (IsVectorCarryOutOpcode(inst.opcode)) { + return LowerVectorCarryOut(inst, block, error); + } + if (VectorByteConvertIndex(inst.opcode) <= 3u) { + return LowerVectorByteConvert(inst, block, error); + } + if (!IsImplemented(inst.opcode)) { + if (error != nullptr) { + *error = fmt::format("decoded opcode has no IR lowering yet: {}", + Decoder::InstructionToString(inst).c_str()); + } + return false; + } + return LowerImplemented(inst, block, error); +} + + +bool LowerImplemented(const Decoder::Instruction& decoded, BasicBlock* block, std::string* error) { + Instruction inst; + inst.pc = decoded.pc; + inst.op = LookupIrOpcode(decoded.opcode); + inst.src_count = decoded.src_count; + + if (!LowerRegisterOperand(decoded.dst, &inst.dst, error)) { + return false; + } + ApplyDppDestinationMask(decoded, inst.dst); + for (uint32_t i = 0; i < decoded.src_count && i < 3u; i++) { + const auto src_index = + (IsReversedBinary(decoded.opcode) && decoded.src_count == 2u) ? 1u - i : i; + if (!LowerSourceOperand(DecodedSourceAt(decoded, src_index), &inst.src[i], error)) { + return false; + } + } + + block->instructions.push_back(inst); + return AppendScalarResultSccNonZero(decoded, block, error); +} + +} // namespace + +bool LowerProgram(const Decoder::Program& decoded, const CFG::Graph& cfg, ShaderType stage, + uint32_t wave_size, Program* program, std::string* error) { + if (program == nullptr) { + SetError(error, "invalid IR output"); + return false; + } + if (wave_size != 32u && wave_size != 64u) { + EXIT("invalid shader wave size: %u\n", wave_size); + } + + *program = {}; + program->stage = stage; + program->wave_size = wave_size; + program->dispatcher_fallback = cfg.irreducible; + program->cfg_failure_kind = cfg.failure_kind; + program->fallback_reason = cfg.unsupported_reason; + program->blocks.clear(); + + if (cfg.blocks.empty()) { + SetError(error, "cannot lower empty CFG"); + return false; + } + + for (const auto& cfg_block: cfg.blocks) { + BasicBlock block; + block.id = cfg_block.id; + block.start_pc = cfg_block.start_pc; + block.end_pc = cfg_block.end_pc; + block.inst_begin = cfg_block.inst_begin; + block.inst_end = cfg_block.inst_end; + block.predecessors = cfg_block.predecessors; + block.successors = cfg_block.successors; + block.terminator = cfg_block.terminator; + + for (uint32_t i = cfg_block.inst_begin; i < cfg_block.inst_end; i++) { + if (i >= decoded.instructions.size()) { + SetError(error, "CFG block references instruction outside decoded program"); + return false; + } + if (std::find(cfg.code_table_load_pcs.begin(), cfg.code_table_load_pcs.end(), + decoded.instructions[i].pc) != cfg.code_table_load_pcs.end()) { + continue; + } + if (!LowerDecodedInstruction(decoded.instructions[i], &block, error)) { + return false; + } + } + + program->blocks.push_back(std::move(block)); + } + + return true; +} + +} // namespace Libs::Graphics::ShaderRecompiler::IR diff --git a/src/graphics/shader/recompiler/shaderIR/ShaderIRInternal.h b/src/graphics/shader/recompiler/shaderIR/ShaderIRInternal.h new file mode 100644 index 000000000..708151679 --- /dev/null +++ b/src/graphics/shader/recompiler/shaderIR/ShaderIRInternal.h @@ -0,0 +1,57 @@ +#ifndef EMULATOR_SRC_GRAPHICS_SHADER_RECOMPILER_SHADERIR_SHADERIRINTERNAL_H_ +#define EMULATOR_SRC_GRAPHICS_SHADER_RECOMPILER_SHADERIR_SHADERIRINTERNAL_H_ + +#include "graphics/shader/recompiler/ShaderIR.h" + +namespace Libs::Graphics::ShaderRecompiler::IR { + +void SetError(std::string* error, const char* message); + +Opcode LookupIrOpcode(Decoder::Opcode opcode); +bool IsImplemented(Decoder::Opcode opcode); +bool IsReversedBinary(Decoder::Opcode opcode); +bool IsVectorCarryOutOpcode(Decoder::Opcode opcode); +bool ScalarResultWritesSccNonZero(Decoder::Opcode opcode); +bool ScalarResultIs64Bit(Decoder::Opcode opcode); + +Operand MakeSccOperand(); +Operand MakeImmediateU32(uint32_t value); +Operand MakePcRelativeU32(uint32_t value); +bool AppendScalarResultSccNonZero(const Decoder::Instruction& decoded, BasicBlock* block, + std::string* error); +uint32_t VectorByteConvertIndex(Decoder::Opcode opcode); +void CopyOperandModifiers(const Decoder::Operand& decoded, Operand& operand); +bool LowerRegisterOperand(const Decoder::Operand& decoded, Operand* operand, + std::string* error); +bool LowerSourceOperand(const Decoder::Operand& decoded, Operand* operand, std::string* error); +void ApplyDppDestinationMask(const Decoder::Instruction& decoded, Operand& dst); +uint32_t ResourceIndexFromOperand(const Decoder::Operand& operand); +MemoryInfo MemoryInfoFromDecoded(const Decoder::Instruction& decoded, ResourceKind kind); +uint32_t RawScalarLoadBase(const Decoder::Operand& operand); +void ClearRegisterOffsetModifiers(Decoder::Operand& operand); +bool TryGetScalarDestinationCode(const Decoder::Operand& operand, uint32_t& code); +bool TryOffsetScalarDestination(const Decoder::Operand& operand, uint32_t index, + Decoder::Operand* result); +Decoder::Operand OffsetDecodedRegister(const Decoder::Operand& operand, uint32_t index); +MemoryInfo OffsetMemoryInfo(const Decoder::Instruction& decoded, ResourceKind kind, + uint32_t dword_index); +uint32_t TypedBufferComponentOffsetBytes(const Decoder::Instruction& decoded, + uint32_t component_index); +uint32_t TypedBufferFormatComponentCount(const Decoder::Instruction& decoded); +bool LowerMoveImmediateU32(uint32_t pc, const Decoder::Operand& dst, uint32_t value, + BasicBlock* block, std::string* error); +MemoryInfo OffsetBufferMemoryInfo(const Decoder::Instruction& decoded, + uint32_t component_index); +MemoryInfo ByteOffsetMemoryInfo(const Decoder::Instruction& decoded, ResourceKind kind, + uint32_t byte_offset); +ResourceKind FlatSegmentResourceKind(uint32_t segment); +const Decoder::Operand& DecodedSourceAt(const Decoder::Instruction& decoded, uint32_t index); +Decoder::Operand M0Operand(); + +bool IsMemoryOpcode(Decoder::Opcode opcode); +bool LowerMemoryInstruction(const Decoder::Instruction& decoded, BasicBlock* block, + std::string* error); + +} // namespace Libs::Graphics::ShaderRecompiler::IR + +#endif // EMULATOR_SRC_GRAPHICS_SHADER_RECOMPILER_SHADERIR_SHADERIRINTERNAL_H_ diff --git a/src/graphics/shader/recompiler/shaderIR/ShaderIRLog.cpp b/src/graphics/shader/recompiler/shaderIR/ShaderIRLog.cpp new file mode 100644 index 000000000..6ce858884 --- /dev/null +++ b/src/graphics/shader/recompiler/shaderIR/ShaderIRLog.cpp @@ -0,0 +1,559 @@ +#include "graphics/shader/recompiler/ShaderIR.h" + +#include "graphics/shader/recompiler/shaderIR/ShaderIRInternal.h" +#include "common/assert.h" +#include "graphics/shader/recompiler/BufferFormat.h" + +#include +#include +#include + +namespace Libs::Graphics::ShaderRecompiler::IR { + +std::string RegisterToString(Register reg) { + switch (reg.file) { + case RegisterFile::Scalar: return fmt::format("s{}", reg.index); + case RegisterFile::Vector: return fmt::format("v{}", reg.index); + case RegisterFile::Vcc: + return reg.index == 0 ? std::string("vcc_lo") : std::string("vcc_hi"); + case RegisterFile::Exec: + return reg.index == 0 ? std::string("exec_lo") : std::string("exec_hi"); + case RegisterFile::Scc: + if (reg.index == static_cast(Decoder::OperandKind::Scc)) { + return "scc"; + } + if (reg.index == static_cast(Decoder::OperandKind::VccZ)) { + return "vccz"; + } + if (reg.index == static_cast(Decoder::OperandKind::ExecZ)) { + return "execz"; + } + return fmt::format("scc_like_{}", reg.index); + case RegisterFile::M0: return "m0"; + default: return "unknown_reg"; + } +} + +std::string OperandToString(const Operand& operand) { + std::string text; + switch (operand.kind) { + case OperandKind::Register: text = RegisterToString(operand.reg); break; + case OperandKind::ImmediateU32: text = fmt::format("0x{:08x}", operand.imm); break; + case OperandKind::PcRelativeU32: text = fmt::format("pc_rel(0x{:08x})", operand.imm); break; + case OperandKind::Null: text = "null"; break; + default: text = "unknown"; break; + } + if (operand.sdwa_sel != 6 || operand.sdwa_sext) { + text += fmt::format(".sdwa(sel={},sext={})", operand.sdwa_sel, operand.sdwa_sext ? 1u : 0u); + } + if (operand.absolute) { + text += ".abs"; + } + if (operand.negate) { + text += ".neg"; + } + if (operand.op_sel || operand.op_sel_hi || operand.negate_hi) { + text += fmt::format(".opsel(lo={},hi={},neghi={})", operand.op_sel ? 1u : 0u, + operand.op_sel_hi ? 1u : 0u, operand.negate_hi ? 1u : 0u); + } + if (operand.omod != 0) { + text += fmt::format(".omod({})", operand.omod); + } + if (operand.clamp) { + text += ".clamp"; + } + if (operand.dpp) { + text += fmt::format(".dpp(ctrl=0x{:x},fi={},bc={})", operand.dpp_ctrl, + operand.dpp_fetch_inactive ? 1u : 0u, operand.dpp_bound_ctrl ? 1u : 0u); + } + return text; +} + +std::string ResourceKindToString(ResourceKind kind) { + switch (kind) { + case ResourceKind::ScalarBuffer: return "scalar_buffer"; + case ResourceKind::Buffer: return "buffer"; + case ResourceKind::Flat: return "flat"; + case ResourceKind::Global: return "global"; + case ResourceKind::Scratch: return "scratch"; + case ResourceKind::Lds: return "lds"; + case ResourceKind::Gds: return "gds"; + case ResourceKind::Image: return "image"; + case ResourceKind::ImageUint: return "image_uint"; + case ResourceKind::StorageImage: return "storage_image"; + case ResourceKind::StorageImageUint: return "storage_image_uint"; + case ResourceKind::Sampler: return "sampler"; + default: return "none"; + } +} + +const char* ImageDimensionToString(Decoder::ImageDimension dimension) { + switch (dimension) { + case Decoder::ImageDimension::Dim2D: return "2d"; + case Decoder::ImageDimension::Dim3D: return "3d"; + case Decoder::ImageDimension::Dim2DArray: return "2d_array"; + default: return "unknown"; + } +} + +std::string MemoryInfoToString(const MemoryInfo& mem) { + if (mem.kind == ResourceKind::None) { + return ""; + } + std::string text = fmt::format( + " ; {} resource={} sampler={} offset={} offset2={} dmask=0x{:x} data_dwords={} " + "data_bits={} component={}/{} dfmt={} nfmt={} signed={} typed={} formatted={} segment={} " + "glc={} slc={} idxen={} offen={} image_flags=0x{:x} image_dim={} image_addr={} " + "image_mip={}", + ResourceKindToString(mem.kind).c_str(), mem.resource, mem.sampler, mem.offset, + mem.secondary_offset, mem.dmask, mem.data_dwords, mem.data_bits, mem.component_index, + mem.component_count, mem.data_format, mem.number_format, mem.data_signed ? 1u : 0u, + mem.typed ? 1u : 0u, mem.formatted ? 1u : 0u, mem.memory_segment, mem.glc ? 1u : 0u, + mem.slc ? 1u : 0u, mem.idxen ? 1u : 0u, mem.offen ? 1u : 0u, mem.image_sample_flags, + ImageDimensionToString(mem.image_dimension), mem.image_address_components, + mem.image_has_mip ? 1u : 0u); + if (mem.image_nsa_dwords != 0) { + text += fmt::format(" image_nsa_dwords={} image_nsa_addr=", mem.image_nsa_dwords); + const auto count = + std::min(mem.image_nsa_dwords * 4u, Decoder::MaxImageNsaAddressComponents); + for (uint32_t i = 0; i < count; i++) { + if (i != 0) { + text += ","; + } + text += fmt::format("{}", mem.image_nsa_addr[i]); + } + } + return text; +} + +std::string ExportInfoToString(const ExportInfo& exp) { + return fmt::format(" ; target={} index={} raw=0x{:02x} en=0x{:x} done={} compr={} vm={}", + ExportTargetKindToString(exp.kind).c_str(), exp.index, exp.target, exp.en, + exp.done ? 1u : 0u, exp.compr ? 1u : 0u, exp.vm ? 1u : 0u); +} + +std::string InputInfoToString(Opcode op, const InputInfo& input) { + if (op != Opcode::LoadInputF32) { + return ""; + } + return fmt::format(" ; input_attr={} input_chan={}", input.attr, input.chan); +} + +std::string TerminatorToString(const CFG::Terminator& term) { + return fmt::format( + "term={} cond={} true={} false={} merge={} continue={} loop={} indirect_sgpr={} " + "indirect_selector={} indirect_targets={} selector_values={}", + static_cast(term.kind), CFG::BranchConditionToString(term.condition).c_str(), + term.true_block, term.false_block, term.merge_block, term.continue_block, + term.loop_header ? 1u : 0u, term.indirect_pc_sgpr, term.indirect_selector_code, + static_cast(term.indirect_targets.size()), + static_cast(term.indirect_selector_values.size())); +} + +std::string ExportSourcesToString(const Instruction& inst) { + std::string text; + for (uint32_t i = 0; i < inst.src_count && i < 4u; i++) { + text += i == 0 ? " " : ", "; + text += OperandToString(inst.src[i]); + } + return text; +} + +std::string InstructionToString(const Instruction& inst) { + std::string text = fmt::format("0x{:08x}: ", inst.pc); + if (inst.op == Opcode::Export) { + text += "Export"; + text += ExportSourcesToString(inst); + text += ExportInfoToString(inst.export_info); + return text; + } + switch (inst.op) { + case Opcode::ControlNop: text += "ControlNop"; break; + case Opcode::Waitcnt: text += "Waitcnt"; break; + case Opcode::Barrier: text += "Barrier"; break; + case Opcode::Sendmsg: text += "Sendmsg"; break; + case Opcode::TtraceData: text += "TtraceData"; break; + case Opcode::InstPrefetch: text += "InstPrefetch"; break; + case Opcode::MoveU32: text += "MoveU32"; break; + case Opcode::MoveF32Bits: text += "MoveF32Bits"; break; + case Opcode::MoveRelDestU32: text += "MoveRelDestU32"; break; + case Opcode::MoveRelSourceU32: text += "MoveRelSourceU32"; break; + case Opcode::MoveU64: text += "MoveU64"; break; + case Opcode::WqmB64: text += "WqmB64"; break; + case Opcode::SaveexecB32: text += "SaveexecB32"; break; + case Opcode::SaveexecB64: text += "SaveexecB64"; break; + case Opcode::ReadFirstLaneU32: text += "ReadFirstLaneU32"; break; + case Opcode::ReadLaneU32: text += "ReadLaneU32"; break; + case Opcode::WriteLaneU32: text += "WriteLaneU32"; break; + case Opcode::Permlane16B32: text += "Permlane16B32"; break; + case Opcode::Permlanex16B32: text += "Permlanex16B32"; break; + case Opcode::AbsI32: text += "AbsI32"; break; + case Opcode::IAddU32: text += "IAddU32"; break; + case Opcode::IAddCarryU32: text += "IAddCarryU32"; break; + case Opcode::ISubBorrowU32: text += "ISubBorrowU32"; break; + case Opcode::ScalarAddCarryU32: text += "ScalarAddCarryU32"; break; + case Opcode::ScalarSubBorrowU32: text += "ScalarSubBorrowU32"; break; + case Opcode::ScalarSubBorrowCarryU32: text += "ScalarSubBorrowCarryU32"; break; + case Opcode::ScalarSignedAddOverflowI32: text += "ScalarSignedAddOverflowI32"; break; + case Opcode::ScalarSignedSubOverflowI32: text += "ScalarSignedSubOverflowI32"; break; + case Opcode::ScalarShiftLeftAddCarryU32: text += "ScalarShiftLeftAddCarryU32"; break; + case Opcode::ISubU32: text += "ISubU32"; break; + case Opcode::IMulU32: text += "IMulU32"; break; + case Opcode::UMulHighU32: text += "UMulHighU32"; break; + case Opcode::SMulHighI32: text += "SMulHighI32"; break; + case Opcode::IMadI24U32: text += "IMadI24U32"; break; + case Opcode::UMadU24U32: text += "UMadU24U32"; break; + case Opcode::UMadU64U32: text += "UMadU64U32"; break; + case Opcode::SadU32: text += "SadU32"; break; + case Opcode::IAdd3U32: text += "IAdd3U32"; break; + case Opcode::IMulI24U32: text += "IMulI24U32"; break; + case Opcode::UMulU24U32: text += "UMulU24U32"; break; + case Opcode::IMinI32: text += "IMinI32"; break; + case Opcode::IMaxI32: text += "IMaxI32"; break; + case Opcode::IMin3I32: text += "IMin3I32"; break; + case Opcode::IMax3I32: text += "IMax3I32"; break; + case Opcode::IMed3I32: text += "IMed3I32"; break; + case Opcode::UMinU32: text += "UMinU32"; break; + case Opcode::UMaxU32: text += "UMaxU32"; break; + case Opcode::UMin3U32: text += "UMin3U32"; break; + case Opcode::UMax3U32: text += "UMax3U32"; break; + case Opcode::UMed3U32: text += "UMed3U32"; break; + case Opcode::BitwiseAndU32: text += "BitwiseAndU32"; break; + case Opcode::BitwiseAndU64: text += "BitwiseAndU64"; break; + case Opcode::BitwiseAndNotU32: text += "BitwiseAndNotU32"; break; + case Opcode::BitwiseAndNotU64: text += "BitwiseAndNotU64"; break; + case Opcode::BitwiseOrU32: text += "BitwiseOrU32"; break; + case Opcode::BitwiseOrU64: text += "BitwiseOrU64"; break; + case Opcode::BitwiseOrNotU32: text += "BitwiseOrNotU32"; break; + case Opcode::BitwiseOrNotU64: text += "BitwiseOrNotU64"; break; + case Opcode::BitwiseAndOrU32: text += "BitwiseAndOrU32"; break; + case Opcode::BitwiseOr3U32: text += "BitwiseOr3U32"; break; + case Opcode::BitwiseXorU32: text += "BitwiseXorU32"; break; + case Opcode::BitwiseXorU64: text += "BitwiseXorU64"; break; + case Opcode::BitwiseXor3U32: text += "BitwiseXor3U32"; break; + case Opcode::BitwiseNandU32: text += "BitwiseNandU32"; break; + case Opcode::BitwiseNandU64: text += "BitwiseNandU64"; break; + case Opcode::BitwiseNorU32: text += "BitwiseNorU32"; break; + case Opcode::BitwiseNorU64: text += "BitwiseNorU64"; break; + case Opcode::BitwiseXnorU32: text += "BitwiseXnorU32"; break; + case Opcode::BitwiseXnorU64: text += "BitwiseXnorU64"; break; + case Opcode::BitwiseNotU32: text += "BitwiseNotU32"; break; + case Opcode::BitwiseNotU64: text += "BitwiseNotU64"; break; + case Opcode::BitClearU32: text += "BitClearU32"; break; + case Opcode::BitSetU32: text += "BitSetU32"; break; + case Opcode::BitReverseU32: text += "BitReverseU32"; break; + case Opcode::BitCountU32: text += "BitCountU32"; break; + case Opcode::BitCountU64: text += "BitCountU64"; break; + case Opcode::BitReplicateB64B32: text += "BitReplicateB64B32"; break; + case Opcode::BitCountAddU32: text += "BitCountAddU32"; break; + case Opcode::MaskedBitCountLowU32: text += "MaskedBitCountLowU32"; break; + case Opcode::MaskedBitCountHighU32: text += "MaskedBitCountHighU32"; break; + case Opcode::FindLsbU32: text += "FindLsbU32"; break; + case Opcode::FindMsbFromHighU32: text += "FindMsbFromHighU32"; break; + case Opcode::FindMsbFromHighU64: text += "FindMsbFromHighU64"; break; + case Opcode::BitFieldMaskU32: text += "BitFieldMaskU32"; break; + case Opcode::BitFieldMaskU64: text += "BitFieldMaskU64"; break; + case Opcode::BitFieldExtractU32: text += "BitFieldExtractU32"; break; + case Opcode::BitFieldExtractU64: text += "BitFieldExtractU64"; break; + case Opcode::BitFieldExtract3U32: text += "BitFieldExtract3U32"; break; + case Opcode::BitFieldExtract3I32: text += "BitFieldExtract3I32"; break; + case Opcode::BitFieldInsertSelectU32: text += "BitFieldInsertSelectU32"; break; + case Opcode::BitCompare0B32: text += "BitCompare0B32"; break; + case Opcode::BitCompare1B32: text += "BitCompare1B32"; break; + case Opcode::AlignBitU32: text += "AlignBitU32"; break; + case Opcode::ShiftLeftAddU32: text += "ShiftLeftAddU32"; break; + case Opcode::AddShiftLeftU32: text += "AddShiftLeftU32"; break; + case Opcode::XorAddU32: text += "XorAddU32"; break; + case Opcode::ShiftLeftOrU32: text += "ShiftLeftOrU32"; break; + case Opcode::ShiftLeftLogicalU32: text += "ShiftLeftLogicalU32"; break; + case Opcode::ShiftLeftLogicalU64: text += "ShiftLeftLogicalU64"; break; + case Opcode::ShiftLeftLogicalU16: text += "ShiftLeftLogicalU16"; break; + case Opcode::ShiftRightLogicalU32: text += "ShiftRightLogicalU32"; break; + case Opcode::ShiftRightLogicalU64: text += "ShiftRightLogicalU64"; break; + case Opcode::ShiftRightLogicalU16: text += "ShiftRightLogicalU16"; break; + case Opcode::ShiftRightArithmeticI32: text += "ShiftRightArithmeticI32"; break; + case Opcode::ShiftRightArithmeticI16: text += "ShiftRightArithmeticI16"; break; + case Opcode::SelectU32: text += "SelectU32"; break; + case Opcode::SelectMaskU32: text += "SelectMaskU32"; break; + case Opcode::SelectF32Bits: text += "SelectF32Bits"; break; + case Opcode::SelectMaskF32Bits: text += "SelectMaskF32Bits"; break; + case Opcode::SelectU64: text += "SelectU64"; break; + case Opcode::PackLowLowU16: text += "PackLowLowU16"; break; + case Opcode::PackLowHighU16: text += "PackLowHighU16"; break; + case Opcode::PackHighHighU16: text += "PackHighHighU16"; break; + case Opcode::CompareFalse: text += "CompareFalse"; break; + case Opcode::CompareTrue: text += "CompareTrue"; break; + case Opcode::CompareEqU32: text += "CompareEqU32"; break; + case Opcode::CompareNeU32: text += "CompareNeU32"; break; + case Opcode::CompareGtU32: text += "CompareGtU32"; break; + case Opcode::CompareGeU32: text += "CompareGeU32"; break; + case Opcode::CompareLtU32: text += "CompareLtU32"; break; + case Opcode::CompareLeU32: text += "CompareLeU32"; break; + case Opcode::CompareNeU64: text += "CompareNeU64"; break; + case Opcode::CompareMaskEqU32: text += "CompareMaskEqU32"; break; + case Opcode::CompareMaskNeU32: text += "CompareMaskNeU32"; break; + case Opcode::CompareMaskGtU32: text += "CompareMaskGtU32"; break; + case Opcode::CompareMaskGeU32: text += "CompareMaskGeU32"; break; + case Opcode::CompareMaskLtU32: text += "CompareMaskLtU32"; break; + case Opcode::CompareMaskLeU32: text += "CompareMaskLeU32"; break; + case Opcode::CompareEqI32: text += "CompareEqI32"; break; + case Opcode::CompareNeI32: text += "CompareNeI32"; break; + case Opcode::CompareGtI32: text += "CompareGtI32"; break; + case Opcode::CompareGeI32: text += "CompareGeI32"; break; + case Opcode::CompareLtI32: text += "CompareLtI32"; break; + case Opcode::CompareLeI32: text += "CompareLeI32"; break; + case Opcode::CompareEqI16: text += "CompareEqI16"; break; + case Opcode::CompareNeI16: text += "CompareNeI16"; break; + case Opcode::CompareGtI16: text += "CompareGtI16"; break; + case Opcode::CompareGeI16: text += "CompareGeI16"; break; + case Opcode::CompareLtI16: text += "CompareLtI16"; break; + case Opcode::CompareLeI16: text += "CompareLeI16"; break; + case Opcode::CompareMaskEqI32: text += "CompareMaskEqI32"; break; + case Opcode::CompareMaskNeI32: text += "CompareMaskNeI32"; break; + case Opcode::CompareMaskGtI32: text += "CompareMaskGtI32"; break; + case Opcode::CompareMaskGeI32: text += "CompareMaskGeI32"; break; + case Opcode::CompareMaskLtI32: text += "CompareMaskLtI32"; break; + case Opcode::CompareMaskLeI32: text += "CompareMaskLeI32"; break; + case Opcode::CompareEqU16: text += "CompareEqU16"; break; + case Opcode::CompareNeU16: text += "CompareNeU16"; break; + case Opcode::CompareGtU16: text += "CompareGtU16"; break; + case Opcode::CompareGeU16: text += "CompareGeU16"; break; + case Opcode::CompareLtU16: text += "CompareLtU16"; break; + case Opcode::CompareLeU16: text += "CompareLeU16"; break; + case Opcode::CompareEqF32: text += "CompareEqF32"; break; + case Opcode::CompareNeF32: text += "CompareNeF32"; break; + case Opcode::CompareGtF32: text += "CompareGtF32"; break; + case Opcode::CompareGeF32: text += "CompareGeF32"; break; + case Opcode::CompareLtF32: text += "CompareLtF32"; break; + case Opcode::CompareLeF32: text += "CompareLeF32"; break; + case Opcode::CompareOrderedF32: text += "CompareOrderedF32"; break; + case Opcode::CompareUnorderedF32: text += "CompareUnorderedF32"; break; + case Opcode::CompareUnordEqF32: text += "CompareUnordEqF32"; break; + case Opcode::CompareUnordNeF32: text += "CompareUnordNeF32"; break; + case Opcode::CompareUnordGtF32: text += "CompareUnordGtF32"; break; + case Opcode::CompareUnordGeF32: text += "CompareUnordGeF32"; break; + case Opcode::CompareUnordLtF32: text += "CompareUnordLtF32"; break; + case Opcode::CompareUnordLeF32: text += "CompareUnordLeF32"; break; + case Opcode::CompareClassF32: text += "CompareClassF32"; break; + case Opcode::CompareEqF16: text += "CompareEqF16"; break; + case Opcode::CompareNeF16: text += "CompareNeF16"; break; + case Opcode::CompareGtF16: text += "CompareGtF16"; break; + case Opcode::CompareGeF16: text += "CompareGeF16"; break; + case Opcode::CompareLtF16: text += "CompareLtF16"; break; + case Opcode::CompareLeF16: text += "CompareLeF16"; break; + case Opcode::CompareUnordNeF16: text += "CompareUnordNeF16"; break; + case Opcode::CompareMaskEqF16: text += "CompareMaskEqF16"; break; + case Opcode::CompareMaskNeF16: text += "CompareMaskNeF16"; break; + case Opcode::CompareMaskGtF16: text += "CompareMaskGtF16"; break; + case Opcode::CompareMaskGeF16: text += "CompareMaskGeF16"; break; + case Opcode::CompareMaskLtF16: text += "CompareMaskLtF16"; break; + case Opcode::CompareMaskLeF16: text += "CompareMaskLeF16"; break; + case Opcode::CompareMaskUnordNeF16: text += "CompareMaskUnordNeF16"; break; + case Opcode::CompareMaskUnordGeF16: text += "CompareMaskUnordGeF16"; break; + case Opcode::CompareMaskEqF32: text += "CompareMaskEqF32"; break; + case Opcode::CompareMaskNeF32: text += "CompareMaskNeF32"; break; + case Opcode::CompareMaskGtF32: text += "CompareMaskGtF32"; break; + case Opcode::CompareMaskGeF32: text += "CompareMaskGeF32"; break; + case Opcode::CompareMaskLtF32: text += "CompareMaskLtF32"; break; + case Opcode::CompareMaskLeF32: text += "CompareMaskLeF32"; break; + case Opcode::CompareMaskUnordEqF32: text += "CompareMaskUnordEqF32"; break; + case Opcode::CompareMaskUnordNeF32: text += "CompareMaskUnordNeF32"; break; + case Opcode::CompareMaskUnordGtF32: text += "CompareMaskUnordGtF32"; break; + case Opcode::CompareMaskUnordGeF32: text += "CompareMaskUnordGeF32"; break; + case Opcode::CompareMaskUnordLtF32: text += "CompareMaskUnordLtF32"; break; + case Opcode::CompareMaskUnordLeF32: text += "CompareMaskUnordLeF32"; break; + case Opcode::ConvertByteU32ToF32: text += "ConvertByteU32ToF32"; break; + case Opcode::ConvertU32ToF32: text += "ConvertU32ToF32"; break; + case Opcode::ConvertI32ToF32: text += "ConvertI32ToF32"; break; + case Opcode::ConvertF32ToU32: text += "ConvertF32ToU32"; break; + case Opcode::ConvertF32ToI32: text += "ConvertF32ToI32"; break; + case Opcode::ConvertF32ToF16: text += "ConvertF32ToF16"; break; + case Opcode::ConvertF16ToF32: text += "ConvertF16ToF32"; break; + case Opcode::ConvertU16ToF16: text += "ConvertU16ToF16"; break; + case Opcode::ConvertF16ToU16: text += "ConvertF16ToU16"; break; + case Opcode::ConvertI16ToF16: text += "ConvertI16ToF16"; break; + case Opcode::ConvertF16ToI16: text += "ConvertF16ToI16"; break; + case Opcode::ConvertRoundPlusInfF32ToI32: text += "ConvertRoundPlusInfF32ToI32"; break; + case Opcode::ConvertFloorF32ToI32: text += "ConvertFloorF32ToI32"; break; + case Opcode::ConvertI4ToOffsetF32: text += "ConvertI4ToOffsetF32"; break; + case Opcode::LdexpF32: text += "LdexpF32"; break; + case Opcode::PackF32ToF16Rtz: text += "PackF32ToF16Rtz"; break; + case Opcode::PackSnorm2x16F32: text += "PackSnorm2x16F32"; break; + case Opcode::PackUnorm2x16F32: text += "PackUnorm2x16F32"; break; + case Opcode::PackU16U32: text += "PackU16U32"; break; + case Opcode::PackU8F32: text += "PackU8F32"; break; + case Opcode::PackB32F16: text += "PackB32F16"; break; + case Opcode::PackedMadI16: text += "PackedMadI16"; break; + case Opcode::PackedMulLoU16: text += "PackedMulLoU16"; break; + case Opcode::PackedAddI16: text += "PackedAddI16"; break; + case Opcode::PackedSubI16: text += "PackedSubI16"; break; + case Opcode::PackedLshlrevB16: text += "PackedLshlrevB16"; break; + case Opcode::PackedLshrrevB16: text += "PackedLshrrevB16"; break; + case Opcode::PackedAshrrevI16: text += "PackedAshrrevI16"; break; + case Opcode::PackedMaxI16: text += "PackedMaxI16"; break; + case Opcode::PackedMinI16: text += "PackedMinI16"; break; + case Opcode::PackedMadU16: text += "PackedMadU16"; break; + case Opcode::PackedAddU16: text += "PackedAddU16"; break; + case Opcode::PackedSubU16: text += "PackedSubU16"; break; + case Opcode::PackedMaxU16: text += "PackedMaxU16"; break; + case Opcode::PackedMinU16: text += "PackedMinU16"; break; + case Opcode::PackedAddF16: text += "PackedAddF16"; break; + case Opcode::PackedMulF16: text += "PackedMulF16"; break; + case Opcode::PackedMinF16: text += "PackedMinF16"; break; + case Opcode::PackedMaxF16: text += "PackedMaxF16"; break; + case Opcode::PackedFmaF16: text += "PackedFmaF16"; break; + case Opcode::AddF16: text += "AddF16"; break; + case Opcode::SubF16: text += "SubF16"; break; + case Opcode::MulF16: text += "MulF16"; break; + case Opcode::MinF16: text += "MinF16"; break; + case Opcode::MaxF16: text += "MaxF16"; break; + case Opcode::FmaF16: text += "FmaF16"; break; + case Opcode::MadMixF16: text += "MadMixF16"; break; + case Opcode::IAddU16: text += "IAddU16"; break; + case Opcode::ISubI16: text += "ISubI16"; break; + case Opcode::IMinI16: text += "IMinI16"; break; + case Opcode::IMaxI16: text += "IMaxI16"; break; + case Opcode::UMinU16: text += "UMinU16"; break; + case Opcode::UMaxU16: text += "UMaxU16"; break; + case Opcode::RcpF32: text += "RcpF32"; break; + case Opcode::FractF32: text += "FractF32"; break; + case Opcode::TruncF32: text += "TruncF32"; break; + case Opcode::CeilF32: text += "CeilF32"; break; + case Opcode::RoundEvenF32: text += "RoundEvenF32"; break; + case Opcode::FloorF32: text += "FloorF32"; break; + case Opcode::Exp2F32: text += "Exp2F32"; break; + case Opcode::Log2F32: text += "Log2F32"; break; + case Opcode::InverseSqrtF32: text += "InverseSqrtF32"; break; + case Opcode::SqrtF32: text += "SqrtF32"; break; + case Opcode::RcpF16: text += "RcpF16"; break; + case Opcode::SqrtF16: text += "SqrtF16"; break; + case Opcode::InverseSqrtF16: text += "InverseSqrtF16"; break; + case Opcode::Log2F16: text += "Log2F16"; break; + case Opcode::Exp2F16: text += "Exp2F16"; break; + case Opcode::FloorF16: text += "FloorF16"; break; + case Opcode::CeilF16: text += "CeilF16"; break; + case Opcode::TruncF16: text += "TruncF16"; break; + case Opcode::RoundEvenF16: text += "RoundEvenF16"; break; + case Opcode::SinF32: text += "SinF32"; break; + case Opcode::CosF32: text += "CosF32"; break; + case Opcode::CubeIdF32: text += "CubeIdF32"; break; + case Opcode::CubeScF32: text += "CubeScF32"; break; + case Opcode::CubeTcF32: text += "CubeTcF32"; break; + case Opcode::CubeMaF32: text += "CubeMaF32"; break; + case Opcode::FAddF32: text += "FAddF32"; break; + case Opcode::FSubF32: text += "FSubF32"; break; + case Opcode::FMulF32: text += "FMulF32"; break; + case Opcode::FMinF32: text += "FMinF32"; break; + case Opcode::FMaxF32: text += "FMaxF32"; break; + case Opcode::FMadF32: text += "FMadF32"; break; + case Opcode::Dot2AccF32F16: text += "Dot2AccF32F16"; break; + case Opcode::FMin3F32: text += "FMin3F32"; break; + case Opcode::FMax3F32: text += "FMax3F32"; break; + case Opcode::FMed3F32: text += "FMed3F32"; break; + case Opcode::Min3F16: text += "Min3F16"; break; + case Opcode::Max3F16: text += "Max3F16"; break; + case Opcode::Med3F16: text += "Med3F16"; break; + case Opcode::LoadSrtDword: text += "LoadSrtDword"; break; + case Opcode::SLoadDword: text += "SLoadDword"; break; + case Opcode::SBufferLoadDword: text += "SBufferLoadDword"; break; + case Opcode::BufferLoadUbyte: text += "BufferLoadUbyte"; break; + case Opcode::BufferLoadSbyte: text += "BufferLoadSbyte"; break; + case Opcode::BufferLoadUshort: text += "BufferLoadUshort"; break; + case Opcode::BufferLoadSshort: text += "BufferLoadSshort"; break; + case Opcode::BufferLoadDword: text += "BufferLoadDword"; break; + case Opcode::BufferStoreByte: text += "BufferStoreByte"; break; + case Opcode::BufferStoreShort: text += "BufferStoreShort"; break; + case Opcode::BufferStoreDword: text += "BufferStoreDword"; break; + case Opcode::AtomicSwapU32: text += "AtomicSwapU32"; break; + case Opcode::AtomicAddU32: text += "AtomicAddU32"; break; + case Opcode::AtomicSubU32: text += "AtomicSubU32"; break; + case Opcode::AtomicSMinI32: text += "AtomicSMinI32"; break; + case Opcode::AtomicUMinU32: text += "AtomicUMinU32"; break; + case Opcode::AtomicSMaxI32: text += "AtomicSMaxI32"; break; + case Opcode::AtomicUMaxU32: text += "AtomicUMaxU32"; break; + case Opcode::AtomicAndU32: text += "AtomicAndU32"; break; + case Opcode::AtomicOrU32: text += "AtomicOrU32"; break; + case Opcode::AtomicXorU32: text += "AtomicXorU32"; break; + case Opcode::FlatLoadUbyte: text += "FlatLoadUbyte"; break; + case Opcode::FlatLoadSbyte: text += "FlatLoadSbyte"; break; + case Opcode::FlatLoadUshort: text += "FlatLoadUshort"; break; + case Opcode::FlatLoadSshort: text += "FlatLoadSshort"; break; + case Opcode::FlatLoadDword: text += "FlatLoadDword"; break; + case Opcode::FlatStoreByte: text += "FlatStoreByte"; break; + case Opcode::FlatStoreShort: text += "FlatStoreShort"; break; + case Opcode::FlatStoreDword: text += "FlatStoreDword"; break; + case Opcode::DsReadUbyte: text += "DsReadUbyte"; break; + case Opcode::DsReadSbyte: text += "DsReadSbyte"; break; + case Opcode::DsReadUshort: text += "DsReadUshort"; break; + case Opcode::DsReadSshort: text += "DsReadSshort"; break; + case Opcode::DsReadB32: text += "DsReadB32"; break; + case Opcode::DsWriteByte: text += "DsWriteByte"; break; + case Opcode::DsWriteShort: text += "DsWriteShort"; break; + case Opcode::DsWriteB32: text += "DsWriteB32"; break; + case Opcode::DsMinF32: text += "DsMinF32"; break; + case Opcode::DsMaxF32: text += "DsMaxF32"; break; + case Opcode::DsSwizzleB32: text += "DsSwizzleB32"; break; + case Opcode::DsConsume: text += "DsConsume"; break; + case Opcode::DsAppend: text += "DsAppend"; break; + case Opcode::DsWriteAddtidB32: text += "DsWriteAddtidB32"; break; + case Opcode::DsReadAddtidB32: text += "DsReadAddtidB32"; break; + case Opcode::ImageGetResinfo: text += "ImageGetResinfo"; break; + case Opcode::ImageGetLod: text += "ImageGetLod"; break; + case Opcode::ImageLoad: text += "ImageLoad"; break; + case Opcode::ImageStore: text += "ImageStore"; break; + case Opcode::ImageSample: text += "ImageSample"; break; + case Opcode::ImageGather4: text += "ImageGather4"; break; + case Opcode::LoadInputF32: text += "LoadInputF32"; break; + case Opcode::Export: text += "Export"; break; + } + text += " "; + text += OperandToString(inst.dst); + if (inst.dst2.kind != OperandKind::Null) { + text += ", "; + text += OperandToString(inst.dst2); + } + for (uint32_t i = 0; i < inst.src_count; i++) { + text += i == 0 ? ", " : ", "; + text += OperandToString(inst.src[i]); + } + text += MemoryInfoToString(inst.memory); + text += InputInfoToString(inst.op, inst.input_info); + return text; +} + +std::string ExportTargetKindToString(ExportTargetKind kind) { + switch (kind) { + case ExportTargetKind::Null: return "null"; + case ExportTargetKind::Position: return "position"; + case ExportTargetKind::Primitive: return "primitive"; + case ExportTargetKind::Parameter: return "parameter"; + case ExportTargetKind::Mrt: return "mrt"; + case ExportTargetKind::MrtZ: return "mrtz"; + default: return "unknown"; + } +} + +std::string ProgramToString(const Program& program) { + std::string text; + text += fmt::format("mode={} masks={} fallback={} reason={}\n", + program.dispatcher_fallback ? "dispatcher" : "structured", + program.lane_mask_mode == ShaderLaneMaskMode::NativeWave ? "native-wave" + : "per-invocation", + CFG::FailureKindToString(program.cfg_failure_kind).c_str(), + program.fallback_reason.c_str()); + for (const auto& block: program.blocks) { + text += fmt::format("block_{} pc=0x{:08x} end=0x{:08x} preds={} succs={} {}\n", block.id, + block.start_pc, block.end_pc, + static_cast(block.predecessors.size()), + static_cast(block.successors.size()), + TerminatorToString(block.terminator).c_str()); + for (const auto& inst: block.instructions) { + text += " "; + text += InstructionToString(inst); + text += "\n"; + } + } + return text; +} + +} // namespace Libs::Graphics::ShaderRecompiler::IR diff --git a/src/graphics/shader/recompiler/shaderIR/ShaderIRMemory.cpp b/src/graphics/shader/recompiler/shaderIR/ShaderIRMemory.cpp new file mode 100644 index 000000000..ac3f6663e --- /dev/null +++ b/src/graphics/shader/recompiler/shaderIR/ShaderIRMemory.cpp @@ -0,0 +1,788 @@ +#include "graphics/shader/recompiler/ShaderIR.h" + +#include "graphics/shader/recompiler/shaderIR/ShaderIRInternal.h" +#include "common/assert.h" +#include "graphics/shader/recompiler/BufferFormat.h" + +#include +#include +#include + +namespace Libs::Graphics::ShaderRecompiler::IR { +namespace { + +bool LowerScalarBufferLoadDword(const Decoder::Instruction& decoded, BasicBlock* block, Opcode op, + std::string* error) { + for (uint32_t i = 0; i < decoded.data_dwords; i++) { + Instruction inst; + inst.pc = decoded.pc; + inst.op = op; + inst.src_count = 1; + inst.memory = OffsetMemoryInfo(decoded, ResourceKind::ScalarBuffer, i); + // Raw s_load uses a two-SGPR pointer base; s_buffer_load uses a four-SGPR + // descriptor resource index. + inst.memory.resource = op == Opcode::SLoadDword ? RawScalarLoadBase(decoded.src0) + : ResourceIndexFromOperand(decoded.src0); + if (!LowerRegisterOperand(OffsetDecodedRegister(decoded.dst, i), &inst.dst, error) || + !LowerSourceOperand(decoded.src1, &inst.src[0], error)) { + return false; + } + block->instructions.push_back(inst); + } + return true; +} + +bool LowerBufferAddressSources(const Decoder::Instruction& decoded, Instruction* inst, + uint32_t first_src, std::string* error) { + uint32_t src = first_src; + if (decoded.idxen) { + if (!LowerSourceOperand(decoded.src0, &inst->src[src++], error)) { + return false; + } + } + if (decoded.offen) { + const auto offset_source = + decoded.idxen ? OffsetDecodedRegister(decoded.src0, 1) : decoded.src0; + if (!LowerSourceOperand(offset_source, &inst->src[src++], error)) { + return false; + } + } + if (!LowerSourceOperand(decoded.src2, &inst->src[src++], error)) { + return false; + } + inst->src_count = src; + return true; +} + +bool LowerBufferLoad(const Decoder::Instruction& decoded, BasicBlock* block, std::string* error) { + const auto ir_op = + decoded.data_bits == 8u + ? (decoded.data_signed ? Opcode::BufferLoadSbyte : Opcode::BufferLoadUbyte) + : decoded.data_bits == 16u + ? (decoded.data_signed ? Opcode::BufferLoadSshort : Opcode::BufferLoadUshort) + : Opcode::BufferLoadDword; + const auto count = decoded.data_bits == 32u ? decoded.data_dwords : 1u; + const auto typed_format_components = TypedBufferFormatComponentCount(decoded); + for (uint32_t i = 0; i < count; i++) { + if (i >= typed_format_components) { + if (!LowerMoveImmediateU32(decoded.pc, OffsetDecodedRegister(decoded.dst, i), 0, block, + error)) { + return false; + } + continue; + } + + Instruction inst; + inst.pc = decoded.pc; + inst.op = ir_op; + inst.memory = OffsetBufferMemoryInfo(decoded, i); + if (!LowerRegisterOperand(OffsetDecodedRegister(decoded.dst, i), &inst.dst, error) || + !LowerBufferAddressSources(decoded, &inst, 0, error)) { + return false; + } + block->instructions.push_back(inst); + } + return true; +} + +bool LowerBufferStore(const Decoder::Instruction& decoded, BasicBlock* block, std::string* error) { + const auto ir_op = decoded.data_bits == 8u ? Opcode::BufferStoreByte + : decoded.data_bits == 16u ? Opcode::BufferStoreShort + : Opcode::BufferStoreDword; + auto count = decoded.data_bits == 32u ? decoded.data_dwords : 1u; + if (decoded.data_bits == 32u) { + const auto typed_format_components = TypedBufferFormatComponentCount(decoded); + if (typed_format_components < count) { + count = typed_format_components; + } + } + for (uint32_t i = 0; i < count; i++) { + Instruction inst; + inst.pc = decoded.pc; + inst.op = ir_op; + inst.memory = OffsetBufferMemoryInfo(decoded, i); + inst.dst.kind = OperandKind::Null; + if (!LowerSourceOperand(OffsetDecodedRegister(decoded.dst, i), &inst.src[0], error) || + !LowerBufferAddressSources(decoded, &inst, 1, error)) { + return false; + } + block->instructions.push_back(inst); + } + return true; +} + +bool LowerBufferAtomicDword(const Decoder::Instruction& decoded, BasicBlock* block, Opcode op, + std::string* error) { + Instruction inst; + inst.pc = decoded.pc; + inst.op = op; + inst.memory = MemoryInfoFromDecoded(decoded, ResourceKind::Buffer); + inst.dst.kind = OperandKind::Null; + if ((decoded.glc && !LowerRegisterOperand(decoded.dst, &inst.dst, error)) || + !LowerSourceOperand(decoded.dst, &inst.src[0], error) || + !LowerBufferAddressSources(decoded, &inst, 1, error)) { + return false; + } + block->instructions.push_back(inst); + return true; +} + +ResourceKind DsMemoryKind(const Decoder::Instruction& decoded) { + return decoded.gds ? ResourceKind::Gds : ResourceKind::Lds; +} + +bool LowerDsRead(const Decoder::Instruction& decoded, BasicBlock* block, std::string* error) { + const auto ir_op = decoded.data_bits == 8u + ? (decoded.data_signed ? Opcode::DsReadSbyte : Opcode::DsReadUbyte) + : decoded.data_bits == 16u + ? (decoded.data_signed ? Opcode::DsReadSshort : Opcode::DsReadUshort) + : Opcode::DsReadB32; + const auto count = decoded.data_bits == 32u ? decoded.data_dwords : 1u; + for (uint32_t i = 0; i < count; i++) { + Instruction inst; + inst.pc = decoded.pc; + inst.op = ir_op; + inst.src_count = 1; + inst.memory = OffsetMemoryInfo(decoded, DsMemoryKind(decoded), i); + if (!LowerRegisterOperand(OffsetDecodedRegister(decoded.dst, i), &inst.dst, error) || + !LowerSourceOperand(decoded.src0, &inst.src[0], error)) { + return false; + } + block->instructions.push_back(inst); + } + return true; +} + +bool LowerDsRead2(const Decoder::Instruction& decoded, BasicBlock* block, uint32_t dwords_per_read, + std::string* error) { + const uint32_t offsets[] = {decoded.offset, decoded.secondary_offset}; + for (uint32_t read = 0; read < 2u; read++) { + for (uint32_t dword = 0; dword < dwords_per_read; dword++) { + const auto index = read * dwords_per_read + dword; + Instruction inst; + inst.pc = decoded.pc; + inst.op = Opcode::DsReadB32; + inst.src_count = 1; + inst.memory = + ByteOffsetMemoryInfo(decoded, DsMemoryKind(decoded), offsets[read] + dword * 4u); + if (!LowerRegisterOperand(OffsetDecodedRegister(decoded.dst, index), &inst.dst, + error) || + !LowerSourceOperand(decoded.src0, &inst.src[0], error)) { + return false; + } + block->instructions.push_back(inst); + } + } + return true; +} + +bool LowerDsAtomicU32(const Decoder::Instruction& decoded, BasicBlock* block, Opcode op, + bool return_old, std::string* error) { + Instruction inst; + inst.pc = decoded.pc; + inst.op = op; + inst.src_count = 2; + inst.memory = MemoryInfoFromDecoded(decoded, DsMemoryKind(decoded)); + inst.dst.kind = return_old ? OperandKind::Register : OperandKind::Null; + if ((return_old && !LowerRegisterOperand(decoded.dst, &inst.dst, error)) || + !LowerSourceOperand(decoded.src1, &inst.src[0], error) || + !LowerSourceOperand(decoded.src0, &inst.src[1], error)) { + return false; + } + block->instructions.push_back(inst); + return true; +} + +bool LowerDsSwizzleB32(const Decoder::Instruction& decoded, BasicBlock* block, std::string* error) { + Instruction inst; + inst.pc = decoded.pc; + inst.op = Opcode::DsSwizzleB32; + inst.src_count = 2; + inst.src[1].kind = OperandKind::ImmediateU32; + inst.src[1].imm = decoded.offset & 0xffffu; + if (!LowerRegisterOperand(decoded.dst, &inst.dst, error) || + !LowerSourceOperand(decoded.src0, &inst.src[0], error)) { + return false; + } + block->instructions.push_back(inst); + return true; +} + +bool LowerDsFloatMinMaxF32(const Decoder::Instruction& decoded, BasicBlock* block, Opcode op, + std::string* error) { + Instruction inst; + inst.pc = decoded.pc; + inst.op = op; + inst.src_count = 2; + inst.memory = MemoryInfoFromDecoded(decoded, DsMemoryKind(decoded)); + inst.memory.resource = 0; + inst.dst.kind = OperandKind::Null; + if (!LowerSourceOperand(decoded.src1, &inst.src[0], error) || + !LowerSourceOperand(decoded.src0, &inst.src[1], error) || + !LowerSourceOperand(decoded.src2, &inst.src[2], error)) { + return false; + } + inst.src_count = 3; + block->instructions.push_back(inst); + return true; +} + +bool LowerDsWriteAddtidB32(const Decoder::Instruction& decoded, BasicBlock* block, + std::string* error) { + Instruction inst; + inst.pc = decoded.pc; + inst.op = Opcode::DsWriteAddtidB32; + inst.src_count = 2; + inst.memory = MemoryInfoFromDecoded(decoded, DsMemoryKind(decoded)); + inst.memory.resource = 0; + inst.dst.kind = OperandKind::Null; + const auto m0 = M0Operand(); + if (!LowerSourceOperand(decoded.src1, &inst.src[0], error) || + !LowerSourceOperand(m0, &inst.src[1], error)) { + return false; + } + block->instructions.push_back(inst); + return true; +} + +bool LowerDsReadAddtidB32(const Decoder::Instruction& decoded, BasicBlock* block, + std::string* error) { + Instruction inst; + inst.pc = decoded.pc; + inst.op = Opcode::DsReadAddtidB32; + inst.src_count = 1; + inst.memory = MemoryInfoFromDecoded(decoded, DsMemoryKind(decoded)); + inst.memory.resource = 0; + const auto m0 = M0Operand(); + if (!LowerRegisterOperand(decoded.dst, &inst.dst, error) || + !LowerSourceOperand(m0, &inst.src[0], error)) { + return false; + } + block->instructions.push_back(inst); + return true; +} + +bool LowerDsAppendConsume(const Decoder::Instruction& decoded, BasicBlock* block, Opcode op, + std::string* error) { + Instruction inst; + inst.pc = decoded.pc; + inst.op = op; + inst.src_count = 1; + inst.memory = MemoryInfoFromDecoded(decoded, DsMemoryKind(decoded)); + const auto m0 = M0Operand(); + if (!LowerRegisterOperand(decoded.dst, &inst.dst, error) || + !LowerSourceOperand(m0, &inst.src[0], error)) { + return false; + } + block->instructions.push_back(inst); + return true; +} + +Opcode ImageAtomicIrOpcode(Decoder::Opcode opcode) { + switch (opcode) { + case Decoder::Opcode::ImageAtomicAdd: return Opcode::AtomicAddU32; + case Decoder::Opcode::ImageAtomicUMin: return Opcode::AtomicUMinU32; + case Decoder::Opcode::ImageAtomicUMax: return Opcode::AtomicUMaxU32; + case Decoder::Opcode::ImageAtomicAnd: return Opcode::AtomicAndU32; + case Decoder::Opcode::ImageAtomicOr: return Opcode::AtomicOrU32; + case Decoder::Opcode::ImageAtomicXor: return Opcode::AtomicXorU32; + default: return Opcode::AtomicAddU32; + } +} + +bool LowerImageAtomicU32(const Decoder::Instruction& decoded, BasicBlock* block, + std::string* error) { + Instruction inst; + inst.pc = decoded.pc; + inst.op = ImageAtomicIrOpcode(decoded.opcode); + inst.src_count = 2; + inst.memory = MemoryInfoFromDecoded(decoded, ResourceKind::StorageImageUint); + inst.memory.sampler = 0; + inst.dst.kind = OperandKind::Null; + if ((decoded.glc && !LowerRegisterOperand(decoded.dst, &inst.dst, error)) || + !LowerSourceOperand(decoded.dst, &inst.src[0], error) || + !LowerSourceOperand(decoded.src0, &inst.src[1], error)) { + return false; + } + block->instructions.push_back(inst); + return true; +} + +bool LowerFlatLoad(const Decoder::Instruction& decoded, BasicBlock* block, std::string* error) { + const auto kind = FlatSegmentResourceKind(decoded.memory_segment); + const auto ir_op = decoded.data_bits == 8u + ? (decoded.data_signed ? Opcode::FlatLoadSbyte : Opcode::FlatLoadUbyte) + : decoded.data_bits == 16u + ? (decoded.data_signed ? Opcode::FlatLoadSshort : Opcode::FlatLoadUshort) + : Opcode::FlatLoadDword; + const auto count = decoded.data_bits == 32u ? decoded.data_dwords : 1u; + if (decoded.data_bits == 32u && count > 1u) { + Instruction inst; + inst.pc = decoded.pc; + inst.op = ir_op; + inst.src_count = decoded.src_count; + inst.memory = MemoryInfoFromDecoded(decoded, kind); + inst.memory.resource = 0; + if (!LowerRegisterOperand(decoded.dst, &inst.dst, error)) { + return false; + } + for (uint32_t i = 0; i < decoded.src_count && i < 3u; i++) { + if (!LowerSourceOperand(DecodedSourceAt(decoded, i), &inst.src[i], error)) { + return false; + } + } + block->instructions.push_back(inst); + return true; + } + + for (uint32_t dword = 0; dword < count; dword++) { + Instruction inst; + inst.pc = decoded.pc; + inst.op = ir_op; + inst.src_count = decoded.src_count; + inst.memory = OffsetMemoryInfo(decoded, kind, dword); + inst.memory.resource = 0; + if (!LowerRegisterOperand(OffsetDecodedRegister(decoded.dst, dword), &inst.dst, error)) { + return false; + } + for (uint32_t i = 0; i < decoded.src_count && i < 3u; i++) { + if (!LowerSourceOperand(DecodedSourceAt(decoded, i), &inst.src[i], error)) { + return false; + } + } + block->instructions.push_back(inst); + } + return true; +} + +bool LowerFlatStore(const Decoder::Instruction& decoded, BasicBlock* block, std::string* error) { + const auto kind = FlatSegmentResourceKind(decoded.memory_segment); + const auto ir_op = decoded.data_bits == 8u ? Opcode::FlatStoreByte + : decoded.data_bits == 16u ? Opcode::FlatStoreShort + : Opcode::FlatStoreDword; + const auto count = decoded.data_bits == 32u ? decoded.data_dwords : 1u; + for (uint32_t dword = 0; dword < count; dword++) { + Instruction inst; + inst.pc = decoded.pc; + inst.op = ir_op; + inst.src_count = decoded.src_count + 1u; + inst.memory = OffsetMemoryInfo(decoded, kind, dword); + inst.memory.resource = 0; + inst.dst.kind = OperandKind::Null; + if (!LowerSourceOperand(OffsetDecodedRegister(decoded.dst, dword), &inst.src[0], error)) { + return false; + } + for (uint32_t i = 0; i < decoded.src_count && i + 1u < 3u; i++) { + if (!LowerSourceOperand(DecodedSourceAt(decoded, i), &inst.src[i + 1u], error)) { + return false; + } + } + block->instructions.push_back(inst); + } + return true; +} + +bool LowerDsWrite(const Decoder::Instruction& decoded, BasicBlock* block, std::string* error) { + const auto ir_op = decoded.data_bits == 8u ? Opcode::DsWriteByte + : decoded.data_bits == 16u ? Opcode::DsWriteShort + : Opcode::DsWriteB32; + const auto count = decoded.data_bits == 32u ? decoded.data_dwords : 1u; + for (uint32_t i = 0; i < count; i++) { + Instruction inst; + inst.pc = decoded.pc; + inst.op = ir_op; + inst.src_count = 2; + inst.memory = OffsetMemoryInfo(decoded, DsMemoryKind(decoded), i); + inst.dst.kind = OperandKind::Null; + if (!LowerSourceOperand(OffsetDecodedRegister(decoded.src1, i), &inst.src[0], error) || + !LowerSourceOperand(decoded.src0, &inst.src[1], error)) { + return false; + } + block->instructions.push_back(inst); + } + return true; +} + +bool LowerDsWrite2(const Decoder::Instruction& decoded, BasicBlock* block, + uint32_t dwords_per_write, std::string* error) { + const Decoder::Operand* data[] = {&decoded.src1, &decoded.src2}; + const uint32_t offsets[] = {decoded.offset, decoded.secondary_offset}; + for (uint32_t write = 0; write < 2u; write++) { + for (uint32_t dword = 0; dword < dwords_per_write; dword++) { + Instruction inst; + inst.pc = decoded.pc; + inst.op = Opcode::DsWriteB32; + inst.src_count = 2; + inst.memory = + ByteOffsetMemoryInfo(decoded, DsMemoryKind(decoded), offsets[write] + dword * 4u); + inst.dst.kind = OperandKind::Null; + if (!LowerSourceOperand(OffsetDecodedRegister(*data[write], dword), &inst.src[0], + error) || + !LowerSourceOperand(decoded.src0, &inst.src[1], error)) { + return false; + } + block->instructions.push_back(inst); + } + } + return true; +} + +bool LowerImageOperation(const Decoder::Instruction& decoded, BasicBlock* block, + std::string* error) { + Instruction inst; + inst.pc = decoded.pc; + if (decoded.opcode == Decoder::Opcode::ImageStore || + decoded.opcode == Decoder::Opcode::ImageStoreMip) { + inst.op = Opcode::ImageStore; + inst.src_count = 2; + inst.memory = MemoryInfoFromDecoded(decoded, ResourceKind::StorageImage); + inst.memory.sampler = 0; + inst.dst.kind = OperandKind::Null; + if (!LowerSourceOperand(decoded.dst, &inst.src[0], error) || + !LowerSourceOperand(decoded.src0, &inst.src[1], error)) { + return false; + } + block->instructions.push_back(inst); + return true; + } + + inst.op = decoded.opcode == Decoder::Opcode::ImageGetResinfo ? Opcode::ImageGetResinfo + : decoded.opcode == Decoder::Opcode::ImageGetLod ? Opcode::ImageGetLod + : decoded.opcode == Decoder::Opcode::ImageLoad || + decoded.opcode == Decoder::Opcode::ImageLoadMip + ? Opcode::ImageLoad + : decoded.opcode == Decoder::Opcode::ImageGather4Lz || + decoded.opcode == Decoder::Opcode::ImageGather4C || + decoded.opcode == Decoder::Opcode::ImageGather4CLz || + decoded.opcode == Decoder::Opcode::ImageGather4LzO || + decoded.opcode == Decoder::Opcode::ImageGather4CO || + decoded.opcode == Decoder::Opcode::ImageGather4CLzO || + decoded.opcode == Decoder::Opcode::ImageGather4H + ? Opcode::ImageGather4 + : Opcode::ImageSample; + inst.src_count = 1; + inst.memory = MemoryInfoFromDecoded(decoded, ResourceKind::Image); + if (!LowerRegisterOperand(decoded.dst, &inst.dst, error) || + !LowerSourceOperand(decoded.src0, &inst.src[0], error)) { + return false; + } + block->instructions.push_back(inst); + return true; +} + + +} // namespace + +bool LowerMemoryInstruction(const Decoder::Instruction& decoded, BasicBlock* block, + std::string* error) { + switch (decoded.opcode) { + case Decoder::Opcode::SLoadDword: + case Decoder::Opcode::SLoadDwordx2: + case Decoder::Opcode::SLoadDwordx4: + case Decoder::Opcode::SLoadDwordx8: + case Decoder::Opcode::SLoadDwordx16: + return LowerScalarBufferLoadDword(decoded, block, Opcode::SLoadDword, error); + case Decoder::Opcode::SBufferLoadDword: + case Decoder::Opcode::SBufferLoadDwordx2: + case Decoder::Opcode::SBufferLoadDwordx4: + case Decoder::Opcode::SBufferLoadDwordx8: + case Decoder::Opcode::SBufferLoadDwordx16: + return LowerScalarBufferLoadDword(decoded, block, Opcode::SBufferLoadDword, error); + case Decoder::Opcode::BufferLoadUbyte: + case Decoder::Opcode::BufferLoadSbyte: + case Decoder::Opcode::BufferLoadUshort: + case Decoder::Opcode::BufferLoadSshort: + case Decoder::Opcode::BufferLoadDword: + case Decoder::Opcode::BufferLoadDwordx2: + case Decoder::Opcode::BufferLoadDwordx3: + case Decoder::Opcode::BufferLoadDwordx4: + case Decoder::Opcode::BufferLoadFormatX: + case Decoder::Opcode::BufferLoadFormatXy: + case Decoder::Opcode::BufferLoadFormatXyz: + case Decoder::Opcode::BufferLoadFormatXyzw: + case Decoder::Opcode::TBufferLoadFormatX: + case Decoder::Opcode::TBufferLoadFormatXy: + case Decoder::Opcode::TBufferLoadFormatXyz: + case Decoder::Opcode::TBufferLoadFormatXyzw: return LowerBufferLoad(decoded, block, error); + case Decoder::Opcode::BufferStoreDword: + case Decoder::Opcode::BufferStoreDwordx2: + case Decoder::Opcode::BufferStoreDwordx3: + case Decoder::Opcode::BufferStoreDwordx4: + case Decoder::Opcode::BufferStoreByte: + case Decoder::Opcode::BufferStoreShort: + case Decoder::Opcode::BufferStoreFormatX: + case Decoder::Opcode::BufferStoreFormatXy: + case Decoder::Opcode::BufferStoreFormatXyz: + case Decoder::Opcode::BufferStoreFormatXyzw: + case Decoder::Opcode::TBufferStoreFormatX: + case Decoder::Opcode::TBufferStoreFormatXy: + case Decoder::Opcode::TBufferStoreFormatXyz: + case Decoder::Opcode::TBufferStoreFormatXyzw: + return LowerBufferStore(decoded, block, error); + case Decoder::Opcode::BufferAtomicSwap: + return LowerBufferAtomicDword(decoded, block, Opcode::AtomicSwapU32, error); + case Decoder::Opcode::BufferAtomicAdd: + return LowerBufferAtomicDword(decoded, block, Opcode::AtomicAddU32, error); + case Decoder::Opcode::BufferAtomicSub: + return LowerBufferAtomicDword(decoded, block, Opcode::AtomicSubU32, error); + case Decoder::Opcode::BufferAtomicSMin: + return LowerBufferAtomicDword(decoded, block, Opcode::AtomicSMinI32, error); + case Decoder::Opcode::BufferAtomicUMin: + return LowerBufferAtomicDword(decoded, block, Opcode::AtomicUMinU32, error); + case Decoder::Opcode::BufferAtomicSMax: + return LowerBufferAtomicDword(decoded, block, Opcode::AtomicSMaxI32, error); + case Decoder::Opcode::BufferAtomicUMax: + return LowerBufferAtomicDword(decoded, block, Opcode::AtomicUMaxU32, error); + case Decoder::Opcode::BufferAtomicAnd: + return LowerBufferAtomicDword(decoded, block, Opcode::AtomicAndU32, error); + case Decoder::Opcode::BufferAtomicOr: + return LowerBufferAtomicDword(decoded, block, Opcode::AtomicOrU32, error); + case Decoder::Opcode::BufferAtomicXor: + return LowerBufferAtomicDword(decoded, block, Opcode::AtomicXorU32, error); + case Decoder::Opcode::FlatLoadUbyte: + case Decoder::Opcode::FlatLoadSbyte: + case Decoder::Opcode::FlatLoadUshort: + case Decoder::Opcode::FlatLoadSshort: + case Decoder::Opcode::FlatLoadDword: + case Decoder::Opcode::FlatLoadDwordx2: + case Decoder::Opcode::FlatLoadDwordx3: + case Decoder::Opcode::FlatLoadDwordx4: return LowerFlatLoad(decoded, block, error); + case Decoder::Opcode::FlatStoreByte: + case Decoder::Opcode::FlatStoreShort: + case Decoder::Opcode::FlatStoreDword: + case Decoder::Opcode::FlatStoreDwordx2: + case Decoder::Opcode::FlatStoreDwordx3: + case Decoder::Opcode::FlatStoreDwordx4: return LowerFlatStore(decoded, block, error); + case Decoder::Opcode::DsAddU32: + return LowerDsAtomicU32(decoded, block, Opcode::AtomicAddU32, false, error); + case Decoder::Opcode::DsAddRtnU32: + return LowerDsAtomicU32(decoded, block, Opcode::AtomicAddU32, true, error); + case Decoder::Opcode::DsSubU32: + return LowerDsAtomicU32(decoded, block, Opcode::AtomicSubU32, false, error); + case Decoder::Opcode::DsSubRtnU32: + return LowerDsAtomicU32(decoded, block, Opcode::AtomicSubU32, true, error); + case Decoder::Opcode::DsMinI32: + return LowerDsAtomicU32(decoded, block, Opcode::AtomicSMinI32, false, error); + case Decoder::Opcode::DsMinRtnI32: + return LowerDsAtomicU32(decoded, block, Opcode::AtomicSMinI32, true, error); + case Decoder::Opcode::DsMaxI32: + return LowerDsAtomicU32(decoded, block, Opcode::AtomicSMaxI32, false, error); + case Decoder::Opcode::DsMaxRtnI32: + return LowerDsAtomicU32(decoded, block, Opcode::AtomicSMaxI32, true, error); + case Decoder::Opcode::DsMinU32: + return LowerDsAtomicU32(decoded, block, Opcode::AtomicUMinU32, false, error); + case Decoder::Opcode::DsMinRtnU32: + return LowerDsAtomicU32(decoded, block, Opcode::AtomicUMinU32, true, error); + case Decoder::Opcode::DsMaxU32: + return LowerDsAtomicU32(decoded, block, Opcode::AtomicUMaxU32, false, error); + case Decoder::Opcode::DsMaxRtnU32: + return LowerDsAtomicU32(decoded, block, Opcode::AtomicUMaxU32, true, error); + case Decoder::Opcode::DsAndB32: + return LowerDsAtomicU32(decoded, block, Opcode::AtomicAndU32, false, error); + case Decoder::Opcode::DsAndRtnB32: + return LowerDsAtomicU32(decoded, block, Opcode::AtomicAndU32, true, error); + case Decoder::Opcode::DsOrB32: + return LowerDsAtomicU32(decoded, block, Opcode::AtomicOrU32, false, error); + case Decoder::Opcode::DsOrRtnB32: + return LowerDsAtomicU32(decoded, block, Opcode::AtomicOrU32, true, error); + case Decoder::Opcode::DsXorB32: + return LowerDsAtomicU32(decoded, block, Opcode::AtomicXorU32, false, error); + case Decoder::Opcode::DsXorRtnB32: + return LowerDsAtomicU32(decoded, block, Opcode::AtomicXorU32, true, error); + case Decoder::Opcode::DsWrxchgRtnB32: + return LowerDsAtomicU32(decoded, block, Opcode::AtomicSwapU32, true, error); + case Decoder::Opcode::DsMinF32: + return LowerDsFloatMinMaxF32(decoded, block, Opcode::DsMinF32, error); + case Decoder::Opcode::DsMaxF32: + return LowerDsFloatMinMaxF32(decoded, block, Opcode::DsMaxF32, error); + case Decoder::Opcode::DsSwizzleB32: return LowerDsSwizzleB32(decoded, block, error); + case Decoder::Opcode::DsConsume: + return LowerDsAppendConsume(decoded, block, Opcode::DsConsume, error); + case Decoder::Opcode::DsAppend: + return LowerDsAppendConsume(decoded, block, Opcode::DsAppend, error); + case Decoder::Opcode::DsWriteAddtidB32: return LowerDsWriteAddtidB32(decoded, block, error); + case Decoder::Opcode::DsReadAddtidB32: return LowerDsReadAddtidB32(decoded, block, error); + case Decoder::Opcode::DsRead2B32: return LowerDsRead2(decoded, block, 1, error); + case Decoder::Opcode::DsRead2B64: + case Decoder::Opcode::DsRead2St64B64: return LowerDsRead2(decoded, block, 2, error); + case Decoder::Opcode::DsReadSbyte: + case Decoder::Opcode::DsReadUbyte: + case Decoder::Opcode::DsReadSshort: + case Decoder::Opcode::DsReadUshort: + case Decoder::Opcode::DsReadB32: + case Decoder::Opcode::DsReadB64: + case Decoder::Opcode::DsReadB96: + case Decoder::Opcode::DsReadB128: return LowerDsRead(decoded, block, error); + case Decoder::Opcode::DsWrite2B32: + case Decoder::Opcode::DsWrite2St64B32: return LowerDsWrite2(decoded, block, 1, error); + case Decoder::Opcode::DsWrite2B64: + case Decoder::Opcode::DsWrite2St64B64: return LowerDsWrite2(decoded, block, 2, error); + case Decoder::Opcode::DsWriteByte: + case Decoder::Opcode::DsWriteShort: + case Decoder::Opcode::DsWriteB32: + case Decoder::Opcode::DsWriteB64: + case Decoder::Opcode::DsWriteB96: + case Decoder::Opcode::DsWriteB128: return LowerDsWrite(decoded, block, error); + case Decoder::Opcode::ImageGetResinfo: + case Decoder::Opcode::ImageGetLod: + case Decoder::Opcode::ImageLoad: + case Decoder::Opcode::ImageLoadMip: + case Decoder::Opcode::ImageStore: + case Decoder::Opcode::ImageStoreMip: + case Decoder::Opcode::ImageGather4Lz: + case Decoder::Opcode::ImageGather4C: + case Decoder::Opcode::ImageGather4CLz: + case Decoder::Opcode::ImageGather4LzO: + case Decoder::Opcode::ImageGather4CO: + case Decoder::Opcode::ImageGather4CLzO: + case Decoder::Opcode::ImageGather4H: + case Decoder::Opcode::ImageSample: return LowerImageOperation(decoded, block, error); + case Decoder::Opcode::ImageAtomicAdd: + case Decoder::Opcode::ImageAtomicUMin: + case Decoder::Opcode::ImageAtomicUMax: + case Decoder::Opcode::ImageAtomicAnd: + case Decoder::Opcode::ImageAtomicOr: + case Decoder::Opcode::ImageAtomicXor: return LowerImageAtomicU32(decoded, block, error); + default: return false; + } +} + + +bool IsMemoryOpcode(Decoder::Opcode opcode) { + switch (opcode) { + case Decoder::Opcode::SLoadDword: + case Decoder::Opcode::SLoadDwordx2: + case Decoder::Opcode::SLoadDwordx4: + case Decoder::Opcode::SLoadDwordx8: + case Decoder::Opcode::SLoadDwordx16: + case Decoder::Opcode::SBufferLoadDword: + case Decoder::Opcode::SBufferLoadDwordx2: + case Decoder::Opcode::SBufferLoadDwordx4: + case Decoder::Opcode::SBufferLoadDwordx8: + case Decoder::Opcode::SBufferLoadDwordx16: + case Decoder::Opcode::BufferLoadUbyte: + case Decoder::Opcode::BufferLoadSbyte: + case Decoder::Opcode::BufferLoadUshort: + case Decoder::Opcode::BufferLoadSshort: + case Decoder::Opcode::BufferLoadDword: + case Decoder::Opcode::BufferLoadDwordx2: + case Decoder::Opcode::BufferLoadDwordx3: + case Decoder::Opcode::BufferLoadDwordx4: + case Decoder::Opcode::BufferLoadFormatX: + case Decoder::Opcode::BufferLoadFormatXy: + case Decoder::Opcode::BufferLoadFormatXyz: + case Decoder::Opcode::BufferLoadFormatXyzw: + case Decoder::Opcode::TBufferLoadFormatX: + case Decoder::Opcode::TBufferLoadFormatXy: + case Decoder::Opcode::TBufferLoadFormatXyz: + case Decoder::Opcode::TBufferLoadFormatXyzw: + case Decoder::Opcode::BufferStoreDword: + case Decoder::Opcode::BufferStoreDwordx2: + case Decoder::Opcode::BufferStoreDwordx3: + case Decoder::Opcode::BufferStoreDwordx4: + case Decoder::Opcode::BufferStoreByte: + case Decoder::Opcode::BufferStoreShort: + case Decoder::Opcode::BufferStoreFormatX: + case Decoder::Opcode::BufferStoreFormatXy: + case Decoder::Opcode::BufferStoreFormatXyz: + case Decoder::Opcode::BufferStoreFormatXyzw: + case Decoder::Opcode::TBufferStoreFormatX: + case Decoder::Opcode::TBufferStoreFormatXy: + case Decoder::Opcode::TBufferStoreFormatXyz: + case Decoder::Opcode::TBufferStoreFormatXyzw: + case Decoder::Opcode::BufferAtomicSwap: + case Decoder::Opcode::BufferAtomicAdd: + case Decoder::Opcode::BufferAtomicSub: + case Decoder::Opcode::BufferAtomicSMin: + case Decoder::Opcode::BufferAtomicUMin: + case Decoder::Opcode::BufferAtomicSMax: + case Decoder::Opcode::BufferAtomicUMax: + case Decoder::Opcode::BufferAtomicAnd: + case Decoder::Opcode::BufferAtomicOr: + case Decoder::Opcode::BufferAtomicXor: + case Decoder::Opcode::FlatLoadUbyte: + case Decoder::Opcode::FlatLoadSbyte: + case Decoder::Opcode::FlatLoadUshort: + case Decoder::Opcode::FlatLoadSshort: + case Decoder::Opcode::FlatLoadDword: + case Decoder::Opcode::FlatLoadDwordx2: + case Decoder::Opcode::FlatLoadDwordx3: + case Decoder::Opcode::FlatLoadDwordx4: + case Decoder::Opcode::FlatStoreByte: + case Decoder::Opcode::FlatStoreShort: + case Decoder::Opcode::FlatStoreDword: + case Decoder::Opcode::FlatStoreDwordx2: + case Decoder::Opcode::FlatStoreDwordx3: + case Decoder::Opcode::FlatStoreDwordx4: + case Decoder::Opcode::DsAddU32: + case Decoder::Opcode::DsAddRtnU32: + case Decoder::Opcode::DsSubU32: + case Decoder::Opcode::DsSubRtnU32: + case Decoder::Opcode::DsMinI32: + case Decoder::Opcode::DsMinRtnI32: + case Decoder::Opcode::DsMaxI32: + case Decoder::Opcode::DsMaxRtnI32: + case Decoder::Opcode::DsMinU32: + case Decoder::Opcode::DsMinRtnU32: + case Decoder::Opcode::DsMaxU32: + case Decoder::Opcode::DsMaxRtnU32: + case Decoder::Opcode::DsAndB32: + case Decoder::Opcode::DsAndRtnB32: + case Decoder::Opcode::DsOrB32: + case Decoder::Opcode::DsOrRtnB32: + case Decoder::Opcode::DsXorB32: + case Decoder::Opcode::DsXorRtnB32: + case Decoder::Opcode::DsWrxchgRtnB32: + case Decoder::Opcode::DsMinF32: + case Decoder::Opcode::DsMaxF32: + case Decoder::Opcode::DsSwizzleB32: + case Decoder::Opcode::DsConsume: + case Decoder::Opcode::DsAppend: + case Decoder::Opcode::DsReadSbyte: + case Decoder::Opcode::DsReadUbyte: + case Decoder::Opcode::DsReadSshort: + case Decoder::Opcode::DsReadUshort: + case Decoder::Opcode::DsRead2B32: + case Decoder::Opcode::DsReadB32: + case Decoder::Opcode::DsReadB64: + case Decoder::Opcode::DsRead2B64: + case Decoder::Opcode::DsRead2St64B64: + case Decoder::Opcode::DsReadB96: + case Decoder::Opcode::DsReadB128: + case Decoder::Opcode::DsWriteByte: + case Decoder::Opcode::DsWriteShort: + case Decoder::Opcode::DsWrite2B32: + case Decoder::Opcode::DsWrite2St64B32: + case Decoder::Opcode::DsWrite2B64: + case Decoder::Opcode::DsWrite2St64B64: + case Decoder::Opcode::DsWriteB32: + case Decoder::Opcode::DsWriteB64: + case Decoder::Opcode::DsWriteB96: + case Decoder::Opcode::DsWriteB128: + case Decoder::Opcode::DsWriteAddtidB32: + case Decoder::Opcode::DsReadAddtidB32: + case Decoder::Opcode::ImageGetResinfo: + case Decoder::Opcode::ImageGetLod: + case Decoder::Opcode::ImageLoad: + case Decoder::Opcode::ImageLoadMip: + case Decoder::Opcode::ImageStore: + case Decoder::Opcode::ImageStoreMip: + case Decoder::Opcode::ImageAtomicAdd: + case Decoder::Opcode::ImageAtomicUMin: + case Decoder::Opcode::ImageAtomicUMax: + case Decoder::Opcode::ImageAtomicAnd: + case Decoder::Opcode::ImageAtomicOr: + case Decoder::Opcode::ImageAtomicXor: + case Decoder::Opcode::ImageGather4Lz: + case Decoder::Opcode::ImageGather4C: + case Decoder::Opcode::ImageGather4CLz: + case Decoder::Opcode::ImageGather4LzO: + case Decoder::Opcode::ImageGather4CO: + case Decoder::Opcode::ImageGather4CLzO: + case Decoder::Opcode::ImageGather4H: + case Decoder::Opcode::ImageSample: return true; + default: return false; + } +} + +} // namespace Libs::Graphics::ShaderRecompiler::IR diff --git a/src/graphics/shader/recompiler/shaderIR/ShaderIROpcodes.cpp b/src/graphics/shader/recompiler/shaderIR/ShaderIROpcodes.cpp new file mode 100644 index 000000000..dba907bd1 --- /dev/null +++ b/src/graphics/shader/recompiler/shaderIR/ShaderIROpcodes.cpp @@ -0,0 +1,534 @@ +#include "graphics/shader/recompiler/ShaderIR.h" + +#include "graphics/shader/recompiler/shaderIR/ShaderIRInternal.h" +#include "common/assert.h" +#include "graphics/shader/recompiler/BufferFormat.h" + +#include +#include +#include + +namespace Libs::Graphics::ShaderRecompiler::IR { +namespace { + +struct LowerMap { + Decoder::Opcode decoded = Decoder::Opcode::Unknown; + Opcode ir = Opcode::MoveU32; +}; + +constexpr LowerMap LOWER_OPS[] = { + {Decoder::Opcode::SMovB32, Opcode::MoveU32}, + {Decoder::Opcode::SMovB64, Opcode::MoveU64}, + {Decoder::Opcode::SWqmB64, Opcode::WqmB64}, + {Decoder::Opcode::SMovkI32, Opcode::MoveU32}, + {Decoder::Opcode::SAbsI32, Opcode::AbsI32}, + {Decoder::Opcode::SBrevB32, Opcode::BitReverseU32}, + {Decoder::Opcode::SBcnt1I32B32, Opcode::BitCountU32}, + {Decoder::Opcode::SBcnt1I32B64, Opcode::BitCountU64}, + {Decoder::Opcode::SFf1I32B32, Opcode::FindLsbU32}, + {Decoder::Opcode::SFlbitI32B64, Opcode::FindMsbFromHighU64}, + {Decoder::Opcode::SBitreplicateB64B32, Opcode::BitReplicateB64B32}, + {Decoder::Opcode::SNotB32, Opcode::BitwiseNotU32}, + {Decoder::Opcode::SNotB64, Opcode::BitwiseNotU64}, + {Decoder::Opcode::SAddU32, Opcode::IAddU32}, + {Decoder::Opcode::SSubU32, Opcode::ISubU32}, + {Decoder::Opcode::SAddI32, Opcode::IAddU32}, + {Decoder::Opcode::SSubI32, Opcode::ISubU32}, + {Decoder::Opcode::SMinI32, Opcode::IMinI32}, + {Decoder::Opcode::SMaxI32, Opcode::IMaxI32}, + {Decoder::Opcode::SMinU32, Opcode::UMinU32}, + {Decoder::Opcode::SMaxU32, Opcode::UMaxU32}, + {Decoder::Opcode::SAndB32, Opcode::BitwiseAndU32}, + {Decoder::Opcode::SAndB64, Opcode::BitwiseAndU64}, + {Decoder::Opcode::SAndn2B32, Opcode::BitwiseAndNotU32}, + {Decoder::Opcode::SAndn2B64, Opcode::BitwiseAndNotU64}, + {Decoder::Opcode::SOrB32, Opcode::BitwiseOrU32}, + {Decoder::Opcode::SOrB64, Opcode::BitwiseOrU64}, + {Decoder::Opcode::SOrn2B32, Opcode::BitwiseOrNotU32}, + {Decoder::Opcode::SOrn2B64, Opcode::BitwiseOrNotU64}, + {Decoder::Opcode::SXorB32, Opcode::BitwiseXorU32}, + {Decoder::Opcode::SXorB64, Opcode::BitwiseXorU64}, + {Decoder::Opcode::SNandB32, Opcode::BitwiseNandU32}, + {Decoder::Opcode::SNandB64, Opcode::BitwiseNandU64}, + {Decoder::Opcode::SNorB32, Opcode::BitwiseNorU32}, + {Decoder::Opcode::SNorB64, Opcode::BitwiseNorU64}, + {Decoder::Opcode::SXnorB32, Opcode::BitwiseXnorU32}, + {Decoder::Opcode::SXnorB64, Opcode::BitwiseXnorU64}, + {Decoder::Opcode::SLshlB32, Opcode::ShiftLeftLogicalU32}, + {Decoder::Opcode::SLshlB64, Opcode::ShiftLeftLogicalU64}, + {Decoder::Opcode::SLshrB32, Opcode::ShiftRightLogicalU32}, + {Decoder::Opcode::SLshrB64, Opcode::ShiftRightLogicalU64}, + {Decoder::Opcode::SAshrI32, Opcode::ShiftRightArithmeticI32}, + {Decoder::Opcode::SMulI32, Opcode::IMulU32}, + {Decoder::Opcode::SMulHiU32, Opcode::UMulHighU32}, + {Decoder::Opcode::SMulkI32, Opcode::IMulU32}, + {Decoder::Opcode::SBfeU32, Opcode::BitFieldExtractU32}, + {Decoder::Opcode::SBfeU64, Opcode::BitFieldExtractU64}, + {Decoder::Opcode::SBfmB32, Opcode::BitFieldMaskU32}, + {Decoder::Opcode::SBfmB64, Opcode::BitFieldMaskU64}, + {Decoder::Opcode::SBitcmp0B32, Opcode::BitCompare0B32}, + {Decoder::Opcode::SBitcmp1B32, Opcode::BitCompare1B32}, + {Decoder::Opcode::SPackLlB32B16, Opcode::PackLowLowU16}, + {Decoder::Opcode::SPackLhB32B16, Opcode::PackLowHighU16}, + {Decoder::Opcode::SPackHhB32B16, Opcode::PackHighHighU16}, + {Decoder::Opcode::SCmpEqI32, Opcode::CompareEqI32}, + {Decoder::Opcode::SCmpLgI32, Opcode::CompareNeI32}, + {Decoder::Opcode::SCmpGtI32, Opcode::CompareGtI32}, + {Decoder::Opcode::SCmpGeI32, Opcode::CompareGeI32}, + {Decoder::Opcode::SCmpLtI32, Opcode::CompareLtI32}, + {Decoder::Opcode::SCmpLeI32, Opcode::CompareLeI32}, + {Decoder::Opcode::SCmpEqU32, Opcode::CompareEqU32}, + {Decoder::Opcode::SCmpLgU32, Opcode::CompareNeU32}, + {Decoder::Opcode::SCmpGtU32, Opcode::CompareGtU32}, + {Decoder::Opcode::SCmpGeU32, Opcode::CompareGeU32}, + {Decoder::Opcode::SCmpLtU32, Opcode::CompareLtU32}, + {Decoder::Opcode::SCmpLeU32, Opcode::CompareLeU32}, + {Decoder::Opcode::SCmpLgU64, Opcode::CompareNeU64}, + {Decoder::Opcode::VReadfirstlaneB32, Opcode::ReadFirstLaneU32}, + {Decoder::Opcode::VReadlaneB32, Opcode::ReadLaneU32}, + {Decoder::Opcode::VWritelaneB32, Opcode::WriteLaneU32}, + {Decoder::Opcode::VPermlane16B32, Opcode::Permlane16B32}, + {Decoder::Opcode::VPermlanex16B32, Opcode::Permlanex16B32}, + {Decoder::Opcode::VCvtF32I32, Opcode::ConvertI32ToF32}, + {Decoder::Opcode::VCvtF32U32, Opcode::ConvertU32ToF32}, + {Decoder::Opcode::VCvtU32F32, Opcode::ConvertF32ToU32}, + {Decoder::Opcode::VCvtI32F32, Opcode::ConvertF32ToI32}, + {Decoder::Opcode::VCvtF16F32, Opcode::ConvertF32ToF16}, + {Decoder::Opcode::VCvtF32F16, Opcode::ConvertF16ToF32}, + {Decoder::Opcode::VCvtF16U16, Opcode::ConvertU16ToF16}, + {Decoder::Opcode::VCvtU16F16, Opcode::ConvertF16ToU16}, + {Decoder::Opcode::VCvtF16I16, Opcode::ConvertI16ToF16}, + {Decoder::Opcode::VCvtI16F16, Opcode::ConvertF16ToI16}, + {Decoder::Opcode::VCvtRpiI32F32, Opcode::ConvertRoundPlusInfF32ToI32}, + {Decoder::Opcode::VCvtFlrI32F32, Opcode::ConvertFloorF32ToI32}, + {Decoder::Opcode::VCvtOffF32I4, Opcode::ConvertI4ToOffsetF32}, + {Decoder::Opcode::VRcpF32, Opcode::RcpF32}, + {Decoder::Opcode::VFractF32, Opcode::FractF32}, + {Decoder::Opcode::VTruncF32, Opcode::TruncF32}, + {Decoder::Opcode::VCeilF32, Opcode::CeilF32}, + {Decoder::Opcode::VRndneF32, Opcode::RoundEvenF32}, + {Decoder::Opcode::VFloorF32, Opcode::FloorF32}, + {Decoder::Opcode::VExpF32, Opcode::Exp2F32}, + {Decoder::Opcode::VLogF32, Opcode::Log2F32}, + {Decoder::Opcode::VRsqF32, Opcode::InverseSqrtF32}, + {Decoder::Opcode::VSqrtF32, Opcode::SqrtF32}, + {Decoder::Opcode::VRcpF16, Opcode::RcpF16}, + {Decoder::Opcode::VSqrtF16, Opcode::SqrtF16}, + {Decoder::Opcode::VRsqF16, Opcode::InverseSqrtF16}, + {Decoder::Opcode::VLogF16, Opcode::Log2F16}, + {Decoder::Opcode::VExpF16, Opcode::Exp2F16}, + {Decoder::Opcode::VFloorF16, Opcode::FloorF16}, + {Decoder::Opcode::VCeilF16, Opcode::CeilF16}, + {Decoder::Opcode::VTruncF16, Opcode::TruncF16}, + {Decoder::Opcode::VRndneF16, Opcode::RoundEvenF16}, + {Decoder::Opcode::VSinF32, Opcode::SinF32}, + {Decoder::Opcode::VCosF32, Opcode::CosF32}, + {Decoder::Opcode::VNotB32, Opcode::BitwiseNotU32}, + {Decoder::Opcode::VBfrevB32, Opcode::BitReverseU32}, + {Decoder::Opcode::VFfblB32, Opcode::FindLsbU32}, + {Decoder::Opcode::VFfbhU32, Opcode::FindMsbFromHighU32}, + {Decoder::Opcode::VAddF32, Opcode::FAddF32}, + {Decoder::Opcode::VSubF32, Opcode::FSubF32}, + {Decoder::Opcode::VSubrevF32, Opcode::FSubF32}, + {Decoder::Opcode::VMulF32, Opcode::FMulF32}, + {Decoder::Opcode::VMacF32, Opcode::FMadF32}, + {Decoder::Opcode::VMadmkF32, Opcode::FMadF32}, + {Decoder::Opcode::VMadakF32, Opcode::FMadF32}, + {Decoder::Opcode::VMinF32, Opcode::FMinF32}, + {Decoder::Opcode::VMaxF32, Opcode::FMaxF32}, + {Decoder::Opcode::VMinI32, Opcode::IMinI32}, + {Decoder::Opcode::VMaxI32, Opcode::IMaxI32}, + {Decoder::Opcode::VMadF32, Opcode::FMadF32}, + {Decoder::Opcode::VMadI32I24, Opcode::IMadI24U32}, + {Decoder::Opcode::VMadU32U24, Opcode::UMadU24U32}, + {Decoder::Opcode::VCubeidF32, Opcode::CubeIdF32}, + {Decoder::Opcode::VCubescF32, Opcode::CubeScF32}, + {Decoder::Opcode::VCubetcF32, Opcode::CubeTcF32}, + {Decoder::Opcode::VCubemaF32, Opcode::CubeMaF32}, + {Decoder::Opcode::VFmaF32, Opcode::FMadF32}, + {Decoder::Opcode::VBfeU32, Opcode::BitFieldExtract3U32}, + {Decoder::Opcode::VBfeI32, Opcode::BitFieldExtract3I32}, + {Decoder::Opcode::VBfiB32, Opcode::BitFieldInsertSelectU32}, + {Decoder::Opcode::VAlignbitB32, Opcode::AlignBitU32}, + {Decoder::Opcode::VMin3F32, Opcode::FMin3F32}, + {Decoder::Opcode::VMin3I32, Opcode::IMin3I32}, + {Decoder::Opcode::VMin3U32, Opcode::UMin3U32}, + {Decoder::Opcode::VMin3F16, Opcode::Min3F16}, + {Decoder::Opcode::VMax3F32, Opcode::FMax3F32}, + {Decoder::Opcode::VMax3I32, Opcode::IMax3I32}, + {Decoder::Opcode::VMax3U32, Opcode::UMax3U32}, + {Decoder::Opcode::VMax3F16, Opcode::Max3F16}, + {Decoder::Opcode::VMed3F32, Opcode::FMed3F32}, + {Decoder::Opcode::VMed3I32, Opcode::IMed3I32}, + {Decoder::Opcode::VMed3U32, Opcode::UMed3U32}, + {Decoder::Opcode::VMed3F16, Opcode::Med3F16}, + {Decoder::Opcode::VSadU32, Opcode::SadU32}, + {Decoder::Opcode::VAdd3U32, Opcode::IAdd3U32}, + {Decoder::Opcode::VLshlAddU32, Opcode::ShiftLeftAddU32}, + {Decoder::Opcode::VAddLshlU32, Opcode::AddShiftLeftU32}, + {Decoder::Opcode::VXadU32, Opcode::XorAddU32}, + {Decoder::Opcode::VLshlOrB32, Opcode::ShiftLeftOrU32}, + {Decoder::Opcode::VAndOrB32, Opcode::BitwiseAndOrU32}, + {Decoder::Opcode::VOr3B32, Opcode::BitwiseOr3U32}, + {Decoder::Opcode::VXor3B32, Opcode::BitwiseXor3U32}, + {Decoder::Opcode::VAddNcU32, Opcode::IAddU32}, + {Decoder::Opcode::VSubNcU32, Opcode::ISubU32}, + {Decoder::Opcode::VSubrevNcU32, Opcode::ISubU32}, + {Decoder::Opcode::VAddNcU16, Opcode::IAddU16}, + {Decoder::Opcode::VSubNcU16, Opcode::ISubI16}, + {Decoder::Opcode::VMaxU16, Opcode::UMaxU16}, + {Decoder::Opcode::VMaxI16, Opcode::IMaxI16}, + {Decoder::Opcode::VMinU16, Opcode::UMinU16}, + {Decoder::Opcode::VMinI16, Opcode::IMinI16}, + {Decoder::Opcode::VAddNcI16, Opcode::IAddU16}, + {Decoder::Opcode::VSubNcI16, Opcode::ISubI16}, + {Decoder::Opcode::VMulI32I24, Opcode::IMulI24U32}, + {Decoder::Opcode::VMulU32U24, Opcode::UMulU24U32}, + {Decoder::Opcode::VMulLoU32, Opcode::IMulU32}, + {Decoder::Opcode::VMulHiU32, Opcode::UMulHighU32}, + {Decoder::Opcode::VMulLoI32, Opcode::IMulU32}, + {Decoder::Opcode::VMulHiI32, Opcode::SMulHighI32}, + {Decoder::Opcode::VAddI32, Opcode::IAddU32}, + {Decoder::Opcode::VSubI32, Opcode::ISubU32}, + {Decoder::Opcode::VSubrevI32, Opcode::ISubU32}, + {Decoder::Opcode::VBfmB32, Opcode::BitFieldMaskU32}, + {Decoder::Opcode::VCvtPkrtzF16F32, Opcode::PackF32ToF16Rtz}, + {Decoder::Opcode::VLdexpF32, Opcode::LdexpF32}, + {Decoder::Opcode::VCvtPknormI16F32, Opcode::PackSnorm2x16F32}, + {Decoder::Opcode::VCvtPknormU16F32, Opcode::PackUnorm2x16F32}, + {Decoder::Opcode::VCvtPkU16U32, Opcode::PackU16U32}, + {Decoder::Opcode::VCvtPkU8F32, Opcode::PackU8F32}, + {Decoder::Opcode::VPackB32F16, Opcode::PackB32F16}, + {Decoder::Opcode::VPkMadI16, Opcode::PackedMadI16}, + {Decoder::Opcode::VPkMulLoU16, Opcode::PackedMulLoU16}, + {Decoder::Opcode::VPkAddI16, Opcode::PackedAddI16}, + {Decoder::Opcode::VPkSubI16, Opcode::PackedSubI16}, + {Decoder::Opcode::VPkLshlrevB16, Opcode::PackedLshlrevB16}, + {Decoder::Opcode::VPkLshrrevB16, Opcode::PackedLshrrevB16}, + {Decoder::Opcode::VPkAshrrevI16, Opcode::PackedAshrrevI16}, + {Decoder::Opcode::VPkMaxI16, Opcode::PackedMaxI16}, + {Decoder::Opcode::VPkMinI16, Opcode::PackedMinI16}, + {Decoder::Opcode::VPkMadU16, Opcode::PackedMadU16}, + {Decoder::Opcode::VPkAddU16, Opcode::PackedAddU16}, + {Decoder::Opcode::VPkSubU16, Opcode::PackedSubU16}, + {Decoder::Opcode::VPkMaxU16, Opcode::PackedMaxU16}, + {Decoder::Opcode::VPkMinU16, Opcode::PackedMinU16}, + {Decoder::Opcode::VPkAddF16, Opcode::PackedAddF16}, + {Decoder::Opcode::VPkMulF16, Opcode::PackedMulF16}, + {Decoder::Opcode::VPkMinF16, Opcode::PackedMinF16}, + {Decoder::Opcode::VPkMaxF16, Opcode::PackedMaxF16}, + {Decoder::Opcode::VPkFmaF16, Opcode::PackedFmaF16}, + {Decoder::Opcode::VPkFmacF16, Opcode::PackedFmaF16}, + {Decoder::Opcode::VAddF16, Opcode::AddF16}, + {Decoder::Opcode::VSubF16, Opcode::SubF16}, + {Decoder::Opcode::VSubrevF16, Opcode::SubF16}, + {Decoder::Opcode::VMulF16, Opcode::MulF16}, + {Decoder::Opcode::VFmacF16, Opcode::FmaF16}, + {Decoder::Opcode::VFmamkF16, Opcode::FmaF16}, + {Decoder::Opcode::VFmaakF16, Opcode::FmaF16}, + {Decoder::Opcode::VFmaF16, Opcode::FmaF16}, + {Decoder::Opcode::VMadMixloF16, Opcode::MadMixF16}, + {Decoder::Opcode::VMadMixhiF16, Opcode::MadMixF16}, + {Decoder::Opcode::VAndB32, Opcode::BitwiseAndU32}, + {Decoder::Opcode::VOrB32, Opcode::BitwiseOrU32}, + {Decoder::Opcode::VXorB32, Opcode::BitwiseXorU32}, + {Decoder::Opcode::VXnorB32, Opcode::BitwiseXnorU32}, + {Decoder::Opcode::VBcntU32B32, Opcode::BitCountAddU32}, + {Decoder::Opcode::VMbcntLoU32B32, Opcode::MaskedBitCountLowU32}, + {Decoder::Opcode::VMbcntHiU32B32, Opcode::MaskedBitCountHighU32}, + {Decoder::Opcode::VLshlB32, Opcode::ShiftLeftLogicalU32}, + {Decoder::Opcode::VLshlrevB32, Opcode::ShiftLeftLogicalU32}, + {Decoder::Opcode::VLshlrevB16, Opcode::ShiftLeftLogicalU16}, + {Decoder::Opcode::VLshrB32, Opcode::ShiftRightLogicalU32}, + {Decoder::Opcode::VLshrrevB32, Opcode::ShiftRightLogicalU32}, + {Decoder::Opcode::VLshrrevB16, Opcode::ShiftRightLogicalU16}, + {Decoder::Opcode::VAshrI32, Opcode::ShiftRightArithmeticI32}, + {Decoder::Opcode::VAshrrevI32, Opcode::ShiftRightArithmeticI32}, + {Decoder::Opcode::VAshrrevI16, Opcode::ShiftRightArithmeticI16}, + {Decoder::Opcode::VMinU32, Opcode::UMinU32}, + {Decoder::Opcode::VMaxU32, Opcode::UMaxU32}, + {Decoder::Opcode::VMaxF16, Opcode::MaxF16}, + {Decoder::Opcode::VMinF16, Opcode::MinF16}, + {Decoder::Opcode::VCmpFF32, Opcode::CompareFalse}, + {Decoder::Opcode::VCmpLtF32, Opcode::CompareLtF32}, + {Decoder::Opcode::VCmpEqF32, Opcode::CompareEqF32}, + {Decoder::Opcode::VCmpLeF32, Opcode::CompareLeF32}, + {Decoder::Opcode::VCmpGtF32, Opcode::CompareGtF32}, + {Decoder::Opcode::VCmpLgF32, Opcode::CompareNeF32}, + {Decoder::Opcode::VCmpGeF32, Opcode::CompareGeF32}, + {Decoder::Opcode::VCmpOF32, Opcode::CompareOrderedF32}, + {Decoder::Opcode::VCmpUF32, Opcode::CompareUnorderedF32}, + {Decoder::Opcode::VCmpNgeF32, Opcode::CompareUnordLtF32}, + {Decoder::Opcode::VCmpNlgF32, Opcode::CompareUnordEqF32}, + {Decoder::Opcode::VCmpNgtF32, Opcode::CompareUnordLeF32}, + {Decoder::Opcode::VCmpNleF32, Opcode::CompareUnordGtF32}, + {Decoder::Opcode::VCmpNeqF32, Opcode::CompareUnordNeF32}, + {Decoder::Opcode::VCmpNltF32, Opcode::CompareUnordGeF32}, + {Decoder::Opcode::VCmpTruF32, Opcode::CompareTrue}, + {Decoder::Opcode::VCmpClassF32, Opcode::CompareClassF32}, + {Decoder::Opcode::VCmpLtF16, Opcode::CompareLtF16}, + {Decoder::Opcode::VCmpEqF16, Opcode::CompareEqF16}, + {Decoder::Opcode::VCmpLeF16, Opcode::CompareLeF16}, + {Decoder::Opcode::VCmpGtF16, Opcode::CompareGtF16}, + {Decoder::Opcode::VCmpLgF16, Opcode::CompareNeF16}, + {Decoder::Opcode::VCmpGeF16, Opcode::CompareGeF16}, + {Decoder::Opcode::VCmpNeqF16, Opcode::CompareUnordNeF16}, + {Decoder::Opcode::VCmpxLtF16, Opcode::CompareMaskLtF16}, + {Decoder::Opcode::VCmpxEqF16, Opcode::CompareMaskEqF16}, + {Decoder::Opcode::VCmpxLeF16, Opcode::CompareMaskLeF16}, + {Decoder::Opcode::VCmpxGtF16, Opcode::CompareMaskGtF16}, + {Decoder::Opcode::VCmpxGeF16, Opcode::CompareMaskGeF16}, + {Decoder::Opcode::VCmpxNeqF16, Opcode::CompareMaskUnordNeF16}, + {Decoder::Opcode::VCmpxNltF16, Opcode::CompareMaskUnordGeF16}, + {Decoder::Opcode::VCmpxLtF32, Opcode::CompareMaskLtF32}, + {Decoder::Opcode::VCmpxEqF32, Opcode::CompareMaskEqF32}, + {Decoder::Opcode::VCmpxLeF32, Opcode::CompareMaskLeF32}, + {Decoder::Opcode::VCmpxGtF32, Opcode::CompareMaskGtF32}, + {Decoder::Opcode::VCmpxLgF32, Opcode::CompareMaskNeF32}, + {Decoder::Opcode::VCmpxGeF32, Opcode::CompareMaskGeF32}, + {Decoder::Opcode::VCmpxNgeF32, Opcode::CompareMaskUnordLtF32}, + {Decoder::Opcode::VCmpxNlgF32, Opcode::CompareMaskUnordEqF32}, + {Decoder::Opcode::VCmpxNgtF32, Opcode::CompareMaskUnordLeF32}, + {Decoder::Opcode::VCmpxNleF32, Opcode::CompareMaskUnordGtF32}, + {Decoder::Opcode::VCmpxNeqF32, Opcode::CompareMaskUnordNeF32}, + {Decoder::Opcode::VCmpxNltF32, Opcode::CompareMaskUnordGeF32}, + {Decoder::Opcode::VCmpFI32, Opcode::CompareFalse}, + {Decoder::Opcode::VCmpLtI32, Opcode::CompareLtI32}, + {Decoder::Opcode::VCmpEqI32, Opcode::CompareEqI32}, + {Decoder::Opcode::VCmpLeI32, Opcode::CompareLeI32}, + {Decoder::Opcode::VCmpGtI32, Opcode::CompareGtI32}, + {Decoder::Opcode::VCmpNeI32, Opcode::CompareNeI32}, + {Decoder::Opcode::VCmpGeI32, Opcode::CompareGeI32}, + {Decoder::Opcode::VCmpTI32, Opcode::CompareTrue}, + {Decoder::Opcode::VCmpLtI16, Opcode::CompareLtI16}, + {Decoder::Opcode::VCmpEqI16, Opcode::CompareEqI16}, + {Decoder::Opcode::VCmpLeI16, Opcode::CompareLeI16}, + {Decoder::Opcode::VCmpGtI16, Opcode::CompareGtI16}, + {Decoder::Opcode::VCmpNeI16, Opcode::CompareNeI16}, + {Decoder::Opcode::VCmpGeI16, Opcode::CompareGeI16}, + {Decoder::Opcode::VCmpxLtI32, Opcode::CompareMaskLtI32}, + {Decoder::Opcode::VCmpxEqI32, Opcode::CompareMaskEqI32}, + {Decoder::Opcode::VCmpxLeI32, Opcode::CompareMaskLeI32}, + {Decoder::Opcode::VCmpxGtI32, Opcode::CompareMaskGtI32}, + {Decoder::Opcode::VCmpxNeI32, Opcode::CompareMaskNeI32}, + {Decoder::Opcode::VCmpxGeI32, Opcode::CompareMaskGeI32}, + {Decoder::Opcode::VCmpLtU16, Opcode::CompareLtU16}, + {Decoder::Opcode::VCmpEqU16, Opcode::CompareEqU16}, + {Decoder::Opcode::VCmpLeU16, Opcode::CompareLeU16}, + {Decoder::Opcode::VCmpGtU16, Opcode::CompareGtU16}, + {Decoder::Opcode::VCmpNeU16, Opcode::CompareNeU16}, + {Decoder::Opcode::VCmpGeU16, Opcode::CompareGeU16}, + {Decoder::Opcode::VCmpFU32, Opcode::CompareFalse}, + {Decoder::Opcode::VCmpLtU32, Opcode::CompareLtU32}, + {Decoder::Opcode::VCmpEqU32, Opcode::CompareEqU32}, + {Decoder::Opcode::VCmpLeU32, Opcode::CompareLeU32}, + {Decoder::Opcode::VCmpGtU32, Opcode::CompareGtU32}, + {Decoder::Opcode::VCmpNeU32, Opcode::CompareNeU32}, + {Decoder::Opcode::VCmpGeU32, Opcode::CompareGeU32}, + {Decoder::Opcode::VCmpTU32, Opcode::CompareTrue}, + {Decoder::Opcode::VCmpNeU64, Opcode::CompareNeU64}, + {Decoder::Opcode::VCmpxLtU32, Opcode::CompareMaskLtU32}, + {Decoder::Opcode::VCmpxEqU32, Opcode::CompareMaskEqU32}, + {Decoder::Opcode::VCmpxLeU32, Opcode::CompareMaskLeU32}, + {Decoder::Opcode::VCmpxGtU32, Opcode::CompareMaskGtU32}, + {Decoder::Opcode::VCmpxNeU32, Opcode::CompareMaskNeU32}, + {Decoder::Opcode::VCmpxGeU32, Opcode::CompareMaskGeU32}, + {Decoder::Opcode::SLoadDword, Opcode::SLoadDword}, + {Decoder::Opcode::SBufferLoadDword, Opcode::SBufferLoadDword}, + {Decoder::Opcode::BufferLoadDword, Opcode::BufferLoadDword}, + {Decoder::Opcode::BufferLoadSbyte, Opcode::BufferLoadSbyte}, + {Decoder::Opcode::BufferLoadSshort, Opcode::BufferLoadSshort}, + {Decoder::Opcode::BufferStoreByte, Opcode::BufferStoreByte}, + {Decoder::Opcode::BufferStoreShort, Opcode::BufferStoreShort}, + {Decoder::Opcode::BufferStoreDword, Opcode::BufferStoreDword}, + {Decoder::Opcode::BufferAtomicSwap, Opcode::AtomicSwapU32}, + {Decoder::Opcode::BufferAtomicAdd, Opcode::AtomicAddU32}, + {Decoder::Opcode::BufferAtomicSub, Opcode::AtomicSubU32}, + {Decoder::Opcode::BufferAtomicSMin, Opcode::AtomicSMinI32}, + {Decoder::Opcode::BufferAtomicUMin, Opcode::AtomicUMinU32}, + {Decoder::Opcode::BufferAtomicSMax, Opcode::AtomicSMaxI32}, + {Decoder::Opcode::BufferAtomicUMax, Opcode::AtomicUMaxU32}, + {Decoder::Opcode::BufferAtomicAnd, Opcode::AtomicAndU32}, + {Decoder::Opcode::BufferAtomicOr, Opcode::AtomicOrU32}, + {Decoder::Opcode::BufferAtomicXor, Opcode::AtomicXorU32}, + {Decoder::Opcode::FlatLoadUbyte, Opcode::FlatLoadUbyte}, + {Decoder::Opcode::FlatLoadSbyte, Opcode::FlatLoadSbyte}, + {Decoder::Opcode::FlatLoadSshort, Opcode::FlatLoadSshort}, + {Decoder::Opcode::FlatStoreByte, Opcode::FlatStoreByte}, + {Decoder::Opcode::FlatStoreShort, Opcode::FlatStoreShort}, + {Decoder::Opcode::FlatStoreDword, Opcode::FlatStoreDword}, + {Decoder::Opcode::DsAddU32, Opcode::AtomicAddU32}, + {Decoder::Opcode::DsAddRtnU32, Opcode::AtomicAddU32}, + {Decoder::Opcode::DsSubU32, Opcode::AtomicSubU32}, + {Decoder::Opcode::DsSubRtnU32, Opcode::AtomicSubU32}, + {Decoder::Opcode::DsMinI32, Opcode::AtomicSMinI32}, + {Decoder::Opcode::DsMinRtnI32, Opcode::AtomicSMinI32}, + {Decoder::Opcode::DsMaxI32, Opcode::AtomicSMaxI32}, + {Decoder::Opcode::DsMaxRtnI32, Opcode::AtomicSMaxI32}, + {Decoder::Opcode::DsMinU32, Opcode::AtomicUMinU32}, + {Decoder::Opcode::DsMinRtnU32, Opcode::AtomicUMinU32}, + {Decoder::Opcode::DsMaxU32, Opcode::AtomicUMaxU32}, + {Decoder::Opcode::DsMaxRtnU32, Opcode::AtomicUMaxU32}, + {Decoder::Opcode::DsAndB32, Opcode::AtomicAndU32}, + {Decoder::Opcode::DsAndRtnB32, Opcode::AtomicAndU32}, + {Decoder::Opcode::DsOrB32, Opcode::AtomicOrU32}, + {Decoder::Opcode::DsOrRtnB32, Opcode::AtomicOrU32}, + {Decoder::Opcode::DsXorB32, Opcode::AtomicXorU32}, + {Decoder::Opcode::DsXorRtnB32, Opcode::AtomicXorU32}, + {Decoder::Opcode::DsWrxchgRtnB32, Opcode::AtomicSwapU32}, + {Decoder::Opcode::DsReadSbyte, Opcode::DsReadSbyte}, + {Decoder::Opcode::DsReadUbyte, Opcode::DsReadUbyte}, + {Decoder::Opcode::DsReadSshort, Opcode::DsReadSshort}, + {Decoder::Opcode::DsReadUshort, Opcode::DsReadUshort}, + {Decoder::Opcode::DsRead2B32, Opcode::DsReadB32}, + {Decoder::Opcode::DsReadB32, Opcode::DsReadB32}, + {Decoder::Opcode::DsReadB64, Opcode::DsReadB32}, + {Decoder::Opcode::DsRead2B64, Opcode::DsReadB32}, + {Decoder::Opcode::DsRead2St64B64, Opcode::DsReadB32}, + {Decoder::Opcode::DsReadB96, Opcode::DsReadB32}, + {Decoder::Opcode::DsReadB128, Opcode::DsReadB32}, + {Decoder::Opcode::DsWriteByte, Opcode::DsWriteByte}, + {Decoder::Opcode::DsWriteShort, Opcode::DsWriteShort}, + {Decoder::Opcode::DsWrite2B32, Opcode::DsWriteB32}, + {Decoder::Opcode::DsWrite2St64B32, Opcode::DsWriteB32}, + {Decoder::Opcode::DsWrite2B64, Opcode::DsWriteB32}, + {Decoder::Opcode::DsWrite2St64B64, Opcode::DsWriteB32}, + {Decoder::Opcode::DsWriteB32, Opcode::DsWriteB32}, + {Decoder::Opcode::DsWriteB64, Opcode::DsWriteB32}, + {Decoder::Opcode::DsWriteB96, Opcode::DsWriteB32}, + {Decoder::Opcode::DsWriteB128, Opcode::DsWriteB32}, + {Decoder::Opcode::DsMinF32, Opcode::DsMinF32}, + {Decoder::Opcode::DsMaxF32, Opcode::DsMaxF32}, + {Decoder::Opcode::DsConsume, Opcode::DsConsume}, + {Decoder::Opcode::DsAppend, Opcode::DsAppend}, + {Decoder::Opcode::DsWriteAddtidB32, Opcode::DsWriteAddtidB32}, + {Decoder::Opcode::DsReadAddtidB32, Opcode::DsReadAddtidB32}, + {Decoder::Opcode::ImageGetResinfo, Opcode::ImageGetResinfo}, + {Decoder::Opcode::ImageGetLod, Opcode::ImageGetLod}, + {Decoder::Opcode::ImageLoad, Opcode::ImageLoad}, + {Decoder::Opcode::ImageLoadMip, Opcode::ImageLoad}, + {Decoder::Opcode::ImageStore, Opcode::ImageStore}, + {Decoder::Opcode::ImageStoreMip, Opcode::ImageStore}, + {Decoder::Opcode::ImageAtomicAdd, Opcode::AtomicAddU32}, + {Decoder::Opcode::ImageAtomicUMin, Opcode::AtomicUMinU32}, + {Decoder::Opcode::ImageAtomicUMax, Opcode::AtomicUMaxU32}, + {Decoder::Opcode::ImageAtomicAnd, Opcode::AtomicAndU32}, + {Decoder::Opcode::ImageAtomicOr, Opcode::AtomicOrU32}, + {Decoder::Opcode::ImageAtomicXor, Opcode::AtomicXorU32}, + {Decoder::Opcode::ImageSample, Opcode::ImageSample}, + {Decoder::Opcode::ImageGather4Lz, Opcode::ImageGather4}, + {Decoder::Opcode::ImageGather4C, Opcode::ImageGather4}, + {Decoder::Opcode::ImageGather4CLz, Opcode::ImageGather4}, + {Decoder::Opcode::ImageGather4LzO, Opcode::ImageGather4}, + {Decoder::Opcode::ImageGather4CO, Opcode::ImageGather4}, + {Decoder::Opcode::ImageGather4CLzO, Opcode::ImageGather4}, + {Decoder::Opcode::ImageGather4H, Opcode::ImageGather4}, +}; + + +} // namespace + +void SetError(std::string* error, const char* message) { + if (error != nullptr) { + *error = message; + } +} + +Opcode LookupIrOpcode(Decoder::Opcode opcode) { + for (const auto& op: LOWER_OPS) { + if (op.decoded == opcode) { + return op.ir; + } + } + return Opcode::MoveU32; +} + +bool IsImplemented(Decoder::Opcode opcode) { + for (const auto& op: LOWER_OPS) { + if (op.decoded == opcode) { + return true; + } + } + return false; +} + +bool IsReversedBinary(Decoder::Opcode opcode) { + switch (opcode) { + case Decoder::Opcode::VSubrevF32: + case Decoder::Opcode::VSubrevF16: + case Decoder::Opcode::VSubrevNcU32: + case Decoder::Opcode::VLshlrevB32: + case Decoder::Opcode::VLshrrevB32: + case Decoder::Opcode::VAshrrevI32: + case Decoder::Opcode::VLshlrevB16: + case Decoder::Opcode::VLshrrevB16: + case Decoder::Opcode::VAshrrevI16: + case Decoder::Opcode::VSubrevI32: return true; + default: return false; + } +} + +bool IsVectorCarryOutOpcode(Decoder::Opcode opcode) { + switch (opcode) { + case Decoder::Opcode::VAddI32: + case Decoder::Opcode::VSubI32: + case Decoder::Opcode::VSubrevI32: return true; + default: return false; + } +} + +bool ScalarResultWritesSccNonZero(Decoder::Opcode opcode) { + switch (opcode) { + case Decoder::Opcode::SAbsI32: + case Decoder::Opcode::SBcnt1I32B32: + case Decoder::Opcode::SBcnt1I32B64: + case Decoder::Opcode::SAndB32: + case Decoder::Opcode::SAndn2B32: + case Decoder::Opcode::SOrB32: + case Decoder::Opcode::SOrn2B32: + case Decoder::Opcode::SXorB32: + case Decoder::Opcode::SNandB32: + case Decoder::Opcode::SNorB32: + case Decoder::Opcode::SXnorB32: + case Decoder::Opcode::SNotB32: + case Decoder::Opcode::SLshlB32: + case Decoder::Opcode::SLshrB32: + case Decoder::Opcode::SAshrI32: + case Decoder::Opcode::SBfeU32: + case Decoder::Opcode::SAndB64: + case Decoder::Opcode::SAndn2B64: + case Decoder::Opcode::SNotB64: + case Decoder::Opcode::SOrB64: + case Decoder::Opcode::SOrn2B64: + case Decoder::Opcode::SXorB64: + case Decoder::Opcode::SNandB64: + case Decoder::Opcode::SNorB64: + case Decoder::Opcode::SXnorB64: + case Decoder::Opcode::SLshlB64: + case Decoder::Opcode::SLshrB64: + case Decoder::Opcode::SBfeU64: + case Decoder::Opcode::SWqmB64: return true; + default: return false; + } +} + +bool ScalarResultIs64Bit(Decoder::Opcode opcode) { + switch (opcode) { + case Decoder::Opcode::SAndB64: + case Decoder::Opcode::SAndn2B64: + case Decoder::Opcode::SNotB64: + case Decoder::Opcode::SOrB64: + case Decoder::Opcode::SOrn2B64: + case Decoder::Opcode::SXorB64: + case Decoder::Opcode::SNandB64: + case Decoder::Opcode::SNorB64: + case Decoder::Opcode::SXnorB64: + case Decoder::Opcode::SLshlB64: + case Decoder::Opcode::SLshrB64: + case Decoder::Opcode::SBfeU64: + case Decoder::Opcode::SWqmB64: return true; + default: return false; + } +} + +} // namespace Libs::Graphics::ShaderRecompiler::IR diff --git a/src/graphics/shader/recompiler/spirvEmitter/spirvEmitterAluOps.cpp b/src/graphics/shader/recompiler/spirvEmitter/spirvEmitterAluOps.cpp new file mode 100644 index 000000000..f619bea01 --- /dev/null +++ b/src/graphics/shader/recompiler/spirvEmitter/spirvEmitterAluOps.cpp @@ -0,0 +1,2549 @@ +#include "graphics/shader/recompiler/spirvEmitter/spirvEmitterInternal.h" + +namespace Libs::Graphics::ShaderRecompiler::Spirv::Emitter { + +void EmitPerInvocationMask(EmitterState* state, const IR::Operand& dst, uint32_t value_bool) { + const auto value = state->builder.AllocateId(); + state->builder.AddFunction({OpSelect, state->uint_type, value, value_bool, + ConstantU32(state, 1), ConstantU32(state, 0)}); + EmitStoreU32(state, dst, value); + EmitStoreU32(state, OffsetRegisterOperand(dst, 1), ConstantU32(state, 0)); +} + +uint32_t EmitLogicalBinary(EmitterState* state, uint32_t opcode, uint32_t lhs, uint32_t rhs) { + const auto logical_opcode = opcode == OpBitwiseAnd ? OpLogicalAnd + : opcode == OpBitwiseOr ? OpLogicalOr + : OpLogicalNotEqual; + const auto value = state->builder.AllocateId(); + state->builder.AddFunction({logical_opcode, state->bool_type, value, lhs, rhs}); + return value; +} + +void EmitAbsI32(EmitterState* state, const IR::Instruction& inst) { + const auto src = EmitValueLoad(state, inst.src[0]); + const auto neg = state->builder.AllocateId(); + const auto cond = state->builder.AllocateId(); + const auto ret = state->builder.AllocateId(); + state->builder.AddFunction({OpSNegate, state->uint_type, neg, src}); + state->builder.AddFunction({OpSLessThan, state->bool_type, cond, src, ConstantU32(state, 0)}); + state->builder.AddFunction({OpSelect, state->uint_type, ret, cond, neg, src}); + EmitStoreU32(state, inst.dst, ret); +} + +void EmitUnaryU32(EmitterState* state, const IR::Instruction& inst, uint32_t opcode) { + const auto src = EmitValueLoad(state, inst.src[0]); + const auto ret = state->builder.AllocateId(); + state->builder.AddFunction({opcode, state->uint_type, ret, src}); + EmitStoreU32(state, inst.dst, ret); +} + +void EmitUnaryU64(EmitterState* state, const IR::Instruction& inst, uint32_t opcode) { + if (state->per_invocation_masks) { + const auto ret = state->builder.AllocateId(); + state->builder.AddFunction({OpLogicalNot, state->bool_type, ret, + EmitLaneMaskOperandActiveBool(state, inst.src[0])}); + EmitPerInvocationMask(state, inst.dst, ret); + return; + } + const auto low = EmitSequentialValueLoad(state, inst.src[0], 0); + const auto high = EmitSequentialValueLoad(state, inst.src[0], 1); + const auto ret_low = state->builder.AllocateId(); + const auto ret_high = state->builder.AllocateId(); + state->builder.AddFunction({opcode, state->uint_type, ret_low, low}); + state->builder.AddFunction({opcode, state->uint_type, ret_high, high}); + EmitStoreU32(state, inst.dst, ret_low); + EmitStoreU32(state, OffsetRegisterOperand(inst.dst, 1), ret_high); +} + +void EmitFindLsbU32(EmitterState* state, const IR::Instruction& inst) { + const auto src = EmitValueLoad(state, inst.src[0]); + const auto i32 = state->builder.AllocateId(); + const auto u32 = state->builder.AllocateId(); + state->builder.AddFunction( + {OpExtInst, state->int_type, i32, state->glsl_std450, GlslFindILsb, src}); + state->builder.AddFunction({OpBitcast, state->uint_type, u32, i32}); + EmitStoreU32(state, inst.dst, u32); +} + +void EmitFindMsbFromHighU32(EmitterState* state, const IR::Instruction& inst) { + const auto src = EmitValueLoad(state, inst.src[0]); + const auto i32 = state->builder.AllocateId(); + const auto msb = state->builder.AllocateId(); + const auto pos = state->builder.AllocateId(); + const auto non_zero = state->builder.AllocateId(); + const auto ret = state->builder.AllocateId(); + state->builder.AddFunction( + {OpExtInst, state->int_type, i32, state->glsl_std450, GlslFindUMsb, src}); + state->builder.AddFunction({OpBitcast, state->uint_type, msb, i32}); + state->builder.AddFunction({OpISub, state->uint_type, pos, ConstantU32(state, 31), msb}); + state->builder.AddFunction( + {OpINotEqual, state->bool_type, non_zero, src, ConstantU32(state, 0)}); + state->builder.AddFunction( + {OpSelect, state->uint_type, ret, non_zero, pos, ConstantU32(state, 0xffffffffu)}); + EmitStoreU32(state, inst.dst, ret); +} + +void EmitFindMsbFromHighU64(EmitterState* state, const IR::Instruction& inst) { + const auto low = EmitSequentialValueLoad(state, inst.src[0], 0); + const auto high = EmitSequentialValueLoad(state, inst.src[0], 1); + const auto high_i32 = state->builder.AllocateId(); + const auto high_msb = state->builder.AllocateId(); + const auto high_pos = state->builder.AllocateId(); + const auto high_non_zero = state->builder.AllocateId(); + const auto low_i32 = state->builder.AllocateId(); + const auto low_msb = state->builder.AllocateId(); + const auto low_pos = state->builder.AllocateId(); + const auto low_total = state->builder.AllocateId(); + const auto low_non_zero = state->builder.AllocateId(); + const auto low_ret = state->builder.AllocateId(); + const auto ret = state->builder.AllocateId(); + state->builder.AddFunction( + {OpExtInst, state->int_type, high_i32, state->glsl_std450, GlslFindUMsb, high}); + state->builder.AddFunction({OpBitcast, state->uint_type, high_msb, high_i32}); + state->builder.AddFunction( + {OpISub, state->uint_type, high_pos, ConstantU32(state, 31), high_msb}); + state->builder.AddFunction( + {OpINotEqual, state->bool_type, high_non_zero, high, ConstantU32(state, 0)}); + state->builder.AddFunction( + {OpExtInst, state->int_type, low_i32, state->glsl_std450, GlslFindUMsb, low}); + state->builder.AddFunction({OpBitcast, state->uint_type, low_msb, low_i32}); + state->builder.AddFunction( + {OpISub, state->uint_type, low_pos, ConstantU32(state, 31), low_msb}); + state->builder.AddFunction( + {OpIAdd, state->uint_type, low_total, low_pos, ConstantU32(state, 32)}); + state->builder.AddFunction( + {OpINotEqual, state->bool_type, low_non_zero, low, ConstantU32(state, 0)}); + state->builder.AddFunction({OpSelect, state->uint_type, low_ret, low_non_zero, low_total, + ConstantU32(state, 0xffffffffu)}); + state->builder.AddFunction({OpSelect, state->uint_type, ret, high_non_zero, high_pos, low_ret}); + EmitStoreU32(state, inst.dst, ret); +} + +void EmitBitCountU64(EmitterState* state, const IR::Instruction& inst) { + const auto low = EmitSequentialValueLoad(state, inst.src[0], 0); + const auto high = EmitSequentialValueLoad(state, inst.src[0], 1); + const auto low_count = state->builder.AllocateId(); + const auto high_count = state->builder.AllocateId(); + const auto ret = state->builder.AllocateId(); + state->builder.AddFunction({OpBitCount, state->uint_type, low_count, low}); + state->builder.AddFunction({OpBitCount, state->uint_type, high_count, high}); + state->builder.AddFunction({OpIAdd, state->uint_type, ret, low_count, high_count}); + EmitStoreU32(state, inst.dst, ret); +} + +void EmitBitCountU32(EmitterState* state, const IR::Instruction& inst) { + const auto bits = EmitValueLoad(state, inst.src[0]); + const auto count = state->builder.AllocateId(); + state->builder.AddFunction({OpBitCount, state->uint_type, count, bits}); + EmitStoreU32(state, inst.dst, count); +} + +void EmitBitCountAddU32(EmitterState* state, const IR::Instruction& inst) { + const auto bits = EmitValueLoad(state, inst.src[0]); + const auto add = EmitValueLoad(state, inst.src[1]); + const auto count = state->builder.AllocateId(); + const auto ret = state->builder.AllocateId(); + state->builder.AddFunction({OpBitCount, state->uint_type, count, bits}); + state->builder.AddFunction({OpIAdd, state->uint_type, ret, count, add}); + EmitStoreU32(state, inst.dst, ret); +} + +void EmitBinaryU32(EmitterState* state, const IR::Instruction& inst, uint32_t opcode) { + const auto lhs = EmitValueLoad(state, inst.src[0]); + auto rhs = EmitValueLoad(state, inst.src[1]); + if (opcode == OpShiftLeftLogical || opcode == OpShiftRightLogical || + opcode == OpShiftRightArithmetic) { + rhs = EmitAndConstant(state, rhs, 31u); + } + const auto ret = state->builder.AllocateId(); + state->builder.AddFunction({opcode, state->uint_type, ret, lhs, rhs}); + EmitStoreU32(state, inst.dst, ret); +} + +void EmitBinaryNotRhsU32(EmitterState* state, const IR::Instruction& inst, uint32_t opcode) { + const auto lhs = EmitValueLoad(state, inst.src[0]); + const auto rhs = EmitValueLoad(state, inst.src[1]); + const auto not_rhs = state->builder.AllocateId(); + const auto ret = state->builder.AllocateId(); + state->builder.AddFunction({OpNot, state->uint_type, not_rhs, rhs}); + state->builder.AddFunction({opcode, state->uint_type, ret, lhs, not_rhs}); + EmitStoreU32(state, inst.dst, ret); +} + +void EmitBinaryThenNotU32(EmitterState* state, const IR::Instruction& inst, uint32_t opcode) { + const auto lhs = EmitValueLoad(state, inst.src[0]); + const auto rhs = EmitValueLoad(state, inst.src[1]); + const auto tmp = state->builder.AllocateId(); + const auto ret = state->builder.AllocateId(); + state->builder.AddFunction({opcode, state->uint_type, tmp, lhs, rhs}); + state->builder.AddFunction({OpNot, state->uint_type, ret, tmp}); + EmitStoreU32(state, inst.dst, ret); +} + +void EmitMaskedBitCountU32(EmitterState* state, const IR::Instruction& inst, uint32_t exec_index) { + const auto bits = EmitValueLoad(state, inst.src[0]); + const auto add = EmitValueLoad(state, inst.src[1]); + const auto thread_mask = EmitThreadMaskBelowPartU32(state, exec_index); + const auto active_bits = state->builder.AllocateId(); + const auto count = state->builder.AllocateId(); + const auto ret = state->builder.AllocateId(); + state->builder.AddFunction({OpBitwiseAnd, state->uint_type, active_bits, bits, thread_mask}); + state->builder.AddFunction({OpBitCount, state->uint_type, count, active_bits}); + state->builder.AddFunction({OpIAdd, state->uint_type, ret, count, add}); + EmitStoreU32(state, inst.dst, ret); +} + +uint32_t EmitMask5U32(EmitterState* state, uint32_t value) { + const auto ret = state->builder.AllocateId(); + state->builder.AddFunction( + {OpBitwiseAnd, state->uint_type, ret, value, ConstantU32(state, 31)}); + return ret; +} + +void EmitChainedBinaryU32(EmitterState* state, const IR::Instruction& inst, uint32_t first_op, + uint32_t second_op, bool mask_first_rhs, bool mask_second_rhs) { + const auto src0 = EmitValueLoad(state, inst.src[0]); + const auto src1_raw = EmitValueLoad(state, inst.src[1]); + const auto src2_raw = EmitValueLoad(state, inst.src[2]); + const auto src1 = mask_first_rhs ? EmitMask5U32(state, src1_raw) : src1_raw; + const auto src2 = mask_second_rhs ? EmitMask5U32(state, src2_raw) : src2_raw; + const auto tmp = state->builder.AllocateId(); + const auto ret = state->builder.AllocateId(); + state->builder.AddFunction({first_op, state->uint_type, tmp, src0, src1}); + state->builder.AddFunction({second_op, state->uint_type, ret, tmp, src2}); + EmitStoreU32(state, inst.dst, ret); +} + +void EmitBinaryU64(EmitterState* state, const IR::Instruction& inst, uint32_t opcode) { + if (state->per_invocation_masks) { + const auto ret = + EmitLogicalBinary(state, opcode, EmitLaneMaskOperandActiveBool(state, inst.src[0]), + EmitLaneMaskOperandActiveBool(state, inst.src[1])); + EmitPerInvocationMask(state, inst.dst, ret); + return; + } + const auto lhs_low = EmitSequentialValueLoad(state, inst.src[0], 0); + const auto lhs_high = EmitSequentialValueLoad(state, inst.src[0], 1); + const auto rhs_low = EmitSequentialValueLoad(state, inst.src[1], 0); + const auto rhs_high = EmitSequentialValueLoad(state, inst.src[1], 1); + const auto ret_low = state->builder.AllocateId(); + const auto ret_high = state->builder.AllocateId(); + state->builder.AddFunction({opcode, state->uint_type, ret_low, lhs_low, rhs_low}); + state->builder.AddFunction({opcode, state->uint_type, ret_high, lhs_high, rhs_high}); + EmitStoreU32(state, inst.dst, ret_low); + EmitStoreU32(state, OffsetRegisterOperand(inst.dst, 1), ret_high); +} + +void EmitBinaryNotRhsU64(EmitterState* state, const IR::Instruction& inst, uint32_t opcode) { + if (state->per_invocation_masks) { + const auto not_rhs = state->builder.AllocateId(); + state->builder.AddFunction({OpLogicalNot, state->bool_type, not_rhs, + EmitLaneMaskOperandActiveBool(state, inst.src[1])}); + const auto ret = EmitLogicalBinary( + state, opcode, EmitLaneMaskOperandActiveBool(state, inst.src[0]), not_rhs); + EmitPerInvocationMask(state, inst.dst, ret); + return; + } + const auto lhs_low = EmitSequentialValueLoad(state, inst.src[0], 0); + const auto lhs_high = EmitSequentialValueLoad(state, inst.src[0], 1); + const auto rhs_low = EmitSequentialValueLoad(state, inst.src[1], 0); + const auto rhs_high = EmitSequentialValueLoad(state, inst.src[1], 1); + const auto not_low = state->builder.AllocateId(); + const auto not_high = state->builder.AllocateId(); + const auto ret_low = state->builder.AllocateId(); + const auto ret_high = state->builder.AllocateId(); + state->builder.AddFunction({OpNot, state->uint_type, not_low, rhs_low}); + state->builder.AddFunction({OpNot, state->uint_type, not_high, rhs_high}); + state->builder.AddFunction({opcode, state->uint_type, ret_low, lhs_low, not_low}); + state->builder.AddFunction({opcode, state->uint_type, ret_high, lhs_high, not_high}); + EmitStoreU32(state, inst.dst, ret_low); + EmitStoreU32(state, OffsetRegisterOperand(inst.dst, 1), ret_high); +} + +void EmitBinaryThenNotU64(EmitterState* state, const IR::Instruction& inst, uint32_t opcode) { + if (state->per_invocation_masks) { + const auto value = + EmitLogicalBinary(state, opcode, EmitLaneMaskOperandActiveBool(state, inst.src[0]), + EmitLaneMaskOperandActiveBool(state, inst.src[1])); + const auto ret = state->builder.AllocateId(); + state->builder.AddFunction({OpLogicalNot, state->bool_type, ret, value}); + EmitPerInvocationMask(state, inst.dst, ret); + return; + } + const auto lhs_low = EmitSequentialValueLoad(state, inst.src[0], 0); + const auto lhs_high = EmitSequentialValueLoad(state, inst.src[0], 1); + const auto rhs_low = EmitSequentialValueLoad(state, inst.src[1], 0); + const auto rhs_high = EmitSequentialValueLoad(state, inst.src[1], 1); + const auto tmp_low = state->builder.AllocateId(); + const auto tmp_high = state->builder.AllocateId(); + const auto ret_low = state->builder.AllocateId(); + const auto ret_high = state->builder.AllocateId(); + state->builder.AddFunction({opcode, state->uint_type, tmp_low, lhs_low, rhs_low}); + state->builder.AddFunction({opcode, state->uint_type, tmp_high, lhs_high, rhs_high}); + state->builder.AddFunction({OpNot, state->uint_type, ret_low, tmp_low}); + state->builder.AddFunction({OpNot, state->uint_type, ret_high, tmp_high}); + EmitStoreU32(state, inst.dst, ret_low); + EmitStoreU32(state, OffsetRegisterOperand(inst.dst, 1), ret_high); +} + +void EmitSelectU64(EmitterState* state, const IR::Instruction& inst) { + const auto cond = EmitConditionBool(state, inst.src[0]); + const auto true_low = EmitSequentialValueLoad(state, inst.src[1], 0); + const auto true_high = EmitSequentialValueLoad(state, inst.src[1], 1); + const auto false_low = EmitSequentialValueLoad(state, inst.src[2], 0); + const auto false_high = EmitSequentialValueLoad(state, inst.src[2], 1); + const auto ret_low = state->builder.AllocateId(); + const auto ret_high = state->builder.AllocateId(); + state->builder.AddFunction({OpSelect, state->uint_type, ret_low, cond, true_low, false_low}); + state->builder.AddFunction({OpSelect, state->uint_type, ret_high, cond, true_high, false_high}); + EmitStoreU32(state, inst.dst, ret_low); + EmitStoreU32(state, OffsetRegisterOperand(inst.dst, 1), ret_high); +} + +uint32_t EmitSelectValueU32(EmitterState* state, uint32_t cond, uint32_t true_value, + uint32_t false_value) { + const auto ret = state->builder.AllocateId(); + state->builder.AddFunction({OpSelect, state->uint_type, ret, cond, true_value, false_value}); + return ret; +} + +void EmitStoreSccBool(EmitterState* state, uint32_t cond) { + const auto value = + EmitSelectValueU32(state, cond, ConstantU32(state, 1), ConstantU32(state, 0)); + EmitStoreU32(state, SccOperand(), value); +} + +uint32_t EmitAndConstant(EmitterState* state, uint32_t value, uint32_t mask); + +void EmitShiftLeftLogicalU64Values(EmitterState* state, uint32_t low, uint32_t high, uint32_t shift, + uint32_t* out_low, uint32_t* out_high) { + const auto amount = EmitAndConstant(state, shift, 63u); + const auto dword_shift = EmitAndConstant(state, amount, 31u); + const auto less32 = state->builder.AllocateId(); + const auto non_zero = state->builder.AllocateId(); + const auto carry_count_base = state->builder.AllocateId(); + const auto carry_count = state->builder.AllocateId(); + const auto low_part = state->builder.AllocateId(); + const auto high_part = state->builder.AllocateId(); + const auto carry_part = state->builder.AllocateId(); + const auto carry = state->builder.AllocateId(); + const auto high_less = state->builder.AllocateId(); + const auto high_ge = state->builder.AllocateId(); + state->builder.AddFunction( + {OpULessThan, state->bool_type, less32, amount, ConstantU32(state, 32)}); + state->builder.AddFunction( + {OpINotEqual, state->bool_type, non_zero, amount, ConstantU32(state, 0)}); + state->builder.AddFunction( + {OpISub, state->uint_type, carry_count_base, ConstantU32(state, 32), dword_shift}); + state->builder.AddFunction( + {OpBitwiseAnd, state->uint_type, carry_count, carry_count_base, ConstantU32(state, 31)}); + state->builder.AddFunction({OpShiftLeftLogical, state->uint_type, low_part, low, dword_shift}); + state->builder.AddFunction( + {OpShiftLeftLogical, state->uint_type, high_part, high, dword_shift}); + state->builder.AddFunction( + {OpShiftRightLogical, state->uint_type, carry_part, low, carry_count}); + state->builder.AddFunction( + {OpSelect, state->uint_type, carry, non_zero, carry_part, ConstantU32(state, 0)}); + state->builder.AddFunction({OpBitwiseOr, state->uint_type, high_less, high_part, carry}); + state->builder.AddFunction({OpShiftLeftLogical, state->uint_type, high_ge, low, dword_shift}); + *out_low = EmitSelectValueU32(state, less32, low_part, ConstantU32(state, 0)); + *out_high = EmitSelectValueU32(state, less32, high_less, high_ge); +} + +void EmitShiftLeftLogicalU64(EmitterState* state, const IR::Instruction& inst) { + const auto low = EmitSequentialValueLoad(state, inst.src[0], 0); + const auto high = EmitSequentialValueLoad(state, inst.src[0], 1); + const auto shift = EmitValueLoad(state, inst.src[1]); + uint32_t ret_low = 0; + uint32_t ret_high = 0; + EmitShiftLeftLogicalU64Values(state, low, high, shift, &ret_low, &ret_high); + EmitStoreU32(state, inst.dst, ret_low); + EmitStoreU32(state, OffsetRegisterOperand(inst.dst, 1), ret_high); +} + +void EmitShiftRightLogicalU64Values(EmitterState* state, uint32_t low, uint32_t high, + uint32_t shift, uint32_t* out_low, uint32_t* out_high) { + const auto amount = EmitAndConstant(state, shift, 63u); + const auto dword_shift = EmitAndConstant(state, amount, 31u); + const auto less32 = state->builder.AllocateId(); + const auto non_zero = state->builder.AllocateId(); + const auto carry_count_base = state->builder.AllocateId(); + const auto carry_count = state->builder.AllocateId(); + const auto low_part = state->builder.AllocateId(); + const auto high_part = state->builder.AllocateId(); + const auto carry_part = state->builder.AllocateId(); + const auto carry = state->builder.AllocateId(); + const auto low_less = state->builder.AllocateId(); + const auto low_ge = state->builder.AllocateId(); + state->builder.AddFunction( + {OpULessThan, state->bool_type, less32, amount, ConstantU32(state, 32)}); + state->builder.AddFunction( + {OpINotEqual, state->bool_type, non_zero, amount, ConstantU32(state, 0)}); + state->builder.AddFunction( + {OpISub, state->uint_type, carry_count_base, ConstantU32(state, 32), dword_shift}); + state->builder.AddFunction( + {OpBitwiseAnd, state->uint_type, carry_count, carry_count_base, ConstantU32(state, 31)}); + state->builder.AddFunction({OpShiftRightLogical, state->uint_type, low_part, low, dword_shift}); + state->builder.AddFunction( + {OpShiftRightLogical, state->uint_type, high_part, high, dword_shift}); + state->builder.AddFunction( + {OpShiftLeftLogical, state->uint_type, carry_part, high, carry_count}); + state->builder.AddFunction( + {OpSelect, state->uint_type, carry, non_zero, carry_part, ConstantU32(state, 0)}); + state->builder.AddFunction({OpBitwiseOr, state->uint_type, low_less, low_part, carry}); + state->builder.AddFunction({OpShiftRightLogical, state->uint_type, low_ge, high, dword_shift}); + *out_low = EmitSelectValueU32(state, less32, low_less, low_ge); + *out_high = EmitSelectValueU32(state, less32, high_part, ConstantU32(state, 0)); +} + +void EmitShiftRightLogicalU64(EmitterState* state, const IR::Instruction& inst) { + const auto low = EmitSequentialValueLoad(state, inst.src[0], 0); + const auto high = EmitSequentialValueLoad(state, inst.src[0], 1); + const auto shift = EmitValueLoad(state, inst.src[1]); + uint32_t ret_low = 0; + uint32_t ret_high = 0; + EmitShiftRightLogicalU64Values(state, low, high, shift, &ret_low, &ret_high); + EmitStoreU32(state, inst.dst, ret_low); + EmitStoreU32(state, OffsetRegisterOperand(inst.dst, 1), ret_high); +} + +void EmitAdd3U32(EmitterState* state, const IR::Instruction& inst) { + const auto lhs = EmitValueLoad(state, inst.src[0]); + const auto rhs = EmitValueLoad(state, inst.src[1]); + const auto add = EmitValueLoad(state, inst.src[2]); + const auto tmp = state->builder.AllocateId(); + const auto ret = state->builder.AllocateId(); + state->builder.AddFunction({OpIAdd, state->uint_type, tmp, lhs, rhs}); + state->builder.AddFunction({OpIAdd, state->uint_type, ret, tmp, add}); + EmitStoreU32(state, inst.dst, ret); +} + +uint32_t EmitAndConstant(EmitterState* state, uint32_t value, uint32_t mask) { + const auto ret = state->builder.AllocateId(); + state->builder.AddFunction( + {OpBitwiseAnd, state->uint_type, ret, value, ConstantU32(state, mask)}); + return ret; +} + +void EmitShiftU32(EmitterState* state, const IR::Instruction& inst, uint32_t opcode) { + const auto lhs = EmitValueLoad(state, inst.src[0]); + const auto rhs = EmitAndConstant(state, EmitValueLoad(state, inst.src[1]), 31u); + const auto ret = state->builder.AllocateId(); + state->builder.AddFunction({opcode, state->uint_type, ret, lhs, rhs}); + EmitStoreU32(state, inst.dst, ret); +} + +void EmitBitClearU32(EmitterState* state, const IR::Instruction& inst) { + const auto value = EmitValueLoad(state, inst.src[0]); + const auto bit = EmitAndConstant(state, EmitValueLoad(state, inst.src[1]), 31u); + const auto mask = state->builder.AllocateId(); + const auto inv = state->builder.AllocateId(); + const auto ret = state->builder.AllocateId(); + state->builder.AddFunction( + {OpShiftLeftLogical, state->uint_type, mask, ConstantU32(state, 1), bit}); + state->builder.AddFunction({OpNot, state->uint_type, inv, mask}); + state->builder.AddFunction({OpBitwiseAnd, state->uint_type, ret, value, inv}); + EmitStoreU32(state, inst.dst, ret); +} + +void EmitBitSetU32(EmitterState* state, const IR::Instruction& inst) { + const auto value = EmitValueLoad(state, inst.src[0]); + const auto bit = EmitAndConstant(state, EmitValueLoad(state, inst.src[1]), 31u); + const auto mask = state->builder.AllocateId(); + const auto ret = state->builder.AllocateId(); + state->builder.AddFunction( + {OpShiftLeftLogical, state->uint_type, mask, ConstantU32(state, 1), bit}); + state->builder.AddFunction({OpBitwiseOr, state->uint_type, ret, value, mask}); + EmitStoreU32(state, inst.dst, ret); +} + +uint32_t EmitNegateU32(EmitterState* state, uint32_t value) { + const auto ret = state->builder.AllocateId(); + state->builder.AddFunction({OpSNegate, state->uint_type, ret, value}); + return ret; +} + +uint32_t EmitSignExtendLow16U32(EmitterState* state, uint32_t value) { + const auto signed_value = state->builder.AllocateId(); + const auto extracted = state->builder.AllocateId(); + const auto bits = state->builder.AllocateId(); + state->builder.AddFunction({OpBitcast, state->int_type, signed_value, value}); + state->builder.AddFunction({OpBitFieldSExtract, state->int_type, extracted, signed_value, + ConstantU32(state, 0), ConstantU32(state, 16)}); + state->builder.AddFunction({OpBitcast, state->uint_type, bits, extracted}); + return bits; +} + +uint32_t EmitU16LaneBits(EmitterState* state, const IR::Operand& operand, bool high_lane, + bool sign_extend) { + const auto raw = EmitValueLoad(state, operand); + const auto lane = high_lane ? (operand.op_sel_hi ? 1u : 0u) : (operand.op_sel ? 1u : 0u); + auto bits = lane != 0u ? EmitShiftRightConstant(state, raw, 16) : raw; + bits = EmitAndConstant(state, bits, 0xffffu); + if (high_lane ? operand.negate_hi : operand.negate) { + bits = EmitAndConstant(state, EmitNegateU32(state, bits), 0xffffu); + } + return sign_extend ? EmitSignExtendLow16U32(state, bits) : bits; +} + +void EmitShiftU16(EmitterState* state, const IR::Instruction& inst, uint32_t opcode, + bool arithmetic) { + const auto lhs = EmitU16LaneBits(state, inst.src[0], false, arithmetic); + const auto rhs = EmitAndConstant(state, EmitU16LaneBits(state, inst.src[1], false), 0xfu); + const auto ret = state->builder.AllocateId(); + state->builder.AddFunction({opcode, state->uint_type, ret, lhs, rhs}); + EmitStoreU32(state, inst.dst, EmitAndConstant(state, ret, 0xffffu)); +} + +void EmitBinaryU16(EmitterState* state, const IR::Instruction& inst, uint32_t opcode) { + const auto lhs = EmitU16LaneBits(state, inst.src[0], false); + const auto rhs = EmitU16LaneBits(state, inst.src[1], false); + const auto ret = state->builder.AllocateId(); + state->builder.AddFunction({opcode, state->uint_type, ret, lhs, rhs}); + EmitStoreU32(state, inst.dst, EmitAndConstant(state, ret, 0xffffu)); +} + +void EmitMinMaxU16(EmitterState* state, const IR::Instruction& inst, bool signed_value, + bool max_value) { + const auto lhs = EmitU16LaneBits(state, inst.src[0], false, signed_value); + const auto rhs = EmitU16LaneBits(state, inst.src[1], false, signed_value); + const auto cond = state->builder.AllocateId(); + const auto ret = state->builder.AllocateId(); + const auto op = signed_value ? (max_value ? OpSGreaterThan : OpSLessThan) + : (max_value ? OpUGreaterThan : OpULessThan); + state->builder.AddFunction({op, state->bool_type, cond, lhs, rhs}); + state->builder.AddFunction({OpSelect, state->uint_type, ret, cond, lhs, rhs}); + EmitStoreU32(state, inst.dst, EmitAndConstant(state, ret, 0xffffu)); +} + +uint32_t EmitXorConstant(EmitterState* state, uint32_t value, uint32_t mask) { + const auto ret = state->builder.AllocateId(); + state->builder.AddFunction( + {OpBitwiseXor, state->uint_type, ret, value, ConstantU32(state, mask)}); + return ret; +} + +AddCarryResult EmitAddCarryValues(EmitterState* state, uint32_t lhs, uint32_t rhs, + uint32_t carry_in) { + const auto pair0 = state->builder.AllocateId(); + const auto sum0 = state->builder.AllocateId(); + const auto carry0 = state->builder.AllocateId(); + const auto pair1 = state->builder.AllocateId(); + const auto sum1 = state->builder.AllocateId(); + const auto carry1 = state->builder.AllocateId(); + const auto carry = state->builder.AllocateId(); + state->builder.AddFunction({OpIAddCarry, state->uint_pair_type, pair0, lhs, rhs}); + state->builder.AddFunction({OpCompositeExtract, state->uint_type, sum0, pair0, 0}); + state->builder.AddFunction({OpCompositeExtract, state->uint_type, carry0, pair0, 1}); + state->builder.AddFunction({OpIAddCarry, state->uint_pair_type, pair1, sum0, carry_in}); + state->builder.AddFunction({OpCompositeExtract, state->uint_type, sum1, pair1, 0}); + state->builder.AddFunction({OpCompositeExtract, state->uint_type, carry1, pair1, 1}); + state->builder.AddFunction({OpBitwiseOr, state->uint_type, carry, carry0, carry1}); + return {sum1, carry}; +} + +void EmitLaneMaskPairFromBool(EmitterState* state, const IR::Operand& dst, uint32_t value_bool) { + const auto write_mask = state->builder.AllocateId(); + state->builder.AddFunction( + {OpLogicalAnd, state->bool_type, write_mask, value_bool, EmitExecActiveBool(state)}); + if (state->per_invocation_masks) { + EmitPerInvocationMask(state, dst, write_mask); + return; + } + const auto ballot = state->builder.AllocateId(); + state->builder.AddFunction({OpGroupNonUniformBallot, state->vec4_uint_type, ballot, + ConstantU32(state, ScopeSubgroup), write_mask}); + for (uint32_t part = 0; part < 2u; part++) { + uint32_t result = ConstantU32(state, 0); + if (state->wave_size != 32u || part == 0u) { + result = state->builder.AllocateId(); + state->builder.AddFunction( + {OpCompositeExtract, state->uint_type, result, ballot, part}); + } + EmitStoreU32(state, OffsetRegisterOperand(dst, part), result); + } +} + +void EmitIAddCarryU32(EmitterState* state, const IR::Instruction& inst) { + const auto lhs = EmitValueLoad(state, inst.src[0]); + const auto rhs = EmitValueLoad(state, inst.src[1]); + const auto carry_in_active = EmitLaneMaskOperandActiveBool(state, inst.src[2]); + const auto carry_in = state->builder.AllocateId(); + state->builder.AddFunction({OpSelect, state->uint_type, carry_in, carry_in_active, + ConstantU32(state, 1), ConstantU32(state, 0)}); + const auto result = EmitAddCarryValues(state, lhs, rhs, carry_in); + const auto carry_bool = state->builder.AllocateId(); + state->builder.AddFunction( + {OpINotEqual, state->bool_type, carry_bool, result.carry, ConstantU32(state, 0)}); + EmitStoreU32(state, inst.dst, result.sum); + EmitLaneMaskPairFromBool(state, inst.dst2, carry_bool); +} + +void EmitISubBorrowU32(EmitterState* state, const IR::Instruction& inst) { + const auto lhs = EmitValueLoad(state, inst.src[0]); + const auto rhs = EmitValueLoad(state, inst.src[1]); + const auto result = state->builder.AllocateId(); + const auto borrow = state->builder.AllocateId(); + state->builder.AddFunction({OpISub, state->uint_type, result, lhs, rhs}); + state->builder.AddFunction({OpUGreaterThan, state->bool_type, borrow, rhs, lhs}); + EmitStoreU32(state, inst.dst, result); + EmitLaneMaskPairFromBool(state, inst.dst2, borrow); +} + +void EmitScalarAddCarryU32(EmitterState* state, const IR::Instruction& inst) { + const auto lhs = EmitValueLoad(state, inst.src[0]); + const auto rhs = EmitValueLoad(state, inst.src[1]); + const auto carry_in = EmitAndConstant(state, EmitValueLoad(state, inst.src[2]), 1u); + const auto result = EmitAddCarryValues(state, lhs, rhs, carry_in); + EmitStoreU32(state, inst.dst, result.sum); + EmitStoreU32(state, SccOperand(), result.carry); +} + +void EmitScalarSubBorrowU32(EmitterState* state, const IR::Instruction& inst) { + const auto lhs = EmitValueLoad(state, inst.src[0]); + const auto rhs = EmitValueLoad(state, inst.src[1]); + const auto result = state->builder.AllocateId(); + const auto borrow = state->builder.AllocateId(); + state->builder.AddFunction({OpISub, state->uint_type, result, lhs, rhs}); + state->builder.AddFunction({OpUGreaterThan, state->bool_type, borrow, rhs, lhs}); + EmitStoreU32(state, inst.dst, result); + EmitStoreSccBool(state, borrow); +} + +void EmitScalarSubBorrowCarryU32(EmitterState* state, const IR::Instruction& inst) { + const auto lhs = EmitValueLoad(state, inst.src[0]); + const auto rhs = EmitValueLoad(state, inst.src[1]); + const auto borrow_in = EmitAndConstant(state, EmitValueLoad(state, inst.src[2]), 1u); + const auto partial = state->builder.AllocateId(); + const auto result = state->builder.AllocateId(); + const auto borrow0 = state->builder.AllocateId(); + const auto borrow1 = state->builder.AllocateId(); + state->builder.AddFunction({OpISub, state->uint_type, partial, lhs, rhs}); + state->builder.AddFunction({OpUGreaterThan, state->bool_type, borrow0, rhs, lhs}); + state->builder.AddFunction({OpISub, state->uint_type, result, partial, borrow_in}); + state->builder.AddFunction({OpUGreaterThan, state->bool_type, borrow1, borrow_in, partial}); + EmitStoreU32(state, inst.dst, result); + EmitStoreSccBool(state, EmitLogicalOrBool(state, borrow0, borrow1)); +} + +void EmitUMadU64U32(EmitterState* state, const IR::Instruction& inst) { + const auto src0 = EmitValueLoad(state, inst.src[0]); + const auto src1 = EmitValueLoad(state, inst.src[1]); + const auto add_low = EmitSequentialValueLoad(state, inst.src[2], 0); + const auto add_high = EmitSequentialValueLoad(state, inst.src[2], 1); + const auto mul_pair = state->builder.AllocateId(); + const auto mul_low = state->builder.AllocateId(); + const auto mul_high = state->builder.AllocateId(); + const auto low_pair = state->builder.AllocateId(); + const auto result_low = state->builder.AllocateId(); + const auto carry_low = state->builder.AllocateId(); + const auto high_pair = state->builder.AllocateId(); + const auto high_sum = state->builder.AllocateId(); + const auto carry_high0 = state->builder.AllocateId(); + const auto carry_pair = state->builder.AllocateId(); + const auto result_high = state->builder.AllocateId(); + const auto carry_high1 = state->builder.AllocateId(); + const auto carry = state->builder.AllocateId(); + const auto carry_bool = state->builder.AllocateId(); + + state->builder.AddFunction({OpUMulExtended, state->uint_pair_type, mul_pair, src0, src1}); + state->builder.AddFunction({OpCompositeExtract, state->uint_type, mul_low, mul_pair, 0}); + state->builder.AddFunction({OpCompositeExtract, state->uint_type, mul_high, mul_pair, 1}); + state->builder.AddFunction({OpIAddCarry, state->uint_pair_type, low_pair, mul_low, add_low}); + state->builder.AddFunction({OpCompositeExtract, state->uint_type, result_low, low_pair, 0}); + state->builder.AddFunction({OpCompositeExtract, state->uint_type, carry_low, low_pair, 1}); + state->builder.AddFunction({OpIAddCarry, state->uint_pair_type, high_pair, mul_high, add_high}); + state->builder.AddFunction({OpCompositeExtract, state->uint_type, high_sum, high_pair, 0}); + state->builder.AddFunction({OpCompositeExtract, state->uint_type, carry_high0, high_pair, 1}); + state->builder.AddFunction( + {OpIAddCarry, state->uint_pair_type, carry_pair, high_sum, carry_low}); + state->builder.AddFunction({OpCompositeExtract, state->uint_type, result_high, carry_pair, 0}); + state->builder.AddFunction({OpCompositeExtract, state->uint_type, carry_high1, carry_pair, 1}); + state->builder.AddFunction({OpBitwiseOr, state->uint_type, carry, carry_high0, carry_high1}); + state->builder.AddFunction( + {OpINotEqual, state->bool_type, carry_bool, carry, ConstantU32(state, 0)}); + + EmitStoreU32(state, inst.dst, result_low); + EmitStoreU32(state, OffsetRegisterOperand(inst.dst, 1), result_high); + EmitLaneMaskPairFromBool(state, inst.dst2, carry_bool); +} + +void EmitScalarSignedOverflowI32(EmitterState* state, const IR::Instruction& inst, bool subtract) { + const auto lhs = EmitValueLoad(state, inst.src[0]); + const auto rhs = EmitValueLoad(state, inst.src[1]); + const auto result = state->builder.AllocateId(); + const auto lhs_sign = state->builder.AllocateId(); + const auto rhs_sign = state->builder.AllocateId(); + const auto result_sign = state->builder.AllocateId(); + const auto sign_match = state->builder.AllocateId(); + const auto sign_change = state->builder.AllocateId(); + const auto overflow = state->builder.AllocateId(); + state->builder.AddFunction({subtract ? OpISub : OpIAdd, state->uint_type, result, lhs, rhs}); + state->builder.AddFunction( + {OpShiftRightLogical, state->uint_type, lhs_sign, lhs, ConstantU32(state, 31)}); + state->builder.AddFunction( + {OpShiftRightLogical, state->uint_type, rhs_sign, rhs, ConstantU32(state, 31)}); + state->builder.AddFunction( + {OpShiftRightLogical, state->uint_type, result_sign, result, ConstantU32(state, 31)}); + state->builder.AddFunction( + {subtract ? OpINotEqual : OpIEqual, state->bool_type, sign_match, lhs_sign, rhs_sign}); + state->builder.AddFunction({OpINotEqual, state->bool_type, sign_change, lhs_sign, result_sign}); + state->builder.AddFunction({OpLogicalAnd, state->bool_type, overflow, sign_match, sign_change}); + EmitStoreU32(state, inst.dst, result); + EmitStoreSccBool(state, overflow); +} + +void EmitScalarShiftLeftAddCarryU32(EmitterState* state, const IR::Instruction& inst) { + const auto lhs = EmitValueLoad(state, inst.src[0]); + const auto rhs = EmitValueLoad(state, inst.src[2]); + const auto shift = inst.src[1].kind == IR::OperandKind::ImmediateU32 ? inst.src[1].imm : 0u; + const auto shifted = state->builder.AllocateId(); + const auto pair = state->builder.AllocateId(); + const auto result = state->builder.AllocateId(); + const auto add_carry = state->builder.AllocateId(); + const auto shifted_out = state->builder.AllocateId(); + const auto shift_carry = state->builder.AllocateId(); + const auto add_carry_bool = state->builder.AllocateId(); + const auto carry = state->builder.AllocateId(); + state->builder.AddFunction( + {OpShiftLeftLogical, state->uint_type, shifted, lhs, ConstantU32(state, shift)}); + state->builder.AddFunction({OpIAddCarry, state->uint_pair_type, pair, shifted, rhs}); + state->builder.AddFunction({OpCompositeExtract, state->uint_type, result, pair, 0}); + state->builder.AddFunction({OpCompositeExtract, state->uint_type, add_carry, pair, 1}); + state->builder.AddFunction( + {OpShiftRightLogical, state->uint_type, shifted_out, lhs, ConstantU32(state, 32u - shift)}); + state->builder.AddFunction( + {OpINotEqual, state->bool_type, shift_carry, shifted_out, ConstantU32(state, 0)}); + state->builder.AddFunction( + {OpINotEqual, state->bool_type, add_carry_bool, add_carry, ConstantU32(state, 0)}); + state->builder.AddFunction({OpLogicalOr, state->bool_type, carry, shift_carry, add_carry_bool}); + EmitStoreU32(state, inst.dst, result); + EmitStoreSccBool(state, carry); +} + +void EmitMulU24U32(EmitterState* state, const IR::Instruction& inst) { + const auto lhs = EmitAndConstant(state, EmitValueLoad(state, inst.src[0]), 0x00ffffffu); + const auto rhs = EmitAndConstant(state, EmitValueLoad(state, inst.src[1]), 0x00ffffffu); + const auto ret = state->builder.AllocateId(); + state->builder.AddFunction({OpIMul, state->uint_type, ret, lhs, rhs}); + EmitStoreU32(state, inst.dst, ret); +} + +void EmitMulI24U32(EmitterState* state, const IR::Instruction& inst) { + const auto lhs_bits = EmitValueLoad(state, inst.src[0]); + const auto rhs_bits = EmitValueLoad(state, inst.src[1]); + const auto lhs_i32 = state->builder.AllocateId(); + const auto rhs_i32 = state->builder.AllocateId(); + const auto lhs_i24 = state->builder.AllocateId(); + const auto rhs_i24 = state->builder.AllocateId(); + const auto mul = state->builder.AllocateId(); + const auto ret = state->builder.AllocateId(); + state->builder.AddFunction({OpBitcast, state->int_type, lhs_i32, lhs_bits}); + state->builder.AddFunction({OpBitcast, state->int_type, rhs_i32, rhs_bits}); + state->builder.AddFunction({OpBitFieldSExtract, state->int_type, lhs_i24, lhs_i32, + ConstantU32(state, 0), ConstantU32(state, 24)}); + state->builder.AddFunction({OpBitFieldSExtract, state->int_type, rhs_i24, rhs_i32, + ConstantU32(state, 0), ConstantU32(state, 24)}); + state->builder.AddFunction({OpIMul, state->int_type, mul, lhs_i24, rhs_i24}); + state->builder.AddFunction({OpBitcast, state->uint_type, ret, mul}); + EmitStoreU32(state, inst.dst, ret); +} + +void EmitMulHighU32(EmitterState* state, const IR::Instruction& inst) { + const auto lhs = EmitValueLoad(state, inst.src[0]); + const auto rhs = EmitValueLoad(state, inst.src[1]); + const auto full = state->builder.AllocateId(); + const auto high = state->builder.AllocateId(); + state->builder.AddFunction({OpUMulExtended, state->uint_pair_type, full, lhs, rhs}); + state->builder.AddFunction({OpCompositeExtract, state->uint_type, high, full, 1}); + EmitStoreU32(state, inst.dst, high); +} + +void EmitMulHighI32(EmitterState* state, const IR::Instruction& inst) { + const auto lhs_bits = EmitValueLoad(state, inst.src[0]); + const auto rhs_bits = EmitValueLoad(state, inst.src[1]); + const auto lhs = state->builder.AllocateId(); + const auto rhs = state->builder.AllocateId(); + const auto full = state->builder.AllocateId(); + const auto high_i32 = state->builder.AllocateId(); + const auto high_u32 = state->builder.AllocateId(); + state->builder.AddFunction({OpBitcast, state->int_type, lhs, lhs_bits}); + state->builder.AddFunction({OpBitcast, state->int_type, rhs, rhs_bits}); + state->builder.AddFunction({OpSMulExtended, state->int_pair_type, full, lhs, rhs}); + state->builder.AddFunction({OpCompositeExtract, state->int_type, high_i32, full, 1}); + state->builder.AddFunction({OpBitcast, state->uint_type, high_u32, high_i32}); + EmitStoreU32(state, inst.dst, high_u32); +} + +void EmitMadU32U24(EmitterState* state, const IR::Instruction& inst) { + const auto lhs = EmitAndConstant(state, EmitValueLoad(state, inst.src[0]), 0x00ffffffu); + const auto rhs = EmitAndConstant(state, EmitValueLoad(state, inst.src[1]), 0x00ffffffu); + const auto add = EmitValueLoad(state, inst.src[2]); + const auto mul = state->builder.AllocateId(); + const auto ret = state->builder.AllocateId(); + state->builder.AddFunction({OpIMul, state->uint_type, mul, lhs, rhs}); + state->builder.AddFunction({OpIAdd, state->uint_type, ret, mul, add}); + EmitStoreU32(state, inst.dst, ret); +} + +void EmitMadI32I24(EmitterState* state, const IR::Instruction& inst) { + const auto lhs_bits = EmitValueLoad(state, inst.src[0]); + const auto rhs_bits = EmitValueLoad(state, inst.src[1]); + const auto add_bits = EmitValueLoad(state, inst.src[2]); + const auto lhs_i32 = state->builder.AllocateId(); + const auto rhs_i32 = state->builder.AllocateId(); + const auto add_i32 = state->builder.AllocateId(); + const auto lhs_i24 = state->builder.AllocateId(); + const auto rhs_i24 = state->builder.AllocateId(); + const auto mul = state->builder.AllocateId(); + const auto add = state->builder.AllocateId(); + const auto ret = state->builder.AllocateId(); + state->builder.AddFunction({OpBitcast, state->int_type, lhs_i32, lhs_bits}); + state->builder.AddFunction({OpBitcast, state->int_type, rhs_i32, rhs_bits}); + state->builder.AddFunction({OpBitcast, state->int_type, add_i32, add_bits}); + state->builder.AddFunction({OpBitFieldSExtract, state->int_type, lhs_i24, lhs_i32, + ConstantU32(state, 0), ConstantU32(state, 24)}); + state->builder.AddFunction({OpBitFieldSExtract, state->int_type, rhs_i24, rhs_i32, + ConstantU32(state, 0), ConstantU32(state, 24)}); + state->builder.AddFunction({OpIMul, state->int_type, mul, lhs_i24, rhs_i24}); + state->builder.AddFunction({OpIAdd, state->int_type, add, mul, add_i32}); + state->builder.AddFunction({OpBitcast, state->uint_type, ret, add}); + EmitStoreU32(state, inst.dst, ret); +} + +uint32_t EmitShiftLeftConstant(EmitterState* state, uint32_t value, uint32_t shift) { + const auto ret = state->builder.AllocateId(); + state->builder.AddFunction( + {OpShiftLeftLogical, state->uint_type, ret, value, ConstantU32(state, shift)}); + return ret; +} + +uint32_t EmitShiftRightConstant(EmitterState* state, uint32_t value, uint32_t shift) { + const auto ret = state->builder.AllocateId(); + state->builder.AddFunction( + {OpShiftRightLogical, state->uint_type, ret, value, ConstantU32(state, shift)}); + return ret; +} + +uint32_t EmitOrU32(EmitterState* state, uint32_t lhs, uint32_t rhs) { + const auto ret = state->builder.AllocateId(); + state->builder.AddFunction({OpBitwiseOr, state->uint_type, ret, lhs, rhs}); + return ret; +} + +uint32_t EmitBitReplicateHalfU32(EmitterState* state, uint32_t value) { + auto ret = EmitOrU32(state, value, EmitShiftLeftConstant(state, value, 8)); + ret = EmitAndConstant(state, ret, 0x00ff00ffu); + ret = EmitOrU32(state, ret, EmitShiftLeftConstant(state, ret, 4)); + ret = EmitAndConstant(state, ret, 0x0f0f0f0fu); + ret = EmitOrU32(state, ret, EmitShiftLeftConstant(state, ret, 2)); + ret = EmitAndConstant(state, ret, 0x33333333u); + ret = EmitOrU32(state, ret, EmitShiftLeftConstant(state, ret, 1)); + ret = EmitAndConstant(state, ret, 0x55555555u); + return EmitOrU32(state, ret, EmitShiftLeftConstant(state, ret, 1)); +} + +void EmitBitReplicateB64B32(EmitterState* state, const IR::Instruction& inst) { + const auto src = EmitValueLoad(state, inst.src[0]); + const auto low = EmitBitReplicateHalfU32(state, EmitAndConstant(state, src, 0x0000ffffu)); + const auto high = EmitBitReplicateHalfU32(state, EmitShiftRightConstant(state, src, 16)); + EmitStoreU32(state, inst.dst, low); + EmitStoreU32(state, OffsetRegisterOperand(inst.dst, 1), high); +} + +uint32_t EmitBitFieldExtractConstant(EmitterState* state, uint32_t value, uint32_t offset, + uint32_t count) { + const auto ret = state->builder.AllocateId(); + state->builder.AddFunction({OpBitFieldUExtract, state->uint_type, ret, value, + ConstantU32(state, offset), ConstantU32(state, count)}); + return ret; +} + +uint32_t EmitRightAlignedMaskU32(EmitterState* state, uint32_t count) { + const auto shifted_one = state->builder.AllocateId(); + const auto mask = state->builder.AllocateId(); + state->builder.AddFunction( + {OpShiftLeftLogical, state->uint_type, shifted_one, ConstantU32(state, 1), count}); + state->builder.AddFunction( + {OpISub, state->uint_type, mask, shifted_one, ConstantU32(state, 1)}); + return mask; +} + +void EmitBitFieldMaskU32(EmitterState* state, const IR::Instruction& inst) { + const auto count = EmitAndConstant(state, EmitValueLoad(state, inst.src[0]), 31u); + const auto offset = EmitAndConstant(state, EmitValueLoad(state, inst.src[1]), 31u); + const auto ret = state->builder.AllocateId(); + state->builder.AddFunction({OpBitFieldInsert, state->uint_type, ret, ConstantU32(state, 0), + ConstantU32(state, 0xffffffffu), offset, count}); + EmitStoreU32(state, inst.dst, ret); +} + +void EmitRightAlignedMaskU64(EmitterState* state, uint32_t count, uint32_t* out_low, + uint32_t* out_high) { + const auto count_lt32 = state->builder.AllocateId(); + const auto count_gt32 = state->builder.AllocateId(); + const auto low_count = state->builder.AllocateId(); + const auto high_base = state->builder.AllocateId(); + const auto high_count = state->builder.AllocateId(); + state->builder.AddFunction( + {OpULessThan, state->bool_type, count_lt32, count, ConstantU32(state, 32)}); + state->builder.AddFunction( + {OpUGreaterThan, state->bool_type, count_gt32, count, ConstantU32(state, 32)}); + state->builder.AddFunction( + {OpSelect, state->uint_type, low_count, count_lt32, count, ConstantU32(state, 32)}); + state->builder.AddFunction( + {OpISub, state->uint_type, high_base, count, ConstantU32(state, 32)}); + state->builder.AddFunction( + {OpSelect, state->uint_type, high_count, count_gt32, high_base, ConstantU32(state, 0)}); + *out_low = state->builder.AllocateId(); + *out_high = state->builder.AllocateId(); + state->builder.AddFunction({OpBitFieldInsert, state->uint_type, *out_low, ConstantU32(state, 0), + ConstantU32(state, 0xffffffffu), ConstantU32(state, 0), low_count}); + state->builder.AddFunction({OpBitFieldInsert, state->uint_type, *out_high, + ConstantU32(state, 0), ConstantU32(state, 0xffffffffu), + ConstantU32(state, 0), high_count}); +} + +void EmitBitFieldMaskU64(EmitterState* state, const IR::Instruction& inst) { + const auto count = EmitAndConstant(state, EmitValueLoad(state, inst.src[0]), 63u); + if (state->per_invocation_masks && inst.dst.kind == IR::OperandKind::Register && + IsMaskRegisterFile(inst.dst.reg.file)) { + const auto active = state->builder.AllocateId(); + state->builder.AddFunction( + {OpINotEqual, state->bool_type, active, count, ConstantU32(state, 0)}); + EmitPerInvocationMask(state, inst.dst, active); + return; + } + const auto offset = EmitAndConstant(state, EmitValueLoad(state, inst.src[1]), 63u); + uint32_t low_mask = 0; + uint32_t high_mask = 0; + uint32_t shifted_low = 0; + uint32_t shifted_high = 0; + EmitRightAlignedMaskU64(state, count, &low_mask, &high_mask); + EmitShiftLeftLogicalU64Values(state, low_mask, high_mask, offset, &shifted_low, &shifted_high); + EmitStoreU32(state, inst.dst, shifted_low); + EmitStoreU32(state, OffsetRegisterOperand(inst.dst, 1), shifted_high); +} + +void EmitBitFieldExtractU64(EmitterState* state, const IR::Instruction& inst) { + const auto src_low = EmitSequentialValueLoad(state, inst.src[0], 0); + const auto src_high = EmitSequentialValueLoad(state, inst.src[0], 1); + const auto field = EmitValueLoad(state, inst.src[1]); + const auto offset = EmitBitFieldExtractConstant(state, field, 0, 6); + const auto raw_count = EmitBitFieldExtractConstant(state, field, 16, 7); + const auto available = state->builder.AllocateId(); + const auto use_raw = state->builder.AllocateId(); + const auto count = state->builder.AllocateId(); + uint32_t shifted_low = 0; + uint32_t shifted_high = 0; + uint32_t mask_low = 0; + uint32_t mask_high = 0; + const auto ret_low = state->builder.AllocateId(); + const auto ret_high = state->builder.AllocateId(); + state->builder.AddFunction( + {OpISub, state->uint_type, available, ConstantU32(state, 64), offset}); + state->builder.AddFunction({OpULessThan, state->bool_type, use_raw, raw_count, available}); + state->builder.AddFunction({OpSelect, state->uint_type, count, use_raw, raw_count, available}); + EmitShiftRightLogicalU64Values(state, src_low, src_high, offset, &shifted_low, &shifted_high); + EmitRightAlignedMaskU64(state, count, &mask_low, &mask_high); + state->builder.AddFunction({OpBitwiseAnd, state->uint_type, ret_low, shifted_low, mask_low}); + state->builder.AddFunction({OpBitwiseAnd, state->uint_type, ret_high, shifted_high, mask_high}); + EmitStoreU32(state, inst.dst, ret_low); + EmitStoreU32(state, OffsetRegisterOperand(inst.dst, 1), ret_high); +} + +void EmitBitFieldExtractU32(EmitterState* state, const IR::Instruction& inst) { + const auto src = EmitValueLoad(state, inst.src[0]); + const auto field = EmitValueLoad(state, inst.src[1]); + const auto offset = EmitBitFieldExtractConstant(state, field, 0, 5); + const auto count = EmitBitFieldExtractConstant(state, field, 16, 7); + const auto ret = state->builder.AllocateId(); + state->builder.AddFunction({OpBitFieldUExtract, state->uint_type, ret, src, offset, count}); + EmitStoreU32(state, inst.dst, ret); +} + +void EmitBitFieldExtract3U32(EmitterState* state, const IR::Instruction& inst, bool signed_value) { + const auto src = EmitValueLoad(state, inst.src[0]); + const auto offset = EmitAndConstant(state, EmitValueLoad(state, inst.src[1]), 31u); + const auto count = EmitAndConstant(state, EmitValueLoad(state, inst.src[2]), 31u); + const auto ret = state->builder.AllocateId(); + if (signed_value) { + const auto shifted = state->builder.AllocateId(); + const auto mask = EmitRightAlignedMaskU32(state, count); + state->builder.AddFunction( + {OpShiftRightArithmetic, state->uint_type, shifted, src, offset}); + state->builder.AddFunction({OpBitwiseAnd, state->uint_type, ret, shifted, mask}); + EmitStoreU32(state, inst.dst, ret); + return; + } + state->builder.AddFunction({OpBitFieldUExtract, state->uint_type, ret, src, offset, count}); + EmitStoreU32(state, inst.dst, ret); +} + +void EmitBitFieldInsertSelectU32(EmitterState* state, const IR::Instruction& inst) { + const auto mask = EmitValueLoad(state, inst.src[0]); + const auto insert = EmitValueLoad(state, inst.src[1]); + const auto base = EmitValueLoad(state, inst.src[2]); + const auto not_mask = state->builder.AllocateId(); + const auto selected = state->builder.AllocateId(); + const auto preserved = state->builder.AllocateId(); + const auto ret = state->builder.AllocateId(); + state->builder.AddFunction({OpNot, state->uint_type, not_mask, mask}); + state->builder.AddFunction({OpBitwiseAnd, state->uint_type, selected, mask, insert}); + state->builder.AddFunction({OpBitwiseAnd, state->uint_type, preserved, not_mask, base}); + state->builder.AddFunction({OpBitwiseOr, state->uint_type, ret, selected, preserved}); + EmitStoreU32(state, inst.dst, ret); +} + +void EmitAlignBitU32(EmitterState* state, const IR::Instruction& inst) { + const auto hi = EmitValueLoad(state, inst.src[0]); + const auto lo = EmitValueLoad(state, inst.src[1]); + const auto shift = EmitAndConstant(state, EmitValueLoad(state, inst.src[2]), 31u); + const auto lo_part = state->builder.AllocateId(); + const auto hi_shift_raw = state->builder.AllocateId(); + const auto hi_shift = state->builder.AllocateId(); + const auto hi_part_raw = state->builder.AllocateId(); + const auto non_zero = state->builder.AllocateId(); + const auto hi_part = state->builder.AllocateId(); + const auto ret = state->builder.AllocateId(); + state->builder.AddFunction({OpShiftRightLogical, state->uint_type, lo_part, lo, shift}); + state->builder.AddFunction( + {OpISub, state->uint_type, hi_shift_raw, ConstantU32(state, 32), shift}); + state->builder.AddFunction( + {OpBitwiseAnd, state->uint_type, hi_shift, hi_shift_raw, ConstantU32(state, 31)}); + state->builder.AddFunction({OpShiftLeftLogical, state->uint_type, hi_part_raw, hi, hi_shift}); + state->builder.AddFunction( + {OpINotEqual, state->bool_type, non_zero, shift, ConstantU32(state, 0)}); + state->builder.AddFunction( + {OpSelect, state->uint_type, hi_part, non_zero, hi_part_raw, ConstantU32(state, 0)}); + state->builder.AddFunction({OpBitwiseOr, state->uint_type, ret, lo_part, hi_part}); + EmitStoreU32(state, inst.dst, ret); +} + +void EmitSelectU32(EmitterState* state, const IR::Instruction& inst) { + const auto cond = EmitConditionBool(state, inst.src[0]); + const auto ret = state->builder.AllocateId(); + state->builder.AddFunction({OpSelect, state->uint_type, ret, cond, + EmitValueLoad(state, inst.src[1]), + EmitValueLoad(state, inst.src[2])}); + EmitStoreU32(state, inst.dst, ret); +} + +void EmitSelectMaskU32(EmitterState* state, const IR::Instruction& inst) { + const auto cond = EmitLaneMaskOperandActiveBool(state, inst.src[0]); + const auto ret = state->builder.AllocateId(); + state->builder.AddFunction({OpSelect, state->uint_type, ret, cond, + EmitValueLoad(state, inst.src[1]), + EmitValueLoad(state, inst.src[2])}); + EmitStoreU32(state, inst.dst, ret); +} + +void EmitSelectF32Bits(EmitterState* state, const IR::Instruction& inst) { + const auto cond = EmitConditionBool(state, inst.src[0]); + const auto selected = state->builder.AllocateId(); + const auto ret = state->builder.AllocateId(); + state->builder.AddFunction({OpSelect, state->float_type, selected, cond, + EmitFloatLoad(state, inst.src[1]), + EmitFloatLoad(state, inst.src[2])}); + state->builder.AddFunction({OpBitcast, state->uint_type, ret, selected}); + EmitStoreU32(state, inst.dst, ret); +} + +void EmitSelectMaskF32Bits(EmitterState* state, const IR::Instruction& inst) { + const auto cond = EmitLaneMaskOperandActiveBool(state, inst.src[0]); + const auto selected = state->builder.AllocateId(); + const auto ret = state->builder.AllocateId(); + state->builder.AddFunction({OpSelect, state->float_type, selected, cond, + EmitFloatLoad(state, inst.src[1]), + EmitFloatLoad(state, inst.src[2])}); + state->builder.AddFunction({OpBitcast, state->uint_type, ret, selected}); + EmitStoreU32(state, inst.dst, ret); +} + +void EmitPackU16(EmitterState* state, const IR::Instruction& inst, bool high_src0, bool high_src1) { + const auto src0 = EmitValueLoad(state, inst.src[0]); + const auto src1 = EmitValueLoad(state, inst.src[1]); + const auto lo = high_src0 ? EmitBitFieldExtractConstant(state, src0, 16, 16) + : EmitAndConstant(state, src0, 0xffffu); + const auto hi = high_src1 ? EmitBitFieldExtractConstant(state, src1, 16, 16) + : EmitAndConstant(state, src1, 0xffffu); + const auto shifted = EmitShiftLeftConstant(state, hi, 16); + const auto ret = state->builder.AllocateId(); + state->builder.AddFunction({OpBitwiseOr, state->uint_type, ret, lo, shifted}); + EmitStoreU32(state, inst.dst, ret); +} + +void EmitPackU8F32(EmitterState* state, const IR::Instruction& inst) { + const auto src = EmitFloatLoad(state, inst.src[0]); + const auto byte_index = EmitAndConstant(state, EmitValueLoad(state, inst.src[1]), 3u); + const auto base = EmitValueLoad(state, inst.src[2]); + + const auto truncated = EmitTruncF32Value(state, src); + const auto below_zero = state->builder.AllocateId(); + const auto above_max = state->builder.AllocateId(); + const auto safe_low = state->builder.AllocateId(); + const auto safe_value = state->builder.AllocateId(); + const auto converted = state->builder.AllocateId(); + state->builder.AddFunction( + {OpFOrdLessThanEqual, state->bool_type, below_zero, src, ConstantF32(state, 0)}); + state->builder.AddFunction({OpFOrdGreaterThanEqual, state->bool_type, above_max, src, + ConstantF32(state, 0x437f0000u)}); + const auto zero_or_nan = EmitLogicalOrBool(state, EmitClassifyF32(state, src).nan, below_zero); + state->builder.AddFunction( + {OpSelect, state->float_type, safe_low, zero_or_nan, ConstantF32(state, 0), truncated}); + state->builder.AddFunction({OpSelect, state->float_type, safe_value, above_max, + ConstantF32(state, 0x437f0000u), safe_low}); + state->builder.AddFunction({OpConvertFToU, state->uint_type, converted, safe_value}); + const auto byte_value = EmitAndConstant(state, converted, 0xffu); + + const auto shift = state->builder.AllocateId(); + state->builder.AddFunction( + {OpShiftLeftLogical, state->uint_type, shift, byte_index, ConstantU32(state, 3)}); + const auto byte_mask = state->builder.AllocateId(); + const auto inv_mask = state->builder.AllocateId(); + const auto cleared = state->builder.AllocateId(); + const auto shifted = state->builder.AllocateId(); + state->builder.AddFunction( + {OpShiftLeftLogical, state->uint_type, byte_mask, ConstantU32(state, 0xffu), shift}); + state->builder.AddFunction({OpNot, state->uint_type, inv_mask, byte_mask}); + state->builder.AddFunction({OpBitwiseAnd, state->uint_type, cleared, base, inv_mask}); + state->builder.AddFunction({OpShiftLeftLogical, state->uint_type, shifted, byte_value, shift}); + EmitStoreU32(state, inst.dst, EmitOrU32(state, cleared, shifted)); +} + +void EmitBinaryF32(EmitterState* state, const IR::Instruction& inst, uint32_t opcode) { + const auto lhs = EmitFloatLoad(state, inst.src[0]); + const auto rhs = EmitFloatLoad(state, inst.src[1]); + const auto f32 = state->builder.AllocateId(); + const auto u32 = state->builder.AllocateId(); + state->builder.AddFunction({opcode, state->float_type, f32, lhs, rhs}); + const auto result = ApplyResultModifiersF32(state, f32, inst.dst); + state->builder.AddFunction({OpBitcast, state->uint_type, u32, result}); + EmitStoreU32(state, inst.dst, u32); +} + +void EmitLdexpF32(EmitterState* state, const IR::Instruction& inst) { + const auto value = EmitFloatLoad(state, inst.src[0]); + const auto exp_bits = EmitValueLoad(state, inst.src[1]); + const auto exp_i32 = state->builder.AllocateId(); + const auto ret_f32 = state->builder.AllocateId(); + const auto ret_u32 = state->builder.AllocateId(); + state->builder.AddFunction({OpBitcast, state->int_type, exp_i32, exp_bits}); + state->builder.AddFunction( + {OpExtInst, state->float_type, ret_f32, state->glsl_std450, GlslLdexp, value, exp_i32}); + const auto result = ApplyResultModifiersF32(state, ret_f32, inst.dst); + state->builder.AddFunction({OpBitcast, state->uint_type, ret_u32, result}); + EmitStoreU32(state, inst.dst, ret_u32); +} + +uint32_t EmitCompareU32Constant(EmitterState* state, uint32_t opcode, uint32_t value, + uint32_t constant) { + const auto ret = state->builder.AllocateId(); + state->builder.AddFunction( + {opcode, state->bool_type, ret, value, ConstantU32(state, constant)}); + return ret; +} + +uint32_t EmitSubConstantMinusU32(EmitterState* state, uint32_t constant, uint32_t value) { + const auto ret = state->builder.AllocateId(); + state->builder.AddFunction( + {OpISub, state->uint_type, ret, ConstantU32(state, constant), value}); + return ret; +} + +uint32_t EmitF32ToF16RtzBits(EmitterState* state, uint32_t f32) { + const auto bits = state->builder.AllocateId(); + state->builder.AddFunction({OpBitcast, state->uint_type, bits, f32}); + + const auto sign = EmitAndConstant(state, EmitShiftRightConstant(state, bits, 16), 0x8000u); + const auto exp = EmitAndConstant(state, EmitShiftRightConstant(state, bits, 23), 0xffu); + const auto mant = EmitAndConstant(state, bits, 0x007fffffu); + + const auto half_exp = state->builder.AllocateId(); + const auto normal_exp = state->builder.AllocateId(); + const auto normal_mant = EmitShiftRightConstant(state, mant, 13); + const auto normal_payload = state->builder.AllocateId(); + const auto normal = state->builder.AllocateId(); + state->builder.AddFunction({OpISub, state->uint_type, half_exp, exp, ConstantU32(state, 112)}); + state->builder.AddFunction( + {OpShiftLeftLogical, state->uint_type, normal_exp, half_exp, ConstantU32(state, 10)}); + state->builder.AddFunction( + {OpBitwiseOr, state->uint_type, normal_payload, normal_exp, normal_mant}); + state->builder.AddFunction({OpBitwiseOr, state->uint_type, normal, sign, normal_payload}); + + const auto mant_with_hidden = EmitOrU32(state, mant, ConstantU32(state, 0x00800000u)); + const auto raw_sub_shift = EmitSubConstantMinusU32(state, 126, exp); + const auto exp_lt_103 = EmitCompareU32Constant(state, OpULessThan, exp, 103); + const auto exp_gt_112 = EmitCompareU32Constant(state, OpUGreaterThan, exp, 112); + const auto sub_shift_low = + EmitSelectValueU32(state, exp_lt_103, ConstantU32(state, 31), raw_sub_shift); + const auto sub_shift = + EmitSelectValueU32(state, exp_gt_112, ConstantU32(state, 14), sub_shift_low); + const auto sub_mant = state->builder.AllocateId(); + const auto subnormal = state->builder.AllocateId(); + state->builder.AddFunction( + {OpShiftRightLogical, state->uint_type, sub_mant, mant_with_hidden, sub_shift}); + state->builder.AddFunction({OpBitwiseOr, state->uint_type, subnormal, sign, sub_mant}); + + const auto nan_payload = + EmitOrU32(state, EmitShiftRightConstant(state, mant, 13), ConstantU32(state, 0x0200u)); + const auto nan = + EmitOrU32(state, sign, EmitOrU32(state, ConstantU32(state, 0x7c00u), nan_payload)); + const auto inf = EmitOrU32(state, sign, ConstantU32(state, 0x7c00u)); + const auto max_finite = EmitOrU32(state, sign, ConstantU32(state, 0x7bffu)); + const auto mant_zero = EmitCompareU32Constant(state, OpIEqual, mant, 0); + const auto special = EmitSelectValueU32(state, mant_zero, inf, nan); + + const auto exp_le_112 = EmitCompareU32Constant(state, OpULessThanEqual, exp, 112); + const auto exp_ge_143 = EmitCompareU32Constant(state, OpUGreaterThanEqual, exp, 143); + const auto exp_eq_255 = EmitCompareU32Constant(state, OpIEqual, exp, 255); + const auto finite0 = EmitSelectValueU32(state, exp_le_112, subnormal, normal); + const auto finite1 = EmitSelectValueU32(state, exp_lt_103, sign, finite0); + const auto finite2 = EmitSelectValueU32(state, exp_ge_143, max_finite, finite1); + return EmitAndConstant(state, EmitSelectValueU32(state, exp_eq_255, special, finite2), 0xffffu); +} + +void EmitPackF32ToF16Rtz(EmitterState* state, const IR::Instruction& inst) { + const auto lo = EmitF32ToF16RtzBits(state, EmitFloatLoad(state, inst.src[0])); + const auto hi = EmitF32ToF16RtzBits(state, EmitFloatLoad(state, inst.src[1])); + const auto shifted = EmitShiftLeftConstant(state, hi, 16); + const auto ret = EmitOrU32(state, lo, shifted); + EmitStoreU32(state, inst.dst, ret); +} + +void EmitPackNormalizedF32(EmitterState* state, const IR::Instruction& inst, uint32_t ext_inst) { + const auto src0 = EmitFloatLoad(state, inst.src[0]); + const auto src1 = EmitFloatLoad(state, inst.src[1]); + const auto pair = state->builder.AllocateId(); + const auto ret = state->builder.AllocateId(); + state->builder.AddFunction({OpCompositeConstruct, state->vec2_float_type, pair, src0, src1}); + state->builder.AddFunction( + {OpExtInst, state->uint_type, ret, state->glsl_std450, ext_inst, pair}); + EmitStoreU32(state, inst.dst, ret); +} + +uint32_t EmitMinMaxU32Value(EmitterState* state, uint32_t lhs, uint32_t rhs, bool max_value) { + const auto cond = state->builder.AllocateId(); + const auto ret = state->builder.AllocateId(); + state->builder.AddFunction( + {max_value ? OpUGreaterThan : OpULessThan, state->bool_type, cond, lhs, rhs}); + state->builder.AddFunction({OpSelect, state->uint_type, ret, cond, lhs, rhs}); + return ret; +} + +void EmitMinMaxU32(EmitterState* state, const IR::Instruction& inst, bool max_value) { + const auto lhs = EmitValueLoad(state, inst.src[0]); + const auto rhs = EmitValueLoad(state, inst.src[1]); + const auto ret = EmitMinMaxU32Value(state, lhs, rhs, max_value); + EmitStoreU32(state, inst.dst, ret); +} + +void EmitSadU32(EmitterState* state, const IR::Instruction& inst) { + const auto lhs = EmitValueLoad(state, inst.src[0]); + const auto rhs = EmitValueLoad(state, inst.src[1]); + const auto add = EmitValueLoad(state, inst.src[2]); + const auto high = EmitMinMaxU32Value(state, lhs, rhs, true); + const auto low = EmitMinMaxU32Value(state, lhs, rhs, false); + const auto diff = state->builder.AllocateId(); + const auto ret = state->builder.AllocateId(); + state->builder.AddFunction({OpISub, state->uint_type, diff, high, low}); + state->builder.AddFunction({OpIAdd, state->uint_type, ret, diff, add}); + EmitStoreU32(state, inst.dst, ret); +} + +uint32_t EmitMinMaxI32Value(EmitterState* state, uint32_t lhs, uint32_t rhs, bool max_value) { + const auto cond = state->builder.AllocateId(); + const auto ret = state->builder.AllocateId(); + state->builder.AddFunction( + {max_value ? OpSGreaterThan : OpSLessThan, state->bool_type, cond, lhs, rhs}); + state->builder.AddFunction({OpSelect, state->uint_type, ret, cond, lhs, rhs}); + return ret; +} + +void EmitMinMaxI32(EmitterState* state, const IR::Instruction& inst, bool max_value) { + const auto lhs = EmitValueLoad(state, inst.src[0]); + const auto rhs = EmitValueLoad(state, inst.src[1]); + const auto ret = EmitMinMaxI32Value(state, lhs, rhs, max_value); + EmitStoreU32(state, inst.dst, ret); +} + +void EmitMinMax3U32(EmitterState* state, const IR::Instruction& inst, bool signed_value, + bool max_value) { + const auto src0 = EmitValueLoad(state, inst.src[0]); + const auto src1 = EmitValueLoad(state, inst.src[1]); + const auto src2 = EmitValueLoad(state, inst.src[2]); + const auto tmp = signed_value ? EmitMinMaxI32Value(state, src0, src1, max_value) + : EmitMinMaxU32Value(state, src0, src1, max_value); + const auto ret = signed_value ? EmitMinMaxI32Value(state, tmp, src2, max_value) + : EmitMinMaxU32Value(state, tmp, src2, max_value); + EmitStoreU32(state, inst.dst, ret); +} + +void EmitMed3U32(EmitterState* state, const IR::Instruction& inst, bool signed_value) { + const auto src0 = EmitValueLoad(state, inst.src[0]); + const auto src1 = EmitValueLoad(state, inst.src[1]); + const auto src2 = EmitValueLoad(state, inst.src[2]); + const auto low_pair = signed_value ? EmitMinMaxI32Value(state, src0, src1, false) + : EmitMinMaxU32Value(state, src0, src1, false); + const auto high_pair = signed_value ? EmitMinMaxI32Value(state, src0, src1, true) + : EmitMinMaxU32Value(state, src0, src1, true); + const auto high_min = signed_value ? EmitMinMaxI32Value(state, high_pair, src2, false) + : EmitMinMaxU32Value(state, high_pair, src2, false); + const auto ret = signed_value ? EmitMinMaxI32Value(state, low_pair, high_min, true) + : EmitMinMaxU32Value(state, low_pair, high_min, true); + EmitStoreU32(state, inst.dst, ret); +} + +uint32_t EmitBitcastF32ToU32(EmitterState* state, uint32_t value) { + const auto ret = state->builder.AllocateId(); + state->builder.AddFunction({OpBitcast, state->uint_type, ret, value}); + return ret; +} + +uint32_t EmitBitcastU32ToF32(EmitterState* state, uint32_t value) { + const auto ret = state->builder.AllocateId(); + state->builder.AddFunction({OpBitcast, state->float_type, ret, value}); + return ret; +} + +uint32_t EmitAndU32(EmitterState* state, uint32_t lhs, uint32_t rhs) { + const auto ret = state->builder.AllocateId(); + state->builder.AddFunction({OpBitwiseAnd, state->uint_type, ret, lhs, rhs}); + return ret; +} + +uint32_t EmitLogicalAndBool(EmitterState* state, uint32_t lhs, uint32_t rhs) { + const auto ret = state->builder.AllocateId(); + state->builder.AddFunction({OpLogicalAnd, state->bool_type, ret, lhs, rhs}); + return ret; +} + +uint32_t EmitLogicalOrBool(EmitterState* state, uint32_t lhs, uint32_t rhs) { + const auto ret = state->builder.AllocateId(); + state->builder.AddFunction({OpLogicalOr, state->bool_type, ret, lhs, rhs}); + return ret; +} + +uint32_t EmitLogicalNotBool(EmitterState* state, uint32_t value) { + const auto ret = state->builder.AllocateId(); + state->builder.AddFunction({OpLogicalNot, state->bool_type, ret, value}); + return ret; +} + +F32Class EmitClassifyF32(EmitterState* state, uint32_t value) { + F32Class cls; + cls.bits = EmitBitcastF32ToU32(state, value); + const auto abs_bits = EmitAndConstant(state, cls.bits, 0x7fffffffu); + const auto exponent_bits = EmitAndConstant(state, abs_bits, 0x7f800000u); + const auto mantissa_bits = EmitAndConstant(state, abs_bits, 0x007fffffu); + const auto exponent_max = EmitCompareU32Constant(state, OpIEqual, exponent_bits, 0x7f800000u); + const auto mantissa_nonzero = EmitCompareU32Constant(state, OpINotEqual, mantissa_bits, 0); + const auto quiet_bit_clear = + EmitCompareU32Constant(state, OpIEqual, EmitAndConstant(state, cls.bits, 0x00400000u), 0); + cls.nan = EmitLogicalAndBool(state, exponent_max, mantissa_nonzero); + cls.snan = EmitLogicalAndBool(state, cls.nan, quiet_bit_clear); + cls.zero = EmitCompareU32Constant(state, OpIEqual, abs_bits, 0); + cls.quiet_bits = EmitOrU32(state, cls.bits, ConstantU32(state, 0x00400000u)); + return cls; +} + +uint32_t EmitClassMaskBitMatch(EmitterState* state, uint32_t mask, uint32_t bit, + uint32_t class_match) { + const auto selected = + EmitCompareU32Constant(state, OpINotEqual, EmitAndConstant(state, mask, 1u << bit), 0); + return EmitLogicalAndBool(state, selected, class_match); +} + +uint32_t EmitClassMaskF32(EmitterState* state, uint32_t value, uint32_t mask) { + const auto bits = EmitBitcastF32ToU32(state, value); + const auto sign_bits = EmitAndConstant(state, bits, 0x80000000u); + const auto abs_bits = EmitAndConstant(state, bits, 0x7fffffffu); + const auto exponent_bits = EmitAndConstant(state, abs_bits, 0x7f800000u); + const auto mantissa_bits = EmitAndConstant(state, abs_bits, 0x007fffffu); + const auto quiet_bits = EmitAndConstant(state, mantissa_bits, 0x00400000u); + + const auto sign = EmitCompareU32Constant(state, OpINotEqual, sign_bits, 0); + const auto positive = EmitLogicalNotBool(state, sign); + const auto exponent_zero = EmitCompareU32Constant(state, OpIEqual, exponent_bits, 0); + const auto exponent_nonzero = EmitLogicalNotBool(state, exponent_zero); + const auto exponent_inf = EmitCompareU32Constant(state, OpIEqual, exponent_bits, 0x7f800000u); + const auto finite_exponent = EmitLogicalNotBool(state, exponent_inf); + const auto mantissa_zero = EmitCompareU32Constant(state, OpIEqual, mantissa_bits, 0); + const auto mantissa_nonzero = EmitLogicalNotBool(state, mantissa_zero); + const auto quiet = EmitCompareU32Constant(state, OpINotEqual, quiet_bits, 0); + + const auto nan_common = EmitLogicalAndBool(state, exponent_inf, mantissa_nonzero); + const auto snan = EmitLogicalAndBool(state, nan_common, EmitLogicalNotBool(state, quiet)); + const auto qnan = EmitLogicalAndBool(state, nan_common, quiet); + const auto inf = EmitLogicalAndBool(state, exponent_inf, mantissa_zero); + const auto normal = EmitLogicalAndBool(state, exponent_nonzero, finite_exponent); + const auto denorm = EmitLogicalAndBool(state, exponent_zero, mantissa_nonzero); + const auto zero = EmitCompareU32Constant(state, OpIEqual, abs_bits, 0); + + uint32_t match = EmitClassMaskBitMatch(state, mask, 0, snan); + match = EmitLogicalOrBool(state, match, EmitClassMaskBitMatch(state, mask, 1, qnan)); + match = EmitLogicalOrBool( + state, match, EmitClassMaskBitMatch(state, mask, 2, EmitLogicalAndBool(state, inf, sign))); + match = EmitLogicalOrBool( + state, match, + EmitClassMaskBitMatch(state, mask, 3, EmitLogicalAndBool(state, normal, sign))); + match = EmitLogicalOrBool( + state, match, + EmitClassMaskBitMatch(state, mask, 4, EmitLogicalAndBool(state, denorm, sign))); + match = EmitLogicalOrBool( + state, match, EmitClassMaskBitMatch(state, mask, 5, EmitLogicalAndBool(state, zero, sign))); + match = EmitLogicalOrBool( + state, match, + EmitClassMaskBitMatch(state, mask, 6, EmitLogicalAndBool(state, zero, positive))); + match = EmitLogicalOrBool( + state, match, + EmitClassMaskBitMatch(state, mask, 7, EmitLogicalAndBool(state, denorm, positive))); + match = EmitLogicalOrBool( + state, match, + EmitClassMaskBitMatch(state, mask, 8, EmitLogicalAndBool(state, normal, positive))); + return EmitLogicalOrBool( + state, match, + EmitClassMaskBitMatch(state, mask, 9, EmitLogicalAndBool(state, inf, positive))); +} + +uint32_t EmitMinMaxF32Value(EmitterState* state, uint32_t lhs, uint32_t rhs, bool max_value) { + const auto lhs_class = EmitClassifyF32(state, lhs); + const auto rhs_class = EmitClassifyF32(state, rhs); + + const auto numeric_cond = state->builder.AllocateId(); + state->builder.AddFunction({max_value ? OpFOrdGreaterThanEqual : OpFOrdLessThan, + state->bool_type, numeric_cond, lhs, rhs}); + const auto ordered_bits = + EmitSelectValueU32(state, numeric_cond, lhs_class.bits, rhs_class.bits); + + const auto both_zero = EmitLogicalAndBool(state, lhs_class.zero, rhs_class.zero); + const auto zero_bits = max_value ? EmitAndU32(state, lhs_class.bits, rhs_class.bits) + : EmitOrU32(state, lhs_class.bits, rhs_class.bits); + const auto numeric_bits = EmitSelectValueU32(state, both_zero, zero_bits, ordered_bits); + + const auto rhs_nan_bits = + EmitSelectValueU32(state, rhs_class.nan, lhs_class.bits, numeric_bits); + const auto non_snan_bits = + EmitSelectValueU32(state, lhs_class.nan, rhs_class.bits, rhs_nan_bits); + const auto rhs_snan_bits = + EmitSelectValueU32(state, rhs_class.snan, rhs_class.quiet_bits, non_snan_bits); + const auto result_bits = + EmitSelectValueU32(state, lhs_class.snan, lhs_class.quiet_bits, rhs_snan_bits); + return EmitBitcastU32ToF32(state, result_bits); +} + +void EmitMinMaxF32(EmitterState* state, const IR::Instruction& inst, bool max_value) { + const auto lhs = EmitFloatLoad(state, inst.src[0]); + const auto rhs = EmitFloatLoad(state, inst.src[1]); + const auto u32 = state->builder.AllocateId(); + const auto f32 = EmitMinMaxF32Value(state, lhs, rhs, max_value); + const auto result = ApplyResultModifiersF32(state, f32, inst.dst); + state->builder.AddFunction({OpBitcast, state->uint_type, u32, result}); + EmitStoreU32(state, inst.dst, u32); +} + +void EmitCompareResult(EmitterState* state, const IR::Operand& dst, uint32_t cond) { + if (IsSccOperand(dst)) { + const auto ret = state->builder.AllocateId(); + state->builder.AddFunction( + {OpSelect, state->uint_type, ret, cond, ConstantU32(state, 1), ConstantU32(state, 0)}); + EmitStoreU32(state, dst, ret); + return; + } + + EmitLaneMaskPairFromBool(state, dst, cond); +} + +void EmitCompareU32(EmitterState* state, const IR::Instruction& inst, uint32_t opcode) { + const auto lhs = EmitValueLoad(state, inst.src[0]); + const auto rhs = EmitValueLoad(state, inst.src[1]); + const auto cond = state->builder.AllocateId(); + state->builder.AddFunction({opcode, state->bool_type, cond, lhs, rhs}); + EmitCompareResult(state, inst.dst, cond); +} + +void EmitCompareU16(EmitterState* state, const IR::Instruction& inst, uint32_t opcode) { + const auto lhs = EmitAndConstant(state, EmitValueLoad(state, inst.src[0]), 0xffffu); + const auto rhs = EmitAndConstant(state, EmitValueLoad(state, inst.src[1]), 0xffffu); + const auto cond = state->builder.AllocateId(); + state->builder.AddFunction({opcode, state->bool_type, cond, lhs, rhs}); + EmitCompareResult(state, inst.dst, cond); +} + +void EmitCompareI16(EmitterState* state, const IR::Instruction& inst, uint32_t opcode) { + const auto lhs = EmitSignExtendLow16U32(state, EmitValueLoad(state, inst.src[0])); + const auto rhs = EmitSignExtendLow16U32(state, EmitValueLoad(state, inst.src[1])); + const auto cond = state->builder.AllocateId(); + state->builder.AddFunction({opcode, state->bool_type, cond, lhs, rhs}); + EmitCompareResult(state, inst.dst, cond); +} + +void EmitBitCompareB32(EmitterState* state, const IR::Instruction& inst, bool bit_set) { + const auto value = EmitValueLoad(state, inst.src[0]); + const auto bit_index = state->builder.AllocateId(); + const auto bit = state->builder.AllocateId(); + const auto cond = state->builder.AllocateId(); + const auto ret = state->builder.AllocateId(); + state->builder.AddFunction({OpBitwiseAnd, state->uint_type, bit_index, + EmitValueLoad(state, inst.src[1]), ConstantU32(state, 31)}); + state->builder.AddFunction( + {OpBitFieldUExtract, state->uint_type, bit, value, bit_index, ConstantU32(state, 1)}); + state->builder.AddFunction( + {OpIEqual, state->bool_type, cond, bit, ConstantU32(state, bit_set ? 1u : 0u)}); + state->builder.AddFunction( + {OpSelect, state->uint_type, ret, cond, ConstantU32(state, 1), ConstantU32(state, 0)}); + EmitStoreU32(state, inst.dst, ret); +} + +void EmitCompareNeU64(EmitterState* state, const IR::Instruction& inst) { + const auto lhs_low = EmitSequentialValueLoad(state, inst.src[0], 0); + const auto lhs_high = EmitSequentialValueLoad(state, inst.src[0], 1); + const auto rhs_low = EmitSequentialValueLoad(state, inst.src[1], 0); + const auto rhs_high = EmitSequentialValueLoad(state, inst.src[1], 1); + const auto ne_low = state->builder.AllocateId(); + const auto ne_high = state->builder.AllocateId(); + const auto cond = state->builder.AllocateId(); + state->builder.AddFunction({OpINotEqual, state->bool_type, ne_low, lhs_low, rhs_low}); + state->builder.AddFunction({OpINotEqual, state->bool_type, ne_high, lhs_high, rhs_high}); + state->builder.AddFunction({OpLogicalOr, state->bool_type, cond, ne_low, ne_high}); + EmitCompareResult(state, inst.dst, cond); +} + +void EmitCompareConstant(EmitterState* state, const IR::Instruction& inst, bool value) { + const auto cond = state->builder.AllocateId(); + state->builder.AddFunction({value ? OpIEqual : OpINotEqual, state->bool_type, cond, + ConstantU32(state, 0), ConstantU32(state, 0)}); + EmitCompareResult(state, inst.dst, cond); +} + +void EmitCompareMaskU32(EmitterState* state, const IR::Instruction& inst, uint32_t opcode) { + const auto lhs = EmitValueLoad(state, inst.src[0]); + const auto rhs = EmitValueLoad(state, inst.src[1]); + const auto cond = state->builder.AllocateId(); + state->builder.AddFunction({opcode, state->bool_type, cond, lhs, rhs}); + EmitCompareResult(state, inst.dst, cond); +} + +void EmitCompareF32(EmitterState* state, const IR::Instruction& inst, uint32_t opcode) { + const auto lhs = EmitFloatLoad(state, inst.src[0]); + const auto rhs = EmitFloatLoad(state, inst.src[1]); + const auto cond = state->builder.AllocateId(); + state->builder.AddFunction({opcode, state->bool_type, cond, lhs, rhs}); + EmitCompareResult(state, inst.dst, cond); +} + +void EmitCompareOrderedF32(EmitterState* state, const IR::Instruction& inst, bool ordered) { + const auto lhs = EmitFloatLoad(state, inst.src[0]); + const auto rhs = EmitFloatLoad(state, inst.src[1]); + const auto lhs_check = state->builder.AllocateId(); + const auto rhs_check = state->builder.AllocateId(); + const auto cond = state->builder.AllocateId(); + if (ordered) { + state->builder.AddFunction({OpFOrdEqual, state->bool_type, lhs_check, lhs, lhs}); + state->builder.AddFunction({OpFOrdEqual, state->bool_type, rhs_check, rhs, rhs}); + state->builder.AddFunction({OpLogicalAnd, state->bool_type, cond, lhs_check, rhs_check}); + } else { + state->builder.AddFunction({OpFUnordNotEqual, state->bool_type, lhs_check, lhs, lhs}); + state->builder.AddFunction({OpFUnordNotEqual, state->bool_type, rhs_check, rhs, rhs}); + state->builder.AddFunction({OpLogicalOr, state->bool_type, cond, lhs_check, rhs_check}); + } + EmitCompareResult(state, inst.dst, cond); +} + +void EmitCompareClassF32(EmitterState* state, const IR::Instruction& inst) { + const auto value = EmitFloatLoad(state, inst.src[0]); + const auto mask = EmitValueLoad(state, inst.src[1]); + EmitCompareResult(state, inst.dst, EmitClassMaskF32(state, value, mask)); +} + +void EmitCompareMaskF32(EmitterState* state, const IR::Instruction& inst, uint32_t opcode) { + const auto lhs = EmitFloatLoad(state, inst.src[0]); + const auto rhs = EmitFloatLoad(state, inst.src[1]); + const auto cond = state->builder.AllocateId(); + state->builder.AddFunction({opcode, state->bool_type, cond, lhs, rhs}); + EmitCompareResult(state, inst.dst, cond); +} + +void EmitConvertU32ToF32(EmitterState* state, const IR::Instruction& inst) { + const auto src = EmitValueLoad(state, inst.src[0]); + const auto f32 = state->builder.AllocateId(); + const auto u32 = state->builder.AllocateId(); + state->builder.AddFunction({OpConvertUToF, state->float_type, f32, src}); + const auto result = ApplyResultModifiersF32(state, f32, inst.dst); + state->builder.AddFunction({OpBitcast, state->uint_type, u32, result}); + EmitStoreU32(state, inst.dst, u32); +} + +void EmitConvertI32ToF32(EmitterState* state, const IR::Instruction& inst) { + const auto src = EmitValueLoad(state, inst.src[0]); + const auto i32 = state->builder.AllocateId(); + const auto f32 = state->builder.AllocateId(); + const auto u32 = state->builder.AllocateId(); + state->builder.AddFunction({OpBitcast, state->int_type, i32, src}); + state->builder.AddFunction({OpConvertSToF, state->float_type, f32, i32}); + const auto result = ApplyResultModifiersF32(state, f32, inst.dst); + state->builder.AddFunction({OpBitcast, state->uint_type, u32, result}); + EmitStoreU32(state, inst.dst, u32); +} + +uint32_t EmitTruncF32Value(EmitterState* state, uint32_t value) { + const auto ret = state->builder.AllocateId(); + state->builder.AddFunction( + {OpExtInst, state->float_type, ret, state->glsl_std450, GlslTrunc, value}); + return ret; +} + +void EmitConvertF32ToU32(EmitterState* state, const IR::Instruction& inst) { + const auto src = EmitFloatLoad(state, inst.src[0]); + const auto trunc = EmitTruncF32Value(state, src); + const auto converted = state->builder.AllocateId(); + const auto below_zero = state->builder.AllocateId(); + const auto above_max = state->builder.AllocateId(); + state->builder.AddFunction({OpConvertFToU, state->uint_type, converted, trunc}); + state->builder.AddFunction( + {OpFOrdLessThanEqual, state->bool_type, below_zero, src, ConstantF32(state, 0)}); + state->builder.AddFunction({OpFOrdGreaterThanEqual, state->bool_type, above_max, src, + ConstantF32(state, 0x4f800000u)}); + const auto zero_or_nan = EmitLogicalOrBool(state, EmitClassifyF32(state, src).nan, below_zero); + const auto saturated = + EmitSelectValueU32(state, above_max, ConstantU32(state, 0xffffffffu), converted); + const auto ret = EmitSelectValueU32(state, zero_or_nan, ConstantU32(state, 0), saturated); + EmitStoreU32(state, inst.dst, ret); +} + +void EmitConvertF32ToI32(EmitterState* state, const IR::Instruction& inst) { + const auto src = EmitFloatLoad(state, inst.src[0]); + const auto trunc = EmitTruncF32Value(state, src); + const auto i32 = state->builder.AllocateId(); + const auto converted = state->builder.AllocateId(); + const auto below_min = state->builder.AllocateId(); + const auto above_max = state->builder.AllocateId(); + state->builder.AddFunction({OpConvertFToS, state->int_type, i32, trunc}); + state->builder.AddFunction({OpBitcast, state->uint_type, converted, i32}); + state->builder.AddFunction( + {OpFOrdLessThanEqual, state->bool_type, below_min, src, ConstantF32(state, 0xcf000000u)}); + state->builder.AddFunction({OpFOrdGreaterThanEqual, state->bool_type, above_max, src, + ConstantF32(state, 0x4f000000u)}); + const auto high_sat = + EmitSelectValueU32(state, above_max, ConstantU32(state, 0x7fffffffu), converted); + const auto low_sat = + EmitSelectValueU32(state, below_min, ConstantU32(state, 0x80000000u), high_sat); + const auto ret = + EmitSelectValueU32(state, EmitClassifyF32(state, src).nan, ConstantU32(state, 0), low_sat); + EmitStoreU32(state, inst.dst, ret); +} + +void EmitConvertF32ToF16(EmitterState* state, const IR::Instruction& inst) { + const auto src = EmitFloatLoad(state, inst.src[0]); + const auto pair = state->builder.AllocateId(); + const auto u32 = state->builder.AllocateId(); + state->builder.AddFunction( + {OpCompositeConstruct, state->vec2_float_type, pair, src, ConstantF32(state, 0)}); + state->builder.AddFunction( + {OpExtInst, state->uint_type, u32, state->glsl_std450, GlslPackHalf2x16, pair}); + EmitStoreU32(state, inst.dst, u32); +} + +void EmitConvertF16ToF32(EmitterState* state, const IR::Instruction& inst) { + const auto src = EmitValueLoad(state, inst.src[0]); + const auto unpacked = state->builder.AllocateId(); + auto f32 = state->builder.AllocateId(); + const auto u32 = state->builder.AllocateId(); + state->builder.AddFunction( + {OpExtInst, state->vec2_float_type, unpacked, state->glsl_std450, GlslUnpackHalf2x16, src}); + state->builder.AddFunction({OpCompositeExtract, state->float_type, f32, unpacked, 0}); + if (inst.src[0].absolute) { + const auto abs_value = state->builder.AllocateId(); + state->builder.AddFunction( + {OpExtInst, state->float_type, abs_value, state->glsl_std450, GlslFAbs, f32}); + f32 = abs_value; + } + if (inst.src[0].negate) { + const auto neg_value = state->builder.AllocateId(); + state->builder.AddFunction({OpFNegate, state->float_type, neg_value, f32}); + f32 = neg_value; + } + const auto result = ApplyResultModifiersF32(state, f32, inst.dst); + state->builder.AddFunction({OpBitcast, state->uint_type, u32, result}); + EmitStoreU32(state, inst.dst, u32); +} + +void EmitConvertF16ToU16(EmitterState* state, const IR::Instruction& inst) { + const auto src = EmitValueLoad(state, inst.src[0]); + const auto unpacked = state->builder.AllocateId(); + auto f32 = state->builder.AllocateId(); + state->builder.AddFunction( + {OpExtInst, state->vec2_float_type, unpacked, state->glsl_std450, GlslUnpackHalf2x16, src}); + state->builder.AddFunction({OpCompositeExtract, state->float_type, f32, unpacked, 0}); + if (inst.src[0].absolute) { + const auto abs_value = state->builder.AllocateId(); + state->builder.AddFunction( + {OpExtInst, state->float_type, abs_value, state->glsl_std450, GlslFAbs, f32}); + f32 = abs_value; + } + if (inst.src[0].negate) { + const auto neg_value = state->builder.AllocateId(); + state->builder.AddFunction({OpFNegate, state->float_type, neg_value, f32}); + f32 = neg_value; + } + + const auto truncated = EmitTruncF32Value(state, f32); + const auto below_zero = state->builder.AllocateId(); + const auto above_max = state->builder.AllocateId(); + const auto safe_low = state->builder.AllocateId(); + const auto safe_value = state->builder.AllocateId(); + const auto converted = state->builder.AllocateId(); + state->builder.AddFunction( + {OpFOrdLessThanEqual, state->bool_type, below_zero, f32, ConstantF32(state, 0)}); + state->builder.AddFunction({OpFOrdGreaterThanEqual, state->bool_type, above_max, f32, + ConstantF32(state, 0x477fff00u)}); + const auto zero_or_nan = EmitLogicalOrBool(state, EmitClassifyF32(state, f32).nan, below_zero); + state->builder.AddFunction( + {OpSelect, state->float_type, safe_low, zero_or_nan, ConstantF32(state, 0), truncated}); + state->builder.AddFunction({OpSelect, state->float_type, safe_value, above_max, + ConstantF32(state, 0x477fff00u), safe_low}); + state->builder.AddFunction({OpConvertFToU, state->uint_type, converted, safe_value}); + EmitStoreU32(state, inst.dst, EmitAndConstant(state, converted, 0xffffu)); +} + +void EmitConvertU16ToF16(EmitterState* state, const IR::Instruction& inst) { + const auto src_bits = EmitU16LaneBits(state, inst.src[0], false, false); + const auto f32 = state->builder.AllocateId(); + state->builder.AddFunction({OpConvertUToF, state->float_type, f32, src_bits}); + EmitStoreU32(state, inst.dst, EmitPackLowF32ToF16Bits(state, f32)); +} + +void EmitConvertI16ToF16(EmitterState* state, const IR::Instruction& inst) { + const auto src_bits = EmitU16LaneBits(state, inst.src[0], false, true); + const auto i32 = state->builder.AllocateId(); + const auto f32 = state->builder.AllocateId(); + state->builder.AddFunction({OpBitcast, state->int_type, i32, src_bits}); + state->builder.AddFunction({OpConvertSToF, state->float_type, f32, i32}); + EmitStoreU32(state, inst.dst, EmitPackLowF32ToF16Bits(state, f32)); +} + +void EmitConvertF16ToI16(EmitterState* state, const IR::Instruction& inst) { + const auto src = EmitValueLoad(state, inst.src[0]); + const auto unpacked = state->builder.AllocateId(); + auto f32 = state->builder.AllocateId(); + state->builder.AddFunction( + {OpExtInst, state->vec2_float_type, unpacked, state->glsl_std450, GlslUnpackHalf2x16, src}); + state->builder.AddFunction({OpCompositeExtract, state->float_type, f32, unpacked, 0}); + if (inst.src[0].absolute) { + const auto abs_value = state->builder.AllocateId(); + state->builder.AddFunction( + {OpExtInst, state->float_type, abs_value, state->glsl_std450, GlslFAbs, f32}); + f32 = abs_value; + } + if (inst.src[0].negate) { + const auto neg_value = state->builder.AllocateId(); + state->builder.AddFunction({OpFNegate, state->float_type, neg_value, f32}); + f32 = neg_value; + } + + const auto truncated = EmitTruncF32Value(state, f32); + const auto below_min = state->builder.AllocateId(); + const auto above_max = state->builder.AllocateId(); + const auto safe_low = state->builder.AllocateId(); + const auto safe_value = state->builder.AllocateId(); + const auto nan_safe = state->builder.AllocateId(); + const auto i32 = state->builder.AllocateId(); + const auto u32 = state->builder.AllocateId(); + state->builder.AddFunction( + {OpFOrdLessThanEqual, state->bool_type, below_min, f32, ConstantF32(state, 0xc7000000u)}); + state->builder.AddFunction({OpFOrdGreaterThanEqual, state->bool_type, above_max, f32, + ConstantF32(state, 0x46fffe00u)}); + state->builder.AddFunction({OpSelect, state->float_type, safe_low, below_min, + ConstantF32(state, 0xc7000000u), truncated}); + state->builder.AddFunction({OpSelect, state->float_type, safe_value, above_max, + ConstantF32(state, 0x46fffe00u), safe_low}); + state->builder.AddFunction({OpSelect, state->float_type, nan_safe, + EmitClassifyF32(state, f32).nan, ConstantF32(state, 0), + safe_value}); + state->builder.AddFunction({OpConvertFToS, state->int_type, i32, nan_safe}); + state->builder.AddFunction({OpBitcast, state->uint_type, u32, i32}); + EmitStoreU32(state, inst.dst, EmitAndConstant(state, u32, 0xffffu)); +} + +void EmitConvertRoundPlusInfF32ToI32(EmitterState* state, const IR::Instruction& inst) { + const auto src = EmitFloatLoad(state, inst.src[0]); + const auto biased = state->builder.AllocateId(); + const auto rounded = state->builder.AllocateId(); + const auto i32 = state->builder.AllocateId(); + const auto u32 = state->builder.AllocateId(); + state->builder.AddFunction( + {OpFAdd, state->float_type, biased, src, ConstantF32(state, 0x3f000000u)}); + state->builder.AddFunction( + {OpExtInst, state->float_type, rounded, state->glsl_std450, GlslFloor, biased}); + state->builder.AddFunction({OpConvertFToS, state->int_type, i32, rounded}); + state->builder.AddFunction({OpBitcast, state->uint_type, u32, i32}); + EmitStoreU32(state, inst.dst, u32); +} + +void EmitConvertFloorF32ToI32(EmitterState* state, const IR::Instruction& inst) { + const auto src = EmitFloatLoad(state, inst.src[0]); + const auto floor = state->builder.AllocateId(); + const auto i32 = state->builder.AllocateId(); + const auto u32 = state->builder.AllocateId(); + state->builder.AddFunction( + {OpExtInst, state->float_type, floor, state->glsl_std450, GlslFloor, src}); + state->builder.AddFunction({OpConvertFToS, state->int_type, i32, floor}); + state->builder.AddFunction({OpBitcast, state->uint_type, u32, i32}); + EmitStoreU32(state, inst.dst, u32); +} + +void EmitConvertI4ToOffsetF32(EmitterState* state, const IR::Instruction& inst) { + const auto src = EmitValueLoad(state, inst.src[0]); + const auto src_i32 = state->builder.AllocateId(); + const auto nibble = state->builder.AllocateId(); + const auto f32 = state->builder.AllocateId(); + const auto scaled = state->builder.AllocateId(); + const auto u32 = state->builder.AllocateId(); + state->builder.AddFunction({OpBitcast, state->int_type, src_i32, src}); + state->builder.AddFunction({OpBitFieldSExtract, state->int_type, nibble, src_i32, + ConstantU32(state, 0), ConstantU32(state, 4)}); + state->builder.AddFunction({OpConvertSToF, state->float_type, f32, nibble}); + state->builder.AddFunction( + {OpFDiv, state->float_type, scaled, f32, ConstantF32(state, 0x41800000u)}); + const auto result = ApplyResultModifiersF32(state, scaled, inst.dst); + state->builder.AddFunction({OpBitcast, state->uint_type, u32, result}); + EmitStoreU32(state, inst.dst, u32); +} + +void EmitConvertByteU32ToF32(EmitterState* state, const IR::Instruction& inst) { + const auto src = EmitValueLoad(state, inst.src[0]); + const auto index = inst.src[1].kind == IR::OperandKind::ImmediateU32 ? inst.src[1].imm : 0u; + const auto byte = EmitBitFieldExtractConstant(state, src, (index & 3u) * 8u, 8u); + const auto f32 = state->builder.AllocateId(); + const auto u32 = state->builder.AllocateId(); + state->builder.AddFunction({OpConvertUToF, state->float_type, f32, byte}); + const auto result = ApplyResultModifiersF32(state, f32, inst.dst); + state->builder.AddFunction({OpBitcast, state->uint_type, u32, result}); + EmitStoreU32(state, inst.dst, u32); +} + +uint32_t EmitFlushF32DenormToSignedZero(EmitterState* state, uint32_t value) { + const auto bits = state->builder.AllocateId(); + const auto abs_bits = state->builder.AllocateId(); + const auto sign_bits = state->builder.AllocateId(); + const auto non_zero = state->builder.AllocateId(); + const auto subnormal = state->builder.AllocateId(); + const auto flush = state->builder.AllocateId(); + const auto selected = state->builder.AllocateId(); + const auto ret = state->builder.AllocateId(); + state->builder.AddFunction({OpBitcast, state->uint_type, bits, value}); + state->builder.AddFunction( + {OpBitwiseAnd, state->uint_type, abs_bits, bits, ConstantU32(state, 0x7fffffffu)}); + state->builder.AddFunction( + {OpBitwiseAnd, state->uint_type, sign_bits, bits, ConstantU32(state, 0x80000000u)}); + state->builder.AddFunction( + {OpINotEqual, state->bool_type, non_zero, abs_bits, ConstantU32(state, 0)}); + state->builder.AddFunction( + {OpULessThan, state->bool_type, subnormal, abs_bits, ConstantU32(state, 0x00800000u)}); + state->builder.AddFunction({OpLogicalAnd, state->bool_type, flush, non_zero, subnormal}); + state->builder.AddFunction({OpSelect, state->uint_type, selected, flush, sign_bits, bits}); + state->builder.AddFunction({OpBitcast, state->float_type, ret, selected}); + return ret; +} + +void EmitRcpF32(EmitterState* state, const IR::Instruction& inst) { + const auto src = EmitFlushF32DenormToSignedZero(state, EmitFloatLoad(state, inst.src[0])); + const auto f32 = state->builder.AllocateId(); + const auto u32 = state->builder.AllocateId(); + state->builder.AddFunction( + {OpFDiv, state->float_type, f32, ConstantF32(state, 0x3f800000u), src}); + const auto result = ApplyResultModifiersF32(state, f32, inst.dst); + state->builder.AddFunction({OpBitcast, state->uint_type, u32, result}); + EmitStoreU32(state, inst.dst, u32); +} + +uint32_t EmitTrigCycleF32(EmitterState* state, uint32_t src, bool preserve_signed_zero) { + const auto fract = state->builder.AllocateId(); + const auto bits = state->builder.AllocateId(); + const auto abs_bits = state->builder.AllocateId(); + const auto large = state->builder.AllocateId(); + const auto finite = state->builder.AllocateId(); + const auto large_finite = state->builder.AllocateId(); + const auto reduced = state->builder.AllocateId(); + state->builder.AddFunction( + {OpExtInst, state->float_type, fract, state->glsl_std450, GlslFract, src}); + state->builder.AddFunction({OpBitcast, state->uint_type, bits, src}); + state->builder.AddFunction( + {OpBitwiseAnd, state->uint_type, abs_bits, bits, ConstantU32(state, 0x7fffffffu)}); + state->builder.AddFunction( + {OpUGreaterThanEqual, state->bool_type, large, abs_bits, ConstantU32(state, 0x4b000000u)}); + state->builder.AddFunction( + {OpULessThan, state->bool_type, finite, abs_bits, ConstantU32(state, 0x7f800000u)}); + state->builder.AddFunction({OpLogicalAnd, state->bool_type, large_finite, large, finite}); + state->builder.AddFunction( + {OpSelect, state->float_type, reduced, large_finite, ConstantF32(state, 0), fract}); + if (!preserve_signed_zero) { + return reduced; + } + const auto zero = state->builder.AllocateId(); + const auto ret = state->builder.AllocateId(); + state->builder.AddFunction({OpIEqual, state->bool_type, zero, abs_bits, ConstantU32(state, 0)}); + state->builder.AddFunction({OpSelect, state->float_type, ret, zero, src, reduced}); + return ret; +} + +void EmitFloatExtInst(EmitterState* state, const IR::Instruction& inst, uint32_t ext_inst, + bool scale_by_two_pi, bool flush_denorm) { + auto src = EmitFloatLoad(state, inst.src[0]); + if (flush_denorm) { + src = EmitFlushF32DenormToSignedZero(state, src); + } + if (scale_by_two_pi) { + src = EmitTrigCycleF32(state, src, ext_inst == GlslSin); + const auto scaled = state->builder.AllocateId(); + state->builder.AddFunction( + {OpFMul, state->float_type, scaled, src, ConstantF32(state, 0x40c90fdbu)}); + src = scaled; + } + const auto f32 = state->builder.AllocateId(); + const auto u32 = state->builder.AllocateId(); + state->builder.AddFunction( + {OpExtInst, state->float_type, f32, state->glsl_std450, ext_inst, src}); + const auto result = ApplyResultModifiersF32(state, f32, inst.dst); + state->builder.AddFunction({OpBitcast, state->uint_type, u32, result}); + EmitStoreU32(state, inst.dst, u32); +} + +void EmitMadF32(EmitterState* state, const IR::Instruction& inst) { + const auto lhs = EmitMixF32Load(state, inst.src[0]); + const auto rhs = EmitMixF32Load(state, inst.src[1]); + const auto add = EmitMixF32Load(state, inst.src[2]); + const auto f32 = state->builder.AllocateId(); + const auto u32 = state->builder.AllocateId(); + state->builder.AddFunction( + {OpExtInst, state->float_type, f32, state->glsl_std450, GlslFma, lhs, rhs, add}); + const auto result = ApplyResultModifiersF32(state, f32, inst.dst); + state->builder.AddFunction({OpBitcast, state->uint_type, u32, result}); + EmitStoreU32(state, inst.dst, u32); +} + +uint32_t EmitFNegateValue(EmitterState* state, uint32_t value) { + const auto ret = state->builder.AllocateId(); + state->builder.AddFunction({OpFNegate, state->float_type, ret, value}); + return ret; +} + +uint32_t EmitFAbsValue(EmitterState* state, uint32_t value) { + const auto ret = state->builder.AllocateId(); + state->builder.AddFunction( + {OpExtInst, state->float_type, ret, state->glsl_std450, GlslFAbs, value}); + return ret; +} + +uint32_t EmitFCompareValue(EmitterState* state, uint32_t opcode, uint32_t lhs, uint32_t rhs) { + const auto ret = state->builder.AllocateId(); + state->builder.AddFunction({opcode, state->bool_type, ret, lhs, rhs}); + return ret; +} + +uint32_t EmitLogicalAndValue(EmitterState* state, uint32_t lhs, uint32_t rhs) { + const auto ret = state->builder.AllocateId(); + state->builder.AddFunction({OpLogicalAnd, state->bool_type, ret, lhs, rhs}); + return ret; +} + +uint32_t EmitFSelectValue(EmitterState* state, uint32_t cond, uint32_t true_value, + uint32_t false_value) { + const auto ret = state->builder.AllocateId(); + state->builder.AddFunction({OpSelect, state->float_type, ret, cond, true_value, false_value}); + return ret; +} + +uint32_t EmitFMulConstant(EmitterState* state, uint32_t value, uint32_t bits) { + const auto ret = state->builder.AllocateId(); + state->builder.AddFunction({OpFMul, state->float_type, ret, value, ConstantF32(state, bits)}); + return ret; +} + +CubeF32Values EmitCubeF32Values(EmitterState* state, const IR::Instruction& inst) { + CubeF32Values cube; + cube.x = EmitFloatLoad(state, inst.src[0]); + cube.y = EmitFloatLoad(state, inst.src[1]); + cube.z = EmitFloatLoad(state, inst.src[2]); + cube.nx = EmitFNegateValue(state, cube.x); + cube.ny = EmitFNegateValue(state, cube.y); + cube.nz = EmitFNegateValue(state, cube.z); + const auto ax = EmitFAbsValue(state, cube.x); + const auto ay = EmitFAbsValue(state, cube.y); + const auto az = EmitFAbsValue(state, cube.z); + const auto z_ge_x = EmitFCompareValue(state, OpFOrdGreaterThanEqual, az, ax); + const auto z_ge_y = EmitFCompareValue(state, OpFOrdGreaterThanEqual, az, ay); + cube.z_face = EmitLogicalAndValue(state, z_ge_x, z_ge_y); + cube.y_face = EmitFCompareValue(state, OpFOrdGreaterThanEqual, ay, ax); + cube.x_neg = EmitFCompareValue(state, OpFOrdLessThan, cube.x, ConstantF32(state, 0)); + cube.y_neg = EmitFCompareValue(state, OpFOrdLessThan, cube.y, ConstantF32(state, 0)); + cube.z_neg = EmitFCompareValue(state, OpFOrdLessThan, cube.z, ConstantF32(state, 0)); + return cube; +} + +uint32_t EmitCubeIdF32(EmitterState* state, const CubeF32Values& cube) { + const auto x_face = + EmitFSelectValue(state, cube.x_neg, ConstantF32(state, 0x3f800000u), ConstantF32(state, 0)); + const auto y_face = EmitFSelectValue(state, cube.y_neg, ConstantF32(state, 0x40400000u), + ConstantF32(state, 0x40000000u)); + const auto z_face = EmitFSelectValue(state, cube.z_neg, ConstantF32(state, 0x40a00000u), + ConstantF32(state, 0x40800000u)); + const auto xy = EmitFSelectValue(state, cube.y_face, y_face, x_face); + return EmitFSelectValue(state, cube.z_face, z_face, xy); +} + +uint32_t EmitCubeScF32(EmitterState* state, const CubeF32Values& cube) { + const auto x_sc = EmitFSelectValue(state, cube.x_neg, cube.z, cube.nz); + const auto z_sc = EmitFSelectValue(state, cube.z_neg, cube.nx, cube.x); + const auto xy = EmitFSelectValue(state, cube.y_face, cube.x, x_sc); + return EmitFSelectValue(state, cube.z_face, z_sc, xy); +} + +uint32_t EmitCubeTcF32(EmitterState* state, const CubeF32Values& cube) { + const auto y_tc = EmitFSelectValue(state, cube.y_neg, cube.nz, cube.z); + const auto xy = EmitFSelectValue(state, cube.y_face, y_tc, cube.ny); + return EmitFSelectValue(state, cube.z_face, cube.ny, xy); +} + +uint32_t EmitCubeMaF32(EmitterState* state, const CubeF32Values& cube) { + const auto x_ma = EmitFMulConstant(state, cube.x, 0x40000000u); + const auto y_ma = EmitFMulConstant(state, cube.y, 0x40000000u); + const auto z_ma = EmitFMulConstant(state, cube.z, 0x40000000u); + const auto xy = EmitFSelectValue(state, cube.y_face, y_ma, x_ma); + return EmitFSelectValue(state, cube.z_face, z_ma, xy); +} + +void EmitCubeF32(EmitterState* state, const IR::Instruction& inst) { + const auto cube = EmitCubeF32Values(state, inst); + uint32_t ret = 0; + switch (inst.op) { + case IR::Opcode::CubeIdF32: ret = EmitCubeIdF32(state, cube); break; + case IR::Opcode::CubeScF32: ret = EmitCubeScF32(state, cube); break; + case IR::Opcode::CubeTcF32: ret = EmitCubeTcF32(state, cube); break; + case IR::Opcode::CubeMaF32: ret = EmitCubeMaF32(state, cube); break; + default: ret = ConstantF32(state, 0); break; + } + const auto bits = state->builder.AllocateId(); + state->builder.AddFunction({OpBitcast, state->uint_type, bits, ret}); + EmitStoreU32(state, inst.dst, bits); +} + +void EmitDot2AccF32F16(EmitterState* state, const IR::Instruction& inst) { + const auto src0 = EmitValueLoad(state, inst.src[0]); + const auto src1 = EmitValueLoad(state, inst.src[1]); + const auto vec0 = state->builder.AllocateId(); + const auto vec1 = state->builder.AllocateId(); + const auto a_lo = state->builder.AllocateId(); + const auto a_hi = state->builder.AllocateId(); + const auto b_lo = state->builder.AllocateId(); + const auto b_hi = state->builder.AllocateId(); + const auto acc = EmitFloatLoad(state, inst.src[2]); + const auto lo = state->builder.AllocateId(); + const auto dot = state->builder.AllocateId(); + const auto u32 = state->builder.AllocateId(); + state->builder.AddFunction( + {OpExtInst, state->vec2_float_type, vec0, state->glsl_std450, GlslUnpackHalf2x16, src0}); + state->builder.AddFunction( + {OpExtInst, state->vec2_float_type, vec1, state->glsl_std450, GlslUnpackHalf2x16, src1}); + state->builder.AddFunction({OpCompositeExtract, state->float_type, a_lo, vec0, 0}); + state->builder.AddFunction({OpCompositeExtract, state->float_type, a_hi, vec0, 1}); + state->builder.AddFunction({OpCompositeExtract, state->float_type, b_lo, vec1, 0}); + state->builder.AddFunction({OpCompositeExtract, state->float_type, b_hi, vec1, 1}); + state->builder.AddFunction( + {OpExtInst, state->float_type, lo, state->glsl_std450, GlslFma, a_lo, b_lo, acc}); + state->builder.AddFunction( + {OpExtInst, state->float_type, dot, state->glsl_std450, GlslFma, a_hi, b_hi, lo}); + state->builder.AddFunction({OpBitcast, state->uint_type, u32, dot}); + EmitStoreU32(state, inst.dst, u32); +} + +uint32_t EmitPackedF16Lane(EmitterState* state, const IR::Operand& operand, bool high_lane) { + const auto raw = EmitValueLoad(state, operand); + const auto unpacked = state->builder.AllocateId(); + auto value = state->builder.AllocateId(); + const auto lane = high_lane ? (operand.op_sel_hi ? 1u : 0u) : (operand.op_sel ? 1u : 0u); + state->builder.AddFunction( + {OpExtInst, state->vec2_float_type, unpacked, state->glsl_std450, GlslUnpackHalf2x16, raw}); + state->builder.AddFunction({OpCompositeExtract, state->float_type, value, unpacked, lane}); + if (operand.absolute) { + const auto abs_value = state->builder.AllocateId(); + state->builder.AddFunction( + {OpExtInst, state->float_type, abs_value, state->glsl_std450, GlslFAbs, value}); + value = abs_value; + } + if (high_lane ? operand.negate_hi : operand.negate) { + const auto neg_value = state->builder.AllocateId(); + state->builder.AddFunction({OpFNegate, state->float_type, neg_value, value}); + value = neg_value; + } + return value; +} + +void EmitCompareF16(EmitterState* state, const IR::Instruction& inst, uint32_t opcode) { + const auto lhs = EmitPackedF16Lane(state, inst.src[0], false); + const auto rhs = EmitPackedF16Lane(state, inst.src[1], false); + const auto cond = state->builder.AllocateId(); + state->builder.AddFunction({opcode, state->bool_type, cond, lhs, rhs}); + EmitCompareResult(state, inst.dst, cond); +} + +uint32_t EmitPackedF16LaneBits(EmitterState* state, const IR::Operand& operand, bool high_lane) { + const auto raw = EmitValueLoad(state, operand); + const auto lane = high_lane ? (operand.op_sel_hi ? 1u : 0u) : (operand.op_sel ? 1u : 0u); + auto bits = lane != 0u ? EmitShiftRightConstant(state, raw, 16) : raw; + bits = EmitAndConstant(state, bits, 0xffffu); + if (operand.absolute) { + bits = EmitAndConstant(state, bits, 0x7fffu); + } + if (high_lane ? operand.negate_hi : operand.negate) { + bits = EmitXorConstant(state, bits, 0x8000u); + } + return bits; +} + +uint32_t EmitF16BitsToF32(EmitterState* state, uint32_t bits) { + const auto unpacked = state->builder.AllocateId(); + const auto ret = state->builder.AllocateId(); + state->builder.AddFunction({OpExtInst, state->vec2_float_type, unpacked, state->glsl_std450, + GlslUnpackHalf2x16, bits}); + state->builder.AddFunction({OpCompositeExtract, state->float_type, ret, unpacked, 0}); + return ret; +} + +uint32_t EmitPackLowF32ToF16Bits(EmitterState* state, uint32_t value) { + const auto pair = state->builder.AllocateId(); + const auto ret = state->builder.AllocateId(); + state->builder.AddFunction( + {OpCompositeConstruct, state->vec2_float_type, pair, value, ConstantF32(state, 0)}); + state->builder.AddFunction( + {OpExtInst, state->uint_type, ret, state->glsl_std450, GlslPackHalf2x16, pair}); + return EmitAndConstant(state, ret, 0xffffu); +} + +void EmitPackB32F16(EmitterState* state, const IR::Instruction& inst) { + const auto lo = EmitPackedF16LaneBits(state, inst.src[0], false); + const auto hi = + EmitShiftLeftConstant(state, EmitPackedF16LaneBits(state, inst.src[1], false), 16); + EmitStoreU32(state, inst.dst, EmitOrU32(state, lo, hi)); +} + +F16Class EmitClassifyF16Bits(EmitterState* state, uint32_t value) { + F16Class cls; + cls.bits = EmitAndConstant(state, value, 0xffffu); + const auto abs_bits = EmitAndConstant(state, cls.bits, 0x7fffu); + const auto exponent_bits = EmitAndConstant(state, abs_bits, 0x7c00u); + const auto mantissa_bits = EmitAndConstant(state, abs_bits, 0x03ffu); + const auto exponent_max = EmitCompareU32Constant(state, OpIEqual, exponent_bits, 0x7c00u); + const auto mantissa_nonzero = EmitCompareU32Constant(state, OpINotEqual, mantissa_bits, 0); + const auto quiet_bit_clear = + EmitCompareU32Constant(state, OpIEqual, EmitAndConstant(state, cls.bits, 0x0200u), 0); + cls.nan = EmitLogicalAndBool(state, exponent_max, mantissa_nonzero); + cls.snan = EmitLogicalAndBool(state, cls.nan, quiet_bit_clear); + cls.zero = EmitCompareU32Constant(state, OpIEqual, abs_bits, 0); + cls.quiet_bits = EmitOrU32(state, cls.bits, ConstantU32(state, 0x0200u)); + return cls; +} + +uint32_t EmitNegativeNumericF16Bool(EmitterState* state, const F16Class& cls) { + const auto sign = + EmitCompareU32Constant(state, OpINotEqual, EmitAndConstant(state, cls.bits, 0x8000u), 0); + const auto nonzero = EmitLogicalNotBool(state, cls.zero); + return EmitLogicalAndBool(state, EmitLogicalAndBool(state, sign, nonzero), + EmitLogicalNotBool(state, cls.nan)); +} + +uint32_t EmitRcpF16Bits(EmitterState* state, uint32_t bits) { + const auto f32 = EmitF16BitsToF32(state, bits); + const auto ret = state->builder.AllocateId(); + state->builder.AddFunction( + {OpFDiv, state->float_type, ret, ConstantF32(state, 0x3f800000u), f32}); + return EmitPackLowF32ToF16Bits(state, ret); +} + +uint32_t EmitSqrtF16Bits(EmitterState* state, uint32_t bits) { + const auto cls = EmitClassifyF16Bits(state, bits); + const auto f32 = EmitF16BitsToF32(state, cls.bits); + const auto sqrt = state->builder.AllocateId(); + state->builder.AddFunction( + {OpExtInst, state->float_type, sqrt, state->glsl_std450, GlslSqrt, f32}); + const auto packed = EmitPackLowF32ToF16Bits(state, sqrt); + const auto numeric = EmitSelectValueU32(state, EmitNegativeNumericF16Bool(state, cls), + ConstantU32(state, 0xfe00u), packed); + return EmitSelectValueU32(state, cls.zero, cls.bits, numeric); +} + +uint32_t EmitRsqF16Bits(EmitterState* state, uint32_t bits) { + const auto cls = EmitClassifyF16Bits(state, bits); + const auto f32 = EmitF16BitsToF32(state, cls.bits); + const auto rsq = state->builder.AllocateId(); + state->builder.AddFunction( + {OpExtInst, state->float_type, rsq, state->glsl_std450, GlslInverseSqrt, f32}); + const auto packed = EmitPackLowF32ToF16Bits(state, rsq); + const auto sign = + EmitCompareU32Constant(state, OpINotEqual, EmitAndConstant(state, cls.bits, 0x8000u), 0); + const auto zero_result = + EmitSelectValueU32(state, sign, ConstantU32(state, 0xfc00u), ConstantU32(state, 0x7c00u)); + const auto numeric = EmitSelectValueU32(state, EmitNegativeNumericF16Bool(state, cls), + ConstantU32(state, 0xfe00u), packed); + return EmitSelectValueU32(state, cls.zero, zero_result, numeric); +} + +uint32_t EmitLog2F16Bits(EmitterState* state, uint32_t bits) { + const auto cls = EmitClassifyF16Bits(state, bits); + const auto f32 = EmitF16BitsToF32(state, cls.bits); + const auto log2 = state->builder.AllocateId(); + state->builder.AddFunction( + {OpExtInst, state->float_type, log2, state->glsl_std450, GlslLog2, f32}); + const auto packed = EmitPackLowF32ToF16Bits(state, log2); + return EmitSelectValueU32(state, EmitNegativeNumericF16Bool(state, cls), + ConstantU32(state, 0xfe00u), packed); +} + +uint32_t EmitExp2F16Bits(EmitterState* state, uint32_t bits) { + const auto f32 = EmitF16BitsToF32(state, bits); + const auto exp2 = state->builder.AllocateId(); + state->builder.AddFunction( + {OpExtInst, state->float_type, exp2, state->glsl_std450, GlslExp2, f32}); + return EmitPackLowF32ToF16Bits(state, exp2); +} + +uint32_t EmitF16MathExtInstBits(EmitterState* state, uint32_t bits, uint32_t ext_inst) { + const auto f32 = EmitF16BitsToF32(state, bits); + const auto result = state->builder.AllocateId(); + state->builder.AddFunction( + {OpExtInst, state->float_type, result, state->glsl_std450, ext_inst, f32}); + return EmitPackLowF32ToF16Bits(state, result); +} + +void EmitSpecialF16(EmitterState* state, const IR::Instruction& inst) { + const auto bits = EmitPackedF16LaneBits(state, inst.src[0], false); + uint32_t ret = ConstantU32(state, 0); + switch (inst.op) { + case IR::Opcode::RcpF16: ret = EmitRcpF16Bits(state, bits); break; + case IR::Opcode::SqrtF16: ret = EmitSqrtF16Bits(state, bits); break; + case IR::Opcode::InverseSqrtF16: ret = EmitRsqF16Bits(state, bits); break; + case IR::Opcode::Log2F16: ret = EmitLog2F16Bits(state, bits); break; + case IR::Opcode::Exp2F16: ret = EmitExp2F16Bits(state, bits); break; + case IR::Opcode::FloorF16: ret = EmitF16MathExtInstBits(state, bits, GlslFloor); break; + case IR::Opcode::CeilF16: ret = EmitF16MathExtInstBits(state, bits, GlslCeil); break; + case IR::Opcode::TruncF16: ret = EmitF16MathExtInstBits(state, bits, GlslTrunc); break; + case IR::Opcode::RoundEvenF16: + ret = EmitF16MathExtInstBits(state, bits, GlslRoundEven); + break; + default: break; + } + EmitStoreU32(state, inst.dst, ret); +} + +void EmitBinaryF16(EmitterState* state, const IR::Instruction& inst, uint32_t opcode) { + const auto lhs = EmitPackedF16Lane(state, inst.src[0], false); + const auto rhs = EmitPackedF16Lane(state, inst.src[1], false); + auto value = state->builder.AllocateId(); + state->builder.AddFunction({opcode, state->float_type, value, lhs, rhs}); + value = ApplyResultModifiersF32(state, value, inst.dst); + EmitStoreU32(state, inst.dst, EmitPackLowF32ToF16Bits(state, value)); +} + +void EmitFmaF16(EmitterState* state, const IR::Instruction& inst) { + const auto fma = + ApplyResultModifiersF32(state, EmitPackedF16FmaLane(state, inst, false), inst.dst); + EmitStoreU32(state, inst.dst, EmitPackLowF32ToF16Bits(state, fma)); +} + +uint32_t EmitMinMaxF16Bits(EmitterState* state, uint32_t lhs, uint32_t rhs, bool max_value) { + const auto lhs_class = EmitClassifyF16Bits(state, lhs); + const auto rhs_class = EmitClassifyF16Bits(state, rhs); + const auto lhs_f32 = EmitF16BitsToF32(state, lhs_class.bits); + const auto rhs_f32 = EmitF16BitsToF32(state, rhs_class.bits); + + const auto numeric_cond = state->builder.AllocateId(); + state->builder.AddFunction({max_value ? OpFOrdGreaterThanEqual : OpFOrdLessThan, + state->bool_type, numeric_cond, lhs_f32, rhs_f32}); + const auto ordered_bits = + EmitSelectValueU32(state, numeric_cond, lhs_class.bits, rhs_class.bits); + + const auto both_zero = EmitLogicalAndBool(state, lhs_class.zero, rhs_class.zero); + const auto zero_bits = max_value ? EmitAndU32(state, lhs_class.bits, rhs_class.bits) + : EmitOrU32(state, lhs_class.bits, rhs_class.bits); + const auto numeric_bits = EmitSelectValueU32(state, both_zero, zero_bits, ordered_bits); + + const auto rhs_nan_bits = + EmitSelectValueU32(state, rhs_class.nan, lhs_class.bits, numeric_bits); + const auto non_snan_bits = + EmitSelectValueU32(state, lhs_class.nan, rhs_class.bits, rhs_nan_bits); + const auto rhs_snan_bits = + EmitSelectValueU32(state, rhs_class.snan, rhs_class.quiet_bits, non_snan_bits); + const auto result_bits = + EmitSelectValueU32(state, lhs_class.snan, lhs_class.quiet_bits, rhs_snan_bits); + return EmitAndConstant(state, result_bits, 0xffffu); +} + +void EmitMinMaxF16(EmitterState* state, const IR::Instruction& inst, bool max_value) { + const auto bits = + EmitMinMaxF16Bits(state, EmitPackedF16LaneBits(state, inst.src[0], false), + EmitPackedF16LaneBits(state, inst.src[1], false), max_value); + EmitStoreU32(state, inst.dst, bits); +} + +uint32_t EmitMinMax3F16Bits(EmitterState* state, uint32_t src0, uint32_t src1, uint32_t src2, + bool max_value) { + return EmitMinMaxF16Bits(state, EmitMinMaxF16Bits(state, src0, src1, max_value), src2, + max_value); +} + +void EmitMinMax3F16(EmitterState* state, const IR::Instruction& inst, bool max_value) { + const auto bits = + EmitMinMax3F16Bits(state, EmitPackedF16LaneBits(state, inst.src[0], false), + EmitPackedF16LaneBits(state, inst.src[1], false), + EmitPackedF16LaneBits(state, inst.src[2], false), max_value); + EmitStoreU32(state, inst.dst, bits); +} + +uint32_t EmitF16BitsEqualBool(EmitterState* state, uint32_t lhs, uint32_t rhs) { + const auto lhs_f32 = EmitF16BitsToF32(state, lhs); + const auto rhs_f32 = EmitF16BitsToF32(state, rhs); + const auto ret = state->builder.AllocateId(); + state->builder.AddFunction({OpFOrdEqual, state->bool_type, ret, lhs_f32, rhs_f32}); + return ret; +} + +uint32_t EmitAnyNanF16Bool(EmitterState* state, uint32_t src0, uint32_t src1, uint32_t src2) { + const auto nan01 = EmitLogicalOrBool(state, EmitClassifyF16Bits(state, src0).nan, + EmitClassifyF16Bits(state, src1).nan); + return EmitLogicalOrBool(state, nan01, EmitClassifyF16Bits(state, src2).nan); +} + +void EmitMed3F16(EmitterState* state, const IR::Instruction& inst) { + const auto src0 = EmitPackedF16LaneBits(state, inst.src[0], false); + const auto src1 = EmitPackedF16LaneBits(state, inst.src[1], false); + const auto src2 = EmitPackedF16LaneBits(state, inst.src[2], false); + + const auto min3 = EmitMinMax3F16Bits(state, src0, src1, src2, false); + const auto max3 = EmitMinMax3F16Bits(state, src0, src1, src2, true); + const auto med_src0 = EmitMinMaxF16Bits(state, src1, src2, true); + const auto med_src1 = EmitMinMaxF16Bits(state, src0, src2, true); + const auto med_src2 = EmitMinMaxF16Bits(state, src0, src1, true); + const auto med01 = + EmitSelectValueU32(state, EmitF16BitsEqualBool(state, max3, src1), med_src1, med_src2); + const auto median = + EmitSelectValueU32(state, EmitF16BitsEqualBool(state, max3, src0), med_src0, med01); + const auto bits = + EmitSelectValueU32(state, EmitAnyNanF16Bool(state, src0, src1, src2), min3, median); + EmitStoreU32(state, inst.dst, bits); +} + +uint32_t EmitPackedMinMaxF16(EmitterState* state, const IR::Instruction& inst, bool max_value) { + const auto lo = EmitMinMaxF16Bits(state, EmitPackedF16LaneBits(state, inst.src[0], false), + EmitPackedF16LaneBits(state, inst.src[1], false), max_value); + const auto hi = EmitMinMaxF16Bits(state, EmitPackedF16LaneBits(state, inst.src[0], true), + EmitPackedF16LaneBits(state, inst.src[1], true), max_value); + return EmitOrU32(state, lo, EmitShiftLeftConstant(state, hi, 16)); +} + +uint32_t EmitPackedF16BinaryLane(EmitterState* state, const IR::Instruction& inst, bool high_lane, + uint32_t opcode) { + const auto lhs = EmitPackedF16Lane(state, inst.src[0], high_lane); + const auto rhs = EmitPackedF16Lane(state, inst.src[1], high_lane); + const auto ret = state->builder.AllocateId(); + state->builder.AddFunction({opcode, state->float_type, ret, lhs, rhs}); + return ret; +} + +uint32_t EmitPackedF16FmaLane(EmitterState* state, const IR::Instruction& inst, bool high_lane) { + const auto lhs = EmitPackedF16Lane(state, inst.src[0], high_lane); + const auto rhs = EmitPackedF16Lane(state, inst.src[1], high_lane); + const auto add = EmitPackedF16Lane(state, inst.src[2], high_lane); + const auto ret = state->builder.AllocateId(); + state->builder.AddFunction( + {OpExtInst, state->float_type, ret, state->glsl_std450, GlslFma, lhs, rhs, add}); + return ret; +} + +uint32_t EmitPackedF16BitsWithResultModifiers(EmitterState* state, uint32_t bits, + const IR::Operand& dst) { + if (!dst.clamp && dst.omod == 0u) { + return bits; + } + auto lo = ApplyResultModifiersF32( + state, EmitF16BitsToF32(state, EmitAndConstant(state, bits, 0xffffu)), dst); + auto hi = ApplyResultModifiersF32( + state, EmitF16BitsToF32(state, EmitShiftRightConstant(state, bits, 16)), dst); + const auto pair = state->builder.AllocateId(); + const auto ret = state->builder.AllocateId(); + state->builder.AddFunction({OpCompositeConstruct, state->vec2_float_type, pair, lo, hi}); + state->builder.AddFunction( + {OpExtInst, state->uint_type, ret, state->glsl_std450, GlslPackHalf2x16, pair}); + return ret; +} + +void EmitPackedF16(EmitterState* state, const IR::Instruction& inst) { + uint32_t lo = 0; + uint32_t hi = 0; + switch (inst.op) { + case IR::Opcode::PackedAddF16: + lo = EmitPackedF16BinaryLane(state, inst, false, OpFAdd); + hi = EmitPackedF16BinaryLane(state, inst, true, OpFAdd); + break; + case IR::Opcode::PackedMulF16: + lo = EmitPackedF16BinaryLane(state, inst, false, OpFMul); + hi = EmitPackedF16BinaryLane(state, inst, true, OpFMul); + break; + case IR::Opcode::PackedMinF16: + EmitStoreU32(state, inst.dst, + EmitPackedF16BitsWithResultModifiers( + state, EmitPackedMinMaxF16(state, inst, false), inst.dst)); + return; + case IR::Opcode::PackedMaxF16: + EmitStoreU32(state, inst.dst, + EmitPackedF16BitsWithResultModifiers( + state, EmitPackedMinMaxF16(state, inst, true), inst.dst)); + return; + case IR::Opcode::PackedFmaF16: + lo = EmitPackedF16FmaLane(state, inst, false); + hi = EmitPackedF16FmaLane(state, inst, true); + break; + default: + lo = ConstantF32(state, 0); + hi = ConstantF32(state, 0); + break; + } + + lo = ApplyResultModifiersF32(state, lo, inst.dst); + hi = ApplyResultModifiersF32(state, hi, inst.dst); + const auto pair = state->builder.AllocateId(); + const auto ret = state->builder.AllocateId(); + state->builder.AddFunction({OpCompositeConstruct, state->vec2_float_type, pair, lo, hi}); + state->builder.AddFunction( + {OpExtInst, state->uint_type, ret, state->glsl_std450, GlslPackHalf2x16, pair}); + EmitStoreU32(state, inst.dst, ret); +} + +uint32_t EmitPackedInteger16Select(EmitterState* state, uint32_t lhs, uint32_t rhs, + bool signed_compare, bool max_value) { + const auto cond = state->builder.AllocateId(); + const auto ret = state->builder.AllocateId(); + const auto op = signed_compare ? (max_value ? OpSGreaterThanEqual : OpSLessThan) + : (max_value ? OpUGreaterThanEqual : OpULessThan); + state->builder.AddFunction({op, state->bool_type, cond, lhs, rhs}); + state->builder.AddFunction({OpSelect, state->uint_type, ret, cond, lhs, rhs}); + return ret; +} + +uint32_t EmitPackedInteger16Lane(EmitterState* state, const IR::Instruction& inst, bool high_lane) { + const auto opcode = inst.op; + if (opcode == IR::Opcode::PackedLshlrevB16 || opcode == IR::Opcode::PackedLshrrevB16 || + opcode == IR::Opcode::PackedAshrrevI16) { + const auto count = + EmitAndConstant(state, EmitU16LaneBits(state, inst.src[0], high_lane), 0xfu); + const auto arithmetic = opcode == IR::Opcode::PackedAshrrevI16; + const auto value = EmitU16LaneBits(state, inst.src[1], high_lane, arithmetic); + const auto ret = state->builder.AllocateId(); + const auto shift_op = opcode == IR::Opcode::PackedLshlrevB16 ? OpShiftLeftLogical + : arithmetic ? OpShiftRightArithmetic + : OpShiftRightLogical; + state->builder.AddFunction({shift_op, state->uint_type, ret, value, count}); + return ret; + } + + const auto signed_compare = + opcode == IR::Opcode::PackedMaxI16 || opcode == IR::Opcode::PackedMinI16; + const auto lhs = EmitU16LaneBits(state, inst.src[0], high_lane, signed_compare); + const auto rhs = EmitU16LaneBits(state, inst.src[1], high_lane, signed_compare); + + switch (opcode) { + case IR::Opcode::PackedMadI16: + case IR::Opcode::PackedMadU16: { + const auto addend = + EmitU16LaneBits(state, inst.src[2], high_lane, opcode == IR::Opcode::PackedMadI16); + const auto mul = state->builder.AllocateId(); + const auto ret = state->builder.AllocateId(); + state->builder.AddFunction({OpIMul, state->uint_type, mul, lhs, rhs}); + state->builder.AddFunction({OpIAdd, state->uint_type, ret, mul, addend}); + return ret; + } + case IR::Opcode::PackedMulLoU16: { + const auto ret = state->builder.AllocateId(); + state->builder.AddFunction({OpIMul, state->uint_type, ret, lhs, rhs}); + return ret; + } + case IR::Opcode::PackedAddI16: + case IR::Opcode::PackedAddU16: { + const auto ret = state->builder.AllocateId(); + state->builder.AddFunction({OpIAdd, state->uint_type, ret, lhs, rhs}); + return ret; + } + case IR::Opcode::PackedSubI16: + case IR::Opcode::PackedSubU16: { + const auto ret = state->builder.AllocateId(); + state->builder.AddFunction({OpISub, state->uint_type, ret, lhs, rhs}); + return ret; + } + case IR::Opcode::PackedMaxI16: + case IR::Opcode::PackedMaxU16: + return EmitPackedInteger16Select(state, lhs, rhs, signed_compare, true); + case IR::Opcode::PackedMinI16: + case IR::Opcode::PackedMinU16: + return EmitPackedInteger16Select(state, lhs, rhs, signed_compare, false); + default: break; + } + + return ConstantU32(state, 0); +} + +void EmitPackedInteger16(EmitterState* state, const IR::Instruction& inst) { + const auto lo = EmitPackedInteger16Lane(state, inst, false); + const auto hi = EmitPackedInteger16Lane(state, inst, true); + const auto lo_bits = EmitAndConstant(state, lo, 0xffffu); + const auto hi_bits = EmitShiftLeftConstant(state, EmitAndConstant(state, hi, 0xffffu), 16); + EmitStoreU32(state, inst.dst, EmitOrU32(state, lo_bits, hi_bits)); +} + +void EmitMadMixF16(EmitterState* state, const IR::Instruction& inst) { + const auto src0 = EmitMixF32Load(state, inst.src[0]); + const auto src1 = EmitMixF32Load(state, inst.src[1]); + const auto src2 = EmitMixF32Load(state, inst.src[2]); + const auto fma = state->builder.AllocateId(); + const auto pair = state->builder.AllocateId(); + const auto packed = state->builder.AllocateId(); + state->builder.AddFunction( + {OpExtInst, state->float_type, fma, state->glsl_std450, GlslFma, src0, src1, src2}); + const auto result = ApplyResultModifiersF32(state, fma, inst.dst); + state->builder.AddFunction( + {OpCompositeConstruct, state->vec2_float_type, pair, result, ConstantF32(state, 0)}); + state->builder.AddFunction( + {OpExtInst, state->uint_type, packed, state->glsl_std450, GlslPackHalf2x16, pair}); + EmitStoreU32(state, inst.dst, packed); +} + +void EmitMinMax3F32(EmitterState* state, const IR::Instruction& inst, bool max_value) { + const auto src0 = EmitFloatLoad(state, inst.src[0]); + const auto src1 = EmitFloatLoad(state, inst.src[1]); + const auto src2 = EmitFloatLoad(state, inst.src[2]); + const auto tmp = EmitMinMaxF32Value(state, src0, src1, max_value); + const auto base = EmitMinMaxF32Value(state, tmp, src2, max_value); + const auto f32 = ApplyResultModifiersF32(state, base, inst.dst); + const auto u32 = state->builder.AllocateId(); + state->builder.AddFunction({OpBitcast, state->uint_type, u32, f32}); + EmitStoreU32(state, inst.dst, u32); +} + +void EmitMed3F32(EmitterState* state, const IR::Instruction& inst) { + const auto src0 = EmitFloatLoad(state, inst.src[0]); + const auto src1 = EmitFloatLoad(state, inst.src[1]); + const auto src2 = EmitFloatLoad(state, inst.src[2]); + const auto low_pair = EmitMinMaxF32Value(state, src0, src1, false); + const auto min3 = EmitMinMaxF32Value(state, low_pair, src2, false); + const auto high_pair = EmitMinMaxF32Value(state, src0, src1, true); + const auto high_min = EmitMinMaxF32Value(state, high_pair, src2, false); + const auto median = EmitMinMaxF32Value(state, low_pair, high_min, true); + const auto nan01 = EmitLogicalOrBool(state, EmitClassifyF32(state, src0).nan, + EmitClassifyF32(state, src1).nan); + const auto any_nan = EmitLogicalOrBool(state, nan01, EmitClassifyF32(state, src2).nan); + const auto base = EmitFSelectValue(state, any_nan, min3, median); + const auto f32 = ApplyResultModifiersF32(state, base, inst.dst); + const auto u32 = state->builder.AllocateId(); + state->builder.AddFunction({OpBitcast, state->uint_type, u32, f32}); + EmitStoreU32(state, inst.dst, u32); +} + +} // namespace Libs::Graphics::ShaderRecompiler::Spirv::Emitter diff --git a/src/graphics/shader/recompiler/spirvEmitter/spirvEmitterAnalysis.cpp b/src/graphics/shader/recompiler/spirvEmitter/spirvEmitterAnalysis.cpp new file mode 100644 index 000000000..419fe72fc --- /dev/null +++ b/src/graphics/shader/recompiler/spirvEmitter/spirvEmitterAnalysis.cpp @@ -0,0 +1,752 @@ +#include "common/assert.h" +#include "graphics/guest_gpu/gpu_defs.h" +#include "graphics/shader/recompiler/SpirvEmitter.h" +#include "graphics/shader/recompiler/spirvEmitter/spirvEmitterInternal.h" + +namespace Libs::Graphics::ShaderRecompiler::Spirv::Emitter { + +uint32_t PixelParameterMappedLocation(const EmitterState& state, uint32_t attr) { + const auto* ps = state.pixel_input_info; + if (state.stage != ShaderType::Pixel || ps == nullptr || attr >= ps->input_num) { + return attr; + } + // VINTRP ATTR selects the PS input slot. SPI_PS_INPUT_CNTL maps that slot to a + // VS parameter export, which is the SPIR-V location we must link against. + return ps->interpolator_settings[attr] & PsInputOffsetMask; +} + +uint32_t PixelParameterLocation(const EmitterState& state, uint32_t attr) { + bool used_locations[32] = {}; + + for (const auto& input: state.inputs) { + if (input.kind != IR::StageInputKind::Parameter) { + continue; + } + + auto location = PixelParameterMappedLocation(state, input.location); + if (location < std::size(used_locations) && used_locations[location]) { + auto fallback_location = input.location; + while (fallback_location < std::size(used_locations) && + used_locations[fallback_location]) { + fallback_location++; + } + EXIT_NOT_IMPLEMENTED(fallback_location >= std::size(used_locations)); + location = fallback_location; + } + + if (input.location == attr) { + return location; + } + + if (location < std::size(used_locations)) { + used_locations[location] = true; + } + } + + return PixelParameterMappedLocation(state, attr); +} + +bool PixelParameterIsFlat(const EmitterState& state, uint32_t attr) { + const auto* ps = state.pixel_input_info; + if (state.stage != ShaderType::Pixel || ps == nullptr || attr >= ps->input_num) { + return false; + } + return (ps->interpolator_settings[attr] & PsInputFlatShade) != 0; +} + +void SetError(std::string* error, const char* message) { + if (error != nullptr) { + *error = message; + } +} + +void CollectRegister(std::vector* registers, IR::Register reg) { + if (std::any_of(registers->begin(), registers->end(), + [reg](const RegisterBinding& binding) { return binding.reg == reg; })) { + return; + } + registers->push_back({reg, 0}); +} + +IR::Operand MakeRegisterOperand(IR::RegisterFile file, uint32_t index) { + IR::Operand operand; + operand.kind = IR::OperandKind::Register; + operand.reg = {file, index}; + return operand; +} + +IR::Register SccRegister() { + return {IR::RegisterFile::Scc, static_cast(Decoder::OperandKind::Scc)}; +} + +IR::Operand SccOperand() { + return MakeRegisterOperand(IR::RegisterFile::Scc, + static_cast(Decoder::OperandKind::Scc)); +} + +bool IsInactiveWave32ExecHigh(const EmitterState& state, IR::Register reg) { + return state.wave_size == 32u && reg.file == IR::RegisterFile::Exec && reg.index == 1u; +} + +bool IsMaskRegisterFile(IR::RegisterFile file) { + return file == IR::RegisterFile::Exec || file == IR::RegisterFile::Vcc; +} + +bool IsSccOperand(const IR::Operand& operand) { + return operand.kind == IR::OperandKind::Register && operand.reg.file == IR::RegisterFile::Scc; +} + +bool IsCompareOpcode(IR::Opcode op); + +void CollectMaskStateRegisters(std::vector* registers) { + CollectRegister(registers, {IR::RegisterFile::Exec, 0}); + CollectRegister(registers, {IR::RegisterFile::Exec, 1}); + CollectRegister(registers, {IR::RegisterFile::Vcc, 0}); + CollectRegister(registers, {IR::RegisterFile::Vcc, 1}); + CollectRegister(registers, SccRegister()); +} + +void CollectSequentialRegisters(std::vector* registers, const IR::Operand& base, + uint32_t count) { + if (base.kind != IR::OperandKind::Register) { + return; + } + for (uint32_t i = 0; i < count; i++) { + auto reg = base.reg; + reg.index += i; + CollectRegister(registers, reg); + } +} + +uint32_t MaxCollectedVectorRegisterEnd(const std::vector& registers) { + uint32_t max_end = 0; + for (const auto& binding: registers) { + if (binding.reg.file == IR::RegisterFile::Vector) { + max_end = std::max(max_end, binding.reg.index + 1u); + } + } + return std::min(max_end, 256u); +} + +void CollectMoveRelSourceRegisters(const IR::Program& program, + std::vector* registers) { + const auto max_vector_end = MaxCollectedVectorRegisterEnd(*registers); + for (const auto& block: program.blocks) { + for (const auto& inst: block.instructions) { + if (inst.op != IR::Opcode::MoveRelSourceU32 || + inst.src[0].kind != IR::OperandKind::Register || + inst.src[0].reg.file != IR::RegisterFile::Vector) { + continue; + } + const auto base_index = inst.src[0].reg.index; + const auto limit = std::min(std::max(max_vector_end, base_index + 1u), 256u); + CollectSequentialRegisters(registers, inst.src[0], limit - base_index); + } + } +} + +bool IsPairDwordOpcode(IR::Opcode op) { + switch (op) { + case IR::Opcode::MoveU64: + case IR::Opcode::WqmB64: + case IR::Opcode::SaveexecB64: + case IR::Opcode::BitwiseAndU64: + case IR::Opcode::BitwiseAndNotU64: + case IR::Opcode::BitwiseOrU64: + case IR::Opcode::BitwiseOrNotU64: + case IR::Opcode::BitwiseXorU64: + case IR::Opcode::BitwiseNandU64: + case IR::Opcode::BitwiseNorU64: + case IR::Opcode::BitwiseXnorU64: + case IR::Opcode::BitwiseNotU64: + case IR::Opcode::BitFieldMaskU64: + case IR::Opcode::BitFieldExtractU64: + case IR::Opcode::BitReplicateB64B32: + case IR::Opcode::ShiftLeftLogicalU64: + case IR::Opcode::ShiftRightLogicalU64: + case IR::Opcode::SelectU64: return true; + default: return false; + } +} + +uint32_t PairDwordSourceCount(IR::Opcode op, uint32_t src_count) { + switch (op) { + case IR::Opcode::BitFieldMaskU64: return 0; + case IR::Opcode::BitReplicateB64B32: return 0; + case IR::Opcode::BitFieldExtractU64: return 1; + case IR::Opcode::ShiftLeftLogicalU64: + case IR::Opcode::ShiftRightLogicalU64: + case IR::Opcode::BitwiseNotU64: + case IR::Opcode::SaveexecB64: + case IR::Opcode::WqmB64: + case IR::Opcode::MoveU64: return std::min(src_count, 1u); + default: return src_count; + } +} + +void CollectRegisters(const IR::Program& program, std::vector* registers) { + CollectMaskStateRegisters(registers); + for (const auto& block: program.blocks) { + for (const auto& inst: block.instructions) { + if (inst.dst.kind == IR::OperandKind::Register) { + CollectRegister(registers, inst.dst.reg); + } + if (inst.dst2.kind == IR::OperandKind::Register) { + CollectRegister(registers, inst.dst2.reg); + } + for (uint32_t i = 0; i < inst.src_count; i++) { + if (inst.src[i].kind == IR::OperandKind::Register) { + CollectRegister(registers, inst.src[i].reg); + } + } + if (IsPairDwordOpcode(inst.op)) { + CollectSequentialRegisters(registers, inst.dst, 2); + const uint32_t first_pair_src = inst.op == IR::Opcode::SelectU64 ? 1u : 0u; + const uint32_t pair_src_count = PairDwordSourceCount(inst.op, inst.src_count); + for (uint32_t i = first_pair_src; i < pair_src_count; i++) { + CollectSequentialRegisters(registers, inst.src[i], 2); + } + } + if (inst.op == IR::Opcode::BitCountU64 || inst.op == IR::Opcode::FindMsbFromHighU64) { + CollectSequentialRegisters(registers, inst.src[0], 2); + } + if (inst.op == IR::Opcode::CompareNeU64) { + CollectSequentialRegisters(registers, inst.src[0], 2); + CollectSequentialRegisters(registers, inst.src[1], 2); + } + if (IsCompareOpcode(inst.op) && inst.dst.kind == IR::OperandKind::Register && + inst.dst.reg.file != IR::RegisterFile::Scc) { + CollectSequentialRegisters(registers, inst.dst, 2); + } + if (inst.op == IR::Opcode::IAddCarryU32 || inst.op == IR::Opcode::ISubBorrowU32) { + CollectSequentialRegisters(registers, inst.dst2, 2); + } + if (inst.op == IR::Opcode::UMadU64U32) { + CollectSequentialRegisters(registers, inst.dst, 2); + CollectSequentialRegisters(registers, inst.dst2, 2); + CollectSequentialRegisters(registers, inst.src[2], 2); + } + if (inst.op == IR::Opcode::SBufferLoadDword && + inst.memory.kind == IR::ResourceKind::ScalarBuffer) { + CollectRegister(registers, {IR::RegisterFile::Scalar, inst.memory.resource * 4u}); + } + if ((inst.op == IR::Opcode::ImageSample || inst.op == IR::Opcode::ImageGather4) && + inst.memory.image_address_components > 1u) { + CollectSequentialRegisters(registers, inst.dst, inst.memory.data_dwords); + CollectSequentialRegisters(registers, inst.src[0], + inst.memory.image_address_components); + } + if (inst.op == IR::Opcode::ImageGetLod) { + CollectSequentialRegisters(registers, inst.dst, inst.memory.data_dwords); + CollectSequentialRegisters(registers, inst.src[0], + inst.memory.image_address_components); + } + if (inst.op == IR::Opcode::ImageLoad) { + CollectSequentialRegisters(registers, inst.dst, inst.memory.data_dwords); + CollectSequentialRegisters(registers, inst.src[0], + inst.memory.image_address_components); + } + if (inst.op == IR::Opcode::ImageStore) { + CollectSequentialRegisters(registers, inst.src[0], inst.memory.data_dwords); + CollectSequentialRegisters(registers, inst.src[1], + inst.memory.image_address_components); + } + if (inst.memory.kind == IR::ResourceKind::StorageImageUint) { + CollectSequentialRegisters(registers, inst.src[1], + inst.memory.image_address_components); + } + } + } + CollectMoveRelSourceRegisters(program, registers); +} + +bool HasOutput(const std::vector& outputs, IR::StageOutputKind kind, + uint32_t index) { + return std::any_of(outputs.begin(), outputs.end(), [kind, index](const OutputBinding& binding) { + return binding.kind == kind && binding.index == index; + }); +} + +void CopyProgramInputsAndOutputs(EmitterState* state, const IR::Program& program) { + for (const auto& input: program.info.inputs) { + state->inputs.push_back( + {input.kind, input.location, input.component_count, 0, input.debug_name}); + } + for (const auto& output: program.info.outputs) { + if (HasOutput(state->outputs, output.kind, output.index)) { + continue; + } + state->outputs.push_back( + {output.kind, output.index, output.location, 0, output.debug_name}); + } +} + +uint32_t OutputVariableForExport(const EmitterState& state, const IR::ExportInfo& exp) { + if (exp.kind == IR::ExportTargetKind::Position) { + return state.per_vertex_variable; + } + if (exp.kind == IR::ExportTargetKind::MrtZ) { + return state.depth_variable; + } + for (const auto& binding: state.outputs) { + const auto expected_kind = exp.kind == IR::ExportTargetKind::Mrt + ? IR::StageOutputKind::Mrt + : IR::StageOutputKind::Parameter; + if (binding.kind == expected_kind && binding.index == exp.index) { + return binding.variable_id; + } + } + return 0; +} + +bool ProgramNeedsComputeDerivatives(const IR::Program& program) { + for (const auto& block: program.blocks) { + for (const auto& inst: block.instructions) { + if (inst.op == IR::Opcode::ImageGetLod) { + return true; + } + } + } + return false; +} + +bool ProgramNeedsImageGatherExtended(const IR::Program& program) { + for (const auto& block: program.blocks) { + for (const auto& inst: block.instructions) { + if (inst.op == IR::Opcode::ImageGather4) { + return true; + } + } + } + return false; +} + +bool IsLdsOpcode(IR::Opcode op) { + switch (op) { + case IR::Opcode::DsReadUbyte: + case IR::Opcode::DsReadSbyte: + case IR::Opcode::DsReadUshort: + case IR::Opcode::DsReadSshort: + case IR::Opcode::DsReadB32: + case IR::Opcode::DsWriteByte: + case IR::Opcode::DsWriteShort: + case IR::Opcode::DsWriteB32: + case IR::Opcode::DsMinF32: + case IR::Opcode::DsMaxF32: + case IR::Opcode::DsSwizzleB32: + case IR::Opcode::DsWriteAddtidB32: + case IR::Opcode::DsReadAddtidB32: return true; + default: return false; + } +} + +bool ProgramNeedsFunctionLds(const IR::Program& program) { + if (program.stage == ShaderType::Compute) { + return false; + } + + for (const auto& block: program.blocks) { + for (const auto& inst: block.instructions) { + if (IsLdsOpcode(inst.op)) { + return true; + } + } + } + return false; +} + +bool ProgramNeedsPixelValidMask(const IR::Program& program) { + if (program.stage != ShaderType::Pixel) { + return false; + } + + for (const auto& block: program.blocks) { + for (const auto& inst: block.instructions) { + if (inst.op == IR::Opcode::Export && inst.export_info.vm) { + return true; + } + } + } + return false; +} + +bool InstructionHasDppSource(const IR::Instruction& inst) { + for (uint32_t i = 0; i < inst.src_count && i < 3u; i++) { + if (inst.src[i].dpp) { + return true; + } + } + return false; +} + +bool ProgramNeedsSubgroupBallot(const IR::Program& program) { + if (program.lane_mask_mode == ShaderLaneMaskMode::PerInvocation) { + for (const auto& block: program.blocks) { + for (const auto& inst: block.instructions) { + if (inst.op == IR::Opcode::WqmB64) { + return true; + } + } + } + } + return ProgramRequiresExactSubgroupSize(program); +} + +bool ProgramNeedsSubgroupShuffle(const IR::Program& program) { + for (const auto& block: program.blocks) { + for (const auto& inst: block.instructions) { + if (inst.op == IR::Opcode::ReadFirstLaneU32 || inst.op == IR::Opcode::ReadLaneU32 || + inst.op == IR::Opcode::Permlane16B32 || inst.op == IR::Opcode::Permlanex16B32 || + inst.op == IR::Opcode::DsSwizzleB32 || inst.op == IR::Opcode::DsConsume || + inst.op == IR::Opcode::DsAppend || InstructionHasDppSource(inst)) { + return true; + } + } + } + return false; +} + +bool IsCompareOpcode(IR::Opcode op) { + switch (op) { + case IR::Opcode::CompareFalse: + case IR::Opcode::CompareTrue: + case IR::Opcode::CompareEqU32: + case IR::Opcode::CompareNeU32: + case IR::Opcode::CompareGtU32: + case IR::Opcode::CompareGeU32: + case IR::Opcode::CompareLtU32: + case IR::Opcode::CompareLeU32: + case IR::Opcode::CompareNeU64: + case IR::Opcode::CompareMaskEqU32: + case IR::Opcode::CompareMaskNeU32: + case IR::Opcode::CompareMaskGtU32: + case IR::Opcode::CompareMaskGeU32: + case IR::Opcode::CompareMaskLtU32: + case IR::Opcode::CompareMaskLeU32: + case IR::Opcode::CompareEqI32: + case IR::Opcode::CompareNeI32: + case IR::Opcode::CompareGtI32: + case IR::Opcode::CompareGeI32: + case IR::Opcode::CompareLtI32: + case IR::Opcode::CompareLeI32: + case IR::Opcode::CompareEqI16: + case IR::Opcode::CompareNeI16: + case IR::Opcode::CompareGtI16: + case IR::Opcode::CompareGeI16: + case IR::Opcode::CompareLtI16: + case IR::Opcode::CompareLeI16: + case IR::Opcode::CompareMaskEqI32: + case IR::Opcode::CompareMaskNeI32: + case IR::Opcode::CompareMaskGtI32: + case IR::Opcode::CompareMaskGeI32: + case IR::Opcode::CompareMaskLtI32: + case IR::Opcode::CompareMaskLeI32: + case IR::Opcode::CompareEqU16: + case IR::Opcode::CompareNeU16: + case IR::Opcode::CompareGtU16: + case IR::Opcode::CompareGeU16: + case IR::Opcode::CompareLtU16: + case IR::Opcode::CompareLeU16: + case IR::Opcode::CompareEqF32: + case IR::Opcode::CompareNeF32: + case IR::Opcode::CompareGtF32: + case IR::Opcode::CompareGeF32: + case IR::Opcode::CompareLtF32: + case IR::Opcode::CompareLeF32: + case IR::Opcode::CompareOrderedF32: + case IR::Opcode::CompareUnorderedF32: + case IR::Opcode::CompareUnordEqF32: + case IR::Opcode::CompareUnordNeF32: + case IR::Opcode::CompareUnordGtF32: + case IR::Opcode::CompareUnordGeF32: + case IR::Opcode::CompareUnordLtF32: + case IR::Opcode::CompareUnordLeF32: + case IR::Opcode::CompareClassF32: + case IR::Opcode::CompareEqF16: + case IR::Opcode::CompareNeF16: + case IR::Opcode::CompareGtF16: + case IR::Opcode::CompareGeF16: + case IR::Opcode::CompareLtF16: + case IR::Opcode::CompareLeF16: + case IR::Opcode::CompareUnordNeF16: + case IR::Opcode::CompareMaskEqF16: + case IR::Opcode::CompareMaskNeF16: + case IR::Opcode::CompareMaskGtF16: + case IR::Opcode::CompareMaskGeF16: + case IR::Opcode::CompareMaskLtF16: + case IR::Opcode::CompareMaskLeF16: + case IR::Opcode::CompareMaskUnordNeF16: + case IR::Opcode::CompareMaskUnordGeF16: + case IR::Opcode::CompareMaskEqF32: + case IR::Opcode::CompareMaskNeF32: + case IR::Opcode::CompareMaskGtF32: + case IR::Opcode::CompareMaskGeF32: + case IR::Opcode::CompareMaskLtF32: + case IR::Opcode::CompareMaskLeF32: + case IR::Opcode::CompareMaskUnordEqF32: + case IR::Opcode::CompareMaskUnordNeF32: + case IR::Opcode::CompareMaskUnordGtF32: + case IR::Opcode::CompareMaskUnordGeF32: + case IR::Opcode::CompareMaskUnordLtF32: + case IR::Opcode::CompareMaskUnordLeF32: return true; + default: return false; + } +} + +bool ProgramNeedsSubgroupLocalInvocationId(const IR::Program& program) { + for (const auto& block: program.blocks) { + for (const auto& inst: block.instructions) { + if (inst.op == IR::Opcode::WqmB64 || inst.op == IR::Opcode::DsSwizzleB32 || + inst.op == IR::Opcode::DsConsume || inst.op == IR::Opcode::DsAppend || + inst.op == IR::Opcode::WriteLaneU32 || inst.op == IR::Opcode::Permlane16B32 || + inst.op == IR::Opcode::Permlanex16B32 || inst.op == IR::Opcode::Export || + InstructionHasDppSource(inst)) { + return true; + } + if (IsCompareOpcode(inst.op)) { + return true; + } + if (inst.dst.kind == IR::OperandKind::Register && + inst.dst.reg.file == IR::RegisterFile::Vector) { + return true; + } + if (inst.dst2.kind == IR::OperandKind::Register && + inst.dst2.reg.file == IR::RegisterFile::Vector) { + return true; + } + } + } + return false; +} + +uint32_t PointerForRegister(const EmitterState& state, IR::Register reg) { + for (const auto& binding: state.registers) { + if (binding.reg == reg) { + return binding.pointer_id; + } + } + return 0; +} + +uint32_t ConstantU32(EmitterState* state, uint32_t value); +void EmitStoreU32(EmitterState* state, const IR::Operand& dst, uint32_t value); +uint32_t EmitSubgroupLocalInvocationId(EmitterState* state); +uint32_t EmitLaneIndexActiveBool(EmitterState* state, uint32_t lane); +[[noreturn]] void ExitDescriptorBindingFailure(const EmitterState& state, + IR::DescriptorBindingKind kind, uint32_t resource, + const char* reason) { + EXIT("shader binding resolution failed during SPIR-V emit: hash=0x%016" PRIx64 + " stage=%u resource=%" PRIu32 " binding_kind=%u reason=%s\n", + state.program->shader_hash, static_cast(state.stage), resource, + static_cast(kind), reason); + std::abort(); +} + +DescriptorResourceBinding ResourceForDescriptor(const EmitterState& state, + IR::DescriptorBindingKind kind, uint32_t resource) { + const auto* descriptor = IR::FindBinding(state.program->bindings, kind); + if (descriptor == nullptr) { + ExitDescriptorBindingFailure(state, kind, resource, "descriptor group was not allocated"); + } + const auto found = + std::find(descriptor->resources.begin(), descriptor->resources.end(), resource); + if (found == descriptor->resources.end()) { + ExitDescriptorBindingFailure(state, kind, resource, + "resource is absent from descriptor group"); + } + return {descriptor, static_cast(found - descriptor->resources.begin())}; +} + +uint32_t DescriptorElementPointer(EmitterState* state, uint32_t result_ptr_type, + uint32_t variable_id, uint32_t array_index, + IR::DescriptorBindingKind kind, uint32_t resource, + const char* variable_name) { + if (variable_id == 0) { + ExitDescriptorBindingFailure(*state, kind, resource, variable_name); + } + const auto pointer = state->builder.AllocateId(); + state->builder.AddFunction( + {OpAccessChain, result_ptr_type, pointer, variable_id, ConstantU32(state, array_index)}); + return pointer; +} + +ImageViewKind ImageViewKindFromDimension(Decoder::ImageDimension dimension) { + switch (dimension) { + case Decoder::ImageDimension::Dim2DArray: return ImageViewKind::Dim2DArray; + case Decoder::ImageDimension::Dim3D: return ImageViewKind::Dim3D; + default: return ImageViewKind::Dim2D; + } +} + +ImageViewKind SampledImageViewKind(const EmitterState& state, const IR::MemoryInfo& mem, + uint32_t use_pc) { + (void)state; + (void)use_pc; + return ImageViewKindFromDimension(mem.image_dimension); +} + +ImageViewKind StorageImageViewKind(const EmitterState& state, const IR::MemoryInfo& mem, + bool uint_image, uint32_t use_pc) { + (void)state; + (void)uint_image; + (void)use_pc; + return ImageViewKindFromDimension(mem.image_dimension); +} + +uint32_t ImageViewCoordinateComponents(ImageViewKind view) { + switch (view) { + case ImageViewKind::Dim2DArray: + case ImageViewKind::Dim3D: return 3u; + default: return 2u; + } +} + +uint32_t ImageViewImageType(const EmitterState& state, ImageViewKind view, bool integer) { + return state.sampled_images[SampledImageIndex(integer, view)].image_type; +} + +uint32_t ImageViewSampledImageType(const EmitterState& state, ImageViewKind view, bool integer) { + return state.sampled_images[SampledImageIndex(integer, view)].sampled_image_type; +} + +uint32_t ImageViewSizeType(const EmitterState& state, ImageViewKind view) { + return ImageViewCoordinateComponents(view) == 3u ? state.vec3_uint_type : state.vec2_uint_type; +} + +uint32_t StorageImageType(const EmitterState& state, bool uint_image, ImageViewKind view) { + if (uint_image) { + switch (view) { + case ImageViewKind::Dim2DArray: return state.storage_image_uint_2d_array_type; + case ImageViewKind::Dim3D: return state.storage_image_uint_3d_type; + default: return state.storage_image_uint_type; + } + } + switch (view) { + case ImageViewKind::Dim2DArray: return state.storage_image_2d_array_type; + case ImageViewKind::Dim3D: return state.storage_image_3d_type; + default: return state.storage_image_type; + } +} + +uint32_t StorageImagePointerType(const EmitterState& state, bool uint_image, ImageViewKind view) { + if (uint_image) { + switch (view) { + case ImageViewKind::Dim2DArray: return state.ptr_uniform_storage_image_uint_2d_array; + case ImageViewKind::Dim3D: return state.ptr_uniform_storage_image_uint_3d; + default: return state.ptr_uniform_storage_image_uint; + } + } + switch (view) { + case ImageViewKind::Dim2DArray: return state.ptr_uniform_storage_image_2d_array; + case ImageViewKind::Dim3D: return state.ptr_uniform_storage_image_3d; + default: return state.ptr_uniform_storage_image; + } +} + +uint32_t StorageImageVariable(const EmitterState& state, bool uint_image, ImageViewKind view) { + if (uint_image) { + switch (view) { + case ImageViewKind::Dim2DArray: return state.storage_image_uint_2d_array_variable; + case ImageViewKind::Dim3D: return state.storage_image_uint_3d_variable; + default: return state.storage_image_uint_variable; + } + } + switch (view) { + case ImageViewKind::Dim2DArray: return state.storage_image_2d_array_variable; + case ImageViewKind::Dim3D: return state.storage_image_3d_variable; + default: return state.storage_image_variable; + } +} + +IR::DescriptorBindingKind StorageBindingKind(bool uint_image, ImageViewKind view) { + if (uint_image) { + switch (view) { + case ImageViewKind::Dim2DArray: return IR::DescriptorBindingKind::StorageUint2DArray; + case ImageViewKind::Dim3D: return IR::DescriptorBindingKind::StorageUint3D; + default: return IR::DescriptorBindingKind::StorageUint2D; + } + } + switch (view) { + case ImageViewKind::Dim2DArray: return IR::DescriptorBindingKind::Storage2DArray; + case ImageViewKind::Dim3D: return IR::DescriptorBindingKind::Storage3D; + default: return IR::DescriptorBindingKind::Storage2D; + } +} + +uint32_t LoadSampledImageDescriptor(EmitterState* state, const IR::MemoryInfo& mem, uint32_t use_pc, + ImageViewKind view) { + (void)use_pc; + const bool integer = mem.kind == IR::ResourceKind::ImageUint; + const auto kind = SampledBindingKind(integer, view); + const auto binding = ResourceForDescriptor(*state, kind, mem.resource); + const auto& descriptors = state->sampled_images[SampledImageIndex(integer, view)]; + const auto pointer = DescriptorElementPointer( + state, descriptors.pointer_type, descriptors.variable, binding.array_index, kind, + mem.resource, "sampled image descriptor array was not emitted"); + const auto image = state->builder.AllocateId(); + state->builder.AddFunction({OpLoad, ImageViewImageType(*state, view, integer), image, pointer}); + return image; +} + +uint32_t LoadSamplerDescriptor(EmitterState* state, uint32_t sampler, uint32_t use_pc) { + (void)use_pc; + const auto binding = + ResourceForDescriptor(*state, IR::DescriptorBindingKind::Samplers, sampler); + const auto pointer = DescriptorElementPointer( + state, state->ptr_uniform_sampler, state->sampler_variable, binding.array_index, + IR::DescriptorBindingKind::Samplers, sampler, "sampler descriptor array was not emitted"); + const auto sampler_id = state->builder.AllocateId(); + state->builder.AddFunction({OpLoad, state->sampler_type, sampler_id, pointer}); + return sampler_id; +} + +uint32_t MakeSampledImage(EmitterState* state, const IR::MemoryInfo& mem, uint32_t use_pc, + ImageViewKind view) { + const auto image = LoadSampledImageDescriptor(state, mem, use_pc, view); + const auto sampler = LoadSamplerDescriptor(state, mem.sampler, use_pc); + if (image == 0 || sampler == 0) { + ExitDescriptorBindingFailure( + *state, SampledBindingKind(mem.kind == IR::ResourceKind::ImageUint, view), mem.resource, + "sampled image or sampler descriptor load failed"); + } + const auto sampled_image = state->builder.AllocateId(); + state->builder.AddFunction( + {OpSampledImage, + ImageViewSampledImageType(*state, view, mem.kind == IR::ResourceKind::ImageUint), + sampled_image, image, sampler}); + return sampled_image; +} + +uint32_t StorageImageDescriptorPointer(EmitterState* state, uint32_t resource, bool uint_image, + uint32_t use_pc, ImageViewKind view) { + (void)use_pc; + const auto kind = StorageBindingKind(uint_image, view); + const auto binding = ResourceForDescriptor(*state, kind, resource); + const auto ptr_type = StorageImagePointerType(*state, uint_image, view); + const auto variable = StorageImageVariable(*state, uint_image, view); + return DescriptorElementPointer(state, ptr_type, variable, binding.array_index, kind, resource, + "storage image descriptor array was not emitted"); +} + +uint32_t LoadStorageImageDescriptor(EmitterState* state, uint32_t resource, bool uint_image, + uint32_t use_pc, ImageViewKind view) { + const auto pointer = StorageImageDescriptorPointer(state, resource, uint_image, use_pc, view); + if (pointer == 0) { + ExitDescriptorBindingFailure(*state, StorageBindingKind(uint_image, view), resource, + "storage image descriptor pointer creation failed"); + } + const auto type = StorageImageType(*state, uint_image, view); + const auto image = state->builder.AllocateId(); + state->builder.AddFunction({OpLoad, type, image, pointer}); + return image; +} + +uint32_t ExecutionModelForStage(ShaderType stage) { + switch (stage) { + case ShaderType::Vertex: return ExecutionModelVertex; + case ShaderType::Pixel: return ExecutionModelFragment; + default: return ExecutionModelGLCompute; + } +} + +} // namespace Libs::Graphics::ShaderRecompiler::Spirv::Emitter diff --git a/src/graphics/shader/recompiler/spirvEmitter/spirvEmitterControlFlow.cpp b/src/graphics/shader/recompiler/spirvEmitter/spirvEmitterControlFlow.cpp new file mode 100644 index 000000000..86e87a512 --- /dev/null +++ b/src/graphics/shader/recompiler/spirvEmitter/spirvEmitterControlFlow.cpp @@ -0,0 +1,1095 @@ +#include "graphics/shader/recompiler/spirvEmitter/spirvEmitterInternal.h" + +namespace Libs::Graphics::ShaderRecompiler::Spirv::Emitter { + +void ComputeReachableBlocks(EmitterState* state, const IR::Program& program) { + state->reachable_blocks.assign(program.blocks.size(), false); + if (program.blocks.empty()) { + return; + } + + std::vector stack; + stack.push_back(0); + while (!stack.empty()) { + const auto block_id = stack.back(); + stack.pop_back(); + if (block_id >= program.blocks.size() || state->reachable_blocks[block_id]) { + continue; + } + state->reachable_blocks[block_id] = true; + for (auto succ: program.blocks[block_id].successors) { + if (succ < program.blocks.size() && !state->reachable_blocks[succ]) { + stack.push_back(succ); + } + } + } +} + +void AllocateBlockLabels(EmitterState* state, const IR::Program& program) { + for (const auto& block: program.blocks) { + if (block.id < state->reachable_blocks.size() && state->reachable_blocks[block.id]) { + state->block_labels.emplace(block.id, state->builder.AllocateId()); + } + } +} + +void AllocateDispatcherState(EmitterState* state, const IR::Program& program) { + state->dispatcher_fallback = program.dispatcher_fallback; + if (!state->dispatcher_fallback) { + return; + } + state->dispatch_pc_variable = state->builder.AllocateId(); + state->dispatch_header_label = state->builder.AllocateId(); + state->dispatch_select_label = state->builder.AllocateId(); + state->dispatch_default_label = state->builder.AllocateId(); + state->dispatch_after_switch_label = state->builder.AllocateId(); + state->dispatch_continue_label = state->builder.AllocateId(); + state->dispatch_merge_label = state->builder.AllocateId(); +} + +uint32_t BlockLabel(const EmitterState& state, uint32_t block_id) { + auto it = state.block_labels.find(block_id); + return it == state.block_labels.end() ? 0 : it->second; +} + +uint32_t EmitBranchCondition(EmitterState* state, CFG::BranchCondition condition) { + switch (condition) { + case CFG::BranchCondition::SccZero: return EmitSccBool(state, false); + case CFG::BranchCondition::SccNonZero: return EmitSccBool(state, true); + case CFG::BranchCondition::VccZero: + return EmitMaskSummaryZeroBool(state, IR::RegisterFile::Vcc, true); + case CFG::BranchCondition::VccNonZero: + return EmitMaskSummaryZeroBool(state, IR::RegisterFile::Vcc, false); + case CFG::BranchCondition::ExecZero: + return EmitMaskSummaryZeroBool(state, IR::RegisterFile::Exec, true); + case CFG::BranchCondition::ExecNonZero: + return EmitMaskSummaryZeroBool(state, IR::RegisterFile::Exec, false); + default: return EmitSccBool(state, true); + } +} + +void EmitStructuredPrefix(EmitterState* state, const CFG::Terminator& term) { + if (term.loop_header) { + const auto merge_label = BlockLabel(*state, term.merge_block); + const auto continue_label = BlockLabel(*state, term.continue_block); + if (merge_label != 0 && continue_label != 0) { + state->builder.AddFunction({OpLoopMerge, merge_label, continue_label, LoopControlNone}); + } + return; + } + if (term.kind == CFG::TerminatorKind::ConditionalBranch && term.merge_block != UINT32_MAX) { + const auto merge_label = BlockLabel(*state, term.merge_block); + if (merge_label != 0) { + state->builder.AddFunction({OpSelectionMerge, merge_label, SelectionControlNone}); + } + } +} + +void EmitReturn(EmitterState* state) { + EmitKillIfPixelValidMaskInactive(state); + state->builder.AddFunction({OpReturn}); +} + +void EmitTerminator(EmitterState* state, const CFG::Terminator& term) { + const auto condition = term.kind == CFG::TerminatorKind::ConditionalBranch + ? EmitBranchCondition(state, term.condition) + : 0; + EmitStructuredPrefix(state, term); + + switch (term.kind) { + case CFG::TerminatorKind::Branch: { + const auto label = BlockLabel(*state, term.true_block); + if (label != 0) { + state->builder.AddFunction({OpBranch, label}); + } else { + EmitReturn(state); + } + break; + } + case CFG::TerminatorKind::ConditionalBranch: { + const auto true_label = BlockLabel(*state, term.true_block); + const auto false_label = BlockLabel(*state, term.false_block); + if (true_label != 0 && false_label != 0) { + state->builder.AddFunction( + {OpBranchConditional, condition, true_label, false_label}); + } else { + EmitReturn(state); + } + break; + } + default: EmitReturn(state); break; + } +} + +bool UserDataDwordIndex(const EmitterState& state, IR::Register reg, uint32_t& dword_index) { + if (state.program == nullptr || reg.file != IR::RegisterFile::Scalar) { + return false; + } + const auto& registers = state.program->bindings.user_data_registers; + const auto found = std::lower_bound(registers.begin(), registers.end(), reg.index); + if (found != registers.end() && *found == reg.index) { + dword_index = static_cast(found - registers.begin()); + return true; + } + return false; +} + +uint32_t EmitVsharpDwordLoad(EmitterState* state, uint32_t dword_index) { + if (state->push_constant_variable != 0) { + const auto pointer = state->builder.AllocateId(); + const auto value = state->builder.AllocateId(); + state->builder.AddFunction({OpAccessChain, state->ptr_push_constant_uint, pointer, + state->push_constant_variable, ConstantU32(state, 0), + ConstantU32(state, dword_index / 4u), + ConstantU32(state, dword_index % 4u)}); + state->builder.AddFunction({OpLoad, state->uint_type, value, pointer}); + return value; + } + if (state->vsharp_storage_variable != 0) { + const auto pointer = state->builder.AllocateId(); + const auto value = state->builder.AllocateId(); + state->builder.AddFunction({OpAccessChain, state->ptr_storage_buffer_uint, pointer, + state->vsharp_storage_variable, ConstantU32(state, 0), + ConstantU32(state, dword_index)}); + state->builder.AddFunction({OpLoad, state->uint_type, value, pointer}); + return value; + } + return ConstantU32(state, 0); +} + +uint32_t InitialRegisterValue(const EmitterState& state, IR::Register reg) { + if (reg.file == IR::RegisterFile::Exec) { + if (state.per_invocation_masks) { + return reg.index == 0 ? 1u : 0u; + } + if (reg.index == 0) { + return 0xffffffffu; + } + return state.wave_size == 64u ? 0xffffffffu : 0u; + } + return 0; +} + +uint32_t EmitInitialRegisterValue(EmitterState* state, IR::Register reg) { + uint32_t dword_index = 0; + if (UserDataDwordIndex(*state, reg, dword_index)) { + return EmitVsharpDwordLoad(state, dword_index); + } + return ConstantU32(state, InitialRegisterValue(*state, reg)); +} + +void EmitRegisterVariables(EmitterState* state) { + if (state->needs_function_lds) { + state->builder.AddFunction( + {OpVariable, state->ptr_workgroup_array, state->lds_variable, StorageClassFunction}); + } + for (const auto& binding: state->registers) { + state->builder.AddFunction( + {OpVariable, state->ptr_func_uint, binding.pointer_id, StorageClassFunction}); + } + if (state->dispatcher_fallback) { + state->builder.AddFunction( + {OpVariable, state->ptr_func_uint, state->dispatch_pc_variable, StorageClassFunction}); + } + if (state->pixel_valid_mask_variable != 0) { + state->builder.AddFunction({OpVariable, state->ptr_func_uint, + state->pixel_valid_mask_variable, StorageClassFunction}); + state->builder.AddFunction( + {OpStore, state->pixel_valid_mask_variable, ConstantU32(state, 1)}); + } + for (const auto& binding: state->registers) { + state->builder.AddFunction( + {OpStore, binding.pointer_id, EmitInitialRegisterValue(state, binding.reg)}); + } +} + +void EmitComputeInputRegisters(EmitterState* state) { + if (state->stage != ShaderType::Compute || state->compute_input_info == nullptr) { + return; + } + + const auto* cs = state->compute_input_info; + const auto thread_ids = cs->thread_ids_num > 0 + ? std::min(static_cast(cs->thread_ids_num), 3u) + : 0u; + for (uint32_t i = 0; i < thread_ids; i++) { + const auto pointer = PointerForRegister(*state, {IR::RegisterFile::Vector, i}); + if (pointer != 0) { + state->builder.AddFunction( + {OpStore, pointer, + EmitInputComponentU32(state, IR::StageInputKind::LocalInvocationId, i)}); + } + } + + uint32_t reg_offset = 0; + for (uint32_t i = 0; i < 3u; i++) { + if (!cs->group_id[i]) { + continue; + } + const auto pointer = PointerForRegister( + *state, + {IR::RegisterFile::Scalar, static_cast(cs->workgroup_register) + reg_offset}); + if (pointer != 0) { + state->builder.AddFunction( + {OpStore, pointer, + EmitInputComponentU32(state, IR::StageInputKind::WorkgroupId, i)}); + } + reg_offset++; + } + + if (cs->tg_size_en) { + const auto pointer = PointerForRegister( + *state, + {IR::RegisterFile::Scalar, static_cast(cs->workgroup_register) + reg_offset}); + if (pointer != 0) { + const uint32_t wave_size = cs->wave_size != 0 ? cs->wave_size : 64u; + const uint32_t total_threads = std::max(cs->threads_num[0], 1u) * + std::max(cs->threads_num[1], 1u) * + std::max(cs->threads_num[2], 1u); + const uint32_t waves = + std::min((total_threads + wave_size - 1u) / wave_size, 0x3fu); + + const auto local_index = EmitLocalInvocationIndex(state); + const auto wave_id = state->builder.AllocateId(); + const auto wave_bits = state->builder.AllocateId(); + const auto is_first = state->builder.AllocateId(); + const auto first_bit = state->builder.AllocateId(); + const auto base = state->builder.AllocateId(); + const auto packed = state->builder.AllocateId(); + state->builder.AddFunction( + {OpUDiv, state->uint_type, wave_id, local_index, ConstantU32(state, wave_size)}); + state->builder.AddFunction( + {OpShiftLeftLogical, state->uint_type, wave_bits, wave_id, ConstantU32(state, 20)}); + state->builder.AddFunction( + {OpIEqual, state->bool_type, is_first, wave_id, ConstantU32(state, 0)}); + state->builder.AddFunction({OpSelect, state->uint_type, first_bit, is_first, + ConstantU32(state, 0x80000000u), ConstantU32(state, 0)}); + state->builder.AddFunction( + {OpBitwiseOr, state->uint_type, base, wave_bits, ConstantU32(state, waves)}); + state->builder.AddFunction({OpBitwiseOr, state->uint_type, packed, base, first_bit}); + state->builder.AddFunction({OpStore, pointer, packed}); + } + } +} + +void EmitPixelFragCoordComponent(EmitterState* state, uint32_t component, uint32_t vgpr) { + const auto input = InputVariableForKind(*state, IR::StageInputKind::FragCoord); + const auto dst = PointerForRegister(*state, {IR::RegisterFile::Vector, vgpr}); + if (input == 0 || dst == 0) { + return; + } + + const auto pointer = state->builder.AllocateId(); + const auto value = state->builder.AllocateId(); + const auto bits = state->builder.AllocateId(); + state->builder.AddFunction( + {OpAccessChain, state->ptr_input_float, pointer, input, ConstantU32(state, component)}); + state->builder.AddFunction({OpLoad, state->float_type, value, pointer}); + state->builder.AddFunction({OpBitcast, state->uint_type, bits, value}); + state->builder.AddFunction({OpStore, dst, bits}); +} + +void EmitPixelFrontFacingRegister(EmitterState* state, uint32_t vgpr) { + const auto input = InputVariableForKind(*state, IR::StageInputKind::FrontFacing); + const auto dst = PointerForRegister(*state, {IR::RegisterFile::Vector, vgpr}); + if (input == 0 || dst == 0) { + return; + } + + const auto value = state->builder.AllocateId(); + const auto bits = state->builder.AllocateId(); + state->builder.AddFunction({OpLoad, state->bool_type, value, input}); + state->builder.AddFunction( + {OpSelect, state->uint_type, bits, value, ConstantU32(state, 1), ConstantU32(state, 0)}); + state->builder.AddFunction({OpStore, dst, bits}); +} + +void EmitPixelInputRegisters(EmitterState* state) { + if (state->stage != ShaderType::Pixel || state->pixel_input_info == nullptr) { + return; + } + + const auto* ps = state->pixel_input_info; + uint32_t reg = ps->ps_system_input_base; + if (ps->HasPositionInput()) { + if (ps->ps_pos_x) { + EmitPixelFragCoordComponent(state, 0, reg); + reg++; + } + if (ps->ps_pos_y) { + EmitPixelFragCoordComponent(state, 1, reg); + reg++; + } + if (ps->ps_pos_z) { + EmitPixelFragCoordComponent(state, 2, reg); + reg++; + } + if (ps->ps_pos_w) { + EmitPixelFragCoordComponent(state, 3, reg); + reg++; + } + } + if (ps->ps_front_face) { + EmitPixelFrontFacingRegister(state, reg); + } +} + +uint32_t EmitInputScalarU32(EmitterState* state, IR::StageInputKind kind) { + const auto variable = InputVariableForKind(*state, kind); + if (variable == 0) { + return ConstantU32(state, 0); + } + const auto raw = state->builder.AllocateId(); + const auto bits = state->builder.AllocateId(); + state->builder.AddFunction({OpLoad, state->int_type, raw, variable}); + state->builder.AddFunction({OpBitcast, state->uint_type, bits, raw}); + return bits; +} + +void EmitVertexInputRegisters(EmitterState* state) { + if (state->stage != ShaderType::Vertex) { + return; + } + + const auto vertex_index = PointerForRegister(*state, {IR::RegisterFile::Vector, 5}); + if (vertex_index != 0) { + state->builder.AddFunction( + {OpStore, vertex_index, EmitInputScalarU32(state, IR::StageInputKind::VertexIndex)}); + } + + const auto instance_index = PointerForRegister(*state, {IR::RegisterFile::Vector, 8}); + if (instance_index != 0) { + state->builder.AddFunction({OpStore, instance_index, + EmitInputScalarU32(state, IR::StageInputKind::InstanceIndex)}); + } +} + +void EmitInstruction(EmitterState* state, const IR::Instruction& inst) { + switch (inst.op) { + case IR::Opcode::ControlNop: EmitControlNop(state, inst); break; + case IR::Opcode::Waitcnt: EmitWaitcnt(state, inst); break; + case IR::Opcode::Barrier: EmitBarrier(state, inst); break; + case IR::Opcode::Sendmsg: + case IR::Opcode::TtraceData: + case IR::Opcode::InstPrefetch: EmitControlNop(state, inst); break; + case IR::Opcode::LoadInputF32: EmitLoadInputF32(state, inst); break; + case IR::Opcode::MoveU32: EmitMoveU32(state, inst); break; + case IR::Opcode::MoveF32Bits: EmitMoveF32Bits(state, inst); break; + case IR::Opcode::MoveRelDestU32: EmitMoveRelDestU32(state, inst); break; + case IR::Opcode::MoveRelSourceU32: EmitMoveRelSourceU32(state, inst); break; + case IR::Opcode::MoveU64: EmitMoveU64(state, inst); break; + case IR::Opcode::WqmB64: EmitWqmB64(state, inst); break; + case IR::Opcode::SaveexecB32: EmitSaveexecB32(state, inst); break; + case IR::Opcode::SaveexecB64: EmitSaveexecB64(state, inst); break; + case IR::Opcode::ReadFirstLaneU32: EmitReadFirstLaneU32(state, inst); break; + case IR::Opcode::ReadLaneU32: EmitReadLaneU32(state, inst); break; + case IR::Opcode::WriteLaneU32: EmitWriteLaneU32(state, inst); break; + case IR::Opcode::Permlane16B32: EmitPermlaneB32(state, inst, false); break; + case IR::Opcode::Permlanex16B32: EmitPermlaneB32(state, inst, true); break; + case IR::Opcode::AbsI32: EmitAbsI32(state, inst); break; + case IR::Opcode::IAddU32: EmitBinaryU32(state, inst, OpIAdd); break; + case IR::Opcode::IAddCarryU32: EmitIAddCarryU32(state, inst); break; + case IR::Opcode::ISubBorrowU32: EmitISubBorrowU32(state, inst); break; + case IR::Opcode::ScalarAddCarryU32: EmitScalarAddCarryU32(state, inst); break; + case IR::Opcode::ScalarSubBorrowU32: EmitScalarSubBorrowU32(state, inst); break; + case IR::Opcode::ScalarSubBorrowCarryU32: EmitScalarSubBorrowCarryU32(state, inst); break; + case IR::Opcode::ScalarSignedAddOverflowI32: + EmitScalarSignedOverflowI32(state, inst, false); + break; + case IR::Opcode::ScalarSignedSubOverflowI32: + EmitScalarSignedOverflowI32(state, inst, true); + break; + case IR::Opcode::ScalarShiftLeftAddCarryU32: + EmitScalarShiftLeftAddCarryU32(state, inst); + break; + case IR::Opcode::ISubU32: EmitBinaryU32(state, inst, OpISub); break; + case IR::Opcode::IMulU32: EmitBinaryU32(state, inst, OpIMul); break; + case IR::Opcode::UMulHighU32: EmitMulHighU32(state, inst); break; + case IR::Opcode::SMulHighI32: EmitMulHighI32(state, inst); break; + case IR::Opcode::IMadI24U32: EmitMadI32I24(state, inst); break; + case IR::Opcode::UMadU24U32: EmitMadU32U24(state, inst); break; + case IR::Opcode::UMadU64U32: EmitUMadU64U32(state, inst); break; + case IR::Opcode::SadU32: EmitSadU32(state, inst); break; + case IR::Opcode::IAdd3U32: EmitAdd3U32(state, inst); break; + case IR::Opcode::IMulI24U32: EmitMulI24U32(state, inst); break; + case IR::Opcode::UMulU24U32: EmitMulU24U32(state, inst); break; + case IR::Opcode::IMinI32: EmitMinMaxI32(state, inst, false); break; + case IR::Opcode::IMaxI32: EmitMinMaxI32(state, inst, true); break; + case IR::Opcode::IMin3I32: EmitMinMax3U32(state, inst, true, false); break; + case IR::Opcode::IMax3I32: EmitMinMax3U32(state, inst, true, true); break; + case IR::Opcode::IMed3I32: EmitMed3U32(state, inst, true); break; + case IR::Opcode::UMinU32: EmitMinMaxU32(state, inst, false); break; + case IR::Opcode::UMaxU32: EmitMinMaxU32(state, inst, true); break; + case IR::Opcode::UMin3U32: EmitMinMax3U32(state, inst, false, false); break; + case IR::Opcode::UMax3U32: EmitMinMax3U32(state, inst, false, true); break; + case IR::Opcode::UMed3U32: EmitMed3U32(state, inst, false); break; + case IR::Opcode::BitwiseAndU32: EmitBinaryU32(state, inst, OpBitwiseAnd); break; + case IR::Opcode::BitwiseAndU64: EmitBinaryU64(state, inst, OpBitwiseAnd); break; + case IR::Opcode::BitwiseAndNotU32: EmitBinaryNotRhsU32(state, inst, OpBitwiseAnd); break; + case IR::Opcode::BitwiseAndNotU64: EmitBinaryNotRhsU64(state, inst, OpBitwiseAnd); break; + case IR::Opcode::BitwiseOrU32: EmitBinaryU32(state, inst, OpBitwiseOr); break; + case IR::Opcode::BitwiseOrU64: EmitBinaryU64(state, inst, OpBitwiseOr); break; + case IR::Opcode::BitwiseOrNotU32: EmitBinaryNotRhsU32(state, inst, OpBitwiseOr); break; + case IR::Opcode::BitwiseOrNotU64: EmitBinaryNotRhsU64(state, inst, OpBitwiseOr); break; + case IR::Opcode::BitwiseAndOrU32: + EmitChainedBinaryU32(state, inst, OpBitwiseAnd, OpBitwiseOr); + break; + case IR::Opcode::BitwiseOr3U32: + EmitChainedBinaryU32(state, inst, OpBitwiseOr, OpBitwiseOr); + break; + case IR::Opcode::BitwiseXorU32: EmitBinaryU32(state, inst, OpBitwiseXor); break; + case IR::Opcode::BitwiseXorU64: EmitBinaryU64(state, inst, OpBitwiseXor); break; + case IR::Opcode::BitwiseXor3U32: + EmitChainedBinaryU32(state, inst, OpBitwiseXor, OpBitwiseXor); + break; + case IR::Opcode::BitwiseNandU32: EmitBinaryThenNotU32(state, inst, OpBitwiseAnd); break; + case IR::Opcode::BitwiseNandU64: EmitBinaryThenNotU64(state, inst, OpBitwiseAnd); break; + case IR::Opcode::BitwiseNorU32: EmitBinaryThenNotU32(state, inst, OpBitwiseOr); break; + case IR::Opcode::BitwiseNorU64: EmitBinaryThenNotU64(state, inst, OpBitwiseOr); break; + case IR::Opcode::BitwiseXnorU32: EmitBinaryThenNotU32(state, inst, OpBitwiseXor); break; + case IR::Opcode::BitwiseXnorU64: EmitBinaryThenNotU64(state, inst, OpBitwiseXor); break; + case IR::Opcode::BitwiseNotU32: EmitUnaryU32(state, inst, OpNot); break; + case IR::Opcode::BitwiseNotU64: EmitUnaryU64(state, inst, OpNot); break; + case IR::Opcode::BitClearU32: EmitBitClearU32(state, inst); break; + case IR::Opcode::BitSetU32: EmitBitSetU32(state, inst); break; + case IR::Opcode::BitReverseU32: EmitUnaryU32(state, inst, OpBitReverse); break; + case IR::Opcode::BitCountU32: EmitBitCountU32(state, inst); break; + case IR::Opcode::BitCountU64: EmitBitCountU64(state, inst); break; + case IR::Opcode::BitCountAddU32: EmitBitCountAddU32(state, inst); break; + case IR::Opcode::MaskedBitCountLowU32: EmitMaskedBitCountU32(state, inst, 0); break; + case IR::Opcode::MaskedBitCountHighU32: EmitMaskedBitCountU32(state, inst, 1); break; + case IR::Opcode::BitReplicateB64B32: EmitBitReplicateB64B32(state, inst); break; + case IR::Opcode::FindLsbU32: EmitFindLsbU32(state, inst); break; + case IR::Opcode::FindMsbFromHighU32: EmitFindMsbFromHighU32(state, inst); break; + case IR::Opcode::FindMsbFromHighU64: EmitFindMsbFromHighU64(state, inst); break; + case IR::Opcode::BitFieldMaskU32: EmitBitFieldMaskU32(state, inst); break; + case IR::Opcode::BitFieldMaskU64: EmitBitFieldMaskU64(state, inst); break; + case IR::Opcode::BitFieldExtractU32: EmitBitFieldExtractU32(state, inst); break; + case IR::Opcode::BitFieldExtractU64: EmitBitFieldExtractU64(state, inst); break; + case IR::Opcode::BitFieldExtract3U32: EmitBitFieldExtract3U32(state, inst, false); break; + case IR::Opcode::BitFieldExtract3I32: EmitBitFieldExtract3U32(state, inst, true); break; + case IR::Opcode::BitFieldInsertSelectU32: EmitBitFieldInsertSelectU32(state, inst); break; + case IR::Opcode::BitCompare0B32: EmitBitCompareB32(state, inst, false); break; + case IR::Opcode::BitCompare1B32: EmitBitCompareB32(state, inst, true); break; + case IR::Opcode::AlignBitU32: EmitAlignBitU32(state, inst); break; + case IR::Opcode::ShiftLeftAddU32: + EmitChainedBinaryU32(state, inst, OpShiftLeftLogical, OpIAdd, true); + break; + case IR::Opcode::AddShiftLeftU32: + EmitChainedBinaryU32(state, inst, OpIAdd, OpShiftLeftLogical, false, true); + break; + case IR::Opcode::XorAddU32: EmitChainedBinaryU32(state, inst, OpBitwiseXor, OpIAdd); break; + case IR::Opcode::ShiftLeftOrU32: + EmitChainedBinaryU32(state, inst, OpShiftLeftLogical, OpBitwiseOr, true); + break; + case IR::Opcode::ShiftLeftLogicalU32: EmitShiftU32(state, inst, OpShiftLeftLogical); break; + case IR::Opcode::ShiftLeftLogicalU64: EmitShiftLeftLogicalU64(state, inst); break; + case IR::Opcode::ShiftLeftLogicalU16: + EmitShiftU16(state, inst, OpShiftLeftLogical, false); + break; + case IR::Opcode::ShiftRightLogicalU32: + EmitShiftU32(state, inst, OpShiftRightLogical); + break; + case IR::Opcode::ShiftRightLogicalU64: EmitShiftRightLogicalU64(state, inst); break; + case IR::Opcode::ShiftRightLogicalU16: + EmitShiftU16(state, inst, OpShiftRightLogical, false); + break; + case IR::Opcode::ShiftRightArithmeticI32: + EmitShiftU32(state, inst, OpShiftRightArithmetic); + break; + case IR::Opcode::ShiftRightArithmeticI16: + EmitShiftU16(state, inst, OpShiftRightArithmetic, true); + break; + case IR::Opcode::SelectU32: EmitSelectU32(state, inst); break; + case IR::Opcode::SelectMaskU32: EmitSelectMaskU32(state, inst); break; + case IR::Opcode::SelectF32Bits: EmitSelectF32Bits(state, inst); break; + case IR::Opcode::SelectMaskF32Bits: EmitSelectMaskF32Bits(state, inst); break; + case IR::Opcode::SelectU64: EmitSelectU64(state, inst); break; + case IR::Opcode::PackLowLowU16: EmitPackU16(state, inst, false, false); break; + case IR::Opcode::PackLowHighU16: EmitPackU16(state, inst, false, true); break; + case IR::Opcode::PackHighHighU16: EmitPackU16(state, inst, true, true); break; + case IR::Opcode::PackU16U32: EmitPackU16(state, inst, false, false); break; + case IR::Opcode::CompareFalse: EmitCompareConstant(state, inst, false); break; + case IR::Opcode::CompareTrue: EmitCompareConstant(state, inst, true); break; + case IR::Opcode::CompareEqU32: EmitCompareU32(state, inst, OpIEqual); break; + case IR::Opcode::CompareNeU32: EmitCompareU32(state, inst, OpINotEqual); break; + case IR::Opcode::CompareGtU32: EmitCompareU32(state, inst, OpUGreaterThan); break; + case IR::Opcode::CompareGeU32: EmitCompareU32(state, inst, OpUGreaterThanEqual); break; + case IR::Opcode::CompareLtU32: EmitCompareU32(state, inst, OpULessThan); break; + case IR::Opcode::CompareLeU32: EmitCompareU32(state, inst, OpULessThanEqual); break; + case IR::Opcode::CompareNeU64: EmitCompareNeU64(state, inst); break; + case IR::Opcode::CompareMaskEqU32: EmitCompareMaskU32(state, inst, OpIEqual); break; + case IR::Opcode::CompareMaskNeU32: EmitCompareMaskU32(state, inst, OpINotEqual); break; + case IR::Opcode::CompareMaskGtU32: EmitCompareMaskU32(state, inst, OpUGreaterThan); break; + case IR::Opcode::CompareMaskGeU32: + EmitCompareMaskU32(state, inst, OpUGreaterThanEqual); + break; + case IR::Opcode::CompareMaskLtU32: EmitCompareMaskU32(state, inst, OpULessThan); break; + case IR::Opcode::CompareMaskLeU32: EmitCompareMaskU32(state, inst, OpULessThanEqual); break; + case IR::Opcode::CompareEqI32: EmitCompareU32(state, inst, OpIEqual); break; + case IR::Opcode::CompareNeI32: EmitCompareU32(state, inst, OpINotEqual); break; + case IR::Opcode::CompareGtI32: EmitCompareU32(state, inst, OpSGreaterThan); break; + case IR::Opcode::CompareGeI32: EmitCompareU32(state, inst, OpSGreaterThanEqual); break; + case IR::Opcode::CompareLtI32: EmitCompareU32(state, inst, OpSLessThan); break; + case IR::Opcode::CompareLeI32: EmitCompareU32(state, inst, OpSLessThanEqual); break; + case IR::Opcode::CompareEqI16: EmitCompareI16(state, inst, OpIEqual); break; + case IR::Opcode::CompareNeI16: EmitCompareI16(state, inst, OpINotEqual); break; + case IR::Opcode::CompareGtI16: EmitCompareI16(state, inst, OpSGreaterThan); break; + case IR::Opcode::CompareGeI16: EmitCompareI16(state, inst, OpSGreaterThanEqual); break; + case IR::Opcode::CompareLtI16: EmitCompareI16(state, inst, OpSLessThan); break; + case IR::Opcode::CompareLeI16: EmitCompareI16(state, inst, OpSLessThanEqual); break; + case IR::Opcode::CompareMaskEqI32: EmitCompareMaskU32(state, inst, OpIEqual); break; + case IR::Opcode::CompareMaskNeI32: EmitCompareMaskU32(state, inst, OpINotEqual); break; + case IR::Opcode::CompareMaskGtI32: EmitCompareMaskU32(state, inst, OpSGreaterThan); break; + case IR::Opcode::CompareMaskGeI32: + EmitCompareMaskU32(state, inst, OpSGreaterThanEqual); + break; + case IR::Opcode::CompareMaskLtI32: EmitCompareMaskU32(state, inst, OpSLessThan); break; + case IR::Opcode::CompareMaskLeI32: EmitCompareMaskU32(state, inst, OpSLessThanEqual); break; + case IR::Opcode::CompareEqU16: EmitCompareU16(state, inst, OpIEqual); break; + case IR::Opcode::CompareNeU16: EmitCompareU16(state, inst, OpINotEqual); break; + case IR::Opcode::CompareGtU16: EmitCompareU16(state, inst, OpUGreaterThan); break; + case IR::Opcode::CompareGeU16: EmitCompareU16(state, inst, OpUGreaterThanEqual); break; + case IR::Opcode::CompareLtU16: EmitCompareU16(state, inst, OpULessThan); break; + case IR::Opcode::CompareLeU16: EmitCompareU16(state, inst, OpULessThanEqual); break; + case IR::Opcode::CompareEqF32: EmitCompareF32(state, inst, OpFOrdEqual); break; + case IR::Opcode::CompareNeF32: EmitCompareF32(state, inst, OpFOrdNotEqual); break; + case IR::Opcode::CompareGtF32: EmitCompareF32(state, inst, OpFOrdGreaterThan); break; + case IR::Opcode::CompareGeF32: EmitCompareF32(state, inst, OpFOrdGreaterThanEqual); break; + case IR::Opcode::CompareLtF32: EmitCompareF32(state, inst, OpFOrdLessThan); break; + case IR::Opcode::CompareLeF32: EmitCompareF32(state, inst, OpFOrdLessThanEqual); break; + case IR::Opcode::CompareOrderedF32: EmitCompareOrderedF32(state, inst, true); break; + case IR::Opcode::CompareUnorderedF32: EmitCompareOrderedF32(state, inst, false); break; + case IR::Opcode::CompareUnordEqF32: EmitCompareF32(state, inst, OpFUnordEqual); break; + case IR::Opcode::CompareUnordNeF32: EmitCompareF32(state, inst, OpFUnordNotEqual); break; + case IR::Opcode::CompareUnordGtF32: EmitCompareF32(state, inst, OpFUnordGreaterThan); break; + case IR::Opcode::CompareUnordGeF32: + EmitCompareF32(state, inst, OpFUnordGreaterThanEqual); + break; + case IR::Opcode::CompareUnordLtF32: EmitCompareF32(state, inst, OpFUnordLessThan); break; + case IR::Opcode::CompareUnordLeF32: + EmitCompareF32(state, inst, OpFUnordLessThanEqual); + break; + case IR::Opcode::CompareClassF32: EmitCompareClassF32(state, inst); break; + case IR::Opcode::CompareEqF16: EmitCompareF16(state, inst, OpFOrdEqual); break; + case IR::Opcode::CompareNeF16: EmitCompareF16(state, inst, OpFOrdNotEqual); break; + case IR::Opcode::CompareGtF16: EmitCompareF16(state, inst, OpFOrdGreaterThan); break; + case IR::Opcode::CompareGeF16: EmitCompareF16(state, inst, OpFOrdGreaterThanEqual); break; + case IR::Opcode::CompareLtF16: EmitCompareF16(state, inst, OpFOrdLessThan); break; + case IR::Opcode::CompareLeF16: EmitCompareF16(state, inst, OpFOrdLessThanEqual); break; + case IR::Opcode::CompareUnordNeF16: EmitCompareF16(state, inst, OpFUnordNotEqual); break; + case IR::Opcode::CompareMaskEqF16: EmitCompareF16(state, inst, OpFOrdEqual); break; + case IR::Opcode::CompareMaskNeF16: EmitCompareF16(state, inst, OpFOrdNotEqual); break; + case IR::Opcode::CompareMaskGtF16: EmitCompareF16(state, inst, OpFOrdGreaterThan); break; + case IR::Opcode::CompareMaskGeF16: + EmitCompareF16(state, inst, OpFOrdGreaterThanEqual); + break; + case IR::Opcode::CompareMaskLtF16: EmitCompareF16(state, inst, OpFOrdLessThan); break; + case IR::Opcode::CompareMaskLeF16: EmitCompareF16(state, inst, OpFOrdLessThanEqual); break; + case IR::Opcode::CompareMaskUnordNeF16: + EmitCompareF16(state, inst, OpFUnordNotEqual); + break; + case IR::Opcode::CompareMaskUnordGeF16: + EmitCompareF16(state, inst, OpFUnordGreaterThanEqual); + break; + case IR::Opcode::CompareMaskEqF32: EmitCompareMaskF32(state, inst, OpFOrdEqual); break; + case IR::Opcode::CompareMaskNeF32: EmitCompareMaskF32(state, inst, OpFOrdNotEqual); break; + case IR::Opcode::CompareMaskGtF32: + EmitCompareMaskF32(state, inst, OpFOrdGreaterThan); + break; + case IR::Opcode::CompareMaskGeF32: + EmitCompareMaskF32(state, inst, OpFOrdGreaterThanEqual); + break; + case IR::Opcode::CompareMaskLtF32: EmitCompareMaskF32(state, inst, OpFOrdLessThan); break; + case IR::Opcode::CompareMaskLeF32: + EmitCompareMaskF32(state, inst, OpFOrdLessThanEqual); + break; + case IR::Opcode::CompareMaskUnordEqF32: + EmitCompareMaskF32(state, inst, OpFUnordEqual); + break; + case IR::Opcode::CompareMaskUnordNeF32: + EmitCompareMaskF32(state, inst, OpFUnordNotEqual); + break; + case IR::Opcode::CompareMaskUnordGtF32: + EmitCompareMaskF32(state, inst, OpFUnordGreaterThan); + break; + case IR::Opcode::CompareMaskUnordGeF32: + EmitCompareMaskF32(state, inst, OpFUnordGreaterThanEqual); + break; + case IR::Opcode::CompareMaskUnordLtF32: + EmitCompareMaskF32(state, inst, OpFUnordLessThan); + break; + case IR::Opcode::CompareMaskUnordLeF32: + EmitCompareMaskF32(state, inst, OpFUnordLessThanEqual); + break; + case IR::Opcode::ConvertU32ToF32: EmitConvertU32ToF32(state, inst); break; + case IR::Opcode::ConvertI32ToF32: EmitConvertI32ToF32(state, inst); break; + case IR::Opcode::ConvertF32ToU32: EmitConvertF32ToU32(state, inst); break; + case IR::Opcode::ConvertF32ToI32: EmitConvertF32ToI32(state, inst); break; + case IR::Opcode::ConvertF32ToF16: EmitConvertF32ToF16(state, inst); break; + case IR::Opcode::ConvertF16ToF32: EmitConvertF16ToF32(state, inst); break; + case IR::Opcode::ConvertU16ToF16: EmitConvertU16ToF16(state, inst); break; + case IR::Opcode::ConvertF16ToU16: EmitConvertF16ToU16(state, inst); break; + case IR::Opcode::ConvertI16ToF16: EmitConvertI16ToF16(state, inst); break; + case IR::Opcode::ConvertF16ToI16: EmitConvertF16ToI16(state, inst); break; + case IR::Opcode::ConvertRoundPlusInfF32ToI32: + EmitConvertRoundPlusInfF32ToI32(state, inst); + break; + case IR::Opcode::ConvertFloorF32ToI32: EmitConvertFloorF32ToI32(state, inst); break; + case IR::Opcode::ConvertI4ToOffsetF32: EmitConvertI4ToOffsetF32(state, inst); break; + case IR::Opcode::LdexpF32: EmitLdexpF32(state, inst); break; + case IR::Opcode::PackF32ToF16Rtz: EmitPackF32ToF16Rtz(state, inst); break; + case IR::Opcode::PackSnorm2x16F32: + EmitPackNormalizedF32(state, inst, GlslPackSnorm2x16); + break; + case IR::Opcode::PackUnorm2x16F32: + EmitPackNormalizedF32(state, inst, GlslPackUnorm2x16); + break; + case IR::Opcode::PackU8F32: EmitPackU8F32(state, inst); break; + case IR::Opcode::PackB32F16: EmitPackB32F16(state, inst); break; + case IR::Opcode::PackedMadI16: + case IR::Opcode::PackedMulLoU16: + case IR::Opcode::PackedAddI16: + case IR::Opcode::PackedSubI16: + case IR::Opcode::PackedLshlrevB16: + case IR::Opcode::PackedLshrrevB16: + case IR::Opcode::PackedAshrrevI16: + case IR::Opcode::PackedMaxI16: + case IR::Opcode::PackedMinI16: + case IR::Opcode::PackedMadU16: + case IR::Opcode::PackedAddU16: + case IR::Opcode::PackedSubU16: + case IR::Opcode::PackedMaxU16: + case IR::Opcode::PackedMinU16: EmitPackedInteger16(state, inst); break; + case IR::Opcode::PackedAddF16: + case IR::Opcode::PackedMulF16: + case IR::Opcode::PackedMinF16: + case IR::Opcode::PackedMaxF16: + case IR::Opcode::PackedFmaF16: EmitPackedF16(state, inst); break; + case IR::Opcode::AddF16: EmitBinaryF16(state, inst, OpFAdd); break; + case IR::Opcode::SubF16: EmitBinaryF16(state, inst, OpFSub); break; + case IR::Opcode::MulF16: EmitBinaryF16(state, inst, OpFMul); break; + case IR::Opcode::MinF16: EmitMinMaxF16(state, inst, false); break; + case IR::Opcode::MaxF16: EmitMinMaxF16(state, inst, true); break; + case IR::Opcode::FmaF16: EmitFmaF16(state, inst); break; + case IR::Opcode::Min3F16: EmitMinMax3F16(state, inst, false); break; + case IR::Opcode::Max3F16: EmitMinMax3F16(state, inst, true); break; + case IR::Opcode::Med3F16: EmitMed3F16(state, inst); break; + case IR::Opcode::MadMixF16: EmitMadMixF16(state, inst); break; + case IR::Opcode::IAddU16: EmitBinaryU16(state, inst, OpIAdd); break; + case IR::Opcode::ISubI16: EmitBinaryU16(state, inst, OpISub); break; + case IR::Opcode::IMinI16: EmitMinMaxU16(state, inst, true, false); break; + case IR::Opcode::IMaxI16: EmitMinMaxU16(state, inst, true, true); break; + case IR::Opcode::UMinU16: EmitMinMaxU16(state, inst, false, false); break; + case IR::Opcode::UMaxU16: EmitMinMaxU16(state, inst, false, true); break; + case IR::Opcode::RcpF16: + case IR::Opcode::SqrtF16: + case IR::Opcode::InverseSqrtF16: + case IR::Opcode::Log2F16: + case IR::Opcode::Exp2F16: + case IR::Opcode::FloorF16: + case IR::Opcode::CeilF16: + case IR::Opcode::TruncF16: + case IR::Opcode::RoundEvenF16: EmitSpecialF16(state, inst); break; + case IR::Opcode::ConvertByteU32ToF32: EmitConvertByteU32ToF32(state, inst); break; + case IR::Opcode::RcpF32: EmitRcpF32(state, inst); break; + case IR::Opcode::FractF32: EmitFloatExtInst(state, inst, GlslFract); break; + case IR::Opcode::TruncF32: EmitFloatExtInst(state, inst, GlslTrunc); break; + case IR::Opcode::CeilF32: EmitFloatExtInst(state, inst, GlslCeil); break; + case IR::Opcode::RoundEvenF32: EmitFloatExtInst(state, inst, GlslRoundEven); break; + case IR::Opcode::FloorF32: EmitFloatExtInst(state, inst, GlslFloor); break; + case IR::Opcode::Exp2F32: EmitFloatExtInst(state, inst, GlslExp2, false, true); break; + case IR::Opcode::Log2F32: EmitFloatExtInst(state, inst, GlslLog2, false, true); break; + case IR::Opcode::InverseSqrtF32: + EmitFloatExtInst(state, inst, GlslInverseSqrt, false, true); + break; + case IR::Opcode::SqrtF32: EmitFloatExtInst(state, inst, GlslSqrt, false, true); break; + case IR::Opcode::SinF32: EmitFloatExtInst(state, inst, GlslSin, true); break; + case IR::Opcode::CosF32: EmitFloatExtInst(state, inst, GlslCos, true); break; + case IR::Opcode::CubeIdF32: EmitCubeF32(state, inst); break; + case IR::Opcode::CubeScF32: EmitCubeF32(state, inst); break; + case IR::Opcode::CubeTcF32: EmitCubeF32(state, inst); break; + case IR::Opcode::CubeMaF32: EmitCubeF32(state, inst); break; + case IR::Opcode::FAddF32: EmitBinaryF32(state, inst, OpFAdd); break; + case IR::Opcode::FSubF32: EmitBinaryF32(state, inst, OpFSub); break; + case IR::Opcode::FMulF32: EmitBinaryF32(state, inst, OpFMul); break; + case IR::Opcode::FMinF32: EmitMinMaxF32(state, inst, false); break; + case IR::Opcode::FMaxF32: EmitMinMaxF32(state, inst, true); break; + case IR::Opcode::FMadF32: EmitMadF32(state, inst); break; + case IR::Opcode::Dot2AccF32F16: EmitDot2AccF32F16(state, inst); break; + case IR::Opcode::FMin3F32: EmitMinMax3F32(state, inst, false); break; + case IR::Opcode::FMax3F32: EmitMinMax3F32(state, inst, true); break; + case IR::Opcode::FMed3F32: EmitMed3F32(state, inst); break; + case IR::Opcode::SLoadDword: EmitSLoadDword(state, inst); break; + case IR::Opcode::SBufferLoadDword: + EmitMemoryLoadU32(state, inst, IR::ResourceKind::ScalarBuffer, 0, 1); + break; + case IR::Opcode::LoadSrtDword: EmitLoadSrtDword(state, inst); break; + case IR::Opcode::BufferLoadUbyte: EmitBufferLoadUbyte(state, inst); break; + case IR::Opcode::BufferLoadSbyte: EmitBufferLoadSbyte(state, inst); break; + case IR::Opcode::BufferLoadUshort: EmitBufferLoadUshort(state, inst); break; + case IR::Opcode::BufferLoadSshort: EmitBufferLoadSshort(state, inst); break; + case IR::Opcode::BufferLoadDword: EmitBufferLoadDword(state, inst); break; + case IR::Opcode::BufferStoreByte: + EmitGuardedByExec(state, [&]() { + EmitMemoryStoreSubDwordU32(state, inst, IR::ResourceKind::Buffer, 1, + AddressSourceCount(inst, 1), 8); + }); + break; + case IR::Opcode::BufferStoreShort: + EmitGuardedByExec(state, [&]() { + EmitMemoryStoreSubDwordU32(state, inst, IR::ResourceKind::Buffer, 1, + AddressSourceCount(inst, 1), 16); + }); + break; + case IR::Opcode::BufferStoreDword: + EmitGuardedByExec(state, [&]() { EmitBufferStoreDword(state, inst); }); + break; + case IR::Opcode::AtomicSwapU32: + EmitGuardedByExec(state, [&]() { EmitAtomicU32(state, inst, OpAtomicExchange); }); + break; + case IR::Opcode::AtomicAddU32: + EmitGuardedByExec(state, [&]() { EmitAtomicU32(state, inst, OpAtomicIAdd); }); + break; + case IR::Opcode::AtomicSubU32: + EmitGuardedByExec(state, [&]() { EmitAtomicU32(state, inst, OpAtomicISub); }); + break; + case IR::Opcode::AtomicSMinI32: + EmitGuardedByExec(state, [&]() { EmitAtomicU32(state, inst, OpAtomicSMin); }); + break; + case IR::Opcode::AtomicUMinU32: + EmitGuardedByExec(state, [&]() { EmitAtomicU32(state, inst, OpAtomicUMin); }); + break; + case IR::Opcode::AtomicSMaxI32: + EmitGuardedByExec(state, [&]() { EmitAtomicU32(state, inst, OpAtomicSMax); }); + break; + case IR::Opcode::AtomicUMaxU32: + EmitGuardedByExec(state, [&]() { EmitAtomicU32(state, inst, OpAtomicUMax); }); + break; + case IR::Opcode::AtomicAndU32: + EmitGuardedByExec(state, [&]() { EmitAtomicU32(state, inst, OpAtomicAnd); }); + break; + case IR::Opcode::AtomicOrU32: + EmitGuardedByExec(state, [&]() { EmitAtomicU32(state, inst, OpAtomicOr); }); + break; + case IR::Opcode::AtomicXorU32: + EmitGuardedByExec(state, [&]() { EmitAtomicU32(state, inst, OpAtomicXor); }); + break; + case IR::Opcode::FlatLoadUbyte: EmitFlatLoadUbyte(state, inst); break; + case IR::Opcode::FlatLoadSbyte: EmitFlatLoadSbyte(state, inst); break; + case IR::Opcode::FlatLoadUshort: EmitFlatLoadUshort(state, inst); break; + case IR::Opcode::FlatLoadSshort: EmitFlatLoadSshort(state, inst); break; + case IR::Opcode::FlatLoadDword: EmitFlatLoadDword(state, inst); break; + case IR::Opcode::FlatStoreByte: + EmitGuardedByExec(state, [&]() { + EmitMemoryStoreSubDwordU32(state, inst, inst.memory.kind, 1, inst.src_count - 1u, + 8); + }); + break; + case IR::Opcode::FlatStoreShort: + EmitGuardedByExec(state, [&]() { + EmitMemoryStoreSubDwordU32(state, inst, inst.memory.kind, 1, inst.src_count - 1u, + 16); + }); + break; + case IR::Opcode::FlatStoreDword: + EmitGuardedByExec(state, [&]() { + EmitMemoryStoreU32(state, inst, inst.memory.kind, 1, inst.src_count - 1u); + }); + break; + case IR::Opcode::DsReadUbyte: + EmitMemoryLoadSubDwordU32(state, inst, inst.memory.kind, 0, 1, 8, false); + break; + case IR::Opcode::DsReadSbyte: + EmitMemoryLoadSubDwordU32(state, inst, inst.memory.kind, 0, 1, 8, true); + break; + case IR::Opcode::DsReadUshort: + EmitMemoryLoadSubDwordU32(state, inst, inst.memory.kind, 0, 1, 16, false); + break; + case IR::Opcode::DsReadSshort: + EmitMemoryLoadSubDwordU32(state, inst, inst.memory.kind, 0, 1, 16, true); + break; + case IR::Opcode::DsReadB32: EmitMemoryLoadU32(state, inst, inst.memory.kind, 0, 1); break; + case IR::Opcode::DsSwizzleB32: EmitDsSwizzleB32(state, inst); break; + case IR::Opcode::DsConsume: EmitDsAppendConsume(state, inst, OpAtomicISub); break; + case IR::Opcode::DsAppend: EmitDsAppendConsume(state, inst, OpAtomicIAdd); break; + case IR::Opcode::DsReadAddtidB32: EmitDsReadAddtidB32(state, inst); break; + case IR::Opcode::DsMinF32: + EmitGuardedByExec(state, [&]() { EmitDsFloatMinMaxF32(state, inst, false); }); + break; + case IR::Opcode::DsMaxF32: + EmitGuardedByExec(state, [&]() { EmitDsFloatMinMaxF32(state, inst, true); }); + break; + case IR::Opcode::DsWriteByte: + EmitGuardedByExec(state, [&]() { + EmitMemoryStoreSubDwordU32(state, inst, inst.memory.kind, 1, 1, 8); + }); + break; + case IR::Opcode::DsWriteShort: + EmitGuardedByExec(state, [&]() { + EmitMemoryStoreSubDwordU32(state, inst, inst.memory.kind, 1, 1, 16); + }); + break; + case IR::Opcode::DsWriteB32: + EmitGuardedByExec(state, + [&]() { EmitMemoryStoreU32(state, inst, inst.memory.kind, 1, 1); }); + break; + case IR::Opcode::DsWriteAddtidB32: + EmitGuardedByExec(state, [&]() { EmitDsWriteAddtidB32(state, inst); }); + break; + case IR::Opcode::ImageGetResinfo: EmitImageGetResinfo(state, inst); break; + case IR::Opcode::ImageGetLod: EmitImageGetLod(state, inst); break; + case IR::Opcode::ImageLoad: EmitImageLoad(state, inst); break; + case IR::Opcode::ImageStore: + EmitGuardedByExec(state, [&]() { EmitImageStore(state, inst); }); + break; + case IR::Opcode::ImageSample: EmitImageSample(state, inst); break; + case IR::Opcode::ImageGather4: EmitImageGather4(state, inst); break; + case IR::Opcode::Export: + if (ExportUsesPixelValidMask(*state, inst)) { + EmitUpdatePixelValidMask(state); + } + EmitGuardedByExec(state, [&]() { EmitExport(state, inst); }); + break; + default: break; + } +} + +uint32_t DispatcherTargetValue(EmitterState* state, uint32_t block_id) { + return BlockLabel(*state, block_id) != 0 ? block_id : UINT32_MAX; +} + +void EmitDispatcherStoreTarget(EmitterState* state, uint32_t block_id) { + state->builder.AddFunction({OpStore, state->dispatch_pc_variable, + ConstantU32(state, DispatcherTargetValue(state, block_id))}); +} + +void EmitDispatcherExit(EmitterState* state); + +bool RegisterForScalarCode(uint32_t code, IR::Register& reg) { + if (code < 106u) { + reg = {IR::RegisterFile::Scalar, code}; + return true; + } + switch (code) { + case 106u: reg = {IR::RegisterFile::Vcc, 0}; return true; + case 107u: reg = {IR::RegisterFile::Vcc, 1}; return true; + case 124u: reg = {IR::RegisterFile::M0, 0}; return true; + case 126u: reg = {IR::RegisterFile::Exec, 0}; return true; + case 127u: reg = {IR::RegisterFile::Exec, 1}; return true; + default: return false; + } +} + +bool EmitDispatcherLoadScalarCode(EmitterState* state, uint32_t code, uint32_t& value) { + IR::Register reg; + if (!RegisterForScalarCode(code, reg)) { + return false; + } + const auto pointer = PointerForRegister(*state, reg); + if (pointer == 0) { + return false; + } + value = state->builder.AllocateId(); + state->builder.AddFunction({OpLoad, state->uint_type, value, pointer}); + return true; +} + +void EmitDispatcherStoreSelectorTarget(EmitterState* state, const CFG::Terminator& term) { + if (term.indirect_selector_code == UINT32_MAX || term.indirect_selector_values.empty() || + term.indirect_selector_values.size() != term.indirect_selector_targets.size()) { + EmitDispatcherExit(state); + return; + } + + uint32_t selector_value = 0; + if (!EmitDispatcherLoadScalarCode(state, term.indirect_selector_code, selector_value)) { + EmitDispatcherExit(state); + return; + } + + uint32_t selected = ConstantU32(state, UINT32_MAX); + for (uint32_t i = 0; i < term.indirect_selector_values.size(); i++) { + const auto match = state->builder.AllocateId(); + state->builder.AddFunction({OpIEqual, state->bool_type, match, selector_value, + ConstantU32(state, term.indirect_selector_values[i])}); + const auto next = state->builder.AllocateId(); + state->builder.AddFunction( + {OpSelect, state->uint_type, next, match, + ConstantU32(state, DispatcherTargetValue(state, term.indirect_selector_targets[i])), + selected}); + selected = next; + } + state->builder.AddFunction({OpStore, state->dispatch_pc_variable, selected}); + state->builder.AddFunction({OpBranch, state->dispatch_after_switch_label}); +} + +void EmitDispatcherStoreIndirectTarget(EmitterState* state, const CFG::Terminator& term) { + if (term.indirect_selector_code != UINT32_MAX) { + EmitDispatcherStoreSelectorTarget(state, term); + return; + } + + if (term.indirect_pc_sgpr == UINT32_MAX) { + EmitDispatcherExit(state); + return; + } + + const auto pointer = + PointerForRegister(*state, IR::Register {IR::RegisterFile::Scalar, term.indirect_pc_sgpr}); + if (pointer == 0) { + EmitDispatcherExit(state); + return; + } + + const auto pc_value = state->builder.AllocateId(); + state->builder.AddFunction({OpLoad, state->uint_type, pc_value, pointer}); + uint32_t selected = ConstantU32(state, UINT32_MAX); + const auto count = std::min(term.indirect_target_pcs.size(), term.indirect_targets.size()); + for (uint32_t i = 0; i < count; i++) { + const auto match = state->builder.AllocateId(); + state->builder.AddFunction({OpIEqual, state->bool_type, match, pc_value, + ConstantU32(state, term.indirect_target_pcs[i])}); + const auto next = state->builder.AllocateId(); + state->builder.AddFunction( + {OpSelect, state->uint_type, next, match, + ConstantU32(state, DispatcherTargetValue(state, term.indirect_targets[i])), selected}); + selected = next; + } + state->builder.AddFunction({OpStore, state->dispatch_pc_variable, selected}); + state->builder.AddFunction({OpBranch, state->dispatch_after_switch_label}); +} + +void EmitDispatcherExit(EmitterState* state) { + state->builder.AddFunction( + {OpStore, state->dispatch_pc_variable, ConstantU32(state, UINT32_MAX)}); + state->builder.AddFunction({OpBranch, state->dispatch_after_switch_label}); +} + +void EmitDispatcherTerminator(EmitterState* state, const CFG::Terminator& term) { + switch (term.kind) { + case CFG::TerminatorKind::Branch: + if (BlockLabel(*state, term.true_block) == 0) { + EmitDispatcherExit(state); + return; + } + EmitDispatcherStoreTarget(state, term.true_block); + state->builder.AddFunction({OpBranch, state->dispatch_after_switch_label}); + return; + case CFG::TerminatorKind::ConditionalBranch: { + const auto condition = EmitBranchCondition(state, term.condition); + const auto selected = state->builder.AllocateId(); + state->builder.AddFunction( + {OpSelect, state->uint_type, selected, condition, + ConstantU32(state, DispatcherTargetValue(state, term.true_block)), + ConstantU32(state, DispatcherTargetValue(state, term.false_block))}); + state->builder.AddFunction({OpStore, state->dispatch_pc_variable, selected}); + state->builder.AddFunction({OpBranch, state->dispatch_after_switch_label}); + return; + } + case CFG::TerminatorKind::IndirectBranch: + EmitDispatcherStoreIndirectTarget(state, term); + return; + default: EmitDispatcherExit(state); return; + } +} + +void EmitDispatcherSwitch(EmitterState* state, const IR::Program& program) { + state->builder.AddFunction({OpLabel, state->dispatch_header_label}); + const auto pc_value = state->builder.AllocateId(); + state->builder.AddFunction({OpLoad, state->uint_type, pc_value, state->dispatch_pc_variable}); + const auto done = state->builder.AllocateId(); + state->builder.AddFunction( + {OpIEqual, state->bool_type, done, pc_value, ConstantU32(state, UINT32_MAX)}); + state->builder.AddFunction({OpLoopMerge, state->dispatch_merge_label, + state->dispatch_continue_label, LoopControlNone}); + state->builder.AddFunction( + {OpBranchConditional, done, state->dispatch_merge_label, state->dispatch_select_label}); + + state->builder.AddFunction({OpLabel, state->dispatch_select_label}); + state->builder.AddFunction( + {OpSelectionMerge, state->dispatch_after_switch_label, SelectionControlNone}); + std::vector switch_words = {OpSwitch, pc_value, state->dispatch_default_label}; + for (const auto& block: program.blocks) { + if (block.id >= state->reachable_blocks.size() || !state->reachable_blocks[block.id]) { + continue; + } + const auto label = BlockLabel(*state, block.id); + if (label != 0) { + switch_words.push_back(block.id); + switch_words.push_back(label); + } + } + state->builder.AddFunction(switch_words); + + state->builder.AddFunction({OpLabel, state->dispatch_default_label}); + EmitDispatcherExit(state); +} + +void EmitDispatcherBlocks(EmitterState* state, const IR::Program& program) { + for (const auto& block: program.blocks) { + if (block.id >= state->reachable_blocks.size() || !state->reachable_blocks[block.id]) { + continue; + } + state->builder.AddFunction({OpLabel, BlockLabel(*state, block.id)}); + for (const auto& inst: block.instructions) { + EmitInstruction(state, inst); + } + EmitDispatcherTerminator(state, block.terminator); + } +} + +void EmitDispatcherLoopTail(EmitterState* state) { + state->builder.AddFunction({OpLabel, state->dispatch_after_switch_label}); + state->builder.AddFunction({OpBranch, state->dispatch_continue_label}); + state->builder.AddFunction({OpLabel, state->dispatch_continue_label}); + state->builder.AddFunction({OpBranch, state->dispatch_header_label}); + state->builder.AddFunction({OpLabel, state->dispatch_merge_label}); + EmitReturn(state); +} + +void EmitDispatcherFunction(EmitterState* state, const IR::Program& program) { + state->builder.AddFunction({OpStore, state->dispatch_pc_variable, ConstantU32(state, 0)}); + state->builder.AddFunction({OpBranch, state->dispatch_header_label}); + EmitDispatcherSwitch(state, program); + EmitDispatcherBlocks(state, program); + EmitDispatcherLoopTail(state); +} + +void EmitFunction(EmitterState* state, const IR::Program& program) { + state->builder.AddFunction( + {OpFunction, state->void_type, state->main_func, FunctionControlNone, state->func_type}); + state->builder.AddFunction({OpLabel, state->entry_label}); + + EmitRegisterVariables(state); + EmitComputeInputRegisters(state); + EmitPixelInputRegisters(state); + EmitVertexInputRegisters(state); + if (state->dispatcher_fallback) { + EmitDispatcherFunction(state, program); + state->builder.AddFunction({OpFunctionEnd}); + return; + } + + const auto entry_block_label = BlockLabel(*state, 0); + if (entry_block_label != 0) { + state->builder.AddFunction({OpBranch, entry_block_label}); + } else { + EmitReturn(state); + } + + for (const auto& block: program.blocks) { + if (block.id >= state->reachable_blocks.size() || !state->reachable_blocks[block.id]) { + continue; + } + state->builder.AddFunction({OpLabel, BlockLabel(*state, block.id)}); + for (const auto& inst: block.instructions) { + EmitInstruction(state, inst); + } + EmitTerminator(state, block.terminator); + } + + state->builder.AddFunction({OpFunctionEnd}); +} + +} // namespace Libs::Graphics::ShaderRecompiler::Spirv::Emitter diff --git a/src/graphics/shader/recompiler/spirvEmitter/spirvEmitterExport.cpp b/src/graphics/shader/recompiler/spirvEmitter/spirvEmitterExport.cpp new file mode 100644 index 000000000..2c26cca16 --- /dev/null +++ b/src/graphics/shader/recompiler/spirvEmitter/spirvEmitterExport.cpp @@ -0,0 +1,154 @@ +#include "graphics/shader/recompiler/spirvEmitter/spirvEmitterInternal.h" + +namespace Libs::Graphics::ShaderRecompiler::Spirv::Emitter { + +uint32_t EmitExportComponentF32(EmitterState* state, const IR::Instruction& inst, + uint32_t component) { + const bool enabled = ((inst.export_info.en >> component) & 1u) != 0; + if (!enabled || component >= inst.src_count || component >= 4u) { + return ConstantF32(state, component == 3u ? 0x3f800000u : 0u); + } + return EmitFloatLoad(state, inst.src[component]); +} + +uint32_t EmitExportVec4F32(EmitterState* state, const IR::Instruction& inst) { + if (inst.export_info.compr) { + uint32_t components[4] = { + ConstantF32(state, 0u), + ConstantF32(state, 0u), + ConstantF32(state, 0u), + ConstantF32(state, 0x3f800000u), + }; + for (uint32_t pair_index = 0; pair_index < 2u && pair_index < inst.src_count; + pair_index++) { + const auto raw = EmitValueLoad(state, inst.src[pair_index]); + const auto unpacked = state->builder.AllocateId(); + state->builder.AddFunction({OpExtInst, state->vec2_float_type, unpacked, + state->glsl_std450, GlslUnpackHalf2x16, raw}); + for (uint32_t lane = 0; lane < 2u; lane++) { + const auto component = pair_index * 2u + lane; + if (((inst.export_info.en >> component) & 1u) == 0) { + continue; + } + components[component] = state->builder.AllocateId(); + state->builder.AddFunction( + {OpCompositeExtract, state->float_type, components[component], unpacked, lane}); + } + } + const auto vec = state->builder.AllocateId(); + state->builder.AddFunction({OpCompositeConstruct, state->vec4_float_type, vec, + components[0], components[1], components[2], components[3]}); + return vec; + } + + const auto x = EmitExportComponentF32(state, inst, 0); + const auto y = EmitExportComponentF32(state, inst, 1); + const auto z = EmitExportComponentF32(state, inst, 2); + const auto w = EmitExportComponentF32(state, inst, 3); + const auto vec = state->builder.AllocateId(); + state->builder.AddFunction({OpCompositeConstruct, state->vec4_float_type, vec, x, y, z, w}); + return vec; +} + +bool ExportWritesData(const IR::Instruction& inst) { + switch (inst.export_info.kind) { + case IR::ExportTargetKind::Null: + case IR::ExportTargetKind::Primitive: return false; + case IR::ExportTargetKind::MrtZ: return (inst.export_info.en & 0x5u) != 0; + default: return inst.export_info.en != 0; + } +} + +void EmitMrtZExport(EmitterState* state, const IR::Instruction& inst) { + if ((inst.export_info.en & 0x1u) != 0 && state->depth_variable != 0) { + state->builder.AddFunction( + {OpStore, state->depth_variable, EmitExportComponentF32(state, inst, 0)}); + } + + if ((inst.export_info.en & 0x4u) != 0 && state->sample_mask_variable != 0) { + const auto raw = + inst.src_count > 2u ? EmitValueLoad(state, inst.src[2]) : ConstantU32(state, 0); + const auto mask = state->builder.AllocateId(); + const auto ptr = state->builder.AllocateId(); + state->builder.AddFunction({OpBitcast, state->int_type, mask, raw}); + state->builder.AddFunction({OpAccessChain, state->ptr_output_int, ptr, + state->sample_mask_variable, ConstantU32(state, 0)}); + state->builder.AddFunction({OpStore, ptr, mask}); + } +} + +void EmitExport(EmitterState* state, const IR::Instruction& inst) { + if (inst.export_info.kind == IR::ExportTargetKind::Null || + inst.export_info.kind == IR::ExportTargetKind::Primitive) { + return; + } + + if (!ExportWritesData(inst)) { + return; + } + + if (inst.export_info.kind == IR::ExportTargetKind::MrtZ) { + EmitMrtZExport(state, inst); + return; + } + + const auto variable = OutputVariableForExport(*state, inst.export_info); + if (variable == 0) { + return; + } + + const auto value = EmitExportVec4F32(state, inst); + if (inst.export_info.kind == IR::ExportTargetKind::Position) { + const auto pointer = state->builder.AllocateId(); + state->builder.AddFunction({OpAccessChain, state->ptr_output_vec4_float, pointer, variable, + ConstantU32(state, 0)}); + state->builder.AddFunction({OpStore, pointer, value}); + return; + } + + state->builder.AddFunction({OpStore, variable, value}); +} + +bool ExportUsesPixelValidMask(const EmitterState& state, const IR::Instruction& inst) { + return state.stage == ShaderType::Pixel && inst.export_info.vm && state.needs_pixel_valid_mask; +} + +void EmitKillIfBoolFalse(EmitterState* state, uint32_t active) { + const auto kill_label = state->builder.AllocateId(); + const auto merge_label = state->builder.AllocateId(); + const auto inactive = state->builder.AllocateId(); + state->builder.AddFunction({OpLogicalNot, state->bool_type, inactive, active}); + state->builder.AddFunction({OpSelectionMerge, merge_label, SelectionControlNone}); + state->builder.AddFunction({OpBranchConditional, inactive, kill_label, merge_label}); + state->builder.AddFunction({OpLabel, kill_label}); + state->builder.AddFunction({OpKill}); + state->builder.AddFunction({OpLabel, merge_label}); +} + +void EmitUpdatePixelValidMask(EmitterState* state) { + if (state->pixel_valid_mask_variable == 0) { + return; + } + + const auto active = EmitExecActiveBool(state); + const auto value = state->builder.AllocateId(); + state->builder.AddFunction( + {OpSelect, state->uint_type, value, active, ConstantU32(state, 1), ConstantU32(state, 0)}); + state->builder.AddFunction({OpStore, state->pixel_valid_mask_variable, value}); +} + +void EmitKillIfPixelValidMaskInactive(EmitterState* state) { + if (state->pixel_valid_mask_variable == 0) { + return; + } + + const auto mask_value = state->builder.AllocateId(); + const auto active = state->builder.AllocateId(); + state->builder.AddFunction( + {OpLoad, state->uint_type, mask_value, state->pixel_valid_mask_variable}); + state->builder.AddFunction( + {OpINotEqual, state->bool_type, active, mask_value, ConstantU32(state, 0)}); + EmitKillIfBoolFalse(state, active); +} + +} // namespace Libs::Graphics::ShaderRecompiler::Spirv::Emitter diff --git a/src/graphics/shader/recompiler/spirvEmitter/spirvEmitterFlowOps.cpp b/src/graphics/shader/recompiler/spirvEmitter/spirvEmitterFlowOps.cpp new file mode 100644 index 000000000..e3afa8802 --- /dev/null +++ b/src/graphics/shader/recompiler/spirvEmitter/spirvEmitterFlowOps.cpp @@ -0,0 +1,381 @@ +#include "graphics/shader/recompiler/spirvEmitter/spirvEmitterInternal.h" + +namespace Libs::Graphics::ShaderRecompiler::Spirv::Emitter { + +void EmitMoveU32(EmitterState* state, const IR::Instruction& inst) { + const auto value = EmitValueLoad(state, inst.src[0]); + EmitStoreU32(state, inst.dst, value); +} + +void EmitMoveF32Bits(EmitterState* state, const IR::Instruction& inst) { + const auto value = EmitFloatLoad(state, inst.src[0]); + const auto bits = state->builder.AllocateId(); + state->builder.AddFunction({OpBitcast, state->uint_type, bits, value}); + EmitStoreU32(state, inst.dst, bits); +} + +uint32_t MoveRelRegisterLimit(const EmitterState& state, IR::Register base) { + uint32_t limit = base.index + 1u; + for (const auto& binding: state.registers) { + if (binding.reg.file == IR::RegisterFile::Vector) { + limit = std::max(limit, binding.reg.index + 1u); + } + } + return std::min(limit, 256u); +} + +void EmitMoveRelSourceU32(EmitterState* state, const IR::Instruction& inst) { + if (inst.src[0].kind != IR::OperandKind::Register || + inst.src[0].reg.file != IR::RegisterFile::Vector) { + EmitMoveU32(state, inst); + return; + } + + const auto base_reg = inst.src[0].reg; + const auto m0 = inst.src_count > 1u ? EmitValueLoad(state, inst.src[1]) : ConstantU32(state, 0); + auto selected = EmitRegisterLoad(state, base_reg); + const auto limit = MoveRelRegisterLimit(*state, base_reg); + for (uint32_t reg = base_reg.index + 1u; reg < limit; reg++) { + const auto candidate = EmitRegisterLoad(state, {IR::RegisterFile::Vector, reg}); + const auto match = state->builder.AllocateId(); + const auto next = state->builder.AllocateId(); + state->builder.AddFunction( + {OpIEqual, state->bool_type, match, m0, ConstantU32(state, reg - base_reg.index)}); + state->builder.AddFunction({OpSelect, state->uint_type, next, match, candidate, selected}); + selected = next; + } + EmitStoreU32(state, inst.dst, selected); +} + +void EmitMoveRelDestU32(EmitterState* state, const IR::Instruction& inst) { + if (inst.dst.kind != IR::OperandKind::Register || + inst.dst.reg.file != IR::RegisterFile::Vector) { + EmitMoveU32(state, inst); + return; + } + + const auto base_reg = inst.dst.reg; + const auto value = EmitValueLoad(state, inst.src[0]); + const auto m0 = inst.src_count > 1u ? EmitValueLoad(state, inst.src[1]) : ConstantU32(state, 0); + const auto exec_active = EmitExecActiveBool(state); + const auto limit = MoveRelRegisterLimit(*state, base_reg); + for (uint32_t reg = base_reg.index; reg < limit; reg++) { + const auto pointer = PointerForRegister(*state, {IR::RegisterFile::Vector, reg}); + if (pointer == 0) { + continue; + } + const auto old_value = state->builder.AllocateId(); + const auto match = state->builder.AllocateId(); + const auto write = state->builder.AllocateId(); + const auto merged = state->builder.AllocateId(); + state->builder.AddFunction({OpLoad, state->uint_type, old_value, pointer}); + state->builder.AddFunction( + {OpIEqual, state->bool_type, match, m0, ConstantU32(state, reg - base_reg.index)}); + state->builder.AddFunction({OpLogicalAnd, state->bool_type, write, exec_active, match}); + state->builder.AddFunction({OpSelect, state->uint_type, merged, write, value, old_value}); + state->builder.AddFunction({OpStore, pointer, merged}); + } +} + +void EmitMoveU64(EmitterState* state, const IR::Instruction& inst) { + if (state->per_invocation_masks && inst.dst.kind == IR::OperandKind::Register && + IsMaskRegisterFile(inst.dst.reg.file)) { + EmitPerInvocationMask(state, inst.dst, EmitLaneMaskOperandActiveBool(state, inst.src[0])); + return; + } + const auto low = EmitSequentialValueLoad(state, inst.src[0], 0); + const auto high = EmitSequentialValueLoad(state, inst.src[0], 1); + EmitStoreU32(state, inst.dst, low); + EmitStoreU32(state, OffsetRegisterOperand(inst.dst, 1), high); +} + +uint32_t EmitWqmLaneU32(EmitterState* state, uint32_t src) { + uint32_t ret = ConstantU32(state, 0); + for (uint32_t i = 0; i < 8u; i++) { + const auto mask = 0x0fu << (i * 4u); + const auto masked = state->builder.AllocateId(); + const auto non_zero = state->builder.AllocateId(); + const auto expanded = state->builder.AllocateId(); + const auto combined = state->builder.AllocateId(); + state->builder.AddFunction( + {OpBitwiseAnd, state->uint_type, masked, src, ConstantU32(state, mask)}); + state->builder.AddFunction( + {OpINotEqual, state->bool_type, non_zero, masked, ConstantU32(state, 0)}); + state->builder.AddFunction({OpSelect, state->uint_type, expanded, non_zero, + ConstantU32(state, mask), ConstantU32(state, 0)}); + state->builder.AddFunction({OpBitwiseOr, state->uint_type, combined, ret, expanded}); + ret = combined; + } + return ret; +} + +void EmitWqmB64(EmitterState* state, const IR::Instruction& inst) { + if (!state->per_invocation_masks && inst.dst.kind == IR::OperandKind::Register && + inst.dst.reg.file == IR::RegisterFile::Scalar) { + const auto low = EmitSequentialValueLoad(state, inst.src[0], 0); + const auto high = EmitSequentialValueLoad(state, inst.src[0], 1); + const auto ret_low = EmitWqmLaneU32(state, low); + const auto ret_high = EmitWqmLaneU32(state, high); + EmitStoreU32(state, inst.dst, ret_low); + EmitStoreU32(state, OffsetRegisterOperand(inst.dst, 1), ret_high); + return; + } + const auto ballot = state->builder.AllocateId(); + state->builder.AddFunction({OpGroupNonUniformBallot, state->vec4_uint_type, ballot, + ConstantU32(state, ScopeSubgroup), + EmitLaneMaskOperandActiveBool(state, inst.src[0])}); + const auto low = state->builder.AllocateId(); + const auto high = state->builder.AllocateId(); + state->builder.AddFunction({OpCompositeExtract, state->uint_type, low, ballot, 0}); + state->builder.AddFunction({OpCompositeExtract, state->uint_type, high, ballot, 1}); + const auto wqm_low = EmitWqmLaneU32(state, low); + const auto wqm_high = EmitWqmLaneU32(state, high); + const auto lane = EmitSubgroupLocalInvocationId(state); + const auto high_half = state->builder.AllocateId(); + const auto mask = state->builder.AllocateId(); + const auto bit_index = state->builder.AllocateId(); + const auto bit = state->builder.AllocateId(); + const auto hit = state->builder.AllocateId(); + const auto active = state->builder.AllocateId(); + state->builder.AddFunction( + {OpUGreaterThanEqual, state->bool_type, high_half, lane, ConstantU32(state, 32)}); + state->builder.AddFunction({OpSelect, state->uint_type, mask, high_half, wqm_high, wqm_low}); + state->builder.AddFunction( + {OpBitwiseAnd, state->uint_type, bit_index, lane, ConstantU32(state, 31)}); + state->builder.AddFunction( + {OpShiftLeftLogical, state->uint_type, bit, ConstantU32(state, 1), bit_index}); + state->builder.AddFunction({OpBitwiseAnd, state->uint_type, hit, mask, bit}); + state->builder.AddFunction({OpINotEqual, state->bool_type, active, hit, ConstantU32(state, 0)}); + if (state->per_invocation_masks) { + EmitPerInvocationMask(state, inst.dst, active); + } else { + const auto result = state->builder.AllocateId(); + state->builder.AddFunction({OpSelect, state->uint_type, result, active, + ConstantU32(state, 1), ConstantU32(state, 0)}); + EmitStoreU32(state, inst.dst, result); + EmitStoreU32(state, OffsetRegisterOperand(inst.dst, 1), ConstantU32(state, 0)); + } +} + +void EmitSaveexecPerInvocation(EmitterState* state, const IR::Instruction& inst) { + const auto old_exec = EmitExecActiveBool(state); + const auto src = EmitLaneMaskOperandActiveBool(state, inst.src[0]); + EmitPerInvocationMask(state, inst.dst, old_exec); + + uint32_t lhs = old_exec; + uint32_t rhs = src; + bool use_or = false; + switch (inst.saveexec_mode) { + case IR::SaveexecMode::And: break; + case IR::SaveexecMode::Andn1: rhs = EmitLogicalNotBool(state, src); break; + case IR::SaveexecMode::Orn2: + lhs = EmitLogicalNotBool(state, old_exec); + use_or = true; + break; + } + const auto result = + use_or ? EmitLogicalOrBool(state, lhs, rhs) : EmitLogicalAndBool(state, lhs, rhs); + EmitPerInvocationMask(state, MakeRegisterOperand(IR::RegisterFile::Exec, 0), result); + EmitStoreSccBool(state, result); +} + +void EmitSaveexecB32(EmitterState* state, const IR::Instruction& inst) { + if (state->per_invocation_masks) { + EmitSaveexecPerInvocation(state, inst); + return; + } + const auto old_low = EmitRegisterLoad(state, {IR::RegisterFile::Exec, 0}); + const auto src_low = EmitValueLoad(state, inst.src[0]); + + EmitStoreU32(state, inst.dst, old_low); + + uint32_t rhs_low = src_low; + if (inst.saveexec_mode == IR::SaveexecMode::Andn1) { + rhs_low = state->builder.AllocateId(); + state->builder.AddFunction({OpNot, state->uint_type, rhs_low, src_low}); + } + + const auto new_low = state->builder.AllocateId(); + state->builder.AddFunction({OpBitwiseAnd, state->uint_type, new_low, rhs_low, old_low}); + EmitStoreU32(state, MakeRegisterOperand(IR::RegisterFile::Exec, 0), new_low); + + if (state->wave_size == 32u) { + EmitStoreU32(state, MakeRegisterOperand(IR::RegisterFile::Exec, 1), ConstantU32(state, 0)); + } + + const auto cond = state->builder.AllocateId(); + const auto scc = state->builder.AllocateId(); + state->builder.AddFunction( + {OpINotEqual, state->bool_type, cond, new_low, ConstantU32(state, 0)}); + state->builder.AddFunction( + {OpSelect, state->uint_type, scc, cond, ConstantU32(state, 1), ConstantU32(state, 0)}); + EmitStoreU32(state, SccOperand(), scc); +} + +void EmitSaveexecB64(EmitterState* state, const IR::Instruction& inst) { + if (state->per_invocation_masks) { + EmitSaveexecPerInvocation(state, inst); + return; + } + const auto old_low = EmitRegisterLoad(state, {IR::RegisterFile::Exec, 0}); + const auto old_high = EmitRegisterLoad(state, {IR::RegisterFile::Exec, 1}); + const auto src_low = EmitSequentialValueLoad(state, inst.src[0], 0); + const auto src_high = EmitSequentialValueLoad(state, inst.src[0], 1); + + EmitStoreU32(state, inst.dst, old_low); + EmitStoreU32(state, OffsetRegisterOperand(inst.dst, 1), old_high); + + uint32_t lhs_low = old_low; + uint32_t lhs_high = old_high; + uint32_t rhs_low = src_low; + uint32_t rhs_high = src_high; + uint32_t opcode = OpBitwiseAnd; + if (inst.saveexec_mode == IR::SaveexecMode::Andn1) { + rhs_low = state->builder.AllocateId(); + rhs_high = state->builder.AllocateId(); + state->builder.AddFunction({OpNot, state->uint_type, rhs_low, src_low}); + state->builder.AddFunction({OpNot, state->uint_type, rhs_high, src_high}); + } + if (inst.saveexec_mode == IR::SaveexecMode::Orn2) { + lhs_low = state->builder.AllocateId(); + lhs_high = state->builder.AllocateId(); + state->builder.AddFunction({OpNot, state->uint_type, lhs_low, old_low}); + state->builder.AddFunction({OpNot, state->uint_type, lhs_high, old_high}); + opcode = OpBitwiseOr; + } + + const auto new_low = state->builder.AllocateId(); + const auto new_high = state->builder.AllocateId(); + state->builder.AddFunction({opcode, state->uint_type, new_low, lhs_low, rhs_low}); + state->builder.AddFunction({opcode, state->uint_type, new_high, lhs_high, rhs_high}); + EmitStoreU32(state, MakeRegisterOperand(IR::RegisterFile::Exec, 0), new_low); + EmitStoreU32(state, MakeRegisterOperand(IR::RegisterFile::Exec, 1), new_high); + + const auto active_new_high = state->wave_size == 32u ? ConstantU32(state, 0) : new_high; + const auto mask = state->builder.AllocateId(); + const auto cond = state->builder.AllocateId(); + const auto scc = state->builder.AllocateId(); + state->builder.AddFunction({OpBitwiseOr, state->uint_type, mask, new_low, active_new_high}); + state->builder.AddFunction({OpINotEqual, state->bool_type, cond, mask, ConstantU32(state, 0)}); + state->builder.AddFunction( + {OpSelect, state->uint_type, scc, cond, ConstantU32(state, 1), ConstantU32(state, 0)}); + EmitStoreU32(state, SccOperand(), scc); +} + +void EmitReadFirstLaneU32(EmitterState* state, const IR::Instruction& inst) { + const auto src = EmitValueLoad(state, inst.src[0]); + const auto active = EmitExecActiveBool(state); + const auto ballot = state->builder.AllocateId(); + const auto first_lane = state->builder.AllocateId(); + const auto first_value = state->builder.AllocateId(); + state->builder.AddFunction({OpGroupNonUniformBallot, state->vec4_uint_type, ballot, + ConstantU32(state, ScopeSubgroup), active}); + state->builder.AddFunction({OpGroupNonUniformBallotFindLSB, state->uint_type, first_lane, + ConstantU32(state, ScopeSubgroup), ballot}); + state->builder.AddFunction({OpGroupNonUniformShuffle, state->uint_type, first_value, + ConstantU32(state, ScopeSubgroup), src, first_lane}); + EmitStoreU32(state, inst.dst, first_value); +} + +uint32_t EmitLaneIndex(EmitterState* state, const IR::Operand& operand) { + const auto lane = state->builder.AllocateId(); + state->builder.AddFunction({OpBitwiseAnd, state->uint_type, lane, EmitValueLoad(state, operand), + ConstantU32(state, 63)}); + return lane; +} + +void EmitReadLaneU32(EmitterState* state, const IR::Instruction& inst) { + const auto src = EmitValueLoad(state, inst.src[0]); + const auto lane = EmitLaneIndex(state, inst.src[1]); + const auto value = state->builder.AllocateId(); + state->builder.AddFunction({OpGroupNonUniformShuffle, state->uint_type, value, + ConstantU32(state, ScopeSubgroup), src, lane}); + EmitStoreU32(state, inst.dst, value); +} + +void EmitWriteLaneU32(EmitterState* state, const IR::Instruction& inst) { + const auto value = EmitValueLoad(state, inst.src[0]); + const auto lane = EmitLaneIndex(state, inst.src[1]); + const auto subid = EmitSubgroupLocalInvocationId(state); + const auto hit = state->builder.AllocateId(); + const auto pointer = + inst.dst.kind == IR::OperandKind::Register ? PointerForRegister(*state, inst.dst.reg) : 0; + const auto old = pointer != 0 ? state->builder.AllocateId() : ConstantU32(state, 0); + const auto merged = state->builder.AllocateId(); + if (pointer != 0) { + state->builder.AddFunction({OpLoad, state->uint_type, old, pointer}); + } + state->builder.AddFunction({OpIEqual, state->bool_type, hit, subid, lane}); + state->builder.AddFunction({OpSelect, state->uint_type, merged, hit, value, old}); + if (pointer != 0) { + // RDNA2 V_WRITELANE_B32 ignores EXEC, unlike ordinary VGPR destination writes. + state->builder.AddFunction({OpStore, pointer, merged}); + } +} + +void EmitPermlaneB32(EmitterState* state, const IR::Instruction& inst, bool x16) { + const auto subid = EmitSubgroupLocalInvocationId(state); + const auto row = state->builder.AllocateId(); + const auto lane = state->builder.AllocateId(); + const auto lane8 = state->builder.AllocateId(); + const auto shift = state->builder.AllocateId(); + const auto upper = state->builder.AllocateId(); + const auto selected = state->builder.AllocateId(); + const auto index0 = state->builder.AllocateId(); + const auto index1 = state->builder.AllocateId(); + const auto target = state->builder.AllocateId(); + const auto shuffled = state->builder.AllocateId(); + const auto value = EmitValueLoad(state, inst.src[0]); + const auto src1 = EmitValueLoad(state, inst.src[1]); + const auto src2 = EmitValueLoad(state, inst.src[2]); + uint32_t row_value = row; + state->builder.AddFunction( + {OpBitwiseAnd, state->uint_type, row, subid, ConstantU32(state, 0xfffffff0u)}); + if (x16) { + row_value = state->builder.AllocateId(); + state->builder.AddFunction( + {OpBitwiseXor, state->uint_type, row_value, row, ConstantU32(state, 16)}); + } + state->builder.AddFunction( + {OpBitwiseAnd, state->uint_type, lane, subid, ConstantU32(state, 15)}); + state->builder.AddFunction( + {OpBitwiseAnd, state->uint_type, lane8, lane, ConstantU32(state, 7)}); + state->builder.AddFunction( + {OpShiftLeftLogical, state->uint_type, shift, lane8, ConstantU32(state, 2)}); + state->builder.AddFunction( + {OpUGreaterThanEqual, state->bool_type, upper, lane, ConstantU32(state, 8)}); + state->builder.AddFunction({OpSelect, state->uint_type, selected, upper, src2, src1}); + state->builder.AddFunction({OpShiftRightLogical, state->uint_type, index0, selected, shift}); + state->builder.AddFunction( + {OpBitwiseAnd, state->uint_type, index1, index0, ConstantU32(state, 15)}); + state->builder.AddFunction({OpBitwiseOr, state->uint_type, target, row_value, index1}); + state->builder.AddFunction({OpGroupNonUniformShuffle, state->uint_type, shuffled, + ConstantU32(state, ScopeSubgroup), value, target}); + uint32_t ret = shuffled; + if (!inst.dst.op_sel) { + const auto source_active = EmitLaneIndexActiveBool(state, target); + ret = state->builder.AllocateId(); + state->builder.AddFunction( + {OpSelect, state->uint_type, ret, source_active, shuffled, ConstantU32(state, 0)}); + } + EmitStoreU32(state, inst.dst, ret); +} + +void EmitControlNop(EmitterState* state, const IR::Instruction& inst) { + (void)state; + (void)inst; +} + +void EmitWaitcnt(EmitterState* state, const IR::Instruction& inst) { + (void)state; + (void)inst; +} + +void EmitBarrier(EmitterState* state, const IR::Instruction& inst) { + (void)inst; + const auto semantics = MemorySemanticsAcquireRelease | MemorySemanticsWorkgroupMemory; + state->builder.AddFunction({OpControlBarrier, ConstantU32(state, ScopeWorkgroup), + ConstantU32(state, ScopeWorkgroup), ConstantU32(state, semantics)}); +} + +} // namespace Libs::Graphics::ShaderRecompiler::Spirv::Emitter diff --git a/src/graphics/shader/recompiler/spirvEmitter/spirvEmitterImageHelpers.cpp b/src/graphics/shader/recompiler/spirvEmitter/spirvEmitterImageHelpers.cpp new file mode 100644 index 000000000..e361e3f80 --- /dev/null +++ b/src/graphics/shader/recompiler/spirvEmitter/spirvEmitterImageHelpers.cpp @@ -0,0 +1,314 @@ +#include "common/assert.h" +#include "graphics/shader/recompiler/spirvEmitter/spirvEmitterInternal.h" + +namespace Libs::Graphics::ShaderRecompiler::Spirv::Emitter { + +bool HasImageSampleFlag(const IR::Instruction& inst, uint32_t flag) { + return (inst.memory.image_sample_flags & flag) != 0; +} + +ImageSampleLayout MakeImageSampleLayout(const IR::Instruction& inst, ImageViewKind view) { + ImageSampleLayout layout; + uint32_t cursor = 0; + if (HasImageSampleFlag(inst, Decoder::ImageSampleFlagOffset)) { + layout.offset = cursor++; + } + if (HasImageSampleFlag(inst, Decoder::ImageSampleFlagBias)) { + layout.bias = cursor++; + } + if (HasImageSampleFlag(inst, Decoder::ImageSampleFlagCompare)) { + layout.dref = cursor++; + } + if (HasImageSampleFlag(inst, Decoder::ImageSampleFlagDerivative)) { + layout.grad_x = cursor; + cursor += 2u; + layout.grad_y = cursor; + cursor += 2u; + } + layout.coord = cursor; + cursor += ImageViewCoordinateComponents(view); + if (HasImageSampleFlag(inst, Decoder::ImageSampleFlagLod)) { + layout.lod = cursor++; + } + return layout; +} + +uint32_t EmitImageCoordF32(EmitterState* state, const IR::Instruction& inst, + const ImageSampleLayout& layout, ImageViewKind view) { + const auto x = EmitImageAddressFloatLoad(state, inst, inst.src[0], layout.coord); + const auto y = inst.memory.image_address_components > layout.coord + 1u + ? EmitImageAddressFloatLoad(state, inst, inst.src[0], layout.coord + 1u) + : EmitZeroF32(state); + const auto coord = state->builder.AllocateId(); + if (ImageViewCoordinateComponents(view) == 3u) { + const auto z = inst.memory.image_address_components > layout.coord + 2u + ? EmitImageAddressFloatLoad(state, inst, inst.src[0], layout.coord + 2u) + : EmitZeroF32(state); + state->builder.AddFunction({OpCompositeConstruct, state->vec3_float_type, coord, x, y, z}); + } else { + state->builder.AddFunction({OpCompositeConstruct, state->vec2_float_type, coord, x, y}); + } + return coord; +} + +uint32_t EmitImageLodF32(EmitterState* state, const IR::Instruction& inst, + const ImageSampleLayout& layout) { + if (HasImageSampleFlag(inst, Decoder::ImageSampleFlagLevelZero) || + layout.lod == NoImageComponent || inst.memory.image_address_components <= layout.lod) { + return EmitZeroF32(state); + } + return EmitImageAddressFloatLoad(state, inst, inst.src[0], layout.lod); +} + +uint32_t EmitImageDrefF32(EmitterState* state, const IR::Instruction& inst, + const ImageSampleLayout& layout) { + if (layout.dref == NoImageComponent || inst.memory.image_address_components <= layout.dref) { + return EmitZeroF32(state); + } + return EmitImageAddressFloatLoad(state, inst, inst.src[0], layout.dref); +} + +uint32_t EmitImageBiasF32(EmitterState* state, const IR::Instruction& inst, + const ImageSampleLayout& layout) { + if (layout.bias == NoImageComponent || inst.memory.image_address_components <= layout.bias) { + return EmitZeroF32(state); + } + return EmitImageAddressFloatLoad(state, inst, inst.src[0], layout.bias); +} + +uint32_t EmitImageGradientF32(EmitterState* state, const IR::Instruction& inst, + uint32_t first_component) { + const auto x = inst.memory.image_address_components > first_component + ? EmitImageAddressFloatLoad(state, inst, inst.src[0], first_component) + : EmitZeroF32(state); + const auto y = inst.memory.image_address_components > first_component + 1u + ? EmitImageAddressFloatLoad(state, inst, inst.src[0], first_component + 1u) + : EmitZeroF32(state); + const auto grad = state->builder.AllocateId(); + state->builder.AddFunction({OpCompositeConstruct, state->vec2_float_type, grad, x, y}); + return grad; +} + +uint32_t EmitImagePackedOffset2I32(EmitterState* state, const IR::Instruction& inst, + const ImageSampleLayout& layout) { + if (layout.offset == NoImageComponent || + inst.memory.image_address_components <= layout.offset) { + const auto zero = ConstantI32(state, 0); + const auto ret = state->builder.AllocateId(); + state->builder.AddFunction({OpCompositeConstruct, state->vec2_int_type, ret, zero, zero}); + return ret; + } + + const auto packed_bits = EmitImageAddressValueLoad(state, inst, inst.src[0], layout.offset); + const auto packed_i32 = state->builder.AllocateId(); + const auto offset_x = state->builder.AllocateId(); + const auto offset_y = state->builder.AllocateId(); + const auto offset = state->builder.AllocateId(); + state->builder.AddFunction({OpBitcast, state->int_type, packed_i32, packed_bits}); + state->builder.AddFunction({OpBitFieldSExtract, state->int_type, offset_x, packed_i32, + ConstantI32(state, 0), ConstantI32(state, 6)}); + state->builder.AddFunction({OpBitFieldSExtract, state->int_type, offset_y, packed_i32, + ConstantI32(state, 8), ConstantI32(state, 6)}); + state->builder.AddFunction( + {OpCompositeConstruct, state->vec2_int_type, offset, offset_x, offset_y}); + return offset; +} + +uint32_t EmitImageOffsetCoordF32(EmitterState* state, const IR::Instruction& inst, + const ImageSampleLayout& layout, uint32_t sampled_image, + uint32_t coord, ImageViewKind view) { + if (!HasImageSampleFlag(inst, Decoder::ImageSampleFlagOffset)) { + return coord; + } + + const auto offset_i32 = EmitImagePackedOffset2I32(state, inst, layout); + const auto offset_f32 = state->builder.AllocateId(); + const auto image = state->builder.AllocateId(); + const auto size_u32 = state->builder.AllocateId(); + const auto size_xy_u32 = + ImageViewCoordinateComponents(view) == 3u ? state->builder.AllocateId() : size_u32; + const auto size_f32 = state->builder.AllocateId(); + const auto normalized = state->builder.AllocateId(); + const auto adjusted = state->builder.AllocateId(); + state->builder.AddFunction({OpConvertSToF, state->vec2_float_type, offset_f32, offset_i32}); + state->builder.AddFunction( + {OpImage, ImageViewImageType(*state, view, inst.memory.kind == IR::ResourceKind::ImageUint), + image, sampled_image}); + state->builder.AddFunction({OpImageQuerySizeLod, ImageViewSizeType(*state, view), size_u32, + image, ConstantU32(state, 0)}); + if (ImageViewCoordinateComponents(view) == 3u) { + const auto width = state->builder.AllocateId(); + const auto height = state->builder.AllocateId(); + state->builder.AddFunction({OpCompositeExtract, state->uint_type, width, size_u32, 0}); + state->builder.AddFunction({OpCompositeExtract, state->uint_type, height, size_u32, 1}); + state->builder.AddFunction( + {OpCompositeConstruct, state->vec2_uint_type, size_xy_u32, width, height}); + } + state->builder.AddFunction({OpConvertUToF, state->vec2_float_type, size_f32, size_xy_u32}); + state->builder.AddFunction({OpFDiv, state->vec2_float_type, normalized, offset_f32, size_f32}); + if (ImageViewCoordinateComponents(view) == 2u) { + state->builder.AddFunction({OpFAdd, state->vec2_float_type, adjusted, coord, normalized}); + return adjusted; + } + + const auto x = state->builder.AllocateId(); + const auto y = state->builder.AllocateId(); + const auto z = state->builder.AllocateId(); + const auto normalized_x = state->builder.AllocateId(); + const auto normalized_y = state->builder.AllocateId(); + const auto adjusted_x = state->builder.AllocateId(); + const auto adjusted_y = state->builder.AllocateId(); + state->builder.AddFunction({OpCompositeExtract, state->float_type, x, coord, 0}); + state->builder.AddFunction({OpCompositeExtract, state->float_type, y, coord, 1}); + state->builder.AddFunction({OpCompositeExtract, state->float_type, z, coord, 2}); + state->builder.AddFunction( + {OpCompositeExtract, state->float_type, normalized_x, normalized, 0}); + state->builder.AddFunction( + {OpCompositeExtract, state->float_type, normalized_y, normalized, 1}); + state->builder.AddFunction({OpFAdd, state->float_type, adjusted_x, x, normalized_x}); + state->builder.AddFunction({OpFAdd, state->float_type, adjusted_y, y, normalized_y}); + state->builder.AddFunction( + {OpCompositeConstruct, state->vec3_float_type, adjusted, adjusted_x, adjusted_y, z}); + return adjusted; +} + +uint32_t EmitImageCoordU32(EmitterState* state, const IR::Instruction& inst, ImageViewKind view) { + const auto x = EmitImageAddressValueLoad(state, inst, inst.src[1], 0); + const auto y = inst.memory.image_address_components > 1u + ? EmitImageAddressValueLoad(state, inst, inst.src[1], 1) + : ConstantU32(state, 0); + const auto coord = state->builder.AllocateId(); + if (ImageViewCoordinateComponents(view) == 3u) { + const auto z = inst.memory.image_address_components > 2u + ? EmitImageAddressValueLoad(state, inst, inst.src[1], 2) + : ConstantU32(state, 0); + state->builder.AddFunction({OpCompositeConstruct, state->vec3_uint_type, coord, x, y, z}); + } else { + state->builder.AddFunction({OpCompositeConstruct, state->vec2_uint_type, coord, x, y}); + } + return coord; +} + +uint32_t EmitImageLoadCoordU32(EmitterState* state, const IR::Instruction& inst, + ImageViewKind view) { + const auto x = EmitImageAddressValueLoad(state, inst, inst.src[0], 0); + const auto y = inst.memory.image_address_components > 1u + ? EmitImageAddressValueLoad(state, inst, inst.src[0], 1) + : ConstantU32(state, 0); + const auto coord = state->builder.AllocateId(); + if (ImageViewCoordinateComponents(view) == 3u) { + const auto z = inst.memory.image_address_components > 2u + ? EmitImageAddressValueLoad(state, inst, inst.src[0], 2) + : ConstantU32(state, 0); + state->builder.AddFunction({OpCompositeConstruct, state->vec3_uint_type, coord, x, y, z}); + } else { + state->builder.AddFunction({OpCompositeConstruct, state->vec2_uint_type, coord, x, y}); + } + return coord; +} + +uint32_t EmitImageMipLodU32(EmitterState* state, const IR::Instruction& inst, + const IR::Operand& address, ImageViewKind view) { + if (!inst.memory.image_has_mip || inst.memory.image_address_components == 0u) { + return ConstantU32(state, 0); + } + const auto lod_component = ImageViewCoordinateComponents(view); + if (inst.memory.image_address_components <= lod_component) { + return ConstantU32(state, 0); + } + return EmitImageAddressValueLoad(state, inst, address, lod_component); +} + +uint32_t EmitImageQueryCoordF32(EmitterState* state, const IR::Instruction& inst, + ImageViewKind view) { + const auto x = EmitImageAddressFloatLoad(state, inst, inst.src[0], 0); + const auto y = inst.memory.image_address_components > 1u + ? EmitImageAddressFloatLoad(state, inst, inst.src[0], 1) + : EmitZeroF32(state); + const auto coord = state->builder.AllocateId(); + if (ImageViewCoordinateComponents(view) == 3u) { + const auto z = inst.memory.image_address_components > 2u + ? EmitImageAddressFloatLoad(state, inst, inst.src[0], 2) + : EmitZeroF32(state); + state->builder.AddFunction({OpCompositeConstruct, state->vec3_float_type, coord, x, y, z}); + } else { + state->builder.AddFunction({OpCompositeConstruct, state->vec2_float_type, coord, x, y}); + } + return coord; +} + +uint32_t DmaskComponentIndex(uint32_t dmask, uint32_t component) { + uint32_t index = 0; + for (uint32_t i = 0; i < component; i++) { + index += (dmask >> i) & 1u; + } + return index; +} + +uint32_t EmitImageStoreComponentF32(EmitterState* state, const IR::Instruction& inst, + uint32_t component) { + const auto dmask = inst.memory.dmask != 0 ? inst.memory.dmask : 1u; + if (((dmask >> component) & 1u) == 0) { + return EmitZeroF32(state); + } + return EmitSequentialFloatLoad(state, inst.src[0], DmaskComponentIndex(dmask, component)); +} + +uint32_t InverseStorageSwizzleComponent(uint32_t swizzle, uint32_t component) { + const auto target = 4u + component; + for (uint32_t source = 0; source < 4u; source++) { + if (((swizzle >> (source * 3u)) & 0x7u) == target) { + return source; + } + } + return UINT32_MAX; +} + +uint32_t StorageImageSwizzle(const EmitterState& state, const IR::Instruction& inst) { + if (state.program == nullptr || inst.memory.resource >= state.program->info.images.size()) { + EXIT("storage image instruction has no specialized swizzle\n"); + } + return state.program->info.images[inst.memory.resource].storage_swizzle; +} + +uint32_t EmitImageStoreTexelF32(EmitterState* state, const IR::Instruction& inst) { + const auto swizzle = StorageImageSwizzle(*state, inst); + const auto component_value = [&](uint32_t component) { + const auto source = InverseStorageSwizzleComponent(swizzle, component); + return source < 4u ? EmitImageStoreComponentF32(state, inst, source) : EmitZeroF32(state); + }; + const auto x = component_value(0); + const auto y = component_value(1); + const auto z = component_value(2); + const auto w = component_value(3); + const auto texel = state->builder.AllocateId(); + state->builder.AddFunction({OpCompositeConstruct, state->vec4_float_type, texel, x, y, z, w}); + return texel; +} + +uint32_t EmitImageStoreComponentU32(EmitterState* state, const IR::Instruction& inst, + uint32_t component) { + const auto dmask = inst.memory.dmask != 0 ? inst.memory.dmask : 1u; + if (((dmask >> component) & 1u) == 0) { + return ConstantU32(state, 0); + } + return EmitSequentialValueLoad(state, inst.src[0], DmaskComponentIndex(dmask, component)); +} + +uint32_t EmitImageStoreTexelU32(EmitterState* state, const IR::Instruction& inst) { + const auto swizzle = StorageImageSwizzle(*state, inst); + const auto component_value = [&](uint32_t component) { + const auto source = InverseStorageSwizzleComponent(swizzle, component); + return source < 4u ? EmitImageStoreComponentU32(state, inst, source) + : ConstantU32(state, 0); + }; + const auto x = component_value(0); + const auto y = component_value(1); + const auto z = component_value(2); + const auto w = component_value(3); + const auto texel = state->builder.AllocateId(); + state->builder.AddFunction({OpCompositeConstruct, state->vec4_uint_type, texel, x, y, z, w}); + return texel; +} + +} // namespace Libs::Graphics::ShaderRecompiler::Spirv::Emitter diff --git a/src/graphics/shader/recompiler/spirvEmitter/spirvEmitterImageOps.cpp b/src/graphics/shader/recompiler/spirvEmitter/spirvEmitterImageOps.cpp new file mode 100644 index 000000000..c776dff6d --- /dev/null +++ b/src/graphics/shader/recompiler/spirvEmitter/spirvEmitterImageOps.cpp @@ -0,0 +1,361 @@ +#include "graphics/shader/recompiler/spirvEmitter/spirvEmitterInternal.h" + +namespace Libs::Graphics::ShaderRecompiler::Spirv::Emitter { + +namespace { + +uint32_t ConstantVec2I32(EmitterState* state, int32_t x, int32_t y) { + const auto value = state->builder.AllocateId(); + state->builder.AddType({OpConstantComposite, state->vec2_int_type, value, ConstantI32(state, x), + ConstantI32(state, y)}); + return value; +} + +uint32_t ConstantImageGatherHorizontalOffsets(EmitterState* state) { + const auto offset0 = ConstantVec2I32(state, -1, 0); + const auto offset1 = ConstantVec2I32(state, 0, 0); + const auto offset2 = ConstantVec2I32(state, 1, 0); + const auto offset3 = ConstantVec2I32(state, 2, 0); + const auto type = state->builder.AllocateId(); + state->builder.AddType({OpTypeArray, type, state->vec2_int_type, ConstantU32(state, 4)}); + const auto value = state->builder.AllocateId(); + state->builder.AddType({OpConstantComposite, type, value, offset0, offset1, offset2, offset3}); + return value; +} + +uint32_t LoadStorageImageDescriptorAtIndex(EmitterState* state, uint32_t resource, + uint32_t array_index, bool uint_image, + ImageViewKind view) { + uint32_t pointer_type = + uint_image ? state->ptr_uniform_storage_image_uint : state->ptr_uniform_storage_image; + uint32_t variable = + uint_image ? state->storage_image_uint_variable : state->storage_image_variable; + uint32_t image_type = uint_image ? state->storage_image_uint_type : state->storage_image_type; + IR::DescriptorBindingKind kind = uint_image ? IR::DescriptorBindingKind::StorageUint2D + : IR::DescriptorBindingKind::Storage2D; + switch (view) { + case ImageViewKind::Dim2D: break; + case ImageViewKind::Dim2DArray: + pointer_type = uint_image ? state->ptr_uniform_storage_image_uint_2d_array + : state->ptr_uniform_storage_image_2d_array; + variable = uint_image ? state->storage_image_uint_2d_array_variable + : state->storage_image_2d_array_variable; + image_type = uint_image ? state->storage_image_uint_2d_array_type + : state->storage_image_2d_array_type; + kind = uint_image ? IR::DescriptorBindingKind::StorageUint2DArray + : IR::DescriptorBindingKind::Storage2DArray; + break; + case ImageViewKind::Dim3D: + pointer_type = uint_image ? state->ptr_uniform_storage_image_uint_3d + : state->ptr_uniform_storage_image_3d; + variable = uint_image ? state->storage_image_uint_3d_variable + : state->storage_image_3d_variable; + image_type = + uint_image ? state->storage_image_uint_3d_type : state->storage_image_3d_type; + kind = uint_image ? IR::DescriptorBindingKind::StorageUint3D + : IR::DescriptorBindingKind::Storage3D; + break; + } + const auto pointer = + DescriptorElementPointer(state, pointer_type, variable, array_index, kind, resource, + "storage image descriptor array was not emitted"); + const auto image = state->builder.AllocateId(); + state->builder.AddFunction({OpLoad, image_type, image, pointer}); + return image; +} + +} // namespace + +uint32_t EmitImageGetResinfoComponent(EmitterState* state, uint32_t image, uint32_t size, + uint32_t component_index, ImageViewKind view) { + switch (component_index) { + case 0: { + const auto width = state->builder.AllocateId(); + state->builder.AddFunction({OpCompositeExtract, state->uint_type, width, size, 0}); + return width; + } + case 1: { + const auto height = state->builder.AllocateId(); + state->builder.AddFunction({OpCompositeExtract, state->uint_type, height, size, 1}); + return height; + } + case 2: + if (ImageViewCoordinateComponents(view) == 3u) { + const auto depth = state->builder.AllocateId(); + state->builder.AddFunction({OpCompositeExtract, state->uint_type, depth, size, 2}); + return depth; + } + return ConstantU32(state, 1); + case 3: { + const auto levels = state->builder.AllocateId(); + state->builder.AddFunction({OpImageQueryLevels, state->uint_type, levels, image}); + return levels; + } + default: return ConstantU32(state, 0); + } +} + +void EmitImageGetResinfo(EmitterState* state, const IR::Instruction& inst) { + const auto view = SampledImageViewKind(*state, inst.memory, inst.pc); + const auto image = LoadSampledImageDescriptor(state, inst.memory, inst.pc, view); + const auto mip_level = EmitValueLoad(state, inst.src[0]); + + const auto size = state->builder.AllocateId(); + state->builder.AddFunction( + {OpImageQuerySizeLod, ImageViewSizeType(*state, view), size, image, mip_level}); + + const auto dmask = inst.memory.dmask != 0 ? inst.memory.dmask : 1u; + uint32_t dst_index = 0; + for (uint32_t component_index = 0; component_index < 4u; component_index++) { + if (((dmask >> component_index) & 1u) == 0) { + continue; + } + const auto component = + EmitImageGetResinfoComponent(state, image, size, component_index, view); + EmitStoreU32(state, OffsetRegisterOperand(inst.dst, dst_index++), component); + } +} + +void EmitImageGetLod(EmitterState* state, const IR::Instruction& inst) { + const auto view = SampledImageViewKind(*state, inst.memory, inst.pc); + const auto sampled_image = MakeSampledImage(state, inst.memory, inst.pc, view); + + const auto lod = state->builder.AllocateId(); + state->builder.AddFunction({OpImageQueryLod, state->vec2_float_type, lod, sampled_image, + EmitImageQueryCoordF32(state, inst, view)}); + + const auto dmask = inst.memory.dmask != 0 ? inst.memory.dmask : 1u; + uint32_t dst_index = 0; + for (uint32_t component_index = 0; component_index < 2u; component_index++) { + if (((dmask >> component_index) & 1u) == 0) { + continue; + } + const auto component = state->builder.AllocateId(); + const auto bits = state->builder.AllocateId(); + state->builder.AddFunction( + {OpCompositeExtract, state->float_type, component, lod, component_index}); + state->builder.AddFunction({OpBitcast, state->uint_type, bits, component}); + EmitStoreU32(state, OffsetRegisterOperand(inst.dst, dst_index++), bits); + } +} + +void EmitImageLoad(EmitterState* state, const IR::Instruction& inst) { + const auto view = SampledImageViewKind(*state, inst.memory, inst.pc); + const auto image = LoadSampledImageDescriptor(state, inst.memory, inst.pc, view); + const bool integer = inst.memory.kind == IR::ResourceKind::ImageUint; + + const auto color = state->builder.AllocateId(); + state->builder.AddFunction( + {OpImageFetch, integer ? state->vec4_uint_type : state->vec4_float_type, color, image, + EmitImageLoadCoordU32(state, inst, view), ImageOperandsLodMask, + EmitImageMipLodU32(state, inst, inst.src[0], view)}); + + const auto dmask = inst.memory.dmask != 0 ? inst.memory.dmask : 1u; + uint32_t dst_index = 0; + for (uint32_t component_index = 0; component_index < 4u; component_index++) { + if (((dmask >> component_index) & 1u) == 0) { + continue; + } + const auto component = state->builder.AllocateId(); + state->builder.AddFunction({OpCompositeExtract, + integer ? state->uint_type : state->float_type, component, + color, component_index}); + const auto bits = integer ? component : state->builder.AllocateId(); + if (!integer) { + state->builder.AddFunction({OpBitcast, state->uint_type, bits, component}); + } + EmitStoreU32(state, OffsetRegisterOperand(inst.dst, dst_index++), bits); + } +} + +void EmitImageStore(EmitterState* state, const IR::Instruction& inst) { + const auto uint_image = inst.memory.kind == IR::ResourceKind::StorageImageUint; + const auto view = StorageImageViewKind(*state, inst.memory, uint_image, inst.pc); + const auto binding = ResourceForDescriptor( + *state, + uint_image + ? (view == ImageViewKind::Dim2DArray ? IR::DescriptorBindingKind::StorageUint2DArray + : view == ImageViewKind::Dim3D ? IR::DescriptorBindingKind::StorageUint3D + : IR::DescriptorBindingKind::StorageUint2D) + : (view == ImageViewKind::Dim2DArray ? IR::DescriptorBindingKind::Storage2DArray + : view == ImageViewKind::Dim3D ? IR::DescriptorBindingKind::Storage3D + : IR::DescriptorBindingKind::Storage2D), + inst.memory.resource); + const auto image = LoadStorageImageDescriptorAtIndex(state, inst.memory.resource, + binding.array_index, uint_image, view); + + state->builder.AddFunction( + {OpImageWrite, image, EmitImageCoordU32(state, inst, view), + uint_image ? EmitImageStoreTexelU32(state, inst) : EmitImageStoreTexelF32(state, inst)}); +} + +void EmitImageSampleResult(EmitterState* state, const IR::Instruction& inst, uint32_t sample, + bool dref, bool integer) { + if (dref) { + const auto bits = state->builder.AllocateId(); + state->builder.AddFunction({OpBitcast, state->uint_type, bits, sample}); + for (uint32_t i = 0; i < inst.memory.data_dwords; i++) { + EmitStoreU32(state, OffsetRegisterOperand(inst.dst, i), bits); + } + return; + } + + const auto dmask = inst.memory.dmask != 0 ? inst.memory.dmask : 1u; + uint32_t dst_index = 0; + for (uint32_t component_index = 0; component_index < 4u; component_index++) { + if (((dmask >> component_index) & 1u) == 0) { + continue; + } + const auto component = state->builder.AllocateId(); + state->builder.AddFunction({OpCompositeExtract, + integer ? state->uint_type : state->float_type, component, + sample, component_index}); + const auto bits = integer ? component : state->builder.AllocateId(); + if (!integer) { + state->builder.AddFunction({OpBitcast, state->uint_type, bits, component}); + } + EmitStoreU32(state, OffsetRegisterOperand(inst.dst, dst_index++), bits); + } +} + +bool ImageSampleNeedsExplicitLod(const EmitterState& state, const IR::Instruction& inst) { + // RDNA2 plain IMAGE_SAMPLE is derivative-based in pixel shaders. Do not translate it + // to Lod(0): only _L/_LZ/_D variants supply explicit LOD/gradient information. + // SPIR-V implicit-lod sampling is fragment-only, so non-pixel stages still need + // an explicit operand even for the unsuffixed sample opcode. + if (HasImageSampleFlag(inst, Decoder::ImageSampleFlagDerivative) || + HasImageSampleFlag(inst, Decoder::ImageSampleFlagLod) || + HasImageSampleFlag(inst, Decoder::ImageSampleFlagLevelZero)) { + return true; + } + return state.stage != ShaderType::Pixel; +} + +void AddImageSampleOperands(EmitterState* state, const IR::Instruction& inst, + const ImageSampleLayout& layout, bool explicit_lod, + std::vector* words) { + if (words == nullptr) { + return; + } + uint32_t mask = 0; + std::vector operands; + + // Keep the SPIR-V operand class aligned with the RDNA2 opcode suffix. Grad/Lod + // operands belong to explicit-lod instructions; bias belongs to implicit-lod + // sampling because it modifies the hardware-computed LOD. + if (HasImageSampleFlag(inst, Decoder::ImageSampleFlagDerivative)) { + mask |= ImageOperandsGradMask; + operands.push_back(EmitImageGradientF32(state, inst, layout.grad_x)); + operands.push_back(EmitImageGradientF32(state, inst, layout.grad_y)); + } else if (explicit_lod) { + mask |= ImageOperandsLodMask; + operands.push_back(EmitImageLodF32(state, inst, layout)); + } else if (HasImageSampleFlag(inst, Decoder::ImageSampleFlagBias)) { + mask |= ImageOperandsBiasMask; + operands.push_back(EmitImageBiasF32(state, inst, layout)); + } + if (mask != 0) { + words->push_back(mask); + words->insert(words->end(), operands.begin(), operands.end()); + } +} + +uint32_t ImageSampleOpcode(const EmitterState& state, const IR::Instruction& inst) { + const auto flags = inst.memory.image_sample_flags; + const bool dref = (flags & Decoder::ImageSampleFlagCompare) != 0; + const bool explicit_lod = ImageSampleNeedsExplicitLod(state, inst); + if (explicit_lod) { + return dref ? OpImageSampleDrefExplicitLod : OpImageSampleExplicitLod; + } + return dref ? OpImageSampleDrefImplicitLod : OpImageSampleImplicitLod; +} + +void EmitImageSample(EmitterState* state, const IR::Instruction& inst) { + const auto view = SampledImageViewKind(*state, inst.memory, inst.pc); + const auto sampled_image = MakeSampledImage(state, inst.memory, inst.pc, view); + + const auto layout = MakeImageSampleLayout(inst, view); + const auto base_coord = EmitImageCoordF32(state, inst, layout, view); + const auto sample = state->builder.AllocateId(); + const auto dref = HasImageSampleFlag(inst, Decoder::ImageSampleFlagCompare); + const bool integer = inst.memory.kind == IR::ResourceKind::ImageUint; + const auto result_type = dref ? state->float_type + : integer ? state->vec4_uint_type + : state->vec4_float_type; + const auto explicit_lod = ImageSampleNeedsExplicitLod(*state, inst); + const auto opcode = ImageSampleOpcode(*state, inst); + const auto coord = + EmitImageOffsetCoordF32(state, inst, layout, sampled_image, base_coord, view); + + std::vector words = {opcode, result_type, sample, sampled_image, coord}; + if (dref) { + words.push_back(EmitImageDrefF32(state, inst, layout)); + } + AddImageSampleOperands(state, inst, layout, explicit_lod, &words); + state->builder.AddFunction(words); + EmitImageSampleResult(state, inst, sample, dref, integer); +} + +uint32_t ImageGatherComponent(uint32_t dmask) { + switch (dmask) { + case 0x2u: return 1; + case 0x4u: return 2; + case 0x8u: return 3; + default: return 0; + } +} + +void AddImageGatherOperands(EmitterState* state, const IR::Instruction& inst, + const ImageSampleLayout& layout, std::vector* words) { + if (words == nullptr) { + return; + } + if (HasImageSampleFlag(inst, Decoder::ImageSampleFlagGatherHorizontal)) { + words->push_back(ImageOperandsConstOffsetsMask); + words->push_back(ConstantImageGatherHorizontalOffsets(state)); + return; + } + if (HasImageSampleFlag(inst, Decoder::ImageSampleFlagOffset)) { + words->push_back(ImageOperandsOffsetMask); + words->push_back(EmitImagePackedOffset2I32(state, inst, layout)); + } +} + +void EmitImageGather4(EmitterState* state, const IR::Instruction& inst) { + const auto view = SampledImageViewKind(*state, inst.memory, inst.pc); + const auto sampled_image = MakeSampledImage(state, inst.memory, inst.pc, view); + + const auto layout = MakeImageSampleLayout(inst, view); + const auto coord = EmitImageCoordF32(state, inst, layout, view); + const auto texels = state->builder.AllocateId(); + const auto dref = HasImageSampleFlag(inst, Decoder::ImageSampleFlagCompare); + const bool integer = inst.memory.kind == IR::ResourceKind::ImageUint; + const auto result_type = integer ? state->vec4_uint_type : state->vec4_float_type; + std::vector words = + dref ? std::vector {OpImageDrefGather, + state->vec4_float_type, + texels, + sampled_image, + coord, + EmitImageDrefF32(state, inst, layout)} + : std::vector { + OpImageGather, result_type, + texels, sampled_image, + coord, ConstantU32(state, ImageGatherComponent(inst.memory.dmask))}; + AddImageGatherOperands(state, inst, layout, &words); + state->builder.AddFunction(words); + + for (uint32_t i = 0; i < inst.memory.data_dwords; i++) { + const auto component = state->builder.AllocateId(); + state->builder.AddFunction({OpCompositeExtract, + integer ? state->uint_type : state->float_type, component, + texels, i}); + const auto bits = integer ? component : state->builder.AllocateId(); + if (!integer) { + state->builder.AddFunction({OpBitcast, state->uint_type, bits, component}); + } + EmitStoreU32(state, OffsetRegisterOperand(inst.dst, i), bits); + } +} + +} // namespace Libs::Graphics::ShaderRecompiler::Spirv::Emitter diff --git a/src/graphics/shader/recompiler/spirvEmitter/spirvEmitterInternal.h b/src/graphics/shader/recompiler/spirvEmitter/spirvEmitterInternal.h new file mode 100644 index 000000000..678d3edcd --- /dev/null +++ b/src/graphics/shader/recompiler/spirvEmitter/spirvEmitterInternal.h @@ -0,0 +1,1555 @@ +#ifndef EMULATOR_INCLUDE_EMULATOR_GRAPHICS_SHADER_RECOMPILER_SPIRVEMITTER_INTERNAL_H_ +#define EMULATOR_INCLUDE_EMULATOR_GRAPHICS_SHADER_RECOMPILER_SPIRVEMITTER_INTERNAL_H_ + +#include "common/common.h" +#include "common/stringUtils.h" +#include "graphics/shader/recompiler/BindingLayout.h" +#include "graphics/shader/recompiler/BufferFormat.h" +#include "graphics/shader/recompiler/ResourceMaterialization.h" +#include "graphics/shader/recompiler/ShaderIR.h" +#include "graphics/shader/recompiler/SpirvBuilder.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace Libs::Graphics::ShaderRecompiler::Spirv::Emitter { + +enum : uint32_t { + ExecutionModelVertex = 0, + ExecutionModelFragment = 4, + ExecutionModelGLCompute = 5, + ExecutionModeOriginUpperLeft = 7, + ExecutionModeEarlyFragmentTests = 9, + ExecutionModeDepthReplacing = 12, + ExecutionModeLocalSize = 17, + ExecutionModeDerivativeGroupQuadsKHR = 5289, + AddressingModelLogical = 0, + MemoryModelGLSL450 = 1, + CapabilityShader = 1, + CapabilityImageGatherExtended = 25, + CapabilityImageQuery = 50, + CapabilityStorageImageReadWithoutFormat = 55, + CapabilityStorageImageWriteWithoutFormat = 56, + CapabilityGroupNonUniform = 61, + CapabilityGroupNonUniformBallot = 64, + CapabilityGroupNonUniformShuffle = 65, + CapabilityComputeDerivativeGroupQuadsKHR = 5288, + StorageClassUniformConstant = 0, + StorageClassInput = 1, + StorageClassOutput = 3, + StorageClassWorkgroup = 4, + StorageClassFunction = 7, + StorageClassPushConstant = 9, + StorageClassImage = 11, + StorageClassStorageBuffer = 12, + FunctionControlNone = 0, + SelectionControlNone = 0, + LoopControlNone = 0, +}; + +enum : uint32_t { + DecorationBlock = 2, + DecorationBuiltIn = 11, + DecorationNoPerspective = 13, + DecorationFlat = 14, + DecorationLocation = 30, + DecorationArrayStride = 6, + DecorationBinding = 33, + DecorationDescriptorSet = 34, + DecorationOffset = 35, +}; + +enum : uint32_t { + BuiltInPosition = 0, + BuiltInFragCoord = 15, + BuiltInFrontFacing = 17, + BuiltInSampleMask = 20, + BuiltInFragDepth = 22, + BuiltInWorkgroupId = 26, + BuiltInLocalInvocationId = 27, + BuiltInGlobalInvocationId = 28, + BuiltInLocalInvocationIndex = 29, + BuiltInSubgroupLocalInvocationId = 41, + BuiltInVertexIndex = 42, + BuiltInInstanceIndex = 43, +}; + +enum : uint32_t { + Dim3D = 2, + Dim2D = 1, + ImageFormatUnknown = 0, + ImageFormatRgba32f = 1, + ImageFormatR32ui = 33, +}; + +enum : uint32_t { + ImageOperandsBiasMask = 0x00000001u, + ImageOperandsLodMask = 0x00000002u, + ImageOperandsGradMask = 0x00000004u, + ImageOperandsOffsetMask = 0x00000010u, + ImageOperandsConstOffsetsMask = 0x00000020u, +}; + +enum : uint32_t { + ScopeDevice = 1, + ScopeWorkgroup = 2, + ScopeSubgroup = 3, + MemorySemanticsNone = 0, + MemorySemanticsAcquireRelease = 0x00000008u, + MemorySemanticsUniformMemory = 0x00000040u, + MemorySemanticsWorkgroupMemory = 0x00000100u, +}; + +enum : uint32_t { + OpExtInst = 12, + OpTypeVoid = 19, + OpTypeBool = 20, + OpTypeInt = 21, + OpTypeFloat = 22, + OpTypeVector = 23, + OpTypeImage = 25, + OpTypeSampler = 26, + OpTypeSampledImage = 27, + OpTypeArray = 28, + OpTypeRuntimeArray = 29, + OpTypeStruct = 30, + OpTypePointer = 32, + OpTypeFunction = 33, + OpConstant = 43, + OpConstantComposite = 44, + OpFunction = 54, + OpFunctionEnd = 56, + OpVariable = 59, + OpImageTexelPointer = 60, + OpLoad = 61, + OpStore = 62, + OpAccessChain = 65, + OpArrayLength = 68, + OpDecorate = 71, + OpMemberDecorate = 72, + OpCompositeConstruct = 80, + OpCompositeExtract = 81, + OpSampledImage = 86, + OpImageSampleImplicitLod = 87, + OpImageSampleExplicitLod = 88, + OpImageSampleDrefImplicitLod = 89, + OpImageSampleDrefExplicitLod = 90, + OpImageFetch = 95, + OpImageGather = 96, + OpImageDrefGather = 97, + OpImageWrite = 99, + OpImage = 100, + OpImageQuerySizeLod = 103, + OpImageQueryLod = 105, + OpImageQueryLevels = 106, + OpConvertFToU = 109, + OpConvertFToS = 110, + OpConvertSToF = 111, + OpConvertUToF = 112, + OpBitcast = 124, + OpSNegate = 126, + OpFNegate = 127, + OpIAdd = 128, + OpFAdd = 129, + OpISub = 130, + OpFSub = 131, + OpIMul = 132, + OpFMul = 133, + OpUDiv = 134, + OpFDiv = 136, + OpIAddCarry = 149, + OpUMulExtended = 151, + OpSMulExtended = 152, + OpLogicalNotEqual = 165, + OpLogicalOr = 166, + OpLogicalAnd = 167, + OpLogicalNot = 168, + OpSelect = 169, + OpIEqual = 170, + OpINotEqual = 171, + OpUGreaterThan = 172, + OpSGreaterThan = 173, + OpUGreaterThanEqual = 174, + OpSGreaterThanEqual = 175, + OpULessThan = 176, + OpSLessThan = 177, + OpULessThanEqual = 178, + OpSLessThanEqual = 179, + OpFOrdEqual = 180, + OpFUnordEqual = 181, + OpFOrdNotEqual = 182, + OpFUnordNotEqual = 183, + OpFOrdLessThan = 184, + OpFUnordLessThan = 185, + OpFOrdGreaterThan = 186, + OpFUnordGreaterThan = 187, + OpFOrdLessThanEqual = 188, + OpFUnordLessThanEqual = 189, + OpFOrdGreaterThanEqual = 190, + OpFUnordGreaterThanEqual = 191, + OpShiftRightLogical = 194, + OpShiftRightArithmetic = 195, + OpShiftLeftLogical = 196, + OpBitwiseOr = 197, + OpBitwiseXor = 198, + OpBitwiseAnd = 199, + OpNot = 200, + OpBitFieldInsert = 201, + OpBitFieldSExtract = 202, + OpBitFieldUExtract = 203, + OpBitReverse = 204, + OpBitCount = 205, + OpControlBarrier = 224, + OpMemoryBarrier = 225, + OpAtomicLoad = 227, + OpAtomicExchange = 229, + OpAtomicCompareExchange = 230, + OpAtomicIAdd = 234, + OpAtomicISub = 235, + OpAtomicSMin = 236, + OpAtomicUMin = 237, + OpAtomicSMax = 238, + OpAtomicUMax = 239, + OpAtomicAnd = 240, + OpAtomicOr = 241, + OpAtomicXor = 242, + OpPhi = 245, + OpLoopMerge = 246, + OpSelectionMerge = 247, + OpLabel = 248, + OpBranch = 249, + OpBranchConditional = 250, + OpSwitch = 251, + OpKill = 252, + OpReturn = 253, + OpGroupNonUniformBallot = 339, + OpGroupNonUniformBallotFindLSB = 343, + OpGroupNonUniformShuffle = 345, +}; + +enum : uint32_t { + GlslRoundEven = 2, + GlslTrunc = 3, + GlslFAbs = 4, + GlslFloor = 8, + GlslCeil = 9, + GlslFract = 10, + GlslSin = 13, + GlslCos = 14, + GlslExp2 = 29, + GlslLog2 = 30, + GlslSqrt = 31, + GlslInverseSqrt = 32, + GlslFMin = 37, + GlslFMax = 40, + GlslFClamp = 43, + GlslLdexp = 53, + GlslFma = 50, + GlslPackSnorm2x16 = 56, + GlslPackUnorm2x16 = 57, + GlslPackHalf2x16 = 58, + GlslUnpackHalf2x16 = 62, + GlslFindILsb = 73, + GlslFindUMsb = 75, +}; + +struct RegisterBinding { + IR::Register reg; + uint32_t pointer_id = 0; +}; + +struct InputBinding { + IR::StageInputKind kind = IR::StageInputKind::VertexIndex; + uint32_t location = 0; + uint32_t component_count = 1; + uint32_t variable_id = 0; + std::string debug_name; +}; + +struct OutputBinding { + IR::StageOutputKind kind = IR::StageOutputKind::Parameter; + uint32_t index = 0; + uint32_t location = 0; + uint32_t variable_id = 0; + std::string debug_name; +}; + +struct DescriptorResourceBinding { + const IR::DescriptorBinding* descriptor = nullptr; + uint32_t array_index = 0; +}; + +struct SampledImageDescriptors { + uint32_t image_type = 0; + uint32_t sampled_image_type = 0; + uint32_t pointer_type = 0; + uint32_t array_type = 0; + uint32_t array_pointer_type = 0; + uint32_t variable = 0; +}; + +struct EmitterState { + Builder builder; + const IR::Program* program = nullptr; + const IR::ResourceSnapshot* resources = nullptr; + const ShaderVertexInputInfo* vertex_input_info = nullptr; + const ShaderPixelInputInfo* pixel_input_info = nullptr; + const ShaderComputeInputInfo* compute_input_info = nullptr; + ShaderType stage = ShaderType::Unknown; + uint32_t wave_size = 64; + bool exact_subgroup_operations = false; + bool per_invocation_masks = false; + uint32_t void_type = 0; + uint32_t bool_type = 0; + uint32_t uint_type = 0; + uint32_t uint_pair_type = 0; + uint32_t int_pair_type = 0; + uint32_t int_type = 0; + uint32_t float_type = 0; + uint32_t vec2_uint_type = 0; + uint32_t vec3_uint_type = 0; + uint32_t vec4_uint_type = 0; + uint32_t vec2_int_type = 0; + uint32_t vec3_int_type = 0; + uint32_t vec4_int_type = 0; + uint32_t vec2_float_type = 0; + uint32_t vec3_float_type = 0; + uint32_t vec4_float_type = 0; + uint32_t ptr_func_uint = 0; + uint32_t ptr_input_float = 0; + uint32_t ptr_input_bool = 0; + uint32_t ptr_input_int = 0; + uint32_t ptr_input_uint = 0; + uint32_t ptr_input_vec2_float = 0; + uint32_t ptr_input_vec3_float = 0; + uint32_t ptr_input_vec2_int = 0; + uint32_t ptr_input_vec3_int = 0; + uint32_t ptr_input_vec4_int = 0; + uint32_t ptr_input_vec2_uint = 0; + uint32_t ptr_input_vec3_uint = 0; + uint32_t ptr_input_vec4_uint = 0; + uint32_t ptr_input_vec4_float = 0; + uint32_t sample_mask_array_type = 0; + uint32_t ptr_output_int = 0; + uint32_t ptr_output_sample_mask_array = 0; + uint32_t ptr_output_float = 0; + uint32_t ptr_output_vec4_float = 0; + uint32_t per_vertex_type = 0; + uint32_t ptr_output_per_vertex = 0; + uint32_t storage_runtime_array_type = 0; + uint32_t storage_buffer_type = 0; + uint32_t ptr_storage_buffer = 0; + uint32_t ptr_storage_buffer_uint = 0; + uint32_t storage_buffer_array_type = 0; + uint32_t ptr_storage_buffer_array = 0; + uint32_t storage_buffer_variable = 0; + uint32_t address_memory_array_type = 0; + uint32_t ptr_address_memory_array = 0; + uint32_t address_memory_variable = 0; + uint32_t gds_variable = 0; + uint32_t push_constant_u32x4_array_type = 0; + uint32_t push_constant_rows_array_type = 0; + uint32_t push_constant_block_type = 0; + uint32_t ptr_push_constant_block = 0; + uint32_t ptr_push_constant_uint = 0; + uint32_t push_constant_variable = 0; + uint32_t vsharp_storage_variable = 0; + uint32_t flattened_srt_variable = 0; + uint32_t lds_array_type = 0; + uint32_t ptr_workgroup_array = 0; + uint32_t ptr_workgroup_uint = 0; + uint32_t lds_variable = 0; + std::array sampled_images; + uint32_t sampler_type = 0; + uint32_t sampler_array_type = 0; + uint32_t ptr_uniform_sampler = 0; + uint32_t ptr_uniform_sampler_array = 0; + uint32_t sampler_variable = 0; + uint32_t storage_image_type = 0; + uint32_t ptr_uniform_storage_image = 0; + uint32_t storage_image_array_type = 0; + uint32_t ptr_uniform_storage_image_array = 0; + uint32_t storage_image_variable = 0; + uint32_t storage_image_2d_array_type = 0; + uint32_t ptr_uniform_storage_image_2d_array = 0; + uint32_t storage_image_2d_array_array_type = 0; + uint32_t ptr_uniform_storage_image_2d_array_array = 0; + uint32_t storage_image_2d_array_variable = 0; + uint32_t storage_image_3d_type = 0; + uint32_t ptr_uniform_storage_image_3d = 0; + uint32_t storage_image_3d_array_type = 0; + uint32_t ptr_uniform_storage_image_3d_array = 0; + uint32_t storage_image_3d_variable = 0; + uint32_t storage_image_uint_type = 0; + uint32_t ptr_uniform_storage_image_uint = 0; + uint32_t storage_image_uint_array_type = 0; + uint32_t ptr_uniform_storage_image_uint_array = 0; + uint32_t storage_image_uint_variable = 0; + uint32_t storage_image_uint_2d_array_type = 0; + uint32_t ptr_uniform_storage_image_uint_2d_array = 0; + uint32_t storage_image_uint_2d_array_array_type = 0; + uint32_t ptr_uniform_storage_image_uint_2d_array_array = 0; + uint32_t storage_image_uint_2d_array_variable = 0; + uint32_t storage_image_uint_3d_type = 0; + uint32_t ptr_uniform_storage_image_uint_3d = 0; + uint32_t storage_image_uint_3d_array_type = 0; + uint32_t ptr_uniform_storage_image_uint_3d_array = 0; + uint32_t storage_image_uint_3d_variable = 0; + uint32_t ptr_image_uint = 0; + uint32_t func_type = 0; + uint32_t main_func = 0; + uint32_t entry_label = 0; + uint32_t pixel_valid_mask_variable = 0; + bool dispatcher_fallback = false; + uint32_t dispatch_pc_variable = 0; + uint32_t dispatch_header_label = 0; + uint32_t dispatch_select_label = 0; + uint32_t dispatch_default_label = 0; + uint32_t dispatch_after_switch_label = 0; + uint32_t dispatch_continue_label = 0; + uint32_t dispatch_merge_label = 0; + uint32_t glsl_std450 = 0; + uint32_t subgroup_local_invocation_id_variable = 0; + uint32_t per_vertex_variable = 0; + uint32_t depth_variable = 0; + uint32_t sample_mask_variable = 0; + bool needs_subgroup_ballot = false; + bool needs_subgroup_shuffle = false; + bool needs_subgroup_local_invocation_id = false; + bool needs_compute_derivatives = false; + bool needs_image_gather_extended = false; + bool needs_function_lds = false; + bool needs_pixel_valid_mask = false; + std::vector registers; + std::vector inputs; + std::vector outputs; + std::vector interface_variables; + std::vector reachable_blocks; + std::map block_labels; + std::map constants; + std::map signed_constants; + std::map float_constants; +}; + +constexpr uint32_t PsInputOffsetMask = 0x0000001fu; +constexpr uint32_t PsInputFlatShade = 0x00000400u; + +enum class VertexInputScalarKind { Float, Sint, Uint }; + +constexpr uint32_t NoImageComponent = 0xffffffffu; + +struct DppTargetLane { + uint32_t lane = 0; + uint32_t valid = 0; +}; + +struct ImageSampleLayout { + uint32_t offset = NoImageComponent; + uint32_t dref = NoImageComponent; + uint32_t bias = NoImageComponent; + uint32_t coord = 0; + uint32_t lod = NoImageComponent; + uint32_t grad_x = NoImageComponent; + uint32_t grad_y = NoImageComponent; +}; + +enum class ImageViewKind { + Dim2D, + Dim2DArray, + Dim3D, +}; + +constexpr uint32_t SampledImageIndex(bool integer, ImageViewKind view) { + return static_cast(view) + (integer ? 3u : 0u); +} + +constexpr IR::DescriptorBindingKind SampledBindingKind(bool integer, ImageViewKind view) { + if (integer) { + switch (view) { + case ImageViewKind::Dim2DArray: return IR::DescriptorBindingKind::SampledUint2DArray; + case ImageViewKind::Dim3D: return IR::DescriptorBindingKind::SampledUint3D; + default: return IR::DescriptorBindingKind::SampledUint2D; + } + } + switch (view) { + case ImageViewKind::Dim2DArray: return IR::DescriptorBindingKind::Sampled2DArray; + case ImageViewKind::Dim3D: return IR::DescriptorBindingKind::Sampled3D; + default: return IR::DescriptorBindingKind::Sampled2D; + } +} + +struct AddCarryResult { + uint32_t sum = 0; + uint32_t carry = 0; +}; + +struct F32Class { + uint32_t bits = 0; + uint32_t nan = 0; + uint32_t snan = 0; + uint32_t zero = 0; + uint32_t quiet_bits = 0; +}; + +struct CubeF32Values { + uint32_t x = 0; + uint32_t y = 0; + uint32_t z = 0; + uint32_t nx = 0; + uint32_t ny = 0; + uint32_t nz = 0; + uint32_t z_face = 0; + uint32_t y_face = 0; + uint32_t x_neg = 0; + uint32_t y_neg = 0; + uint32_t z_neg = 0; +}; + +struct F16Class { + uint32_t bits = 0; + uint32_t nan = 0; + uint32_t snan = 0; + uint32_t zero = 0; + uint32_t quiet_bits = 0; +}; + +uint32_t PixelParameterMappedLocation(const EmitterState& state, uint32_t attr); + +uint32_t PixelParameterLocation(const EmitterState& state, uint32_t attr); + +bool PixelParameterIsFlat(const EmitterState& state, uint32_t attr); + +VertexInputScalarKind VertexParameterScalarKind(const EmitterState& state, uint32_t location); + +uint32_t VertexParameterComponentCount(const EmitterState& state, const InputBinding& input); + +uint32_t VertexParameterScalarType(const EmitterState& state, VertexInputScalarKind kind); + +uint32_t VertexParameterScalarPointerType(const EmitterState& state, VertexInputScalarKind kind); + +uint32_t VertexParameterVectorOrScalarType(const EmitterState& state, VertexInputScalarKind kind, + uint32_t components); + +uint32_t VertexParameterInputPointerType(const EmitterState& state, VertexInputScalarKind kind, + uint32_t components); + +void SetError(std::string* error, const char* message); + +void CollectRegister(std::vector* registers, IR::Register reg); + +IR::Operand MakeRegisterOperand(IR::RegisterFile file, uint32_t index); + +IR::Register SccRegister(); + +IR::Operand SccOperand(); + +bool IsInactiveWave32ExecHigh(const EmitterState& state, IR::Register reg); + +bool IsMaskRegisterFile(IR::RegisterFile file); + +bool IsSccOperand(const IR::Operand& operand); + +bool IsCompareOpcode(IR::Opcode op); + +void CollectMaskStateRegisters(std::vector* registers); + +void CollectSequentialRegisters(std::vector* registers, const IR::Operand& base, + uint32_t count); + +uint32_t MaxCollectedVectorRegisterEnd(const std::vector& registers); + +void CollectMoveRelSourceRegisters(const IR::Program& program, + std::vector* registers); + +bool IsPairDwordOpcode(IR::Opcode op); + +uint32_t PairDwordSourceCount(IR::Opcode op, uint32_t src_count); + +void CollectRegisters(const IR::Program& program, std::vector* registers); + +bool HasOutput(const std::vector& outputs, IR::StageOutputKind kind, uint32_t index); + +void CopyProgramInputsAndOutputs(EmitterState* state, const IR::Program& program); + +uint32_t OutputVariableForExport(const EmitterState& state, const IR::ExportInfo& exp); + +bool ProgramNeedsComputeDerivatives(const IR::Program& program); + +bool ProgramNeedsImageGatherExtended(const IR::Program& program); + +bool IsLdsOpcode(IR::Opcode op); + +bool ProgramNeedsFunctionLds(const IR::Program& program); + +bool ProgramNeedsPixelValidMask(const IR::Program& program); + +bool InstructionHasDppSource(const IR::Instruction& inst); + +bool ProgramNeedsSubgroupBallot(const IR::Program& program); + +bool ProgramNeedsSubgroupShuffle(const IR::Program& program); + +bool IsCompareOpcode(IR::Opcode op); + +bool ProgramNeedsSubgroupLocalInvocationId(const IR::Program& program); + +uint32_t PointerForRegister(const EmitterState& state, IR::Register reg); + +uint32_t ConstantU32(EmitterState* state, uint32_t value); + +void EmitStoreU32(EmitterState* state, const IR::Operand& dst, uint32_t value); + +uint32_t EmitSubgroupLocalInvocationId(EmitterState* state); + +uint32_t EmitLaneIndexActiveBool(EmitterState* state, uint32_t lane); + +[[noreturn]] void ExitDescriptorBindingFailure(const EmitterState& state, + IR::DescriptorBindingKind kind, uint32_t resource, + const char* reason); + +DescriptorResourceBinding ResourceForDescriptor(const EmitterState& state, + IR::DescriptorBindingKind kind, uint32_t resource); + +uint32_t DescriptorElementPointer(EmitterState* state, uint32_t result_ptr_type, + uint32_t variable_id, uint32_t array_index, + IR::DescriptorBindingKind kind, uint32_t resource, + const char* variable_name); + +ImageViewKind SampledImageViewKind(const EmitterState& state, const IR::MemoryInfo& mem, + uint32_t use_pc); + +uint32_t ImageViewCoordinateComponents(ImageViewKind view); + +uint32_t ImageViewImageType(const EmitterState& state, ImageViewKind view, bool integer); + +uint32_t ImageViewSampledImageType(const EmitterState& state, ImageViewKind view, bool integer); + +uint32_t ImageViewSizeType(const EmitterState& state, ImageViewKind view); + +uint32_t LoadSampledImageDescriptor(EmitterState* state, const IR::MemoryInfo& mem, uint32_t use_pc, + ImageViewKind view); + +uint32_t LoadSamplerDescriptor(EmitterState* state, uint32_t sampler, uint32_t use_pc); + +uint32_t MakeSampledImage(EmitterState* state, const IR::MemoryInfo& mem, uint32_t use_pc, + ImageViewKind view); + +ImageViewKind StorageImageViewKind(const EmitterState& state, const IR::MemoryInfo& mem, + bool uint_image, uint32_t use_pc); + +uint32_t StorageImageDescriptorPointer(EmitterState* state, uint32_t resource, bool uint_image, + uint32_t use_pc = UINT32_MAX, + ImageViewKind view = ImageViewKind::Dim2D); + +uint32_t LoadStorageImageDescriptor(EmitterState* state, uint32_t resource, bool uint_image, + uint32_t use_pc, ImageViewKind view = ImageViewKind::Dim2D); + +uint32_t ExecutionModelForStage(ShaderType stage); + +uint32_t ConstantU32(EmitterState* state, uint32_t value); + +uint32_t ConstantI32(EmitterState* state, int32_t value); + +uint32_t ConstantF32(EmitterState* state, uint32_t bits); + +uint32_t FloatBits(float value); + +uint32_t ConstantF32Value(EmitterState* state, float value); + +void AllocateInputVariables(EmitterState* state); + +void AllocateOutputVariables(EmitterState* state); + +uint32_t BuiltInForInput(IR::StageInputKind kind); + +void AddInputAnnotationsAndNames(EmitterState* state); + +void AddOutputAnnotationsAndNames(EmitterState* state); + +void DecorateDescriptor(EmitterState* state, uint32_t variable, const char* name, uint32_t set, + uint32_t binding); + +void AddDescriptorAnnotationsAndNames(EmitterState* state); + +void EmitHeaderAndTypes(EmitterState* state); + +void AllocateRegisterVariables(EmitterState* state); + +void AllocateDescriptorVariables(EmitterState* state); + +bool SdwaSelectorOffsetWidth(uint32_t sel, uint32_t& offset, uint32_t& width); + +uint32_t EmitSdwaExtractU32(EmitterState* state, const IR::Operand& operand, uint32_t value); + +uint32_t EmitTrueBool(EmitterState* state); + +DppTargetLane EmitDppQuadPermTargetLane(EmitterState* state, uint32_t subid, uint32_t control); + +DppTargetLane EmitDppRowShiftTargetLane(EmitterState* state, uint32_t subid, uint32_t amount, + bool left); + +DppTargetLane EmitDppRowRotateRightTargetLane(EmitterState* state, uint32_t subid, uint32_t amount); + +DppTargetLane EmitDppMirrorTargetLane(EmitterState* state, uint32_t subid, bool half_row); + +DppTargetLane EmitDppTargetLane(EmitterState* state, uint32_t control); + +uint32_t EmitDppValueU32(EmitterState* state, const IR::Operand& operand, uint32_t value); + +uint32_t EmitValueLoad(EmitterState* state, const IR::Operand& operand); + +uint32_t EmitRegisterLoad(EmitterState* state, IR::Register reg); + +uint32_t WaveMaskHighLoad(EmitterState* state, IR::RegisterFile file); + +uint32_t EmitLaneMaskPartU32(EmitterState* state, uint32_t part); + +uint32_t EmitThreadMaskBelowPartU32(EmitterState* state, uint32_t part); + +uint32_t EmitMaskActiveBool(EmitterState* state, IR::RegisterFile file); + +uint32_t EmitMaskZeroBool(EmitterState* state, IR::RegisterFile file, bool zero); + +uint32_t EmitMaskSummaryZeroBool(EmitterState* state, IR::RegisterFile file, bool zero); + +uint32_t EmitExecActiveBool(EmitterState* state); + +uint32_t EmitConditionBool(EmitterState* state, const IR::Operand& operand); + +uint32_t EmitSubgroupLocalInvocationId(EmitterState* state); + +uint32_t InputVariableForKind(const EmitterState& state, IR::StageInputKind kind); + +uint32_t InputVariableForParameter(const EmitterState& state, uint32_t location); + +uint32_t EmitInputComponentU32(EmitterState* state, IR::StageInputKind kind, uint32_t component); + +uint32_t EmitLocalInvocationIndex(EmitterState* state); + +void EmitLoadInputF32(EmitterState* state, const IR::Instruction& inst); + +uint32_t EmitSccBool(EmitterState* state, bool non_zero); + +uint32_t EmitFloatLoad(EmitterState* state, const IR::Operand& operand); + +uint32_t ApplyResultModifiersF32(EmitterState* state, uint32_t value, const IR::Operand& dst); + +uint32_t EmitMixF32Load(EmitterState* state, const IR::Operand& operand); + +IR::Operand OffsetRegisterOperand(const IR::Operand& operand, uint32_t offset); + +uint32_t EmitLaneMaskOperandActiveBool(EmitterState* state, const IR::Operand& operand); + +uint32_t EmitBallotLaneActiveBool(EmitterState* state, uint32_t ballot, uint32_t lane); + +uint32_t EmitLaneIndexActiveBool(EmitterState* state, uint32_t lane); + +uint32_t EmitSubgroupLaneActiveBool(EmitterState* state, uint32_t lane); + +uint32_t EmitSequentialValueLoad(EmitterState* state, const IR::Operand& operand, uint32_t offset); + +uint32_t EmitSequentialFloatLoad(EmitterState* state, const IR::Operand& operand, uint32_t offset); + +uint32_t ImageAddressHalfComponent(const IR::Instruction& inst, uint32_t component); + +IR::Operand ImageAddressOperand(const IR::Instruction& inst, const IR::Operand& base, + uint32_t component); + +uint32_t EmitImageAddressValueLoad(EmitterState* state, const IR::Instruction& inst, + const IR::Operand& base, uint32_t component); + +uint32_t EmitImageAddressFloatLoad(EmitterState* state, const IR::Instruction& inst, + const IR::Operand& base, uint32_t component); + +uint32_t EmitZeroF32(EmitterState* state); + +bool HasImageSampleFlag(const IR::Instruction& inst, uint32_t flag); + +ImageSampleLayout MakeImageSampleLayout(const IR::Instruction& inst, ImageViewKind view); + +uint32_t EmitImageCoordF32(EmitterState* state, const IR::Instruction& inst, + const ImageSampleLayout& layout, ImageViewKind view); + +uint32_t EmitImageLodF32(EmitterState* state, const IR::Instruction& inst, + const ImageSampleLayout& layout); + +uint32_t EmitImageDrefF32(EmitterState* state, const IR::Instruction& inst, + const ImageSampleLayout& layout); + +uint32_t EmitImageBiasF32(EmitterState* state, const IR::Instruction& inst, + const ImageSampleLayout& layout); + +uint32_t EmitImageGradientF32(EmitterState* state, const IR::Instruction& inst, + uint32_t first_component); + +uint32_t EmitImagePackedOffset2I32(EmitterState* state, const IR::Instruction& inst, + const ImageSampleLayout& layout); + +uint32_t EmitImageOffsetCoordF32(EmitterState* state, const IR::Instruction& inst, + const ImageSampleLayout& layout, uint32_t sampled_image, + uint32_t coord, ImageViewKind view); + +uint32_t EmitImageCoordU32(EmitterState* state, const IR::Instruction& inst, + ImageViewKind view = ImageViewKind::Dim2D); + +uint32_t EmitImageLoadCoordU32(EmitterState* state, const IR::Instruction& inst, + ImageViewKind view); + +uint32_t EmitImageMipLodU32(EmitterState* state, const IR::Instruction& inst, + const IR::Operand& address, ImageViewKind view); + +uint32_t EmitImageQueryCoordF32(EmitterState* state, const IR::Instruction& inst, + ImageViewKind view); + +uint32_t DmaskComponentIndex(uint32_t dmask, uint32_t component); + +uint32_t EmitImageStoreComponentF32(EmitterState* state, const IR::Instruction& inst, + uint32_t component); + +uint32_t EmitImageStoreTexelF32(EmitterState* state, const IR::Instruction& inst); + +uint32_t EmitImageStoreTexelU32(EmitterState* state, const IR::Instruction& inst); + +uint32_t EmitDppWriteActiveBool(EmitterState* state, const IR::Operand& dst); + +uint32_t EmitSdwaDestinationMerge(EmitterState* state, uint32_t pointer, const IR::Operand& dst, + uint32_t value); + +void EmitStoreU32(EmitterState* state, const IR::Operand& dst, uint32_t value); + +uint32_t EmitAddU32(EmitterState* state, uint32_t lhs, uint32_t rhs); + +uint32_t EmitBinaryU32(EmitterState* state, uint32_t opcode, uint32_t lhs, uint32_t rhs); + +uint32_t EmitNotEqualZeroBool(EmitterState* state, uint32_t value); + +uint32_t EmitSelectU32Value(EmitterState* state, uint32_t condition, uint32_t true_value, + uint32_t false_value); + +uint32_t EmitByteAddress(EmitterState* state, const IR::Instruction& inst, uint32_t first_src, + uint32_t src_count); + +uint32_t StorageBufferPackedStride(const EmitterState& state, const IR::MemoryInfo& mem, + uint32_t use_pc); + +uint32_t StorageBufferFormat(const EmitterState& state, const IR::MemoryInfo& mem, uint32_t use_pc); + +bool ShouldApplyBufferAddTid(const IR::Instruction& inst); + +uint32_t EmitBufferIndexWithAddTid(EmitterState* state, const IR::Instruction& inst, uint32_t index, + uint32_t packed); + +uint32_t EmitOptionalLogicalAndBool(EmitterState* state, uint32_t lhs, uint32_t rhs); + +bool IsStorageBufferMemoryKind(IR::ResourceKind kind); + +uint32_t EmitBufferAddressFromParts(EmitterState* state, const IR::Instruction& inst, + uint32_t index, uint32_t offset, uint32_t soffset); + +uint32_t EmitBufferByteAddress(EmitterState* state, const IR::Instruction& inst, uint32_t first_src, + uint32_t src_count); + +uint32_t EmitMemoryByteAddress(EmitterState* state, const IR::Instruction& inst, + const IR::MemoryInfo& mem, uint32_t first_src, uint32_t src_count); + +uint32_t EmitDwordIndex(EmitterState* state, const IR::Instruction& inst, uint32_t first_src, + uint32_t src_count); + +uint32_t EmitMemoryDwordIndex(EmitterState* state, const IR::Instruction& inst, + const IR::MemoryInfo& mem, uint32_t first_src, uint32_t src_count); + +DescriptorResourceBinding StorageBufferBindingForMemory(EmitterState* state, + const IR::MemoryInfo& mem, uint32_t use_pc); + +uint32_t EmitStorageBufferObjectPointer(EmitterState* state, const IR::MemoryInfo& mem, + uint32_t use_pc); + +uint32_t EmitStorageBufferElementInBounds(EmitterState* state, const IR::MemoryInfo& mem, + uint32_t index, uint32_t use_pc); + +uint32_t EmitStorageBufferElementPointer(EmitterState* state, const IR::MemoryInfo& mem, + uint32_t index, uint32_t use_pc); + +uint32_t EmitLdsElementPointer(EmitterState* state, uint32_t index); + +uint32_t EmitGdsElementInBounds(EmitterState* state, uint32_t index); + +uint32_t EmitGdsElementPointer(EmitterState* state, uint32_t index); + +uint32_t EmitMemoryElementPointer(EmitterState* state, const IR::MemoryInfo& mem, uint32_t index, + uint32_t use_pc); + +uint32_t EmitMemoryLoadDwordValueU32(EmitterState* state, const IR::Instruction& inst, + IR::ResourceKind kind, uint32_t first_src, uint32_t src_count); + +void EmitMemoryLoadU32(EmitterState* state, const IR::Instruction& inst, IR::ResourceKind kind, + uint32_t first_src, uint32_t src_count); + +uint32_t EmitMemoryLoadSubDwordValueU32(EmitterState* state, const IR::Instruction& inst, + IR::ResourceKind kind, uint32_t first_src, + uint32_t src_count, uint32_t data_bits, bool sign_extend); + +void EmitMemoryLoadSubDwordU32(EmitterState* state, const IR::Instruction& inst, + IR::ResourceKind kind, uint32_t first_src, uint32_t src_count, + uint32_t data_bits, bool sign_extend); + +void EmitMemoryStoreU32(EmitterState* state, const IR::Instruction& inst, IR::ResourceKind kind, + uint32_t first_src, uint32_t src_count); + +void EmitMemoryStoreSubDwordU32(EmitterState* state, const IR::Instruction& inst, + IR::ResourceKind kind, uint32_t first_src, uint32_t src_count, + uint32_t data_bits); + +uint32_t AddressSourceCount(const IR::Instruction& inst, uint32_t first_src); + +bool IsFormattedBufferComponent(const IR::Instruction& inst); + +Prospero::BufferFormat FormattedBufferFormat(const EmitterState& state, const IR::Instruction& inst); + +IR::Instruction WithFormatComponentByteOffset(const IR::Instruction& inst, Prospero::BufferFormat format); + +uint32_t EmitTBufferBitcastF32ToU32(EmitterState* state, uint32_t value); + +uint32_t EmitTBufferBitcastU32ToF32(EmitterState* state, uint32_t value); + +uint32_t EmitTBufferBitcastU32ToI32(EmitterState* state, uint32_t value); + +uint32_t EmitTBufferCompareU32Constant(EmitterState* state, uint32_t opcode, uint32_t value, + uint32_t constant); + +uint32_t EmitTBufferSelectF32(EmitterState* state, uint32_t condition, uint32_t true_value, + uint32_t false_value); + +bool IsSignedFormatComponent(Format::ComponentType type); + +uint32_t EmitExtractFormatFieldU32(EmitterState* state, uint32_t raw_word, uint32_t offset, + uint32_t bits, bool sign_extend); + +uint32_t EmitHalfToF32Bits(EmitterState* state, uint32_t raw); + +uint32_t EmitUFloatToF32Bits(EmitterState* state, uint32_t raw, uint32_t bits); + +uint32_t EmitFormatRawComponent(EmitterState* state, const IR::Instruction& inst, + const Format::BufferFormatInfo& info); + +uint32_t NormalizeFormatComponent(EmitterState* state, const Format::BufferFormatInfo& info, + uint32_t component, uint32_t raw); + +uint32_t UnpackTBufferFormat(EmitterState* state, const IR::Instruction& inst, + const Format::BufferFormatInfo& info); + +bool EmitTypedTBufferLoad(EmitterState* state, const IR::Instruction& inst, + const Format::BufferFormatInfo& info); + +bool EmitFormattedBufferLoad(EmitterState* state, const IR::Instruction& inst); + +uint32_t FormattedBufferDwordStoreComponentCount(Prospero::BufferFormat format, + uint32_t opcode_components); + +bool EmitBufferIntegerFormatStore(EmitterState* state, const IR::Instruction& inst); + +uint32_t EmitAtomicPointer(EmitterState* state, const IR::Instruction& inst); + +void EmitDeviceAtomicMemoryBarrier(EmitterState* state); + +void EmitAtomicU32(EmitterState* state, const IR::Instruction& inst, uint32_t opcode); + +void EmitSLoadDword(EmitterState* state, const IR::Instruction& inst); + +void EmitLoadSrtDword(EmitterState* state, const IR::Instruction& inst); + +void EmitBufferLoadUbyte(EmitterState* state, const IR::Instruction& inst); + +void EmitBufferLoadSbyte(EmitterState* state, const IR::Instruction& inst); + +void EmitBufferLoadUshort(EmitterState* state, const IR::Instruction& inst); + +void EmitBufferLoadSshort(EmitterState* state, const IR::Instruction& inst); + +void EmitBufferLoadDword(EmitterState* state, const IR::Instruction& inst); + +void EmitBufferStoreDword(EmitterState* state, const IR::Instruction& inst); + +void EmitFlatLoadUbyte(EmitterState* state, const IR::Instruction& inst); + +void EmitFlatLoadSbyte(EmitterState* state, const IR::Instruction& inst); + +void EmitFlatLoadUshort(EmitterState* state, const IR::Instruction& inst); + +void EmitFlatLoadSshort(EmitterState* state, const IR::Instruction& inst); + +void EmitFlatLoadDword(EmitterState* state, const IR::Instruction& inst); + +uint32_t EmitDsAddtidDwordIndex(EmitterState* state, const IR::Instruction& inst, + uint32_t m0_src_index); + +void EmitDsWriteAddtidB32(EmitterState* state, const IR::Instruction& inst); + +void EmitDsReadAddtidB32(EmitterState* state, const IR::Instruction& inst); + +void EmitDsAppendConsume(EmitterState* state, const IR::Instruction& inst, uint32_t atomic_opcode); + +void EmitDsFloatMinMaxF32(EmitterState* state, const IR::Instruction& inst, bool max_value); + +uint32_t EmitDsSwizzleTargetLane(EmitterState* state, uint32_t subid, uint32_t control); + +void EmitDsSwizzleB32(EmitterState* state, const IR::Instruction& inst); + +uint32_t EmitImageGetResinfoComponent(EmitterState* state, uint32_t image, uint32_t size, + uint32_t component_index); + +void EmitImageGetResinfo(EmitterState* state, const IR::Instruction& inst); + +void EmitImageGetLod(EmitterState* state, const IR::Instruction& inst); + +void EmitImageLoad(EmitterState* state, const IR::Instruction& inst); + +void EmitImageStore(EmitterState* state, const IR::Instruction& inst); + +void EmitImageSampleResult(EmitterState* state, const IR::Instruction& inst, uint32_t sample, + bool dref); + +bool ImageSampleNeedsExplicitLod(const EmitterState& state, const IR::Instruction& inst); + +void AddImageSampleOperands(EmitterState* state, const IR::Instruction& inst, + const ImageSampleLayout& layout, bool explicit_lod, + std::vector* words); + +uint32_t ImageSampleOpcode(const EmitterState& state, const IR::Instruction& inst); + +void EmitImageSample(EmitterState* state, const IR::Instruction& inst); + +uint32_t ImageGatherComponent(uint32_t dmask); + +void EmitImageGather4(EmitterState* state, const IR::Instruction& inst); + +void EmitMoveU32(EmitterState* state, const IR::Instruction& inst); + +void EmitMoveF32Bits(EmitterState* state, const IR::Instruction& inst); + +uint32_t MoveRelRegisterLimit(const EmitterState& state, IR::Register base); + +void EmitMoveRelSourceU32(EmitterState* state, const IR::Instruction& inst); + +void EmitMoveRelDestU32(EmitterState* state, const IR::Instruction& inst); + +void EmitMoveU64(EmitterState* state, const IR::Instruction& inst); + +uint32_t EmitWqmLaneU32(EmitterState* state, uint32_t src); + +void EmitWqmB64(EmitterState* state, const IR::Instruction& inst); + +void EmitSaveexecB32(EmitterState* state, const IR::Instruction& inst); + +void EmitSaveexecB64(EmitterState* state, const IR::Instruction& inst); + +void EmitReadFirstLaneU32(EmitterState* state, const IR::Instruction& inst); + +uint32_t EmitLaneIndex(EmitterState* state, const IR::Operand& operand); + +void EmitReadLaneU32(EmitterState* state, const IR::Instruction& inst); + +void EmitWriteLaneU32(EmitterState* state, const IR::Instruction& inst); + +void EmitPermlaneB32(EmitterState* state, const IR::Instruction& inst, bool x16); + +void EmitControlNop(EmitterState* state, const IR::Instruction& inst); + +void EmitWaitcnt(EmitterState* state, const IR::Instruction& inst); + +void EmitBarrier(EmitterState* state, const IR::Instruction& inst); + +void EmitAbsI32(EmitterState* state, const IR::Instruction& inst); + +void EmitUnaryU32(EmitterState* state, const IR::Instruction& inst, uint32_t opcode); + +void EmitUnaryU64(EmitterState* state, const IR::Instruction& inst, uint32_t opcode); + +void EmitFindLsbU32(EmitterState* state, const IR::Instruction& inst); + +void EmitFindMsbFromHighU32(EmitterState* state, const IR::Instruction& inst); + +void EmitFindMsbFromHighU64(EmitterState* state, const IR::Instruction& inst); + +void EmitBitCountU64(EmitterState* state, const IR::Instruction& inst); + +void EmitBitCountU32(EmitterState* state, const IR::Instruction& inst); + +void EmitBitCountAddU32(EmitterState* state, const IR::Instruction& inst); + +void EmitBinaryU32(EmitterState* state, const IR::Instruction& inst, uint32_t opcode); + +void EmitBinaryNotRhsU32(EmitterState* state, const IR::Instruction& inst, uint32_t opcode); + +void EmitBinaryThenNotU32(EmitterState* state, const IR::Instruction& inst, uint32_t opcode); + +void EmitMaskedBitCountU32(EmitterState* state, const IR::Instruction& inst, uint32_t exec_index); + +uint32_t EmitMask5U32(EmitterState* state, uint32_t value); + +void EmitChainedBinaryU32(EmitterState* state, const IR::Instruction& inst, uint32_t first_op, + uint32_t second_op, bool mask_first_rhs = false, + bool mask_second_rhs = false); + +void EmitBinaryU64(EmitterState* state, const IR::Instruction& inst, uint32_t opcode); + +void EmitBinaryNotRhsU64(EmitterState* state, const IR::Instruction& inst, uint32_t opcode); + +void EmitBinaryThenNotU64(EmitterState* state, const IR::Instruction& inst, uint32_t opcode); + +void EmitSelectU64(EmitterState* state, const IR::Instruction& inst); + +uint32_t EmitSelectValueU32(EmitterState* state, uint32_t cond, uint32_t true_value, + uint32_t false_value); + +void EmitStoreSccBool(EmitterState* state, uint32_t cond); + +uint32_t EmitAndConstant(EmitterState* state, uint32_t value, uint32_t mask); + +void EmitShiftLeftLogicalU64Values(EmitterState* state, uint32_t low, uint32_t high, uint32_t shift, + uint32_t* out_low, uint32_t* out_high); + +void EmitShiftLeftLogicalU64(EmitterState* state, const IR::Instruction& inst); + +void EmitShiftRightLogicalU64Values(EmitterState* state, uint32_t low, uint32_t high, + uint32_t shift, uint32_t* out_low, uint32_t* out_high); + +void EmitShiftRightLogicalU64(EmitterState* state, const IR::Instruction& inst); + +void EmitAdd3U32(EmitterState* state, const IR::Instruction& inst); + +uint32_t EmitAndConstant(EmitterState* state, uint32_t value, uint32_t mask); + +void EmitShiftU32(EmitterState* state, const IR::Instruction& inst, uint32_t opcode); + +uint32_t EmitU16LaneBits(EmitterState* state, const IR::Operand& operand, bool high_lane, + bool sign_extend = false); + +void EmitShiftU16(EmitterState* state, const IR::Instruction& inst, uint32_t opcode, + bool arithmetic); + +void EmitBinaryU16(EmitterState* state, const IR::Instruction& inst, uint32_t opcode); + +void EmitMinMaxU16(EmitterState* state, const IR::Instruction& inst, bool signed_value, + bool max_value); + +uint32_t EmitXorConstant(EmitterState* state, uint32_t value, uint32_t mask); + +AddCarryResult EmitAddCarryValues(EmitterState* state, uint32_t lhs, uint32_t rhs, + uint32_t carry_in); + +void EmitLaneMaskPairFromBool(EmitterState* state, const IR::Operand& dst, uint32_t value_bool); + +void EmitPerInvocationMask(EmitterState* state, const IR::Operand& dst, uint32_t value_bool); + +void EmitIAddCarryU32(EmitterState* state, const IR::Instruction& inst); + +void EmitISubBorrowU32(EmitterState* state, const IR::Instruction& inst); + +void EmitScalarAddCarryU32(EmitterState* state, const IR::Instruction& inst); + +void EmitScalarSubBorrowU32(EmitterState* state, const IR::Instruction& inst); + +void EmitScalarSubBorrowCarryU32(EmitterState* state, const IR::Instruction& inst); + +void EmitBitClearU32(EmitterState* state, const IR::Instruction& inst); + +void EmitBitSetU32(EmitterState* state, const IR::Instruction& inst); + +void EmitUMadU64U32(EmitterState* state, const IR::Instruction& inst); + +void EmitScalarSignedOverflowI32(EmitterState* state, const IR::Instruction& inst, bool subtract); + +void EmitScalarShiftLeftAddCarryU32(EmitterState* state, const IR::Instruction& inst); + +void EmitMulU24U32(EmitterState* state, const IR::Instruction& inst); + +void EmitMulI24U32(EmitterState* state, const IR::Instruction& inst); + +void EmitMulHighU32(EmitterState* state, const IR::Instruction& inst); + +void EmitMulHighI32(EmitterState* state, const IR::Instruction& inst); + +void EmitMadU32U24(EmitterState* state, const IR::Instruction& inst); + +void EmitMadI32I24(EmitterState* state, const IR::Instruction& inst); + +uint32_t EmitShiftLeftConstant(EmitterState* state, uint32_t value, uint32_t shift); + +uint32_t EmitShiftRightConstant(EmitterState* state, uint32_t value, uint32_t shift); + +uint32_t EmitOrU32(EmitterState* state, uint32_t lhs, uint32_t rhs); + +uint32_t EmitBitReplicateHalfU32(EmitterState* state, uint32_t value); + +void EmitBitReplicateB64B32(EmitterState* state, const IR::Instruction& inst); + +uint32_t EmitBitFieldExtractConstant(EmitterState* state, uint32_t value, uint32_t offset, + uint32_t count); + +uint32_t EmitRightAlignedMaskU32(EmitterState* state, uint32_t count); + +void EmitBitFieldMaskU32(EmitterState* state, const IR::Instruction& inst); + +void EmitRightAlignedMaskU64(EmitterState* state, uint32_t count, uint32_t* out_low, + uint32_t* out_high); + +void EmitBitFieldMaskU64(EmitterState* state, const IR::Instruction& inst); + +void EmitBitFieldExtractU64(EmitterState* state, const IR::Instruction& inst); + +void EmitBitFieldExtractU32(EmitterState* state, const IR::Instruction& inst); + +void EmitBitFieldExtract3U32(EmitterState* state, const IR::Instruction& inst, bool signed_value); + +void EmitBitFieldInsertSelectU32(EmitterState* state, const IR::Instruction& inst); + +void EmitAlignBitU32(EmitterState* state, const IR::Instruction& inst); + +void EmitSelectU32(EmitterState* state, const IR::Instruction& inst); + +void EmitSelectMaskU32(EmitterState* state, const IR::Instruction& inst); + +void EmitSelectF32Bits(EmitterState* state, const IR::Instruction& inst); + +void EmitSelectMaskF32Bits(EmitterState* state, const IR::Instruction& inst); + +void EmitPackU16(EmitterState* state, const IR::Instruction& inst, bool high_src0, bool high_src1); + +void EmitPackU8F32(EmitterState* state, const IR::Instruction& inst); + +void EmitBinaryF32(EmitterState* state, const IR::Instruction& inst, uint32_t opcode); + +void EmitLdexpF32(EmitterState* state, const IR::Instruction& inst); + +uint32_t EmitCompareU32Constant(EmitterState* state, uint32_t opcode, uint32_t value, + uint32_t constant); + +uint32_t EmitSubConstantMinusU32(EmitterState* state, uint32_t constant, uint32_t value); + +uint32_t EmitF32ToF16RtzBits(EmitterState* state, uint32_t f32); + +void EmitPackF32ToF16Rtz(EmitterState* state, const IR::Instruction& inst); + +void EmitPackNormalizedF32(EmitterState* state, const IR::Instruction& inst, uint32_t ext_inst); + +uint32_t EmitMinMaxU32Value(EmitterState* state, uint32_t lhs, uint32_t rhs, bool max_value); + +void EmitMinMaxU32(EmitterState* state, const IR::Instruction& inst, bool max_value); + +void EmitSadU32(EmitterState* state, const IR::Instruction& inst); + +uint32_t EmitMinMaxI32Value(EmitterState* state, uint32_t lhs, uint32_t rhs, bool max_value); + +void EmitMinMaxI32(EmitterState* state, const IR::Instruction& inst, bool max_value); + +void EmitMinMax3U32(EmitterState* state, const IR::Instruction& inst, bool signed_value, + bool max_value); + +void EmitMed3U32(EmitterState* state, const IR::Instruction& inst, bool signed_value); + +uint32_t EmitBitcastF32ToU32(EmitterState* state, uint32_t value); + +uint32_t EmitBitcastU32ToF32(EmitterState* state, uint32_t value); + +uint32_t EmitAndU32(EmitterState* state, uint32_t lhs, uint32_t rhs); + +uint32_t EmitLogicalAndBool(EmitterState* state, uint32_t lhs, uint32_t rhs); + +uint32_t EmitLogicalOrBool(EmitterState* state, uint32_t lhs, uint32_t rhs); + +uint32_t EmitLogicalNotBool(EmitterState* state, uint32_t value); + +F32Class EmitClassifyF32(EmitterState* state, uint32_t value); + +uint32_t EmitClassMaskBitMatch(EmitterState* state, uint32_t mask, uint32_t bit, + uint32_t class_match); + +uint32_t EmitClassMaskF32(EmitterState* state, uint32_t value, uint32_t mask); + +uint32_t EmitMinMaxF32Value(EmitterState* state, uint32_t lhs, uint32_t rhs, bool max_value); + +void EmitMinMaxF32(EmitterState* state, const IR::Instruction& inst, bool max_value); + +void EmitCompareResult(EmitterState* state, const IR::Operand& dst, uint32_t cond); + +void EmitCompareU32(EmitterState* state, const IR::Instruction& inst, uint32_t opcode); + +void EmitCompareU16(EmitterState* state, const IR::Instruction& inst, uint32_t opcode); + +void EmitCompareI16(EmitterState* state, const IR::Instruction& inst, uint32_t opcode); + +void EmitBitCompareB32(EmitterState* state, const IR::Instruction& inst, bool bit_set); + +void EmitCompareNeU64(EmitterState* state, const IR::Instruction& inst); + +void EmitCompareConstant(EmitterState* state, const IR::Instruction& inst, bool value); + +void EmitCompareMaskU32(EmitterState* state, const IR::Instruction& inst, uint32_t opcode); + +void EmitCompareF32(EmitterState* state, const IR::Instruction& inst, uint32_t opcode); + +void EmitCompareOrderedF32(EmitterState* state, const IR::Instruction& inst, bool ordered); + +void EmitCompareClassF32(EmitterState* state, const IR::Instruction& inst); + +void EmitCompareMaskF32(EmitterState* state, const IR::Instruction& inst, uint32_t opcode); + +void EmitConvertU32ToF32(EmitterState* state, const IR::Instruction& inst); + +void EmitConvertI32ToF32(EmitterState* state, const IR::Instruction& inst); + +uint32_t EmitTruncF32Value(EmitterState* state, uint32_t value); + +void EmitConvertF32ToU32(EmitterState* state, const IR::Instruction& inst); + +void EmitConvertF32ToI32(EmitterState* state, const IR::Instruction& inst); + +void EmitConvertF32ToF16(EmitterState* state, const IR::Instruction& inst); + +void EmitConvertF16ToF32(EmitterState* state, const IR::Instruction& inst); + +void EmitConvertU16ToF16(EmitterState* state, const IR::Instruction& inst); + +void EmitConvertF16ToU16(EmitterState* state, const IR::Instruction& inst); + +void EmitConvertI16ToF16(EmitterState* state, const IR::Instruction& inst); + +void EmitConvertF16ToI16(EmitterState* state, const IR::Instruction& inst); + +void EmitConvertRoundPlusInfF32ToI32(EmitterState* state, const IR::Instruction& inst); + +void EmitConvertFloorF32ToI32(EmitterState* state, const IR::Instruction& inst); + +void EmitConvertI4ToOffsetF32(EmitterState* state, const IR::Instruction& inst); + +void EmitConvertByteU32ToF32(EmitterState* state, const IR::Instruction& inst); + +uint32_t EmitFlushF32DenormToSignedZero(EmitterState* state, uint32_t value); + +void EmitRcpF32(EmitterState* state, const IR::Instruction& inst); + +uint32_t EmitTrigCycleF32(EmitterState* state, uint32_t src, bool preserve_signed_zero); + +void EmitFloatExtInst(EmitterState* state, const IR::Instruction& inst, uint32_t ext_inst, + bool scale_by_two_pi = false, bool flush_denorm = false); + +void EmitMadF32(EmitterState* state, const IR::Instruction& inst); + +uint32_t EmitFNegateValue(EmitterState* state, uint32_t value); + +uint32_t EmitFAbsValue(EmitterState* state, uint32_t value); + +uint32_t EmitFCompareValue(EmitterState* state, uint32_t opcode, uint32_t lhs, uint32_t rhs); + +uint32_t EmitLogicalAndValue(EmitterState* state, uint32_t lhs, uint32_t rhs); + +uint32_t EmitFSelectValue(EmitterState* state, uint32_t cond, uint32_t true_value, + uint32_t false_value); + +uint32_t EmitFMulConstant(EmitterState* state, uint32_t value, uint32_t bits); + +CubeF32Values EmitCubeF32Values(EmitterState* state, const IR::Instruction& inst); + +uint32_t EmitCubeIdF32(EmitterState* state, const CubeF32Values& cube); + +uint32_t EmitCubeScF32(EmitterState* state, const CubeF32Values& cube); + +uint32_t EmitCubeTcF32(EmitterState* state, const CubeF32Values& cube); + +uint32_t EmitCubeMaF32(EmitterState* state, const CubeF32Values& cube); + +void EmitCubeF32(EmitterState* state, const IR::Instruction& inst); + +void EmitDot2AccF32F16(EmitterState* state, const IR::Instruction& inst); + +uint32_t EmitPackedF16Lane(EmitterState* state, const IR::Operand& operand, bool high_lane); + +void EmitCompareF16(EmitterState* state, const IR::Instruction& inst, uint32_t opcode); + +uint32_t EmitPackedF16LaneBits(EmitterState* state, const IR::Operand& operand, bool high_lane); + +uint32_t EmitF16BitsToF32(EmitterState* state, uint32_t bits); + +uint32_t EmitPackLowF32ToF16Bits(EmitterState* state, uint32_t value); + +void EmitPackB32F16(EmitterState* state, const IR::Instruction& inst); + +F16Class EmitClassifyF16Bits(EmitterState* state, uint32_t value); + +uint32_t EmitNegativeNumericF16Bool(EmitterState* state, const F16Class& cls); + +uint32_t EmitRcpF16Bits(EmitterState* state, uint32_t bits); + +uint32_t EmitSqrtF16Bits(EmitterState* state, uint32_t bits); + +uint32_t EmitRsqF16Bits(EmitterState* state, uint32_t bits); + +uint32_t EmitLog2F16Bits(EmitterState* state, uint32_t bits); + +uint32_t EmitExp2F16Bits(EmitterState* state, uint32_t bits); + +void EmitSpecialF16(EmitterState* state, const IR::Instruction& inst); + +void EmitBinaryF16(EmitterState* state, const IR::Instruction& inst, uint32_t opcode); + +void EmitFmaF16(EmitterState* state, const IR::Instruction& inst); + +uint32_t EmitMinMaxF16Bits(EmitterState* state, uint32_t lhs, uint32_t rhs, bool max_value); + +void EmitMinMaxF16(EmitterState* state, const IR::Instruction& inst, bool max_value); + +uint32_t EmitMinMax3F16Bits(EmitterState* state, uint32_t src0, uint32_t src1, uint32_t src2, + bool max_value); + +void EmitMinMax3F16(EmitterState* state, const IR::Instruction& inst, bool max_value); + +uint32_t EmitF16BitsEqualBool(EmitterState* state, uint32_t lhs, uint32_t rhs); + +uint32_t EmitAnyNanF16Bool(EmitterState* state, uint32_t src0, uint32_t src1, uint32_t src2); + +void EmitMed3F16(EmitterState* state, const IR::Instruction& inst); + +uint32_t EmitPackedMinMaxF16(EmitterState* state, const IR::Instruction& inst, bool max_value); + +uint32_t EmitPackedF16BinaryLane(EmitterState* state, const IR::Instruction& inst, bool high_lane, + uint32_t opcode); + +uint32_t EmitPackedF16FmaLane(EmitterState* state, const IR::Instruction& inst, bool high_lane); + +void EmitPackedF16(EmitterState* state, const IR::Instruction& inst); + +void EmitPackedInteger16(EmitterState* state, const IR::Instruction& inst); + +void EmitMadMixF16(EmitterState* state, const IR::Instruction& inst); + +void EmitMinMax3F32(EmitterState* state, const IR::Instruction& inst, bool max_value); + +void EmitMed3F32(EmitterState* state, const IR::Instruction& inst); + +uint32_t EmitExportComponentF32(EmitterState* state, const IR::Instruction& inst, + uint32_t component); + +uint32_t EmitExportVec4F32(EmitterState* state, const IR::Instruction& inst); + +bool ExportWritesData(const IR::Instruction& inst); + +void EmitExport(EmitterState* state, const IR::Instruction& inst); + +bool ExportUsesPixelValidMask(const EmitterState& state, const IR::Instruction& inst); + +void EmitKillIfBoolFalse(EmitterState* state, uint32_t active); + +void EmitUpdatePixelValidMask(EmitterState* state); + +void EmitKillIfPixelValidMaskInactive(EmitterState* state); + +void ComputeReachableBlocks(EmitterState* state, const IR::Program& program); + +void AllocateBlockLabels(EmitterState* state, const IR::Program& program); + +void AllocateDispatcherState(EmitterState* state, const IR::Program& program); + +uint32_t BlockLabel(const EmitterState& state, uint32_t block_id); + +uint32_t EmitBranchCondition(EmitterState* state, CFG::BranchCondition condition); + +void EmitStructuredPrefix(EmitterState* state, const CFG::Terminator& term); + +void EmitReturn(EmitterState* state); + +void EmitTerminator(EmitterState* state, const CFG::Terminator& term); + +uint32_t InitialRegisterValue(const EmitterState& state, IR::Register reg); + +void EmitRegisterVariables(EmitterState* state); + +void EmitComputeInputRegisters(EmitterState* state); + +void EmitPixelInputRegisters(EmitterState* state); + +void EmitInstruction(EmitterState* state, const IR::Instruction& inst); + +uint32_t DispatcherTargetValue(EmitterState* state, uint32_t block_id); + +void EmitDispatcherStoreTarget(EmitterState* state, uint32_t block_id); + +void EmitDispatcherExit(EmitterState* state); + +void EmitDispatcherTerminator(EmitterState* state, const CFG::Terminator& term); + +void EmitDispatcherSwitch(EmitterState* state, const IR::Program& program); + +void EmitDispatcherBlocks(EmitterState* state, const IR::Program& program); + +void EmitDispatcherLoopTail(EmitterState* state); + +void EmitDispatcherFunction(EmitterState* state, const IR::Program& program); + +void EmitFunction(EmitterState* state, const IR::Program& program); + +// These templates accept local lambdas from several emitter translation units. +template +void EmitIfCondition(EmitterState* state, uint32_t condition, Fn&& fn) { + if (condition == 0) { + fn(); + return; + } + + const auto then_label = state->builder.AllocateId(); + const auto merge_label = state->builder.AllocateId(); + state->builder.AddFunction({OpSelectionMerge, merge_label, SelectionControlNone}); + state->builder.AddFunction({OpBranchConditional, condition, then_label, merge_label}); + state->builder.AddFunction({OpLabel, then_label}); + fn(); + state->builder.AddFunction({OpBranch, merge_label}); + state->builder.AddFunction({OpLabel, merge_label}); +} + +template +uint32_t EmitValueOrZeroIfCondition(EmitterState* state, uint32_t condition, Fn&& fn) { + if (condition == 0) { + return fn(); + } + + const auto then_label = state->builder.AllocateId(); + const auto else_label = state->builder.AllocateId(); + const auto merge_label = state->builder.AllocateId(); + state->builder.AddFunction({OpSelectionMerge, merge_label, SelectionControlNone}); + state->builder.AddFunction({OpBranchConditional, condition, then_label, else_label}); + state->builder.AddFunction({OpLabel, then_label}); + const auto then_value = fn(); + state->builder.AddFunction({OpBranch, merge_label}); + state->builder.AddFunction({OpLabel, else_label}); + state->builder.AddFunction({OpBranch, merge_label}); + state->builder.AddFunction({OpLabel, merge_label}); + const auto value = state->builder.AllocateId(); + state->builder.AddFunction({OpPhi, state->uint_type, value, then_value, then_label, + ConstantU32(state, 0), else_label}); + return value; +} + +template +void EmitGuardedByExec(EmitterState* state, Fn&& fn) { + const auto then_label = state->builder.AllocateId(); + const auto merge_label = state->builder.AllocateId(); + const auto condition = EmitExecActiveBool(state); + state->builder.AddFunction({OpSelectionMerge, merge_label, SelectionControlNone}); + state->builder.AddFunction({OpBranchConditional, condition, then_label, merge_label}); + state->builder.AddFunction({OpLabel, then_label}); + fn(); + state->builder.AddFunction({OpBranch, merge_label}); + state->builder.AddFunction({OpLabel, merge_label}); +} + +} // namespace Libs::Graphics::ShaderRecompiler::Spirv::Emitter + +#endif /* EMULATOR_INCLUDE_EMULATOR_GRAPHICS_SHADER_RECOMPILER_SPIRVEMITTER_INTERNAL_H_ */ diff --git a/src/graphics/shader/recompiler/spirvEmitter/spirvEmitterMemory.cpp b/src/graphics/shader/recompiler/spirvEmitter/spirvEmitterMemory.cpp new file mode 100644 index 000000000..a25d5b8d6 --- /dev/null +++ b/src/graphics/shader/recompiler/spirvEmitter/spirvEmitterMemory.cpp @@ -0,0 +1,1587 @@ +#include "graphics/shader/recompiler/spirvEmitter/spirvEmitterInternal.h" + +namespace Libs::Graphics::ShaderRecompiler::Spirv::Emitter { + +uint32_t EmitDppWriteActiveBool(EmitterState* state, const IR::Operand& dst) { + const auto exec_active = EmitExecActiveBool(state); + if (!dst.dpp) { + return exec_active; + } + + const auto subid = EmitSubgroupLocalInvocationId(state); + const auto bank_shift = state->builder.AllocateId(); + const auto row_shift = state->builder.AllocateId(); + const auto bank = state->builder.AllocateId(); + const auto row = state->builder.AllocateId(); + const auto bank_bit = state->builder.AllocateId(); + const auto row_bit = state->builder.AllocateId(); + const auto bank_hit = state->builder.AllocateId(); + const auto row_hit = state->builder.AllocateId(); + const auto bank_active = state->builder.AllocateId(); + const auto row_active = state->builder.AllocateId(); + const auto dpp_active = state->builder.AllocateId(); + const auto ret = state->builder.AllocateId(); + + state->builder.AddFunction( + {OpShiftRightLogical, state->uint_type, bank_shift, subid, ConstantU32(state, 2)}); + state->builder.AddFunction( + {OpBitwiseAnd, state->uint_type, bank, bank_shift, ConstantU32(state, 3)}); + state->builder.AddFunction( + {OpShiftRightLogical, state->uint_type, row_shift, subid, ConstantU32(state, 4)}); + state->builder.AddFunction( + {OpBitwiseAnd, state->uint_type, row, row_shift, ConstantU32(state, 3)}); + state->builder.AddFunction( + {OpShiftLeftLogical, state->uint_type, bank_bit, ConstantU32(state, 1), bank}); + state->builder.AddFunction( + {OpShiftLeftLogical, state->uint_type, row_bit, ConstantU32(state, 1), row}); + state->builder.AddFunction({OpBitwiseAnd, state->uint_type, bank_hit, + ConstantU32(state, dst.dpp_bank_mask), bank_bit}); + state->builder.AddFunction( + {OpBitwiseAnd, state->uint_type, row_hit, ConstantU32(state, dst.dpp_row_mask), row_bit}); + state->builder.AddFunction( + {OpINotEqual, state->bool_type, bank_active, bank_hit, ConstantU32(state, 0)}); + state->builder.AddFunction( + {OpINotEqual, state->bool_type, row_active, row_hit, ConstantU32(state, 0)}); + state->builder.AddFunction( + {OpLogicalAnd, state->bool_type, dpp_active, bank_active, row_active}); + uint32_t write_active = dpp_active; + if (!dst.dpp_bound_ctrl) { + const auto target = EmitDppTargetLane(state, dst.dpp_ctrl); + const auto bounded = state->builder.AllocateId(); + state->builder.AddFunction( + {OpLogicalAnd, state->bool_type, bounded, write_active, target.valid}); + write_active = bounded; + } + state->builder.AddFunction({OpLogicalAnd, state->bool_type, ret, exec_active, write_active}); + return ret; +} + +uint32_t EmitSdwaDestinationMerge(EmitterState* state, uint32_t pointer, const IR::Operand& dst, + uint32_t value) { + if (dst.sdwa_sel == 6u) { + return value; + } + + uint32_t offset = 0; + uint32_t width = 0; + if (!SdwaSelectorOffsetWidth(dst.sdwa_sel, offset, width) || width == 32u) { + return value; + } + + const auto value_mask = width == 16u ? 0x0000ffffu : 0x000000ffu; + const auto field_mask = value_mask << offset; + const auto part = state->builder.AllocateId(); + const auto shifted = offset != 0u ? state->builder.AllocateId() : part; + const auto old_value = state->builder.AllocateId(); + const auto kept = state->builder.AllocateId(); + const auto merged = state->builder.AllocateId(); + state->builder.AddFunction( + {OpBitwiseAnd, state->uint_type, part, value, ConstantU32(state, value_mask)}); + if (offset != 0u) { + state->builder.AddFunction( + {OpShiftLeftLogical, state->uint_type, shifted, part, ConstantU32(state, offset)}); + } + state->builder.AddFunction({OpLoad, state->uint_type, old_value, pointer}); + state->builder.AddFunction( + {OpBitwiseAnd, state->uint_type, kept, old_value, ConstantU32(state, ~field_mask)}); + state->builder.AddFunction({OpBitwiseOr, state->uint_type, merged, kept, shifted}); + return merged; +} + +void EmitStoreU32(EmitterState* state, const IR::Operand& dst, uint32_t value) { + if (dst.kind != IR::OperandKind::Register) { + return; + } + const auto pointer = PointerForRegister(*state, dst.reg); + if (pointer != 0) { + const auto store_value = EmitSdwaDestinationMerge(state, pointer, dst, value); + const auto wave_value = + IsInactiveWave32ExecHigh(*state, dst.reg) ? ConstantU32(state, 0) : store_value; + if (dst.reg.file == IR::RegisterFile::Vector) { + const auto old_value = state->builder.AllocateId(); + const auto selected = state->builder.AllocateId(); + state->builder.AddFunction({OpLoad, state->uint_type, old_value, pointer}); + state->builder.AddFunction({OpSelect, state->uint_type, selected, + EmitDppWriteActiveBool(state, dst), wave_value, old_value}); + state->builder.AddFunction({OpStore, pointer, selected}); + return; + } + state->builder.AddFunction({OpStore, pointer, wave_value}); + } +} + +uint32_t EmitAddU32(EmitterState* state, uint32_t lhs, uint32_t rhs) { + const auto ret = state->builder.AllocateId(); + state->builder.AddFunction({OpIAdd, state->uint_type, ret, lhs, rhs}); + return ret; +} + +uint32_t EmitBinaryU32(EmitterState* state, uint32_t opcode, uint32_t lhs, uint32_t rhs) { + const auto ret = state->builder.AllocateId(); + state->builder.AddFunction({opcode, state->uint_type, ret, lhs, rhs}); + return ret; +} + +uint32_t EmitNotEqualZeroBool(EmitterState* state, uint32_t value) { + const auto ret = state->builder.AllocateId(); + state->builder.AddFunction({OpINotEqual, state->bool_type, ret, value, ConstantU32(state, 0)}); + return ret; +} + +uint32_t EmitSelectU32Value(EmitterState* state, uint32_t condition, uint32_t true_value, + uint32_t false_value) { + const auto ret = state->builder.AllocateId(); + state->builder.AddFunction( + {OpSelect, state->uint_type, ret, condition, true_value, false_value}); + return ret; +} + +uint32_t EmitByteAddress(EmitterState* state, const IR::Instruction& inst, uint32_t first_src, + uint32_t src_count) { + uint32_t address = ConstantU32(state, inst.memory.offset); + for (uint32_t i = 0; i < src_count; i++) { + address = EmitAddU32(state, address, EmitValueLoad(state, inst.src[first_src + i])); + } + return address; +} + +uint32_t StorageBufferPackedStride(const EmitterState& state, const IR::MemoryInfo& mem, + uint32_t use_pc) { + (void)use_pc; + if (mem.resource >= state.program->info.buffers.size()) { + ExitDescriptorBindingFailure(state, IR::DescriptorBindingKind::Buffers, mem.resource, + "buffer specialization is missing"); + } + return state.program->info.buffers[mem.resource].packed_stride; +} + +uint32_t StorageBufferFormat(const EmitterState& state, const IR::MemoryInfo& mem, + uint32_t use_pc) { + (void)use_pc; + if (mem.resource >= state.program->info.buffers.size()) { + ExitDescriptorBindingFailure(state, IR::DescriptorBindingKind::Buffers, mem.resource, + "buffer specialization is missing"); + } + return state.program->info.buffers[mem.resource].descriptor_format; +} + +bool ShouldApplyBufferAddTid(const IR::Instruction& inst) { + return inst.memory.kind == IR::ResourceKind::Buffer && inst.memory.formatted && + !inst.memory.typed && inst.op == IR::Opcode::BufferStoreDword; +} + +uint32_t EmitBufferIndexWithAddTid(EmitterState* state, const IR::Instruction& inst, uint32_t index, + uint32_t packed) { + if (!ShouldApplyBufferAddTid(inst)) { + return index; + } + + const auto add_tid_bit = + EmitBinaryU32(state, OpBitwiseAnd, + EmitBinaryU32(state, OpShiftRightLogical, packed, ConstantU32(state, 20)), + ConstantU32(state, 1)); + const auto add_tid = EmitNotEqualZeroBool(state, add_tid_bit); + const auto lane = + EmitBinaryU32(state, OpBitwiseAnd, EmitLocalInvocationIndex(state), ConstantU32(state, 63)); + const auto tid_index = EmitAddU32(state, index, lane); + return EmitSelectU32Value(state, add_tid, tid_index, index); +} + +uint32_t EmitOptionalLogicalAndBool(EmitterState* state, uint32_t lhs, uint32_t rhs) { + if (lhs == 0) { + return rhs; + } + if (rhs == 0) { + return lhs; + } + const auto ret = state->builder.AllocateId(); + state->builder.AddFunction({OpLogicalAnd, state->bool_type, ret, lhs, rhs}); + return ret; +} + +bool IsStorageBufferMemoryKind(IR::ResourceKind kind) { + switch (kind) { + case IR::ResourceKind::ScalarBuffer: + case IR::ResourceKind::Buffer: + case IR::ResourceKind::Flat: + case IR::ResourceKind::Global: + case IR::ResourceKind::Scratch: return true; + default: return false; + } +} + +uint32_t EmitBufferAddressFromParts(EmitterState* state, const IR::Instruction& inst, + uint32_t index, uint32_t offset, uint32_t soffset) { + const auto& mem = inst.memory; + const auto packed = ConstantU32(state, StorageBufferPackedStride(*state, mem, inst.pc)); + const auto address_index = EmitBufferIndexWithAddTid(state, inst, index, packed); + const auto stride = EmitBinaryU32(state, OpBitwiseAnd, packed, ConstantU32(state, 0x3fffu)); + const auto swizzle_bit = + EmitBinaryU32(state, OpBitwiseAnd, + EmitBinaryU32(state, OpShiftRightLogical, packed, ConstantU32(state, 14)), + ConstantU32(state, 1)); + const auto stride_nonzero = EmitNotEqualZeroBool(state, stride); + const auto swizzle_nonzero = EmitNotEqualZeroBool(state, swizzle_bit); + const auto use_swizzle = state->builder.AllocateId(); + state->builder.AddFunction( + {OpLogicalAnd, state->bool_type, use_swizzle, swizzle_nonzero, stride_nonzero}); + + const auto index_stride_enum = + EmitBinaryU32(state, OpBitwiseAnd, + EmitBinaryU32(state, OpShiftRightLogical, packed, ConstantU32(state, 16)), + ConstantU32(state, 3)); + const auto index_shift = EmitAddU32(state, index_stride_enum, ConstantU32(state, 3)); + const auto index_stride = + EmitBinaryU32(state, OpShiftLeftLogical, ConstantU32(state, 8), index_stride_enum); + const auto index_stride_mask = + EmitBinaryU32(state, OpISub, index_stride, ConstantU32(state, 1)); + + const auto linear_index = EmitBinaryU32(state, OpIMul, address_index, stride); + const auto linear = EmitAddU32(state, linear_index, offset); + + const auto index_msb = EmitBinaryU32(state, OpShiftRightLogical, address_index, index_shift); + const auto index_lsb = EmitBinaryU32(state, OpBitwiseAnd, address_index, index_stride_mask); + const auto offset_msb_bytes = + EmitBinaryU32(state, OpBitwiseAnd, offset, ConstantU32(state, ~3u)); + const auto offset_lsb = EmitBinaryU32(state, OpBitwiseAnd, offset, ConstantU32(state, 3)); + const auto index_part = EmitBinaryU32(state, OpIMul, index_msb, stride); + const auto msb_sum = EmitAddU32(state, index_part, offset_msb_bytes); + const auto msb = EmitBinaryU32(state, OpIMul, msb_sum, index_stride); + const auto lsb_index = + EmitBinaryU32(state, OpShiftLeftLogical, index_lsb, ConstantU32(state, 2)); + const auto lsb = EmitAddU32(state, lsb_index, offset_lsb); + const auto swizzled = EmitAddU32(state, msb, lsb); + const auto buffer_offset = EmitSelectU32Value(state, use_swizzle, swizzled, linear); + return EmitAddU32(state, buffer_offset, soffset); +} + +uint32_t EmitBufferByteAddress(EmitterState* state, const IR::Instruction& inst, uint32_t first_src, + uint32_t src_count) { + const auto end = first_src + src_count; + auto cursor = first_src; + auto LoadAddressSource = [&]() { + if (cursor >= end) { + return ConstantU32(state, 0); + } + return EmitValueLoad(state, inst.src[cursor++]); + }; + + uint32_t index = ConstantU32(state, 0); + if (inst.memory.idxen) { + index = LoadAddressSource(); + } + uint32_t offset = ConstantU32(state, inst.memory.offset); + if (inst.memory.offen) { + offset = EmitAddU32(state, offset, LoadAddressSource()); + } + const auto soffset = LoadAddressSource(); + return EmitBufferAddressFromParts(state, inst, index, offset, soffset); +} + +bool IsFlatMemoryKind(IR::ResourceKind kind) { + return kind == IR::ResourceKind::Flat || kind == IR::ResourceKind::Global || + kind == IR::ResourceKind::Scratch; +} + +uint32_t EmitRelativeAddress(EmitterState* state, const IR::Instruction& inst, uint32_t first_src, + uint32_t src_count, bool vector_only = true, + bool align_components = false) { + const auto delta = state->program->info.addresses[inst.memory.resource].specialized_base; + auto immediate = static_cast(inst.memory.offset); + if (align_components) { + immediate = static_cast(static_cast(immediate) & ~3u); + } + const auto initial = delta + static_cast(static_cast(immediate)); + uint32_t low = ConstantU32(state, static_cast(initial)); + uint32_t high = ConstantU32(state, static_cast(initial >> 32u)); + for (uint32_t i = 0; i < src_count; i++) { + const auto& src = inst.src[first_src + i]; + if (!vector_only || + (src.kind == IR::OperandKind::Register && src.reg.file == IR::RegisterFile::Vector)) { + auto value = EmitValueLoad(state, src); + if (align_components) { + value = EmitBinaryU32(state, OpBitwiseAnd, value, ConstantU32(state, ~3u)); + } + const auto next = EmitAddU32(state, low, value); + const auto carry = state->builder.AllocateId(); + state->builder.AddFunction({OpULessThan, state->bool_type, carry, next, low}); + high = EmitAddU32( + state, high, + EmitSelectU32Value(state, carry, ConstantU32(state, 1), ConstantU32(state, 0))); + low = next; + } + } + const auto valid = state->builder.AllocateId(); + state->builder.AddFunction({OpIEqual, state->bool_type, valid, high, ConstantU32(state, 0)}); + return EmitSelectU32Value(state, valid, low, ConstantU32(state, UINT32_MAX)); +} + +uint32_t EmitFlatVirtualAddress(EmitterState* state, const IR::Instruction& inst, + uint32_t first_src, uint32_t src_count) { + if (state->resources == nullptr || inst.memory.resource >= state->resources->addresses.size() || + src_count < 2) { + ExitDescriptorBindingFailure(*state, IR::DescriptorBindingKind::AddressMemory, + inst.memory.resource, "flat address snapshot is missing"); + } + const auto low0 = EmitValueLoad(state, inst.src[first_src]); + const auto high0 = EmitValueLoad(state, inst.src[first_src + 1u]); + const auto immediate = ConstantU32(state, inst.memory.offset); + const auto low = EmitAddU32(state, low0, immediate); + const auto carry_bool = state->builder.AllocateId(); + state->builder.AddFunction({OpULessThan, state->bool_type, carry_bool, low, low0}); + const auto carry = + EmitSelectU32Value(state, carry_bool, ConstantU32(state, 1), ConstantU32(state, 0)); + const auto immediate_high = + ConstantU32(state, (inst.memory.offset & 0x80000000u) != 0 ? UINT32_MAX : 0u); + const auto high = EmitAddU32(state, EmitAddU32(state, high0, immediate_high), carry); + + const auto base = state->program->info.addresses[inst.memory.resource].specialized_base; + const auto base_low = static_cast(base); + const auto base_high = static_cast(base >> 32u); + const auto relative_low = EmitBinaryU32(state, OpISub, low, ConstantU32(state, base_low)); + const auto borrow_bool = state->builder.AllocateId(); + state->builder.AddFunction( + {OpULessThan, state->bool_type, borrow_bool, low, ConstantU32(state, base_low)}); + const auto borrow = + EmitSelectU32Value(state, borrow_bool, ConstantU32(state, 1), ConstantU32(state, 0)); + const auto relative_high = EmitBinaryU32( + state, OpISub, EmitBinaryU32(state, OpISub, high, ConstantU32(state, base_high)), borrow); + const auto valid = state->builder.AllocateId(); + state->builder.AddFunction( + {OpIEqual, state->bool_type, valid, relative_high, ConstantU32(state, 0)}); + return EmitSelectU32Value(state, valid, relative_low, ConstantU32(state, UINT32_MAX)); +} + +uint32_t EmitMemoryByteAddress(EmitterState* state, const IR::Instruction& inst, + const IR::MemoryInfo& mem, uint32_t first_src, uint32_t src_count) { + if (mem.kind == IR::ResourceKind::Buffer) { + return EmitBufferByteAddress(state, inst, first_src, src_count); + } + if (mem.kind == IR::ResourceKind::Flat || + ((mem.kind == IR::ResourceKind::Global || mem.kind == IR::ResourceKind::Scratch) && + state->program->info.addresses[mem.resource].source == IR::ScalarProvenance::Unknown)) { + return EmitFlatVirtualAddress(state, inst, first_src, src_count); + } + if (mem.kind == IR::ResourceKind::Global || mem.kind == IR::ResourceKind::Scratch) { + return EmitRelativeAddress(state, inst, first_src, src_count); + } + return EmitByteAddress(state, inst, first_src, src_count); +} + +uint32_t EmitDwordIndex(EmitterState* state, const IR::Instruction& inst, uint32_t first_src, + uint32_t src_count) { + const auto address = EmitByteAddress(state, inst, first_src, src_count); + const auto index = state->builder.AllocateId(); + state->builder.AddFunction( + {OpShiftRightLogical, state->uint_type, index, address, ConstantU32(state, 2)}); + return index; +} + +uint32_t EmitMemoryDwordIndex(EmitterState* state, const IR::Instruction& inst, + const IR::MemoryInfo& mem, uint32_t first_src, uint32_t src_count) { + const auto address = EmitMemoryByteAddress(state, inst, mem, first_src, src_count); + const auto index = state->builder.AllocateId(); + state->builder.AddFunction( + {OpShiftRightLogical, state->uint_type, index, address, ConstantU32(state, 2)}); + return index; +} + +DescriptorResourceBinding +StorageBufferBindingForMemory(EmitterState* state, const IR::MemoryInfo& mem, uint32_t use_pc) { + (void)use_pc; + const auto binding = + ResourceForDescriptor(*state, IR::DescriptorBindingKind::Buffers, mem.resource); + if (state->storage_buffer_variable == 0) { + ExitDescriptorBindingFailure(*state, IR::DescriptorBindingKind::Buffers, mem.resource, + "storage buffer descriptor array was not emitted"); + } + return binding; +} + +uint32_t EmitStorageBufferObjectPointer(EmitterState* state, const IR::MemoryInfo& mem, + uint32_t use_pc) { + if (IsFlatMemoryKind(mem.kind)) { + if (state->address_memory_variable == 0) { + ExitDescriptorBindingFailure(*state, IR::DescriptorBindingKind::AddressMemory, + mem.resource, "address memory binding was not emitted"); + } + const auto binding = + ResourceForDescriptor(*state, IR::DescriptorBindingKind::AddressMemory, mem.resource); + const auto pointer = state->builder.AllocateId(); + state->builder.AddFunction({OpAccessChain, state->ptr_storage_buffer, pointer, + state->address_memory_variable, + ConstantU32(state, binding.array_index)}); + return pointer; + } + const auto binding = StorageBufferBindingForMemory(state, mem, use_pc); + const auto pointer = state->builder.AllocateId(); + state->builder.AddFunction({OpAccessChain, state->ptr_storage_buffer, pointer, + state->storage_buffer_variable, + ConstantU32(state, binding.array_index)}); + return pointer; +} + +uint32_t EmitStorageBufferElementInBounds(EmitterState* state, const IR::MemoryInfo& mem, + uint32_t index, uint32_t use_pc) { + const auto object = EmitStorageBufferObjectPointer(state, mem, use_pc); + const auto length = state->builder.AllocateId(); + const auto in_bounds = state->builder.AllocateId(); + state->builder.AddFunction({OpArrayLength, state->uint_type, length, object, 0}); + state->builder.AddFunction({OpULessThan, state->bool_type, in_bounds, index, length}); + return in_bounds; +} + +uint32_t EmitStorageBufferElementPointer(EmitterState* state, const IR::MemoryInfo& mem, + uint32_t index, uint32_t use_pc) { + if (IsFlatMemoryKind(mem.kind)) { + if (state->address_memory_variable == 0) { + ExitDescriptorBindingFailure(*state, IR::DescriptorBindingKind::AddressMemory, + mem.resource, "address memory binding was not emitted"); + } + const auto binding = + ResourceForDescriptor(*state, IR::DescriptorBindingKind::AddressMemory, mem.resource); + const auto pointer = state->builder.AllocateId(); + state->builder.AddFunction( + {OpAccessChain, state->ptr_storage_buffer_uint, pointer, state->address_memory_variable, + ConstantU32(state, binding.array_index), ConstantU32(state, 0), index}); + return pointer; + } + const auto binding = StorageBufferBindingForMemory(state, mem, use_pc); + const auto pointer = state->builder.AllocateId(); + state->builder.AddFunction( + {OpAccessChain, state->ptr_storage_buffer_uint, pointer, state->storage_buffer_variable, + ConstantU32(state, binding.array_index), ConstantU32(state, 0), index}); + return pointer; +} + +uint32_t EmitLdsElementPointer(EmitterState* state, uint32_t index) { + const auto pointer = state->builder.AllocateId(); + state->builder.AddFunction( + {OpAccessChain, state->ptr_workgroup_uint, pointer, state->lds_variable, index}); + return pointer; +} + +uint32_t EmitLdsElementInBounds(EmitterState* state, uint32_t index) { + const auto in_bounds = state->builder.AllocateId(); + const auto dwords = state->needs_function_lds ? 8192u : 1024u; + state->builder.AddFunction( + {OpULessThan, state->bool_type, in_bounds, index, ConstantU32(state, dwords)}); + return in_bounds; +} + +uint32_t EmitMemoryElementPointer(EmitterState* state, const IR::MemoryInfo& mem, uint32_t index, + uint32_t use_pc) { + if (mem.kind == IR::ResourceKind::Lds) { + return EmitLdsElementPointer(state, index); + } + if (mem.kind == IR::ResourceKind::Gds) { + return EmitGdsElementPointer(state, index); + } + return EmitStorageBufferElementPointer(state, mem, index, use_pc); +} + +uint32_t EmitMemoryLoadDwordValueU32(EmitterState* state, const IR::Instruction& inst, + IR::ResourceKind kind, uint32_t first_src, + uint32_t src_count) { + auto mem = inst.memory; + mem.kind = kind; + const auto index = EmitMemoryDwordIndex(state, inst, mem, first_src, src_count); + auto LoadFromPointer = [&](uint32_t pointer) { + const auto value = state->builder.AllocateId(); + state->builder.AddFunction({OpLoad, state->uint_type, value, pointer}); + return value; + }; + if (mem.kind == IR::ResourceKind::Lds) { + return LoadFromPointer(EmitLdsElementPointer(state, index)); + } + if (mem.kind == IR::ResourceKind::Gds) { + const auto in_bounds = EmitGdsElementInBounds(state, index); + return EmitValueOrZeroIfCondition(state, in_bounds, [&]() { + return LoadFromPointer(EmitGdsElementPointer(state, index)); + }); + } + if (IsStorageBufferMemoryKind(mem.kind)) { + const auto in_bounds = EmitStorageBufferElementInBounds(state, mem, index, inst.pc); + return EmitValueOrZeroIfCondition(state, in_bounds, [&]() { + return LoadFromPointer(EmitStorageBufferElementPointer(state, mem, index, inst.pc)); + }); + } + return ConstantU32(state, 0); +} + +void EmitMemoryLoadU32(EmitterState* state, const IR::Instruction& inst, IR::ResourceKind kind, + uint32_t first_src, uint32_t src_count) { + const auto value = EmitMemoryLoadDwordValueU32(state, inst, kind, first_src, src_count); + EmitStoreU32(state, inst.dst, value); +} + +void EmitMemoryLoadDwordsU32(EmitterState* state, const IR::Instruction& inst, + IR::ResourceKind kind, uint32_t first_src, uint32_t src_count) { + const auto count = std::max(1u, inst.memory.data_dwords); + if (count == 1u) { + EmitMemoryLoadU32(state, inst, kind, first_src, src_count); + return; + } + + std::vector values; + values.reserve(count); + for (uint32_t i = 0; i < count; i++) { + auto load_inst = inst; + load_inst.memory.offset += i * 4u; + load_inst.memory.data_dwords = 1u; + load_inst.memory.component_index = i; + load_inst.memory.component_count = count; + values.push_back(EmitMemoryLoadDwordValueU32(state, load_inst, kind, first_src, src_count)); + } + for (uint32_t i = 0; i < count; i++) { + EmitStoreU32(state, OffsetRegisterOperand(inst.dst, i), values[i]); + } +} + +uint32_t EmitMemoryLoadSubDwordValueU32(EmitterState* state, const IR::Instruction& inst, + IR::ResourceKind kind, uint32_t first_src, + uint32_t src_count, uint32_t data_bits, bool sign_extend) { + auto mem = inst.memory; + mem.kind = kind; + const auto address = EmitMemoryByteAddress(state, inst, mem, first_src, src_count); + const auto index = state->builder.AllocateId(); + state->builder.AddFunction( + {OpShiftRightLogical, state->uint_type, index, address, ConstantU32(state, 2)}); + auto LoadFromPointer = [&](uint32_t pointer) { + const auto word = state->builder.AllocateId(); + const auto byte_in_word = state->builder.AllocateId(); + const auto shift = state->builder.AllocateId(); + const auto shifted = state->builder.AllocateId(); + const auto masked = state->builder.AllocateId(); + const auto value = sign_extend ? state->builder.AllocateId() : masked; + const auto mask = data_bits == 8u ? 0xffu : 0xffffu; + state->builder.AddFunction({OpLoad, state->uint_type, word, pointer}); + state->builder.AddFunction( + {OpBitwiseAnd, state->uint_type, byte_in_word, address, ConstantU32(state, 3)}); + state->builder.AddFunction( + {OpShiftLeftLogical, state->uint_type, shift, byte_in_word, ConstantU32(state, 3)}); + state->builder.AddFunction({OpShiftRightLogical, state->uint_type, shifted, word, shift}); + state->builder.AddFunction( + {OpBitwiseAnd, state->uint_type, masked, shifted, ConstantU32(state, mask)}); + if (sign_extend) { + const auto left = state->builder.AllocateId(); + const auto sign_shift = 32u - data_bits; + state->builder.AddFunction({OpShiftLeftLogical, state->uint_type, left, masked, + ConstantU32(state, sign_shift)}); + state->builder.AddFunction({OpShiftRightArithmetic, state->uint_type, value, left, + ConstantU32(state, sign_shift)}); + } + return value; + }; + if (mem.kind == IR::ResourceKind::Lds) { + return LoadFromPointer(EmitLdsElementPointer(state, index)); + } + if (mem.kind == IR::ResourceKind::Gds) { + const auto in_bounds = EmitGdsElementInBounds(state, index); + return EmitValueOrZeroIfCondition(state, in_bounds, [&]() { + return LoadFromPointer(EmitGdsElementPointer(state, index)); + }); + } + if (IsStorageBufferMemoryKind(mem.kind)) { + const auto in_bounds = EmitStorageBufferElementInBounds(state, mem, index, inst.pc); + return EmitValueOrZeroIfCondition(state, in_bounds, [&]() { + return LoadFromPointer(EmitStorageBufferElementPointer(state, mem, index, inst.pc)); + }); + } + return ConstantU32(state, 0); +} + +void EmitMemoryLoadSubDwordU32(EmitterState* state, const IR::Instruction& inst, + IR::ResourceKind kind, uint32_t first_src, uint32_t src_count, + uint32_t data_bits, bool sign_extend) { + const auto value = EmitMemoryLoadSubDwordValueU32(state, inst, kind, first_src, src_count, + data_bits, sign_extend); + EmitStoreU32(state, inst.dst, value); +} + +void EmitMemoryStoreU32(EmitterState* state, const IR::Instruction& inst, IR::ResourceKind kind, + uint32_t first_src, uint32_t src_count) { + auto store_inst = inst; + store_inst.memory.kind = kind; + const auto value = EmitValueLoad(state, inst.src[0]); + uint32_t in_bounds = 0; + const auto index = + EmitMemoryDwordIndex(state, store_inst, store_inst.memory, first_src, src_count); + if (IsStorageBufferMemoryKind(store_inst.memory.kind)) { + in_bounds = EmitOptionalLogicalAndBool( + state, in_bounds, + EmitStorageBufferElementInBounds(state, store_inst.memory, index, inst.pc)); + } else if (store_inst.memory.kind == IR::ResourceKind::Gds) { + in_bounds = EmitGdsElementInBounds(state, index); + } + EmitIfCondition(state, in_bounds, [&]() { + const auto pointer = EmitMemoryElementPointer(state, store_inst.memory, index, inst.pc); + state->builder.AddFunction({OpStore, pointer, value}); + }); +} + +template +void EmitAtomicUpdateU32(EmitterState* state, uint32_t pointer, IR::ResourceKind kind, + Fn&& desired_value) { + const auto scope = kind == IR::ResourceKind::Lds ? ScopeWorkgroup : ScopeDevice; + const auto memory = kind == IR::ResourceKind::Lds ? MemorySemanticsWorkgroupMemory + : MemorySemanticsUniformMemory; + const auto preheader = state->builder.AllocateId(); + const auto header = state->builder.AllocateId(); + const auto continue_label = state->builder.AllocateId(); + const auto merge = state->builder.AllocateId(); + const auto initial = state->builder.AllocateId(); + const auto observed = state->builder.AllocateId(); + const auto exchanged = state->builder.AllocateId(); + + state->builder.AddFunction({OpBranch, preheader}); + state->builder.AddFunction({OpLabel, preheader}); + state->builder.AddFunction({OpAtomicLoad, state->uint_type, initial, pointer, + ConstantU32(state, scope), + ConstantU32(state, MemorySemanticsNone)}); + state->builder.AddFunction({OpBranch, header}); + state->builder.AddFunction({OpLabel, header}); + state->builder.AddFunction( + {OpPhi, state->uint_type, observed, initial, preheader, exchanged, continue_label}); + const auto desired = desired_value(observed); + state->builder.AddFunction({OpAtomicCompareExchange, state->uint_type, exchanged, pointer, + ConstantU32(state, scope), ConstantU32(state, MemorySemanticsNone), + ConstantU32(state, MemorySemanticsNone), desired, observed}); + const auto success = state->builder.AllocateId(); + state->builder.AddFunction({OpIEqual, state->bool_type, success, exchanged, observed}); + state->builder.AddFunction({OpLoopMerge, merge, continue_label, LoopControlNone}); + state->builder.AddFunction({OpBranchConditional, success, merge, continue_label}); + state->builder.AddFunction({OpLabel, continue_label}); + state->builder.AddFunction({OpBranch, header}); + state->builder.AddFunction({OpLabel, merge}); + const auto semantics = MemorySemanticsAcquireRelease | memory; + state->builder.AddFunction( + {OpMemoryBarrier, ConstantU32(state, scope), ConstantU32(state, semantics)}); +} + +void EmitMemoryStoreSubDwordU32(EmitterState* state, const IR::Instruction& inst, + IR::ResourceKind kind, uint32_t first_src, uint32_t src_count, + uint32_t data_bits) { + auto store_inst = inst; + store_inst.memory.kind = kind; + uint32_t in_bounds = 0; + const auto address = + EmitMemoryByteAddress(state, store_inst, store_inst.memory, first_src, src_count); + const auto index = state->builder.AllocateId(); + state->builder.AddFunction( + {OpShiftRightLogical, state->uint_type, index, address, ConstantU32(state, 2)}); + if (IsStorageBufferMemoryKind(store_inst.memory.kind)) { + in_bounds = EmitOptionalLogicalAndBool( + state, in_bounds, + EmitStorageBufferElementInBounds(state, store_inst.memory, index, inst.pc)); + } else if (store_inst.memory.kind == IR::ResourceKind::Gds) { + in_bounds = EmitGdsElementInBounds(state, index); + } + EmitIfCondition(state, in_bounds, [&]() { + const auto pointer = EmitMemoryElementPointer(state, store_inst.memory, index, inst.pc); + const auto value = EmitValueLoad(state, inst.src[0]); + const auto byte_in_word = state->builder.AllocateId(); + const auto shift = state->builder.AllocateId(); + const auto field_mask = state->builder.AllocateId(); + const auto value_masked = state->builder.AllocateId(); + const auto value_shifted = state->builder.AllocateId(); + const auto mask = data_bits == 8u ? 0xffu : 0xffffu; + state->builder.AddFunction( + {OpBitwiseAnd, state->uint_type, byte_in_word, address, ConstantU32(state, 3)}); + state->builder.AddFunction( + {OpShiftLeftLogical, state->uint_type, shift, byte_in_word, ConstantU32(state, 3)}); + state->builder.AddFunction( + {OpShiftLeftLogical, state->uint_type, field_mask, ConstantU32(state, mask), shift}); + state->builder.AddFunction( + {OpBitwiseAnd, state->uint_type, value_masked, value, ConstantU32(state, mask)}); + state->builder.AddFunction( + {OpShiftLeftLogical, state->uint_type, value_shifted, value_masked, shift}); + auto Merge = [&](uint32_t word) { + const auto inverted = state->builder.AllocateId(); + const auto cleared = state->builder.AllocateId(); + const auto merged = state->builder.AllocateId(); + state->builder.AddFunction({OpNot, state->uint_type, inverted, field_mask}); + state->builder.AddFunction({OpBitwiseAnd, state->uint_type, cleared, word, inverted}); + state->builder.AddFunction( + {OpBitwiseOr, state->uint_type, merged, cleared, value_shifted}); + return merged; + }; + if (store_inst.memory.kind == IR::ResourceKind::Lds || + store_inst.memory.kind == IR::ResourceKind::Gds) { + EmitAtomicUpdateU32(state, pointer, store_inst.memory.kind, Merge); + } else { + const auto word = state->builder.AllocateId(); + state->builder.AddFunction({OpLoad, state->uint_type, word, pointer}); + state->builder.AddFunction({OpStore, pointer, Merge(word)}); + } + }); +} + +uint32_t AddressSourceCount(const IR::Instruction& inst, uint32_t first_src) { + return inst.src_count > first_src ? inst.src_count - first_src : 0u; +} + +bool IsFormattedBufferComponent(const IR::Instruction& inst) { + return inst.memory.kind == IR::ResourceKind::Buffer && inst.memory.formatted && + inst.memory.data_dwords == 1u; +} + +Prospero::BufferFormat FormattedBufferFormat(const EmitterState& state, const IR::Instruction& inst) { + return inst.memory.typed + ? Format::DecodeTBufferFormat(inst.memory.data_format, inst.memory.number_format) + : static_cast(StorageBufferFormat(state, inst.memory, inst.pc)); +} + +IR::Instruction WithFormatComponentByteOffset(const IR::Instruction& inst, Prospero::BufferFormat format) { + auto rebased = inst; + if (inst.memory.typed || inst.memory.component_index == 0u) { + return rebased; + } + + const auto default_component_offset = inst.memory.component_index * 4u; + if (rebased.memory.offset >= default_component_offset) { + rebased.memory.offset = + rebased.memory.offset - default_component_offset + + Format::GetFormatComponentByteOffset(format, inst.memory.component_index); + } + return rebased; +} + +uint32_t EmitTBufferBitcastF32ToU32(EmitterState* state, uint32_t value) { + const auto ret = state->builder.AllocateId(); + state->builder.AddFunction({OpBitcast, state->uint_type, ret, value}); + return ret; +} + +uint32_t EmitTBufferBitcastU32ToF32(EmitterState* state, uint32_t value) { + const auto ret = state->builder.AllocateId(); + state->builder.AddFunction({OpBitcast, state->float_type, ret, value}); + return ret; +} + +uint32_t EmitTBufferBitcastU32ToI32(EmitterState* state, uint32_t value) { + const auto ret = state->builder.AllocateId(); + state->builder.AddFunction({OpBitcast, state->int_type, ret, value}); + return ret; +} + +uint32_t EmitTBufferCompareU32Constant(EmitterState* state, uint32_t opcode, uint32_t value, + uint32_t constant) { + const auto ret = state->builder.AllocateId(); + state->builder.AddFunction( + {opcode, state->bool_type, ret, value, ConstantU32(state, constant)}); + return ret; +} + +uint32_t EmitTBufferSelectF32(EmitterState* state, uint32_t condition, uint32_t true_value, + uint32_t false_value) { + const auto ret = state->builder.AllocateId(); + state->builder.AddFunction( + {OpSelect, state->float_type, ret, condition, true_value, false_value}); + return ret; +} + +bool IsSignedFormatComponent(Format::ComponentType type) { + return type == Format::ComponentType::Sint || type == Format::ComponentType::Snorm || + type == Format::ComponentType::Sscaled; +} + +uint32_t EmitExtractFormatFieldU32(EmitterState* state, uint32_t raw_word, uint32_t offset, + uint32_t bits, bool sign_extend) { + if (bits == 32u && offset == 0u) { + return raw_word; + } + + if (sign_extend) { + const auto signed_word = EmitTBufferBitcastU32ToI32(state, raw_word); + const auto extracted = state->builder.AllocateId(); + state->builder.AddFunction({OpBitFieldSExtract, state->int_type, extracted, signed_word, + ConstantU32(state, offset), ConstantU32(state, bits)}); + const auto ret = state->builder.AllocateId(); + state->builder.AddFunction({OpBitcast, state->uint_type, ret, extracted}); + return ret; + } + + const auto extracted = state->builder.AllocateId(); + state->builder.AddFunction({OpBitFieldUExtract, state->uint_type, extracted, raw_word, + ConstantU32(state, offset), ConstantU32(state, bits)}); + return extracted; +} + +uint32_t EmitHalfToF32Bits(EmitterState* state, uint32_t raw) { + const auto unpacked = state->builder.AllocateId(); + const auto value = state->builder.AllocateId(); + state->builder.AddFunction( + {OpExtInst, state->vec2_float_type, unpacked, state->glsl_std450, GlslUnpackHalf2x16, raw}); + state->builder.AddFunction({OpCompositeExtract, state->float_type, value, unpacked, 0}); + return EmitTBufferBitcastF32ToU32(state, value); +} + +uint32_t EmitUFloatToF32Bits(EmitterState* state, uint32_t raw, uint32_t bits) { + const auto mantissa_bits = bits == 11u ? 6u : 5u; + const auto mantissa_mask = (1u << mantissa_bits) - 1u; + const auto mantissa = + EmitBinaryU32(state, OpBitwiseAnd, raw, ConstantU32(state, mantissa_mask)); + const auto exponent = EmitBinaryU32( + state, OpBitwiseAnd, + EmitBinaryU32(state, OpShiftRightLogical, raw, ConstantU32(state, mantissa_bits)), + ConstantU32(state, 0x1fu)); + + const auto exponent_32 = EmitAddU32(state, exponent, ConstantU32(state, 127u - 15u)); + const auto exponent_bits = + EmitBinaryU32(state, OpShiftLeftLogical, exponent_32, ConstantU32(state, 23)); + const auto mantissa_bits_32 = + EmitBinaryU32(state, OpShiftLeftLogical, mantissa, ConstantU32(state, 23u - mantissa_bits)); + const auto normal_bits = EmitBinaryU32(state, OpBitwiseOr, exponent_bits, mantissa_bits_32); + const auto normal = EmitTBufferBitcastU32ToF32(state, normal_bits); + + const auto special_bits = + EmitBinaryU32(state, OpBitwiseOr, ConstantU32(state, 0x7f800000u), mantissa_bits_32); + const auto special = EmitTBufferBitcastU32ToF32(state, special_bits); + + const auto mantissa_f32 = state->builder.AllocateId(); + const auto subnormal = state->builder.AllocateId(); + state->builder.AddFunction({OpConvertUToF, state->float_type, mantissa_f32, mantissa}); + state->builder.AddFunction( + {OpFMul, state->float_type, subnormal, mantissa_f32, + ConstantF32Value(state, std::ldexp(1.0f, 1 - 15 - static_cast(mantissa_bits)))}); + + const auto zero_exp = EmitTBufferCompareU32Constant(state, OpIEqual, exponent, 0); + const auto special_exp = EmitTBufferCompareU32Constant(state, OpIEqual, exponent, 31); + const auto finite = EmitTBufferSelectF32(state, zero_exp, subnormal, normal); + const auto result = EmitTBufferSelectF32(state, special_exp, special, finite); + return EmitTBufferBitcastF32ToU32(state, result); +} + +uint32_t EmitFormatRawComponent(EmitterState* state, const IR::Instruction& inst, + const Format::BufferFormatInfo& info) { + const auto component = inst.memory.component_index; + if (component >= info.component_count) { + return ConstantU32(state, 0); + } + + auto load_inst = WithFormatComponentByteOffset(inst, info.format); + const auto bits = info.component_bits[component]; + const auto sign_extend = IsSignedFormatComponent(info.type); + if (info.packed_bitfield) { + const auto raw_word = EmitMemoryLoadDwordValueU32( + state, load_inst, IR::ResourceKind::Buffer, 0, AddressSourceCount(load_inst, 0)); + return EmitExtractFormatFieldU32(state, raw_word, info.component_bit_offset[component], + bits, sign_extend); + } + + if (bits == 32u) { + return EmitMemoryLoadDwordValueU32(state, load_inst, IR::ResourceKind::Buffer, 0, + AddressSourceCount(load_inst, 0)); + } + + return EmitMemoryLoadSubDwordValueU32(state, load_inst, IR::ResourceKind::Buffer, 0, + AddressSourceCount(load_inst, 0), bits, sign_extend); +} + +uint32_t NormalizeFormatComponent(EmitterState* state, const Format::BufferFormatInfo& info, + uint32_t component, uint32_t raw) { + const auto bits = info.component_bits[component]; + switch (info.type) { + case Format::ComponentType::Uint: + case Format::ComponentType::Sint: return raw; + case Format::ComponentType::Uscaled: { + const auto value = state->builder.AllocateId(); + state->builder.AddFunction({OpConvertUToF, state->float_type, value, raw}); + return EmitTBufferBitcastF32ToU32(state, value); + } + case Format::ComponentType::Sscaled: { + const auto signed_raw = EmitTBufferBitcastU32ToI32(state, raw); + const auto value = state->builder.AllocateId(); + state->builder.AddFunction({OpConvertSToF, state->float_type, value, signed_raw}); + return EmitTBufferBitcastF32ToU32(state, value); + } + case Format::ComponentType::Unorm: { + const auto value = state->builder.AllocateId(); + const auto normalized = state->builder.AllocateId(); + const auto max_value = static_cast((1u << bits) - 1u); + state->builder.AddFunction({OpConvertUToF, state->float_type, value, raw}); + state->builder.AddFunction( + {OpFDiv, state->float_type, normalized, value, ConstantF32Value(state, max_value)}); + return EmitTBufferBitcastF32ToU32(state, normalized); + } + case Format::ComponentType::Snorm: { + const auto signed_raw = EmitTBufferBitcastU32ToI32(state, raw); + const auto value = state->builder.AllocateId(); + const auto normalized = state->builder.AllocateId(); + const auto clamped = state->builder.AllocateId(); + const auto max_value = static_cast((1u << (bits - 1u)) - 1u); + state->builder.AddFunction({OpConvertSToF, state->float_type, value, signed_raw}); + state->builder.AddFunction( + {OpFDiv, state->float_type, normalized, value, ConstantF32Value(state, max_value)}); + state->builder.AddFunction({OpExtInst, state->float_type, clamped, state->glsl_std450, + GlslFMax, normalized, ConstantF32Value(state, -1.0f)}); + return EmitTBufferBitcastF32ToU32(state, clamped); + } + case Format::ComponentType::Float: + if (bits == 32u) { + return raw; + } + if (bits == 16u) { + return EmitHalfToF32Bits(state, raw); + } + return EmitUFloatToF32Bits(state, raw, bits); + default: return raw; + } +} + +uint32_t UnpackTBufferFormat(EmitterState* state, const IR::Instruction& inst, + const Format::BufferFormatInfo& info) { + const auto raw = EmitFormatRawComponent(state, inst, info); + return NormalizeFormatComponent(state, info, inst.memory.component_index, raw); +} + +bool EmitTypedTBufferLoad(EmitterState* state, const IR::Instruction& inst, + const Format::BufferFormatInfo& info) { + if (!Format::CanUseTypedBufferLoad(info.format)) { + return false; + } + const auto value = EmitMemoryLoadDwordValueU32(state, inst, IR::ResourceKind::Buffer, 0, + AddressSourceCount(inst, 0)); + EmitStoreU32(state, inst.dst, value); + return true; +} + +bool EmitFormattedBufferLoad(EmitterState* state, const IR::Instruction& inst) { + if (!IsFormattedBufferComponent(inst)) { + return false; + } + + const auto format = FormattedBufferFormat(*state, inst); + if (!Format::IsKnownFormat(format)) { + return false; + } + + const auto info = Format::GetFormatInfo(format); + if (inst.memory.component_index >= info.component_count) { + EmitStoreU32(state, inst.dst, ConstantU32(state, 0)); + return true; + } + + if (EmitTypedTBufferLoad(state, inst, info)) { + return true; + } + + EmitStoreU32(state, inst.dst, UnpackTBufferFormat(state, inst, info)); + return true; +} + +uint32_t FormattedBufferDwordStoreComponentCount(Prospero::BufferFormat format, + uint32_t opcode_components) { + switch (format) { + case Prospero::BufferFormat::k32UInt: + case Prospero::BufferFormat::k32SInt: + case Prospero::BufferFormat::k32Float: return opcode_components >= 1u ? 1u : 0u; + case Prospero::BufferFormat::k32_32UInt: + case Prospero::BufferFormat::k32_32SInt: + case Prospero::BufferFormat::k32_32Float: return opcode_components >= 2u ? 2u : 0u; + case Prospero::BufferFormat::k32_32_32UInt: + case Prospero::BufferFormat::k32_32_32SInt: + case Prospero::BufferFormat::k32_32_32Float: return opcode_components >= 3u ? 3u : 0u; + case Prospero::BufferFormat::k32_32_32_32UInt: + case Prospero::BufferFormat::k32_32_32_32SInt: + case Prospero::BufferFormat::k32_32_32_32Float: return opcode_components >= 4u ? 4u : 0u; + default: return 0u; + } +} + +bool EmitBufferIntegerFormatStore(EmitterState* state, const IR::Instruction& inst) { + if (!IsFormattedBufferComponent(inst)) { + return false; + } + + const auto format = FormattedBufferFormat(*state, inst); + if (!inst.memory.typed) { + const auto dword_components = + FormattedBufferDwordStoreComponentCount(format, inst.memory.component_count); + if (dword_components != 0u) { + if (inst.memory.component_index >= dword_components) { + return true; + } + EmitMemoryStoreU32(state, inst, IR::ResourceKind::Buffer, 1, + AddressSourceCount(inst, 1)); + return true; + } + + switch (format) { + case Prospero::BufferFormat::k8UInt: + case Prospero::BufferFormat::k8SInt: + if (inst.memory.component_index != 0u) { + return true; + } + EmitMemoryStoreSubDwordU32(state, inst, IR::ResourceKind::Buffer, 1, + AddressSourceCount(inst, 1), 8); + return true; + case Prospero::BufferFormat::k16UInt: + case Prospero::BufferFormat::k16SInt: + if (inst.memory.component_index != 0u) { + return true; + } + EmitMemoryStoreSubDwordU32(state, inst, IR::ResourceKind::Buffer, 1, + AddressSourceCount(inst, 1), 16); + return true; + case Prospero::BufferFormat::k8_8UInt: + case Prospero::BufferFormat::k8_8SInt: { + if (inst.memory.component_index >= 2u) { + return true; + } + const auto rebased = WithFormatComponentByteOffset(inst, format); + EmitMemoryStoreSubDwordU32(state, rebased, IR::ResourceKind::Buffer, 1, + AddressSourceCount(rebased, 1), 8); + return true; + } + case Prospero::BufferFormat::k16_16UInt: + case Prospero::BufferFormat::k16_16SInt: { + if (inst.memory.component_index >= 2u) { + return true; + } + const auto rebased = WithFormatComponentByteOffset(inst, format); + EmitMemoryStoreSubDwordU32(state, rebased, IR::ResourceKind::Buffer, 1, + AddressSourceCount(rebased, 1), 16); + return true; + } + default: return false; + } + } + + switch (format) { + case Prospero::BufferFormat::k8UInt: + case Prospero::BufferFormat::k8SInt: + EmitMemoryStoreSubDwordU32(state, inst, IR::ResourceKind::Buffer, 1, + AddressSourceCount(inst, 1), 8); + return true; + case Prospero::BufferFormat::k16UInt: + case Prospero::BufferFormat::k16SInt: + EmitMemoryStoreSubDwordU32(state, inst, IR::ResourceKind::Buffer, 1, + AddressSourceCount(inst, 1), 16); + return true; + case Prospero::BufferFormat::k8_8UInt: + case Prospero::BufferFormat::k8_8SInt: + EmitMemoryStoreSubDwordU32(state, inst, IR::ResourceKind::Buffer, 1, + AddressSourceCount(inst, 1), 8); + return true; + case Prospero::BufferFormat::k16_16UInt: + case Prospero::BufferFormat::k16_16SInt: + EmitMemoryStoreSubDwordU32(state, inst, IR::ResourceKind::Buffer, 1, + AddressSourceCount(inst, 1), 16); + return true; + default: break; + } + return false; +} + +uint32_t EmitAtomicPointer(EmitterState* state, const IR::Instruction& inst) { + switch (inst.memory.kind) { + case IR::ResourceKind::Lds: + return EmitLdsElementPointer(state, EmitDwordIndex(state, inst, 1, 1)); + case IR::ResourceKind::Gds: + return EmitGdsElementPointer(state, EmitDwordIndex(state, inst, 1, 1)); + case IR::ResourceKind::Buffer: { + const auto index = + EmitMemoryDwordIndex(state, inst, inst.memory, 1, AddressSourceCount(inst, 1)); + return EmitStorageBufferElementPointer(state, inst.memory, index, inst.pc); + } + case IR::ResourceKind::StorageImageUint: { + const auto view = StorageImageViewKind(*state, inst.memory, true, inst.pc); + const auto image_pointer = + StorageImageDescriptorPointer(state, inst.memory.resource, true, inst.pc, view); + const auto pointer = state->builder.AllocateId(); + state->builder.AddFunction({OpImageTexelPointer, state->ptr_image_uint, pointer, + image_pointer, EmitImageCoordU32(state, inst, view), + ConstantU32(state, 0)}); + return pointer; + } + default: return 0; + } +} + +void EmitDeviceAtomicMemoryBarrier(EmitterState* state) { + const auto semantics = MemorySemanticsAcquireRelease | MemorySemanticsUniformMemory; + state->builder.AddFunction( + {OpMemoryBarrier, ConstantU32(state, ScopeDevice), ConstantU32(state, semantics)}); +} + +void EmitAtomicU32(EmitterState* state, const IR::Instruction& inst, uint32_t opcode) { + if (IsStorageBufferMemoryKind(inst.memory.kind)) { + const auto index = + EmitMemoryDwordIndex(state, inst, inst.memory, 1, AddressSourceCount(inst, 1)); + const auto in_bounds = EmitStorageBufferElementInBounds(state, inst.memory, index, inst.pc); + const auto value = EmitValueLoad(state, inst.src[0]); + const auto old = EmitValueOrZeroIfCondition(state, in_bounds, [&]() { + const auto pointer = + EmitStorageBufferElementPointer(state, inst.memory, index, inst.pc); + const auto result = state->builder.AllocateId(); + state->builder.AddFunction({opcode, state->uint_type, result, pointer, + ConstantU32(state, ScopeDevice), + ConstantU32(state, MemorySemanticsNone), value}); + EmitDeviceAtomicMemoryBarrier(state); + return result; + }); + EmitStoreU32(state, inst.dst, old); + return; + } + if (inst.memory.kind == IR::ResourceKind::Gds) { + const auto index = EmitDwordIndex(state, inst, 1, 1); + const auto in_bounds = EmitGdsElementInBounds(state, index); + const auto value = EmitValueLoad(state, inst.src[0]); + const auto old = EmitValueOrZeroIfCondition(state, in_bounds, [&]() { + const auto pointer = EmitGdsElementPointer(state, index); + const auto result = state->builder.AllocateId(); + state->builder.AddFunction({opcode, state->uint_type, result, pointer, + ConstantU32(state, ScopeDevice), + ConstantU32(state, MemorySemanticsNone), value}); + EmitDeviceAtomicMemoryBarrier(state); + return result; + }); + EmitStoreU32(state, inst.dst, old); + return; + } + + const auto pointer = EmitAtomicPointer(state, inst); + if (pointer == 0) { + EmitStoreU32(state, inst.dst, ConstantU32(state, 0)); + return; + } + + const auto value = EmitValueLoad(state, inst.src[0]); + const auto old = state->builder.AllocateId(); + const auto scope = inst.memory.kind == IR::ResourceKind::Lds ? ScopeWorkgroup : ScopeDevice; + state->builder.AddFunction({opcode, state->uint_type, old, pointer, ConstantU32(state, scope), + ConstantU32(state, MemorySemanticsNone), value}); + if (inst.memory.kind == IR::ResourceKind::StorageImageUint || + inst.memory.kind == IR::ResourceKind::Gds) { + EmitDeviceAtomicMemoryBarrier(state); + } + EmitStoreU32(state, inst.dst, old); +} + +void EmitSLoadDword(EmitterState* state, const IR::Instruction& inst) { + if (state->address_memory_variable == 0) { + ExitDescriptorBindingFailure(*state, IR::DescriptorBindingKind::AddressMemory, + inst.memory.resource, + "scalar memory descriptor array was not emitted"); + } + const auto binding = ResourceForDescriptor(*state, IR::DescriptorBindingKind::AddressMemory, + inst.memory.resource); + const auto address = EmitRelativeAddress(state, inst, 0, 1, false, true); + const auto index = state->builder.AllocateId(); + state->builder.AddFunction( + {OpShiftRightLogical, state->uint_type, index, address, ConstantU32(state, 2)}); + const auto object = state->builder.AllocateId(); + state->builder.AddFunction({OpAccessChain, state->ptr_storage_buffer, object, + state->address_memory_variable, + ConstantU32(state, binding.array_index)}); + const auto length = state->builder.AllocateId(); + const auto in_bounds = state->builder.AllocateId(); + state->builder.AddFunction({OpArrayLength, state->uint_type, length, object, 0}); + state->builder.AddFunction({OpULessThan, state->bool_type, in_bounds, index, length}); + const auto value = EmitValueOrZeroIfCondition(state, in_bounds, [&]() { + const auto pointer = state->builder.AllocateId(); + const auto loaded = state->builder.AllocateId(); + state->builder.AddFunction( + {OpAccessChain, state->ptr_storage_buffer_uint, pointer, state->address_memory_variable, + ConstantU32(state, binding.array_index), ConstantU32(state, 0), index}); + state->builder.AddFunction({OpLoad, state->uint_type, loaded, pointer}); + return loaded; + }); + EmitStoreU32(state, inst.dst, value); +} + +void EmitLoadSrtDword(EmitterState* state, const IR::Instruction& inst) { + if (state->flattened_srt_variable == 0 || inst.src_count != 1 || + inst.src[0].kind != IR::OperandKind::ImmediateU32) { + ExitDescriptorBindingFailure(*state, IR::DescriptorBindingKind::FlattenedSrt, + inst.src_count != 0 ? inst.src[0].imm : 0, + "invalid flattened SRT load"); + } + const auto pointer = state->builder.AllocateId(); + const auto value = state->builder.AllocateId(); + state->builder.AddFunction({OpAccessChain, state->ptr_storage_buffer_uint, pointer, + state->flattened_srt_variable, ConstantU32(state, 0), + ConstantU32(state, inst.src[0].imm)}); + state->builder.AddFunction({OpLoad, state->uint_type, value, pointer}); + EmitStoreU32(state, inst.dst, value); +} + +void EmitBufferLoadUbyte(EmitterState* state, const IR::Instruction& inst) { + EmitGuardedByExec(state, [&]() { + EmitMemoryLoadSubDwordU32(state, inst, IR::ResourceKind::Buffer, 0, + AddressSourceCount(inst, 0), 8, false); + }); +} + +void EmitBufferLoadSbyte(EmitterState* state, const IR::Instruction& inst) { + EmitGuardedByExec(state, [&]() { + EmitMemoryLoadSubDwordU32(state, inst, IR::ResourceKind::Buffer, 0, + AddressSourceCount(inst, 0), 8, true); + }); +} + +void EmitBufferLoadUshort(EmitterState* state, const IR::Instruction& inst) { + EmitGuardedByExec(state, [&]() { + EmitMemoryLoadSubDwordU32(state, inst, IR::ResourceKind::Buffer, 0, + AddressSourceCount(inst, 0), 16, false); + }); +} + +void EmitBufferLoadSshort(EmitterState* state, const IR::Instruction& inst) { + EmitGuardedByExec(state, [&]() { + EmitMemoryLoadSubDwordU32(state, inst, IR::ResourceKind::Buffer, 0, + AddressSourceCount(inst, 0), 16, true); + }); +} + +void EmitBufferLoadDword(EmitterState* state, const IR::Instruction& inst) { + EmitGuardedByExec(state, [&]() { + if (EmitFormattedBufferLoad(state, inst)) { + return; + } + EmitMemoryLoadU32(state, inst, IR::ResourceKind::Buffer, 0, AddressSourceCount(inst, 0)); + }); +} + +void EmitBufferStoreDword(EmitterState* state, const IR::Instruction& inst) { + if (EmitBufferIntegerFormatStore(state, inst)) { + return; + } + EmitMemoryStoreU32(state, inst, IR::ResourceKind::Buffer, 1, AddressSourceCount(inst, 1)); +} + +void EmitFlatLoadUbyte(EmitterState* state, const IR::Instruction& inst) { + EmitGuardedByExec(state, [&]() { + EmitMemoryLoadSubDwordU32(state, inst, inst.memory.kind, 0, inst.src_count, 8, false); + }); +} + +void EmitFlatLoadSbyte(EmitterState* state, const IR::Instruction& inst) { + EmitGuardedByExec(state, [&]() { + EmitMemoryLoadSubDwordU32(state, inst, inst.memory.kind, 0, inst.src_count, 8, true); + }); +} + +void EmitFlatLoadUshort(EmitterState* state, const IR::Instruction& inst) { + EmitGuardedByExec(state, [&]() { + EmitMemoryLoadSubDwordU32(state, inst, inst.memory.kind, 0, inst.src_count, 16, false); + }); +} + +void EmitFlatLoadSshort(EmitterState* state, const IR::Instruction& inst) { + EmitGuardedByExec(state, [&]() { + EmitMemoryLoadSubDwordU32(state, inst, inst.memory.kind, 0, inst.src_count, 16, true); + }); +} + +void EmitFlatLoadDword(EmitterState* state, const IR::Instruction& inst) { + EmitGuardedByExec(state, [&]() { + EmitMemoryLoadDwordsU32(state, inst, inst.memory.kind, 0, inst.src_count); + }); +} + +uint32_t EmitDsAddtidDwordIndex(EmitterState* state, const IR::Instruction& inst, + uint32_t m0_src_index) { + if (state->stage != ShaderType::Compute) { + const auto m0 = m0_src_index < inst.src_count ? EmitValueLoad(state, inst.src[m0_src_index]) + : ConstantU32(state, 0); + const auto m0_lo = state->builder.AllocateId(); + const auto base = state->builder.AllocateId(); + const auto index = state->builder.AllocateId(); + state->builder.AddFunction( + {OpBitwiseAnd, state->uint_type, m0_lo, m0, ConstantU32(state, 0x0000ffffu)}); + state->builder.AddFunction( + {OpIAdd, state->uint_type, base, m0_lo, ConstantU32(state, inst.memory.offset)}); + state->builder.AddFunction( + {OpShiftRightLogical, state->uint_type, index, base, ConstantU32(state, 2)}); + return index; + } + + const auto local_index = EmitLocalInvocationIndex(state); + const auto lane = state->builder.AllocateId(); + const auto lane_bytes = state->builder.AllocateId(); + const auto m0 = m0_src_index < inst.src_count ? EmitValueLoad(state, inst.src[m0_src_index]) + : ConstantU32(state, 0); + const auto m0_lo = state->builder.AllocateId(); + const auto base = state->builder.AllocateId(); + const auto address = state->builder.AllocateId(); + const auto index = state->builder.AllocateId(); + state->builder.AddFunction( + {OpBitwiseAnd, state->uint_type, lane, local_index, ConstantU32(state, 63)}); + state->builder.AddFunction( + {OpShiftLeftLogical, state->uint_type, lane_bytes, lane, ConstantU32(state, 2)}); + state->builder.AddFunction( + {OpBitwiseAnd, state->uint_type, m0_lo, m0, ConstantU32(state, 0x0000ffffu)}); + state->builder.AddFunction( + {OpIAdd, state->uint_type, base, m0_lo, ConstantU32(state, inst.memory.offset)}); + state->builder.AddFunction({OpIAdd, state->uint_type, address, base, lane_bytes}); + state->builder.AddFunction( + {OpShiftRightLogical, state->uint_type, index, address, ConstantU32(state, 2)}); + return index; +} + +void EmitDsWriteAddtidB32(EmitterState* state, const IR::Instruction& inst) { + const auto value = EmitValueLoad(state, inst.src[0]); + const auto index = EmitDsAddtidDwordIndex(state, inst, 1); + const auto pointer = EmitLdsElementPointer(state, index); + state->builder.AddFunction({OpStore, pointer, value}); +} + +void EmitDsReadAddtidB32(EmitterState* state, const IR::Instruction& inst) { + const auto index = EmitDsAddtidDwordIndex(state, inst, 0); + const auto pointer = EmitLdsElementPointer(state, index); + const auto value = state->builder.AllocateId(); + state->builder.AddFunction({OpLoad, state->uint_type, value, pointer}); + EmitStoreU32(state, inst.dst, value); +} + +struct DsCounterAddress { + uint32_t index = 0; + uint32_t size = 0; + uint32_t in_lds_bounds = 0; +}; + +DsCounterAddress EmitAppendConsumeAddress(EmitterState* state, const IR::Instruction& inst) { + const auto m0 = inst.src_count > 0 ? EmitValueLoad(state, inst.src[0]) : ConstantU32(state, 0); + const auto base = state->builder.AllocateId(); + const auto size = state->builder.AllocateId(); + const auto address = state->builder.AllocateId(); + const auto index = state->builder.AllocateId(); + const auto in_bounds = state->builder.AllocateId(); + // RDNA2 puts the DS append/consume byte base in M0[31:16]. + state->builder.AddFunction( + {OpShiftRightLogical, state->uint_type, base, m0, ConstantU32(state, 16)}); + state->builder.AddFunction( + {OpBitwiseAnd, state->uint_type, size, m0, ConstantU32(state, 0xffffu)}); + state->builder.AddFunction( + {OpIAdd, state->uint_type, address, base, ConstantU32(state, inst.memory.offset)}); + state->builder.AddFunction( + {OpShiftRightLogical, state->uint_type, index, address, ConstantU32(state, 2)}); + state->builder.AddFunction({OpULessThan, state->bool_type, in_bounds, + ConstantU32(state, inst.memory.offset + 3u), size}); + return {index, size, in_bounds}; +} + +uint32_t EmitGdsElementInBounds(EmitterState* state, uint32_t index) { + const auto length = state->builder.AllocateId(); + const auto in_bounds = state->builder.AllocateId(); + state->builder.AddFunction({OpArrayLength, state->uint_type, length, state->gds_variable, 0}); + state->builder.AddFunction({OpULessThan, state->bool_type, in_bounds, index, length}); + return in_bounds; +} + +uint32_t EmitGdsElementPointer(EmitterState* state, uint32_t index) { + const auto pointer = state->builder.AllocateId(); + state->builder.AddFunction({OpAccessChain, state->ptr_storage_buffer_uint, pointer, + state->gds_variable, ConstantU32(state, 0), index}); + return pointer; +} + +uint32_t EmitBitCountU32Value(EmitterState* state, uint32_t value) { + const auto ret = state->builder.AllocateId(); + state->builder.AddFunction({OpBitCount, state->uint_type, ret, value}); + return ret; +} + +uint32_t EmitFindLsbU32Value(EmitterState* state, uint32_t value) { + const auto i32 = state->builder.AllocateId(); + const auto ret = state->builder.AllocateId(); + state->builder.AddFunction( + {OpExtInst, state->int_type, i32, state->glsl_std450, GlslFindILsb, value}); + state->builder.AddFunction({OpBitcast, state->uint_type, ret, i32}); + return ret; +} + +struct ExecMaskInfo { + uint32_t active_count = 0; + uint32_t first_lane = 0; + uint32_t any_active = 0; +}; + +ExecMaskInfo EmitExecMaskInfo(EmitterState* state) { + uint32_t exec_lo = 0; + uint32_t exec_hi = 0; + if (state->per_invocation_masks) { + const auto ballot = state->builder.AllocateId(); + state->builder.AddFunction({OpGroupNonUniformBallot, state->vec4_uint_type, ballot, + ConstantU32(state, ScopeSubgroup), EmitExecActiveBool(state)}); + exec_lo = state->builder.AllocateId(); + state->builder.AddFunction({OpCompositeExtract, state->uint_type, exec_lo, ballot, 0}); + if (state->wave_size == 64u) { + exec_hi = state->builder.AllocateId(); + state->builder.AddFunction({OpCompositeExtract, state->uint_type, exec_hi, ballot, 1}); + } else { + exec_hi = ConstantU32(state, 0); + } + } else { + exec_lo = EmitRegisterLoad(state, {IR::RegisterFile::Exec, 0}); + exec_hi = WaveMaskHighLoad(state, IR::RegisterFile::Exec); + } + const auto lo_count = EmitBitCountU32Value(state, exec_lo); + const auto hi_count = EmitBitCountU32Value(state, exec_hi); + const auto active_count = state->builder.AllocateId(); + state->builder.AddFunction({OpIAdd, state->uint_type, active_count, lo_count, hi_count}); + + const auto lo_nonzero = EmitNotEqualZeroBool(state, exec_lo); + const auto hi_nonzero = EmitNotEqualZeroBool(state, exec_hi); + const auto any_active = EmitLogicalOrBool(state, lo_nonzero, hi_nonzero); + const auto hi_lsb = EmitFindLsbU32Value(state, exec_hi); + const auto hi_lane = state->builder.AllocateId(); + state->builder.AddFunction({OpIAdd, state->uint_type, hi_lane, hi_lsb, ConstantU32(state, 32)}); + const auto lo_lsb = EmitFindLsbU32Value(state, exec_lo); + const auto selected = state->builder.AllocateId(); + state->builder.AddFunction({OpSelect, state->uint_type, selected, lo_nonzero, lo_lsb, hi_lane}); + const auto first_lane = state->builder.AllocateId(); + state->builder.AddFunction( + {OpSelect, state->uint_type, first_lane, any_active, selected, ConstantU32(state, 0)}); + return {active_count, first_lane, any_active}; +} + +void EmitDsAppendConsume(EmitterState* state, const IR::Instruction& inst, uint32_t atomic_opcode) { + const auto gds = inst.memory.kind == IR::ResourceKind::Gds; + if (gds && state->gds_variable == 0) { + EmitStoreU32(state, inst.dst, ConstantU32(state, 0)); + return; + } + + const auto address = EmitAppendConsumeAddress(state, inst); + const auto storage_bounds = gds ? EmitGdsElementInBounds(state, address.index) + : EmitLdsElementInBounds(state, address.index); + const auto m0_bounds = gds ? EmitNotEqualZeroBool(state, address.size) : address.in_lds_bounds; + const auto in_bounds = EmitLogicalAndBool(state, m0_bounds, storage_bounds); + const auto exec = EmitExecMaskInfo(state); + const auto subid = EmitSubgroupLocalInvocationId(state); + const auto first_lane = state->builder.AllocateId(); + const auto do_atomic = state->builder.AllocateId(); + state->builder.AddFunction({OpIEqual, state->bool_type, first_lane, subid, exec.first_lane}); + const auto first_active = EmitLogicalAndBool(state, first_lane, exec.any_active); + state->builder.AddFunction( + {OpLogicalAnd, state->bool_type, do_atomic, first_active, in_bounds}); + + const auto atomic_value = EmitValueOrZeroIfCondition(state, do_atomic, [&]() { + const auto pointer = gds ? EmitGdsElementPointer(state, address.index) + : EmitLdsElementPointer(state, address.index); + const auto result = state->builder.AllocateId(); + state->builder.AddFunction({atomic_opcode, state->uint_type, result, pointer, + ConstantU32(state, gds ? ScopeDevice : ScopeWorkgroup), + ConstantU32(state, MemorySemanticsNone), exec.active_count}); + if (gds) { + EmitDeviceAtomicMemoryBarrier(state); + } else { + const auto semantics = MemorySemanticsAcquireRelease | MemorySemanticsWorkgroupMemory; + state->builder.AddFunction({OpMemoryBarrier, ConstantU32(state, ScopeWorkgroup), + ConstantU32(state, semantics)}); + } + return result; + }); + const auto broadcast = state->builder.AllocateId(); + state->builder.AddFunction({OpGroupNonUniformShuffle, state->uint_type, broadcast, + ConstantU32(state, ScopeSubgroup), atomic_value, exec.first_lane}); + const auto value = EmitSelectU32Value(state, exec.any_active, broadcast, ConstantU32(state, 0)); + EmitStoreU32(state, inst.dst, value); +} + +void EmitDsFloatMinMaxF32(EmitterState* state, const IR::Instruction& inst, bool max_value) { + const auto index = EmitDwordIndex(state, inst, 1, 1); + const auto in_bounds = + inst.memory.kind == IR::ResourceKind::Gds ? EmitGdsElementInBounds(state, index) : 0u; + EmitIfCondition(state, in_bounds, [&]() { + const auto pointer = EmitMemoryElementPointer(state, inst.memory, index, inst.pc); + const auto data_f32 = EmitFloatLoad(state, inst.src[0]); + const auto cmp_f32 = EmitFloatLoad(state, inst.src[2]); + EmitAtomicUpdateU32(state, pointer, inst.memory.kind, [&](uint32_t old_u32) { + const auto old_f32 = state->builder.AllocateId(); + const auto store_src = state->builder.AllocateId(); + const auto value_f32 = state->builder.AllocateId(); + const auto value_u32 = state->builder.AllocateId(); + state->builder.AddFunction({OpBitcast, state->float_type, old_f32, old_u32}); + state->builder.AddFunction({max_value ? OpFOrdGreaterThan : OpFOrdLessThan, + state->bool_type, store_src, max_value ? old_f32 : cmp_f32, + max_value ? cmp_f32 : old_f32}); + state->builder.AddFunction( + {OpSelect, state->float_type, value_f32, store_src, data_f32, old_f32}); + state->builder.AddFunction({OpBitcast, state->uint_type, value_u32, value_f32}); + return value_u32; + }); + }); +} + +uint32_t EmitDsSwizzleTargetLane(EmitterState* state, uint32_t subid, uint32_t control) { + if ((control & 0xc000u) == 0xc000u) { + const uint32_t mask = control & 0x1fu; + const uint32_t rotate = (control >> 5u) & 0x1fu; + const uint32_t rotate_delta = (control & 0x400u) != 0u ? ((32u - rotate) & 0x1fu) : rotate; + const auto lane = EmitAndConstant(state, subid, 31); + const auto rotated_sum = EmitAddU32(state, lane, ConstantU32(state, rotate_delta)); + const auto rotated = EmitAndConstant(state, rotated_sum, 31); + const auto kept = EmitAndConstant(state, lane, mask); + const auto moved = EmitAndConstant(state, rotated, (~mask) & 31u); + const auto combined = EmitOrU32(state, kept, moved); + const auto base = EmitAndConstant(state, subid, 0xffffffe0u); + return EmitOrU32(state, base, combined); + } + + if ((control & 0x8000u) != 0) { + const auto lane2 = state->builder.AllocateId(); + const auto shift = state->builder.AllocateId(); + const auto perm0 = state->builder.AllocateId(); + const auto perm = state->builder.AllocateId(); + const auto base = state->builder.AllocateId(); + const auto target = state->builder.AllocateId(); + state->builder.AddFunction( + {OpBitwiseAnd, state->uint_type, lane2, subid, ConstantU32(state, 3)}); + state->builder.AddFunction( + {OpShiftLeftLogical, state->uint_type, shift, lane2, ConstantU32(state, 1)}); + state->builder.AddFunction( + {OpShiftRightLogical, state->uint_type, perm0, ConstantU32(state, control), shift}); + state->builder.AddFunction( + {OpBitwiseAnd, state->uint_type, perm, perm0, ConstantU32(state, 3)}); + state->builder.AddFunction( + {OpBitwiseAnd, state->uint_type, base, subid, ConstantU32(state, 0xfffffffcu)}); + state->builder.AddFunction({OpBitwiseOr, state->uint_type, target, base, perm}); + return target; + } + + const auto lane = state->builder.AllocateId(); + const auto masked = state->builder.AllocateId(); + const auto ored = state->builder.AllocateId(); + const auto xored = state->builder.AllocateId(); + const auto base = state->builder.AllocateId(); + const auto target = state->builder.AllocateId(); + state->builder.AddFunction( + {OpBitwiseAnd, state->uint_type, lane, subid, ConstantU32(state, 31)}); + state->builder.AddFunction( + {OpBitwiseAnd, state->uint_type, masked, lane, ConstantU32(state, control & 0x1fu)}); + state->builder.AddFunction( + {OpBitwiseOr, state->uint_type, ored, masked, ConstantU32(state, (control >> 5u) & 0x1fu)}); + state->builder.AddFunction({OpBitwiseXor, state->uint_type, xored, ored, + ConstantU32(state, (control >> 10u) & 0x1fu)}); + state->builder.AddFunction( + {OpBitwiseAnd, state->uint_type, base, subid, ConstantU32(state, 0xffffffe0u)}); + state->builder.AddFunction({OpBitwiseOr, state->uint_type, target, base, xored}); + return target; +} + +void EmitDsSwizzleB32(EmitterState* state, const IR::Instruction& inst) { + const auto source = EmitValueLoad(state, inst.src[0]); + const auto control = inst.src_count > 1u ? inst.src[1].imm & 0xffffu : 0u; + const auto subid = EmitSubgroupLocalInvocationId(state); + const auto target = EmitDsSwizzleTargetLane(state, subid, control); + const auto value = state->builder.AllocateId(); + state->builder.AddFunction({OpGroupNonUniformShuffle, state->uint_type, value, + ConstantU32(state, ScopeSubgroup), source, target}); + const auto exec_active = EmitLaneIndexActiveBool(state, target); + const auto subgroup_active = EmitSubgroupLaneActiveBool(state, target); + const auto source_active = state->builder.AllocateId(); + const auto selected = state->builder.AllocateId(); + // RDNA2 DS_SWIZZLE returns zero when the selected source thread is invalid. + state->builder.AddFunction( + {OpLogicalAnd, state->bool_type, source_active, exec_active, subgroup_active}); + state->builder.AddFunction( + {OpSelect, state->uint_type, selected, source_active, value, ConstantU32(state, 0)}); + EmitStoreU32(state, inst.dst, selected); +} + +} // namespace Libs::Graphics::ShaderRecompiler::Spirv::Emitter diff --git a/src/graphics/shader/recompiler/spirvEmitter/spirvEmitterModule.cpp b/src/graphics/shader/recompiler/spirvEmitter/spirvEmitterModule.cpp new file mode 100644 index 000000000..ce75be3a4 --- /dev/null +++ b/src/graphics/shader/recompiler/spirvEmitter/spirvEmitterModule.cpp @@ -0,0 +1,929 @@ +#include "graphics/shader/recompiler/spirvEmitter/spirvEmitterInternal.h" + +namespace Libs::Graphics::ShaderRecompiler::Spirv::Emitter { + +const IR::DescriptorBinding* DescriptorBinding(const EmitterState& state, + IR::DescriptorBindingKind kind) { + return IR::FindBinding(state.program->bindings, kind); +} + +uint32_t DescriptorCount(const EmitterState& state, IR::DescriptorBindingKind kind) { + const auto* binding = DescriptorBinding(state, kind); + return binding != nullptr ? static_cast(binding->resources.size()) : 0; +} + +uint32_t ConstantU32(EmitterState* state, uint32_t value) { + if (auto it = state->constants.find(value); it != state->constants.end()) { + return it->second; + } + + const auto id = state->builder.AllocateId(); + state->constants.emplace(value, id); + state->builder.AddType({OpConstant, state->uint_type, id, value}); + return id; +} + +uint32_t ConstantI32(EmitterState* state, int32_t value) { + const auto bits = static_cast(value); + if (auto it = state->signed_constants.find(bits); it != state->signed_constants.end()) { + return it->second; + } + + const auto id = state->builder.AllocateId(); + state->signed_constants.emplace(bits, id); + state->builder.AddType({OpConstant, state->int_type, id, bits}); + return id; +} + +uint32_t ConstantF32(EmitterState* state, uint32_t bits) { + if (auto it = state->float_constants.find(bits); it != state->float_constants.end()) { + return it->second; + } + + const auto id = state->builder.AllocateId(); + state->float_constants.emplace(bits, id); + state->builder.AddType({OpConstant, state->float_type, id, bits}); + return id; +} + +uint32_t FloatBits(float value) { + uint32_t bits = 0; + std::memcpy(&bits, &value, sizeof(bits)); + return bits; +} + +uint32_t ConstantF32Value(EmitterState* state, float value) { + return ConstantF32(state, FloatBits(value)); +} + +VertexInputScalarKind VertexParameterScalarKind(const EmitterState& state, uint32_t location) { + if (state.stage != ShaderType::Vertex || state.vertex_input_info == nullptr || + location >= ShaderVertexInputInfo::RES_MAX || + location >= static_cast(state.vertex_input_info->resources_num)) { + return VertexInputScalarKind::Float; + } + + switch (state.vertex_input_info->resources[location].Format()) { + case 5: + case 11: + case 18: + case 20: + case 27: + case 60: + case 62: + case 69: + case 72: + case 75: return VertexInputScalarKind::Uint; + case 6: + case 12: + case 19: + case 21: + case 28: + case 61: + case 63: + case 70: + case 73: + case 76: return VertexInputScalarKind::Sint; + default: return VertexInputScalarKind::Float; + } +} + +uint32_t VertexParameterComponentCount(const EmitterState& state, const InputBinding& input) { + uint32_t count = input.component_count; + if (state.stage == ShaderType::Vertex && state.vertex_input_info != nullptr && + input.location < ShaderVertexInputInfo::RES_MAX && + input.location < static_cast(state.vertex_input_info->resources_num) && + state.vertex_input_info->resources_dst[input.location].registers_num > 0) { + count = static_cast( + state.vertex_input_info->resources_dst[input.location].registers_num); + } + return std::clamp(count, 1u, 4u); +} + +uint32_t VertexParameterScalarType(const EmitterState& state, VertexInputScalarKind kind) { + switch (kind) { + case VertexInputScalarKind::Sint: return state.int_type; + case VertexInputScalarKind::Uint: return state.uint_type; + case VertexInputScalarKind::Float: + default: return state.float_type; + } +} + +uint32_t VertexParameterScalarPointerType(const EmitterState& state, VertexInputScalarKind kind) { + switch (kind) { + case VertexInputScalarKind::Sint: return state.ptr_input_int; + case VertexInputScalarKind::Uint: return state.ptr_input_uint; + case VertexInputScalarKind::Float: + default: return state.ptr_input_float; + } +} + +uint32_t VertexParameterVectorOrScalarType(const EmitterState& state, VertexInputScalarKind kind, + uint32_t components) { + switch (kind) { + case VertexInputScalarKind::Sint: + switch (components) { + case 1: return state.int_type; + case 2: return state.vec2_int_type; + case 3: return state.vec3_int_type; + default: return state.vec4_int_type; + } + case VertexInputScalarKind::Uint: + switch (components) { + case 1: return state.uint_type; + case 2: return state.vec2_uint_type; + case 3: return state.vec3_uint_type; + default: return state.vec4_uint_type; + } + case VertexInputScalarKind::Float: + default: + switch (components) { + case 1: return state.float_type; + case 2: return state.vec2_float_type; + case 3: return state.vec3_float_type; + default: return state.vec4_float_type; + } + } +} + +uint32_t VertexParameterInputPointerType(const EmitterState& state, VertexInputScalarKind kind, + uint32_t components) { + switch (kind) { + case VertexInputScalarKind::Sint: + switch (components) { + case 1: return state.ptr_input_int; + case 2: return state.ptr_input_vec2_int; + case 3: return state.ptr_input_vec3_int; + default: return state.ptr_input_vec4_int; + } + case VertexInputScalarKind::Uint: + switch (components) { + case 1: return state.ptr_input_uint; + case 2: return state.ptr_input_vec2_uint; + case 3: return state.ptr_input_vec3_uint; + default: return state.ptr_input_vec4_uint; + } + case VertexInputScalarKind::Float: + default: + switch (components) { + case 1: return state.ptr_input_float; + case 2: return state.ptr_input_vec2_float; + case 3: return state.ptr_input_vec3_float; + default: return state.ptr_input_vec4_float; + } + } +} + +void AllocateInputVariables(EmitterState* state) { + for (auto& binding: state->inputs) { + binding.variable_id = state->builder.AllocateId(); + state->interface_variables.push_back(binding.variable_id); + } + if (state->needs_subgroup_local_invocation_id) { + state->subgroup_local_invocation_id_variable = state->builder.AllocateId(); + state->interface_variables.push_back(state->subgroup_local_invocation_id_variable); + } +} + +static uint32_t AllocateInterfaceVariable(EmitterState* state) { + const auto variable = state->builder.AllocateId(); + state->interface_variables.push_back(variable); + return variable; +} + +static uint32_t AllocateSharedOutputVariable(EmitterState* state, uint32_t& variable) { + if (variable == 0) { + variable = AllocateInterfaceVariable(state); + } + return variable; +} + +void AllocateOutputVariables(EmitterState* state) { + for (auto& binding: state->outputs) { + switch (binding.kind) { + case IR::StageOutputKind::Position: + binding.variable_id = + AllocateSharedOutputVariable(state, state->per_vertex_variable); + break; + case IR::StageOutputKind::Depth: + binding.variable_id = AllocateSharedOutputVariable(state, state->depth_variable); + break; + case IR::StageOutputKind::SampleMask: + binding.variable_id = + AllocateSharedOutputVariable(state, state->sample_mask_variable); + break; + case IR::StageOutputKind::Parameter: + case IR::StageOutputKind::Mrt: + binding.variable_id = AllocateInterfaceVariable(state); + break; + } + } +} + +uint32_t BuiltInForInput(IR::StageInputKind kind) { + switch (kind) { + case IR::StageInputKind::VertexIndex: return BuiltInVertexIndex; + case IR::StageInputKind::InstanceIndex: return BuiltInInstanceIndex; + case IR::StageInputKind::FragCoord: return BuiltInFragCoord; + case IR::StageInputKind::FrontFacing: return BuiltInFrontFacing; + case IR::StageInputKind::WorkgroupId: return BuiltInWorkgroupId; + case IR::StageInputKind::LocalInvocationId: return BuiltInLocalInvocationId; + case IR::StageInputKind::LocalInvocationIndex: return BuiltInLocalInvocationIndex; + case IR::StageInputKind::GlobalInvocationId: return BuiltInGlobalInvocationId; + default: return UINT32_MAX; + } +} + +void AddInputAnnotationsAndNames(EmitterState* state) { + if (state->subgroup_local_invocation_id_variable != 0) { + state->builder.AddName(state->subgroup_local_invocation_id_variable, + "gl_SubgroupInvocationID"); + state->builder.AddAnnotation({OpDecorate, state->subgroup_local_invocation_id_variable, + DecorationBuiltIn, BuiltInSubgroupLocalInvocationId}); + if (state->stage == ShaderType::Pixel) { + state->builder.AddAnnotation( + {OpDecorate, state->subgroup_local_invocation_id_variable, DecorationFlat}); + } + } + for (const auto& input: state->inputs) { + state->builder.AddName(input.variable_id, input.debug_name.c_str()); + if (input.kind == IR::StageInputKind::Parameter) { + const auto flat = PixelParameterIsFlat(*state, input.location); + if (flat) { + state->builder.AddAnnotation({OpDecorate, input.variable_id, DecorationFlat}); + } + if (state->stage == ShaderType::Pixel && state->pixel_input_info != nullptr && + state->pixel_input_info->ps_no_perspective && !flat) { + state->builder.AddAnnotation( + {OpDecorate, input.variable_id, DecorationNoPerspective}); + } + const auto location = PixelParameterLocation(*state, input.location); + state->builder.AddAnnotation( + {OpDecorate, input.variable_id, DecorationLocation, location}); + continue; + } + const auto builtin = BuiltInForInput(input.kind); + if (builtin != UINT32_MAX) { + state->builder.AddAnnotation( + {OpDecorate, input.variable_id, DecorationBuiltIn, builtin}); + } + } +} + +void AddOutputAnnotationsAndNames(EmitterState* state) { + if (state->per_vertex_variable != 0) { + state->builder.AddName(state->per_vertex_type, "gl_PerVertex"); + state->builder.AddName(state->per_vertex_variable, "outPerVertex"); + state->builder.AddAnnotation( + {OpMemberDecorate, state->per_vertex_type, 0, DecorationBuiltIn, BuiltInPosition}); + state->builder.AddAnnotation({OpDecorate, state->per_vertex_type, DecorationBlock}); + } + if (state->depth_variable != 0) { + state->builder.AddName(state->depth_variable, "gl_FragDepth"); + state->builder.AddAnnotation( + {OpDecorate, state->depth_variable, DecorationBuiltIn, BuiltInFragDepth}); + } + if (state->sample_mask_variable != 0) { + state->builder.AddName(state->sample_mask_variable, "gl_SampleMask"); + state->builder.AddAnnotation( + {OpDecorate, state->sample_mask_variable, DecorationBuiltIn, BuiltInSampleMask}); + } + for (const auto& binding: state->outputs) { + if (binding.kind == IR::StageOutputKind::Parameter || + binding.kind == IR::StageOutputKind::Mrt) { + state->builder.AddName(binding.variable_id, binding.debug_name.c_str()); + state->builder.AddAnnotation( + {OpDecorate, binding.variable_id, DecorationLocation, binding.location}); + } + } +} + +void DecorateDescriptor(EmitterState* state, uint32_t variable, const char* name, uint32_t set, + uint32_t binding) { + if (variable == 0) { + return; + } + state->builder.AddName(variable, name); + state->builder.AddAnnotation({OpDecorate, variable, DecorationDescriptorSet, set}); + state->builder.AddAnnotation({OpDecorate, variable, DecorationBinding, binding}); +} + +void AddDescriptorAnnotationsAndNames(EmitterState* state) { + const auto set = state->program->bindings.descriptor_set; + auto Decorate = [&](uint32_t variable, const char* name, IR::DescriptorBindingKind kind) { + const auto* binding = DescriptorBinding(*state, kind); + if (binding != nullptr) { + DecorateDescriptor(state, variable, name, set, binding->binding); + } + }; + if (state->storage_buffer_variable != 0) { + Decorate(state->storage_buffer_variable, "buffers", IR::DescriptorBindingKind::Buffers); + } + if (state->address_memory_variable != 0) { + Decorate(state->address_memory_variable, "address_memory", + IR::DescriptorBindingKind::AddressMemory); + } + constexpr const char* SampledNames[] = { + "sampled_2d", "sampled_2d_array", "sampled_3d", + "sampled_uint_2d", "sampled_uint_2d_array", "sampled_uint_3d"}; + for (uint32_t i = 0; i < state->sampled_images.size(); i++) { + const auto view = static_cast(i % 3u); + Decorate(state->sampled_images[i].variable, SampledNames[i], + SampledBindingKind(i >= 3u, view)); + } + if (state->storage_image_variable != 0) { + Decorate(state->storage_image_variable, "textures2D_L", + IR::DescriptorBindingKind::Storage2D); + } + if (state->storage_image_uint_variable != 0) { + Decorate(state->storage_image_uint_variable, "textures2D_L_U", + IR::DescriptorBindingKind::StorageUint2D); + } + if (state->storage_image_2d_array_variable != 0) { + Decorate(state->storage_image_2d_array_variable, "textures2D_L_A", + IR::DescriptorBindingKind::Storage2DArray); + } + if (state->storage_image_uint_2d_array_variable != 0) { + Decorate(state->storage_image_uint_2d_array_variable, "textures2D_L_U_A", + IR::DescriptorBindingKind::StorageUint2DArray); + } + if (state->storage_image_3d_variable != 0) { + Decorate(state->storage_image_3d_variable, "textures2D_L_3D", + IR::DescriptorBindingKind::Storage3D); + } + if (state->storage_image_uint_3d_variable != 0) { + Decorate(state->storage_image_uint_3d_variable, "textures2D_L_U_3D", + IR::DescriptorBindingKind::StorageUint3D); + } + if (state->sampler_variable != 0) { + Decorate(state->sampler_variable, "samplers", IR::DescriptorBindingKind::Samplers); + } + if (state->gds_variable != 0) { + Decorate(state->gds_variable, "gds", IR::DescriptorBindingKind::Gds); + } + if (state->flattened_srt_variable != 0) { + Decorate(state->flattened_srt_variable, "flattened_srt", + IR::DescriptorBindingKind::FlattenedSrt); + } +} + +void AddVsharpAnnotationsAndNames(EmitterState* state) { + const auto& bind = state->program->bindings; + if (state->push_constant_variable != 0) { + state->builder.AddName(state->push_constant_block_type, "BufferResource"); + state->builder.AddName(state->push_constant_variable, "vsharp"); + state->builder.AddAnnotation( + {OpDecorate, state->push_constant_u32x4_array_type, DecorationArrayStride, 4}); + state->builder.AddAnnotation( + {OpDecorate, state->push_constant_rows_array_type, DecorationArrayStride, 16}); + state->builder.AddAnnotation({OpMemberDecorate, state->push_constant_block_type, 0, + DecorationOffset, bind.push_constant_offset}); + state->builder.AddAnnotation( + {OpDecorate, state->push_constant_block_type, DecorationBlock}); + } + if (state->vsharp_storage_variable != 0) { + const auto* storage = DescriptorBinding(*state, IR::DescriptorBindingKind::UserData); + if (storage != nullptr) { + DecorateDescriptor(state, state->vsharp_storage_variable, "user_data", + bind.descriptor_set, storage->binding); + } + } +} + +void EmitHeaderAndTypes(EmitterState* state) { + state->void_type = state->builder.AllocateId(); + state->bool_type = state->builder.AllocateId(); + state->uint_type = state->builder.AllocateId(); + state->uint_pair_type = state->builder.AllocateId(); + state->int_type = state->builder.AllocateId(); + state->int_pair_type = state->builder.AllocateId(); + state->float_type = state->builder.AllocateId(); + state->vec2_uint_type = state->builder.AllocateId(); + state->vec3_uint_type = state->builder.AllocateId(); + state->vec4_uint_type = state->builder.AllocateId(); + state->vec2_int_type = state->builder.AllocateId(); + state->vec3_int_type = state->builder.AllocateId(); + state->vec4_int_type = state->builder.AllocateId(); + state->vec2_float_type = state->builder.AllocateId(); + state->vec3_float_type = state->builder.AllocateId(); + state->vec4_float_type = state->builder.AllocateId(); + state->ptr_func_uint = state->builder.AllocateId(); + state->ptr_input_float = state->builder.AllocateId(); + state->ptr_input_bool = state->builder.AllocateId(); + state->ptr_input_int = state->builder.AllocateId(); + state->ptr_input_uint = state->builder.AllocateId(); + state->ptr_input_vec2_float = state->builder.AllocateId(); + state->ptr_input_vec3_float = state->builder.AllocateId(); + state->ptr_input_vec2_int = state->builder.AllocateId(); + state->ptr_input_vec3_int = state->builder.AllocateId(); + state->ptr_input_vec4_int = state->builder.AllocateId(); + state->ptr_input_vec2_uint = state->builder.AllocateId(); + state->ptr_input_vec3_uint = state->builder.AllocateId(); + state->ptr_input_vec4_uint = state->builder.AllocateId(); + state->ptr_input_vec4_float = state->builder.AllocateId(); + state->sample_mask_array_type = state->builder.AllocateId(); + state->ptr_output_int = state->builder.AllocateId(); + state->ptr_output_sample_mask_array = state->builder.AllocateId(); + state->ptr_output_float = state->builder.AllocateId(); + state->ptr_output_vec4_float = state->builder.AllocateId(); + state->per_vertex_type = state->builder.AllocateId(); + state->ptr_output_per_vertex = state->builder.AllocateId(); + state->storage_runtime_array_type = state->builder.AllocateId(); + state->storage_buffer_type = state->builder.AllocateId(); + state->ptr_storage_buffer = state->builder.AllocateId(); + state->ptr_storage_buffer_uint = state->builder.AllocateId(); + state->storage_buffer_array_type = state->builder.AllocateId(); + state->ptr_storage_buffer_array = state->builder.AllocateId(); + if (state->address_memory_variable != 0) { + state->address_memory_array_type = state->builder.AllocateId(); + state->ptr_address_memory_array = state->builder.AllocateId(); + } + if (state->push_constant_variable != 0) { + state->push_constant_u32x4_array_type = state->builder.AllocateId(); + state->push_constant_rows_array_type = state->builder.AllocateId(); + state->push_constant_block_type = state->builder.AllocateId(); + state->ptr_push_constant_block = state->builder.AllocateId(); + state->ptr_push_constant_uint = state->builder.AllocateId(); + } + state->lds_array_type = state->builder.AllocateId(); + state->ptr_workgroup_array = state->builder.AllocateId(); + state->ptr_workgroup_uint = state->builder.AllocateId(); + state->lds_variable = state->builder.AllocateId(); + for (auto& image: state->sampled_images) { + image.image_type = state->builder.AllocateId(); + image.sampled_image_type = state->builder.AllocateId(); + image.pointer_type = state->builder.AllocateId(); + image.array_type = state->builder.AllocateId(); + image.array_pointer_type = state->builder.AllocateId(); + } + state->sampler_type = state->builder.AllocateId(); + state->sampler_array_type = state->builder.AllocateId(); + state->ptr_uniform_sampler = state->builder.AllocateId(); + state->ptr_uniform_sampler_array = state->builder.AllocateId(); + state->storage_image_type = state->builder.AllocateId(); + state->ptr_uniform_storage_image = state->builder.AllocateId(); + state->storage_image_array_type = state->builder.AllocateId(); + state->ptr_uniform_storage_image_array = state->builder.AllocateId(); + state->storage_image_2d_array_type = state->builder.AllocateId(); + state->ptr_uniform_storage_image_2d_array = state->builder.AllocateId(); + state->storage_image_2d_array_array_type = state->builder.AllocateId(); + state->ptr_uniform_storage_image_2d_array_array = state->builder.AllocateId(); + state->storage_image_3d_type = state->builder.AllocateId(); + state->ptr_uniform_storage_image_3d = state->builder.AllocateId(); + state->storage_image_3d_array_type = state->builder.AllocateId(); + state->ptr_uniform_storage_image_3d_array = state->builder.AllocateId(); + state->storage_image_uint_type = state->builder.AllocateId(); + state->ptr_uniform_storage_image_uint = state->builder.AllocateId(); + state->storage_image_uint_array_type = state->builder.AllocateId(); + state->ptr_uniform_storage_image_uint_array = state->builder.AllocateId(); + state->storage_image_uint_2d_array_type = state->builder.AllocateId(); + state->ptr_uniform_storage_image_uint_2d_array = state->builder.AllocateId(); + state->storage_image_uint_2d_array_array_type = state->builder.AllocateId(); + state->ptr_uniform_storage_image_uint_2d_array_array = state->builder.AllocateId(); + state->storage_image_uint_3d_type = state->builder.AllocateId(); + state->ptr_uniform_storage_image_uint_3d = state->builder.AllocateId(); + state->storage_image_uint_3d_array_type = state->builder.AllocateId(); + state->ptr_uniform_storage_image_uint_3d_array = state->builder.AllocateId(); + state->ptr_image_uint = state->builder.AllocateId(); + state->func_type = state->builder.AllocateId(); + state->main_func = state->builder.AllocateId(); + state->entry_label = state->builder.AllocateId(); + state->glsl_std450 = state->builder.AllocateId(); + + state->builder.AddCapability({CapabilityShader}); + state->builder.AddCapability({CapabilityImageQuery}); + if (state->needs_image_gather_extended) { + state->builder.AddCapability({CapabilityImageGatherExtended}); + } + if (state->storage_image_variable != 0 || state->storage_image_2d_array_variable != 0 || + state->storage_image_3d_variable != 0) { + state->builder.AddCapability({CapabilityStorageImageReadWithoutFormat}); + state->builder.AddCapability({CapabilityStorageImageWriteWithoutFormat}); + } + if (state->needs_subgroup_ballot || state->needs_subgroup_shuffle || + state->needs_subgroup_local_invocation_id) { + state->builder.AddCapability({CapabilityGroupNonUniform}); + } + if (state->needs_subgroup_ballot) { + state->builder.AddCapability({CapabilityGroupNonUniformBallot}); + } + if (state->needs_subgroup_shuffle) { + state->builder.AddCapability({CapabilityGroupNonUniformShuffle}); + } + if (state->needs_compute_derivatives && state->stage == ShaderType::Compute) { + state->builder.AddCapability({CapabilityComputeDerivativeGroupQuadsKHR}); + state->builder.AddExtension("SPV_KHR_compute_shader_derivatives"); + } + state->builder.AddExtInstImport(state->glsl_std450, "GLSL.std.450"); + state->builder.AddMemoryModel({AddressingModelLogical, MemoryModelGLSL450}); + state->builder.AddEntryPoint(ExecutionModelForStage(state->stage), state->main_func, "main", + state->interface_variables); + if (state->stage == ShaderType::Compute) { + uint32_t local_x = state->needs_compute_derivatives ? 2u : 1u; + uint32_t local_y = state->needs_compute_derivatives ? 2u : 1u; + uint32_t local_z = 1u; + if (state->compute_input_info != nullptr) { + const auto* cs = state->compute_input_info; + local_x = cs->threads_num[0] != 0u ? cs->threads_num[0] : local_x; + local_y = cs->threads_num[1] != 0u ? cs->threads_num[1] : local_y; + local_z = cs->threads_num[2] != 0u ? cs->threads_num[2] : local_z; + } + state->builder.AddExecutionMode( + {state->main_func, ExecutionModeLocalSize, local_x, local_y, local_z}); + } + if (state->stage == ShaderType::Pixel) { + state->builder.AddExecutionMode({state->main_func, ExecutionModeOriginUpperLeft}); + if (state->depth_variable != 0) { + state->builder.AddExecutionMode({state->main_func, ExecutionModeDepthReplacing}); + } + if (state->pixel_input_info != nullptr && state->pixel_input_info->ps_early_z && + !state->pixel_input_info->ps_pixel_kill_enable && + !state->pixel_input_info->ps_depth_export_enable && + !state->pixel_input_info->ps_sample_mask_export_enable) { + state->builder.AddExecutionMode({state->main_func, ExecutionModeEarlyFragmentTests}); + } + } + if (state->needs_compute_derivatives && state->stage == ShaderType::Compute) { + state->builder.AddExecutionMode({state->main_func, ExecutionModeDerivativeGroupQuadsKHR}); + } + state->builder.AddName(state->main_func, "main"); + if (state->stage == ShaderType::Compute || state->needs_function_lds) { + state->builder.AddName(state->lds_variable, "lds_dwords"); + } + AddInputAnnotationsAndNames(state); + AddOutputAnnotationsAndNames(state); + AddDescriptorAnnotationsAndNames(state); + AddVsharpAnnotationsAndNames(state); + state->builder.AddAnnotation( + {OpDecorate, state->storage_runtime_array_type, DecorationArrayStride, 4}); + state->builder.AddAnnotation( + {OpMemberDecorate, state->storage_buffer_type, 0, DecorationOffset, 0}); + state->builder.AddAnnotation({OpDecorate, state->storage_buffer_type, DecorationBlock}); + + state->builder.AddType({OpTypeVoid, state->void_type}); + state->builder.AddType({OpTypeBool, state->bool_type}); + state->builder.AddType({OpTypeInt, state->uint_type, 32, 0}); + state->builder.AddType( + {OpTypeStruct, state->uint_pair_type, state->uint_type, state->uint_type}); + state->builder.AddType({OpTypeInt, state->int_type, 32, 1}); + state->builder.AddType({OpTypeStruct, state->int_pair_type, state->int_type, state->int_type}); + state->builder.AddType({OpTypeFloat, state->float_type, 32}); + state->builder.AddType({OpTypeVector, state->vec2_uint_type, state->uint_type, 2}); + state->builder.AddType({OpTypeVector, state->vec3_uint_type, state->uint_type, 3}); + state->builder.AddType({OpTypeVector, state->vec4_uint_type, state->uint_type, 4}); + state->builder.AddType({OpTypeVector, state->vec2_int_type, state->int_type, 2}); + state->builder.AddType({OpTypeVector, state->vec3_int_type, state->int_type, 3}); + state->builder.AddType({OpTypeVector, state->vec4_int_type, state->int_type, 4}); + state->builder.AddType({OpTypeVector, state->vec2_float_type, state->float_type, 2}); + state->builder.AddType({OpTypeVector, state->vec3_float_type, state->float_type, 3}); + state->builder.AddType({OpTypeVector, state->vec4_float_type, state->float_type, 4}); + state->builder.AddType( + {OpTypePointer, state->ptr_func_uint, StorageClassFunction, state->uint_type}); + state->builder.AddType( + {OpTypePointer, state->ptr_input_float, StorageClassInput, state->float_type}); + state->builder.AddType( + {OpTypePointer, state->ptr_input_bool, StorageClassInput, state->bool_type}); + state->builder.AddType( + {OpTypePointer, state->ptr_input_int, StorageClassInput, state->int_type}); + state->builder.AddType( + {OpTypePointer, state->ptr_input_uint, StorageClassInput, state->uint_type}); + state->builder.AddType( + {OpTypePointer, state->ptr_input_vec2_float, StorageClassInput, state->vec2_float_type}); + state->builder.AddType( + {OpTypePointer, state->ptr_input_vec3_float, StorageClassInput, state->vec3_float_type}); + state->builder.AddType( + {OpTypePointer, state->ptr_input_vec2_int, StorageClassInput, state->vec2_int_type}); + state->builder.AddType( + {OpTypePointer, state->ptr_input_vec3_int, StorageClassInput, state->vec3_int_type}); + state->builder.AddType( + {OpTypePointer, state->ptr_input_vec4_int, StorageClassInput, state->vec4_int_type}); + state->builder.AddType( + {OpTypePointer, state->ptr_input_vec2_uint, StorageClassInput, state->vec2_uint_type}); + state->builder.AddType( + {OpTypePointer, state->ptr_input_vec3_uint, StorageClassInput, state->vec3_uint_type}); + state->builder.AddType( + {OpTypePointer, state->ptr_input_vec4_uint, StorageClassInput, state->vec4_uint_type}); + state->builder.AddType( + {OpTypePointer, state->ptr_input_vec4_float, StorageClassInput, state->vec4_float_type}); + if (state->subgroup_local_invocation_id_variable != 0) { + state->builder.AddType({OpVariable, state->ptr_input_uint, + state->subgroup_local_invocation_id_variable, StorageClassInput}); + } + for (const auto& input: state->inputs) { + uint32_t ptr_type = state->ptr_input_uint; + switch (input.kind) { + case IR::StageInputKind::VertexIndex: + case IR::StageInputKind::InstanceIndex: ptr_type = state->ptr_input_int; break; + case IR::StageInputKind::WorkgroupId: + case IR::StageInputKind::LocalInvocationId: + case IR::StageInputKind::GlobalInvocationId: + ptr_type = state->ptr_input_vec3_uint; + break; + case IR::StageInputKind::FragCoord: ptr_type = state->ptr_input_vec4_float; break; + case IR::StageInputKind::FrontFacing: ptr_type = state->ptr_input_bool; break; + case IR::StageInputKind::Parameter: + if (state->stage == ShaderType::Vertex) { + const auto kind = VertexParameterScalarKind(*state, input.location); + const auto components = VertexParameterComponentCount(*state, input); + ptr_type = VertexParameterInputPointerType(*state, kind, components); + } else { + ptr_type = state->ptr_input_vec4_float; + } + break; + default: break; + } + state->builder.AddType({OpVariable, ptr_type, input.variable_id, StorageClassInput}); + } + state->builder.AddType( + {OpTypePointer, state->ptr_output_float, StorageClassOutput, state->float_type}); + state->builder.AddType( + {OpTypePointer, state->ptr_output_int, StorageClassOutput, state->int_type}); + state->builder.AddType( + {OpTypePointer, state->ptr_output_vec4_float, StorageClassOutput, state->vec4_float_type}); + if (state->per_vertex_variable != 0) { + state->builder.AddType({OpTypeStruct, state->per_vertex_type, state->vec4_float_type}); + state->builder.AddType({OpTypePointer, state->ptr_output_per_vertex, StorageClassOutput, + state->per_vertex_type}); + state->builder.AddType({OpVariable, state->ptr_output_per_vertex, + state->per_vertex_variable, StorageClassOutput}); + } + for (const auto& binding: state->outputs) { + if (binding.kind == IR::StageOutputKind::Parameter || + binding.kind == IR::StageOutputKind::Mrt) { + state->builder.AddType({OpVariable, state->ptr_output_vec4_float, binding.variable_id, + StorageClassOutput}); + } + } + if (state->depth_variable != 0) { + state->builder.AddType( + {OpVariable, state->ptr_output_float, state->depth_variable, StorageClassOutput}); + } + if (state->sample_mask_variable != 0) { + state->builder.AddType( + {OpTypeArray, state->sample_mask_array_type, state->int_type, ConstantU32(state, 1)}); + state->builder.AddType({OpTypePointer, state->ptr_output_sample_mask_array, + StorageClassOutput, state->sample_mask_array_type}); + state->builder.AddType({OpVariable, state->ptr_output_sample_mask_array, + state->sample_mask_variable, StorageClassOutput}); + } + state->builder.AddType( + {OpTypeRuntimeArray, state->storage_runtime_array_type, state->uint_type}); + state->builder.AddType( + {OpTypeStruct, state->storage_buffer_type, state->storage_runtime_array_type}); + state->builder.AddType({OpTypePointer, state->ptr_storage_buffer, StorageClassStorageBuffer, + state->storage_buffer_type}); + state->builder.AddType({OpTypePointer, state->ptr_storage_buffer_uint, + StorageClassStorageBuffer, state->uint_type}); + if (state->storage_buffer_variable != 0) { + const auto count = + ConstantU32(state, DescriptorCount(*state, IR::DescriptorBindingKind::Buffers)); + state->builder.AddType( + {OpTypeArray, state->storage_buffer_array_type, state->storage_buffer_type, count}); + state->builder.AddType({OpTypePointer, state->ptr_storage_buffer_array, + StorageClassStorageBuffer, state->storage_buffer_array_type}); + state->builder.AddType({OpVariable, state->ptr_storage_buffer_array, + state->storage_buffer_variable, StorageClassStorageBuffer}); + } + if (state->gds_variable != 0) { + state->builder.AddType({OpVariable, state->ptr_storage_buffer, state->gds_variable, + StorageClassStorageBuffer}); + } + if (state->push_constant_variable != 0) { + const auto row_width = ConstantU32(state, 4); + const auto row_count = ConstantU32( + state, std::max((state->program->bindings.push_constant_size + 15u) / 16u, 1u)); + state->builder.AddType( + {OpTypeArray, state->push_constant_u32x4_array_type, state->uint_type, row_width}); + state->builder.AddType({OpTypeArray, state->push_constant_rows_array_type, + state->push_constant_u32x4_array_type, row_count}); + state->builder.AddType( + {OpTypeStruct, state->push_constant_block_type, state->push_constant_rows_array_type}); + state->builder.AddType({OpTypePointer, state->ptr_push_constant_block, + StorageClassPushConstant, state->push_constant_block_type}); + state->builder.AddType({OpTypePointer, state->ptr_push_constant_uint, + StorageClassPushConstant, state->uint_type}); + state->builder.AddType({OpVariable, state->ptr_push_constant_block, + state->push_constant_variable, StorageClassPushConstant}); + } + if (state->vsharp_storage_variable != 0) { + state->builder.AddType({OpVariable, state->ptr_storage_buffer, + state->vsharp_storage_variable, StorageClassStorageBuffer}); + } + if (state->address_memory_variable != 0) { + const auto count = + ConstantU32(state, DescriptorCount(*state, IR::DescriptorBindingKind::AddressMemory)); + state->builder.AddType( + {OpTypeArray, state->address_memory_array_type, state->storage_buffer_type, count}); + state->builder.AddType({OpTypePointer, state->ptr_address_memory_array, + StorageClassStorageBuffer, state->address_memory_array_type}); + state->builder.AddType({OpVariable, state->ptr_address_memory_array, + state->address_memory_variable, StorageClassStorageBuffer}); + } + if (state->flattened_srt_variable != 0) { + state->builder.AddType({OpVariable, state->ptr_storage_buffer, + state->flattened_srt_variable, StorageClassStorageBuffer}); + } + if (state->stage == ShaderType::Compute || state->needs_function_lds) { + const auto storage_class = + state->needs_function_lds ? StorageClassFunction : StorageClassWorkgroup; + const auto lds_size = ConstantU32(state, state->needs_function_lds ? 8192u : 1024u); + state->builder.AddType({OpTypeArray, state->lds_array_type, state->uint_type, lds_size}); + state->builder.AddType( + {OpTypePointer, state->ptr_workgroup_array, storage_class, state->lds_array_type}); + state->builder.AddType( + {OpTypePointer, state->ptr_workgroup_uint, storage_class, state->uint_type}); + if (state->stage == ShaderType::Compute) { + state->builder.AddType({OpVariable, state->ptr_workgroup_array, state->lds_variable, + StorageClassWorkgroup}); + } + } + for (uint32_t i = 0; i < state->sampled_images.size(); i++) { + auto& image = state->sampled_images[i]; + const auto view = static_cast(i % 3u); + const auto component = i >= 3u ? state->uint_type : state->float_type; + const auto dimension = view == ImageViewKind::Dim3D ? Dim3D : Dim2D; + const auto arrayed = view == ImageViewKind::Dim2DArray ? 1u : 0u; + state->builder.AddType({OpTypeImage, image.image_type, component, dimension, 0, arrayed, 0, + 1, ImageFormatUnknown}); + state->builder.AddType({OpTypeSampledImage, image.sampled_image_type, image.image_type}); + state->builder.AddType( + {OpTypePointer, image.pointer_type, StorageClassUniformConstant, image.image_type}); + if (image.variable != 0) { + const auto kind = SampledBindingKind(i >= 3u, view); + const auto count = ConstantU32(state, DescriptorCount(*state, kind)); + state->builder.AddType({OpTypeArray, image.array_type, image.image_type, count}); + state->builder.AddType({OpTypePointer, image.array_pointer_type, + StorageClassUniformConstant, image.array_type}); + state->builder.AddType({OpVariable, image.array_pointer_type, image.variable, + StorageClassUniformConstant}); + } + } + state->builder.AddType({OpTypeSampler, state->sampler_type}); + state->builder.AddType({OpTypePointer, state->ptr_uniform_sampler, StorageClassUniformConstant, + state->sampler_type}); + if (state->sampler_variable != 0) { + const auto count = + ConstantU32(state, DescriptorCount(*state, IR::DescriptorBindingKind::Samplers)); + state->builder.AddType( + {OpTypeArray, state->sampler_array_type, state->sampler_type, count}); + state->builder.AddType({OpTypePointer, state->ptr_uniform_sampler_array, + StorageClassUniformConstant, state->sampler_array_type}); + state->builder.AddType({OpVariable, state->ptr_uniform_sampler_array, + state->sampler_variable, StorageClassUniformConstant}); + } + state->builder.AddType({OpTypeImage, state->storage_image_type, state->float_type, Dim2D, 0, 0, + 0, 2, ImageFormatUnknown}); + state->builder.AddType({OpTypePointer, state->ptr_uniform_storage_image, + StorageClassUniformConstant, state->storage_image_type}); + if (state->storage_image_variable != 0) { + const auto count = + ConstantU32(state, DescriptorCount(*state, IR::DescriptorBindingKind::Storage2D)); + state->builder.AddType( + {OpTypeArray, state->storage_image_array_type, state->storage_image_type, count}); + state->builder.AddType({OpTypePointer, state->ptr_uniform_storage_image_array, + StorageClassUniformConstant, state->storage_image_array_type}); + state->builder.AddType({OpVariable, state->ptr_uniform_storage_image_array, + state->storage_image_variable, StorageClassUniformConstant}); + } + state->builder.AddType({OpTypeImage, state->storage_image_2d_array_type, state->float_type, + Dim2D, 0, 1, 0, 2, ImageFormatUnknown}); + state->builder.AddType({OpTypePointer, state->ptr_uniform_storage_image_2d_array, + StorageClassUniformConstant, state->storage_image_2d_array_type}); + if (state->storage_image_2d_array_variable != 0) { + const auto count = + ConstantU32(state, DescriptorCount(*state, IR::DescriptorBindingKind::Storage2DArray)); + state->builder.AddType({OpTypeArray, state->storage_image_2d_array_array_type, + state->storage_image_2d_array_type, count}); + state->builder.AddType({OpTypePointer, state->ptr_uniform_storage_image_2d_array_array, + StorageClassUniformConstant, + state->storage_image_2d_array_array_type}); + state->builder.AddType({OpVariable, state->ptr_uniform_storage_image_2d_array_array, + state->storage_image_2d_array_variable, + StorageClassUniformConstant}); + } + state->builder.AddType({OpTypeImage, state->storage_image_3d_type, state->float_type, Dim3D, 0, + 0, 0, 2, ImageFormatUnknown}); + state->builder.AddType({OpTypePointer, state->ptr_uniform_storage_image_3d, + StorageClassUniformConstant, state->storage_image_3d_type}); + if (state->storage_image_3d_variable != 0) { + const auto count = + ConstantU32(state, DescriptorCount(*state, IR::DescriptorBindingKind::Storage3D)); + state->builder.AddType( + {OpTypeArray, state->storage_image_3d_array_type, state->storage_image_3d_type, count}); + state->builder.AddType({OpTypePointer, state->ptr_uniform_storage_image_3d_array, + StorageClassUniformConstant, state->storage_image_3d_array_type}); + state->builder.AddType({OpVariable, state->ptr_uniform_storage_image_3d_array, + state->storage_image_3d_variable, StorageClassUniformConstant}); + } + state->builder.AddType({OpTypeImage, state->storage_image_uint_type, state->uint_type, Dim2D, 0, + 0, 0, 2, ImageFormatR32ui}); + state->builder.AddType({OpTypePointer, state->ptr_uniform_storage_image_uint, + StorageClassUniformConstant, state->storage_image_uint_type}); + if (state->storage_image_uint_variable != 0) { + const auto count = + ConstantU32(state, DescriptorCount(*state, IR::DescriptorBindingKind::StorageUint2D)); + state->builder.AddType({OpTypeArray, state->storage_image_uint_array_type, + state->storage_image_uint_type, count}); + state->builder.AddType({OpTypePointer, state->ptr_uniform_storage_image_uint_array, + StorageClassUniformConstant, state->storage_image_uint_array_type}); + state->builder.AddType({OpVariable, state->ptr_uniform_storage_image_uint_array, + state->storage_image_uint_variable, StorageClassUniformConstant}); + } + state->builder.AddType({OpTypeImage, state->storage_image_uint_2d_array_type, state->uint_type, + Dim2D, 0, 1, 0, 2, ImageFormatR32ui}); + state->builder.AddType({OpTypePointer, state->ptr_uniform_storage_image_uint_2d_array, + StorageClassUniformConstant, state->storage_image_uint_2d_array_type}); + if (state->storage_image_uint_2d_array_variable != 0) { + const auto count = ConstantU32( + state, DescriptorCount(*state, IR::DescriptorBindingKind::StorageUint2DArray)); + state->builder.AddType({OpTypeArray, state->storage_image_uint_2d_array_array_type, + state->storage_image_uint_2d_array_type, count}); + state->builder.AddType({OpTypePointer, state->ptr_uniform_storage_image_uint_2d_array_array, + StorageClassUniformConstant, + state->storage_image_uint_2d_array_array_type}); + state->builder.AddType({OpVariable, state->ptr_uniform_storage_image_uint_2d_array_array, + state->storage_image_uint_2d_array_variable, + StorageClassUniformConstant}); + } + state->builder.AddType({OpTypeImage, state->storage_image_uint_3d_type, state->uint_type, Dim3D, + 0, 0, 0, 2, ImageFormatR32ui}); + state->builder.AddType({OpTypePointer, state->ptr_uniform_storage_image_uint_3d, + StorageClassUniformConstant, state->storage_image_uint_3d_type}); + if (state->storage_image_uint_3d_variable != 0) { + const auto count = + ConstantU32(state, DescriptorCount(*state, IR::DescriptorBindingKind::StorageUint3D)); + state->builder.AddType({OpTypeArray, state->storage_image_uint_3d_array_type, + state->storage_image_uint_3d_type, count}); + state->builder.AddType({OpTypePointer, state->ptr_uniform_storage_image_uint_3d_array, + StorageClassUniformConstant, + state->storage_image_uint_3d_array_type}); + state->builder.AddType({OpVariable, state->ptr_uniform_storage_image_uint_3d_array, + state->storage_image_uint_3d_variable, + StorageClassUniformConstant}); + } + state->builder.AddType( + {OpTypePointer, state->ptr_image_uint, StorageClassImage, state->uint_type}); + state->builder.AddType({OpTypeFunction, state->func_type, state->void_type}); +} + +void AllocateRegisterVariables(EmitterState* state) { + if (state->stage == ShaderType::Pixel && state->needs_pixel_valid_mask) { + state->pixel_valid_mask_variable = state->builder.AllocateId(); + state->builder.AddName(state->pixel_valid_mask_variable, "pixel_valid_mask_active"); + } + for (auto& binding: state->registers) { + binding.pointer_id = state->builder.AllocateId(); + const auto name = IR::RegisterToString(binding.reg); + state->builder.AddName(binding.pointer_id, name.c_str()); + } +} + +void AllocateDescriptorVariables(EmitterState* state) { + if (DescriptorBinding(*state, IR::DescriptorBindingKind::Buffers) != nullptr) { + state->storage_buffer_variable = state->builder.AllocateId(); + } + if (DescriptorBinding(*state, IR::DescriptorBindingKind::AddressMemory) != nullptr) { + state->address_memory_variable = state->builder.AllocateId(); + } + if (state->program->bindings.push_constant_size != 0) { + state->push_constant_variable = state->builder.AllocateId(); + } + if (DescriptorBinding(*state, IR::DescriptorBindingKind::UserData) != nullptr) { + state->vsharp_storage_variable = state->builder.AllocateId(); + } + if (DescriptorBinding(*state, IR::DescriptorBindingKind::FlattenedSrt) != nullptr) { + state->flattened_srt_variable = state->builder.AllocateId(); + } + for (uint32_t i = 0; i < state->sampled_images.size(); i++) { + const auto view = static_cast(i % 3u); + if (DescriptorBinding(*state, SampledBindingKind(i >= 3u, view)) != nullptr) { + state->sampled_images[i].variable = state->builder.AllocateId(); + } + } + if (DescriptorBinding(*state, IR::DescriptorBindingKind::Storage2D) != nullptr) { + state->storage_image_variable = state->builder.AllocateId(); + } + if (DescriptorBinding(*state, IR::DescriptorBindingKind::StorageUint2D) != nullptr) { + state->storage_image_uint_variable = state->builder.AllocateId(); + } + if (DescriptorBinding(*state, IR::DescriptorBindingKind::Storage2DArray) != nullptr) { + state->storage_image_2d_array_variable = state->builder.AllocateId(); + } + if (DescriptorBinding(*state, IR::DescriptorBindingKind::StorageUint2DArray) != nullptr) { + state->storage_image_uint_2d_array_variable = state->builder.AllocateId(); + } + if (DescriptorBinding(*state, IR::DescriptorBindingKind::Storage3D) != nullptr) { + state->storage_image_3d_variable = state->builder.AllocateId(); + } + if (DescriptorBinding(*state, IR::DescriptorBindingKind::StorageUint3D) != nullptr) { + state->storage_image_uint_3d_variable = state->builder.AllocateId(); + } + if (DescriptorBinding(*state, IR::DescriptorBindingKind::Samplers) != nullptr) { + state->sampler_variable = state->builder.AllocateId(); + } + if (DescriptorBinding(*state, IR::DescriptorBindingKind::Gds) != nullptr) { + state->gds_variable = state->builder.AllocateId(); + } +} + +} // namespace Libs::Graphics::ShaderRecompiler::Spirv::Emitter diff --git a/src/graphics/shader/recompiler/spirvEmitter/spirvEmitterValues.cpp b/src/graphics/shader/recompiler/spirvEmitter/spirvEmitterValues.cpp new file mode 100644 index 000000000..44758a094 --- /dev/null +++ b/src/graphics/shader/recompiler/spirvEmitter/spirvEmitterValues.cpp @@ -0,0 +1,840 @@ +#include "graphics/shader/recompiler/spirvEmitter/spirvEmitterInternal.h" + +namespace Libs::Graphics::ShaderRecompiler::Spirv::Emitter { + +bool SdwaSelectorOffsetWidth(uint32_t sel, uint32_t& offset, uint32_t& width) { + if (sel <= 3u) { + offset = sel * 8u; + width = 8u; + return true; + } + if (sel == 4u || sel == 5u) { + offset = sel == 5u ? 16u : 0u; + width = 16u; + return true; + } + if (sel == 6u) { + offset = 0; + width = 32u; + return true; + } + return false; +} + +uint32_t EmitSdwaExtractU32(EmitterState* state, const IR::Operand& operand, uint32_t value) { + if (operand.sdwa_sel == 6u && !operand.sdwa_sext) { + return value; + } + + uint32_t offset = 0; + uint32_t width = 0; + if (!SdwaSelectorOffsetWidth(operand.sdwa_sel, offset, width) || width == 32u) { + return value; + } + + if (operand.sdwa_sext) { + const auto signed_value = state->builder.AllocateId(); + const auto extracted = state->builder.AllocateId(); + const auto bits = state->builder.AllocateId(); + state->builder.AddFunction({OpBitcast, state->int_type, signed_value, value}); + state->builder.AddFunction({OpBitFieldSExtract, state->int_type, extracted, signed_value, + ConstantU32(state, offset), ConstantU32(state, width)}); + state->builder.AddFunction({OpBitcast, state->uint_type, bits, extracted}); + return bits; + } + + const auto extracted = state->builder.AllocateId(); + state->builder.AddFunction({OpBitFieldUExtract, state->uint_type, extracted, value, + ConstantU32(state, offset), ConstantU32(state, width)}); + return extracted; +} + +uint32_t EmitTrueBool(EmitterState* state) { + const auto ret = state->builder.AllocateId(); + state->builder.AddFunction( + {OpINotEqual, state->bool_type, ret, ConstantU32(state, 1), ConstantU32(state, 0)}); + return ret; +} + +DppTargetLane EmitDppQuadPermTargetLane(EmitterState* state, uint32_t subid, uint32_t control) { + const auto quad_base = state->builder.AllocateId(); + const auto lane = state->builder.AllocateId(); + const auto shift = state->builder.AllocateId(); + const auto selected0 = state->builder.AllocateId(); + const auto selected = state->builder.AllocateId(); + const auto target = state->builder.AllocateId(); + state->builder.AddFunction( + {OpBitwiseAnd, state->uint_type, quad_base, subid, ConstantU32(state, 0xfffffffcu)}); + state->builder.AddFunction( + {OpBitwiseAnd, state->uint_type, lane, subid, ConstantU32(state, 3)}); + state->builder.AddFunction( + {OpShiftLeftLogical, state->uint_type, shift, lane, ConstantU32(state, 1)}); + state->builder.AddFunction( + {OpShiftRightLogical, state->uint_type, selected0, ConstantU32(state, control), shift}); + state->builder.AddFunction( + {OpBitwiseAnd, state->uint_type, selected, selected0, ConstantU32(state, 3)}); + state->builder.AddFunction({OpBitwiseOr, state->uint_type, target, quad_base, selected}); + return {target, EmitTrueBool(state)}; +} + +DppTargetLane EmitDppRowShiftTargetLane(EmitterState* state, uint32_t subid, uint32_t amount, + bool left) { + const auto row = state->builder.AllocateId(); + const auto lane = state->builder.AllocateId(); + const auto lane_shifted = state->builder.AllocateId(); + const auto target = state->builder.AllocateId(); + const auto valid = state->builder.AllocateId(); + state->builder.AddFunction( + {OpBitwiseAnd, state->uint_type, row, subid, ConstantU32(state, 0xfffffff0u)}); + state->builder.AddFunction( + {OpBitwiseAnd, state->uint_type, lane, subid, ConstantU32(state, 15)}); + if (left) { + state->builder.AddFunction( + {OpIAdd, state->uint_type, lane_shifted, lane, ConstantU32(state, amount)}); + state->builder.AddFunction( + {OpULessThan, state->bool_type, valid, lane, ConstantU32(state, 16u - amount)}); + } else { + state->builder.AddFunction( + {OpISub, state->uint_type, lane_shifted, lane, ConstantU32(state, amount)}); + state->builder.AddFunction( + {OpUGreaterThanEqual, state->bool_type, valid, lane, ConstantU32(state, amount)}); + } + state->builder.AddFunction({OpBitwiseOr, state->uint_type, target, row, lane_shifted}); + return {target, valid}; +} + +DppTargetLane EmitDppRowRotateRightTargetLane(EmitterState* state, uint32_t subid, + uint32_t amount) { + const auto row = state->builder.AllocateId(); + const auto lane = state->builder.AllocateId(); + const auto in_high = state->builder.AllocateId(); + const auto minus = state->builder.AllocateId(); + const auto plus = state->builder.AllocateId(); + const auto selected = state->builder.AllocateId(); + const auto target = state->builder.AllocateId(); + state->builder.AddFunction( + {OpBitwiseAnd, state->uint_type, row, subid, ConstantU32(state, 0xfffffff0u)}); + state->builder.AddFunction( + {OpBitwiseAnd, state->uint_type, lane, subid, ConstantU32(state, 15)}); + state->builder.AddFunction( + {OpUGreaterThanEqual, state->bool_type, in_high, lane, ConstantU32(state, amount)}); + state->builder.AddFunction({OpISub, state->uint_type, minus, lane, ConstantU32(state, amount)}); + state->builder.AddFunction( + {OpIAdd, state->uint_type, plus, lane, ConstantU32(state, 16u - amount)}); + state->builder.AddFunction({OpSelect, state->uint_type, selected, in_high, minus, plus}); + state->builder.AddFunction({OpBitwiseOr, state->uint_type, target, row, selected}); + return {target, EmitTrueBool(state)}; +} + +DppTargetLane EmitDppMirrorTargetLane(EmitterState* state, uint32_t subid, bool half_row) { + const auto base_mask = half_row ? 0xfffffff8u : 0xfffffff0u; + const auto lane_mask = half_row ? 7u : 15u; + const auto base = state->builder.AllocateId(); + const auto lane = state->builder.AllocateId(); + const auto mirrored = state->builder.AllocateId(); + const auto target = state->builder.AllocateId(); + state->builder.AddFunction( + {OpBitwiseAnd, state->uint_type, base, subid, ConstantU32(state, base_mask)}); + state->builder.AddFunction( + {OpBitwiseAnd, state->uint_type, lane, subid, ConstantU32(state, lane_mask)}); + state->builder.AddFunction( + {OpISub, state->uint_type, mirrored, ConstantU32(state, lane_mask), lane}); + state->builder.AddFunction({OpBitwiseOr, state->uint_type, target, base, mirrored}); + return {target, EmitTrueBool(state)}; +} + +DppTargetLane EmitDppTargetLane(EmitterState* state, uint32_t control) { + const auto subid = EmitSubgroupLocalInvocationId(state); + if (control <= 0xffu) { + return EmitDppQuadPermTargetLane(state, subid, control); + } + if (control >= 0x101u && control <= 0x10fu) { + return EmitDppRowShiftTargetLane(state, subid, control & 0xfu, true); + } + if (control >= 0x111u && control <= 0x11fu) { + return EmitDppRowShiftTargetLane(state, subid, control & 0xfu, false); + } + if (control >= 0x121u && control <= 0x12fu) { + return EmitDppRowRotateRightTargetLane(state, subid, control & 0xfu); + } + if (control == 0x140u) { + return EmitDppMirrorTargetLane(state, subid, false); + } + if (control == 0x141u) { + return EmitDppMirrorTargetLane(state, subid, true); + } + return {subid, EmitTrueBool(state)}; +} + +uint32_t EmitDppValueU32(EmitterState* state, const IR::Operand& operand, uint32_t value) { + if (!operand.dpp) { + return value; + } + + const auto target = EmitDppTargetLane(state, operand.dpp_ctrl); + const auto shuffled = state->builder.AllocateId(); + state->builder.AddFunction({OpGroupNonUniformShuffle, state->uint_type, shuffled, + ConstantU32(state, ScopeSubgroup), value, target.lane}); + if (operand.dpp_fetch_inactive) { + return shuffled; + } + + const auto source_active = EmitLaneIndexActiveBool(state, target.lane); + const auto can_fetch = state->builder.AllocateId(); + const auto ret = state->builder.AllocateId(); + state->builder.AddFunction( + {OpLogicalAnd, state->bool_type, can_fetch, target.valid, source_active}); + state->builder.AddFunction( + {OpSelect, state->uint_type, ret, can_fetch, shuffled, ConstantU32(state, 0)}); + return ret; +} + +uint32_t EmitValueLoad(EmitterState* state, const IR::Operand& operand) { + uint32_t value = 0; + switch (operand.kind) { + case IR::OperandKind::ImmediateU32: + case IR::OperandKind::PcRelativeU32: value = ConstantU32(state, operand.imm); break; + case IR::OperandKind::Register: { + const auto pointer = PointerForRegister(*state, operand.reg); + if (pointer == 0) { + value = ConstantU32(state, 0); + break; + } + value = state->builder.AllocateId(); + state->builder.AddFunction({OpLoad, state->uint_type, value, pointer}); + break; + } + case IR::OperandKind::Null: value = ConstantU32(state, 0); break; + default: value = ConstantU32(state, 0); break; + } + value = EmitDppValueU32(state, operand, value); + return EmitSdwaExtractU32(state, operand, value); +} + +uint32_t EmitRegisterLoad(EmitterState* state, IR::Register reg) { + if (IsInactiveWave32ExecHigh(*state, reg)) { + return ConstantU32(state, 0); + } + const auto pointer = PointerForRegister(*state, reg); + if (pointer == 0) { + return ConstantU32(state, 0); + } + const auto value = state->builder.AllocateId(); + state->builder.AddFunction({OpLoad, state->uint_type, value, pointer}); + return value; +} + +uint32_t WaveMaskHighLoad(EmitterState* state, IR::RegisterFile file) { + if (state->wave_size == 32u && IsMaskRegisterFile(file)) { + return ConstantU32(state, 0); + } + return EmitRegisterLoad(state, {file, 1}); +} + +uint32_t EmitLaneMaskPartU32(EmitterState* state, uint32_t part) { + const auto subid = EmitSubgroupLocalInvocationId(state); + const auto lane = state->builder.AllocateId(); + const auto bit = state->builder.AllocateId(); + state->builder.AddFunction( + {OpBitwiseAnd, state->uint_type, lane, subid, ConstantU32(state, 31)}); + state->builder.AddFunction( + {OpShiftLeftLogical, state->uint_type, bit, ConstantU32(state, 1), lane}); + if (part == 0) { + const auto in_low = state->builder.AllocateId(); + const auto ret = state->builder.AllocateId(); + state->builder.AddFunction( + {OpULessThan, state->bool_type, in_low, subid, ConstantU32(state, 32)}); + state->builder.AddFunction( + {OpSelect, state->uint_type, ret, in_low, bit, ConstantU32(state, 0)}); + return ret; + } + if (state->wave_size == 32u) { + return ConstantU32(state, 0); + } + const auto in_high = state->builder.AllocateId(); + const auto ret = state->builder.AllocateId(); + state->builder.AddFunction( + {OpUGreaterThanEqual, state->bool_type, in_high, subid, ConstantU32(state, 32)}); + state->builder.AddFunction( + {OpSelect, state->uint_type, ret, in_high, bit, ConstantU32(state, 0)}); + return ret; +} + +uint32_t EmitThreadMaskBelowPartU32(EmitterState* state, uint32_t part) { + const auto subid = EmitSubgroupLocalInvocationId(state); + const auto lane = state->builder.AllocateId(); + const auto bit = state->builder.AllocateId(); + const auto below = state->builder.AllocateId(); + state->builder.AddFunction( + {OpBitwiseAnd, state->uint_type, lane, subid, ConstantU32(state, 31)}); + state->builder.AddFunction( + {OpShiftLeftLogical, state->uint_type, bit, ConstantU32(state, 1), lane}); + state->builder.AddFunction({OpISub, state->uint_type, below, bit, ConstantU32(state, 1)}); + + if (part == 0) { + const auto in_high = state->builder.AllocateId(); + const auto ret = state->builder.AllocateId(); + state->builder.AddFunction( + {OpUGreaterThanEqual, state->bool_type, in_high, subid, ConstantU32(state, 32)}); + state->builder.AddFunction( + {OpSelect, state->uint_type, ret, in_high, ConstantU32(state, 0xffffffffu), below}); + return ret; + } + if (state->wave_size == 32u) { + return ConstantU32(state, 0); + } + const auto in_high = state->builder.AllocateId(); + const auto ret = state->builder.AllocateId(); + state->builder.AddFunction( + {OpUGreaterThanEqual, state->bool_type, in_high, subid, ConstantU32(state, 32)}); + state->builder.AddFunction( + {OpSelect, state->uint_type, ret, in_high, below, ConstantU32(state, 0)}); + return ret; +} + +uint32_t EmitMaskActiveBool(EmitterState* state, IR::RegisterFile file) { + const auto low = EmitRegisterLoad(state, {file, 0}); + const auto condition = state->builder.AllocateId(); + if (state->per_invocation_masks) { + const auto combined = state->builder.AllocateId(); + state->builder.AddFunction( + {OpBitwiseOr, state->uint_type, combined, low, WaveMaskHighLoad(state, file)}); + state->builder.AddFunction( + {OpINotEqual, state->bool_type, condition, combined, ConstantU32(state, 0)}); + return condition; + } + if (state->exact_subgroup_operations) { + const auto high = WaveMaskHighLoad(state, file); + const auto low_hit = state->builder.AllocateId(); + const auto high_hit = state->builder.AllocateId(); + const auto active = state->builder.AllocateId(); + state->builder.AddFunction( + {OpBitwiseAnd, state->uint_type, low_hit, low, EmitLaneMaskPartU32(state, 0)}); + state->builder.AddFunction( + {OpBitwiseAnd, state->uint_type, high_hit, high, EmitLaneMaskPartU32(state, 1)}); + state->builder.AddFunction({OpBitwiseOr, state->uint_type, active, low_hit, high_hit}); + state->builder.AddFunction( + {OpINotEqual, state->bool_type, condition, active, ConstantU32(state, 0)}); + return condition; + } + state->builder.AddFunction( + {OpINotEqual, state->bool_type, condition, low, ConstantU32(state, 0)}); + return condition; +} + +uint32_t EmitMaskZeroBool(EmitterState* state, IR::RegisterFile file, bool zero) { + const auto active = EmitMaskActiveBool(state, file); + if (!zero) { + return active; + } + const auto cond = state->builder.AllocateId(); + state->builder.AddFunction({OpLogicalNot, state->bool_type, cond, active}); + return cond; +} + +uint32_t EmitMaskSummaryZeroBool(EmitterState* state, IR::RegisterFile file, bool zero) { + if (state->per_invocation_masks) { + return EmitMaskZeroBool(state, file, zero); + } + if (state->exact_subgroup_operations) { + const auto low = EmitRegisterLoad(state, {file, 0}); + const auto high = WaveMaskHighLoad(state, file); + const auto combined = state->builder.AllocateId(); + const auto active = state->builder.AllocateId(); + state->builder.AddFunction({OpBitwiseOr, state->uint_type, combined, low, high}); + state->builder.AddFunction( + {OpINotEqual, state->bool_type, active, combined, ConstantU32(state, 0)}); + if (!zero) { + return active; + } + const auto inactive = state->builder.AllocateId(); + state->builder.AddFunction({OpLogicalNot, state->bool_type, inactive, active}); + return inactive; + } + const auto active = EmitMaskActiveBool(state, file); + if (!zero) { + return active; + } + const auto inactive = state->builder.AllocateId(); + state->builder.AddFunction({OpLogicalNot, state->bool_type, inactive, active}); + return inactive; +} + +uint32_t EmitExecActiveBool(EmitterState* state) { + return EmitMaskZeroBool(state, IR::RegisterFile::Exec, false); +} + +uint32_t EmitConditionBool(EmitterState* state, const IR::Operand& operand) { + if (operand.kind == IR::OperandKind::Register) { + if (operand.reg.file == IR::RegisterFile::Exec || + operand.reg.file == IR::RegisterFile::Vcc) { + return EmitMaskActiveBool(state, operand.reg.file); + } + } + const auto cond_value = EmitValueLoad(state, operand); + const auto cond = state->builder.AllocateId(); + state->builder.AddFunction( + {OpINotEqual, state->bool_type, cond, cond_value, ConstantU32(state, 0)}); + return cond; +} + +uint32_t EmitSubgroupLocalInvocationId(EmitterState* state) { + if (state->subgroup_local_invocation_id_variable == 0) { + return ConstantU32(state, 0); + } + const auto value = state->builder.AllocateId(); + state->builder.AddFunction( + {OpLoad, state->uint_type, value, state->subgroup_local_invocation_id_variable}); + return value; +} + +uint32_t InputVariableForKind(const EmitterState& state, IR::StageInputKind kind) { + for (const auto& input: state.inputs) { + if (input.kind == kind) { + return input.variable_id; + } + } + return 0; +} + +uint32_t InputVariableForParameter(const EmitterState& state, uint32_t location) { + for (const auto& input: state.inputs) { + if (input.kind == IR::StageInputKind::Parameter && input.location == location) { + return input.variable_id; + } + } + return 0; +} + +const InputBinding* InputBindingForParameter(const EmitterState& state, uint32_t location) { + for (const auto& input: state.inputs) { + if (input.kind == IR::StageInputKind::Parameter && input.location == location) { + return &input; + } + } + return nullptr; +} + +uint32_t EmitInputComponentU32(EmitterState* state, IR::StageInputKind kind, uint32_t component) { + const auto variable = InputVariableForKind(*state, kind); + if (variable == 0) { + return ConstantU32(state, 0); + } + const auto pointer = state->builder.AllocateId(); + const auto value = state->builder.AllocateId(); + state->builder.AddFunction( + {OpAccessChain, state->ptr_input_uint, pointer, variable, ConstantU32(state, component)}); + state->builder.AddFunction({OpLoad, state->uint_type, value, pointer}); + return value; +} + +uint32_t EmitLocalInvocationIndex(EmitterState* state) { + const auto variable = InputVariableForKind(*state, IR::StageInputKind::LocalInvocationIndex); + if (variable == 0) { + return ConstantU32(state, 0); + } + const auto value = state->builder.AllocateId(); + state->builder.AddFunction({OpLoad, state->uint_type, value, variable}); + return value; +} + +uint32_t VertexInputDefaultComponentU32(EmitterState* state, VertexInputScalarKind kind, + uint32_t component) { + if ((component & 3u) != 3u) { + return ConstantU32(state, 0); + } + return ConstantU32(state, kind == VertexInputScalarKind::Float ? 0x3f800000u : 1u); +} + +uint32_t EmitVertexParameterComponentU32(EmitterState* state, const InputBinding& input, + uint32_t component) { + const auto count = VertexParameterComponentCount(*state, input); + const auto kind = VertexParameterScalarKind(*state, input.location); + if (component >= count) { + return VertexInputDefaultComponentU32(state, kind, component); + } + + const auto scalar_type = VertexParameterScalarType(*state, kind); + uint32_t raw = state->builder.AllocateId(); + if (count == 1u) { + state->builder.AddFunction({OpLoad, scalar_type, raw, input.variable_id}); + } else { + const auto pointer_type = VertexParameterScalarPointerType(*state, kind); + const auto pointer = state->builder.AllocateId(); + state->builder.AddFunction({OpAccessChain, pointer_type, pointer, input.variable_id, + ConstantU32(state, component)}); + state->builder.AddFunction({OpLoad, scalar_type, raw, pointer}); + } + + if (kind == VertexInputScalarKind::Uint) { + return raw; + } + + const auto bits = state->builder.AllocateId(); + state->builder.AddFunction({OpBitcast, state->uint_type, bits, raw}); + return bits; +} + +void EmitLoadInputF32(EmitterState* state, const IR::Instruction& inst) { + const auto* input = InputBindingForParameter(*state, inst.input_info.attr); + if (input == nullptr || input->variable_id == 0) { + EmitStoreU32(state, inst.dst, ConstantU32(state, 0)); + return; + } + + if (state->stage == ShaderType::Vertex) { + EmitStoreU32(state, inst.dst, + EmitVertexParameterComponentU32(state, *input, inst.input_info.chan & 3u)); + return; + } + + const auto input_value = state->builder.AllocateId(); + const auto component = state->builder.AllocateId(); + const auto bits = state->builder.AllocateId(); + state->builder.AddFunction({OpLoad, state->vec4_float_type, input_value, input->variable_id}); + state->builder.AddFunction( + {OpCompositeExtract, state->float_type, component, input_value, inst.input_info.chan & 3u}); + state->builder.AddFunction({OpBitcast, state->uint_type, bits, component}); + EmitStoreU32(state, inst.dst, bits); +} + +uint32_t EmitSccBool(EmitterState* state, bool non_zero) { + const auto value = EmitRegisterLoad(state, SccRegister()); + const auto cond = state->builder.AllocateId(); + state->builder.AddFunction( + {non_zero ? OpINotEqual : OpIEqual, state->bool_type, cond, value, ConstantU32(state, 0)}); + return cond; +} + +uint32_t EmitFloatLoad(EmitterState* state, const IR::Operand& operand) { + const auto uint_value = EmitValueLoad(state, operand); + auto float_value = state->builder.AllocateId(); + state->builder.AddFunction({OpBitcast, state->float_type, float_value, uint_value}); + if (operand.absolute) { + const auto abs_value = state->builder.AllocateId(); + state->builder.AddFunction( + {OpExtInst, state->float_type, abs_value, state->glsl_std450, GlslFAbs, float_value}); + float_value = abs_value; + } + if (operand.negate) { + const auto neg_value = state->builder.AllocateId(); + state->builder.AddFunction({OpFNegate, state->float_type, neg_value, float_value}); + float_value = neg_value; + } + return float_value; +} + +uint32_t ApplyResultModifiersF32(EmitterState* state, uint32_t value, const IR::Operand& dst) { + uint32_t result = value; + if (dst.omod != 0) { + const uint32_t bits = dst.omod == 1u ? 0x40000000u + : dst.omod == 2u ? 0x40800000u + : 0x3f000000u; + const auto scaled = state->builder.AllocateId(); + state->builder.AddFunction( + {OpFMul, state->float_type, scaled, result, ConstantF32(state, bits)}); + result = scaled; + } + if (!dst.clamp) { + return result; + } + const auto ret = state->builder.AllocateId(); + state->builder.AddFunction({OpExtInst, state->float_type, ret, state->glsl_std450, GlslFClamp, + result, ConstantF32(state, 0), ConstantF32(state, 0x3f800000u)}); + return ret; +} + +uint32_t EmitMixF32Load(EmitterState* state, const IR::Operand& operand) { + auto value_operand = operand; + value_operand.negate_hi = false; + + if (operand.op_sel_hi) { + auto raw = operand; + raw.sdwa_sel = 6; + raw.sdwa_sext = false; + raw.op_sel = false; + raw.op_sel_hi = false; + raw.negate = false; + raw.negate_hi = false; + raw.absolute = false; + raw.dpp = false; + raw.dpp_fetch_inactive = false; + raw.dpp_bound_ctrl = false; + + const auto bits = EmitValueLoad(state, raw); + const auto unpacked = state->builder.AllocateId(); + auto f32 = state->builder.AllocateId(); + state->builder.AddFunction({OpExtInst, state->vec2_float_type, unpacked, state->glsl_std450, + GlslUnpackHalf2x16, bits}); + state->builder.AddFunction( + {OpCompositeExtract, state->float_type, f32, unpacked, operand.op_sel ? 1u : 0u}); + if (operand.absolute) { + const auto abs_value = state->builder.AllocateId(); + state->builder.AddFunction( + {OpExtInst, state->float_type, abs_value, state->glsl_std450, GlslFAbs, f32}); + f32 = abs_value; + } + if (value_operand.negate) { + const auto neg_value = state->builder.AllocateId(); + state->builder.AddFunction({OpFNegate, state->float_type, neg_value, f32}); + f32 = neg_value; + } + return f32; + } + + value_operand.op_sel = false; + value_operand.op_sel_hi = false; + return EmitFloatLoad(state, value_operand); +} + +IR::Operand OffsetRegisterOperand(const IR::Operand& operand, uint32_t offset) { + if (offset == 0 || operand.kind != IR::OperandKind::Register) { + return operand; + } + auto ret = operand; + ret.reg.index += offset; + ret.sdwa_sel = 6; + ret.omod = 0; + ret.sdwa_sext = false; + ret.op_sel = false; + ret.op_sel_hi = false; + ret.negate = false; + ret.negate_hi = false; + ret.absolute = false; + ret.clamp = false; + ret.dpp_ctrl = 0; + ret.dpp_row_mask = 0xf; + ret.dpp_bank_mask = 0xf; + ret.dpp_fetch_inactive = false; + ret.dpp_bound_ctrl = false; + ret.dpp = false; + return ret; +} + +uint32_t EmitLaneMaskOperandActiveBool(EmitterState* state, const IR::Operand& operand) { + const auto low = EmitValueLoad(state, operand); + const auto active = state->builder.AllocateId(); + if (state->per_invocation_masks || state->exact_subgroup_operations) { + uint32_t high = ConstantU32(state, 0); + if (operand.kind == IR::OperandKind::Register && + (operand.reg.file == IR::RegisterFile::Scalar || + operand.reg.file == IR::RegisterFile::Vcc || + operand.reg.file == IR::RegisterFile::Exec)) { + high = IsMaskRegisterFile(operand.reg.file) + ? WaveMaskHighLoad(state, operand.reg.file) + : EmitValueLoad(state, OffsetRegisterOperand(operand, 1)); + } + if (state->per_invocation_masks) { + const auto combined = state->builder.AllocateId(); + state->builder.AddFunction({OpBitwiseOr, state->uint_type, combined, low, high}); + state->builder.AddFunction( + {OpINotEqual, state->bool_type, active, combined, ConstantU32(state, 0)}); + return active; + } + const auto low_hit = state->builder.AllocateId(); + const auto high_hit = state->builder.AllocateId(); + const auto hit = state->builder.AllocateId(); + state->builder.AddFunction( + {OpBitwiseAnd, state->uint_type, low_hit, low, EmitLaneMaskPartU32(state, 0)}); + state->builder.AddFunction( + {OpBitwiseAnd, state->uint_type, high_hit, high, EmitLaneMaskPartU32(state, 1)}); + state->builder.AddFunction({OpBitwiseOr, state->uint_type, hit, low_hit, high_hit}); + state->builder.AddFunction( + {OpINotEqual, state->bool_type, active, hit, ConstantU32(state, 0)}); + return active; + } + state->builder.AddFunction({OpINotEqual, state->bool_type, active, low, ConstantU32(state, 0)}); + return active; +} + +uint32_t EmitLaneIndexActiveBool(EmitterState* state, uint32_t lane) { + if (state->per_invocation_masks) { + const auto ballot = state->builder.AllocateId(); + state->builder.AddFunction({OpGroupNonUniformBallot, state->vec4_uint_type, ballot, + ConstantU32(state, ScopeSubgroup), EmitExecActiveBool(state)}); + return EmitBallotLaneActiveBool(state, ballot, lane); + } + const auto lane_low = state->builder.AllocateId(); + const auto bit = state->builder.AllocateId(); + state->builder.AddFunction( + {OpBitwiseAnd, state->uint_type, lane_low, lane, ConstantU32(state, 31)}); + state->builder.AddFunction( + {OpShiftLeftLogical, state->uint_type, bit, ConstantU32(state, 1), lane_low}); + + uint32_t mask = EmitRegisterLoad(state, {IR::RegisterFile::Exec, 0}); + if (state->wave_size == 64u) { + const auto high = EmitRegisterLoad(state, {IR::RegisterFile::Exec, 1}); + const auto in_high = state->builder.AllocateId(); + const auto selected = state->builder.AllocateId(); + state->builder.AddFunction( + {OpUGreaterThanEqual, state->bool_type, in_high, lane, ConstantU32(state, 32)}); + state->builder.AddFunction({OpSelect, state->uint_type, selected, in_high, high, mask}); + mask = selected; + } + + const auto hit = state->builder.AllocateId(); + const auto ret = state->builder.AllocateId(); + state->builder.AddFunction({OpBitwiseAnd, state->uint_type, hit, mask, bit}); + state->builder.AddFunction({OpINotEqual, state->bool_type, ret, hit, ConstantU32(state, 0)}); + return ret; +} + +uint32_t EmitSubgroupLaneActiveBool(EmitterState* state, uint32_t lane) { + const auto active_ballot = state->builder.AllocateId(); + state->builder.AddFunction({OpGroupNonUniformBallot, state->vec4_uint_type, active_ballot, + ConstantU32(state, ScopeSubgroup), EmitTrueBool(state)}); + return EmitBallotLaneActiveBool(state, active_ballot, lane); +} +uint32_t EmitBallotLaneActiveBool(EmitterState* state, uint32_t active_ballot, uint32_t lane) { + const auto low = state->builder.AllocateId(); + state->builder.AddFunction({OpCompositeExtract, state->uint_type, low, active_ballot, 0}); + uint32_t mask = low; + if (state->wave_size == 64u) { + const auto high = state->builder.AllocateId(); + const auto in_high = state->builder.AllocateId(); + const auto selected = state->builder.AllocateId(); + state->builder.AddFunction({OpCompositeExtract, state->uint_type, high, active_ballot, 1}); + state->builder.AddFunction( + {OpUGreaterThanEqual, state->bool_type, in_high, lane, ConstantU32(state, 32)}); + state->builder.AddFunction({OpSelect, state->uint_type, selected, in_high, high, low}); + mask = selected; + } + + const auto lane_low = state->builder.AllocateId(); + const auto bit = state->builder.AllocateId(); + state->builder.AddFunction( + {OpBitwiseAnd, state->uint_type, lane_low, lane, ConstantU32(state, 31)}); + state->builder.AddFunction( + {OpShiftLeftLogical, state->uint_type, bit, ConstantU32(state, 1), lane_low}); + + const auto hit = state->builder.AllocateId(); + const auto active = state->builder.AllocateId(); + const auto in_range = state->builder.AllocateId(); + const auto ret = state->builder.AllocateId(); + state->builder.AddFunction({OpBitwiseAnd, state->uint_type, hit, mask, bit}); + state->builder.AddFunction({OpINotEqual, state->bool_type, active, hit, ConstantU32(state, 0)}); + state->builder.AddFunction( + {OpULessThan, state->bool_type, in_range, lane, ConstantU32(state, state->wave_size)}); + state->builder.AddFunction({OpLogicalAnd, state->bool_type, ret, active, in_range}); + return ret; +} + +uint32_t EmitSequentialValueLoad(EmitterState* state, const IR::Operand& operand, uint32_t offset) { + if (offset == 0 || operand.kind == IR::OperandKind::Register) { + return EmitValueLoad(state, OffsetRegisterOperand(operand, offset)); + } + if (operand.kind == IR::OperandKind::ImmediateU32 && offset == 1) { + return ConstantU32(state, operand.sext_64 ? 0xffffffffu : 0u); + } + return ConstantU32(state, 0); +} + +uint32_t EmitSequentialFloatLoad(EmitterState* state, const IR::Operand& operand, uint32_t offset) { + const auto value = EmitSequentialValueLoad(state, operand, offset); + const auto f32 = state->builder.AllocateId(); + state->builder.AddFunction({OpBitcast, state->float_type, f32, value}); + return f32; +} + +bool ImageAddressUsesA16(const IR::Instruction& inst) { + return (inst.memory.image_sample_flags & Decoder::ImageSampleFlagA16) != 0u; +} + +bool ImageAddressComponentUsesA16(const IR::Instruction& inst, uint32_t component) { + if (!ImageAddressUsesA16(inst)) { + return false; + } + + uint32_t cursor = 0; + if ((inst.memory.image_sample_flags & Decoder::ImageSampleFlagOffset) != 0u) { + if (component == cursor) { + return false; + } + cursor++; + } + if ((inst.memory.image_sample_flags & Decoder::ImageSampleFlagBias) != 0u) { + if (component == cursor) { + return true; + } + cursor++; + } + if ((inst.memory.image_sample_flags & Decoder::ImageSampleFlagCompare) != 0u && + component == cursor) { + return false; + } + return true; +} + +uint32_t ImageAddressHalfComponent(const IR::Instruction& inst, uint32_t component) { + if (!ImageAddressUsesA16(inst)) { + return component * 2u; + } + + uint32_t half_component = 0; + for (uint32_t i = 0; i < component; i++) { + const auto half_width = ImageAddressComponentUsesA16(inst, i) ? 1u : 2u; + if (half_width == 2u && (half_component & 1u) != 0u) { + half_component++; + } + half_component += half_width; + } + if (!ImageAddressComponentUsesA16(inst, component) && (half_component & 1u) != 0u) { + half_component++; + } + return half_component; +} + +uint32_t ImageAddressPackedComponent(const IR::Instruction& inst, uint32_t component) { + return ImageAddressHalfComponent(inst, component) / 2u; +} + +IR::Operand ImageAddressOperand(const IR::Instruction& inst, const IR::Operand& base, + uint32_t component) { + const auto packed_component = ImageAddressPackedComponent(inst, component); + if (packed_component > 0 && inst.memory.image_nsa_dwords != 0) { + const auto nsa_index = packed_component - 1u; + const auto nsa_count = std::min(inst.memory.image_nsa_dwords * 4u, + Decoder::MaxImageNsaAddressComponents); + if (nsa_index < nsa_count) { + return MakeRegisterOperand(IR::RegisterFile::Vector, + inst.memory.image_nsa_addr[nsa_index]); + } + } + return OffsetRegisterOperand(base, packed_component); +} + +uint32_t EmitImageAddressValueLoad(EmitterState* state, const IR::Instruction& inst, + const IR::Operand& base, uint32_t component) { + const auto operand = ImageAddressOperand(inst, base, component); + const auto packed_component = ImageAddressPackedComponent(inst, component); + if (packed_component == 0 || operand.kind == IR::OperandKind::Register) { + auto value = EmitValueLoad(state, operand); + if (ImageAddressComponentUsesA16(inst, component)) { + if ((ImageAddressHalfComponent(inst, component) & 1u) != 0u) { + value = EmitShiftRightConstant(state, value, 16u); + } + value = EmitAndConstant(state, value, 0xffffu); + } + return value; + } + return ConstantU32(state, 0); +} + +uint32_t EmitImageAddressFloatLoad(EmitterState* state, const IR::Instruction& inst, + const IR::Operand& base, uint32_t component) { + const auto value = EmitImageAddressValueLoad(state, inst, base, component); + if (ImageAddressComponentUsesA16(inst, component)) { + return EmitF16BitsToF32(state, value); + } + const auto f32 = state->builder.AllocateId(); + state->builder.AddFunction({OpBitcast, state->float_type, f32, value}); + return f32; +} + +uint32_t EmitZeroF32(EmitterState* state) { + const auto f32 = state->builder.AllocateId(); + state->builder.AddFunction({OpBitcast, state->float_type, f32, ConstantU32(state, 0)}); + return f32; +} + +} // namespace Libs::Graphics::ShaderRecompiler::Spirv::Emitter diff --git a/src/graphics/shader/shader.cpp b/src/graphics/shader/shader.cpp new file mode 100644 index 000000000..ccfed3e0f --- /dev/null +++ b/src/graphics/shader/shader.cpp @@ -0,0 +1,1721 @@ +#include "graphics/shader/shader.h" + +#include "common/assert.h" +#include "common/common.h" +#include "common/emulatorConfig.h" +#include "common/file.h" +#include "common/hash.h" +#include "common/logging/log.h" +#include "common/magicEnum.h" +#include "common/profiler.h" +#include "common/stringUtils.h" +#include "graphics/guest_gpu/graphicsRun.h" +#include "graphics/guest_gpu/hardwareContext.h" +#include "graphics/guest_gpu/gpu_defs.h" +#include "graphics/host_gpu/hostMemory.h" +#include "graphics/shader/recompiler/ShaderDecoder.h" +#include "graphics/shader/recompiler/ShaderRecompiler.h" +#include "graphics/shader/shaderVertexMetadata.h" +#include "libs/errno.h" +#include "spirv-tools/libspirv.h" +#include "spirv-tools/libspirv.hpp" +#include "spirv-tools/optimizer.hpp" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS +#ifndef NOMINMAX +#define NOMINMAX +#endif +#include +#undef min +#undef max +#endif + +namespace Libs::Graphics { + +struct ShaderBinaryInfo { + uint8_t signature[7]; + uint8_t version; + uint32_t pssl_or_cg : 1; + uint32_t cached : 1; + uint32_t type : 4; + uint32_t source_type : 2; + uint32_t length : 24; + uint8_t chunk_usage_base_offset_dw; + uint8_t num_input_usage_slots; + uint8_t is_srt : 1; + uint8_t is_srt_used_info_valid : 1; + uint8_t is_extended_usage_info : 1; + uint8_t reserved2 : 5; + uint8_t reserved3; + uint32_t hash0; + uint32_t hash1; + uint32_t crc32; +}; + +static std::unique_ptr> g_shader_map; +static std::mutex g_shader_map_mutex; + +struct ShaderStageProgramKey { + ShaderType stage = ShaderType::Unknown; + ShaderLaneMaskMode lane_mask_mode = ShaderLaneMaskMode::NativeWave; + uint64_t shader_hash = 0; + ShaderId program_id; + Config::ShaderOptimizationType optimization_type = Config::ShaderOptimizationType::None; + bool validation = false; + + bool operator==(const ShaderStageProgramKey& other) const { + return stage == other.stage && lane_mask_mode == other.lane_mask_mode && + shader_hash == other.shader_hash && program_id == other.program_id && + optimization_type == other.optimization_type && validation == other.validation; + } +}; + +struct ShaderStageProgramKeyHash { + size_t operator()(const ShaderStageProgramKey& key) const { + uint32_t hash = 0; + auto mix = [&hash](uint32_t value) { + hash ^= Common::hash32(value) + 0x9e3779b9u + (hash << 6u) + (hash >> 2u); + }; + + mix(static_cast(key.stage)); + mix(static_cast(key.lane_mask_mode)); + mix(Common::hash64(key.shader_hash)); + mix(key.program_id.hash0); + mix(key.program_id.crc32); + mix(static_cast(key.program_id.ids.size())); + for (auto value: key.program_id.ids) { + mix(value); + } + mix(static_cast(key.optimization_type)); + mix(key.validation ? 1u : 0u); + return hash; + } +}; + +struct ShaderProgramPermutation { + std::vector spirv; + std::shared_ptr program; +}; + +static std::unordered_map>, + ShaderStageProgramKeyHash> + g_shader_program_cache; +static std::mutex g_shader_program_cache_mutex; + +static constexpr uint32_t ShaderMaxPermutationsPerProgram = 64; + +static std::span MakeShaderSpirvView(const std::vector& spirv) { + return {spirv.data(), spirv.size()}; +} + +void ShaderInit() { + EXIT_IF(g_shader_map != nullptr); + + g_shader_map = std::make_unique>(); +} + +void ShaderMapUserData(uint64_t addr, const ShaderMappedData& data) { + EXIT_IF(g_shader_map == nullptr); + + std::scoped_lock lock(g_shader_map_mutex); + + (*g_shader_map)[addr] = data; +} + +static bool ShaderGetMappedData(uint64_t addr, ShaderMappedData* data) { + EXIT_IF(g_shader_map == nullptr); + EXIT_IF(data == nullptr); + + std::scoped_lock lock(g_shader_map_mutex); + + if (auto iter = g_shader_map->find(addr); iter != g_shader_map->end()) { + *data = iter->second; + return true; + } + + return false; +} + +static bool SpirvDisassemble(const uint32_t* src_binary, size_t src_binary_size, + std::string* dst_disassembly) { + if (dst_disassembly != nullptr) { + spvtools::SpirvTools core(SPV_ENV_VULKAN_1_2); + + std::string disassembly; + if (!core.Disassemble(src_binary, src_binary_size, &disassembly, + static_cast(SPV_BINARY_TO_TEXT_OPTION_NO_HEADER) | + static_cast(SPV_BINARY_TO_TEXT_OPTION_FRIENDLY_NAMES) | + static_cast(SPV_BINARY_TO_TEXT_OPTION_COMMENT) | + static_cast(SPV_BINARY_TO_TEXT_OPTION_INDENT) | + static_cast(SPV_BINARY_TO_TEXT_OPTION_COLOR))) { + *dst_disassembly = disassembly.c_str(); + + LOGF("Disassemble failed\n"); + return false; + } + + *dst_disassembly = disassembly.c_str(); + } + return true; +} + +static bool SpirvValidateBinary(const char* label, uint64_t shader_hash, + const std::vector& spirv) { + if (!Config::ShaderValidationEnabled()) { + return true; + } + + spvtools::SpirvTools core(SPV_ENV_VULKAN_1_3); + std::string messages; + core.SetMessageConsumer([&messages](spv_message_level_t /*level*/, const char* /*source*/, + const spv_position_t& position, const char* message) { + messages += fmt::format("{}: {} ({}) {}\n", static_cast(position.line), + static_cast(position.column), static_cast(position.index), + message); + }); + + if (core.Validate(spirv)) { + return true; + } + + std::string disassembly; + SpirvDisassemble(spirv.data(), spirv.size(), &disassembly); + LOGF_COLOR(Log::Color::BrightRed, "%s SPIR-V validation failed hash=0x%016" PRIx64 ":\n%s", + label, shader_hash, messages.c_str()); + LOGF("%s\n", disassembly.c_str()); + return false; +} + +static void ExitShaderRecompilerFailure(const char* label, uint64_t shader_hash, + const char* reason) { + EXIT("%s failed hash=0x%016" PRIx64 ": %s\n", label, shader_hash, + reason != nullptr ? reason : ""); +} + +bool ShaderBindingResearchGuardEnabled() { + return Config::GraphicsDebugDumpEnabled() || + Config::GetShaderLogDirection() != Config::ShaderLogDirection::Silent; +} + +static const ShaderBinaryInfo* GetBinaryInfo(const uint32_t* code) { + EXIT_IF(code == nullptr); + + if (code[0] == 0xBEEB03FF) { + return reinterpret_cast(code + + static_cast(code[1] + 1) * 2); + } + + return nullptr; +} + +static std::span ShaderGetMappedCode(uint64_t shader_addr, const char* label, + uint64_t shader_hash) { + ShaderMappedData data; + if (!ShaderGetMappedData(shader_addr, &data)) { + EXIT("%s hash=0x%016" PRIx64 " shader=0x%016" PRIx64 " is missing from ShaderMap\n", label, + shader_hash, shader_addr); + } + if (data.code_size_bytes == 0 || data.code_size_bytes % sizeof(uint32_t) != 0) { + EXIT("%s hash=0x%016" PRIx64 " shader=0x%016" PRIx64 + " has invalid AGC shader_size=0x%08" PRIx32 "\n", + label, shader_hash, shader_addr, data.code_size_bytes); + } + const auto code_words = data.code_size_bytes / sizeof(uint32_t); + LOGF("%s hash=0x%016" PRIx64 " shader=0x%016" PRIx64 " using AGC shader_size=0x%08" PRIx32 + " (%" PRIu32 " dwords)\n", + label, shader_hash, shader_addr, data.code_size_bytes, code_words); + return {reinterpret_cast(shader_addr), code_words}; +} + +#if 0 +// Kept as disabled debugging guards for investigating unusual stage register state. +static void vs_check(const HW::VertexShaderInfo& vs, const HW::ShaderRegisters& sh) { + const auto is_zero_or_wave64_subgroup = [](uint32_t value) { + return value == 0 || value <= 0x40; + }; + const auto is_known_gs_out_prim_type = [](uint32_t value) { + switch (static_cast(value)) { + case Prospero::GsOutputPrimitiveType::kPoints: + case Prospero::GsOutputPrimitiveType::kLines: + case Prospero::GsOutputPrimitiveType::kTriangles: + case Prospero::GsOutputPrimitiveType::k2dRectangle: + case Prospero::GsOutputPrimitiveType::kRectList: return true; + } + + return false; + }; + const bool ps5_ngg_passthrough_triangle_path = + vs.es_regs.data_addr != 0 && vs.gs_regs.data_addr == vs.es_regs.data_addr && + vs.gs_regs.chksum != 0 && + sh.m_geNggSubgrpCntl == 0x00000001 && sh.m_vgtGsMaxVertOut == 0x00000003 && + sh.m_vgtGsOutPrimType == 0x00000002 && sh.m_geMaxOutputPerSubgroup <= 0x000000c0; + + if (vs.es_regs.data_addr != 0 || vs.gs_regs.data_addr != 0) { + EXIT_NOT_IMPLEMENTED(vs.gs_regs.rsrc1.priority != 0); + EXIT_NOT_IMPLEMENTED(vs.gs_regs.rsrc1.float_mode != 192); + EXIT_NOT_IMPLEMENTED(vs.gs_regs.rsrc1.dx10_clamp != true); + EXIT_NOT_IMPLEMENTED(vs.gs_regs.rsrc1.debug_mode != false); + EXIT_NOT_IMPLEMENTED(vs.gs_regs.rsrc1.ieee_mode != false); + EXIT_NOT_IMPLEMENTED(vs.gs_regs.rsrc1.cu_group_enable != false); + EXIT_NOT_IMPLEMENTED(vs.gs_regs.rsrc1.require_forward_progress != false); + EXIT_NOT_IMPLEMENTED(vs.gs_regs.rsrc1.lds_configuration != false); + EXIT_NOT_IMPLEMENTED(vs.gs_regs.rsrc1.gs_vgpr_component_count != 3); + EXIT_NOT_IMPLEMENTED(vs.gs_regs.rsrc1.fp16_overflow != false); + EXIT_NOT_IMPLEMENTED(vs.gs_regs.rsrc2.scratch_en != false); + EXIT_NOT_IMPLEMENTED(vs.gs_regs.rsrc2.offchip_lds != false); + EXIT_NOT_IMPLEMENTED(vs.gs_regs.rsrc2.es_vgpr_component_count != 3); + EXIT_NOT_IMPLEMENTED(vs.gs_regs.rsrc2.lds_size != 0); + EXIT_NOT_IMPLEMENTED(vs.gs_regs.rsrc2.shared_vgprs != 0); + } + + for (uint32_t value = sh.m_spiShaderPosFormat; value != 0; value >>= 4u) { + EXIT_NOT_IMPLEMENTED((value & 0xfu) != 0 && (value & 0xfu) != 0x4u); + } + if (sh.m_paClVsOutCntl != 0x00000000) { + static bool logged = false; + if (!logged) { + LOGF("\t temporary: accepting PA_CL_VS_OUT_CNTL = 0x%08" PRIx32 "\n", + sh.m_paClVsOutCntl); + logged = true; + } + } + + EXIT_NOT_IMPLEMENTED(sh.m_spiShaderIdxFormat != 0x00000000 && + sh.m_spiShaderIdxFormat != 0x00000001); + EXIT_NOT_IMPLEMENTED(sh.m_geNggSubgrpCntl != 0x00000000 && sh.m_geNggSubgrpCntl != 0x00000001); + EXIT_NOT_IMPLEMENTED(sh.m_vgtGsInstanceCnt != 0x00000000); + EXIT_NOT_IMPLEMENTED(!is_zero_or_wave64_subgroup(sh.GetEsVertsPerSubgrp())); + EXIT_NOT_IMPLEMENTED(!is_zero_or_wave64_subgroup(sh.GetGsPrimsPerSubgrp())); + EXIT_NOT_IMPLEMENTED(!is_zero_or_wave64_subgroup(sh.GetGsInstPrimsInSubgrp())); + EXIT_NOT_IMPLEMENTED(!is_zero_or_wave64_subgroup(sh.m_geMaxOutputPerSubgroup) && + !ps5_ngg_passthrough_triangle_path); + EXIT_NOT_IMPLEMENTED(sh.m_vgtEsgsRingItemsize != 0x00000000 && + sh.m_vgtEsgsRingItemsize != 0x00000004); + EXIT_NOT_IMPLEMENTED(sh.m_vgtGsMaxVertOut != 0x00000000 && !ps5_ngg_passthrough_triangle_path); + EXIT_NOT_IMPLEMENTED(!is_known_gs_out_prim_type(sh.m_vgtGsOutPrimType)); +} + +static void ps_check(const HW::PsStageRegisters& ps, const HW::ShaderRegisters& sh) { + if (sh.target_output_mode[0] != 0 && sh.target_output_mode[0] != 2 && + sh.target_output_mode[0] != 4 && sh.target_output_mode[0] != 5 && + sh.target_output_mode[0] != 7 && sh.target_output_mode[0] != 9) { + EXIT("Not implemented (sh.target_output_mode[0] != 0 && sh.target_output_mode[0] != 2 && " + "sh.target_output_mode[0] != 4 && sh.target_output_mode[0] != 5 && " + "sh.target_output_mode[0] != 7 && sh.target_output_mode[0] != 9)\n"); + } + EXIT_NOT_IMPLEMENTED(sh.db_shader_control.conservative_z_export_value != 0x00000000); + EXIT_NOT_IMPLEMENTED(sh.db_shader_control.shader_z_behavior != 0x00000001 && + sh.db_shader_control.shader_z_behavior != 0x00000000); + // EXIT_NOT_IMPLEMENTED(ps.shader_kill_enable != false); + // EXIT_NOT_IMPLEMENTED(ps.shader_execute_on_noop != false); + // EXIT_NOT_IMPLEMENTED(ps.m_spiShaderPgmRsrc1Ps != 0x002c0000); + // EXIT_NOT_IMPLEMENTED(ps.m_spiShaderPgmRsrc2Ps != 0x00000000); + // EXIT_NOT_IMPLEMENTED(ps.vgprs != 0x00 && ps.vgprs != 0x01); + // EXIT_NOT_IMPLEMENTED(ps.sgprs != 0x00 && ps.sgprs != 0x01); + EXIT_NOT_IMPLEMENTED(ps.rsrc1.priority != 0); + EXIT_NOT_IMPLEMENTED(ps.rsrc1.float_mode != 192); + EXIT_NOT_IMPLEMENTED(ps.rsrc1.dx10_clamp != true); + EXIT_NOT_IMPLEMENTED(ps.rsrc1.debug_mode != false); + EXIT_NOT_IMPLEMENTED(ps.rsrc1.ieee_mode != false); + EXIT_NOT_IMPLEMENTED(ps.rsrc1.cu_group_disable != false); + EXIT_NOT_IMPLEMENTED(ps.rsrc1.require_forward_progress != false); + EXIT_NOT_IMPLEMENTED(ps.rsrc1.fp16_overflow != false); + EXIT_NOT_IMPLEMENTED(ps.rsrc2.scratch_en != false); + // EXIT_NOT_IMPLEMENTED(ps.user_sgpr != 0 && ps.user_sgpr != 4 && ps.user_sgpr != 12); + EXIT_NOT_IMPLEMENTED(ps.rsrc2.wave_cnt_en != false); + if (ps.rsrc2.extra_lds_size != 0) { + static std::atomic_uint log_count {0}; + if (log_count.fetch_add(1, std::memory_order_relaxed) < 32) { + LOGF("\t PS extra LDS reservation = 0x%02" PRIx8 ", continuing\n", + ps.rsrc2.extra_lds_size); + } + } + EXIT_NOT_IMPLEMENTED(ps.rsrc2.raster_ordered_shading != false); + EXIT_NOT_IMPLEMENTED(ps.rsrc2.shared_vgprs != 0); + + if (sh.shader_z_format != 0x00000000 && sh.shader_z_format != 0x00000001 && + !sh.db_shader_control.shader_z_export_enable) { + static std::atomic_uint log_count {0}; + if (log_count.fetch_add(1, std::memory_order_relaxed) < 32) { + LOGF("\t shader_z_format = 0x%08" PRIx32 + " with z export disabled, ignoring depth export format\n", + sh.shader_z_format); + } + } + EXIT_NOT_IMPLEMENTED(sh.db_shader_control.shader_z_export_enable && + sh.shader_z_format != 0x00000000 && sh.shader_z_format != 0x00000001); + constexpr uint32_t ps_input_linear_center = 0x00000020u; + constexpr uint32_t ps_input_pos_w = 0x00000800u; + constexpr uint32_t ps_input_front_face = 0x00001000u; + constexpr uint32_t supported_ps_input_bits = + 0x00000702u | ps_input_linear_center | ps_input_pos_w | ps_input_front_face; + EXIT_NOT_IMPLEMENTED((sh.ps_input_ena & ~supported_ps_input_bits) != 0); + EXIT_NOT_IMPLEMENTED((sh.ps_input_addr & ~supported_ps_input_bits) != 0); + EXIT_NOT_IMPLEMENTED(sh.ps_input_ena != sh.ps_input_addr); + // EXIT_NOT_IMPLEMENTED(ps.m_spiPsInControl != 0x00000000); + constexpr uint32_t baryc_persp_mask = + 0x00000003u | 0x00000030u | 0x00000300u | 0x00003000u; + constexpr uint32_t baryc_linear_mask = 0x00030000u | 0x00300000u | 0x03000000u; + constexpr uint32_t baryc_known_mask = baryc_persp_mask | baryc_linear_mask; + EXIT_NOT_IMPLEMENTED((sh.baryc_cntl & ~baryc_known_mask) != 0); + EXIT_NOT_IMPLEMENTED((sh.baryc_cntl & baryc_persp_mask) != 0); + if ((sh.ps_input_ena & ps_input_linear_center) == 0 && (sh.baryc_cntl & baryc_linear_mask) != 0) { + static std::atomic_uint log_count {0}; + if (log_count.fetch_add(1, std::memory_order_relaxed) < 32) { + LOGF("\t ignoring inactive linear SPI_BARYC_CNTL bits: 0x%08" PRIx32 "\n", + sh.baryc_cntl & baryc_linear_mask); + } + } else { + EXIT_NOT_IMPLEMENTED((sh.baryc_cntl & baryc_linear_mask) != 0x00000000 && + (sh.baryc_cntl & baryc_linear_mask) != 0x01000000); + } + if ((sh.m_cbShaderMask & 0x0000000f) != 0x0000000f) { + static bool logged = false; + if (!logged) { + LOGF("\t temporary: accepting partial CB_SHADER_MASK = 0x%08" PRIx32 "\n", + sh.m_cbShaderMask); + logged = true; + } + } + if ((sh.m_cbShaderMask & ~0x0000000fu) != 0) { + static bool logged = false; + if (!logged) { + LOGF("\t temporary: ignoring extra CB_SHADER_MASK MRT bits: 0x%08" PRIx32 "\n", + sh.m_cbShaderMask); + logged = true; + } + } + + if (sh.db_shader_control.other_bits != 0x00000000) { + static std::atomic_uint log_count {0}; + if (log_count.fetch_add(1, std::memory_order_relaxed) < 32) { + LOGF("\t temporary: ignoring unsupported DB_SHADER_CONTROL bits 0x%08" PRIx32 "\n", + sh.db_shader_control.other_bits); + } + } + EXIT_NOT_IMPLEMENTED(sh.m_paScShaderControl != 0x00000000); +} + +static void cs_check(const HW::CsStageRegisters& cs, const HW::ShaderRegisters& /*sh*/) { + // EXIT_NOT_IMPLEMENTED(cs.num_thread_x != 0x00000040); + // EXIT_NOT_IMPLEMENTED(cs.num_thread_y != 0x00000001); + // EXIT_NOT_IMPLEMENTED(cs.num_thread_z != 0x00000001); + // EXIT_NOT_IMPLEMENTED(cs.vgprs != 0x00 && cs.vgprs != 0x01); + // EXIT_NOT_IMPLEMENTED(cs.sgprs != 0x01 && cs.sgprs != 0x02); + EXIT_NOT_IMPLEMENTED(cs.bulky != 0x00); + EXIT_NOT_IMPLEMENTED(cs.scratch_en != 0x00); + // EXIT_NOT_IMPLEMENTED(cs.user_sgpr != 0x0c); + if (cs.tgid_x_en == 0x00) { + static bool logged = false; + if (!logged) { + LOGF("\t temporary: compute shader has TGID X disabled\n"); + logged = true; + } + } else { + EXIT_NOT_IMPLEMENTED(cs.tgid_x_en != 0x01); + } + // EXIT_NOT_IMPLEMENTED(cs.tgid_y_en != 0x00); + // EXIT_NOT_IMPLEMENTED(cs.tgid_z_en != 0x00); + EXIT_NOT_IMPLEMENTED(cs.tg_size_en != 0x00); + EXIT_NOT_IMPLEMENTED(cs.tidig_comp_cnt > 2); + + // EXIT_NOT_IMPLEMENTED(cs.m_computePgmRsrc1 != 0x002c0040); + // EXIT_NOT_IMPLEMENTED(cs.m_computePgmRsrc2 != 0x00000098); + // EXIT_NOT_IMPLEMENTED(cs.m_computeNumThreadX != 0x00000040); + // EXIT_NOT_IMPLEMENTED(cs.m_computeNumThreadY != 0x00000001); + // EXIT_NOT_IMPLEMENTED(cs.m_computeNumThreadZ != 0x00000001); +} +#endif + +static void GetNextGenFallbackShaderId(uint64_t addr, uint32_t* hash0, uint32_t* crc32) { + EXIT_IF(hash0 == nullptr); + EXIT_IF(crc32 == nullptr); + + auto x = addr; + x ^= x >> 33u; + x *= 0xff51afd7ed558ccdull; + x ^= x >> 33u; + x *= 0xc4ceb9fe1a85ec53ull; + x ^= x >> 33u; + + *hash0 = static_cast((x >> 32u) & 0xffffffffu); + *crc32 = static_cast(x & 0xffffffffu); +} + +static void ShaderDetectBuffers(ShaderVertexInputInfo* info) { + KYTY_PROFILER_FUNCTION(); + + EXIT_IF(info == nullptr); + + info->buffers_num = 0; + + for (int ri = 0; ri < info->resources_num; ri++) { + const auto& r = info->resources[ri]; + + bool merged = false; + for (int bi = 0; bi < info->buffers_num; bi++) { + auto& b = info->buffers[bi]; + + uint64_t stride = b.stride; + + if (stride == r.Stride() && + b.fetch_index == static_cast(info->resources_dst[ri].fetch_index)) { + uint64_t rbase = r.Base48(); + uint64_t base = std::min(rbase, b.addr); + uint64_t offset1 = rbase - base; + uint64_t offset2 = b.addr - base; + + if (offset1 < stride && offset2 < stride) { + EXIT_NOT_IMPLEMENTED(b.num_records != r.NumRecords()); + b.addr = base; + EXIT_NOT_IMPLEMENTED(b.attr_num >= ShaderVertexInputBuffer::ATTR_MAX); + b.attr_indices[b.attr_num++] = ri; + merged = true; + break; + } + } + } + + if (!merged) { + EXIT_NOT_IMPLEMENTED(info->buffers_num >= ShaderVertexInputInfo::RES_MAX); + int bi = info->buffers_num++; + info->buffers[bi].addr = r.Base48(); + info->buffers[bi].stride = r.Stride(); + info->buffers[bi].num_records = r.NumRecords(); + info->buffers[bi].fetch_index = info->resources_dst[ri].fetch_index; + info->buffers[bi].attr_num = 1; + info->buffers[bi].attr_indices[0] = ri; + } + } + + for (int bi = 0; bi < info->buffers_num; bi++) { + auto& b = info->buffers[bi]; + for (int ri = 0; ri < b.attr_num; ri++) { + b.attr_offsets[ri] = info->resources[b.attr_indices[ri]].Base48() - b.addr; + } + } +} + +static uint32_t VertexAttribFormatToBufferFormat(uint32_t format) { + struct FormatMap { + Prospero::VertexAttribFormat vertex; + Prospero::BufferFormat buffer; + }; + + static constexpr FormatMap format_map[] = { + {Prospero::VertexAttribFormat::kInvalid, Prospero::BufferFormat::kInvalid}, + {Prospero::VertexAttribFormat::k8UNorm, Prospero::BufferFormat::k8UNorm}, + {Prospero::VertexAttribFormat::k8SNorm, Prospero::BufferFormat::k8SNorm}, + {Prospero::VertexAttribFormat::k8UScaled, Prospero::BufferFormat::k8UScaled}, + {Prospero::VertexAttribFormat::k8SScaled, Prospero::BufferFormat::k8SScaled}, + {Prospero::VertexAttribFormat::k8UInt, Prospero::BufferFormat::k8UInt}, + {Prospero::VertexAttribFormat::k8SInt, Prospero::BufferFormat::k8SInt}, + {Prospero::VertexAttribFormat::k16UNorm, Prospero::BufferFormat::k16UNorm}, + {Prospero::VertexAttribFormat::k16SNorm, Prospero::BufferFormat::k16SNorm}, + {Prospero::VertexAttribFormat::k16UScaled, Prospero::BufferFormat::k16UScaled}, + {Prospero::VertexAttribFormat::k16SScaled, Prospero::BufferFormat::k16SScaled}, + {Prospero::VertexAttribFormat::k16UInt, Prospero::BufferFormat::k16UInt}, + {Prospero::VertexAttribFormat::k16SInt, Prospero::BufferFormat::k16SInt}, + {Prospero::VertexAttribFormat::k16Float, Prospero::BufferFormat::k16Float}, + {Prospero::VertexAttribFormat::k8_8UNorm, Prospero::BufferFormat::k8_8UNorm}, + {Prospero::VertexAttribFormat::k8_8SNorm, Prospero::BufferFormat::k8_8SNorm}, + {Prospero::VertexAttribFormat::k8_8UScaled, Prospero::BufferFormat::k8_8UScaled}, + {Prospero::VertexAttribFormat::k8_8SScaled, Prospero::BufferFormat::k8_8SScaled}, + {Prospero::VertexAttribFormat::k8_8UInt, Prospero::BufferFormat::k8_8UInt}, + {Prospero::VertexAttribFormat::k8_8SInt, Prospero::BufferFormat::k8_8SInt}, + {Prospero::VertexAttribFormat::k32UInt, Prospero::BufferFormat::k32UInt}, + {Prospero::VertexAttribFormat::k32SInt, Prospero::BufferFormat::k32SInt}, + {Prospero::VertexAttribFormat::k32Float, Prospero::BufferFormat::k32Float}, + {Prospero::VertexAttribFormat::k16_16UNorm, Prospero::BufferFormat::k16_16UNorm}, + {Prospero::VertexAttribFormat::k16_16SNorm, Prospero::BufferFormat::k16_16SNorm}, + {Prospero::VertexAttribFormat::k16_16UScaled, Prospero::BufferFormat::k16_16UScaled}, + {Prospero::VertexAttribFormat::k16_16SScaled, Prospero::BufferFormat::k16_16SScaled}, + {Prospero::VertexAttribFormat::k16_16UInt, Prospero::BufferFormat::k16_16UInt}, + {Prospero::VertexAttribFormat::k16_16SInt, Prospero::BufferFormat::k16_16SInt}, + {Prospero::VertexAttribFormat::k16_16Float, Prospero::BufferFormat::k16_16Float}, + {Prospero::VertexAttribFormat::k11_11_10UNorm, Prospero::BufferFormat::k11_11_10UNorm}, + {Prospero::VertexAttribFormat::k11_11_10SNorm, Prospero::BufferFormat::k11_11_10SNorm}, + {Prospero::VertexAttribFormat::k11_11_10UScaled, Prospero::BufferFormat::k11_11_10UScaled}, + {Prospero::VertexAttribFormat::k11_11_10SScaled, Prospero::BufferFormat::k11_11_10SScaled}, + {Prospero::VertexAttribFormat::k11_11_10UInt, Prospero::BufferFormat::k11_11_10UInt}, + {Prospero::VertexAttribFormat::k11_11_10SInt, Prospero::BufferFormat::k11_11_10SInt}, + {Prospero::VertexAttribFormat::k11_11_10Float, Prospero::BufferFormat::k11_11_10Float}, + {Prospero::VertexAttribFormat::k10_11_11UNorm, Prospero::BufferFormat::k10_11_11UNorm}, + {Prospero::VertexAttribFormat::k10_11_11SNorm, Prospero::BufferFormat::k10_11_11SNorm}, + {Prospero::VertexAttribFormat::k10_11_11UScaled, Prospero::BufferFormat::k10_11_11UScaled}, + {Prospero::VertexAttribFormat::k10_11_11SScaled, Prospero::BufferFormat::k10_11_11SScaled}, + {Prospero::VertexAttribFormat::k10_11_11UInt, Prospero::BufferFormat::k10_11_11UInt}, + {Prospero::VertexAttribFormat::k10_11_11SInt, Prospero::BufferFormat::k10_11_11SInt}, + {Prospero::VertexAttribFormat::k10_11_11Float, Prospero::BufferFormat::k10_11_11Float}, + {Prospero::VertexAttribFormat::k2_10_10_10UNorm, Prospero::BufferFormat::k2_10_10_10UNorm}, + {Prospero::VertexAttribFormat::k2_10_10_10SNorm, Prospero::BufferFormat::k2_10_10_10SNorm}, + {Prospero::VertexAttribFormat::k2_10_10_10UScaled, Prospero::BufferFormat::k2_10_10_10UScaled}, + {Prospero::VertexAttribFormat::k2_10_10_10SScaled, Prospero::BufferFormat::k2_10_10_10SScaled}, + {Prospero::VertexAttribFormat::k2_10_10_10UInt, Prospero::BufferFormat::k2_10_10_10UInt}, + {Prospero::VertexAttribFormat::k2_10_10_10SInt, Prospero::BufferFormat::k2_10_10_10SInt}, + {Prospero::VertexAttribFormat::k10_10_10_2UNorm, Prospero::BufferFormat::k10_10_10_2UNorm}, + {Prospero::VertexAttribFormat::k10_10_10_2SNorm, Prospero::BufferFormat::k10_10_10_2SNorm}, + {Prospero::VertexAttribFormat::k10_10_10_2UScaled, Prospero::BufferFormat::k10_10_10_2UScaled}, + {Prospero::VertexAttribFormat::k10_10_10_2SScaled, Prospero::BufferFormat::k10_10_10_2SScaled}, + {Prospero::VertexAttribFormat::k10_10_10_2UInt, Prospero::BufferFormat::k10_10_10_2UInt}, + {Prospero::VertexAttribFormat::k10_10_10_2SInt, Prospero::BufferFormat::k10_10_10_2SInt}, + {Prospero::VertexAttribFormat::k8_8_8_8UNorm, Prospero::BufferFormat::k8_8_8_8UNorm}, + {Prospero::VertexAttribFormat::k8_8_8_8SNorm, Prospero::BufferFormat::k8_8_8_8SNorm}, + {Prospero::VertexAttribFormat::k8_8_8_8UScaled, Prospero::BufferFormat::k8_8_8_8UScaled}, + {Prospero::VertexAttribFormat::k8_8_8_8SScaled, Prospero::BufferFormat::k8_8_8_8SScaled}, + {Prospero::VertexAttribFormat::k8_8_8_8UInt, Prospero::BufferFormat::k8_8_8_8UInt}, + {Prospero::VertexAttribFormat::k8_8_8_8SInt, Prospero::BufferFormat::k8_8_8_8SInt}, + {Prospero::VertexAttribFormat::k32_32UInt, Prospero::BufferFormat::k32_32UInt}, + {Prospero::VertexAttribFormat::k32_32SInt, Prospero::BufferFormat::k32_32SInt}, + {Prospero::VertexAttribFormat::k32_32Float, Prospero::BufferFormat::k32_32Float}, + {Prospero::VertexAttribFormat::k16_16_16_16UNorm, Prospero::BufferFormat::k16_16_16_16UNorm}, + {Prospero::VertexAttribFormat::k16_16_16_16SNorm, Prospero::BufferFormat::k16_16_16_16SNorm}, + {Prospero::VertexAttribFormat::k16_16_16_16UScaled, Prospero::BufferFormat::k16_16_16_16UScaled}, + {Prospero::VertexAttribFormat::k16_16_16_16SScaled, Prospero::BufferFormat::k16_16_16_16SScaled}, + {Prospero::VertexAttribFormat::k16_16_16_16UInt, Prospero::BufferFormat::k16_16_16_16UInt}, + {Prospero::VertexAttribFormat::k16_16_16_16SInt, Prospero::BufferFormat::k16_16_16_16SInt}, + {Prospero::VertexAttribFormat::k16_16_16_16Float, Prospero::BufferFormat::k16_16_16_16Float}, + {Prospero::VertexAttribFormat::k32_32_32UInt, Prospero::BufferFormat::k32_32_32UInt}, + {Prospero::VertexAttribFormat::k32_32_32SInt, Prospero::BufferFormat::k32_32_32SInt}, + {Prospero::VertexAttribFormat::k32_32_32Float, Prospero::BufferFormat::k32_32_32Float}, + {Prospero::VertexAttribFormat::k32_32_32_32UInt, Prospero::BufferFormat::k32_32_32_32UInt}, + {Prospero::VertexAttribFormat::k32_32_32_32SInt, Prospero::BufferFormat::k32_32_32_32SInt}, + {Prospero::VertexAttribFormat::k32_32_32_32Float, Prospero::BufferFormat::k32_32_32_32Float}, + }; + + for (const auto& entry: format_map) { + if (format == Prospero::GpuEnumValue(entry.vertex)) { + return Prospero::GpuEnumValue(entry.buffer); + } + } + + return format; +} + +static void ShaderApplyAttribSemantics(ShaderVertexInputInfo* info, + const ShaderSemantic* input_semantics, + uint32_t num_input_semantics, const uint32_t* attrib, + const uint32_t* buffer) { + KYTY_PROFILER_FUNCTION(); + + EXIT_IF(info == nullptr || attrib == nullptr || buffer == nullptr); + + for (uint32_t i = 0; i < num_input_semantics; i++) { + const auto& in = input_semantics[i]; + + EXIT_NOT_IMPLEMENTED(in.static_vb_index == 1 || in.static_attribute == 1); + + uint32_t reg = in.hardware_mapping; + uint32_t size = in.size_in_elements; + + LOGF("reg = %u, size = %u, va[%u] = 0x%08" PRIx32 "\n", reg, size, i, attrib[in.semantic]); + + size_t index = attrib[in.semantic] & 0x1fu; + uint32_t format = (attrib[in.semantic] >> 5u) & 0x1ffu; + uint32_t offset = (attrib[in.semantic] >> 14u) & 0xfffu; + uint32_t fetch_index = (attrib[in.semantic] >> 26u) & 0x1u; + + if (fetch_index != 0) { + static std::atomic log_count = 0; + auto log_id = log_count.fetch_add(1); + if (log_id < 64) { + LOGF("\t temporary: PS5 vertex attrib semantic %u uses fetch index %u, buffer " + "index %zu\n", + static_cast(in.semantic), fetch_index, index); + } + } + + EXIT_NOT_IMPLEMENTED(index >= ShaderVertexInputInfo::RES_MAX); + + const auto* sharp = &buffer[index * 4]; + + EXIT_NOT_IMPLEMENTED(info->resources_num >= ShaderVertexInputInfo::RES_MAX); + + auto& r = info->resources[info->resources_num]; + auto& rd = info->resources_dst[info->resources_num]; + rd.register_start = static_cast(reg); + rd.registers_num = static_cast(size); + rd.attr_id = static_cast(in.semantic); + rd.fetch_index = fetch_index; + r.fields[0] = sharp[0]; + r.fields[1] = sharp[1]; + r.fields[2] = sharp[2]; + r.fields[3] = sharp[3]; + if (format != 0) { + auto buffer_format = VertexAttribFormatToBufferFormat(format); + static std::atomic log_count = 0; + auto log_id = log_count.fetch_add(1); + if (log_id < 64) { + LOGF("\t temporary: PS5 vertex attrib semantic %u uses attrib format %u -> buffer " + "format %u, offset %u, buffer index %zu\n", + static_cast(in.semantic), format, buffer_format, offset, index); + } + r.fields[3] = (r.fields[3] & ~((0x7fu << 12u) | 0xfffu)) | + ((buffer_format & 0x7fu) << 12u) | DstSel(4, 5, 6, 7); + } + if (offset != 0) { + r.UpdateAddress48(r.Base48() + offset); + } + + info->resources_num++; + } +} + +static uint32_t ShaderCalcPsSystemInputBase(const HW::ShaderRegisters& regs) { + constexpr uint32_t ps_input_persp_sample = 0x00000001u; + constexpr uint32_t ps_input_persp_center = 0x00000002u; + constexpr uint32_t ps_input_persp_centroid = 0x00000004u; + constexpr uint32_t ps_input_persp_pull = 0x00000008u; + constexpr uint32_t ps_input_linear_sample = 0x00000010u; + constexpr uint32_t ps_input_linear_center = 0x00000020u; + constexpr uint32_t ps_input_linear_centroid = 0x00000040u; + constexpr uint32_t ps_input_line_stipple = 0x00000080u; + constexpr uint32_t ps_input_pos_xy = 0x00000300u; + constexpr uint32_t ps_input_pos_z = 0x00000400u; + constexpr uint32_t ps_input_pos_w = 0x00000800u; + constexpr uint32_t ps_input_front_face = 0x00001000u; + constexpr uint32_t ps_input_ancillary = 0x00002000u; + constexpr uint32_t ps_input_sample_coverage = 0x00004000u; + constexpr uint32_t ps_input_pos_fixed_pt = 0x00008000u; + constexpr uint32_t supported_ps_input_bits = + ps_input_persp_sample | ps_input_persp_center | ps_input_persp_centroid | + ps_input_persp_pull | ps_input_linear_sample | ps_input_linear_center | + ps_input_linear_centroid | ps_input_line_stipple | ps_input_pos_xy | ps_input_pos_z | + ps_input_pos_w | ps_input_front_face | ps_input_ancillary | ps_input_sample_coverage | + ps_input_pos_fixed_pt; + + EXIT_NOT_IMPLEMENTED((regs.ps_input_ena & ~supported_ps_input_bits) != 0); + EXIT_NOT_IMPLEMENTED((regs.ps_input_addr & ~supported_ps_input_bits) != 0); + EXIT_NOT_IMPLEMENTED(regs.ps_input_ena != regs.ps_input_addr); + + const uint32_t inputs = regs.ps_input_addr; + uint32_t reg = 0; + if ((inputs & ps_input_persp_sample) != 0) { + reg += 2; + } + if ((inputs & ps_input_persp_center) != 0) { + reg += 2; + } + if ((inputs & ps_input_persp_centroid) != 0) { + reg += 2; + } + if ((inputs & ps_input_persp_pull) != 0) { + reg += 3; + } + if ((inputs & ps_input_linear_sample) != 0) { + reg += 2; + } + if ((inputs & ps_input_linear_center) != 0) { + reg += 2; + } + if ((inputs & ps_input_linear_centroid) != 0) { + reg += 2; + } + if ((inputs & ps_input_line_stipple) != 0) { + reg += 1; + } + return reg; +} + +static bool ShaderGetStaticInputInfoVS(const HW::VertexShaderInfo* regs, + const HW::ShaderRegisters* sh, ShaderVertexInputInfo* info) { + KYTY_PROFILER_FUNCTION(); + + EXIT_IF(info == nullptr || regs == nullptr || sh == nullptr); + + *info = {}; + + info->export_count = static_cast(sh->GetExportCount()); + + EXIT_NOT_IMPLEMENTED(regs->es_regs.data_addr == 0 || regs->gs_regs.chksum == 0); + + uint64_t shader_addr = regs->es_regs.data_addr; + const HW::UserSgprInfo& user_sgpr = regs->gs_user_sgpr; + auto user_sgpr_num = regs->gs_regs.rsrc2.user_sgpr; + ShaderMappedData data; + if (!ShaderGetMappedData(shader_addr, &data)) { + LOGF("ShaderGetInputInfoVS(): shader=0x%016" PRIx64 " is missing from ShaderMap\n", + shader_addr); + return false; + } + + if (data.user_data == nullptr) { + LOGF("ShaderGetInputInfoVS(): no AGC user data for shader=0x%016" PRIx64 " es=0x%016" PRIx64 + " gs=0x%016" PRIx64 " chksum=0x%016" PRIx64 " user_sgpr_num=%u\n", + shader_addr, regs->es_regs.data_addr, regs->gs_regs.data_addr, regs->gs_regs.chksum, + static_cast(user_sgpr_num)); + } + ShaderVertexMetadata metadata; + std::string metadata_error; + if (!ShaderReadVertexMetadata(data, HW::UserSgprInfo::SGPRS_MAX, &metadata, &metadata_error)) { + LOGF("ShaderGetInputInfoVS(): invalid AGC metadata shader=0x%016" PRIx64 ": %s\n", + shader_addr, metadata_error.c_str()); + return false; + } + + if (metadata.vertex_buffer_reg >= 0) { + info->fetch_external = false; + info->fetch_embedded = true; + info->fetch_attrib_reg = metadata.vertex_attrib_reg; + info->fetch_buffer_reg = metadata.vertex_buffer_reg; + + const auto* attrib = reinterpret_cast( + static_cast(user_sgpr.value[metadata.vertex_attrib_reg]) | + (static_cast(user_sgpr.value[metadata.vertex_attrib_reg + 1]) << 32u)); + const auto* buffer = reinterpret_cast( + static_cast(user_sgpr.value[metadata.vertex_buffer_reg]) | + (static_cast(user_sgpr.value[metadata.vertex_buffer_reg + 1]) << 32u)); + + if (attrib == nullptr || buffer == nullptr) { + LOGF("ShaderGetInputInfoVS(): null vertex table pointer shader=0x%016" PRIx64 "\n", + shader_addr); + return false; + } + uint32_t max_semantic = 0; + for (uint32_t i = 0; i < metadata.input_semantics_count; i++) { + max_semantic = + std::max(max_semantic, static_cast(metadata.input_semantics[i].semantic)); + } + if (!HostMemoryRangeIsReadable(reinterpret_cast(attrib), + static_cast(max_semantic + 1) * + sizeof(uint32_t))) { + LOGF("ShaderGetInputInfoVS(): unreadable vertex attribute table shader=0x%016" PRIx64 + "\n", + shader_addr); + return false; + } + uint32_t max_buffer = 0; + for (uint32_t i = 0; i < metadata.input_semantics_count; i++) { + max_buffer = std::max(max_buffer, attrib[metadata.input_semantics[i].semantic] & 0x1fu); + } + if (!HostMemoryRangeIsReadable(reinterpret_cast(buffer), + static_cast(max_buffer + 1) * 4 * + sizeof(uint32_t))) { + LOGF("ShaderGetInputInfoVS(): unreadable vertex buffer table shader=0x%016" PRIx64 "\n", + shader_addr); + return false; + } + + ShaderApplyAttribSemantics(info, metadata.input_semantics.data(), + metadata.input_semantics_count, attrib, buffer); + ShaderDetectBuffers(info); + } + return true; +} + +static void ShaderGetStaticInputInfoPS(const HW::PixelShaderInfo* regs, + const HW::ShaderRegisters* sh, + const ShaderVertexInputInfo* vs_info, + ShaderPixelInputInfo* ps_info) { + KYTY_PROFILER_FUNCTION(); + + EXIT_IF(vs_info == nullptr); + EXIT_IF(ps_info == nullptr); + EXIT_IF(regs == nullptr); + EXIT_IF(sh == nullptr); + + *ps_info = {}; + + ps_info->input_num = sh->ps_in_control; + ps_info->ps_system_input_base = ShaderCalcPsSystemInputBase(*sh); + const uint32_t active_inputs = sh->ps_input_ena & sh->ps_input_addr; + ps_info->ps_pos_x = (active_inputs & 0x00000100u) != 0; + ps_info->ps_pos_y = (active_inputs & 0x00000200u) != 0; + ps_info->ps_pos_xy = ps_info->ps_pos_x && ps_info->ps_pos_y; + ps_info->ps_pos_z = (active_inputs & 0x00000400u) != 0; + ps_info->ps_pos_w = (active_inputs & 0x00000800u) != 0; + ps_info->ps_front_face = (active_inputs & 0x00001000u) != 0; + ps_info->ps_no_perspective = (sh->ps_input_ena & sh->ps_input_addr & 0x00000020u) != 0; + ps_info->ps_pixel_kill_enable = sh->db_shader_control.shader_kill_enable; + ps_info->ps_depth_export_enable = sh->db_shader_control.shader_z_export_enable; + ps_info->ps_sample_mask_export_enable = sh->db_shader_control.shader_mask_export_enable; + ps_info->ps_early_z = (sh->db_shader_control.shader_z_behavior == 1 && + !sh->db_shader_control.shader_kill_enable && + !sh->db_shader_control.shader_z_export_enable && + !sh->db_shader_control.shader_mask_export_enable); + ps_info->ps_execute_on_noop = sh->db_shader_control.shader_execute_on_noop; + + for (uint32_t i = 0; i < ps_info->input_num; i++) { + ps_info->interpolator_settings[i] = sh->ps_interpolator_settings[i]; + } + + ps_info->descriptor_set = + vs_info->stage.program != nullptr && !vs_info->stage.program->bindings.descriptors.empty() + ? 1 + : 0; + + for (int i = 0; i < 8; i++) { + ps_info->target_output_mode[i] = sh->target_output_mode[i]; + } + ps_info->mrt_output_mask = 0; +} + +static void ShaderGetStaticInputInfoCS(const HW::ComputeShaderInfo* regs, + const HW::ShaderRegisters* /*sh*/, + ShaderComputeInputInfo* info) { + EXIT_IF(info == nullptr); + EXIT_IF(regs == nullptr); + + *info = {}; + + info->threads_num[0] = regs->cs_regs.num_thread_x; + info->threads_num[1] = regs->cs_regs.num_thread_y; + info->threads_num[2] = regs->cs_regs.num_thread_z; + info->group_id[0] = regs->cs_regs.tgid_x_en != 0; + info->group_id[1] = regs->cs_regs.tgid_y_en != 0; + info->group_id[2] = regs->cs_regs.tgid_z_en != 0; + info->wave_size = regs->cs_regs.wave_size; + info->thread_ids_num = regs->cs_regs.tidig_comp_cnt + 1; + info->tg_size_en = regs->cs_regs.tg_size_en != 0; + + info->workgroup_register = regs->cs_regs.user_sgpr; +} + +static ShaderStageProgramKey MakeShaderStageProgramKey(ShaderType stage, uint64_t shader_hash, + const ShaderId& program_id, + ShaderLaneMaskMode lane_mask_mode) { + ShaderStageProgramKey key {}; + key.stage = stage; + key.lane_mask_mode = lane_mask_mode; + key.shader_hash = shader_hash; + key.program_id = program_id; + key.optimization_type = Config::GetShaderOptimizationType(); + key.validation = Config::ShaderValidationEnabled(); + return key; +} + +static void ApplyVertexOutputs(ShaderVertexInputInfo* info, + const ShaderRecompiler::IR::Program& program) { + info->export_count = 0; + info->param_export_mask = 0; + for (const auto& output: program.info.outputs) { + if (output.kind == ShaderRecompiler::IR::StageOutputKind::Parameter && output.index < 32) { + info->param_export_mask |= 1u << output.index; + info->export_count = std::max(info->export_count, static_cast(output.index + 1)); + } + } +} + +static void ApplyPixelOutputs(ShaderPixelInputInfo* info, + const ShaderRecompiler::IR::Program& program) { + info->mrt_output_mask = 0; + for (const auto& output: program.info.outputs) { + if (output.kind == ShaderRecompiler::IR::StageOutputKind::Mrt && output.index < 8) { + info->mrt_output_mask |= 1u << output.index; + } + } +} + +static std::string ShaderDescribeSpecialization(const ShaderRecompiler::IR::Program& program); + +static bool LogPermutationMismatch(const ShaderProgramPermutation& permutation, const char* stage, + uint64_t shader_hash, const std::string& error) { + static std::atomic log_count {0}; + if (Config::GraphicsDebugDumpEnabled() && + log_count.fetch_add(1, std::memory_order_relaxed) < 64) { + LOGF("ShaderProgramCache native runtime mismatch %s shader=0x%016" PRIx64 + ": %s\n specialization: %s\n", + stage, shader_hash, error.c_str(), + ShaderDescribeSpecialization(*permutation.program).c_str()); + } + return false; +} + +static bool TryUseVertexPermutation(const ShaderProgramPermutation& permutation, + const HW::VertexShaderInfo* regs, ShaderVertexInputInfo* info, + uint64_t shader_hash) { + EXIT_IF(regs == nullptr || info == nullptr); + std::string error; + if (!ShaderMaterializeStageRuntime( + permutation.program, + std::span(regs->gs_user_sgpr.value, regs->gs_regs.rsrc2.user_sgpr), + regs->es_regs.data_addr, &info->stage, &error)) { + return LogPermutationMismatch(permutation, "VS", shader_hash, error); + } + ApplyVertexOutputs(info, *permutation.program); + return true; +} + +static bool TryUsePixelPermutation(const ShaderProgramPermutation& permutation, + const HW::PixelShaderInfo* regs, ShaderPixelInputInfo* info, + uint64_t shader_hash) { + EXIT_IF(regs == nullptr || info == nullptr); + std::string error; + if (!ShaderMaterializeStageRuntime( + permutation.program, + std::span(regs->ps_user_sgpr.value, regs->ps_regs.rsrc2.user_sgpr), + regs->ps_regs.data_addr, &info->stage, &error)) { + return LogPermutationMismatch(permutation, "PS", shader_hash, error); + } + ApplyPixelOutputs(info, *permutation.program); + return true; +} + +static bool TryUseComputePermutation(const ShaderProgramPermutation& permutation, + const HW::ComputeShaderInfo* regs, + ShaderComputeInputInfo* info, uint64_t shader_hash) { + EXIT_IF(regs == nullptr || info == nullptr); + std::string error; + if (!ShaderMaterializeStageRuntime( + permutation.program, + std::span(regs->cs_user_sgpr.value, regs->cs_regs.user_sgpr), + regs->cs_regs.data_addr, &info->stage, &error)) { + return LogPermutationMismatch(permutation, "CS", shader_hash, error); + } + return true; +} + +static void LogShaderProgramCacheHit(const char* stage, uint64_t shader_hash, uint64_t words) { + if (!Config::GraphicsDebugDumpEnabled()) { + return; + } + + static std::atomic log_count {0}; + if (log_count.fetch_add(1, std::memory_order_relaxed) >= 512) { + return; + } + + LOGF("ShaderProgramCache: reused %s shader=0x%016" PRIx64 " words=%" PRIu64 "\n", stage, + shader_hash, words); +} + +static std::string ShaderDescribeSpecialization(const ShaderRecompiler::IR::Program& program) { + std::string ret = fmt::format( + "set={} push={} groups={} user={} buffers={} images={} samplers={} addresses={} srt={}", + program.bindings.descriptor_set, program.bindings.push_constant_size, + program.bindings.descriptors.size(), program.bindings.user_data_registers.size(), + program.info.buffers.size(), program.info.images.size(), program.info.samplers.size(), + program.info.addresses.size(), program.srt.reads.size()); + for (uint32_t i = 0; i < program.info.buffers.size(); i++) { + const auto& buffer = program.info.buffers[i]; + ret += fmt::format(" b{}[stride={} format={}]", i, buffer.packed_stride, + buffer.descriptor_format); + } + for (uint32_t i = 0; i < program.info.images.size(); i++) { + const auto& image = program.info.images[i]; + ret += fmt::format(" i{}[kind={} dim={} swizzle=0x{:03x}]", i, + static_cast(image.kind), + static_cast(image.dimension), image.storage_swizzle); + } + for (uint32_t i = 0; i < program.info.addresses.size(); i++) { + ret += fmt::format(" a{}[base=0x{:x}]", i, program.info.addresses[i].specialized_base); + } + return ret; +} + +static void ShaderAppendNativeSpecialization(std::vector* ids, + const ShaderRecompiler::IR::Program& program) { + EXIT_IF(ids == nullptr || !program.binding_layout_complete); + ids->push_back(static_cast(program.lane_mask_mode)); + ids->push_back(program.bindings.descriptor_set); + ids->push_back(program.bindings.push_constant_offset); + ids->push_back(program.bindings.push_constant_size); + ids->push_back(static_cast(program.bindings.user_data_registers.size())); + ids->insert(ids->end(), program.bindings.user_data_registers.begin(), + program.bindings.user_data_registers.end()); + ids->push_back(static_cast(program.bindings.descriptors.size())); + for (const auto& binding: program.bindings.descriptors) { + ids->push_back(static_cast(binding.kind)); + ids->push_back(binding.binding); + ids->push_back(static_cast(binding.resources.size())); + ids->insert(ids->end(), binding.resources.begin(), binding.resources.end()); + } + ids->push_back(static_cast(program.info.buffers.size())); + for (const auto& buffer: program.info.buffers) { + ids->push_back(buffer.packed_stride); + ids->push_back(buffer.descriptor_format); + } + ids->push_back(static_cast(program.info.images.size())); + for (const auto& image: program.info.images) { + ids->push_back(static_cast(image.kind)); + ids->push_back(static_cast(image.dimension)); + ids->push_back(static_cast(image.mip_mode)); + ids->push_back(image.storage_swizzle); + } + ids->push_back(static_cast(program.info.addresses.size())); + for (const auto& address: program.info.addresses) { + ids->push_back(static_cast(address.specialized_base)); + ids->push_back(static_cast(address.specialized_base >> 32u)); + } +} + +static std::span AddShaderProgramPermutation(const char* stage, + uint64_t shader_hash, + const ShaderStageProgramKey& key, + ShaderProgramPermutation permutation) { + static std::atomic compiled {0}; + const auto compiled_count = compiled.fetch_add(1, std::memory_order_relaxed) + 1; + std::printf("Num compiled %d shaders\n", compiled_count); + + std::scoped_lock lock(g_shader_program_cache_mutex); + auto& permutations = g_shader_program_cache[key]; + + if (permutations.size() >= ShaderMaxPermutationsPerProgram) { + LOGF("ShaderProgramCache overflow: %s\n", + ShaderDescribeSpecialization(*permutation.program).c_str()); + for (uint64_t i = 0; i < static_cast(permutations.size()); i++) { + LOGF("ShaderProgramCache overflow existing[%" PRIu64 "]: %s\n", i, + ShaderDescribeSpecialization(*permutations[static_cast(i)]->program) + .c_str()); + } + EXIT("ShaderProgramCache: more than %u permutations for same %s shader: " + "shader=0x%016" PRIx64 " existing=%" PRIu64 " incoming_words=%" PRIu64 + " program_hash0=0x%08" PRIx32 " program_crc32=0x%08" PRIx32 " program_ids=%" PRIu64 + "\n", + ShaderMaxPermutationsPerProgram, stage, shader_hash, + static_cast(permutations.size()), + static_cast(permutation.program->bindings.descriptors.size()), + key.program_id.hash0, key.program_id.crc32, + static_cast(key.program_id.ids.size())); + } + + auto cached = std::make_unique(std::move(permutation)); + auto spirv = MakeShaderSpirvView(cached->spirv); + permutations.push_back(std::move(cached)); + return spirv; +} + +bool ShaderCompileInfoVS(const HW::VertexShaderInfo* regs, const HW::ShaderRegisters* sh, + ShaderLaneMaskMode lane_mask_mode, ShaderVertexInputInfo* info, + std::span* spirv) { + EXIT_IF(spirv == nullptr); + *spirv = {}; + + if (!ShaderGetStaticInputInfoVS(regs, sh, info)) { + return false; + } + const auto shader_hash = regs->gs_regs.chksum; + const auto program_id = ShaderGetIdVS(regs, info, false); + const auto key = + MakeShaderStageProgramKey(ShaderType::Vertex, shader_hash, program_id, lane_mask_mode); + + { + std::scoped_lock lock(g_shader_program_cache_mutex); + if (auto iter = g_shader_program_cache.find(key); iter != g_shader_program_cache.end()) { + for (const auto& permutation: iter->second) { + if (TryUseVertexPermutation(*permutation, regs, info, shader_hash)) { + *spirv = MakeShaderSpirvView(permutation->spirv); + LogShaderProgramCacheHit("VS", shader_hash, + static_cast(spirv->size())); + return true; + } + } + } + } + + std::vector compiled_spirv; + if (!ShaderCompileSpirvVS(regs, sh, lane_mask_mode, info, &compiled_spirv)) { + return false; + } + + ShaderProgramPermutation permutation {}; + permutation.spirv = std::move(compiled_spirv); + permutation.program = info->stage.program; + *spirv = AddShaderProgramPermutation("VS", shader_hash, key, std::move(permutation)); + return true; +} + +bool ShaderCompileInfoPS(const HW::PixelShaderInfo* regs, const HW::ShaderRegisters* sh, + ShaderLaneMaskMode lane_mask_mode, const ShaderVertexInputInfo* vs_info, + ShaderPixelInputInfo* ps_info, std::span* spirv) { + EXIT_IF(spirv == nullptr); + *spirv = {}; + + ShaderGetStaticInputInfoPS(regs, sh, vs_info, ps_info); + const auto shader_hash = + regs->ps_regs.chksum != 0 ? regs->ps_regs.chksum : regs->ps_regs.data_addr; + const auto program_id = ShaderGetIdPS(regs, ps_info, false); + const auto key = + MakeShaderStageProgramKey(ShaderType::Pixel, shader_hash, program_id, lane_mask_mode); + + { + std::scoped_lock lock(g_shader_program_cache_mutex); + if (auto iter = g_shader_program_cache.find(key); iter != g_shader_program_cache.end()) { + for (const auto& permutation: iter->second) { + if (TryUsePixelPermutation(*permutation, regs, ps_info, shader_hash)) { + *spirv = MakeShaderSpirvView(permutation->spirv); + LogShaderProgramCacheHit("PS", shader_hash, + static_cast(spirv->size())); + return true; + } + } + } + } + + std::vector compiled_spirv; + if (!ShaderCompileSpirvPS(regs, sh, lane_mask_mode, ps_info, &compiled_spirv)) { + return false; + } + + ShaderProgramPermutation permutation {}; + permutation.spirv = std::move(compiled_spirv); + permutation.program = ps_info->stage.program; + *spirv = AddShaderProgramPermutation("PS", shader_hash, key, std::move(permutation)); + return true; +} + +bool ShaderCompileInfoCS(const HW::ComputeShaderInfo* regs, const HW::ShaderRegisters* sh, + ShaderComputeInputInfo* info, std::span* spirv) { + EXIT_IF(spirv == nullptr); + *spirv = {}; + + ShaderGetStaticInputInfoCS(regs, sh, info); + const auto shader_hash = regs->cs_regs.data_addr; + const auto program_id = ShaderGetIdCS(regs, info, false); + const auto key = MakeShaderStageProgramKey(ShaderType::Compute, shader_hash, program_id, + ShaderLaneMaskMode::NativeWave); + + { + std::scoped_lock lock(g_shader_program_cache_mutex); + if (auto iter = g_shader_program_cache.find(key); iter != g_shader_program_cache.end()) { + for (const auto& permutation: iter->second) { + if (TryUseComputePermutation(*permutation, regs, info, shader_hash)) { + *spirv = MakeShaderSpirvView(permutation->spirv); + LogShaderProgramCacheHit("CS", shader_hash, + static_cast(spirv->size())); + return true; + } + } + } + } + + std::vector compiled_spirv; + if (!ShaderCompileSpirvCS(regs, sh, info, &compiled_spirv)) { + return false; + } + + ShaderProgramPermutation permutation {}; + permutation.spirv = std::move(compiled_spirv); + permutation.program = info->stage.program; + *spirv = AddShaderProgramPermutation("CS", shader_hash, key, std::move(permutation)); + return true; +} + +// NOLINTNEXTLINE(readability-function-cognitive-complexity) +void ShaderDbgDumpInputInfo(const ShaderVertexInputInfo* info) { + KYTY_PROFILER_BLOCK("ShaderDbgDumpInputInfo(Vs)"); + + LOGF("ShaderDbgDumpInputInfo()\n"); + + LOGF("\t fetch_external = %s\n" + "\t fetch_embedded = %s\n" + "\t param_export_mask = 0x%08" PRIx32 "\n" + "\t export_count = %d\n", + info->fetch_external ? "true" : "false", info->fetch_embedded ? "true" : "false", + info->param_export_mask, info->export_count); + + for (int i = 0; i < info->resources_num; i++) { + LOGF("\t input %d\n", i); + + const auto& r = info->resources[i]; + const auto& rd = info->resources_dst[i]; + + LOGF("\t\t register_start = %d\n" + "\t\t registers_num = %d\n" + "\t\t fetch_index = %" PRIu32 "\n", + rd.register_start, rd.registers_num, rd.fetch_index); + LOGF("\t\t fields = %08" PRIx32 "%08" PRIx32 "%08" PRIx32 "%08" PRIx32 "\n", + r.fields[3], r.fields[2], r.fields[1], r.fields[0]); + LOGF("\t\t Base() = %" PRIx64 "\n" + "\t\t Stride() = %" PRIu16 "\n" + "\t\t SwizzleEnabled() = %s\n" + "\t\t NumRecords() = %" PRIu32 "\n" + "\t\t DstSelX() = %" PRIu8 "\n" + "\t\t DstSelY() = %" PRIu8 "\n" + "\t\t DstSelZ() = %" PRIu8 "\n" + "\t\t DstSelW() = %" PRIu8 "\n", + r.Base48(), r.Stride(), r.SwizzleEnabled() ? "true" : "false", r.NumRecords(), + r.DstSelX(), r.DstSelY(), r.DstSelZ(), r.DstSelW()); + LOGF("\t\t Format() = %" PRIu8 "\n" + "\t\t OutOfBounds() = %" PRIu8 "\n", + r.Format(), r.OutOfBounds()); + LOGF("\t\t AddTid() = %s\n", r.AddTid() ? "true" : "false"); + } + + for (int i = 0; i < info->buffers_num; i++) { + LOGF("\t buffer %d\n", i); + + const auto& r = info->buffers[i]; + LOGF("\t\t addr = %" PRIx64 "\n" + "\t\t stride = %" PRIu32 "\n" + "\t\t num_records = %" PRIu32 "\n" + "\t\t fetch_index = %" PRIu32 "\n" + "\t\t attr_num = %" PRId32 "\n", + r.addr, r.stride, r.num_records, r.fetch_index, r.attr_num); + for (int j = 0; j < r.attr_num; j++) { + LOGF("\t\t attr_indices[%d] = %d\n" + "\t\t attr_offsets[%d] = %u\n", + j, r.attr_indices[j], j, r.attr_offsets[j]); + } + } +} + +void ShaderDbgDumpInputInfo(const ShaderPixelInputInfo* info) { + KYTY_PROFILER_BLOCK("ShaderDbgDumpInputInfo(Ps)"); + + LOGF("ShaderDbgDumpInputInfo()\n"); + + LOGF("\t input_num = %u\n" + "\t ps_system_input_base = %u\n" + "\t ps_pos_x = %s\n" + "\t ps_pos_y = %s\n" + "\t ps_pos_z = %s\n" + "\t ps_pos_w = %s\n" + "\t ps_front_face = %s\n" + "\t ps_no_perspective = %s\n" + "\t ps_pixel_kill_enable = %s\n" + "\t ps_early_z = %s\n" + "\t ps_execute_on_noop = %s\n", + info->input_num, info->ps_system_input_base, info->ps_pos_x ? "true" : "false", + info->ps_pos_y ? "true" : "false", info->ps_pos_z ? "true" : "false", + info->ps_pos_w ? "true" : "false", info->ps_front_face ? "true" : "false", + info->ps_no_perspective ? "true" : "false", info->ps_pixel_kill_enable ? "true" : "false", + info->ps_early_z ? "true" : "false", info->ps_execute_on_noop ? "true" : "false"); + + for (uint32_t i = 0; i < info->input_num; i++) { + LOGF("\t interpolator_settings[%u] = %u\n", i, info->interpolator_settings[i]); + } +} + +void ShaderDbgDumpInputInfo(const ShaderComputeInputInfo* info) { + LOGF("ShaderDbgDumpInputInfo()\n"); + + LOGF("\t workgroup_register = %d\n" + "\t thread_ids_num = %d\n" + "\t wave_size = %u\n" + "\t threads_num = {%u, %u, %u}\n" + "\t tg_size_en = %s\n", + info->workgroup_register, info->thread_ids_num, info->wave_size, info->threads_num[0], + info->threads_num[1], info->threads_num[2], info->tg_size_en ? "true" : "false"); + LOGF("\t threadgroup_id = {%s, %s, %s}\n", info->group_id[0] ? "true" : "false", + info->group_id[1] ? "true" : "false", info->group_id[2] ? "true" : "false"); +} + +static bool ShaderRecompilerTextDumpEnabled() { + // Graphics debug dump already writes SPIR-V binaries to _Shaders. Keep the very large + // disassembly/IR text behind the shader-log switch so file logging cannot stall boot. + return Config::GetShaderLogDirection() != Config::ShaderLogDirection::Silent; +} + +static void DumpShaderRecompilerSpirv(const char* type, uint64_t shader_hash, + const std::vector& bin) { + if (!Config::GraphicsDebugDumpEnabled()) { + return; + } + + static std::atomic_int id = 0; + + const auto base_name = Config::GetShaderLogFolder() / + fmt::format("{:04d}_{:04d}_new_shader_{}_{:016x}", + GraphicsRunGetFrameNum(), id++, type, shader_hash); + Common::File::CreateDirectories(base_name.parent_path()); + + Common::File spv_file; + auto spv_name = base_name; + spv_name += ".spv"; + spv_file.Create(spv_name); + if (spv_file.IsInvalid()) { + auto spv_name_text = Common::PathToString(spv_name); + LOGF_COLOR(Log::Color::BrightRed, "Can't create file: %s\n", spv_name_text.c_str()); + } else { + spv_file.Write(bin.data(), bin.size() * 4); + spv_file.Close(); + } + + return; + + std::string text; + if (!SpirvDisassemble(bin.data(), bin.size(), &text)) { + auto spv_name_text = Common::PathToString(spv_name); + LOGF_COLOR(Log::Color::BrightRed, "SpirvDisassemble() failed for %s\n", + spv_name_text.c_str()); + return; + } + + Common::File asm_file; + auto asm_name = base_name; + asm_name += ".spvasm"; + asm_file.Create(asm_name); + if (asm_file.IsInvalid()) { + auto asm_name_text = Common::PathToString(asm_name); + LOGF_COLOR(Log::Color::BrightRed, "Can't create file: %s\n", asm_name_text.c_str()); + } else { + asm_file.Printf("%s", text.c_str()); + asm_file.Close(); + } +} + +static void DumpShaderRecompilerOriginal(const char* type, uint64_t shader_hash, + std::span code, + const std::string& decoded_dump) { + // if (!Config::GraphicsDebugDumpEnabled()) { + // return; + // } + return; + EXIT_IF(code.empty()); + + static std::atomic_int id = 0; + + const auto base_name = Config::GetShaderLogFolder() / "original" / + fmt::format("{:04d}_{:04d}_new_shader_{}_{:016x}", + GraphicsRunGetFrameNum(), id++, type, shader_hash); + Common::File::CreateDirectories(base_name.parent_path()); + + Common::File bin_file; + auto bin_name = base_name; + bin_name += ".bin"; + bin_file.Create(bin_name); + if (bin_file.IsInvalid()) { + auto bin_name_text = Common::PathToString(bin_name); + LOGF_COLOR(Log::Color::BrightRed, "Can't create file: %s\n", bin_name_text.c_str()); + } else { + bin_file.Write(code.data(), code.size_bytes()); + bin_file.Close(); + } + + Common::File text_file; + auto text_name = base_name; + text_name += ".rdna2"; + text_file.Create(text_name); + if (text_file.IsInvalid()) { + auto text_name_text = Common::PathToString(text_name); + LOGF_COLOR(Log::Color::BrightRed, "Can't create file: %s\n", text_name_text.c_str()); + } else { + text_file.Printf("%s", decoded_dump.c_str()); + text_file.Close(); + } +} + +bool ShaderCompileSpirvVS(const HW::VertexShaderInfo* regs, const HW::ShaderRegisters* sh, + ShaderLaneMaskMode lane_mask_mode, ShaderVertexInputInfo* input_info, + std::vector* spirv) { + KYTY_PROFILER_FUNCTION(profiler::colors::Amber300); + + EXIT_IF(regs == nullptr); + EXIT_IF(sh == nullptr); + EXIT_IF(input_info == nullptr); + EXIT_IF(spirv == nullptr); + + EXIT_NOT_IMPLEMENTED(regs->es_regs.data_addr == 0 || regs->gs_regs.chksum == 0); + + const uint64_t shader_addr = regs->es_regs.data_addr; + const auto code = ShaderGetMappedCode(shader_addr, "ShaderRecompiler VS", regs->gs_regs.chksum); + + ShaderRecompiler::CompileOptions options; + options.stage = ShaderType::Vertex; + options.lane_mask_mode = lane_mask_mode; + options.shader_hash = regs->gs_regs.chksum; + options.shader_base = shader_addr; + options.user_data_base = 8; + options.user_data_count = regs->gs_regs.rsrc2.user_sgpr; + options.user_data = regs->gs_user_sgpr.value; + options.descriptor_set = 0; + options.push_constant_offset = 0; + options.vertex_input_info = input_info; + options.dump_ir = ShaderRecompilerTextDumpEnabled(); + options.early_dump = options.dump_ir; + options.dump_label = "ShaderRecompiler VS"; + + ShaderRecompiler::CompileResult result; + std::string error; + if (!ShaderRecompiler::TryRecompile(code, options, &result, &error)) { + ExitShaderRecompilerFailure("ShaderRecompiler VS", options.shader_hash, error.c_str()); + } + DumpShaderRecompilerOriginal("vs", options.shader_hash, code, result.decoded_dump); + if (!SpirvValidateBinary("ShaderRecompiler VS", options.shader_hash, result.spirv)) { + DumpShaderRecompilerSpirv("vs", options.shader_hash, result.spirv); + ExitShaderRecompilerFailure("ShaderRecompiler VS", options.shader_hash, + "SPIR-V validation failed"); + } + + input_info->stage.program = + std::make_shared(std::move(result.program)); + input_info->stage.resources = + std::make_shared(std::move(result.resources)); + ApplyVertexOutputs(input_info, *input_info->stage.program); + *spirv = std::move(result.spirv); + DumpShaderRecompilerSpirv("vs", options.shader_hash, *spirv); + + if (options.dump_ir) { + if (!options.early_dump) { + LOGF("ShaderRecompiler VS decoded RDNA2:\n%s", result.decoded_dump.c_str()); + LOGF("ShaderRecompiler VS IR:\n%s", result.ir_dump.c_str()); + } + LOGF("ShaderRecompiler VS SPIR-V words=%" PRIu64 "\n", + static_cast(spirv->size())); + } + return true; +} + +bool ShaderCompileSpirvPS(const HW::PixelShaderInfo* regs, const HW::ShaderRegisters* sh, + ShaderLaneMaskMode lane_mask_mode, ShaderPixelInputInfo* input_info, + std::vector* spirv) { + KYTY_PROFILER_FUNCTION(profiler::colors::Blue300); + + EXIT_IF(regs == nullptr); + EXIT_IF(sh == nullptr); + EXIT_IF(input_info == nullptr); + EXIT_IF(spirv == nullptr); + + const uint64_t shader_addr = regs->ps_regs.data_addr; + const uint64_t shader_hash = regs->ps_regs.chksum != 0 ? regs->ps_regs.chksum : shader_addr; + const auto code = ShaderGetMappedCode(shader_addr, "ShaderRecompiler PS", shader_hash); + + ShaderRecompiler::CompileOptions options; + options.stage = ShaderType::Pixel; + options.lane_mask_mode = lane_mask_mode; + options.shader_hash = shader_hash; + options.shader_base = shader_addr; + options.user_data_count = regs->ps_regs.rsrc2.user_sgpr; + options.user_data = regs->ps_user_sgpr.value; + options.descriptor_set = input_info->descriptor_set; + options.push_constant_offset = 0; + options.pixel_input_info = input_info; + options.dump_ir = ShaderRecompilerTextDumpEnabled(); + options.early_dump = options.dump_ir; + options.dump_label = "ShaderRecompiler PS"; + + ShaderRecompiler::CompileResult result; + std::string error; + if (!ShaderRecompiler::TryRecompile(code, options, &result, &error)) { + ExitShaderRecompilerFailure("ShaderRecompiler PS", options.shader_hash, error.c_str()); + } + DumpShaderRecompilerOriginal("ps", options.shader_hash, code, result.decoded_dump); + if (!SpirvValidateBinary("ShaderRecompiler PS", options.shader_hash, result.spirv)) { + DumpShaderRecompilerSpirv("ps", options.shader_hash, result.spirv); + ExitShaderRecompilerFailure("ShaderRecompiler PS", options.shader_hash, + "SPIR-V validation failed"); + } + input_info->stage.program = + std::make_shared(std::move(result.program)); + input_info->stage.resources = + std::make_shared(std::move(result.resources)); + ApplyPixelOutputs(input_info, *input_info->stage.program); + *spirv = std::move(result.spirv); + DumpShaderRecompilerSpirv("ps", options.shader_hash, *spirv); + + if (options.dump_ir) { + if (!options.early_dump) { + LOGF("ShaderRecompiler PS decoded RDNA2:\n%s", result.decoded_dump.c_str()); + LOGF("ShaderRecompiler PS IR:\n%s", result.ir_dump.c_str()); + } + LOGF("ShaderRecompiler PS SPIR-V words=%" PRIu64 "\n", + static_cast(spirv->size())); + } + return true; +} + +bool ShaderCompileSpirvCS(const HW::ComputeShaderInfo* regs, const HW::ShaderRegisters* sh, + ShaderComputeInputInfo* input_info, std::vector* spirv) { + KYTY_PROFILER_FUNCTION(profiler::colors::CyanA700); + + EXIT_IF(regs == nullptr); + EXIT_IF(sh == nullptr); + EXIT_IF(input_info == nullptr); + EXIT_IF(spirv == nullptr); + + const uint64_t shader_addr = regs->cs_regs.data_addr; + const auto code = ShaderGetMappedCode(shader_addr, "ShaderRecompiler CS", shader_addr); + + ShaderRecompiler::CompileOptions options; + options.stage = ShaderType::Compute; + options.shader_hash = shader_addr; + options.shader_base = shader_addr; + options.user_data_count = regs->cs_regs.user_sgpr; + options.user_data = regs->cs_user_sgpr.value; + options.descriptor_set = 0; + options.push_constant_offset = 0; + options.compute_input_info = input_info; + options.wave_size = input_info->wave_size; + options.dump_ir = ShaderRecompilerTextDumpEnabled(); + options.early_dump = options.dump_ir; + options.dump_label = "ShaderRecompiler CS"; + + ShaderRecompiler::CompileResult result; + std::string error; + if (!ShaderRecompiler::TryRecompile(code, options, &result, &error)) { + ExitShaderRecompilerFailure("ShaderRecompiler CS", options.shader_hash, error.c_str()); + } + DumpShaderRecompilerOriginal("cs", options.shader_hash, code, result.decoded_dump); + if (!SpirvValidateBinary("ShaderRecompiler CS", options.shader_hash, result.spirv)) { + DumpShaderRecompilerSpirv("cs", options.shader_hash, result.spirv); + ExitShaderRecompilerFailure("ShaderRecompiler CS", options.shader_hash, + "SPIR-V validation failed"); + } + input_info->stage.program = + std::make_shared(std::move(result.program)); + input_info->stage.resources = + std::make_shared(std::move(result.resources)); + *spirv = std::move(result.spirv); + DumpShaderRecompilerSpirv("cs", options.shader_hash, *spirv); + + if (options.dump_ir) { + if (!options.early_dump) { + LOGF("ShaderRecompiler CS decoded RDNA2:\n%s", result.decoded_dump.c_str()); + LOGF("ShaderRecompiler CS IR:\n%s", result.ir_dump.c_str()); + } + LOGF("ShaderRecompiler CS SPIR-V words=%" PRIu64 " wave_size=%u\n", + static_cast(spirv->size()), options.wave_size); + } + return true; +} + +ShaderId ShaderGetIdVS(const HW::VertexShaderInfo* regs, const ShaderVertexInputInfo* input_info, + bool include_bind_specialization) { + KYTY_PROFILER_FUNCTION(); + + ShaderId ret; + + ret.ids.reserve(64); + + EXIT_NOT_IMPLEMENTED(regs->es_regs.data_addr == 0 || regs->gs_regs.chksum == 0); + + ret.hash0 = (regs->gs_regs.chksum >> 32u) & 0xffffffffu; + ret.crc32 = regs->gs_regs.chksum & 0xffffffffu; + + ret.ids.push_back(static_cast(input_info->fetch_external)); + ret.ids.push_back(static_cast(input_info->fetch_embedded)); + ret.ids.push_back(input_info->resources_num); + ret.ids.push_back(input_info->export_count); + + for (int i = 0; i < input_info->resources_num; i++) { + const auto& r = input_info->resources[i]; + const auto& rd = input_info->resources_dst[i]; + + ret.ids.push_back(rd.register_start); + ret.ids.push_back(rd.registers_num); + ret.ids.push_back(rd.fetch_index); + ret.ids.push_back(r.Stride()); + ret.ids.push_back(static_cast(r.SwizzleEnabled())); + ret.ids.push_back(r.DstSelX()); + ret.ids.push_back(r.DstSelY()); + ret.ids.push_back(r.DstSelZ()); + ret.ids.push_back(r.DstSelW()); + ret.ids.push_back(r.Format()); + ret.ids.push_back(r.OutOfBounds()); + ret.ids.push_back(static_cast(r.AddTid())); + } + + ret.ids.push_back(input_info->buffers_num); + + for (int i = 0; i < input_info->buffers_num; i++) { + const auto& r = input_info->buffers[i]; + ret.ids.push_back(r.attr_num); + ret.ids.push_back(r.stride); + ret.ids.push_back(r.fetch_index); + for (int j = 0; j < r.attr_num; j++) { + ret.ids.push_back(r.attr_indices[j]); + ret.ids.push_back(r.attr_offsets[j]); + } + } + + if (include_bind_specialization) { + EXIT_IF(!input_info->stage); + ShaderAppendNativeSpecialization(&ret.ids, *input_info->stage.program); + } + + return ret; +} + +ShaderId ShaderGetIdPS(const HW::PixelShaderInfo* regs, const ShaderPixelInputInfo* input_info, + bool include_bind_specialization) { + KYTY_PROFILER_FUNCTION(); + + ShaderId ret; + + ret.ids.reserve(64); + + ret.hash0 = (regs->ps_regs.chksum >> 32u) & 0xffffffffu; + ret.crc32 = regs->ps_regs.chksum & 0xffffffffu; + + ret.ids.push_back(input_info->descriptor_set); + ret.ids.push_back(input_info->input_num); + ret.ids.push_back(input_info->ps_system_input_base); + ret.ids.push_back(static_cast(input_info->ps_pos_x)); + ret.ids.push_back(static_cast(input_info->ps_pos_y)); + ret.ids.push_back(static_cast(input_info->ps_pos_z)); + ret.ids.push_back(static_cast(input_info->ps_pos_w)); + ret.ids.push_back(static_cast(input_info->ps_front_face)); + ret.ids.push_back(static_cast(input_info->ps_pixel_kill_enable)); + ret.ids.push_back(static_cast(input_info->ps_sample_mask_export_enable)); + ret.ids.push_back(static_cast(input_info->ps_early_z)); + ret.ids.push_back(static_cast(input_info->ps_execute_on_noop)); + + for (auto mode: input_info->target_output_mode) { + ret.ids.push_back(mode); + } + ret.ids.push_back(input_info->mrt_output_mask); + + for (uint32_t i = 0; i < input_info->input_num; i++) { + ret.ids.push_back(input_info->interpolator_settings[i]); + } + + if (include_bind_specialization) { + EXIT_IF(!input_info->stage); + ShaderAppendNativeSpecialization(&ret.ids, *input_info->stage.program); + } + + return ret; +} + +ShaderId ShaderGetIdCS(const HW::ComputeShaderInfo* regs, const ShaderComputeInputInfo* input_info, + bool include_bind_specialization) { + const auto* src = reinterpret_cast(regs->cs_regs.data_addr); + + EXIT_NOT_IMPLEMENTED(src == nullptr); + + const auto* header = GetBinaryInfo(src); + + ShaderId ret; + ret.ids.reserve(64); + + if (header != nullptr) { + ret.hash0 = header->hash0; + ret.crc32 = header->crc32; + ret.ids.push_back(header->length); + } else { + GetNextGenFallbackShaderId(regs->cs_regs.data_addr, &ret.hash0, &ret.crc32); + ret.ids.push_back(static_cast(regs->cs_regs.data_addr & 0xffffffffu)); + ret.ids.push_back(static_cast((regs->cs_regs.data_addr >> 32u) & 0xffffffffu)); + } + + ret.ids.push_back(input_info->workgroup_register); + ret.ids.push_back(input_info->wave_size); + ret.ids.push_back(input_info->thread_ids_num); + + for (int i = 0; i < 3; i++) { + ret.ids.push_back(input_info->threads_num[i]); + ret.ids.push_back(static_cast(input_info->group_id[i])); + } + ret.ids.push_back(static_cast(input_info->dispatch_thread_dimensions)); + for (uint32_t dim: input_info->dispatch_threads_num) { + ret.ids.push_back(dim); + } + + if (include_bind_specialization) { + EXIT_IF(!input_info->stage); + ShaderAppendNativeSpecialization(&ret.ids, *input_info->stage.program); + } + + return ret; +} + +bool ShaderAddressValid(uint64_t addr) { + return reinterpret_cast(addr) != nullptr; +} + +} // namespace Libs::Graphics diff --git a/src/graphics/shader/shader.h b/src/graphics/shader/shader.h new file mode 100644 index 000000000..8186fb48a --- /dev/null +++ b/src/graphics/shader/shader.h @@ -0,0 +1,246 @@ +#ifndef EMULATOR_INCLUDE_EMULATOR_GRAPHICS_SHADER_H_ +#define EMULATOR_INCLUDE_EMULATOR_GRAPHICS_SHADER_H_ + +#include "common/abi.h" +#include "common/common.h" +#include "graphics/shader/shaderBindings.h" + +#include +#include +#include +#include + +namespace Libs::Graphics { + +namespace HW { +struct VertexShaderInfo; +struct PixelShaderInfo; +struct ComputeShaderInfo; +struct ShaderRegisters; +} // namespace HW + +enum class ShaderType { Unknown, Vertex, Pixel, Fetch, Compute }; + +enum class ShaderLaneMaskMode { NativeWave, PerInvocation }; + +namespace ShaderRecompiler::IR { +struct Program; +struct ResourceSnapshot; +} // namespace ShaderRecompiler::IR + +struct ShaderStageRuntime { + std::shared_ptr program; + std::shared_ptr resources; + + [[nodiscard]] explicit operator bool() const { + return program != nullptr && resources != nullptr; + } +}; + +// Resolves an immutable native shader plan against current user data. The prior stage is preserved +// if any ReadConst, snapshot, or specialization check fails. +bool ShaderMaterializeStageRuntime(std::shared_ptr program, + std::span user_data, uint64_t shader_base, + ShaderStageRuntime* stage, std::string* error); + +struct ShaderId { + uint32_t hash0 = 0; + uint32_t crc32 = 0; + std::vector ids; + + bool operator==(const ShaderId& other) const { + return hash0 == other.hash0 && crc32 == other.crc32 && ids == other.ids; + } + bool operator!=(const ShaderId& other) const { return !(*this == other); } +}; + +constexpr uint32_t DstSel(uint32_t x, uint32_t y = 0, uint32_t z = 0, uint32_t w = 0) { + return x | (y << 3u) | (z << 6u) | (w << 9u); +} + +inline uint8_t GetDstSel(uint32_t swizzle, uint32_t channel) { + return (swizzle >> (channel * 3u)) & 0x7u; +} + +struct ShaderVertexInputInfo { + static constexpr int RES_MAX = 32; + + ShaderBufferResource resources[RES_MAX]; + ShaderVertexDestination resources_dst[RES_MAX]; + int resource_fetch_components[RES_MAX] = {}; + ShaderVertexInputBuffer buffers[RES_MAX]; + ShaderStageRuntime stage; + int resources_num = 0; + int fetch_shader_reg = 0; + int fetch_attrib_reg = 0; + int fetch_buffer_reg = 0; + int buffers_num = 0; + int export_count = 0; + uint32_t param_export_mask = 0; + bool fetch_external = false; + bool fetch_embedded = false; +}; + +struct ShaderComputeInputInfo { + uint32_t threads_num[3] = {0, 0, 0}; + uint32_t dispatch_threads_num[3] = {0, 0, 0}; + bool group_id[3] = {false, false, false}; + bool dispatch_thread_dimensions = false; + uint32_t wave_size = 64; + int thread_ids_num = 0; + int workgroup_register = 0; + bool tg_size_en = false; + ShaderStageRuntime stage; +}; + +struct ShaderPixelInputInfo { + uint32_t interpolator_settings[32] = {0}; + uint32_t input_num = 0; + uint32_t ps_system_input_base = 0; + uint8_t target_output_mode[8] = {}; + uint32_t mrt_output_mask = 0; + uint32_t descriptor_set = 0; + bool ps_pos_x = false; + bool ps_pos_y = false; + bool ps_pos_xy = false; + bool ps_pos_z = false; + bool ps_pos_w = false; + bool ps_front_face = false; + bool ps_no_perspective = false; + bool ps_pixel_kill_enable = false; + bool ps_depth_export_enable = false; + bool ps_sample_mask_export_enable = false; + bool ps_early_z = false; + bool ps_execute_on_noop = false; + ShaderStageRuntime stage; + + bool HasPositionInput() const { return ps_pos_x || ps_pos_y || ps_pos_z || ps_pos_w; } +}; + +struct ShaderSharp { + uint16_t offset_dw : 15; + uint16_t size : 1; +}; + +struct ShaderUserData { + uint16_t* direct_resource_offset; + ShaderSharp* sharp_resource_offset[4]; + uint16_t eud_size_dw; + uint16_t srt_size_dw; + uint16_t direct_resource_count; + uint16_t sharp_resource_count[4]; +}; + +struct ShaderRegisterRange { + uint16_t start; + uint16_t end; +}; + +struct ShaderDrawModifier { + uint32_t enbl_start_vertex_offset : 1; + uint32_t enbl_start_index_offset : 1; + uint32_t enbl_start_instance_offset : 1; + uint32_t enbl_draw_index : 1; + uint32_t enbl_user_vgprs : 1; + uint32_t render_target_slice_offset : 3; + uint32_t fuse_draws : 1; + uint32_t compiler_flags : 23; + uint32_t is_default : 1; + uint32_t reserved : 31; +}; + +struct ShaderRegister { + uint32_t offset; + uint32_t value; +}; + +struct ShaderSpecialRegs { + ShaderRegister ge_cntl; + ShaderRegister vgt_shader_stages_en; + uint32_t dispatch_modifier; + ShaderRegisterRange user_data_range; + ShaderDrawModifier draw_modifier; + ShaderRegister vgt_gs_out_prim_type; + ShaderRegister ge_user_vgpr_en; +}; + +struct ShaderSemantic { + uint32_t semantic : 8; + uint32_t hardware_mapping : 8; + uint32_t size_in_elements : 4; + uint32_t is_f16 : 2; + uint32_t is_flat_shaded : 1; + uint32_t is_linear : 1; + uint32_t is_custom : 1; + uint32_t static_vb_index : 1; + uint32_t static_attribute : 1; + uint32_t reserved : 1; + uint32_t default_value : 2; + uint32_t default_value_hi : 2; +}; + +struct Shader { + uint32_t file_header; + uint32_t version; + ShaderUserData* user_data; + const volatile void* code; + ShaderRegister* cx_registers; + ShaderRegister* sh_registers; + ShaderSpecialRegs* specials; + ShaderSemantic* input_semantics; + ShaderSemantic* output_semantics; + uint32_t header_size; + uint32_t shader_size; + uint32_t embedded_constant_buffer_size_dqw; + uint32_t target; + uint32_t num_input_semantics; + uint16_t scratch_size_dw_per_thread; + uint16_t num_output_semantics; + uint16_t special_sizes_bytes; + uint8_t type; + uint8_t num_cx_registers; + uint8_t num_sh_registers; +}; + +struct ShaderMappedData { + ShaderUserData* user_data = nullptr; + ShaderSemantic* input_semantics = nullptr; + uint32_t num_input_semantics = 0; + uint32_t code_size_bytes = 0; +}; + +void ShaderInit(); +void ShaderMapUserData(uint64_t addr, const ShaderMappedData& data); + +void ShaderDbgDumpInputInfo(const ShaderVertexInputInfo* info); +void ShaderDbgDumpInputInfo(const ShaderPixelInputInfo* info); +void ShaderDbgDumpInputInfo(const ShaderComputeInputInfo* info); +ShaderId ShaderGetIdVS(const HW::VertexShaderInfo* regs, const ShaderVertexInputInfo* input_info, + bool include_bind_specialization); +ShaderId ShaderGetIdPS(const HW::PixelShaderInfo* regs, const ShaderPixelInputInfo* input_info, + bool include_bind_specialization); +ShaderId ShaderGetIdCS(const HW::ComputeShaderInfo* regs, const ShaderComputeInputInfo* input_info, + bool include_bind_specialization); +// Returned SPIR-V spans are read-only views backed by the shader program cache. +bool ShaderCompileInfoVS(const HW::VertexShaderInfo* regs, const HW::ShaderRegisters* sh, + ShaderLaneMaskMode lane_mask_mode, ShaderVertexInputInfo* input_info, + std::span* spirv); +bool ShaderCompileInfoPS(const HW::PixelShaderInfo* regs, const HW::ShaderRegisters* sh, + ShaderLaneMaskMode lane_mask_mode, const ShaderVertexInputInfo* vs_info, + ShaderPixelInputInfo* input_info, std::span* spirv); +bool ShaderCompileInfoCS(const HW::ComputeShaderInfo* regs, const HW::ShaderRegisters* sh, + ShaderComputeInputInfo* input_info, std::span* spirv); +bool ShaderCompileSpirvVS(const HW::VertexShaderInfo* regs, const HW::ShaderRegisters* sh, + ShaderLaneMaskMode lane_mask_mode, ShaderVertexInputInfo* input_info, + std::vector* spirv); +bool ShaderCompileSpirvPS(const HW::PixelShaderInfo* regs, const HW::ShaderRegisters* sh, + ShaderLaneMaskMode lane_mask_mode, ShaderPixelInputInfo* input_info, + std::vector* spirv); +bool ShaderCompileSpirvCS(const HW::ComputeShaderInfo* regs, const HW::ShaderRegisters* sh, + ShaderComputeInputInfo* input_info, std::vector* spirv); +bool ShaderBindingResearchGuardEnabled(); +bool ShaderAddressValid(uint64_t addr); + +} // namespace Libs::Graphics + +#endif /* EMULATOR_INCLUDE_EMULATOR_GRAPHICS_SHADER_H_ */ diff --git a/src/graphics/shader/shaderBindings.cpp b/src/graphics/shader/shaderBindings.cpp new file mode 100644 index 000000000..244f893f5 --- /dev/null +++ b/src/graphics/shader/shaderBindings.cpp @@ -0,0 +1,37 @@ +#include "graphics/shader/shaderBindings.h" + +#include "common/assert.h" +#include "graphics/guest_gpu/gpu_defs.h" + +namespace Libs::Graphics { + +ResourceDescriptorType ShaderClassifyResourceDescriptor(const uint32_t* desc) { + EXIT_IF(desc == nullptr); + + const auto raw_type = (desc[3] >> 28u) & 0xfu; + switch (static_cast(raw_type)) { + case Prospero::ImageType::kColor1D: + case Prospero::ImageType::kColor2D: + case Prospero::ImageType::kColor3D: + case Prospero::ImageType::kCube: + case Prospero::ImageType::kColor1DArray: + case Prospero::ImageType::kColor2DArray: + case Prospero::ImageType::kColor2DMsaa: + case Prospero::ImageType::kColor2DMsaaArray: return ResourceDescriptorType::Texture; + default: break; + } + + // The mixed resource tag shares a byte with texture fields, so texture type wins. + switch (static_cast((desc[5] >> 27u) & 0x3u)) { + case Prospero::DescriptorKind::kBuffer: return ResourceDescriptorType::Buffer; + case Prospero::DescriptorKind::kSampler: return ResourceDescriptorType::Sampler; + case Prospero::DescriptorKind::kUnused: return ResourceDescriptorType::Unused; + default: break; + } + + return (raw_type & Prospero::GpuEnumValue(Prospero::ImageType::kColor1D)) == 0 + ? ResourceDescriptorType::Buffer + : ResourceDescriptorType::Unused; +} + +} // namespace Libs::Graphics diff --git a/src/graphics/shader/shaderBindings.h b/src/graphics/shader/shaderBindings.h new file mode 100644 index 000000000..dad1f77d3 --- /dev/null +++ b/src/graphics/shader/shaderBindings.h @@ -0,0 +1,191 @@ +#ifndef EMULATOR_INCLUDE_EMULATOR_GRAPHICS_SHADERBINDINGS_H_ +#define EMULATOR_INCLUDE_EMULATOR_GRAPHICS_SHADERBINDINGS_H_ + +#include "common/abi.h" +#include "common/common.h" + +namespace Libs::Graphics { + +inline constexpr uint16_t AGC_ILLEGAL_DIRECT_OFFSET = 0xffff; + +enum class AgcDirectResourceType : uint32_t { + GdsCounterRange = 0, + ShaderResourceTable = 1, + SubPtrFetchShader = 2, + PtrSoBufferTable = 3, + PtrInternalGlobalTable = 4, + PtrExtendedUserData = 5, + GsFlags = 6, + GdsMemoryRange = 7, + PtrVertexBufferTable = 8, + StereoXOffset = 9, + PtrVertexAttribDescTable = 10, + Last = PtrVertexAttribDescTable, +}; + +enum class ResourceDescriptorType { + Texture, + Buffer, + Sampler, + Unused, +}; + +struct ShaderBufferResource { + uint32_t fields[4] = {0}; + + void UpdateAddress48(uint64_t gpu_addr) { + auto lo = static_cast(gpu_addr & 0xffffffffu); + auto hi = static_cast(gpu_addr >> 32u); + fields[0] = lo; + fields[1] = (fields[1] & 0xffff0000u) | (hi & 0x0000ffffu); + } + + [[nodiscard]] uint16_t Stride() const { return (fields[1] >> 16u) & 0x3FFFu; } + [[nodiscard]] bool SwizzleEnabled() const { return ((fields[1] >> 31u) & 0x1u) == 1; } + [[nodiscard]] uint32_t NumRecords() const { return fields[2]; } + [[nodiscard]] uint8_t DstSelX() const { return (fields[3] >> 0u) & 0x7u; } + [[nodiscard]] uint8_t DstSelY() const { return (fields[3] >> 3u) & 0x7u; } + [[nodiscard]] uint8_t DstSelZ() const { return (fields[3] >> 6u) & 0x7u; } + [[nodiscard]] uint8_t DstSelW() const { return (fields[3] >> 9u) & 0x7u; } + [[nodiscard]] uint32_t DstSelXY() const { return (fields[3] >> 0u) & 0x3Fu; } + [[nodiscard]] uint32_t DstSelXYZ() const { return (fields[3] >> 0u) & 0x1FFu; } + [[nodiscard]] uint32_t DstSelXYZW() const { return (fields[3] >> 0u) & 0xFFFu; } + [[nodiscard]] bool AddTid() const { return ((fields[3] >> 23u) & 0x1u) == 1; } + [[nodiscard]] uint8_t IndexStride() const { return (fields[3] >> 21u) & 0x3u; } + [[nodiscard]] uint32_t IndexStrideRecords() const { return 8u << IndexStride(); } + [[nodiscard]] uint32_t PackedStride() const { + return Stride() | (static_cast(SwizzleEnabled()) << 14u) | + (static_cast(IndexStride()) << 16u) | + (static_cast(AddTid()) << 20u); + } + + [[nodiscard]] uint64_t Base48() const { + return (fields[0] | (static_cast(fields[1]) << 32u)) & 0xFFFFFFFFFFFFu; + } + [[nodiscard]] uint8_t Format() const { return (fields[3] >> 12u) & 0x7Fu; } + [[nodiscard]] uint8_t OutOfBounds() const { return (fields[3] >> 28u) & 0x3u; } +}; + +struct ShaderTextureResource { + uint32_t fields[8] = {0}; + + void UpdateAddress40(uint64_t gpu_addr) { + auto lo = static_cast(gpu_addr & 0xffffffffu); + auto hi = static_cast(gpu_addr >> 32u); + fields[0] = lo; + fields[1] = (fields[1] & 0xffffff00u) | (hi & 0x000000ffu); + } + + [[nodiscard]] uint16_t MinLod() const { return (fields[1] >> 8u) & 0xFFFu; } + [[nodiscard]] uint8_t DstSelX() const { return (fields[3] >> 0u) & 0x7u; } + [[nodiscard]] uint8_t DstSelY() const { return (fields[3] >> 3u) & 0x7u; } + [[nodiscard]] uint8_t DstSelZ() const { return (fields[3] >> 6u) & 0x7u; } + [[nodiscard]] uint8_t DstSelW() const { return (fields[3] >> 9u) & 0x7u; } + [[nodiscard]] uint32_t DstSelXY() const { return (fields[3] >> 0u) & 0x3Fu; } + [[nodiscard]] uint32_t DstSelXYZ() const { return (fields[3] >> 0u) & 0x1FFu; } + [[nodiscard]] uint32_t DstSelXYZW() const { return (fields[3] >> 0u) & 0xFFFu; } + [[nodiscard]] uint8_t BaseLevel() const { return (fields[3] >> 12u) & 0xFu; } + [[nodiscard]] uint8_t LastLevel() const { return (fields[3] >> 16u) & 0xFu; } + [[nodiscard]] uint8_t TileMode() const { return (fields[3] >> 20u) & 0x1Fu; } + [[nodiscard]] uint8_t Type() const { return (fields[3] >> 28u) & 0xFu; } + [[nodiscard]] uint16_t Depth() const { return (fields[4] >> 0u) & 0x1FFFu; } + + [[nodiscard]] uint64_t Base40() const { + return ((fields[0] | (static_cast(fields[1]) << 32u)) & 0xFFFFFFFFFFu) << 8u; + } + [[nodiscard]] bool IsNull() const { return Base40() == 0; } + [[nodiscard]] uint16_t Format() const { return (fields[1] >> 20u) & 0x1FFu; } + [[nodiscard]] uint16_t Width5() const { + return ((fields[1] >> 30u) & 0x3u) | (((fields[2] >> 0u) & 0xFFFu) << 2u); + } + [[nodiscard]] uint16_t Height5() const { return (fields[2] >> 14u) & 0x3FFFu; } + [[nodiscard]] uint8_t BCSwizzle() const { return (fields[3] >> 25u) & 0x7u; } + [[nodiscard]] uint16_t BaseArray5() const { return (fields[4] >> 16u) & 0x1FFFu; } + [[nodiscard]] uint8_t ArrayPitch() const { return (fields[5] >> 0u) & 0xFu; } + [[nodiscard]] uint8_t MaxMip() const { return (fields[5] >> 4u) & 0xFu; } + [[nodiscard]] uint16_t MinLodWarn5() const { return (fields[5] >> 8u) & 0xFFFu; } + [[nodiscard]] uint8_t PerfMod5() const { return (fields[5] >> 20u) & 0x7u; } + [[nodiscard]] bool CornerSample() const { return ((fields[5] >> 23u) & 0x1u) == 1; } + [[nodiscard]] bool MipStatsCntEn() const { return ((fields[5] >> 25u) & 0x1u) == 1; } + [[nodiscard]] bool PrtDefColor() const { return ((fields[5] >> 26u) & 0x1u) == 1; } + [[nodiscard]] uint8_t MipStatsCntId() const { return (fields[6] >> 0u) & 0xFFu; } + [[nodiscard]] bool MsaaDepth() const { return ((fields[6] >> 10u) & 0x1u) == 1; } + [[nodiscard]] uint8_t MaxUncompBlkSize() const { return (fields[6] >> 15u) & 0x3u; } + [[nodiscard]] uint8_t MaxCompBlkSize() const { return (fields[6] >> 17u) & 0x3u; } + [[nodiscard]] bool MetaPipeAligned() const { return ((fields[6] >> 19u) & 0x1u) == 1; } + [[nodiscard]] bool WriteCompress() const { return ((fields[6] >> 20u) & 0x1u) == 1; } + [[nodiscard]] bool MetaCompress() const { return ((fields[6] >> 21u) & 0x1u) == 1; } + [[nodiscard]] bool DccAlphaPos() const { return ((fields[6] >> 22u) & 0x1u) == 1; } + [[nodiscard]] bool DccColorTransf() const { return ((fields[6] >> 23u) & 0x1u) == 1; } + [[nodiscard]] uint64_t MetaAddr() const { + return ((fields[6] >> 24u) & 0xFFu) | (static_cast(fields[7]) << 8u); + } +}; + +struct ShaderSamplerResource { + uint32_t fields[4] = {0}; + + void UpdateIndex(uint32_t index) { fields[0] = index; } + + [[nodiscard]] uint8_t ClampX() const { return (fields[0] >> 0u) & 0x7u; } + [[nodiscard]] uint8_t ClampY() const { return (fields[0] >> 3u) & 0x7u; } + [[nodiscard]] uint8_t ClampZ() const { return (fields[0] >> 6u) & 0x7u; } + [[nodiscard]] uint8_t MaxAnisoRatio() const { return (fields[0] >> 9u) & 0x7u; } + [[nodiscard]] uint8_t DepthCompareFunc() const { return (fields[0] >> 12u) & 0x7u; } + [[nodiscard]] bool ForceUnormCoords() const { return ((fields[0] >> 15u) & 0x1u) == 1; } + [[nodiscard]] uint8_t AnisoThreshold() const { return (fields[0] >> 16u) & 0x7u; } + [[nodiscard]] bool ForceSrgb() const { return ((fields[0] >> 20u) & 0x1u) == 1; } + [[nodiscard]] uint8_t AnisoBias() const { return (fields[0] >> 21u) & 0x3Fu; } + [[nodiscard]] bool TruncCoord() const { return ((fields[0] >> 27u) & 0x1u) == 1; } + [[nodiscard]] bool DisableCubeWrap() const { return ((fields[0] >> 28u) & 0x1u) == 1; } + [[nodiscard]] uint8_t FilterMode() const { return (fields[0] >> 29u) & 0x3u; } + [[nodiscard]] uint16_t MinLod() const { return (fields[1] >> 0u) & 0xFFFu; } + [[nodiscard]] uint16_t MaxLod() const { return (fields[1] >> 12u) & 0xFFFu; } + [[nodiscard]] uint8_t PerfMip() const { return (fields[1] >> 24u) & 0xFu; } + [[nodiscard]] uint8_t PerfZ() const { return (fields[1] >> 28u) & 0xFu; } + [[nodiscard]] uint16_t LodBias() const { return (fields[2] >> 0u) & 0x3FFFu; } + [[nodiscard]] uint8_t LodBiasSec() const { return (fields[2] >> 14u) & 0x3Fu; } + [[nodiscard]] uint8_t XyMagFilter() const { return (fields[2] >> 20u) & 0x3u; } + [[nodiscard]] uint8_t XyMinFilter() const { return (fields[2] >> 22u) & 0x3u; } + [[nodiscard]] uint8_t ZFilter() const { return (fields[2] >> 24u) & 0x3u; } + [[nodiscard]] uint8_t MipFilter() const { return (fields[2] >> 26u) & 0x3u; } + [[nodiscard]] uint16_t BorderColorPtr() const { return (fields[3] >> 0u) & 0xFFFu; } + [[nodiscard]] uint8_t BorderColorType() const { return (fields[3] >> 30u) & 0x3u; } + + [[nodiscard]] bool DisableDegamma() const { return ((fields[0] >> 31u) & 0x1u) == 1; } + [[nodiscard]] bool PointPreclamp() const { return ((fields[2] >> 28u) & 0x1u) == 1; } + [[nodiscard]] bool AnisoOverride() const { return ((fields[2] >> 29u) & 0x1u) == 1; } + [[nodiscard]] bool BlendZeroPrt() const { return ((fields[2] >> 30u) & 0x1u) == 1; } +}; + +struct ShaderVertexInputBuffer { + static constexpr int ATTR_MAX = 32; + + uint64_t addr = 0; + uint32_t stride = 0; + uint32_t num_records = 0; + uint32_t fetch_index = 0; + int attr_num = 0; + int attr_indices[ATTR_MAX] = {0}; + uint32_t attr_offsets[ATTR_MAX] = {0}; +}; + +struct ShaderVertexDestination { + int register_start = 0; + int registers_num = 0; + int attr_id = -1; + uint32_t fetch_index = 0; +}; + +enum class ShaderStorageUsage { + Unknown, + Constant, + ReadOnly, + ReadWrite, +}; + +ResourceDescriptorType ShaderClassifyResourceDescriptor(const uint32_t* desc); + +} // namespace Libs::Graphics + +#endif /* EMULATOR_INCLUDE_EMULATOR_GRAPHICS_SHADERBINDINGS_H_ */ diff --git a/src/graphics/shader/shaderStageRuntime.cpp b/src/graphics/shader/shaderStageRuntime.cpp new file mode 100644 index 000000000..3df5f4b1c --- /dev/null +++ b/src/graphics/shader/shaderStageRuntime.cpp @@ -0,0 +1,32 @@ +#include "graphics/shader/recompiler/ResourceMaterialization.h" +#include "graphics/shader/shader.h" + +#include + +namespace Libs::Graphics { + +bool ShaderMaterializeStageRuntime(std::shared_ptr program, + std::span user_data, uint64_t shader_base, + ShaderStageRuntime* stage, std::string* error) { + if (program == nullptr || stage == nullptr) { + if (error != nullptr) { + *error = + program == nullptr ? "missing native shader plan" : "missing shader stage output"; + } + return false; + } + ShaderRecompiler::IR::SrtRuntime runtime; + runtime.user_data = user_data; + runtime.shader_base = shader_base; + ShaderRecompiler::IR::ResourceSnapshot snapshot; + if (!ShaderRecompiler::IR::MaterializeResources(*program, runtime, &snapshot, error) || + !ShaderRecompiler::IR::ValidateResourceSpecialization(*program, snapshot, error)) { + return false; + } + auto resources = + std::make_shared(std::move(snapshot)); + *stage = {std::move(program), std::move(resources)}; + return true; +} + +} // namespace Libs::Graphics diff --git a/src/graphics/shader/shaderVertexMetadata.cpp b/src/graphics/shader/shaderVertexMetadata.cpp new file mode 100644 index 000000000..6f1f56b64 --- /dev/null +++ b/src/graphics/shader/shaderVertexMetadata.cpp @@ -0,0 +1,96 @@ +#include "graphics/shader/shaderVertexMetadata.h" + +#include "graphics/host_gpu/hostMemory.h" + +#include +#include + +namespace Libs::Graphics { + +namespace { + +bool Fail(std::string* error, const char* message) { + if (error != nullptr) { + *error = message; + } + return false; +} + +bool Readable(const void* data, uint64_t size) { + return data != nullptr && HostMemoryRangeIsReadable(reinterpret_cast(data), size); +} + +} // namespace + +bool ShaderReadVertexMetadata(const ShaderMappedData& data, uint32_t max_user_sgprs, + ShaderVertexMetadata* metadata, std::string* error) { + if (metadata == nullptr) { + return Fail(error, "missing vertex metadata output"); + } + if (!Readable(data.user_data, sizeof(ShaderUserData))) { + return Fail(error, "unreadable AGC user-data header"); + } + + ShaderUserData user_data {}; + std::memcpy(&user_data, data.user_data, sizeof(user_data)); + + constexpr uint32_t DirectResourceCount = + static_cast(AgcDirectResourceType::Last) + 1u; + if (user_data.direct_resource_count > DirectResourceCount) { + return Fail(error, "AGC direct-resource count exceeds the known resource domain"); + } + + std::array direct_offsets {}; + const auto direct_size = + static_cast(user_data.direct_resource_count) * sizeof(uint16_t); + if (direct_size != 0) { + if (!Readable(user_data.direct_resource_offset, direct_size)) { + return Fail(error, "unreadable AGC direct-resource offsets"); + } + std::memcpy(direct_offsets.data(), user_data.direct_resource_offset, direct_size); + } + + ShaderVertexMetadata next; + for (uint32_t type = 0; type < user_data.direct_resource_count; type++) { + const auto reg = direct_offsets[type]; + if (reg == AGC_ILLEGAL_DIRECT_OFFSET) { + continue; + } + switch (static_cast(type)) { + case AgcDirectResourceType::PtrVertexBufferTable: next.vertex_buffer_reg = reg; break; + case AgcDirectResourceType::PtrVertexAttribDescTable: + next.vertex_attrib_reg = reg; + break; + default: break; + } + } + + const bool embedded = next.vertex_buffer_reg >= 0 || next.vertex_attrib_reg >= 0; + if (!embedded) { + *metadata = next; + return true; + } + if (next.vertex_buffer_reg < 0 || next.vertex_attrib_reg < 0) { + return Fail(error, "vertex buffer and attribute tables must be supplied together"); + } + if (static_cast(next.vertex_buffer_reg) + 1u >= max_user_sgprs || + static_cast(next.vertex_attrib_reg) + 1u >= max_user_sgprs) { + return Fail(error, "vertex table pointer exceeds the user-SGPR domain"); + } + if (data.num_input_semantics == 0 || + data.num_input_semantics > ShaderVertexInputInfo::RES_MAX) { + return Fail(error, "vertex semantic count is outside the supported domain"); + } + + const auto semantic_size = + static_cast(data.num_input_semantics) * sizeof(ShaderSemantic); + if (!Readable(data.input_semantics, semantic_size)) { + return Fail(error, "unreadable vertex input semantics"); + } + std::memcpy(next.input_semantics.data(), data.input_semantics, semantic_size); + next.input_semantics_count = data.num_input_semantics; + *metadata = next; + return true; +} + +} // namespace Libs::Graphics diff --git a/src/graphics/shader/shaderVertexMetadata.h b/src/graphics/shader/shaderVertexMetadata.h new file mode 100644 index 000000000..122ca0954 --- /dev/null +++ b/src/graphics/shader/shaderVertexMetadata.h @@ -0,0 +1,24 @@ +#ifndef EMULATOR_INCLUDE_EMULATOR_GRAPHICS_SHADERVERTEXMETADATA_H_ +#define EMULATOR_INCLUDE_EMULATOR_GRAPHICS_SHADERVERTEXMETADATA_H_ + +#include "graphics/shader/shader.h" + +#include +#include + +namespace Libs::Graphics { + +struct ShaderVertexMetadata { + int vertex_buffer_reg = -1; + int vertex_attrib_reg = -1; + std::array input_semantics {}; + uint32_t input_semantics_count = 0; +}; + +// Copies the small AGC metadata subset used by the vertex path after validating every guest range. +bool ShaderReadVertexMetadata(const ShaderMappedData& data, uint32_t max_user_sgprs, + ShaderVertexMetadata* metadata, std::string* error); + +} // namespace Libs::Graphics + +#endif /* EMULATOR_INCLUDE_EMULATOR_GRAPHICS_SHADERVERTEXMETADATA_H_ */ diff --git a/src/kernel/eventFlag.cpp b/src/kernel/eventFlag.cpp new file mode 100644 index 000000000..594e7e148 --- /dev/null +++ b/src/kernel/eventFlag.cpp @@ -0,0 +1,380 @@ +#include "kernel/eventFlag.h" + +#include "common/assert.h" +#include "common/common.h" +#include "common/logging/log.h" +#include "common/stringUtils.h" +#include "common/threads.h" +#include "common/timer.h" +#include "libs/errno.h" +#include "libs/libs.h" + +namespace Libs::LibKernel::EventFlag { + +LIB_NAME("libkernel", "libkernel"); + +class KernelEventFlagPrivate { +public: + enum class Result { Ok, AlreadyWaiting, TimedOut, Canceled, Deleted }; + + enum class ClearMode { None, All, Bits }; + + enum class WaitMode { And, Or }; + + KernelEventFlagPrivate(const std::string& name, bool single, bool /*fifo*/, uint64_t bits) + : m_name(name), m_single_thread(single), m_bits(bits) {} + virtual ~KernelEventFlagPrivate(); + + KYTY_CLASS_NO_COPY(KernelEventFlagPrivate); + + void Set(uint64_t bits); + void Clear(uint64_t bits); + void Cancel(uint64_t bits, int* num_waiting_threads); + Result Wait(uint64_t bits, WaitMode wait_mode, ClearMode clear_mode, uint64_t* result, + uint32_t* ptr_micros); + + Result Poll(uint64_t bits, WaitMode wait_mode, ClearMode clear_mode, uint64_t* result) { + uint32_t micros = 0; + return Wait(bits, wait_mode, clear_mode, result, µs); + } + +private: + enum class Status { Set, Canceled, Deleted }; + + Common::Mutex m_mutex; + Common::CondVar m_cond_var; + Status m_status = Status::Set; + int m_waiting_threads = 0; + std::string m_name; + bool m_single_thread = false; + uint64_t m_bits = 0; +}; + +namespace { + +struct EventFlagWaitMode { + KernelEventFlagPrivate::WaitMode wait = KernelEventFlagPrivate::WaitMode::And; + KernelEventFlagPrivate::ClearMode clear = KernelEventFlagPrivate::ClearMode::None; +}; + +EventFlagWaitMode DecodeEventFlagWaitMode(uint32_t wait_mode) { + EventFlagWaitMode mode; + + switch (wait_mode & 0xfu) { + case 0x01: mode.wait = KernelEventFlagPrivate::WaitMode::And; break; + case 0x02: mode.wait = KernelEventFlagPrivate::WaitMode::Or; break; + default: EXIT("unknown mode: %u\n", wait_mode); + } + + switch (wait_mode & 0xf0u) { + case 0x00: mode.clear = KernelEventFlagPrivate::ClearMode::None; break; + case 0x10: mode.clear = KernelEventFlagPrivate::ClearMode::All; break; + case 0x20: mode.clear = KernelEventFlagPrivate::ClearMode::Bits; break; + default: EXIT("unknown mode: %u\n", wait_mode); + } + + return mode; +} + +} // namespace + +KernelEventFlagPrivate::~KernelEventFlagPrivate() { + Common::LockGuard lock(m_mutex); + + while (m_status != Status::Set) { + m_mutex.Unlock(); + Common::Thread::SleepMicro(10); + m_mutex.Lock(); + } + + m_status = Status::Deleted; + + m_cond_var.SignalAll(); + + while (m_waiting_threads > 0) { + m_mutex.Unlock(); + Common::Thread::SleepMicro(10); + m_mutex.Lock(); + } +} + +KernelEventFlagPrivate::Result KernelEventFlagPrivate::Wait(uint64_t bits, WaitMode wait_mode, + ClearMode clear_mode, uint64_t* result, + uint32_t* ptr_micros) { + Common::LockGuard lock(m_mutex); + + uint32_t micros = 0; + bool infinitely = true; + if (ptr_micros != nullptr) { + micros = *ptr_micros; + infinitely = false; + } + + uint32_t elapsed = 0; + Common::Timer t; + t.Start(); + + auto update_timeout = [&]() { + if (ptr_micros != nullptr) { + *ptr_micros = (elapsed >= micros ? 0 : micros - elapsed); + } + }; + + if (m_single_thread && m_waiting_threads > 0) { + return Result::AlreadyWaiting; + } + + while (!((wait_mode == WaitMode::And && (m_bits & bits) == bits) || + (wait_mode == WaitMode::Or && (m_bits & bits) != 0))) { + if ((elapsed >= micros && !infinitely)) { + if (result != nullptr) { + *result = m_bits; + } + update_timeout(); + return Result::TimedOut; + } + + m_waiting_threads++; + + if (infinitely) { + m_cond_var.Wait(&m_mutex); + } else { + m_cond_var.WaitFor(&m_mutex, micros - elapsed); + } + + m_waiting_threads--; + + elapsed = static_cast(t.GetTimeS() * 1000000.0); + + switch (m_status) { + case Status::Canceled: + if (result != nullptr) { + *result = m_bits; + } + update_timeout(); + return Result::Canceled; + case Status::Deleted: + if (result != nullptr) { + *result = m_bits; + } + update_timeout(); + return Result::Deleted; + case Status::Set: break; + } + } + + if (result != nullptr) { + *result = m_bits; + } + update_timeout(); + + switch (clear_mode) { + case ClearMode::All: m_bits = 0; break; + case ClearMode::Bits: m_bits &= ~bits; break; + case ClearMode::None: break; + } + + return Result::Ok; +} + +void KernelEventFlagPrivate::Set(uint64_t bits) { + Common::LockGuard lock(m_mutex); + + EXIT_NOT_IMPLEMENTED(m_status == Status::Deleted); + + while (m_status != Status::Set) { + m_mutex.Unlock(); + Common::Thread::SleepMicro(10); + m_mutex.Lock(); + } + + m_bits |= bits; + + m_cond_var.SignalAll(); +} + +void KernelEventFlagPrivate::Clear(uint64_t bits) { + Common::LockGuard lock(m_mutex); + + EXIT_NOT_IMPLEMENTED(m_status == Status::Deleted); + + while (m_status != Status::Set) { + m_mutex.Unlock(); + Common::Thread::SleepMicro(10); + m_mutex.Lock(); + } + + m_bits &= bits; +} + +void KernelEventFlagPrivate::Cancel(uint64_t bits, int* num_waiting_threads) { + Common::LockGuard lock(m_mutex); + + EXIT_NOT_IMPLEMENTED(m_status == Status::Deleted); + + while (m_status != Status::Set) { + m_mutex.Unlock(); + Common::Thread::SleepMicro(10); + m_mutex.Lock(); + } + + if (num_waiting_threads != nullptr) { + *num_waiting_threads = m_waiting_threads; + } + + m_status = Status::Canceled; + m_bits = bits; + + m_cond_var.SignalAll(); + + while (m_waiting_threads > 0) { + m_mutex.Unlock(); + Common::Thread::SleepMicro(10); + m_mutex.Lock(); + } + + m_status = Status::Set; +} + +int KYTY_SYSV_ABI KernelCreateEventFlag(KernelEventFlag* ef, const char* name, uint32_t attr, + uint64_t init_pattern, const void* param) { + PRINT_NAME(); + + if (ef == nullptr || name == nullptr) { + return KERNEL_ERROR_EINVAL; + } + + if (param != nullptr || (attr & ~0x33u) != 0) { + return KERNEL_ERROR_EINVAL; + } + + bool single = true; + bool fifo = true; + + switch (attr & 0x0fu) { + case 0x00: + case 0x01: fifo = true; break; + case 0x02: fifo = false; break; + default: return KERNEL_ERROR_EINVAL; + } + + switch (attr & 0xf0u) { + case 0x00: + case 0x10: single = true; break; + case 0x20: single = false; break; + default: return KERNEL_ERROR_EINVAL; + } + + *ef = new KernelEventFlagPrivate(std::string(name), single, fifo, init_pattern); + + LOGF("\tEventFlag create: %s\n", name); + + return OK; +} + +int KYTY_SYSV_ABI KernelDeleteEventFlag(KernelEventFlag ef) { + PRINT_NAME(); + + if (ef == nullptr) { + return KERNEL_ERROR_ESRCH; + } + + delete ef; + + return OK; +} + +int KYTY_SYSV_ABI KernelWaitEventFlag(KernelEventFlag ef, uint64_t bit_pattern, uint32_t wait_mode, + uint64_t* result_pat, KernelUseconds* timeout) { + PRINT_NAME(); + + if (ef == nullptr) { + return KERNEL_ERROR_ESRCH; + } + + if (bit_pattern == 0) { + return KERNEL_ERROR_EINVAL; + } + + const auto mode = DecodeEventFlagWaitMode(wait_mode); + auto result = ef->Wait(bit_pattern, mode.wait, mode.clear, result_pat, timeout); + + int ret = OK; + + switch (result) { + case KernelEventFlagPrivate::Result::Ok: ret = OK; break; + case KernelEventFlagPrivate::Result::AlreadyWaiting: ret = KERNEL_ERROR_EPERM; break; + case KernelEventFlagPrivate::Result::TimedOut: ret = KERNEL_ERROR_ETIMEDOUT; break; + case KernelEventFlagPrivate::Result::Canceled: ret = KERNEL_ERROR_ECANCELED; break; + case KernelEventFlagPrivate::Result::Deleted: ret = KERNEL_ERROR_EACCES; break; + } + + return ret; +} + +int KYTY_SYSV_ABI KernelPollEventFlag(KernelEventFlag ef, uint64_t bit_pattern, uint32_t wait_mode, + uint64_t* result_pat) { + PRINT_NAME(); + + if (ef == nullptr) { + return KERNEL_ERROR_ESRCH; + } + + if (bit_pattern == 0) { + return KERNEL_ERROR_EINVAL; + } + + const auto mode = DecodeEventFlagWaitMode(wait_mode); + auto result = ef->Poll(bit_pattern, mode.wait, mode.clear, result_pat); + + int ret = OK; + + switch (result) { + case KernelEventFlagPrivate::Result::Ok: ret = OK; break; + case KernelEventFlagPrivate::Result::AlreadyWaiting: ret = KERNEL_ERROR_EPERM; break; + case KernelEventFlagPrivate::Result::TimedOut: + case KernelEventFlagPrivate::Result::Canceled: + case KernelEventFlagPrivate::Result::Deleted: ret = KERNEL_ERROR_EBUSY; break; + } + + return ret; +} + +int KYTY_SYSV_ABI KernelSetEventFlag(KernelEventFlag ef, uint64_t bit_pattern) { + PRINT_NAME(); + + if (ef == nullptr) { + return KERNEL_ERROR_ESRCH; + } + + ef->Set(bit_pattern); + + return OK; +} + +int KYTY_SYSV_ABI KernelClearEventFlag(KernelEventFlag ef, uint64_t bit_pattern) { + PRINT_NAME(); + + if (ef == nullptr) { + return KERNEL_ERROR_ESRCH; + } + + ef->Clear(bit_pattern); + + return OK; +} + +int KYTY_SYSV_ABI KernelCancelEventFlag(KernelEventFlag ef, uint64_t set_pattern, + int* num_wait_threads) { + PRINT_NAME(); + + if (ef == nullptr) { + return KERNEL_ERROR_ESRCH; + } + + ef->Cancel(set_pattern, num_wait_threads); + + return OK; +} + +} // namespace Libs::LibKernel::EventFlag diff --git a/src/kernel/eventFlag.h b/src/kernel/eventFlag.h new file mode 100644 index 000000000..99f789beb --- /dev/null +++ b/src/kernel/eventFlag.h @@ -0,0 +1,28 @@ +#ifndef EMULATOR_INCLUDE_EMULATOR_KERNEL_EVENTFLAG_H_ +#define EMULATOR_INCLUDE_EMULATOR_KERNEL_EVENTFLAG_H_ + +#include "common/abi.h" +#include "common/common.h" +#include "kernel/pthread.h" + +namespace Libs::LibKernel::EventFlag { + +class KernelEventFlagPrivate; + +using KernelEventFlag = KernelEventFlagPrivate*; + +int KYTY_SYSV_ABI KernelCreateEventFlag(KernelEventFlag* ef, const char* name, uint32_t attr, + uint64_t init_pattern, const void* param); +int KYTY_SYSV_ABI KernelDeleteEventFlag(KernelEventFlag ef); +int KYTY_SYSV_ABI KernelWaitEventFlag(KernelEventFlag ef, uint64_t bit_pattern, uint32_t wait_mode, + uint64_t* result_pat, KernelUseconds* timeout); +int KYTY_SYSV_ABI KernelPollEventFlag(KernelEventFlag ef, uint64_t bit_pattern, uint32_t wait_mode, + uint64_t* result_pat); +int KYTY_SYSV_ABI KernelSetEventFlag(KernelEventFlag ef, uint64_t bit_pattern); +int KYTY_SYSV_ABI KernelClearEventFlag(KernelEventFlag ef, uint64_t bit_pattern); +int KYTY_SYSV_ABI KernelCancelEventFlag(KernelEventFlag ef, uint64_t set_pattern, + int* num_wait_threads); + +} // namespace Libs::LibKernel::EventFlag + +#endif /* EMULATOR_INCLUDE_EMULATOR_KERNEL_EVENTFLAG_H_ */ diff --git a/src/kernel/eventQueue.cpp b/src/kernel/eventQueue.cpp new file mode 100644 index 000000000..e27d0457c --- /dev/null +++ b/src/kernel/eventQueue.cpp @@ -0,0 +1,621 @@ +#include "kernel/eventQueue.h" + +#include "common/assert.h" +#include "common/common.h" +#include "common/logging/log.h" +#include "common/stringUtils.h" +#include "common/threads.h" +#include "common/timer.h" +#include "libs/errno.h" +#include "libs/libs.h" + +#include +#include +#include +#include + +namespace Libs::LibKernel::EventQueue { + +LIB_NAME("libkernel", "libkernel"); + +constexpr uint16_t EV_ADD = 0x01; +constexpr uint16_t EV_ONESHOT = 0x10; +constexpr uint16_t EV_CLEAR = 0x20; +constexpr uint16_t EV_ERROR = 0x4000; + +static uint64_t MonotonicTimeNs() { + return static_cast(std::chrono::duration_cast( + std::chrono::steady_clock::now().time_since_epoch()) + .count()); +} + +static std::list g_equeues; +static Common::Mutex g_equeues_mutex; + +class KernelEqueuePrivate { +public: + KernelEqueuePrivate() = default; + virtual ~KernelEqueuePrivate(); + + KYTY_CLASS_NO_COPY(KernelEqueuePrivate); + + [[nodiscard]] const std::string& GetName() const { return m_name; } + void SetName(const std::string& m_name) { this->m_name = m_name; } + + void AddEvent(const KernelEqueueEvent& event); + bool TriggerEvent(uintptr_t ident, int16_t filter, void* trigger_data); + bool DeleteEvent(uintptr_t ident, int16_t filter); + + int GetTriggeredEvents(KernelEvent* ev, int num); + int WaitForEvents(KernelEvent* ev, int num, uint32_t micros); + +private: + void TriggerExpiredTimers(uint64_t now_ns); + bool GetNextTimerWaitMicros(uint64_t now_ns, uint32_t* wait_micros) const; + + std::list m_events; + Common::Mutex m_mutex; + Common::CondVar m_cond_var; + std::string m_name; +}; + +KernelEqueuePrivate::~KernelEqueuePrivate() { + Common::LockGuard lock(m_mutex); + + for (auto& event: m_events) { + if (event.filter.delete_event_func != nullptr) { + event.filter.delete_event_func(this, &event); + } + } +} + +int KernelEqueuePrivate::GetTriggeredEvents(KernelEvent* ev, int num) { + Common::LockGuard lock(m_mutex); + + EXIT_IF(num < 1); + + TriggerExpiredTimers(MonotonicTimeNs()); + + int ret = 0; + + for (auto it = m_events.begin(); it != m_events.end();) { + auto& event = *it; + bool erase = false; + while (event.triggered) { + ev[ret++] = event.event; + if ((event.event.flags & EV_ONESHOT) != 0) { + erase = true; + break; + } + + if (event.filter.reset_func != nullptr) { + event.filter.reset_func(&event); + } else if ((event.event.flags & EV_CLEAR) != 0) { + event.triggered = false; + event.event.fflags = 0; + event.event.data = 0; + } + + if (!event.pending_events.empty()) { + event.event = event.pending_events.front(); + event.pending_events.pop_front(); + event.triggered = true; + } + + if (ret >= num) { + break; + } + } + it = (erase ? m_events.erase(it) : std::next(it)); + if (ret >= num) { + break; + } + } + + return ret; +} + +void KernelEqueuePrivate::TriggerExpiredTimers(uint64_t now_ns) { + for (auto& event: m_events) { + if (!event.triggered && event.deadline_ns != 0 && event.deadline_ns <= now_ns) { + event.triggered = true; + } + } +} + +bool KernelEqueuePrivate::GetNextTimerWaitMicros(uint64_t now_ns, uint32_t* wait_micros) const { + EXIT_IF(wait_micros == nullptr); + + uint64_t nearest_deadline = UINT64_MAX; + for (const auto& event: m_events) { + if (!event.triggered && event.deadline_ns != 0) { + nearest_deadline = std::min(nearest_deadline, event.deadline_ns); + } + } + if (nearest_deadline == UINT64_MAX) { + return false; + } + + const auto remaining_ns = nearest_deadline > now_ns ? nearest_deadline - now_ns : 0; + const auto rounded_us = std::max(1, (remaining_ns + 999u) / 1000u); + *wait_micros = static_cast(std::min(rounded_us, UINT32_MAX)); + return true; +} + +int KernelEqueuePrivate::WaitForEvents(KernelEvent* ev, int num, uint32_t micros) { + Common::LockGuard lock(m_mutex); + + EXIT_IF(num < 1); + + uint32_t elapsed = 0; + Common::Timer t; + t.Start(); + + for (;;) { + int ret = GetTriggeredEvents(ev, num); + + if (ret > 0 || (elapsed >= micros && micros != 0)) { + return ret; + } + + uint32_t timer_wait = 0; + const bool has_timer = GetNextTimerWaitMicros(MonotonicTimeNs(), &timer_wait); + if (micros == 0 && !has_timer) { + m_cond_var.Wait(&m_mutex); + } else { + const auto external_wait = micros != 0 ? micros - elapsed : UINT32_MAX; + m_cond_var.WaitFor(&m_mutex, + has_timer ? std::min(external_wait, timer_wait) : external_wait); + } + + elapsed = static_cast(t.GetTimeS() * 1000000.0); + } + + return 0; +} + +void KernelEqueuePrivate::AddEvent(const KernelEqueueEvent& event) { + Common::LockGuard lock(m_mutex); + + auto it = std::find_if(m_events.begin(), m_events.end(), + [ident = event.event.ident, filter = event.event.filter](const auto& e) { + return e.event.ident == ident && e.event.filter == filter; + }); + if (it != m_events.end()) { + *it = event; + } else { + m_events.push_back(event); + } + + m_cond_var.Signal(); +} + +bool KernelEqueuePrivate::TriggerEvent(uintptr_t ident, int16_t filter, void* trigger_data) { + Common::LockGuard lock(m_mutex); + + auto it = std::find_if(m_events.begin(), m_events.end(), [ident, filter](const auto& e) { + return e.event.ident == ident && e.event.filter == filter; + }); + if (it != m_events.end()) { + auto& event = *it; + + if (event.filter.trigger_func != nullptr) { + event.filter.trigger_func(&event, trigger_data); + } else { + event.triggered = true; + } + + m_cond_var.Signal(); + + return true; + } + + return false; +} + +static void UserEventTriggerFunc(KernelEqueueEvent* event, void* trigger_data) { + EXIT_IF(event == nullptr); + event->triggered = true; + event->event.data = reinterpret_cast(trigger_data); + event->event.udata = trigger_data; +} + +static void AmprEventTriggerFunc(KernelEqueueEvent* event, void* trigger_data) { + EXIT_IF(event == nullptr); + auto triggered_event = event->event; + triggered_event.data = static_cast(reinterpret_cast(trigger_data)); + if (event->triggered) { + event->pending_events.push_back(triggered_event); + } else { + event->event = triggered_event; + event->triggered = true; + } +} + +static void UserEventResetFunc(KernelEqueueEvent* event) { + EXIT_IF(event == nullptr); + if ((event->event.flags & EV_CLEAR) != 0) { + event->triggered = false; + event->event.fflags = 0; + event->event.data = 0; + } +} + +bool KernelEqueuePrivate::DeleteEvent(uintptr_t ident, int16_t filter) { + Common::LockGuard lock(m_mutex); + + auto it = std::find_if(m_events.begin(), m_events.end(), [ident, filter](const auto& e) { + return e.event.ident == ident && e.event.filter == filter; + }); + if (it != m_events.end()) { + auto& event = *it; + + if (event.filter.delete_event_func != nullptr) { + event.filter.delete_event_func(this, &event); + } + + m_events.erase(it); + + return true; + } + + return false; +} + +int KYTY_SYSV_ABI KernelCreateEqueue(KernelEqueue* eq, const char* name) { + PRINT_NAME(); + + if (eq == nullptr || name == nullptr) { + return KERNEL_ERROR_EINVAL; + } + + *eq = new KernelEqueuePrivate; + + (*eq)->SetName(std::string(name)); + + { + Common::LockGuard lock(g_equeues_mutex); + g_equeues.push_back(*eq); + } + + LOGF("\tEqueue create: %s\n", name); + + return OK; +} + +int KYTY_SYSV_ABI KernelAddEvent(KernelEqueue eq, const KernelEqueueEvent& event) { + if (eq == nullptr) { + return KERNEL_ERROR_EBADF; + } + + eq->AddEvent(event); + + return OK; +} + +int KYTY_SYSV_ABI KernelTriggerEvent(KernelEqueue eq, uintptr_t ident, int16_t filter, + void* trigger_data) { + if (eq == nullptr) { + return KERNEL_ERROR_EBADF; + } + + if (!eq->TriggerEvent(ident, filter, trigger_data)) { + return KERNEL_ERROR_ENOENT; + } + + return OK; +} + +int KYTY_SYSV_ABI KernelDeleteEvent(KernelEqueue eq, uintptr_t ident, int16_t filter) { + if (eq == nullptr) { + return KERNEL_ERROR_EBADF; + } + + if (!eq->DeleteEvent(ident, filter)) { + return KERNEL_ERROR_ENOENT; + } + + return OK; +} + +int KYTY_SYSV_ABI KernelDeleteEqueue(KernelEqueue eq) { + PRINT_NAME(); + + if (eq == nullptr) { + return KERNEL_ERROR_EBADF; + } + + LOGF("\tEqueue delete: %s\n", eq->GetName().c_str()); + + { + Common::LockGuard lock(g_equeues_mutex); + g_equeues.remove(eq); + } + + delete eq; + + return OK; +} + +int KYTY_SYSV_ABI KernelWaitEqueue(KernelEqueue eq, KernelEvent* ev, int num, int* out, + const KernelUseconds* timo) { + PRINT_NAME(); + + if (eq == nullptr) { + return KERNEL_ERROR_EBADF; + } + + if (ev == nullptr) { + return KERNEL_ERROR_EFAULT; + } + + if (num < 1) { + return KERNEL_ERROR_EINVAL; + } + + EXIT_NOT_IMPLEMENTED(out == nullptr); + + LOGF("\tEqueue wait: %s, caller = 0x%016" PRIx64 ", eq = 0x%016" PRIx64 ", ev = 0x%016" PRIx64 + ", num = %d, timo = %s, thread_id = %d\n", + eq->GetName().c_str(), reinterpret_cast(__builtin_return_address(0)), + reinterpret_cast(eq), reinterpret_cast(ev), num, + (timo == nullptr ? "inf" : fmt::format("{}", *timo).c_str()), + Common::Thread::GetThreadIdUnique()); + + if (timo == nullptr) { + *out = eq->WaitForEvents(ev, num, 0); + } + + if (timo != nullptr) { + if (*timo == 0) { + *out = eq->GetTriggeredEvents(ev, num); + } else { + *out = eq->WaitForEvents(ev, num, *timo); + } + } + + if (*out == 0) { + LOGF("\tEqueue wait timedout: %s\n", eq->GetName().c_str()); + return KERNEL_ERROR_ETIMEDOUT; + } + + LOGF("\tEqueue wait received %u events: ident = 0x%016" PRIx64 + ", filter = %d, flags = 0x%04" PRIx16 ", fflags = 0x%08" PRIx32 ", data = 0x%016" PRIx64 + ", udata = 0x%016" PRIx64 "\n", + *out, static_cast(ev[0].ident), ev[0].filter, ev[0].flags, ev[0].fflags, + static_cast(ev[0].data), reinterpret_cast(ev[0].udata)); + + return OK; +} + +int KYTY_SYSV_ABI KernelAddUserEvent(KernelEqueue eq, int id) { + PRINT_NAME(); + + LOGF("\t user event add: eq = 0x%016" PRIx64 ", id = %d\n", reinterpret_cast(eq), id); + + if (eq == nullptr) { + return KERNEL_ERROR_EBADF; + } + + KernelEqueueEvent event {}; + event.event.ident = static_cast(id); + event.event.filter = KERNEL_EVFILT_USER; + event.event.flags = EV_ADD; + event.event.fflags = 0; + event.event.data = 0; + event.event.udata = nullptr; + event.filter.trigger_func = UserEventTriggerFunc; + event.filter.reset_func = UserEventResetFunc; + + eq->AddEvent(event); + + return OK; +} + +int KYTY_SYSV_ABI KernelAddUserEventEdge(KernelEqueue eq, int id) { + PRINT_NAME(); + + LOGF("\t user event edge add: eq = 0x%016" PRIx64 ", id = %d\n", reinterpret_cast(eq), + id); + + if (eq == nullptr) { + return KERNEL_ERROR_EBADF; + } + + KernelEqueueEvent event {}; + event.event.ident = static_cast(id); + event.event.filter = KERNEL_EVFILT_USER; + event.event.flags = EV_ADD | EV_CLEAR; + event.event.fflags = 0; + event.event.data = 0; + event.event.udata = nullptr; + event.filter.trigger_func = UserEventTriggerFunc; + event.filter.reset_func = UserEventResetFunc; + + eq->AddEvent(event); + + return OK; +} + +int KYTY_SYSV_ABI KernelTriggerUserEvent(KernelEqueue eq, int id, void* udata) { + PRINT_NAME(); + + LOGF("\t user event trigger: eq = 0x%016" PRIx64 ", id = %d, udata = 0x%016" PRIx64 "\n", + reinterpret_cast(eq), id, reinterpret_cast(udata)); + + return KernelTriggerEvent(eq, static_cast(id), KERNEL_EVFILT_USER, udata); +} + +int KYTY_SYSV_ABI KernelTriggerUserEventForAll(int id, void* udata) { + int triggered = 0; + + Common::LockGuard lock(g_equeues_mutex); + + for (auto* eq: g_equeues) { + if (eq != nullptr && + eq->TriggerEvent(static_cast(id), KERNEL_EVFILT_USER, udata)) { + triggered++; + } + } + + return (triggered > 0 ? OK : KERNEL_ERROR_ENOENT); +} + +int KYTY_SYSV_ABI KernelDeleteUserEvent(KernelEqueue eq, int id) { + PRINT_NAME(); + + LOGF("\t user event delete: eq = 0x%016" PRIx64 ", id = %d\n", reinterpret_cast(eq), + id); + + return KernelDeleteEvent(eq, static_cast(id), KERNEL_EVFILT_USER); +} + +int KYTY_SYSV_ABI KernelAddHRTimerEvent(KernelEqueue eq, int id, const KernelTimespec* ts, + void* udata) { + if (eq == nullptr) { + return KERNEL_ERROR_EBADF; + } + if (ts == nullptr) { + return KERNEL_ERROR_EFAULT; + } + if (ts->tv_sec < 0 || ts->tv_nsec < 0 || ts->tv_nsec >= 1000000000ll || + static_cast(ts->tv_sec) > + (UINT64_MAX - static_cast(ts->tv_nsec)) / 1000000000ull) { + return KERNEL_ERROR_EINVAL; + } + + const auto delay_ns = + static_cast(ts->tv_sec) * 1000000000ull + static_cast(ts->tv_nsec); + const auto now_ns = MonotonicTimeNs(); + + KernelEqueueEvent event {}; + event.deadline_ns = delay_ns <= UINT64_MAX - now_ns ? now_ns + delay_ns : UINT64_MAX; + event.event.ident = static_cast(id); + event.event.filter = KERNEL_EVFILT_HRTIMER; + event.event.flags = EV_ADD | EV_ONESHOT; + event.event.fflags = 0; + event.event.data = 0; + event.event.udata = udata; + eq->AddEvent(event); + + return OK; +} + +int KYTY_SYSV_ABI KernelDeleteHRTimerEvent(KernelEqueue eq, int id) { + return KernelDeleteEvent(eq, static_cast(id), KERNEL_EVFILT_HRTIMER); +} + +int KYTY_SYSV_ABI KernelAddAmprEvent(KernelEqueue eq, int id, void* udata) { + PRINT_NAME(); + + LOGF("\t AMPR event add: eq = 0x%016" PRIx64 ", id = %d, udata = 0x%016" PRIx64 "\n", + reinterpret_cast(eq), id, reinterpret_cast(udata)); + + if (eq != nullptr) { + KernelEqueueEvent event {}; + event.event.ident = static_cast(id); + event.event.filter = KERNEL_EVFILT_USER; + event.event.flags = EV_ADD | EV_CLEAR; + event.event.fflags = 0; + event.event.data = 0; + event.event.udata = udata; + event.filter.trigger_func = AmprEventTriggerFunc; + event.filter.reset_func = UserEventResetFunc; + eq->AddEvent(event); + } + + return OK; +} + +int KYTY_SYSV_ABI KernelAddAmprSystemEvent(KernelEqueue eq, int id, void* udata) { + PRINT_NAME(); + + LOGF("\t AMPR system event add: eq = 0x%016" PRIx64 ", id = %d, udata = 0x%016" PRIx64 "\n", + reinterpret_cast(eq), id, reinterpret_cast(udata)); + + return KernelAddAmprEvent(eq, id, udata); +} + +int KYTY_SYSV_ABI KernelDeleteAmprEvent(KernelEqueue eq, int id) { + PRINT_NAME(); + + LOGF("\t AMPR event delete: eq = 0x%016" PRIx64 ", id = %d\n", reinterpret_cast(eq), + id); + + if (eq != nullptr) { + eq->DeleteEvent(static_cast(id), KERNEL_EVFILT_USER); + } + + return OK; +} + +int KYTY_SYSV_ABI KernelDeleteAmprSystemEvent(KernelEqueue eq, int id) { + PRINT_NAME(); + + LOGF("\t AMPR system event delete: eq = 0x%016" PRIx64 ", id = %d\n", + reinterpret_cast(eq), id); + + return KernelDeleteAmprEvent(eq, id); +} + +intptr_t KYTY_SYSV_ABI KernelGetEventData(const KernelEvent* ev) { + PRINT_NAME(); + + if (ev != nullptr) { + return ev->data; + } + + return 0; +} + +intptr_t KYTY_SYSV_ABI KernelGetEventFflags(const KernelEvent* ev) { + PRINT_NAME(); + + if (ev != nullptr) { + return ev->fflags; + } + + return 0; +} + +int KYTY_SYSV_ABI KernelGetEventFilter(const KernelEvent* ev) { + PRINT_NAME(); + + if (ev != nullptr) { + return ev->filter; + } + + return 0; +} + +uintptr_t KYTY_SYSV_ABI KernelGetEventId(const KernelEvent* ev) { + PRINT_NAME(); + + if (ev != nullptr) { + return ev->ident; + } + + return 0; +} + +void* KYTY_SYSV_ABI KernelGetEventUserData(const KernelEvent* ev) { + PRINT_NAME(); + + if (ev != nullptr) { + return ev->udata; + } + + return nullptr; +} + +int KYTY_SYSV_ABI KernelGetEventError(const KernelEvent* ev) { + PRINT_NAME(); + + if (ev != nullptr && (ev->flags & EV_ERROR) != 0) { + return static_cast(ev->data); + } + + return 0; +} + +} // namespace Libs::LibKernel::EventQueue diff --git a/src/kernel/eventQueue.h b/src/kernel/eventQueue.h new file mode 100644 index 000000000..564d22762 --- /dev/null +++ b/src/kernel/eventQueue.h @@ -0,0 +1,85 @@ +#ifndef EMULATOR_INCLUDE_EMULATOR_KERNEL_EVENTQUEUE_H_ +#define EMULATOR_INCLUDE_EMULATOR_KERNEL_EVENTQUEUE_H_ + +#include "common/abi.h" +#include "common/common.h" +#include "kernel/pthread.h" + +#include + +namespace Libs::LibKernel::EventQueue { + +constexpr int16_t KERNEL_EVFILT_TIMER = -7; +constexpr int16_t KERNEL_EVFILT_READ = -1; +constexpr int16_t KERNEL_EVFILT_WRITE = -2; +constexpr int16_t KERNEL_EVFILT_USER = -11; +constexpr int16_t KERNEL_EVFILT_FILE = -4; +constexpr int16_t KERNEL_EVFILT_GRAPHICS = -14; +constexpr int16_t KERNEL_EVFILT_VIDEO_OUT = -13; +constexpr int16_t KERNEL_EVFILT_HRTIMER = -15; + +class KernelEqueuePrivate; +struct KernelEqueueEvent; + +using KernelEqueue = KernelEqueuePrivate*; + +using trigger_func_t = void (*)(KernelEqueueEvent* event, void* trigger_data); +using reset_func_t = void (*)(KernelEqueueEvent* event); +using delete_func_t = void (*)(KernelEqueue eq, KernelEqueueEvent* event); + +struct KernelEvent { + uintptr_t ident = 0; + int16_t filter = 0; + uint16_t flags = 0; + uint32_t fflags = 0; + intptr_t data = 0; + void* udata = nullptr; +}; + +struct KernelFilter { + void* data = nullptr; + trigger_func_t trigger_func = nullptr; + reset_func_t reset_func = nullptr; + delete_func_t delete_event_func = nullptr; +}; + +struct KernelEqueueEvent { + bool triggered = false; + uint64_t deadline_ns = 0; + KernelEvent event; + KernelFilter filter; + std::deque pending_events; +}; + +int KYTY_SYSV_ABI KernelAddEvent(KernelEqueue eq, const KernelEqueueEvent& event); +int KYTY_SYSV_ABI KernelTriggerEvent(KernelEqueue eq, uintptr_t ident, int16_t filter, + void* trigger_data); +int KYTY_SYSV_ABI KernelDeleteEvent(KernelEqueue eq, uintptr_t ident, int16_t filter); + +int KYTY_SYSV_ABI KernelCreateEqueue(KernelEqueue* eq, const char* name); +int KYTY_SYSV_ABI KernelDeleteEqueue(KernelEqueue eq); +int KYTY_SYSV_ABI KernelWaitEqueue(KernelEqueue eq, KernelEvent* ev, int num, int* out, + const KernelUseconds* timo); +int KYTY_SYSV_ABI KernelAddUserEvent(KernelEqueue eq, int id); +int KYTY_SYSV_ABI KernelAddUserEventEdge(KernelEqueue eq, int id); +int KYTY_SYSV_ABI KernelTriggerUserEvent(KernelEqueue eq, int id, void* udata); +int KYTY_SYSV_ABI KernelTriggerUserEventForAll(int id, void* udata); +int KYTY_SYSV_ABI KernelDeleteUserEvent(KernelEqueue eq, int id); +int KYTY_SYSV_ABI KernelAddHRTimerEvent(KernelEqueue eq, int id, const KernelTimespec* ts, + void* udata); +int KYTY_SYSV_ABI KernelDeleteHRTimerEvent(KernelEqueue eq, int id); +int KYTY_SYSV_ABI KernelAddAmprEvent(KernelEqueue eq, int id, void* udata); +int KYTY_SYSV_ABI KernelAddAmprSystemEvent(KernelEqueue eq, int id, void* udata); +int KYTY_SYSV_ABI KernelDeleteAmprEvent(KernelEqueue eq, int id); +int KYTY_SYSV_ABI KernelDeleteAmprSystemEvent(KernelEqueue eq, int id); + +intptr_t KYTY_SYSV_ABI KernelGetEventData(const KernelEvent* ev); +intptr_t KYTY_SYSV_ABI KernelGetEventFflags(const KernelEvent* ev); +int KYTY_SYSV_ABI KernelGetEventFilter(const KernelEvent* ev); +uintptr_t KYTY_SYSV_ABI KernelGetEventId(const KernelEvent* ev); +void* KYTY_SYSV_ABI KernelGetEventUserData(const KernelEvent* ev); +int KYTY_SYSV_ABI KernelGetEventError(const KernelEvent* ev); + +} // namespace Libs::LibKernel::EventQueue + +#endif /* EMULATOR_INCLUDE_EMULATOR_KERNEL_EVENTQUEUE_H_ */ diff --git a/src/kernel/fileSystem.cpp b/src/kernel/fileSystem.cpp new file mode 100644 index 000000000..1e17cf114 --- /dev/null +++ b/src/kernel/fileSystem.cpp @@ -0,0 +1,1161 @@ +#include "kernel/fileSystem.h" + +#include "common/assert.h" +#include "common/common.h" +#include "common/dateTime.h" +#include "common/emulatorConfig.h" +#include "common/file.h" +#include "common/hash.h" +#include "common/logging/log.h" +#include "common/stringUtils.h" +#include "common/threads.h" +#include "libs/errno.h" +#include "libs/libs.h" +#include "libs/network.h" + +#include +#include +#include +#include +#include +#include +#include +#include + +namespace Libs::LibKernel::FileSystem { + +LIB_NAME("libkernel", "libkernel"); + +constexpr int DESCRIPTOR_MIN = 3; + +enum class SpecialFile { + None, + Random, +}; + +class MountPoints { +public: + struct MountPair { + std::filesystem::path dir; + std::string point; + }; + + MountPoints() { EXIT_NOT_IMPLEMENTED(!Common::Thread::IsMainThread()); } + virtual ~MountPoints() { KYTY_NOT_IMPLEMENTED; } + + KYTY_CLASS_NO_COPY(MountPoints); + + void Mount(const std::filesystem::path& folder, const std::string& point); + void Umount(const std::string& folder_or_point); + + [[nodiscard]] std::filesystem::path GetRealFilename(const std::string& mounted_file_name); + [[nodiscard]] std::filesystem::path GetRealDirectory(const std::string& mounted_directory); + +private: + std::vector m_mount_pairs; + Common::Mutex m_mutex; +}; + +struct File { + Common::File f; + std::string name; + std::filesystem::path real_name; + std::atomic_bool opened; + std::atomic_bool directory; + std::atomic_bool append; + std::atomic_bool sync_writes; + SpecialFile special; + Common::Mutex mutex; + std::vector dents; + uint64_t dents_offset; +}; + +class FileDescriptors { +public: + FileDescriptors() { EXIT_NOT_IMPLEMENTED(!Common::Thread::IsMainThread()); } + virtual ~FileDescriptors() { KYTY_NOT_IMPLEMENTED; } + + KYTY_CLASS_NO_COPY(FileDescriptors); + + int CreateDescriptor(); + void DeleteDescriptor(int d); + File* GetFile(int d); + File* GetFile(const std::filesystem::path& real_name); + void CloseAll(); + +private: + std::vector m_files; + Common::Mutex m_mutex; +}; + +static MountPoints* g_mount_points = nullptr; +static FileDescriptors* g_files = nullptr; + +static bool IsRandomDevice(const std::string& path) { + return path == "/dev/random" || path == "/dev/urandom"; +} + +static void FillRandomBuffer(void* buf, size_t nbytes) { + auto* out = reinterpret_cast(buf); + std::random_device random; + + for (size_t i = 0; i < nbytes; i++) { + out[i] = static_cast(random()); + } +} + +static void SecToTimespec(KernelTimespec* ts, double sec) { + ts->tv_sec = static_cast(sec); + ts->tv_nsec = static_cast((sec - static_cast(ts->tv_sec)) * 1000000000.0); +} + +static uint64_t AlignDown(uint64_t value, uint64_t alignment) { + return (alignment != 0 ? value & ~(alignment - 1) : value); +} + +static uint64_t AlignUp(uint64_t value, uint64_t alignment) { + return (alignment != 0 ? (value + alignment - 1) & ~(alignment - 1) : value); +} + +static int PosixToKernel(int posix_errno) { + return KERNEL_ERROR_UNKNOWN + posix_errno; +} + +int FileDescriptors::CreateDescriptor() { + Common::LockGuard lock(m_mutex); + + auto* file = new File {}; + file->opened = false; + file->directory = false; + file->append = false; + file->sync_writes = false; + file->special = SpecialFile::None; + + int files_num = static_cast(m_files.size()); + for (int index = 0; index < files_num; index++) { + if (m_files[index] == nullptr) { + m_files[index] = file; + return index + DESCRIPTOR_MIN; + } + } + + m_files.push_back(file); + return static_cast(m_files.size()) + DESCRIPTOR_MIN - 1; +} + +void FileDescriptors::DeleteDescriptor(int d) { + Common::LockGuard lock(m_mutex); + + auto index = static_cast(d - DESCRIPTOR_MIN); + + EXIT_IF(index >= m_files.size()); + EXIT_IF(m_files[index] == nullptr); + EXIT_IF(m_files[index]->opened); + + delete m_files[index]; + m_files[index] = nullptr; +} + +File* FileDescriptors::GetFile(int d) { + Common::LockGuard lock(m_mutex); + + auto index = static_cast(d - DESCRIPTOR_MIN); + + if (index >= m_files.size()) { + return nullptr; + } + + return m_files[index]; +} + +File* FileDescriptors::GetFile(const std::filesystem::path& real_name) { + Common::LockGuard lock(m_mutex); + + for (auto* f: m_files) { + if (f != nullptr && f->real_name == real_name) { + return f; + } + } + + return nullptr; +} + +void FileDescriptors::CloseAll() { + Common::LockGuard lock(m_mutex); + + for (auto& f: m_files) { + if (f != nullptr && f->opened) { + f->f.Close(); + delete f; + f = nullptr; + } + } +} + +void MountPoints::Mount(const std::filesystem::path& folder, const std::string& point) { + Common::LockGuard lock(m_mutex); + + auto folder_str = Common::FixDirectorySlash(Common::PathToGenericString(folder)); + auto point_str = Common::FixDirectorySlash(point); + + Umount(point_str); + + MountPair p; + p.dir = folder_str; + p.point = point_str; + + m_mount_pairs.push_back(p); +} + +void MountPoints::Umount(const std::string& folder_or_point) { + Common::LockGuard lock(m_mutex); + + auto folder_or_point_str = Common::FixDirectorySlash(folder_or_point); + + const auto it = std::find_if( + m_mount_pairs.begin(), m_mount_pairs.end(), [&folder_or_point_str](const MountPair& p) { + return Common::PathToGenericString(p.dir) == folder_or_point_str || + p.point == folder_or_point_str; + }); + if (it != m_mount_pairs.end()) { + m_mount_pairs.erase(it); + } +} + +std::filesystem::path MountPoints::GetRealFilename(const std::string& mounted_file_name) { + Common::LockGuard lock(m_mutex); + + auto mounted_path = + Common::DirectoryWithoutFilename(Common::FixFilenameSlash(mounted_file_name)); + + const auto it = std::find_if( + m_mount_pairs.begin(), m_mount_pairs.end(), + [&mounted_path](const MountPair& p) { return Common::StartsWith(mounted_path, p.point); }); + if (it != m_mount_pairs.end()) { + const auto& p = *it; + auto rel_path = + Common::RemoveFirst(Common::FixFilenameSlash(mounted_file_name), p.point.size()); + while (Common::StartsWith(rel_path, '/')) { + rel_path = Common::RemoveFirst(rel_path, 1); + } + return p.dir / rel_path; + } + + return mounted_file_name; +} + +std::filesystem::path MountPoints::GetRealDirectory(const std::string& mounted_directory) { + Common::LockGuard lock(m_mutex); + + auto mounted_path = Common::FixDirectorySlash(mounted_directory); + + const auto it = std::find_if( + m_mount_pairs.begin(), m_mount_pairs.end(), + [&mounted_path](const MountPair& p) { return Common::StartsWith(mounted_path, p.point); }); + if (it != m_mount_pairs.end()) { + const auto& p = *it; + auto rel_path = + Common::RemoveFirst(Common::FixDirectorySlash(mounted_directory), p.point.size()); + while (Common::StartsWith(rel_path, '/')) { + rel_path = Common::RemoveFirst(rel_path, 1); + } + return p.dir / rel_path; + } + + return mounted_directory; +} + +KYTY_SUBSYSTEM_INIT(FileSystem) { + g_mount_points = new MountPoints; + g_files = new FileDescriptors; +} + +KYTY_SUBSYSTEM_UNEXPECTED_SHUTDOWN(FileSystem) { + if (g_files != nullptr) { + g_files->CloseAll(); + } +} + +KYTY_SUBSYSTEM_DESTROY(FileSystem) { + if (g_files != nullptr) { + g_files->CloseAll(); + } +} + +void Mount(const std::filesystem::path& folder, const std::string& point) { + + g_mount_points->Mount(folder, point); +} + +void Umount(const std::string& folder_or_point) { + + g_mount_points->Umount(folder_or_point); +} + +std::filesystem::path GetRealFilename(const std::string& mounted_file_name) { + + return g_mount_points->GetRealFilename(mounted_file_name); +} + +// NOLINTNEXTLINE(readability-function-cognitive-complexity) +int KYTY_SYSV_ABI KernelOpen(const char* path, int flags, uint16_t mode) { + PRINT_NAME(); + + if (path == nullptr) { + return KERNEL_ERROR_EINVAL; + } + + auto flags_u = static_cast(flags); + + LOGF("\t path = %s\n" + "\t flags = %08" PRIx32 "\n" + "\t mode = %04" PRIx16 "\n", + path, flags_u, mode); + + bool append = (flags_u & 0x0008u) != 0; + bool fsync = (flags_u & 0x0080u) != 0; + bool sync = (flags_u & 0x0080u) != 0; + bool creat = (flags_u & 0x0200u) != 0; + bool trunc = (flags_u & 0x0400u) != 0; + bool excl = (flags_u & 0x0800u) != 0; + bool dsync = (flags_u & 0x1000u) != 0; + bool direct = (flags_u & 0x00010000u) != 0; + bool directory = (flags_u & 0x00020000u) != 0; + + if (direct) { + LOGF("\t O_DIRECT ignored on host file backend\n"); + } + + flags_u &= 0x3u; + + Common::File::Mode rw_mode = Common::File::Mode::Read; + + switch (flags_u) { + case 0: rw_mode = Common::File::Mode::Read; break; + case 1: rw_mode = Common::File::Mode::Write; break; + case 2: rw_mode = Common::File::Mode::ReadWrite; break; + default: EXIT("invalid flag_u: %u\n", flags_u); + } + + EXIT_NOT_IMPLEMENTED(directory && rw_mode != Common::File::Mode::Read); + EXIT_NOT_IMPLEMENTED(directory && (trunc || creat)); + + int descriptor = g_files->CreateDescriptor(); + auto* file = g_files->GetFile(descriptor); + + EXIT_IF(file == nullptr || file->opened || file->directory); + + file->name = path; + file->append = append; + file->sync_writes = fsync || sync || dsync; + + if (!directory && IsRandomDevice(file->name)) { + file->real_name = file->name; + file->special = SpecialFile::Random; + file->opened = true; + + LOGF_COLOR(Log::Color::Green, "\tOpen device: %s, [ok]\n", + Common::PathToString(file->real_name).c_str()); + + return descriptor; + } + + file->real_name = (directory ? g_mount_points->GetRealDirectory(file->name) + : g_mount_points->GetRealFilename(file->name)); + + if (trunc && rw_mode == Common::File::Mode::Read) { + return KERNEL_ERROR_EACCES; + } + + bool dir_exist = Common::File::IsDirectoryExisting(file->real_name); + bool file_exist = Common::File::IsFileExisting(file->real_name); + + if (directory || dir_exist) { + if (!dir_exist) { + g_files->DeleteDescriptor(descriptor); + return KERNEL_ERROR_ENOTDIR; + } + + EXIT_NOT_IMPLEMENTED(!directory && rw_mode != Common::File::Mode::Read); + EXIT_NOT_IMPLEMENTED(!directory && (trunc || creat)); + + file->dents = Common::File::GetDirEntries(file->real_name); + file->dents_offset = 0; + file->directory = true; + + LOGF_COLOR(Log::Color::Green, "\tOpen dir: %s, entries = %" PRIu32 ", [ok]\n", + Common::PathToString(file->real_name).c_str(), + static_cast(file->dents.size())); + + for (const auto& f: file->dents) { + LOGF("\t\t%s %s\n", f.is_file ? "[file]" : "[dir ]", f.name.c_str()); + } + } else { + bool result = false; + + if (excl && creat && file_exist) { + g_files->DeleteDescriptor(descriptor); + return KERNEL_ERROR_EEXIST; + } + + if (creat && (!file_exist || trunc)) { + Common::File::CreateDirectories(file->real_name.parent_path()); + result = file->f.Create(file->real_name); + + LOGF_COLOR(result ? Log::Color::Green : Log::Color::Red, "\tCreate: %s, %s\n", + Common::PathToString(file->real_name).c_str(), result ? "[ok]" : "[fail]"); + } else { + result = file->f.Open(file->real_name, rw_mode); + + LOGF_COLOR(result ? Log::Color::Green : Log::Color::Red, "\tOpen: %s, %s\n", + Common::PathToString(file->real_name).c_str(), result ? "[ok]" : "[fail]"); + } + + if (result && trunc) { + result = file->f.Truncate(0); + } + + if (result && append) { + file->f.Seek(file->f.Size()); + } + + if (!result || file->f.IsInvalid()) { + g_files->DeleteDescriptor(descriptor); + return KERNEL_ERROR_EACCES; + } + } + + file->opened = true; + return descriptor; +} + +int KYTY_SYSV_ABI KernelClose(int d) { + PRINT_NAME(); + + if (d < DESCRIPTOR_MIN) { + return KERNEL_ERROR_EPERM; + } + + if (::Libs::Network::Net::IsSocket(d)) { + const auto result = ::Libs::Network::Net::SocketClose(d); + return (result == OK ? OK : PosixToKernel(::Libs::Network::NetToPosix(result))); + } + + auto* file = g_files->GetFile(d); + + if (file == nullptr) { + return KERNEL_ERROR_EBADF; + } + + EXIT_IF(!file->opened); + + if (!file->directory && file->special == SpecialFile::None) { + file->f.Close(); + } + + file->opened = false; + + LOGF("\tClose: %s\n", Common::PathToString(file->real_name).c_str()); + + g_files->DeleteDescriptor(d); + + return OK; +} + +int64_t KYTY_SYSV_ABI KernelRead(int d, void* buf, size_t nbytes) { + PRINT_NAME(); + + if (d < DESCRIPTOR_MIN) { + return KERNEL_ERROR_EPERM; + } + + if (buf == nullptr) { + return KERNEL_ERROR_EFAULT; + } + + if (::Libs::Network::Net::IsSocket(d)) { + const auto result = ::Libs::Network::Net::Recv(d, buf, nbytes, 0); + return (result >= 0 ? result : PosixToKernel(*Posix::GetErrorAddr())); + } + + auto* file = g_files->GetFile(d); + + if (file == nullptr) { + return KERNEL_ERROR_EBADF; + } + + EXIT_NOT_IMPLEMENTED(file->directory); + + EXIT_IF(!file->opened); + + EXIT_NOT_IMPLEMENTED(nbytes > UINT_MAX); + + if (file->special == SpecialFile::Random) { + FillRandomBuffer(buf, nbytes); + + LOGF("\tRead %" PRIu64 " random bytes from: %s\n", static_cast(nbytes), + Common::PathToString(file->real_name).c_str()); + + return static_cast(nbytes); + } + + file->mutex.Lock(); + + bool is_invalid = file->f.IsInvalid(); + uint32_t bytes_read = 0; + file->f.Read(buf, static_cast(nbytes), &bytes_read); + + file->mutex.Unlock(); + + if (is_invalid) { + LOGF("\tfile is invalid\n"); + return KERNEL_ERROR_EIO; + } + + LOGF("\tRead %u bytes from: %s\n", bytes_read, Common::PathToString(file->real_name).c_str()); + + return bytes_read; +} + +int64_t KYTY_SYSV_ABI KernelWrite(int d, const void* buf, size_t nbytes) { + PRINT_NAME(); + + if (buf == nullptr) { + return KERNEL_ERROR_EFAULT; + } + + if (d == 1 || d == 2) { + auto* out = (d == 1 ? stdout : stderr); + const auto written = std::fwrite(buf, 1, nbytes, out); + const auto flush_ok = std::fflush(out) == 0; + + if (written != nbytes || !flush_ok) { + return KERNEL_ERROR_EIO; + } + + return static_cast(written); + } + + if (d < DESCRIPTOR_MIN) { + return KERNEL_ERROR_EBADF; + } + + if (::Libs::Network::Net::IsSocket(d)) { + const auto result = ::Libs::Network::Net::Send(d, buf, nbytes, 0); + return (result >= 0 ? result : PosixToKernel(*Posix::GetErrorAddr())); + } + + auto* file = g_files->GetFile(d); + + if (file == nullptr) { + return KERNEL_ERROR_EBADF; + } + + EXIT_NOT_IMPLEMENTED(file->directory); + EXIT_NOT_IMPLEMENTED(file->special != SpecialFile::None); + + EXIT_IF(!file->opened); + + EXIT_NOT_IMPLEMENTED(nbytes > UINT_MAX); + + file->mutex.Lock(); + + bool is_invalid = file->f.IsInvalid(); + uint32_t bytes_written = 0; + if (file->append) { + file->f.Seek(file->f.Size()); + } + file->f.Write(buf, static_cast(nbytes), &bytes_written); + if (file->sync_writes) { + file->f.Flush(); + } + + file->mutex.Unlock(); + + if (is_invalid) { + LOGF("\tfile is invalid\n"); + return KERNEL_ERROR_EIO; + } + + LOGF("\tWrite %u bytes to: %s\n", bytes_written, Common::PathToString(file->real_name).c_str()); + + return bytes_written; +} + +int64_t KYTY_SYSV_ABI KernelPread(int d, void* buf, size_t nbytes, int64_t offset) { + PRINT_NAME(); + + if (d < DESCRIPTOR_MIN) { + return KERNEL_ERROR_EPERM; + } + + if (buf == nullptr) { + return KERNEL_ERROR_EFAULT; + } + + if (offset < 0) { + return KERNEL_ERROR_EINVAL; + } + + auto* file = g_files->GetFile(d); + + if (file == nullptr) { + return KERNEL_ERROR_EBADF; + } + + EXIT_NOT_IMPLEMENTED(file->directory); + + EXIT_IF(!file->opened); + + EXIT_NOT_IMPLEMENTED(nbytes > UINT_MAX); + + if (file->special == SpecialFile::Random) { + FillRandomBuffer(buf, nbytes); + + LOGF("\tRead %" PRIu64 " random bytes (pos = %" PRId64 ") from: %s\n", + static_cast(nbytes), offset, Common::PathToString(file->real_name).c_str()); + + return static_cast(nbytes); + } + + file->mutex.Lock(); + + bool is_invalid = file->f.IsInvalid(); + auto pos = file->f.Tell(); + uint32_t bytes_read = 0; + file->f.Seek(offset); + file->f.Read(buf, static_cast(nbytes), &bytes_read); + file->f.Seek(pos); + + file->mutex.Unlock(); + + if (is_invalid) { + LOGF("\tfile is invalid\n"); + return KERNEL_ERROR_EIO; + } + + LOGF("\tRead %u bytes (pos = %" PRId64 ") from: %s\n", bytes_read, offset, + Common::PathToString(file->real_name).c_str()); + + return bytes_read; +} + +int64_t KYTY_SYSV_ABI KernelPwrite(int d, const void* buf, size_t nbytes, int64_t offset) { + PRINT_NAME(); + + if (d < DESCRIPTOR_MIN) { + return KERNEL_ERROR_EPERM; + } + + if (buf == nullptr) { + return KERNEL_ERROR_EFAULT; + } + + if (offset < 0) { + return KERNEL_ERROR_EINVAL; + } + + auto* file = g_files->GetFile(d); + + if (file == nullptr) { + return KERNEL_ERROR_EBADF; + } + + EXIT_NOT_IMPLEMENTED(file->directory); + EXIT_NOT_IMPLEMENTED(file->special != SpecialFile::None); + + EXIT_IF(!file->opened); + + EXIT_NOT_IMPLEMENTED(nbytes > UINT_MAX); + + file->mutex.Lock(); + + bool is_invalid = file->f.IsInvalid(); + auto pos = file->f.Tell(); + uint32_t bytes_written = 0; + file->f.Seek(file->append ? file->f.Size() : static_cast(offset)); + file->f.Write(buf, static_cast(nbytes), &bytes_written); + if (file->sync_writes) { + file->f.Flush(); + } + file->f.Seek(pos); + + file->mutex.Unlock(); + + if (is_invalid) { + LOGF("\tfile is invalid\n"); + return KERNEL_ERROR_EIO; + } + + LOGF("\tWrite %u bytes (pos = %" PRId64 ") to: %s\n", bytes_written, offset, + Common::PathToString(file->real_name).c_str()); + + return bytes_written; +} + +int64_t KYTY_SYSV_ABI KernelLseek(int d, int64_t offset, int whence) { + PRINT_NAME(); + + if (d < DESCRIPTOR_MIN) { + return KERNEL_ERROR_EPERM; + } + + auto* file = g_files->GetFile(d); + + if (file == nullptr) { + return KERNEL_ERROR_EBADF; + } + + EXIT_NOT_IMPLEMENTED(file->directory); + + EXIT_IF(!file->opened); + + if (file->special != SpecialFile::None) { + return KERNEL_ERROR_ESPIPE; + } + + file->mutex.Lock(); + + bool is_invalid = file->f.IsInvalid(); + + if (whence == 1) { + offset = static_cast(file->f.Tell()) + offset; + whence = 0; + } + + if (whence == 2) { + offset = static_cast(file->f.Size()) + offset; + whence = 0; + } + + EXIT_NOT_IMPLEMENTED(whence != 0); + + if (offset < 0) { + return KERNEL_ERROR_EINVAL; + } + + file->f.Seek(offset); + auto pos = static_cast(file->f.Tell()); + + EXIT_IF(pos != offset); + + file->mutex.Unlock(); + + if (is_invalid) { + LOGF("\tfile is invalid\n"); + return KERNEL_ERROR_EIO; + } + + LOGF("\tLseek (pos = %" PRId64 ") to: %s\n", offset, + Common::PathToString(file->real_name).c_str()); + + return pos; +} + +int KYTY_SYSV_ABI KernelStat(const char* path, FileStat* sb) { + PRINT_NAME(); + + if (path == nullptr || sb == nullptr) { + return KERNEL_ERROR_EINVAL; + } + + LOGF("\t KernelStat: %s\n", path); + + std::string path_s = std::string(path); + auto real_file_name = g_mount_points->GetRealFilename(path_s); + auto real_directory = g_mount_points->GetRealDirectory(path_s); + + bool is_dir = Common::File::IsDirectoryExisting(real_file_name) || + Common::File::IsDirectoryExisting(real_directory); + bool is_file = Common::File::IsFileExisting(real_file_name); + + if (!is_dir && !is_file) { + LOGF("\t file not found\n"); + return KERNEL_ERROR_ENOENT; + } + + EXIT_NOT_IMPLEMENTED(is_dir && is_file); + + FileStat stat {}; + stat.st_mode = 0000777u | (is_dir ? 0040000u : 0100000u); + + auto at = Common::DateTime::FromSystemUTC(); + auto wt = at; + + if (is_dir) { + stat.st_size = 0; + stat.st_blksize = 512; + stat.st_blocks = 0; + } else { + stat.st_size = static_cast(Common::File::Size(real_file_name)); + stat.st_blksize = 512; + stat.st_blocks = (stat.st_size + 511) / 512; + + Common::File::GetLastAccessAndWriteTimeUTC(real_file_name, &at, &wt); + } + + SecToTimespec(&stat.st_atim, at.ToUnix()); + SecToTimespec(&stat.st_mtim, wt.ToUnix()); + stat.st_ctim = stat.st_atim; + stat.st_birthtim = stat.st_mtim; + *sb = stat; + + return OK; +} + +int KYTY_SYSV_ABI KernelFstat(int d, FileStat* sb) { + PRINT_NAME(); + + if (d < DESCRIPTOR_MIN) { + return KERNEL_ERROR_EPERM; + } + + if (sb == nullptr) { + return KERNEL_ERROR_EFAULT; + } + + auto* file = g_files->GetFile(d); + + if (file == nullptr) { + return KERNEL_ERROR_EBADF; + } + + EXIT_IF(!file->opened); + + LOGF("\tKernelFstat: %s\n", Common::PathToString(file->real_name).c_str()); + + FileStat stat {}; + stat.st_mode = 0000777u | (file->directory ? 0040000u : 0100000u); + + auto at = Common::DateTime::FromSystemUTC(); + auto wt = at; + + if (file->special != SpecialFile::None) { + stat.st_size = 0; + stat.st_blksize = 512; + stat.st_blocks = 0; + SecToTimespec(&stat.st_atim, at.ToUnix()); + SecToTimespec(&stat.st_mtim, wt.ToUnix()); + stat.st_ctim = stat.st_atim; + stat.st_birthtim = stat.st_mtim; + *sb = stat; + + return OK; + } + + if (!file->directory) { + file->mutex.Lock(); + + bool is_invalid = file->f.IsInvalid(); + auto size = file->f.Size(); + file->f.GetLastAccessAndWriteTimeUTC(&at, &wt); + + file->mutex.Unlock(); + + if (is_invalid) { + LOGF("\tfile is invalid\n"); + return KERNEL_ERROR_EIO; + } + + stat.st_size = static_cast(size); + stat.st_blksize = 512; + stat.st_blocks = (stat.st_size + 511) / 512; + } else { + stat.st_size = 0; + stat.st_blksize = 512; + stat.st_blocks = 0; + } + + SecToTimespec(&stat.st_atim, at.ToUnix()); + SecToTimespec(&stat.st_mtim, wt.ToUnix()); + stat.st_ctim = stat.st_atim; + stat.st_birthtim = stat.st_mtim; + *sb = stat; + + return OK; +} + +int KYTY_SYSV_ABI KernelUnlink(const char* path) { + PRINT_NAME(); + + if (path == nullptr) { + return KERNEL_ERROR_EINVAL; + } + + auto path_s = std::string(path); + auto real_file_name = g_mount_points->GetRealFilename(path_s); + auto real_directory = g_mount_points->GetRealDirectory(path_s); + + bool is_dir = Common::File::IsDirectoryExisting(real_file_name) || + Common::File::IsDirectoryExisting(real_directory); + bool is_file = Common::File::IsFileExisting(real_file_name); + + if (is_dir) { + return KERNEL_ERROR_EPERM; + } + + if (!is_file) { + return KERNEL_ERROR_ENOENT; + } + + auto* open_file = g_files->GetFile(real_file_name); + bool ok = (open_file != nullptr && open_file->opened && !open_file->directory + ? open_file->f.Unlink() + : Common::File::DeleteFile(real_file_name)); + + if (!ok) { + return KERNEL_ERROR_EIO; + } + + LOGF("\tKernelUnlink: %s\n", path); + + return OK; +} + +int KYTY_SYSV_ABI KernelRename(const char* from, const char* to) { + PRINT_NAME(); + + if (from == nullptr || to == nullptr) { + return KERNEL_ERROR_EINVAL; + } + + auto from_path = std::string(from); + auto to_path = std::string(to); + auto real_from = g_mount_points->GetRealFilename(from_path); + auto real_to = g_mount_points->GetRealFilename(to_path); + + EXIT_NOT_IMPLEMENTED(g_files->GetFile(real_from) != nullptr); + EXIT_NOT_IMPLEMENTED(g_files->GetFile(real_to) != nullptr); + + if (!Common::File::IsFileExisting(real_from)) { + return KERNEL_ERROR_ENOENT; + } + + Common::File::CreateDirectories(real_to.parent_path()); + + if (Common::File::IsFileExisting(real_to)) { + Common::File::DeleteFile(real_to); + } + + if (!Common::File::MoveFile(real_from, real_to)) { + return KERNEL_ERROR_EIO; + } + + LOGF("\tKernelRename: %s -> %s\n", from, to); + + return OK; +} + +int KYTY_SYSV_ABI KernelGetdirentries(int fd, char* buf, int nbytes, int64_t* basep) { + PRINT_NAME(); + + if (fd < DESCRIPTOR_MIN) { + return KERNEL_ERROR_EBADF; + } + + if (buf == nullptr) { + return KERNEL_ERROR_EFAULT; + } + + auto* file = g_files->GetFile(fd); + + if (file == nullptr) { + return KERNEL_ERROR_EBADF; + } + + constexpr uint64_t DIR_BLOCK_SIZE = 512; + constexpr uint64_t DIRENT_META_SIZE = 8; + + if (!file->directory || nbytes < static_cast(DIR_BLOCK_SIZE)) { + return KERNEL_ERROR_EINVAL; + } + + EXIT_IF(!file->opened); + + LOGF("\t dir = %s\n" + "\t nbytes = %d\n" + "\t offset = %" PRIu64 "\n", + Common::PathToString(file->real_name).c_str(), nbytes, file->dents_offset); + + if (basep != nullptr) { + *basep = static_cast(file->dents_offset); + } + + std::vector dirents; + dirents.reserve( + std::max(static_cast(DIR_BLOCK_SIZE), file->dents.size() * 64u)); + + uint64_t next_ceiling = 0; + uint64_t dirent_offset = 0; + int64_t last_reclen_offset = -1; + + for (const auto& entry: file->dents) { + const auto& str = entry.name; + auto str_size = str.size(); + EXIT_NOT_IMPLEMENTED(str_size > 255); + + uint64_t reclen = AlignUp(DIRENT_META_SIZE + str_size + 1, 4); + if (dirent_offset + reclen > next_ceiling) { + if (last_reclen_offset >= 0) { + auto* last_reclen = + reinterpret_cast(dirents.data() + last_reclen_offset); + *last_reclen = static_cast(*last_reclen + (next_ceiling - dirent_offset)); + } + + dirent_offset = next_ceiling; + next_ceiling += DIR_BLOCK_SIZE; + dirents.resize(next_ceiling); + std::fill(dirents.begin() + static_cast(dirent_offset), dirents.end(), + 0); + } + + *reinterpret_cast(dirents.data() + dirent_offset + 0) = + Common::hash(str.data(), static_cast(str.size())); + *reinterpret_cast(dirents.data() + dirent_offset + 4) = + static_cast(reclen); + *reinterpret_cast(dirents.data() + dirent_offset + 6) = (entry.is_file ? 8 : 4); + *reinterpret_cast(dirents.data() + dirent_offset + 7) = + static_cast(str_size); + memcpy(dirents.data() + dirent_offset + 8, str.data(), str_size); + dirents[dirent_offset + 8 + str_size] = '\0'; + + last_reclen_offset = static_cast(dirent_offset + 4); + dirent_offset += reclen; + + LOGF("\t name = %s\n", str.data()); + } + + if (last_reclen_offset >= 0) { + auto* last_reclen = reinterpret_cast(dirents.data() + last_reclen_offset); + *last_reclen = static_cast(*last_reclen + (next_ceiling - dirent_offset)); + } + + const uint64_t directory_size = next_ceiling; + if (file->dents_offset >= directory_size) { + return 0; + } + + uint64_t bytes_written = 0; + uint64_t working_offset = file->dents_offset; + uint64_t dirent_buffer_offset = 0; + uint64_t aligned_count = AlignDown(static_cast(nbytes), DIR_BLOCK_SIZE); + + while (dirent_buffer_offset < dirents.size()) { + const auto* normal_dirent = dirents.data() + dirent_buffer_offset; + uint16_t reclen = *reinterpret_cast(normal_dirent + 4); + uint8_t namlen = *(normal_dirent + 7); + + if (namlen == 0 || reclen == 0) { + break; + } + + if (working_offset >= reclen) { + dirent_buffer_offset += reclen; + working_offset -= reclen; + continue; + } + + if (bytes_written + reclen > aligned_count) { + break; + } + + const auto bytes_to_copy = reclen - working_offset; + memcpy(buf + bytes_written, normal_dirent + working_offset, bytes_to_copy); + bytes_written += bytes_to_copy; + dirent_buffer_offset += reclen; + working_offset = 0; + } + + file->dents_offset += bytes_written; + return static_cast(bytes_written); +} + +int KYTY_SYSV_ABI KernelGetdents(int fd, char* buf, int nbytes) { + PRINT_NAME(); + + return KernelGetdirentries(fd, buf, nbytes, nullptr); +} + +int KYTY_SYSV_ABI KernelMkdir(const char* path, uint16_t mode) { + PRINT_NAME(); + + if (path == nullptr) { + return KERNEL_ERROR_EINVAL; + } + + if (std::strlen(path) > 255) { + return KERNEL_ERROR_ENAMETOOLONG; + } + + LOGF("\t path = %s\n" + "\t mode = %04" PRIx16 "\n", + path, mode); + + auto real_name = g_mount_points->GetRealDirectory(std::string(path)); + + if (Common::File::IsDirectoryExisting(real_name)) { + return KERNEL_ERROR_EEXIST; + } + + if (!Common::File::CreateDirectory(real_name)) { + return KERNEL_ERROR_EIO; + } + + if (!Common::File::IsDirectoryExisting(real_name)) { + return KERNEL_ERROR_ENOENT; + } + + return OK; +} + +int KYTY_SYSV_ABI KernelRmdir(const char* path) { + PRINT_NAME(); + + if (path == nullptr) { + return KERNEL_ERROR_EINVAL; + } + + LOGF("\t path = %s\n", path); + + auto real_name = g_mount_points->GetRealDirectory(std::string(path)); + + if (!Common::File::IsDirectoryExisting(real_name)) { + return KERNEL_ERROR_ENOENT; + } + + if (!Common::File::DeleteDirectory(real_name)) { + return KERNEL_ERROR_EIO; + } + + return OK; +} + +int KYTY_SYSV_ABI KernelCheckReachability(const char* path) { + PRINT_NAME(); + + if (path == nullptr) { + return KERNEL_ERROR_EINVAL; + } + + if (std::strlen(path) > 255) { + return KERNEL_ERROR_ENAMETOOLONG; + } + + LOGF("\t KernelCheckReachability: %s\n", path); + + std::string mounted_path = std::string(path); + + if (IsRandomDevice(mounted_path)) { + return OK; + } + + auto real_name = g_mount_points->GetRealFilename(mounted_path); + + if (Common::File::IsFileExisting(real_name) || Common::File::IsDirectoryExisting(real_name)) { + return OK; + } + + return KERNEL_ERROR_ENOENT; +} + +} // namespace Libs::LibKernel::FileSystem diff --git a/src/kernel/fileSystem.h b/src/kernel/fileSystem.h new file mode 100644 index 000000000..0c5e5530a --- /dev/null +++ b/src/kernel/fileSystem.h @@ -0,0 +1,61 @@ +#ifndef EMULATOR_INCLUDE_EMULATOR_KERNEL_FILESYSTEM_H_ +#define EMULATOR_INCLUDE_EMULATOR_KERNEL_FILESYSTEM_H_ + +#include "common/abi.h" +#include "common/common.h" +#include "common/stringUtils.h" +#include "common/subsystems.h" +#include "kernel/pthread.h" + +#include + +namespace Libs::LibKernel::FileSystem { + +struct FileStat { + uint32_t st_dev; + uint32_t st_ino; + uint16_t st_mode; + uint16_t st_nlink; + uint32_t st_uid; + uint32_t st_gid; + uint32_t st_rdev; + KernelTimespec st_atim; + KernelTimespec st_mtim; + KernelTimespec st_ctim; + int64_t st_size; + int64_t st_blocks; + uint32_t st_blksize; + uint32_t st_flags; + uint32_t st_gen; + int32_t st_lspare; + KernelTimespec st_birthtim; + unsigned int: (8 / 2) * (16 - static_cast(sizeof(KernelTimespec))); + unsigned int: (8 / 2) * (16 - static_cast(sizeof(KernelTimespec))); +}; + +KYTY_SUBSYSTEM_DEFINE(FileSystem); + +void Mount(const std::filesystem::path& folder, const std::string& point); +void Umount(const std::string& folder_or_point); +std::filesystem::path GetRealFilename(const std::string& mounted_file_name); + +int KYTY_SYSV_ABI KernelOpen(const char* path, int flags, uint16_t mode); +int KYTY_SYSV_ABI KernelClose(int d); +int64_t KYTY_SYSV_ABI KernelRead(int d, void* buf, size_t nbytes); +int64_t KYTY_SYSV_ABI KernelPread(int d, void* buf, size_t nbytes, int64_t offset); +int64_t KYTY_SYSV_ABI KernelWrite(int d, const void* buf, size_t nbytes); +int64_t KYTY_SYSV_ABI KernelPwrite(int d, const void* buf, size_t nbytes, int64_t offset); +int64_t KYTY_SYSV_ABI KernelLseek(int d, int64_t offset, int whence); +int KYTY_SYSV_ABI KernelStat(const char* path, FileStat* sb); +int KYTY_SYSV_ABI KernelFstat(int d, FileStat* sb); +int KYTY_SYSV_ABI KernelUnlink(const char* path); +int KYTY_SYSV_ABI KernelRename(const char* from, const char* to); +int KYTY_SYSV_ABI KernelGetdirentries(int fd, char* buf, int nbytes, int64_t* basep); +int KYTY_SYSV_ABI KernelGetdents(int fd, char* buf, int nbytes); +int KYTY_SYSV_ABI KernelMkdir(const char* path, uint16_t mode); +int KYTY_SYSV_ABI KernelRmdir(const char* path); +int KYTY_SYSV_ABI KernelCheckReachability(const char* path); + +} // namespace Libs::LibKernel::FileSystem + +#endif /* EMULATOR_INCLUDE_EMULATOR_KERNEL_FILESYSTEM_H_ */ diff --git a/src/kernel/memory.cpp b/src/kernel/memory.cpp new file mode 100644 index 000000000..055971e4c --- /dev/null +++ b/src/kernel/memory.cpp @@ -0,0 +1,3971 @@ +#include "kernel/memory.h" + +#include "common/assert.h" +#include "common/logging/log.h" +#include "common/magicEnum.h" +#include "common/stringUtils.h" +#include "common/threads.h" +#include "common/virtualMemory.h" +#include "graphics/guest_gpu/graphicsRun.h" +#include "graphics/host_gpu/renderer/renderContext.h" +#include "libs/errno.h" +#include "libs/libs.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS +#ifndef NOMINMAX +#define NOMINMAX +#endif +#include // IWYU pragma: keep +#ifndef MEM_RESERVE_PLACEHOLDER +#define MEM_RESERVE_PLACEHOLDER 0x00040000 +#endif +#ifndef MEM_REPLACE_PLACEHOLDER +#define MEM_REPLACE_PLACEHOLDER 0x00004000 +#endif +#ifndef MEM_PRESERVE_PLACEHOLDER +#define MEM_PRESERVE_PLACEHOLDER 0x00000002 +#endif +#ifndef MEM_COALESCE_PLACEHOLDERS +#define MEM_COALESCE_PLACEHOLDERS 0x00000001 +#endif +#elif KYTY_PLATFORM == KYTY_PLATFORM_LINUX +#include +#include +#include +#include +#endif + +namespace Libs::LibKernel::Memory { + +namespace VirtualMemory = Common::VirtualMemory; + +LIB_NAME("libkernel", "libkernel"); + +constexpr int PROT_CPU_READ = 0x01; +constexpr int PROT_CPU_WRITE = 0x02; +constexpr int PROT_CPU_EXEC = 0x04; +constexpr int PROT_GPU_READ = 0x10; +constexpr int PROT_GPU_WRITE = 0x20; + +enum class GpuAccessMode { NoAccess, Read, Write, ReadWrite }; + +constexpr uint64_t PAGE_TABLE_POOL_SIZE = 4ull * 1024ull * 1024ull * 1024ull; +constexpr uint64_t PAGE_TABLE_GRANULARITY = 2ull * 1024ull * 1024ull; +constexpr int PAGE_TABLE_POOL_ENTRIES = + static_cast(PAGE_TABLE_POOL_SIZE / PAGE_TABLE_GRANULARITY); +constexpr uint64_t DEFAULT_FLEXIBLE_MEMORY_SIZE = 4ull * 1024ull * 1024ull * 1024ull; + +static uint64_t g_flexible_memory_size = DEFAULT_FLEXIBLE_MEMORY_SIZE; + +static Graphics::GpuResourceManager& GetGpuResources() { + return *Graphics::g_render_ctx->GetGpuResources(); +} + +static bool IsGpuAddressRange(uint64_t vaddr, uint64_t size) { + constexpr uint64_t GPU_ADDRESS_LIMIT = 1ull << 40u; + return vaddr != 0 && size != 0 && vaddr < GPU_ADDRESS_LIMIT && + size <= GPU_ADDRESS_LIMIT - vaddr; +} + +static void MapGpuRange(uint64_t vaddr, uint64_t size, GpuAccessMode mode) { + if (mode == GpuAccessMode::NoAccess) { + return; + } + if (!IsGpuAddressRange(vaddr, size)) { + EXIT("invalid GPU map range: addr=0x%016" PRIx64 " size=0x%016" PRIx64 "\n", vaddr, size); + } + const auto access = mode == GpuAccessMode::Read ? Graphics::GpuAccess::Read + : mode == GpuAccessMode::Write ? Graphics::GpuAccess::Write + : Graphics::GpuAccess::ReadWrite; + GetGpuResources().MapMemory(vaddr, size, access); +} + +static void UnmapGpuRange(uint64_t vaddr, uint64_t size, GpuAccessMode mode) { + if (mode == GpuAccessMode::NoAccess) { + return; + } + if (!IsGpuAddressRange(vaddr, size)) { + EXIT("invalid GPU unmap range: addr=0x%016" PRIx64 " size=0x%016" PRIx64 "\n", vaddr, size); + } + auto& resources = GetGpuResources(); + Graphics::GraphicsRunSubmissionLock submission_lock; + const auto access = mode == GpuAccessMode::Read ? Graphics::GpuAccess::Read + : mode == GpuAccessMode::Write ? Graphics::GpuAccess::Write + : Graphics::GpuAccess::ReadWrite; + resources.UnmapMemory(vaddr, size, access); +} + +static bool DecodeMemoryProtection(int prot, VirtualMemory::Mode* mode, GpuAccessMode* gpu_mode) { + EXIT_IF(mode == nullptr); + EXIT_IF(gpu_mode == nullptr); + + bool cpu_read = (prot & PROT_CPU_READ) != 0; + bool cpu_write = (prot & PROT_CPU_WRITE) != 0; + bool cpu_exec = (prot & PROT_CPU_EXEC) != 0; + bool gpu_read = (prot & PROT_GPU_READ) != 0; + bool gpu_write = (prot & PROT_GPU_WRITE) != 0; + + if ((prot & + (PROT_CPU_READ | PROT_CPU_WRITE | PROT_CPU_EXEC | PROT_GPU_READ | PROT_GPU_WRITE)) == 0 && + prot != 0) { + return false; + } + + if (gpu_read && gpu_write) { + *gpu_mode = GpuAccessMode::ReadWrite; + } else if (gpu_read) { + *gpu_mode = GpuAccessMode::Read; + } else if (gpu_write) { + *gpu_mode = GpuAccessMode::Write; + } else { + *gpu_mode = GpuAccessMode::NoAccess; + } + + bool host_read = cpu_read || gpu_read; + bool host_write = cpu_write || gpu_write; + + if (host_write) { + host_read = true; + } + + *mode = VirtualMemory::Mode::NoAccess; + if (cpu_exec) { + *mode = (host_write ? VirtualMemory::Mode::ExecuteReadWrite + : (host_read ? VirtualMemory::Mode::ExecuteRead + : VirtualMemory::Mode::Execute)); + } else if (host_write) { + *mode = VirtualMemory::Mode::ReadWrite; + } else if (host_read) { + *mode = VirtualMemory::Mode::Read; + } + + return true; +} + +static GpuAccessMode GetGpuAccessMode(int prot) { + VirtualMemory::Mode mode {}; + GpuAccessMode gpu_mode {}; + if (!DecodeMemoryProtection(prot, &mode, &gpu_mode)) { + EXIT("unsupported GPU memory protection: 0x%08x\n", prot); + } + return gpu_mode; +} + +static bool ProtectCommittedHostMemory(uint64_t start, uint64_t size, VirtualMemory::Mode mode, + VirtualMemory::Mode* old_mode) { + if (size == 0) { + if (old_mode != nullptr) { + *old_mode = VirtualMemory::Mode::NoAccess; + } + return true; + } + +#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS + const auto end = start + size; + auto cur = start; + bool protected_any = false; + + while (cur < end) { + MEMORY_BASIC_INFORMATION mbi {}; + if (VirtualQuery(reinterpret_cast(cur), &mbi, sizeof(mbi)) == 0) { + return false; + } + + const auto region_start = reinterpret_cast(mbi.BaseAddress); + const auto region_end = region_start + static_cast(mbi.RegionSize); + const auto chunk_start = std::max(cur, region_start); + const auto chunk_end = std::min(end, region_end); + + if (chunk_end <= chunk_start) { + return false; + } + + if (mbi.State == MEM_COMMIT) { + VirtualMemory::Mode chunk_old {}; + if (!VirtualMemory::Protect(chunk_start, chunk_end - chunk_start, mode, &chunk_old)) { + return false; + } + + if (!protected_any && old_mode != nullptr) { + *old_mode = chunk_old; + } + protected_any = true; + } + + cur = chunk_end; + } + + if (!protected_any && old_mode != nullptr) { + *old_mode = VirtualMemory::Mode::NoAccess; + } + + return true; +#else + return VirtualMemory::Protect(start, size, mode, old_mode); +#endif +} + +static void CopyVirtualRangeName(char* dst, const char* name) { + EXIT_IF(dst == nullptr); + + std::memset(dst, 0, KERNEL_MAXIMUM_NAME_LENGTH); + + if (name != nullptr) { + std::strncpy(dst, name, KERNEL_MAXIMUM_NAME_LENGTH - 1); + } +} + +static bool VirtualRangesOverlap(uint64_t left_start, uint64_t left_size, uint64_t right_start, + uint64_t right_size) { + if (left_size == 0 || right_size == 0) { + return false; + } + + auto left_end = (UINT64_MAX - left_start < left_size ? UINT64_MAX : left_start + left_size); + auto right_end = + (UINT64_MAX - right_start < right_size ? UINT64_MAX : right_start + right_size); + + return left_start < right_end && right_start < left_end; +} + +static bool CommitFixedHostRange(uint64_t start, uint64_t size, VirtualMemory::Mode mode) { + constexpr uint64_t PAGE_SIZE = 0x4000; + + for (uint64_t addr = start; addr < start + size; addr += PAGE_SIZE) { +#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS + MEMORY_BASIC_INFORMATION info {}; + if (VirtualQuery(reinterpret_cast(addr), &info, sizeof(info)) != 0) { + if (info.State == MEM_COMMIT) { + if (!VirtualMemory::Protect(addr, PAGE_SIZE, mode)) { + return false; + } + continue; + } + if (info.State == MEM_RESERVE) { + if (!VirtualMemory::Commit(addr, PAGE_SIZE, mode)) { + return false; + } + continue; + } + } +#endif + if (VirtualMemory::AllocFixed(addr, PAGE_SIZE, mode)) { + continue; + } + if (VirtualMemory::Commit(addr, PAGE_SIZE, mode)) { + continue; + } + if (VirtualMemory::Protect(addr, PAGE_SIZE, mode)) { + continue; + } + return false; + } + + return true; +} + +#include "memoryAddressSpace.inc" + +enum class VirtualRangeType { + Reserved, + PoolReserved, + Direct, + Flexible, + Pooled, + Stack, + Code, +}; + +static bool IsReservedRangeType(VirtualRangeType type) { + return type == VirtualRangeType::Reserved || type == VirtualRangeType::PoolReserved; +} + +static bool IsPooledRangeType(VirtualRangeType type) { + return type == VirtualRangeType::Pooled || type == VirtualRangeType::PoolReserved; +} + +static bool IsCommittedRangeType(VirtualRangeType type) { + return !IsReservedRangeType(type); +} + +class VirtualRanges { +public: + struct Range { + uint64_t start = 0; + uint64_t size = 0; + uint64_t offset = 0; + int protection = 0; + int memory_type = 0; + VirtualRangeType type = VirtualRangeType::Reserved; + bool committed_from_reserved = false; + // Reserved placeholders need OS-specific release/commit paths; plain reserves still use + // Common::VirtualMemory. + bool placeholder_backed = false; + char name[KERNEL_MAXIMUM_NAME_LENGTH]; + }; + + bool Add(uint64_t start, uint64_t size, uint64_t offset, int protection, int memory_type, + VirtualRangeType type, const char* name, bool committed_from_reserved = false, + bool placeholder_backed = false) { + Common::LockGuard lock(m_mutex); + + if (start == 0 || size == 0) { + return false; + } + auto position = LowerBound(start); + if ((position != m_ranges.end() && + VirtualRangesOverlap(start, size, position->start, position->size)) || + (position != m_ranges.begin() && + VirtualRangesOverlap(start, size, std::prev(position)->start, + std::prev(position)->size))) { + return false; + } + + Range r {}; + r.start = start; + r.size = size; + r.offset = offset; + r.protection = protection; + r.memory_type = memory_type; + r.type = type; + r.committed_from_reserved = committed_from_reserved; + r.placeholder_backed = placeholder_backed; + CopyVirtualRangeName(r.name, name); + const auto index = static_cast(position - m_ranges.begin()); + m_ranges.insert(position, r); + MergeAroundUnlocked(index); + return true; + } + + bool Remove(uint64_t start, uint64_t size) { + Common::LockGuard lock(m_mutex); + + auto position = LowerBound(start); + if (position != m_ranges.end() && position->start == start && position->size == size) { + m_ranges.erase(position); + return true; + } + auto removed = RemoveUnlocked(start, size); + MergeUnlocked(); + return removed; + } + + bool HasOverlap(uint64_t start, uint64_t size) { + Common::LockGuard lock(m_mutex); + + return FindOverlap(start, size) != nullptr; + } + + bool HasGpuAccess(uint64_t start, uint64_t size) { + Common::LockGuard lock(m_mutex); + + return std::any_of(m_ranges.begin(), m_ranges.end(), [start, size](const auto& range) { + return VirtualRangesOverlap(start, size, range.start, range.size) && + (range.protection & (PROT_GPU_READ | PROT_GPU_WRITE)) != 0; + }); + } + + bool ReleaseReserved(uint64_t start, uint64_t size) { + Common::LockGuard lock(m_mutex); + + for (size_t index = 0; index < m_ranges.size(); index++) { + auto& r = m_ranges[index]; + if (r.start == start && r.size == size && IsReservedRangeType(r.type)) { + m_ranges.erase(m_ranges.begin() + static_cast(index)); + return VirtualMemory::Free(start); + } + } + return true; + } + + bool ConsumeReserved(uint64_t start, uint64_t size, + VirtualRangeType type = VirtualRangeType::Reserved) { + Common::LockGuard lock(m_mutex); + + auto end = End(start, size); + for (const auto& r: m_ranges) { + if (r.type == type && start >= r.start && end <= End(r.start, r.size)) { + RemoveUnlocked(start, size); + MergeUnlocked(); + return true; + } + } + + return false; + } + + bool ConsumeReservedSpan(uint64_t start, uint64_t size, Range* first_range = nullptr, + VirtualRangeType type = VirtualRangeType::Reserved) { + Common::LockGuard lock(m_mutex); + + if (size == 0) { + return false; + } + + auto current = start; + auto end = End(start, size); + while (current < end) { + const Range* candidate = nullptr; + for (const auto& r: m_ranges) { + if (r.type == type && current >= r.start && current < End(r.start, r.size)) { + candidate = &r; + break; + } + } + if (candidate == nullptr) { + return false; + } + if (current == start && first_range != nullptr) { + *first_range = *candidate; + } + current = std::min(end, End(candidate->start, candidate->size)); + } + + RemoveUnlocked(start, size); + MergeUnlocked(); + return true; + } + + void Rename(uint64_t start, uint64_t size, const char* name) { + Common::LockGuard lock(m_mutex); + + auto position = LowerBound(start); + if (position != m_ranges.end() && position->start == start && position->size == size) { + CopyVirtualRangeName(position->name, name); + MergeAroundUnlocked(static_cast(position - m_ranges.begin())); + return; + } + EditUnlocked(start, size, [name](Range* r) { CopyVirtualRangeName(r->name, name); }); + } + + void Protect(uint64_t start, uint64_t size, int protection) { + Common::LockGuard lock(m_mutex); + + EditUnlocked(start, size, [protection](Range* r) { r->protection = protection; }); + } + + void SetMemoryType(uint64_t start, uint64_t size, int memory_type) { + Common::LockGuard lock(m_mutex); + + EditUnlocked(start, size, [memory_type](Range* r) { r->memory_type = memory_type; }); + } + + bool Query(uint64_t addr, int flags, Range* out) { + EXIT_IF(out == nullptr); + + Common::LockGuard lock(m_mutex); + + auto next = std::upper_bound( + m_ranges.begin(), m_ranges.end(), addr, + [](uint64_t value, const Range& range) { return value < range.start; }); + if (next != m_ranges.begin()) { + auto current = std::prev(next); + if (addr < End(current->start, current->size)) { + *out = *current; + return true; + } + } + if (flags != 1 || next == m_ranges.end()) { + return false; + } + + *out = *next; + return true; + } + + bool QuerySpan(uint64_t start, uint64_t size, std::vector* out) { + EXIT_IF(out == nullptr); + + Common::LockGuard lock(m_mutex); + out->clear(); + if (start == 0 || size == 0 || size > UINT64_MAX - start) { + return false; + } + + const auto end = start + size; + auto current = start; + for (const auto& range: m_ranges) { + const auto range_end = End(range.start, range.size); + if (range_end <= current) { + continue; + } + if (range.start > current) { + break; + } + + Range part = range; + part.start = current; + part.size = std::min(end, range_end) - current; + if (part.type == VirtualRangeType::Direct) { + part.offset += current - range.start; + } + out->push_back(part); + current += part.size; + if (current == end) { + return true; + } + } + + out->clear(); + return false; + } + + uint64_t CountPageTableEntries(bool gpu) { + Common::LockGuard lock(m_mutex); + + uint64_t used = 0; + for (const auto& r: m_ranges) { + if (!IsCommittedRangeType(r.type) || r.size == 0) { + continue; + } + + const bool has_cpu_access = + (r.protection & (PROT_CPU_READ | PROT_CPU_WRITE | PROT_CPU_EXEC)) != 0; + const bool has_gpu_access = (r.protection & (PROT_GPU_READ | PROT_GPU_WRITE)) != 0; + if (gpu ? !has_gpu_access : !has_cpu_access) { + continue; + } + + const auto end = End(r.start, r.size); + const auto first_entry = r.start / PAGE_TABLE_GRANULARITY; + const auto last_entry = (end - 1) / PAGE_TABLE_GRANULARITY; + used += last_entry - first_entry + 1; + } + + return used; + } + +private: + static uint64_t End(uint64_t start, uint64_t size) { + return (UINT64_MAX - start < size ? UINT64_MAX : start + size); + } + + static bool SameMergeKey(const Range& left, const Range& right) { + if (left.type == VirtualRangeType::Direct || right.type == VirtualRangeType::Direct) { + return false; + } + + return left.type == right.type && left.protection == right.protection && + left.memory_type == right.memory_type && + left.committed_from_reserved == right.committed_from_reserved && + left.placeholder_backed == right.placeholder_backed && + std::strncmp(left.name, right.name, KERNEL_MAXIMUM_NAME_LENGTH) == 0; + } + + static void AddPiece(std::vector* ranges, const Range& source, uint64_t start, + uint64_t end) { + EXIT_IF(ranges == nullptr); + + if (end <= start) { + return; + } + + Range piece = source; + piece.start = start; + piece.size = end - start; + if (piece.type == VirtualRangeType::Direct) { + piece.offset += start - source.start; + } + ranges->push_back(piece); + } + + std::vector::iterator LowerBound(uint64_t start) { + return std::lower_bound( + m_ranges.begin(), m_ranges.end(), start, + [](const Range& range, uint64_t value) { return range.start < value; }); + } + + void MergeAroundUnlocked(size_t index) { + if (index >= m_ranges.size()) { + return; + } + if (index != 0) { + auto& previous = m_ranges[index - 1]; + auto& current = m_ranges[index]; + if (End(previous.start, previous.size) == current.start && + SameMergeKey(previous, current)) { + previous.size += current.size; + m_ranges.erase(m_ranges.begin() + static_cast(index)); + index--; + } + } + while (index + 1 < m_ranges.size()) { + auto& current = m_ranges[index]; + auto& next = m_ranges[index + 1]; + if (End(current.start, current.size) != next.start || !SameMergeKey(current, next)) { + break; + } + current.size += next.size; + m_ranges.erase(m_ranges.begin() + static_cast(index + 1)); + } + } + + template + void EditUnlocked(uint64_t start, uint64_t size, EditFunc edit) { + if (size == 0) { + return; + } + + std::vector out; + auto edit_end = End(start, size); + + for (const auto& r: m_ranges) { + auto r_end = End(r.start, r.size); + if (!VirtualRangesOverlap(start, size, r.start, r.size)) { + out.push_back(r); + continue; + } + + auto mid_start = std::max(start, r.start); + auto mid_end = std::min(edit_end, r_end); + + AddPiece(&out, r, r.start, mid_start); + + Range mid = r; + mid.start = mid_start; + mid.size = mid_end - mid_start; + if (mid.type == VirtualRangeType::Direct) { + mid.offset += mid_start - r.start; + } + edit(&mid); + out.push_back(mid); + + AddPiece(&out, r, mid_end, r_end); + } + + m_ranges = out; + MergeUnlocked(); + } + + bool RemoveUnlocked(uint64_t start, uint64_t size) { + if (size == 0) { + return false; + } + + std::vector out; + bool removed = false; + auto rem_end = End(start, size); + + for (const auto& r: m_ranges) { + auto r_end = End(r.start, r.size); + if (!VirtualRangesOverlap(start, size, r.start, r.size)) { + out.push_back(r); + continue; + } + + removed = true; + AddPiece(&out, r, r.start, std::max(start, r.start)); + AddPiece(&out, r, std::min(rem_end, r_end), r_end); + } + + m_ranges = out; + return removed; + } + + void MergeUnlocked() { + if (m_ranges.size() < 2) { + return; + } + + std::sort(m_ranges.begin(), m_ranges.end(), + [](const Range& left, const Range& right) { return left.start < right.start; }); + + std::vector merged; + for (const auto& r: m_ranges) { + if (!merged.empty()) { + auto& last = merged[merged.size() - 1]; + if (End(last.start, last.size) == r.start && SameMergeKey(last, r)) { + last.size += r.size; + continue; + } + } + merged.push_back(r); + } + m_ranges = merged; + } + + Range* FindOverlap(uint64_t start, uint64_t size) { + auto position = LowerBound(start); + if (position != m_ranges.end() && + VirtualRangesOverlap(start, size, position->start, position->size)) { + return &*position; + } + if (position != m_ranges.begin()) { + auto previous = std::prev(position); + if (VirtualRangesOverlap(start, size, previous->start, previous->size)) { + return &*previous; + } + } + return nullptr; + } + + std::vector m_ranges; + Common::Mutex m_mutex; +}; + +class PhysicalMemory { +public: + struct AllocatedBlock { + uint64_t start_addr; + uint64_t size; + uint64_t map_vaddr; + uint64_t map_size; + uint64_t host_vaddr; + uint64_t host_size; + int prot; + VirtualMemory::Mode mode; + GpuAccessMode gpu_mode; + int memory_type; + bool pool_expansion; + char name[KERNEL_MAXIMUM_NAME_LENGTH]; + }; + + PhysicalMemory() { + EXIT_NOT_IMPLEMENTED(!Common::Thread::IsMainThread()); + m_free.emplace(0, Size()); + } + virtual ~PhysicalMemory() { KYTY_NOT_IMPLEMENTED; } + + KYTY_CLASS_NO_COPY(PhysicalMemory); + + static uint64_t Size() { return static_cast(13824) * 1024 * 1024; } + + bool Alloc(uint64_t search_start, uint64_t search_end, size_t len, size_t alignment, + uint64_t* phys_addr_out, int memory_type, bool pool_expansion = false); + bool Available(uint64_t search_start, uint64_t search_end, size_t alignment, + uint64_t* phys_addr_out, uint64_t* size_out); + bool Release(uint64_t start, size_t len, uint64_t* vaddr, uint64_t* size, + GpuAccessMode* gpu_mode); + bool Map(uint64_t vaddr, uint64_t phys_addr, size_t len, int prot, VirtualMemory::Mode mode, + GpuAccessMode gpu_mode); + bool Unmap(uint64_t vaddr, uint64_t size, GpuAccessMode* gpu_mode, + uint64_t* host_vaddr_to_release = nullptr); + bool Find(uint64_t vaddr, uint64_t* base_addr, size_t* len, int* prot, + VirtualMemory::Mode* mode, GpuAccessMode* gpu_mode); + bool Find(uint64_t phys_addr, bool next, PhysicalMemory::AllocatedBlock* out); + bool CanMapDirect(uint64_t phys_addr, size_t len); + bool ReleasePoolExpansion(uint64_t phys_addr, size_t len); + std::vector FindMappings(uint64_t phys_addr, size_t len); + void SetVirtualRangeName(uint64_t vaddr, uint64_t len, const char* name); + void SetVirtualRangeMemoryType(uint64_t vaddr, uint64_t len, int memory_type); + + [[nodiscard]] Common::Mutex& GetMutex() { return m_mutex; } + [[nodiscard]] const std::map& GetPhysicalBlocks() const { + return m_physical; + } + [[nodiscard]] const std::vector& GetMappings() const { return m_mappings; } + +private: + void ConsumeFreeRange(std::map::iterator range, uint64_t start, + uint64_t size); + void AddFreeRange(uint64_t start, uint64_t size); + + std::map m_physical; + std::map m_free; + std::vector m_mappings; + Common::Mutex m_mutex; +}; + +class FlexibleMemory { +public: + struct AllocatedBlock { + uint64_t map_vaddr; + uint64_t map_size; + uint64_t host_vaddr; + uint64_t host_size; + int prot; + VirtualMemory::Mode mode; + GpuAccessMode gpu_mode; + char name[KERNEL_MAXIMUM_NAME_LENGTH]; + }; + + FlexibleMemory() { EXIT_NOT_IMPLEMENTED(!Common::Thread::IsMainThread()); } + virtual ~FlexibleMemory() { KYTY_NOT_IMPLEMENTED; } + + KYTY_CLASS_NO_COPY(FlexibleMemory); + + static uint64_t Size() { return g_flexible_memory_size; } + uint64_t Available(); + + bool Map(uint64_t vaddr, size_t len, int prot, VirtualMemory::Mode mode, GpuAccessMode gpu_mode, + const char* name); + bool Unmap(uint64_t vaddr, uint64_t size, GpuAccessMode* gpu_mode, + uint64_t* host_vaddr_to_release = nullptr); + bool Find(uint64_t vaddr, uint64_t* base_addr, size_t* len, int* prot, + VirtualMemory::Mode* mode, GpuAccessMode* gpu_mode); + void SetVirtualRangeName(uint64_t vaddr, uint64_t len, const char* name); + + [[nodiscard]] Common::Mutex& GetMutex() { return m_mutex; } + [[nodiscard]] const std::vector& GetBlocks() const { return m_allocated; } + +private: + std::vector m_allocated; + uint64_t m_allocated_total = 0; + Common::Mutex m_mutex; +}; + +class PooledMemory { +public: + struct Mapping { + uint64_t vaddr; + uint64_t size; + uint64_t phys_addr; + GpuAccessMode gpu_mode; + }; + + void Expand(uint64_t phys_addr, uint64_t size); + bool ReleaseExpansion(uint64_t phys_addr, uint64_t size); + bool Allocate(uint64_t vaddr, uint64_t size, GpuAccessMode gpu_mode, + std::vector* mappings); + bool Query(uint64_t vaddr, uint64_t size, std::vector* mappings); + bool Release(uint64_t vaddr, uint64_t size, GpuAccessMode* gpu_mode); + [[nodiscard]] uint64_t Available(); + [[nodiscard]] std::vector GetMappings(); + +private: + struct PhysicalRange { + uint64_t start; + uint64_t size; + }; + + void AddFreeUnlocked(uint64_t start, uint64_t size); + bool QueryUnlocked(uint64_t vaddr, uint64_t size, std::vector* mappings) const; + + std::vector m_free; + std::vector m_expansions; + std::vector m_mappings; + Common::Mutex m_mutex; +}; + +static PhysicalMemory* g_physical_memory = nullptr; +static FlexibleMemory* g_flexible_memory = nullptr; +static PooledMemory* g_pooled_memory = nullptr; +static VirtualRanges* g_virtual_ranges = nullptr; +static DirectMemoryBacking* g_direct_memory_backing = nullptr; +static PlaceholderAddressSpace* g_placeholder_address_space = nullptr; +static callback_func_t g_alloc_callback = nullptr; +static callback_func_t g_free_callback = nullptr; +static std::atomic g_memory_pool_committed = 0; +static void MemoryPoolSubtractCommitted(uint64_t len); +// Keep host mappings, physical blocks, placeholders, and virtual ranges in step. +static std::recursive_mutex g_memory_operation_mutex; + +bool TryWriteBacking(uint64_t vaddr, const void* data, uint64_t size) { + return g_direct_memory_backing->TryWriteBacking(vaddr, data, size); +} + +bool TryReadBacking(uint64_t vaddr, void* data, uint64_t size) { + return g_direct_memory_backing->TryReadBacking(vaddr, data, size); +} + +void WriteBacking(uint64_t vaddr, const void* data, uint64_t size) noexcept { + if (!TryWriteBacking(vaddr, data, size)) { + EXIT("Memory: required direct-backing write failed, addr=0x%016" PRIx64 + " size=0x%016" PRIx64 "\n", + vaddr, size); + } +} + +struct PrtAperture { + uint64_t address = 0; + uint64_t size = 0; +}; + +constexpr int PRT_APERTURE_MAX_INDEX = 2; +constexpr uint64_t PRT_PAGE_SIZE = 0x4000; +constexpr uint64_t PRT_APERTURE_START = 0x0f00000000ull; +constexpr uint64_t PRT_APERTURE_END = 0xfc00000000ull; + +static std::array g_prt_apertures {}; +static Common::Mutex g_prt_aperture_mutex; + +static bool IsInPrtAperture(uint64_t address) { + Common::LockGuard lock(g_prt_aperture_mutex); + + for (const auto& aperture: g_prt_apertures) { + if (aperture.size != 0 && address >= aperture.address && + address < aperture.address + aperture.size) { + return true; + } + } + + return false; +} + +static void SelfTestSub64SharedPlaceholderAlias() { +#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS + constexpr uint64_t PageSize = 0x4000; + const auto granularity = g_placeholder_address_space->GetGranularity(); + if (granularity < PageSize * 2u) { + LOGF_COLOR( + Log::Color::Yellow, + "\t direct-memory sub-64K placeholder self-test skipped: granularity too small\n"); + return; + } + + const auto base = g_placeholder_address_space->ReserveAligned(0, granularity, granularity); + if (base == 0) { + LOGF_COLOR(Log::Color::Yellow, + "\t direct-memory sub-64K placeholder self-test skipped: reserve unavailable\n"); + return; + } + + const auto alias = base + PageSize; + bool ok = false; + auto failure_reason = DirectMemoryBacking::FailureReason::None; + const bool consumed = g_placeholder_address_space->Consume(alias, PageSize); + + if (consumed && + g_direct_memory_backing->MapExistingPlaceholderFixed( + alias, PageSize, PageSize, VirtualMemory::Mode::ReadWrite, &failure_reason)) { + auto* ptr = reinterpret_cast(alias); + *ptr = 0x4b59545953553634ull; // "KYTYSU64" + ok = (*ptr == 0x4b59545953553634ull); + std::memset(ptr, 0, PageSize); + + bool placeholder_preserved = false; + ok = g_direct_memory_backing->Unmap(alias, PageSize, true, &placeholder_preserved) && + placeholder_preserved && ok; + if (placeholder_preserved) { + g_placeholder_address_space->AddFree(alias, PageSize); + } + } else if (consumed) { + g_placeholder_address_space->AddFree(alias, PageSize); + } + + const bool released = g_placeholder_address_space->ReleaseFree(base, granularity); + LOGF_COLOR(ok && released ? Log::Color::Green : Log::Color::Red, + "\t direct-memory sub-64K placeholder self-test: %s%s%s\n", + ok && released ? "ok" : "failed", ok ? "" : ", reason = ", + ok ? "" : DirectMemoryBacking::GetFailureReasonName(failure_reason)); +#endif +} + +static bool RestoreCommittedPlaceholderOrProtect(uint64_t vaddr, uint64_t size) { + if (g_placeholder_address_space->ReleaseCommitted(vaddr, size)) { + return true; + } + VirtualMemory::Protect(vaddr, size, VirtualMemory::Mode::NoAccess); + return false; +} + +static bool ReleaseReservedRange(uint64_t vaddr, uint64_t size) { + VirtualRanges::Range range {}; + if (g_virtual_ranges->Query(vaddr, 0, &range) && range.start == vaddr && range.size == size && + IsReservedRangeType(range.type) && range.placeholder_backed) { + if (!g_virtual_ranges->ConsumeReserved(vaddr, size, range.type)) { + return false; + } + if (g_placeholder_address_space->ReleaseFree(vaddr, size)) { + return true; + } + if (VirtualMemory::Free(vaddr)) { + return true; + } + g_virtual_ranges->Add(vaddr, size, 0, 0, 0, range.type, range.name, false, true); + return false; + } + return g_virtual_ranges->ReleaseReserved(vaddr, size); +} + +static bool ReplaceFixedRangeWithReserved(uint64_t start, uint64_t size, bool* placeholder_backed); + +KYTY_SUBSYSTEM_INIT(Memory) { + g_flexible_memory_size = DEFAULT_FLEXIBLE_MEMORY_SIZE; + g_physical_memory = new PhysicalMemory; + g_flexible_memory = new FlexibleMemory; + g_pooled_memory = new PooledMemory; + g_virtual_ranges = new VirtualRanges; + g_direct_memory_backing = new DirectMemoryBacking(PhysicalMemory::Size()); + g_placeholder_address_space = new PlaceholderAddressSpace; + + VirtualMemory::Init(); + EXIT_IF(!g_direct_memory_backing->SelfTest()); + g_placeholder_address_space->SelfTest(); + SelfTestSub64SharedPlaceholderAlias(); +} + +KYTY_SUBSYSTEM_UNEXPECTED_SHUTDOWN(Memory) {} + +KYTY_SUBSYSTEM_DESTROY(Memory) { + delete g_placeholder_address_space; + g_placeholder_address_space = nullptr; + delete g_direct_memory_backing; + g_direct_memory_backing = nullptr; + delete g_pooled_memory; + g_pooled_memory = nullptr; +} + +struct AlignedPos { + uint64_t value = 0; + bool valid = false; +}; + +static constexpr AlignedPos GetAlignedPos(uint64_t pos, size_t alignment) { + if (alignment == 0) { + return {pos, true}; + } + + const auto remainder = pos % alignment; + const auto increment = (remainder != 0 ? alignment - remainder : 0); + if (increment > UINT64_MAX - pos) { + return {}; + } + + return {pos + increment, true}; +} + +static_assert(!GetAlignedPos(UINT64_MAX - 1, 4).valid); + +void RegisterCallbacks(callback_func_t alloc_func, callback_func_t free_func) { + EXIT_IF(g_alloc_callback != nullptr || g_free_callback != nullptr); + EXIT_IF(alloc_func == nullptr || free_func == nullptr); + + std::lock_guard memory_operation_lock(g_memory_operation_mutex); + + g_alloc_callback = alloc_func; + g_free_callback = free_func; + + g_physical_memory->GetMutex().Lock(); + for (const auto& b: g_physical_memory->GetMappings()) { + if (b.map_vaddr != 0 && b.map_size != 0) { + g_alloc_callback(b.map_vaddr, b.map_size); + } + } + g_physical_memory->GetMutex().Unlock(); + + g_flexible_memory->GetMutex().Lock(); + for (const auto& b: g_flexible_memory->GetBlocks()) { + g_alloc_callback(b.map_vaddr, b.map_size); + } + g_flexible_memory->GetMutex().Unlock(); + + for (const auto& mapping: g_pooled_memory->GetMappings()) { + g_alloc_callback(mapping.vaddr, mapping.size); + } +} + +void SetFlexibleMemorySize(uint64_t size) { + g_flexible_memory_size = size; + LOGF("\t flexible memory size = 0x%016" PRIx64 " (%" PRIu64 " MiB)\n", size, + size / (1024ull * 1024ull)); +} + +bool PhysicalMemory::Alloc(uint64_t search_start, uint64_t search_end, size_t len, size_t alignment, + uint64_t* phys_addr_out, int memory_type, bool pool_expansion) { + if (phys_addr_out == nullptr) { + return false; + } + + Common::LockGuard lock(m_mutex); + + search_end = std::min(search_end, Size()); + if (search_start >= search_end) { + return false; + } + + auto range = m_free.upper_bound(search_start); + if (range != m_free.begin()) { + range--; + } + for (; range != m_free.end() && range->first < search_end; ++range) { + const auto range_end = std::min(range->first + range->second, search_end); + const auto lower_bound = std::max(range->first, search_start); + const auto aligned = GetAlignedPos(lower_bound, alignment); + const auto free_pos = aligned.value; + if (!aligned.valid || free_pos < lower_bound || free_pos > range_end || + len > range_end - free_pos) { + continue; + } + + AllocatedBlock b {}; + b.size = len; + b.start_addr = free_pos; + b.gpu_mode = GpuAccessMode::NoAccess; + b.map_size = 0; + b.map_vaddr = 0; + b.prot = 0; + b.mode = VirtualMemory::Mode::NoAccess; + b.memory_type = memory_type; + b.pool_expansion = pool_expansion; + + ConsumeFreeRange(range, free_pos, len); + EXIT_IF(!m_physical.emplace(b.start_addr, b).second); + + *phys_addr_out = free_pos; + return true; + } + + return false; +} + +bool PhysicalMemory::Available(uint64_t search_start, uint64_t search_end, size_t alignment, + uint64_t* phys_addr_out, uint64_t* size_out) { + if (phys_addr_out == nullptr || size_out == nullptr) { + return false; + } + + Common::LockGuard lock(m_mutex); + + search_end = std::min(search_end, Size()); + if (search_start >= search_end) { + return false; + } + + uint64_t best_addr = 0; + uint64_t best_size = 0; + + for (const auto& [range_start, range_size]: m_free) { + if (range_start >= search_end) { + break; + } + const auto range_end = std::min(range_start + range_size, search_end); + const auto lower_bound = std::max(range_start, search_start); + const auto aligned = GetAlignedPos(lower_bound, alignment); + const auto free_pos = aligned.value; + if (aligned.valid && free_pos >= lower_bound && free_pos < range_end && + range_end - free_pos > best_size) { + best_addr = free_pos; + best_size = range_end - free_pos; + } + } + + if (best_size == 0) { + return false; + } + + *phys_addr_out = best_addr; + *size_out = best_size; + return true; +} + +void PhysicalMemory::ConsumeFreeRange(std::map::iterator range, uint64_t start, + uint64_t size) { + const auto range_start = range->first; + const auto range_end = range->first + range->second; + m_free.erase(range); + if (range_start < start) { + m_free.emplace(range_start, start - range_start); + } + if (start + size < range_end) { + m_free.emplace(start + size, range_end - start - size); + } +} + +void PhysicalMemory::AddFreeRange(uint64_t start, uint64_t size) { + auto end = start + size; + auto next = m_free.lower_bound(start); + if (next != m_free.begin()) { + auto previous = std::prev(next); + if (previous->first + previous->second >= start) { + start = previous->first; + end = std::max(end, previous->first + previous->second); + next = m_free.erase(previous); + } + } + while (next != m_free.end() && next->first <= end) { + end = std::max(end, next->first + next->second); + next = m_free.erase(next); + } + m_free.emplace(start, end - start); +} + +bool PhysicalMemory::Release(uint64_t start, size_t len, uint64_t* vaddr, uint64_t* size, + GpuAccessMode* gpu_mode) { + EXIT_IF(vaddr == nullptr); + EXIT_IF(size == nullptr); + EXIT_IF(gpu_mode == nullptr); + + Common::LockGuard lock(m_mutex); + + auto next = m_physical.upper_bound(start); + if (next == m_physical.begin()) { + return false; + } + auto it = std::prev(next); + auto& b = it->second; + if (b.pool_expansion || start < b.start_addr || start >= b.start_addr + b.size || + len > b.start_addr + b.size - start) { + return false; + } + + if (start == b.start_addr && len == b.size) { + *vaddr = b.map_vaddr; + *size = b.map_size; + *gpu_mode = b.gpu_mode; + + m_physical.erase(it); + AddFreeRange(start, len); + return true; + } + if (start > b.start_addr && start + len < b.start_addr + b.size) { + auto old_start = b.start_addr; + auto old_end = b.start_addr + b.size; + + *vaddr = (b.map_vaddr != 0 ? b.map_vaddr + (start - old_start) : 0); + *size = (b.map_vaddr != 0 ? len : 0); + *gpu_mode = b.gpu_mode; + + AllocatedBlock right = b; + right.start_addr = start + len; + right.size = old_end - right.start_addr; + if (right.map_vaddr != 0) { + right.map_vaddr += right.start_addr - old_start; + right.map_size = right.size; + } + + b.size = start - old_start; + if (b.map_vaddr != 0) { + b.map_size = b.size; + } + + m_physical.emplace(right.start_addr, right); + AddFreeRange(start, len); + return true; + } + if (start == b.start_addr && len < b.size) { + *vaddr = b.map_vaddr; + *size = (b.map_vaddr != 0 ? len : 0); + *gpu_mode = b.gpu_mode; + + AllocatedBlock remaining = b; + m_physical.erase(it); + remaining.start_addr += len; + remaining.size -= len; + if (remaining.map_vaddr != 0) { + remaining.map_vaddr += len; + remaining.map_size -= len; + } + m_physical.emplace(remaining.start_addr, remaining); + AddFreeRange(start, len); + return true; + } + if (start > b.start_addr && start + len == b.start_addr + b.size) { + *vaddr = (b.map_vaddr != 0 ? b.map_vaddr + (start - b.start_addr) : 0); + *size = (b.map_vaddr != 0 ? len : 0); + *gpu_mode = b.gpu_mode; + + b.size = start - b.start_addr; + if (b.map_vaddr != 0) { + b.map_size = b.size; + } + AddFreeRange(start, len); + return true; + } + + return false; +} + +bool PhysicalMemory::Map(uint64_t vaddr, uint64_t phys_addr, size_t len, int prot, + VirtualMemory::Mode mode, GpuAccessMode gpu_mode) { + Common::LockGuard lock(m_mutex); + + auto next = m_physical.upper_bound(phys_addr); + if (next == m_physical.begin()) { + return false; + } + const auto& block = std::prev(next)->second; + if (block.pool_expansion || phys_addr < block.start_addr || + phys_addr >= block.start_addr + block.size || + len > block.start_addr + block.size - phys_addr) { + return false; + } + + AllocatedBlock mapping = block; + mapping.start_addr = phys_addr; + mapping.size = len; + mapping.map_vaddr = vaddr; + mapping.map_size = len; + mapping.host_vaddr = vaddr; + mapping.host_size = len; + mapping.prot = prot; + mapping.mode = mode; + mapping.gpu_mode = gpu_mode; + m_mappings.push_back(mapping); + + return true; +} + +bool PhysicalMemory::CanMapDirect(uint64_t phys_addr, size_t len) { + Common::LockGuard lock(m_mutex); + + auto next = m_physical.upper_bound(phys_addr); + if (next == m_physical.begin()) { + return false; + } + const auto& block = std::prev(next)->second; + return !block.pool_expansion && phys_addr >= block.start_addr && + phys_addr < block.start_addr + block.size && + len <= block.start_addr + block.size - phys_addr; +} + +bool PhysicalMemory::ReleasePoolExpansion(uint64_t phys_addr, size_t len) { + Common::LockGuard lock(m_mutex); + const auto it = m_physical.find(phys_addr); + if (it == m_physical.end() || !it->second.pool_expansion || it->second.size != len) { + return false; + } + m_physical.erase(it); + AddFreeRange(phys_addr, len); + return true; +} + +bool PhysicalMemory::Unmap(uint64_t vaddr, uint64_t size, GpuAccessMode* gpu_mode, + uint64_t* host_vaddr_to_release) { + EXIT_IF(gpu_mode == nullptr); + + Common::LockGuard lock(m_mutex); + + if (host_vaddr_to_release != nullptr) { + *host_vaddr_to_release = 0; + } + + auto set_host_release_if_last = [this, host_vaddr_to_release](uint64_t host_vaddr, + uint64_t host_size) { + if (host_vaddr_to_release == nullptr || host_vaddr == 0 || host_size == 0) { + return; + } + const bool still_mapped = std::any_of( + m_mappings.begin(), m_mappings.end(), [host_vaddr, host_size](const auto& block) { + return block.host_vaddr == host_vaddr && block.host_size == host_size; + }); + if (!still_mapped) { + *host_vaddr_to_release = host_vaddr; + } + }; + + size_t index = 0; + for (auto& b: m_mappings) { + if (b.map_vaddr == vaddr && b.map_size == size) { + *gpu_mode = b.gpu_mode; + const auto host_vaddr = b.host_vaddr; + const auto host_size = b.host_size; + + m_mappings.erase(m_mappings.begin() + static_cast(index)); + set_host_release_if_last(host_vaddr, host_size); + + return true; + } + if (vaddr > b.map_vaddr && vaddr + size < b.map_vaddr + b.map_size) { + *gpu_mode = b.gpu_mode; + + AllocatedBlock right = b; + right.start_addr += (vaddr + size) - b.map_vaddr; + right.size = b.map_vaddr + b.map_size - (vaddr + size); + right.map_size = right.size; + right.map_vaddr = vaddr + size; + + b.size = vaddr - b.map_vaddr; + b.map_size = b.size; + m_mappings.push_back(right); + return true; + } + if (vaddr == b.map_vaddr && size < b.map_size) { + *gpu_mode = b.gpu_mode; + + b.start_addr += size; + b.size -= size; + b.map_vaddr += size; + b.map_size -= size; + return true; + } + if (vaddr > b.map_vaddr && vaddr + size == b.map_vaddr + b.map_size) { + *gpu_mode = b.gpu_mode; + + b.size = vaddr - b.map_vaddr; + b.map_size = b.size; + return true; + } + index++; + } + + return false; +} + +bool PhysicalMemory::Find(uint64_t phys_addr, bool next, AllocatedBlock* out) { + EXIT_IF(out == nullptr); + + Common::LockGuard lock(m_mutex); + + auto following = m_physical.upper_bound(phys_addr); + if (following != m_physical.begin()) { + const auto& block = std::prev(following)->second; + if (phys_addr < block.start_addr + block.size) { + *out = block; + return true; + } + } + + if (next && following != m_physical.end()) { + *out = following->second; + return true; + } + + return false; +} + +std::vector PhysicalMemory::FindMappings(uint64_t phys_addr, + size_t len) { + Common::LockGuard lock(m_mutex); + + std::vector mappings; + for (const auto& m: m_mappings) { + if (!VirtualRangesOverlap(phys_addr, len, m.start_addr, m.size)) { + continue; + } + + const uint64_t overlap_start = std::max(phys_addr, m.start_addr); + const uint64_t overlap_end = std::min(phys_addr + len, m.start_addr + m.size); + AllocatedBlock part = m; + part.start_addr = overlap_start; + part.size = overlap_end - overlap_start; + part.map_vaddr += overlap_start - m.start_addr; + part.map_size = part.size; + mappings.push_back(part); + } + + std::sort(mappings.begin(), mappings.end(), + [](const auto& a, const auto& b) { return a.map_vaddr < b.map_vaddr; }); + return mappings; +} + +bool PhysicalMemory::Find(uint64_t vaddr, uint64_t* base_addr, size_t* len, int* prot, + VirtualMemory::Mode* mode, GpuAccessMode* gpu_mode) { + Common::LockGuard lock(m_mutex); + + return std::any_of(m_mappings.begin(), m_mappings.end(), + [vaddr, base_addr, len, prot, mode, gpu_mode](auto& b) { + if (vaddr >= b.map_vaddr && vaddr < b.map_vaddr + b.map_size) { + if (base_addr != nullptr) { + *base_addr = b.map_vaddr; + } + if (len != nullptr) { + *len = b.map_size; + } + if (prot != nullptr) { + *prot = b.prot; + } + if (mode != nullptr) { + *mode = b.mode; + } + if (gpu_mode != nullptr) { + *gpu_mode = b.gpu_mode; + } + + return true; + } + return false; + }); +} + +void PhysicalMemory::SetVirtualRangeName(uint64_t vaddr, uint64_t len, const char* name) { + Common::LockGuard lock(m_mutex); + + for (auto& b: m_mappings) { + if (VirtualRangesOverlap(vaddr, len, b.map_vaddr, b.map_size)) { + CopyVirtualRangeName(b.name, name); + } + } +} + +bool FlexibleMemory::Map(uint64_t vaddr, size_t len, int prot, VirtualMemory::Mode mode, + GpuAccessMode gpu_mode, const char* name) { + Common::LockGuard lock(m_mutex); + + const auto available = (Size() >= m_allocated_total ? Size() - m_allocated_total : 0); + if (len == 0 || len > available) { + LOGF_COLOR(Log::Color::Red, + "\t flexible memory exhausted: configured = 0x%016" PRIx64 + ", allocated = 0x%016" PRIx64 ", available = 0x%016" PRIx64 + ", requested = 0x%016" PRIx64 "\n", + Size(), m_allocated_total, available, static_cast(len)); + return false; + } + + AllocatedBlock b {}; + b.map_vaddr = vaddr; + b.map_size = len; + b.host_vaddr = vaddr; + b.host_size = len; + b.prot = prot; + b.mode = mode; + b.gpu_mode = gpu_mode; + CopyVirtualRangeName(b.name, name); + + m_allocated.push_back(b); + m_allocated_total += len; + + return true; +} + +bool FlexibleMemory::Unmap(uint64_t vaddr, uint64_t size, GpuAccessMode* gpu_mode, + uint64_t* host_vaddr_to_release) { + EXIT_IF(gpu_mode == nullptr); + + Common::LockGuard lock(m_mutex); + + if (host_vaddr_to_release != nullptr) { + *host_vaddr_to_release = 0; + } + + auto set_host_release_if_last = [this, host_vaddr_to_release](uint64_t host_vaddr, + uint64_t host_size) { + if (host_vaddr_to_release == nullptr || host_vaddr == 0 || host_size == 0) { + return; + } + const bool still_mapped = std::any_of( + m_allocated.begin(), m_allocated.end(), [host_vaddr, host_size](const auto& block) { + return block.host_vaddr == host_vaddr && block.host_size == host_size; + }); + if (!still_mapped) { + *host_vaddr_to_release = host_vaddr; + } + }; + + size_t index = 0; + for (auto& b: m_allocated) { + if (b.map_vaddr == vaddr && b.map_size == size) { + *gpu_mode = b.gpu_mode; + const auto host_vaddr = b.host_vaddr; + const auto host_size = b.host_size; + + m_allocated.erase(m_allocated.begin() + static_cast(index)); + m_allocated_total -= size; + set_host_release_if_last(host_vaddr, host_size); + return true; + } + if (vaddr > b.map_vaddr && vaddr + size < b.map_vaddr + b.map_size) { + *gpu_mode = b.gpu_mode; + + AllocatedBlock right = b; + right.map_size = b.map_vaddr + b.map_size - (vaddr + size); + right.map_vaddr = vaddr + size; + + b.map_size = vaddr - b.map_vaddr; + m_allocated.push_back(right); + m_allocated_total -= size; + return true; + } + if (vaddr == b.map_vaddr && size < b.map_size) { + *gpu_mode = b.gpu_mode; + + b.map_vaddr += size; + b.map_size -= size; + m_allocated_total -= size; + return true; + } + if (vaddr > b.map_vaddr && vaddr + size == b.map_vaddr + b.map_size) { + *gpu_mode = b.gpu_mode; + + b.map_size = vaddr - b.map_vaddr; + m_allocated_total -= size; + return true; + } + index++; + } + + return false; +} + +bool FlexibleMemory::Find(uint64_t vaddr, uint64_t* base_addr, size_t* len, int* prot, + VirtualMemory::Mode* mode, GpuAccessMode* gpu_mode) { + Common::LockGuard lock(m_mutex); + + return std::any_of(m_allocated.begin(), m_allocated.end(), + [vaddr, base_addr, len, prot, mode, gpu_mode](auto& b) { + if (vaddr >= b.map_vaddr && vaddr < b.map_vaddr + b.map_size) { + if (base_addr != nullptr) { + *base_addr = b.map_vaddr; + } + if (len != nullptr) { + *len = b.map_size; + } + if (prot != nullptr) { + *prot = b.prot; + } + if (mode != nullptr) { + *mode = b.mode; + } + if (gpu_mode != nullptr) { + *gpu_mode = b.gpu_mode; + } + + return true; + } + return false; + }); +} + +void FlexibleMemory::SetVirtualRangeName(uint64_t vaddr, uint64_t len, const char* name) { + Common::LockGuard lock(m_mutex); + + for (auto& b: m_allocated) { + if (VirtualRangesOverlap(vaddr, len, b.map_vaddr, b.map_size)) { + CopyVirtualRangeName(b.name, name); + } + } +} + +void PhysicalMemory::SetVirtualRangeMemoryType(uint64_t vaddr, uint64_t len, int memory_type) { + Common::LockGuard lock(m_mutex); + + for (auto& b: m_mappings) { + if (VirtualRangesOverlap(vaddr, len, b.map_vaddr, b.map_size)) { + b.memory_type = memory_type; + } + } +} + +uint64_t FlexibleMemory::Available() { + Common::LockGuard lock(m_mutex); + + return (Size() >= m_allocated_total ? Size() - m_allocated_total : 0); +} + +void PooledMemory::AddFreeUnlocked(uint64_t start, uint64_t size) { + if (size == 0) { + return; + } + + m_free.push_back({start, size}); + std::sort(m_free.begin(), m_free.end(), + [](const auto& left, const auto& right) { return left.start < right.start; }); + + std::vector merged; + for (const auto& range: m_free) { + if (!merged.empty() && range.start <= merged.back().start + merged.back().size) { + const auto end = + std::max(merged.back().start + merged.back().size, range.start + range.size); + merged.back().size = end - merged.back().start; + } else { + merged.push_back(range); + } + } + m_free = std::move(merged); +} + +void PooledMemory::Expand(uint64_t phys_addr, uint64_t size) { + Common::LockGuard lock(m_mutex); + m_expansions.push_back({phys_addr, size}); + AddFreeUnlocked(phys_addr, size); +} + +bool PooledMemory::ReleaseExpansion(uint64_t phys_addr, uint64_t size) { + Common::LockGuard lock(m_mutex); + const auto expansion = std::find_if(m_expansions.begin(), m_expansions.end(), + [phys_addr, size](const auto& range) { + return range.start == phys_addr && range.size == size; + }); + if (expansion == m_expansions.end()) { + return false; + } + if (std::any_of(m_mappings.begin(), m_mappings.end(), [phys_addr, size](const auto& mapping) { + return VirtualRangesOverlap(phys_addr, size, mapping.phys_addr, mapping.size); + })) { + return false; + } + + const auto end = phys_addr + size; + const auto free_range = + std::find_if(m_free.begin(), m_free.end(), [phys_addr, end](const auto& r) { + return phys_addr >= r.start && end <= r.start + r.size; + }); + if (free_range == m_free.end()) { + return false; + } + + const auto old = *free_range; + const auto old_end = old.start + old.size; + const auto left_size = phys_addr - old.start; + const auto right_size = old_end - end; + m_free.erase(free_range); + if (left_size != 0) { + m_free.push_back({old.start, left_size}); + } + if (right_size != 0) { + m_free.push_back({end, right_size}); + } + std::sort(m_free.begin(), m_free.end(), + [](const auto& left, const auto& right) { return left.start < right.start; }); + m_expansions.erase(expansion); + return true; +} + +bool PooledMemory::Allocate(uint64_t vaddr, uint64_t size, GpuAccessMode gpu_mode, + std::vector* mappings) { + EXIT_IF(mappings == nullptr); + mappings->clear(); + if (vaddr == 0 || size == 0 || UINT64_MAX - vaddr < size) { + return false; + } + + Common::LockGuard lock(m_mutex); + auto free = m_free; + auto current = vaddr; + auto remaining = size; + for (auto& range: free) { + if (remaining == 0 || range.size == 0) { + continue; + } + const auto part_size = std::min(range.size, remaining); + mappings->push_back({current, part_size, range.start, gpu_mode}); + range.start += part_size; + range.size -= part_size; + current += part_size; + remaining -= part_size; + } + if (remaining != 0) { + mappings->clear(); + return false; + } + + free.erase( + std::remove_if(free.begin(), free.end(), [](const auto& range) { return range.size == 0; }), + free.end()); + m_free = std::move(free); + m_mappings.insert(m_mappings.end(), mappings->begin(), mappings->end()); + return true; +} + +bool PooledMemory::QueryUnlocked(uint64_t vaddr, uint64_t size, + std::vector* mappings) const { + EXIT_IF(mappings == nullptr); + mappings->clear(); + if (vaddr == 0 || size == 0 || UINT64_MAX - vaddr < size) { + return false; + } + + const auto end = vaddr + size; + auto current = vaddr; + while (current < end) { + const auto it = + std::find_if(m_mappings.begin(), m_mappings.end(), [current](const auto& m) { + return current >= m.vaddr && current < m.vaddr + m.size; + }); + if (it == m_mappings.end()) { + mappings->clear(); + return false; + } + const auto part_size = std::min(end, it->vaddr + it->size) - current; + mappings->push_back( + {current, part_size, it->phys_addr + (current - it->vaddr), it->gpu_mode}); + current += part_size; + } + return true; +} + +bool PooledMemory::Query(uint64_t vaddr, uint64_t size, std::vector* mappings) { + Common::LockGuard lock(m_mutex); + return QueryUnlocked(vaddr, size, mappings); +} + +bool PooledMemory::Release(uint64_t vaddr, uint64_t size, GpuAccessMode* gpu_mode) { + EXIT_IF(gpu_mode == nullptr); + + Common::LockGuard lock(m_mutex); + std::vector released; + if (!QueryUnlocked(vaddr, size, &released)) { + return false; + } + + *gpu_mode = released.front().gpu_mode; + const auto end = vaddr + size; + std::vector kept; + for (const auto& mapping: m_mappings) { + const auto mapping_end = mapping.vaddr + mapping.size; + const auto cut_start = std::max(vaddr, mapping.vaddr); + const auto cut_end = std::min(end, mapping_end); + if (cut_start >= cut_end) { + kept.push_back(mapping); + continue; + } + if (mapping.vaddr < cut_start) { + auto left = mapping; + left.size = cut_start - mapping.vaddr; + kept.push_back(left); + } + if (cut_end < mapping_end) { + auto right = mapping; + right.vaddr = cut_end; + right.size = mapping_end - cut_end; + right.phys_addr = mapping.phys_addr + (cut_end - mapping.vaddr); + kept.push_back(right); + } + } + m_mappings = std::move(kept); + for (const auto& mapping: released) { + AddFreeUnlocked(mapping.phys_addr, mapping.size); + } + return true; +} + +uint64_t PooledMemory::Available() { + Common::LockGuard lock(m_mutex); + uint64_t available = 0; + for (const auto& range: m_free) { + available += range.size; + } + return available; +} + +std::vector PooledMemory::GetMappings() { + Common::LockGuard lock(m_mutex); + return m_mappings; +} + +static bool UnmapPooledBackingTransactional(const std::vector& mappings, + VirtualMemory::Mode mode) { + struct RemovedMapping { + PooledMemory::Mapping mapping; + bool placeholder_preserved; + }; + + std::vector removed; + for (const auto& mapping: mappings) { + bool placeholder_preserved = false; + if (!g_direct_memory_backing->Unmap(mapping.vaddr, mapping.size, true, + &placeholder_preserved)) { + for (auto it = removed.rbegin(); it != removed.rend(); ++it) { + auto failure_reason = DirectMemoryBacking::FailureReason::None; + const bool restored = it->placeholder_preserved + ? g_direct_memory_backing->MapExistingPlaceholderFixed( + it->mapping.vaddr, it->mapping.size, + it->mapping.phys_addr, mode, &failure_reason) + : g_direct_memory_backing->MapFixed( + it->mapping.vaddr, it->mapping.size, + it->mapping.phys_addr, mode, &failure_reason); + if (!restored) { + EXIT("pooled-memory unmap rollback failed: %s\n", + DirectMemoryBacking::GetFailureReasonName(failure_reason)); + } + } + return false; + } + removed.push_back({mapping, placeholder_preserved}); + } + + for (const auto& entry: removed) { + if (entry.placeholder_preserved) { + g_placeholder_address_space->AddFree(entry.mapping.vaddr, entry.mapping.size); + } else if (!g_placeholder_address_space->ReserveFixed(entry.mapping.vaddr, + entry.mapping.size)) { + EXIT("failed to reserve pooled-memory placeholder\n"); + } + } + return true; +} + +int32_t KYTY_SYSV_ABI KernelMapNamedFlexibleMemory(void** addr_in_out, size_t len, int prot, + int flags, const char* name) { + PRINT_NAME(); + + std::lock_guard memory_operation_lock(g_memory_operation_mutex); + + EXIT_NOT_IMPLEMENTED(addr_in_out == nullptr); + + constexpr size_t PAGE_SIZE = 0x4000; + constexpr size_t MAXIMUM_NAME_SIZE = 32; + constexpr uint64_t DEFAULT_PS5_BASE = 0x200000000; + constexpr int MAP_FIXED = 0x10; + constexpr int MAP_SHARED = 0x01; + constexpr int MAP_PRIVATE = 0x02; + constexpr int MAP_NO_OVERWRITE = 0x80; + constexpr int MAP_VOID = 0x100; + constexpr int MAP_STACK = 0x400; + constexpr int MAP_NO_SYNC = 0x800; + constexpr int MAP_ANON = 0x1000; + constexpr int MAP_UNKNOWN_8000 = 0x8000; + constexpr int MAP_NO_CORE = 0x20000; + constexpr int MAP_NO_COALESCE = 0x400000; + constexpr int SUPPORTED_MAP_BITS = MAP_SHARED | MAP_PRIVATE | MAP_FIXED | MAP_NO_OVERWRITE | + MAP_VOID | MAP_STACK | MAP_NO_SYNC | MAP_ANON | + MAP_UNKNOWN_8000 | MAP_NO_CORE | MAP_NO_COALESCE; + + if (len == 0 || (len & (PAGE_SIZE - 1)) != 0) { + return KERNEL_ERROR_EINVAL; + } + + if (name == nullptr) { + return KERNEL_ERROR_EFAULT; + } + + if (std::strlen(name) >= MAXIMUM_NAME_SIZE) { + return KERNEL_ERROR_ENAMETOOLONG; + } + + if ((flags & ~SUPPORTED_MAP_BITS) != 0) { + LOGF_COLOR(Log::Color::Red, "\t unsupported flags = 0x%08" PRIx32 "\n", + static_cast(flags & ~SUPPORTED_MAP_BITS)); + return KERNEL_ERROR_EINVAL; + } + + VirtualMemory::Mode mode = VirtualMemory::Mode::NoAccess; + GpuAccessMode gpu_mode = GpuAccessMode::NoAccess; + + if (!DecodeMemoryProtection(prot, &mode, &gpu_mode)) { + EXIT("unknown prot: %d\n", prot); + } + + auto in_addr = reinterpret_cast(*addr_in_out); + uint64_t out_addr = 0; + bool committed_from_reserved = false; + bool consumed_reserved = false; + VirtualRanges::Range consumed_range {}; + + if ((flags & MAP_FIXED) != 0) { + if (in_addr == 0 || (in_addr & (PAGE_SIZE - 1)) != 0) { + return KERNEL_ERROR_EINVAL; + } + if ((flags & MAP_NO_OVERWRITE) != 0 && g_virtual_ranges->HasOverlap(in_addr, len)) { + return KERNEL_ERROR_ENOMEM; + } + if (g_virtual_ranges->Query(in_addr, 0, &consumed_range) && + consumed_range.type == VirtualRangeType::Reserved && + g_virtual_ranges->ConsumeReserved(in_addr, len)) { + consumed_reserved = true; + if (g_placeholder_address_space->Commit(in_addr, len, mode) || + CommitFixedHostRange(in_addr, len, mode)) { + out_addr = in_addr; + committed_from_reserved = true; + } + } else if (!ReleaseReservedRange(in_addr, len)) { + return KERNEL_ERROR_ENOMEM; + } else if (VirtualMemory::AllocFixed(in_addr, len, mode)) { + out_addr = in_addr; + } + } else { + const auto search_addr = (in_addr != 0 ? in_addr : DEFAULT_PS5_BASE); + out_addr = VirtualMemory::AllocAligned(search_addr, len, mode, PAGE_SIZE); + } + + *addr_in_out = reinterpret_cast(out_addr); + + if (out_addr == 0) { + if (consumed_reserved) { + g_virtual_ranges->Add(in_addr, len, 0, 0, 0, VirtualRangeType::Reserved, + consumed_range.name, false, consumed_range.placeholder_backed); + } + return KERNEL_ERROR_ENOMEM; + } + + if (!g_flexible_memory->Map(out_addr, len, prot, mode, gpu_mode, name)) { + LOGF_COLOR(Log::Color::Red, "\t [Fail]\n"); + if (committed_from_reserved) { + const bool placeholder_backed = RestoreCommittedPlaceholderOrProtect(out_addr, len); + g_virtual_ranges->Add(out_addr, len, 0, 0, 0, VirtualRangeType::Reserved, name, false, + placeholder_backed); + } else { + VirtualMemory::Free(out_addr); + } + return KERNEL_ERROR_ENOMEM; + } + + const auto range_type = + ((flags & MAP_STACK) != 0 ? VirtualRangeType::Stack : VirtualRangeType::Flexible); + if (!g_virtual_ranges->Add(out_addr, len, 0, prot, 0, range_type, name, + committed_from_reserved)) { + GpuAccessMode rollback_gpu_mode = GpuAccessMode::NoAccess; + g_flexible_memory->Unmap(out_addr, len, &rollback_gpu_mode); + if (committed_from_reserved) { + const bool placeholder_backed = RestoreCommittedPlaceholderOrProtect(out_addr, len); + g_virtual_ranges->Add(out_addr, len, 0, 0, 0, VirtualRangeType::Reserved, name, false, + placeholder_backed); + } else { + VirtualMemory::Free(out_addr); + } + return KERNEL_ERROR_EBUSY; + } + + LOGF("\t in_addr = 0x%016" PRIx64 "\n" + "\t out_addr = 0x%016" PRIx64 "\n" + "\t size = %" PRIu64 "\n" + "\t mode = %s\n" + "\t flags = 0x%08" PRIx32 "\n" + "\t name = %s\n" + "\t gpu_mode = %s\n", + in_addr, out_addr, len, Common::EnumName(mode).c_str(), static_cast(flags), name, + Common::EnumName(gpu_mode).c_str()); + + MapGpuRange(out_addr, len, gpu_mode); + + if (g_alloc_callback != nullptr) { + g_alloc_callback(out_addr, len); + } + + return OK; +} + +int KYTY_SYSV_ABI KernelMapFlexibleMemory(void** addr_in_out, size_t len, int prot, int flags) { + return KernelMapNamedFlexibleMemory(addr_in_out, len, prot, flags, ""); +} + +int KYTY_SYSV_ABI KernelSetPrtAperture(int index, void* addr, size_t len) { + PRINT_NAME(); + + const auto address = reinterpret_cast(addr); + + LOGF("\t index = %d\n" + "\t addr = 0x%016" PRIx64 "\n" + "\t len = 0x%016" PRIx64 "\n", + index, address, static_cast(len)); + + if (index < 0 || index > PRT_APERTURE_MAX_INDEX) { + return KERNEL_ERROR_EINVAL; + } + + if (len == 0) { + Common::LockGuard lock(g_prt_aperture_mutex); + const auto old = g_prt_apertures[static_cast(index)]; + if (old.size != 0) { + UnmapGpuRange(old.address, old.size, GpuAccessMode::ReadWrite); + } + g_prt_apertures[static_cast(index)] = {}; + LOGF_COLOR(Log::Color::Green, "\t[Ok]\n"); + return OK; + } + + if (address == 0 || (address & (PRT_PAGE_SIZE - 1u)) != 0 || + (len & (PRT_PAGE_SIZE - 1u)) != 0) { + return KERNEL_ERROR_EINVAL; + } + + if (address < PRT_APERTURE_START || len > PRT_APERTURE_END - address) { + return KERNEL_ERROR_EINVAL; + } + + { + Common::LockGuard lock(g_prt_aperture_mutex); + const auto old = g_prt_apertures[static_cast(index)]; + if (old.size != 0) { + UnmapGpuRange(old.address, old.size, GpuAccessMode::ReadWrite); + } + MapGpuRange(address, len, GpuAccessMode::ReadWrite); + g_prt_apertures[static_cast(index)] = {address, static_cast(len)}; + } + + LOGF_COLOR(Log::Color::Green, "\t[Ok]\n"); + + return OK; +} + +int KYTY_SYSV_ABI KernelGetPrtAperture(int index, void** addr, size_t* len) { + PRINT_NAME(); + + LOGF("\t index = %d\n" + "\t addr = %p\n" + "\t len = %p\n", + index, static_cast(addr), static_cast(len)); + + if (index < 0 || index > PRT_APERTURE_MAX_INDEX) { + return KERNEL_ERROR_EINVAL; + } + + if (addr == nullptr || len == nullptr) { + return KERNEL_ERROR_EFAULT; + } + + PrtAperture aperture {}; + { + Common::LockGuard lock(g_prt_aperture_mutex); + aperture = g_prt_apertures[static_cast(index)]; + } + + *addr = reinterpret_cast(aperture.address); + *len = static_cast(aperture.size); + + LOGF_COLOR(Log::Color::Green, + "\t *addr = 0x%016" PRIx64 "\n" + "\t *len = 0x%016" PRIx64 "\n" + "\t[Ok]\n", + aperture.address, aperture.size); + + return OK; +} + +int KYTY_SYSV_ABI KernelSetVirtualRangeName(const void* addr, uint64_t len, const char* name) { + PRINT_NAME(); + + std::lock_guard memory_operation_lock(g_memory_operation_mutex); + + auto vaddr = reinterpret_cast(addr); + + LOGF("\t addr = 0x%016" PRIx64 "\n" + "\t len = 0x%016" PRIx64 "\n" + "\t name = %s\n", + vaddr, len, name != nullptr ? name : "(null)"); + + if (name == nullptr) { + return KERNEL_ERROR_EFAULT; + } + + if (std::strlen(name) >= KERNEL_MAXIMUM_NAME_LENGTH) { + return KERNEL_ERROR_ENAMETOOLONG; + } + + g_physical_memory->SetVirtualRangeName(vaddr, len, name); + g_flexible_memory->SetVirtualRangeName(vaddr, len, name); + g_virtual_ranges->Rename(vaddr, len, name); + + return OK; +} + +int KYTY_SYSV_ABI KernelMunmap(uint64_t vaddr, size_t len) { + PRINT_NAME(); + + std::lock_guard memory_operation_lock(g_memory_operation_mutex); + + LOGF("\t start = 0x%016" PRIx64 "\n" + "\t len = 0x%016" PRIx64 "\n", + vaddr, len); + + if (len == 0 || UINT64_MAX - vaddr < len) { + return KERNEL_ERROR_EINVAL; + } + + VirtualRanges::Range range {}; + if (!g_virtual_ranges->Query(vaddr, 0, &range)) { + return KERNEL_ERROR_EACCES; + } + const auto chunk_len = std::min(len, range.size - (vaddr - range.start)); + if (chunk_len < len) { + const int ret = KernelMunmap(vaddr, chunk_len); + return ret == OK ? KernelMunmap(vaddr + chunk_len, len - chunk_len) : ret; + } + if (IsReservedRangeType(range.type)) { + const bool released = range.placeholder_backed + ? g_placeholder_address_space->ReleaseFree(vaddr, len) + : VirtualMemory::FreeRange(vaddr, len); + if (!released) { + return KERNEL_ERROR_EACCES; + } + g_virtual_ranges->Remove(vaddr, len); + return OK; + } + UnmapGpuRange(vaddr, len, GetGpuAccessMode(range.protection)); + + size_t backend_len = 0; + uint64_t backend_base = 0; + GpuAccessMode gpu_mode = GpuAccessMode::NoAccess; + const bool pooled_backend = range.type == VirtualRangeType::Pooled; + bool backend_found = false; + bool exact_host_range = backend_found && backend_base == vaddr && backend_len == len; + bool backend_unmapped = false; + bool shared_unmapped = false; + bool placeholder_restored = false; + bool flexible_backend = false; + uint64_t physical_host_to_release = 0; + uint64_t flexible_host_to_release = 0; + const bool can_restore_placeholder = + range.committed_from_reserved && range.start == vaddr && range.size == len; + + if (pooled_backend) { + std::vector mappings; + backend_found = g_pooled_memory->Query(vaddr, len, &mappings); + exact_host_range = backend_found; + if (backend_found) { + VirtualMemory::Mode mode = VirtualMemory::Mode::NoAccess; + GpuAccessMode decoded_gpu = GpuAccessMode::NoAccess; + shared_unmapped = DecodeMemoryProtection(range.protection, &mode, &decoded_gpu) && + UnmapPooledBackingTransactional(mappings, mode); + if (shared_unmapped && !g_pooled_memory->Release(vaddr, len, &gpu_mode)) { + EXIT("failed to release unmapped pooled-memory range\n"); + } + backend_unmapped = shared_unmapped; + placeholder_restored = shared_unmapped; + } + if (!backend_unmapped) { + EXIT("pooled-memory unmap failed: addr=0x%016" PRIx64 " size=0x%016" PRIx64 "\n", vaddr, + len); + return KERNEL_ERROR_EACCES; + } + } else if ((backend_found = g_physical_memory->Find(vaddr, &backend_base, &backend_len, nullptr, + nullptr, &gpu_mode))) { + exact_host_range = backend_base == vaddr && backend_len == len; + backend_unmapped = + g_physical_memory->Unmap(vaddr, len, &gpu_mode, &physical_host_to_release); + if (backend_unmapped) { + const bool direct_contains = g_direct_memory_backing->Contains(vaddr, len); + if (direct_contains) { + shared_unmapped = g_direct_memory_backing->Unmap( + vaddr, len, can_restore_placeholder, &placeholder_restored); + } + if (placeholder_restored) { + g_placeholder_address_space->AddFree(vaddr, len); + } + } + } else { + flexible_backend = true; + backend_found = g_flexible_memory->Find(vaddr, &backend_base, &backend_len, nullptr, + nullptr, &gpu_mode); + exact_host_range = backend_found && backend_base == vaddr && backend_len == len; + backend_unmapped = (backend_found ? g_flexible_memory->Unmap(vaddr, len, &gpu_mode, + &flexible_host_to_release) + : false); + if (backend_unmapped && can_restore_placeholder) { + placeholder_restored = g_placeholder_address_space->ReleaseCommitted(vaddr, len); + } + } + if (!backend_unmapped) { + EXIT("memory backend unmap failed: addr=0x%016" PRIx64 " size=0x%016" PRIx64 "\n", vaddr, + len); + } + + g_virtual_ranges->Remove(vaddr, len); + + const bool host_range_released_or_shared = + flexible_backend ? flexible_host_to_release != 0 + : (shared_unmapped || physical_host_to_release != 0); + const bool can_release_host_range = exact_host_range && backend_unmapped && + !range.committed_from_reserved && + host_range_released_or_shared; + if (can_release_host_range && (vaddr != 0 || len != 0)) { + if (!shared_unmapped) { + VirtualMemory::Free(flexible_backend ? flexible_host_to_release + : physical_host_to_release); + } + } else if (IsCommittedRangeType(range.type)) { + constexpr uint64_t PAGE_SIZE = 0x4000; + auto aligned_addr = vaddr & ~(PAGE_SIZE - 1); + auto aligned_len = (len + (vaddr - aligned_addr) + PAGE_SIZE - 1) & ~(PAGE_SIZE - 1); + if (placeholder_restored) { + // Exact placeholder-backed unmaps already returned this range to the placeholder + // manager. + } else if (!shared_unmapped) { + VirtualMemory::Protect(aligned_addr, aligned_len, VirtualMemory::Mode::NoAccess); + } else if (range.committed_from_reserved) { + VirtualMemory::ReserveFixed(aligned_addr, aligned_len); + } + if (range.committed_from_reserved) { + g_virtual_ranges->Add(vaddr, len, 0, 0, 0, VirtualRangeType::Reserved, range.name, + false, placeholder_restored); + } + } + + if (g_free_callback != nullptr && IsCommittedRangeType(range.type)) { + g_free_callback(vaddr, len); + } + if (pooled_backend) { + MemoryPoolSubtractCommitted(len); + } + + return OK; +} + +size_t KYTY_SYSV_ABI KernelGetDirectMemorySize() { + PRINT_NAME(); + + return PhysicalMemory::Size(); +} + +int KYTY_SYSV_ABI KernelAvailableDirectMemorySize(int64_t search_start, int64_t search_end, + size_t alignment, int64_t* phys_addr_out, + size_t* size_out) { + PRINT_NAME(); + + std::lock_guard memory_operation_lock(g_memory_operation_mutex); + + LOGF("\t search_start = 0x%016" PRIx64 "\n" + "\t search_end = 0x%016" PRIx64 "\n" + "\t alignment = 0x%016" PRIx64 "\n", + search_start, search_end, static_cast(alignment)); + + if (phys_addr_out == nullptr || size_out == nullptr) { + return KERNEL_ERROR_EINVAL; + } + + *phys_addr_out = 0; + *size_out = 0; + + if (search_start < 0 || search_end < 0) { + return KERNEL_ERROR_EINVAL; + } + + if (search_end <= search_start) { + LOGF_COLOR(Log::Color::Red, "\t[Fail]\n"); + return KERNEL_ERROR_ENOMEM; + } + + uint64_t phys_addr = 0; + uint64_t size = 0; + if (!g_physical_memory->Available(static_cast(search_start), + static_cast(search_end), alignment, &phys_addr, + &size)) { + LOGF_COLOR(Log::Color::Red, "\t[Fail]\n"); + return KERNEL_ERROR_ENOMEM; + } + + *phys_addr_out = static_cast(phys_addr); + *size_out = static_cast(size); + + LOGF_COLOR(Log::Color::Green, + "\t phys_addr = 0x%016" PRIx64 "\n" + "\t size = 0x%016" PRIx64 "\n" + "\t[Ok]\n", + phys_addr, size); + + return OK; +} + +int KYTY_SYSV_ABI KernelGetPageTableStats(int* cpu_total, int* cpu_available, int* gpu_total, + int* gpu_available) { + PRINT_NAME(); + + std::lock_guard memory_operation_lock(g_memory_operation_mutex); + + if (cpu_total == nullptr || cpu_available == nullptr || gpu_total == nullptr || + gpu_available == nullptr) { + return KERNEL_ERROR_EFAULT; + } + + const auto cpu_used = + (g_virtual_ranges != nullptr ? g_virtual_ranges->CountPageTableEntries(false) : 0); + const auto gpu_used = + (g_virtual_ranges != nullptr ? g_virtual_ranges->CountPageTableEntries(true) : 0); + + *cpu_total = PAGE_TABLE_POOL_ENTRIES; + *gpu_total = PAGE_TABLE_POOL_ENTRIES; + *cpu_available = PAGE_TABLE_POOL_ENTRIES - + static_cast(std::min(cpu_used, PAGE_TABLE_POOL_ENTRIES)); + *gpu_available = PAGE_TABLE_POOL_ENTRIES - + static_cast(std::min(gpu_used, PAGE_TABLE_POOL_ENTRIES)); + + LOGF_COLOR(Log::Color::Green, + "\t cpu_total = %d\n" + "\t cpu_available = %d\n" + "\t gpu_total = %d\n" + "\t gpu_available = %d\n" + "\t[Ok]\n", + *cpu_total, *cpu_available, *gpu_total, *gpu_available); + + return OK; +} + +int KYTY_SYSV_ABI KernelDirectMemoryQuery(int64_t offset, int flags, void* info, size_t info_size) { + PRINT_NAME(); + + std::lock_guard memory_operation_lock(g_memory_operation_mutex); + + LOGF("\t offset = 0x%016" PRIx64 "\n" + "\t flags = 0x%08" PRIx32 "\n" + "\t info_size = 0x%016" PRIx64 "\n", + offset, flags, info_size); + + struct QueryInfo { + int64_t start; + int64_t end; + int memory_type; + }; + + if (offset < 0 || (flags != 0 && flags != 1) || info_size != sizeof(QueryInfo) || + info == nullptr) { + return KERNEL_ERROR_EINVAL; + } + + auto* query_info = static_cast(info); + + PhysicalMemory::AllocatedBlock block {}; + { + Common::LockGuard lock(g_physical_memory->GetMutex()); + const auto& blocks = g_physical_memory->GetPhysicalBlocks(); + auto current = blocks.upper_bound(static_cast(offset)); + if (current != blocks.begin()) { + auto previous = std::prev(current); + if (static_cast(offset) < + previous->second.start_addr + previous->second.size) { + current = previous; + } + } + if (current == blocks.end() || + (flags == 0 && (static_cast(offset) < current->second.start_addr || + static_cast(offset) >= + current->second.start_addr + current->second.size))) { + if (flags == 1 && static_cast(offset) < PhysicalMemory::Size()) { + query_info->start = static_cast(PhysicalMemory::Size()); + query_info->end = static_cast(PhysicalMemory::Size()); + query_info->memory_type = 0; + LOGF_COLOR(Log::Color::Green, "\t terminal = true\n\t[Ok]\n"); + return OK; + } + + LOGF_COLOR(Log::Color::Red, "\t[Fail]\n"); + return KERNEL_ERROR_EACCES; + } + + block = current->second; + uint64_t end = block.start_addr + block.size; + for (auto following = std::next(current); + following != blocks.end() && following->second.start_addr == end && + following->second.memory_type == block.memory_type; + ++following) { + end += following->second.size; + } + block.size = end - block.start_addr; + } + + query_info->start = static_cast(block.start_addr); + query_info->end = static_cast(block.start_addr + block.size); + query_info->memory_type = block.memory_type; + + LOGF_COLOR(Log::Color::Green, + "\t start = %016" PRIx64 "\n" + "\t end = %016" PRIx64 "\n" + "\t memory_type = %d\n" + "\t[Ok]\n", + query_info->start, query_info->end, query_info->memory_type); + + return OK; +} + +int KYTY_SYSV_ABI KernelAllocateDirectMemory(int64_t search_start, int64_t search_end, size_t len, + size_t alignment, int memory_type, + int64_t* phys_addr_out) { + PRINT_NAME(); + + std::lock_guard memory_operation_lock(g_memory_operation_mutex); + + LOGF("\t search_start = 0x%016" PRIx64 "\n" + "\t search_end = 0x%016" PRIx64 "\n" + "\t len = 0x%016" PRIx64 "\n" + "\t alignment = 0x%016" PRIx64 "\n" + "\t memory_type = %d\n", + search_start, search_end, len, alignment, memory_type); + + if (search_start < 0 || search_end <= search_start || len == 0 || phys_addr_out == nullptr) { + return KERNEL_ERROR_EINVAL; + } + + uint64_t addr = 0; + if (!g_physical_memory->Alloc(search_start, search_end, len, alignment, &addr, memory_type)) { + LOGF_COLOR(Log::Color::Red, "\t[Fail]\n"); + return KERNEL_ERROR_EAGAIN; + } + + *phys_addr_out = static_cast(addr); + + LOGF_COLOR(Log::Color::Green, "\tphys_addr = %016" PRIx64 "\n\t[Ok]\n", addr); + + return OK; +} + +int KYTY_SYSV_ABI KernelAllocateMainDirectMemory(size_t len, size_t alignment, int memory_type, + int64_t* phys_addr_out) { + PRINT_NAME(); + + std::lock_guard memory_operation_lock(g_memory_operation_mutex); + + LOGF("\t len = 0x%016" PRIx64 "\n" + "\t alignment = 0x%016" PRIx64 "\n" + "\t memory_type = %d\n", + len, alignment, memory_type); + + if (len == 0 || phys_addr_out == nullptr) { + return KERNEL_ERROR_EINVAL; + } + + uint64_t addr = 0; + if (!g_physical_memory->Alloc(0, PhysicalMemory::Size(), len, alignment, &addr, memory_type)) { + LOGF_COLOR(Log::Color::Red, "\t[Fail]\n"); + return KERNEL_ERROR_EAGAIN; + } + + *phys_addr_out = static_cast(addr); + + LOGF_COLOR(Log::Color::Green, "\tphys_addr = %016" PRIx64 "\n\t[Ok]\n", addr); + + return OK; +} + +int KYTY_SYSV_ABI KernelReleaseDirectMemory(int64_t start, size_t len) { + PRINT_NAME(); + + std::lock_guard memory_operation_lock(g_memory_operation_mutex); + + LOGF("\t start = 0x%016" PRIx64 "\n" + "\t len = 0x%016" PRIx64 "\n", + start, len); + + if (start < 0 || len == 0) { + return KERNEL_ERROR_EINVAL; + } + if (g_pooled_memory->ReleaseExpansion(static_cast(start), len)) { + if (!g_physical_memory->ReleasePoolExpansion(static_cast(start), len)) { + EXIT("failed to release physical pool expansion\n"); + } + return OK; + } + + uint64_t vaddr = 0; + uint64_t size = 0; + GpuAccessMode gpu_mode = GpuAccessMode::NoAccess; + + PhysicalMemory::AllocatedBlock block {}; + bool found = g_physical_memory->Find(start, false, &block); + bool exact_host_range = + found && block.start_addr == static_cast(start) && block.size == len; + VirtualRanges::Range range {}; + const bool range_found = (found ? g_virtual_ranges->Query(block.map_vaddr, 0, &range) : false); + bool shared_unmapped = false; + bool placeholder_restored = false; + const auto mapped_aliases = g_physical_memory->FindMappings(start, len); + for (const auto& alias: mapped_aliases) { + UnmapGpuRange(alias.map_vaddr, alias.map_size, alias.gpu_mode); + } + + bool result = g_physical_memory->Release(start, len, &vaddr, &size, &gpu_mode); + + if (!result) { + if (!mapped_aliases.empty()) { + EXIT("physical-memory release failed after GPU alias unmap: addr=0x%016" PRIx64 + " size=0x%016" PRIx64 " aliases=%zu\n", + start, len, mapped_aliases.size()); + } + return KERNEL_ERROR_EACCES; + } + + for (const auto& alias: mapped_aliases) { + GpuAccessMode alias_gpu_mode = GpuAccessMode::NoAccess; + uint64_t alias_host_to_release = 0; + if (!g_physical_memory->Unmap(alias.map_vaddr, alias.map_size, &alias_gpu_mode, + &alias_host_to_release)) { + EXIT("physical-memory alias unmap failed: addr=0x%016" PRIx64 " size=0x%016" PRIx64 + "\n", + alias.map_vaddr, alias.map_size); + } + + VirtualRanges::Range alias_range {}; + const bool alias_range_found = g_virtual_ranges->Query(alias.map_vaddr, 0, &alias_range); + g_virtual_ranges->Remove(alias.map_vaddr, alias.map_size); + + bool alias_shared_unmapped = false; + bool alias_placeholder_restored = false; + if (g_direct_memory_backing->Contains(alias.map_vaddr, alias.map_size)) { + const bool can_restore_placeholder = + alias_range_found && alias_range.committed_from_reserved && + alias_range.start == alias.map_vaddr && alias_range.size == alias.map_size; + alias_shared_unmapped = g_direct_memory_backing->Unmap(alias.map_vaddr, alias.map_size, + can_restore_placeholder, + &alias_placeholder_restored); + if (alias_placeholder_restored) { + g_placeholder_address_space->AddFree(alias.map_vaddr, alias.map_size); + } + } + + if (alias_range_found && alias_range.committed_from_reserved) { + g_virtual_ranges->Add(alias.map_vaddr, alias.map_size, 0, 0, 0, + VirtualRangeType::Reserved, alias_range.name, false, + alias_placeholder_restored); + } else if (!alias_shared_unmapped && alias_host_to_release != 0) { + VirtualMemory::Free(alias_host_to_release); + } else if (!alias_shared_unmapped) { + VirtualMemory::Protect(alias.map_vaddr, alias.map_size, VirtualMemory::Mode::NoAccess); + } + } + + if (vaddr != 0 || size != 0) { + g_virtual_ranges->Remove(vaddr, size); + const bool can_restore_placeholder = range_found && range.committed_from_reserved && + range.start == vaddr && range.size == size; + shared_unmapped = g_direct_memory_backing->Unmap(vaddr, size, can_restore_placeholder, + &placeholder_restored); + if (placeholder_restored) { + g_placeholder_address_space->AddFree(vaddr, size); + } + } + + if (exact_host_range && (!range_found || !range.committed_from_reserved) && + (vaddr != 0 || size != 0)) { + if (!shared_unmapped) { + VirtualMemory::Free(vaddr); + } + } else if (vaddr != 0 || size != 0) { + if (placeholder_restored) { + // Exact placeholder-backed unmaps already returned this range to the placeholder + // manager. + } else if (!shared_unmapped) { + VirtualMemory::Protect(vaddr, size, VirtualMemory::Mode::NoAccess); + } else if (range_found && range.committed_from_reserved) { + VirtualMemory::ReserveFixed(vaddr, size); + } + if (range_found && range.committed_from_reserved) { + g_virtual_ranges->Add(vaddr, size, 0, 0, 0, VirtualRangeType::Reserved, range.name, + false, placeholder_restored); + } + } + + if (g_free_callback != nullptr) { + g_free_callback(vaddr, len); + } + + return OK; +} + +int KYTY_SYSV_ABI KernelCheckedReleaseDirectMemory(int64_t start, size_t len) { + PRINT_NAME(); + + LOGF("\t start = 0x%016" PRIx64 "\n" + "\t len = 0x%016" PRIx64 "\n", + start, len); + + if (start < 0) { + return KERNEL_ERROR_EINVAL; + } + + constexpr size_t PAGE_SIZE = 0x4000; + if ((static_cast(start) & (PAGE_SIZE - 1)) != 0 || (len & (PAGE_SIZE - 1)) != 0) { + return KERNEL_ERROR_EINVAL; + } + + if (len == 0) { + return OK; + } + + return KernelReleaseDirectMemory(start, len); +} + +int KYTY_SYSV_ABI KernelMapDirectMemory(void** addr, size_t len, int prot, int flags, + int64_t direct_memory_start, size_t alignment) { + PRINT_NAME(); + + std::lock_guard memory_operation_lock(g_memory_operation_mutex); + + EXIT_NOT_IMPLEMENTED(addr == nullptr); + constexpr int MAP_FIXED = 0x10; + constexpr int MAP_NO_OVERWRITE = 0x80; + + bool fixed = ((flags & MAP_FIXED) != 0); + bool no_overwrite = ((flags & MAP_NO_OVERWRITE) != 0); + + VirtualMemory::Mode mode = VirtualMemory::Mode::NoAccess; + GpuAccessMode gpu_mode = GpuAccessMode::NoAccess; + + if (!DecodeMemoryProtection(prot, &mode, &gpu_mode)) { + EXIT("unknown prot: %d\n", prot); + } + if (direct_memory_start < 0 || len == 0 || + !g_physical_memory->CanMapDirect(static_cast(direct_memory_start), len)) { + return KERNEL_ERROR_ENOMEM; + } + + auto in_addr = reinterpret_cast(*addr); + uint64_t out_addr = 0; + bool committed_from_reserved = false; + bool shared_backing = false; + bool consumed_reserved = false; + VirtualRanges::Range consumed_range {}; + auto shared_failure = DirectMemoryBacking::FailureReason::None; + // Direct mappings must remain views of the single backing object. Anonymous fallbacks break + // aliasing and lose direct-memory contents when a range is unmapped and mapped again. + auto map_shared_fixed = [&](uint64_t target_addr) -> bool { + const bool placeholder_ready = g_placeholder_address_space->Consume(target_addr, len); + if (placeholder_ready) { + if (g_direct_memory_backing->MapExistingPlaceholderFixed( + target_addr, len, direct_memory_start, mode, &shared_failure)) { + return true; + } + g_placeholder_address_space->AddFree(target_addr, len); + return false; + } + + return g_direct_memory_backing->MapFixed(target_addr, len, direct_memory_start, mode, + &shared_failure); + }; + auto map_consumed_reserved_fixed = [&]() { + consumed_reserved = true; + if (map_shared_fixed(in_addr)) { + out_addr = in_addr; + committed_from_reserved = true; + shared_backing = true; + } + }; + + if (fixed) { + EXIT_NOT_IMPLEMENTED(in_addr == 0); + EXIT_NOT_IMPLEMENTED(alignment != 0 && (in_addr & (alignment - 1)) != 0); + if (no_overwrite && g_virtual_ranges->HasOverlap(in_addr, len)) { + return KERNEL_ERROR_ENOMEM; + } + + VirtualRanges::Range same_range {}; + if (g_virtual_ranges->Query(in_addr, 0, &same_range) && + same_range.type == VirtualRangeType::Direct && + len <= same_range.start + same_range.size - in_addr && + same_range.offset + (in_addr - same_range.start) == + static_cast(direct_memory_start) && + same_range.protection == prot) { + *addr = reinterpret_cast(in_addr); + LOGF_COLOR(Log::Color::Green, + "\t in_addr = 0x%016" PRIx64 "\n" + "\t out_addr = 0x%016" PRIx64 "\n" + "\t dmem = 0x%016" PRIx64 "\n" + "\t size = 0x%016" PRIx64 "\n" + "\t mode = %s\n" + "\t flags = 0x%08" PRIx32 "\n" + "\t align = 0x%016" PRIx64 "\n" + "\t gpu_mode = %s\n" + "\t shared = %s\n" + "\t reason = already-mapped\n" + "\t [Ok]\n", + in_addr, in_addr, static_cast(direct_memory_start), len, + Common::EnumName(mode).c_str(), static_cast(flags), alignment, + Common::EnumName(gpu_mode).c_str(), + g_direct_memory_backing->Contains(in_addr, len) ? "yes" : "no"); + return OK; + } + + if (g_virtual_ranges->ConsumeReservedSpan(in_addr, len, &consumed_range)) { + map_consumed_reserved_fixed(); + } else { + bool placeholder_backed = false; + if (ReplaceFixedRangeWithReserved(in_addr, len, &placeholder_backed) && + g_virtual_ranges->ConsumeReservedSpan(in_addr, len, &consumed_range)) { + map_consumed_reserved_fixed(); + } else { + VirtualRanges::Range existing_range {}; + if (g_virtual_ranges->Query(in_addr, 0, &existing_range)) { + GpuAccessMode old_gpu_mode = GpuAccessMode::NoAccess; + g_physical_memory->Unmap(in_addr, len, &old_gpu_mode); + g_flexible_memory->Unmap(in_addr, len, &old_gpu_mode); + g_virtual_ranges->Remove(in_addr, len); + const bool old_shared = g_direct_memory_backing->Contains(in_addr, len); + + bool old_placeholder_restored = false; + g_direct_memory_backing->Unmap(in_addr, len, + existing_range.committed_from_reserved, + &old_placeholder_restored); + if (old_placeholder_restored) { + g_placeholder_address_space->AddFree(in_addr, len); + } + + if (old_shared && g_direct_memory_backing->Contains(in_addr, len)) { + out_addr = 0; + } else if (map_shared_fixed(in_addr)) { + out_addr = in_addr; + committed_from_reserved = existing_range.committed_from_reserved || + existing_range.start != in_addr || + existing_range.size != len; + shared_backing = true; + } + } else if (!ReleaseReservedRange(in_addr, len)) { + return KERNEL_ERROR_ENOMEM; + } else if (map_shared_fixed(in_addr)) { + out_addr = in_addr; + shared_backing = true; + } + } + } + } else { + constexpr size_t DEFAULT_ALIGNMENT = 0x4000; + alignment = (alignment != 0 ? alignment : DEFAULT_ALIGNMENT); + if (in_addr != 0 && g_virtual_ranges->Query(in_addr, 0, &consumed_range) && + consumed_range.type == VirtualRangeType::Reserved && + g_virtual_ranges->ConsumeReserved(in_addr, len)) { + consumed_reserved = true; + if (map_shared_fixed(in_addr)) { + out_addr = in_addr; + committed_from_reserved = true; + shared_backing = true; + } + } else { + out_addr = g_direct_memory_backing->MapAligned(in_addr, len, direct_memory_start, mode, + alignment, &shared_failure); + shared_backing = out_addr != 0; + } + } + + *addr = reinterpret_cast(out_addr); + + const char* shared_reason = "n/a"; + if (!shared_backing) { + shared_reason = DirectMemoryBacking::GetFailureReasonName(shared_failure); + } + + LOGF("\t in_addr = 0x%016" PRIx64 "\n" + "\t out_addr = 0x%016" PRIx64 "\n" + "\t dmem = 0x%016" PRIx64 "\n" + "\t size = 0x%016" PRIx64 "\n" + "\t mode = %s\n" + "\t flags = 0x%08" PRIx32 "\n" + "\t align = 0x%016" PRIx64 "\n" + "\t gpu_mode = %s\n" + "\t shared = %s\n" + "\t reason = %s\n", + in_addr, out_addr, static_cast(direct_memory_start), len, + Common::EnumName(mode).c_str(), static_cast(flags), alignment, + Common::EnumName(gpu_mode).c_str(), shared_backing ? "yes" : "no", shared_reason); + + if (out_addr == 0) { + if (consumed_reserved) { + g_virtual_ranges->Add(in_addr, len, 0, 0, 0, VirtualRangeType::Reserved, + consumed_range.name, false, consumed_range.placeholder_backed); + } + return KERNEL_ERROR_ENOMEM; + } + + if (!g_physical_memory->Map(out_addr, direct_memory_start, len, prot, mode, gpu_mode)) { + LOGF_COLOR(Log::Color::Red, "\t [Fail]\n"); + bool placeholder_restored = false; + g_direct_memory_backing->Unmap(out_addr, len, committed_from_reserved, + &placeholder_restored); + if (placeholder_restored) { + g_placeholder_address_space->AddFree(out_addr, len); + } + + KYTY_NOT_IMPLEMENTED; + + return KERNEL_ERROR_EBUSY; + } + + PhysicalMemory::AllocatedBlock mapped_block {}; + g_physical_memory->Find(direct_memory_start, false, &mapped_block); + if (!g_virtual_ranges->Add(out_addr, len, direct_memory_start, prot, mapped_block.memory_type, + VirtualRangeType::Direct, "", committed_from_reserved)) { + GpuAccessMode rollback_gpu_mode = GpuAccessMode::NoAccess; + g_physical_memory->Unmap(out_addr, len, &rollback_gpu_mode); + bool placeholder_restored = false; + g_direct_memory_backing->Unmap(out_addr, len, committed_from_reserved, + &placeholder_restored); + if (placeholder_restored) { + g_placeholder_address_space->AddFree(out_addr, len); + } + return KERNEL_ERROR_EBUSY; + } + + MapGpuRange(out_addr, len, gpu_mode); + + if (g_alloc_callback != nullptr) { + g_alloc_callback(out_addr, len); + } + + LOGF_COLOR(Log::Color::Green, "\t [Ok]\n"); + + return OK; +} + +int KYTY_SYSV_ABI KernelMapDirectMemory2(void** addr, size_t len, int type, int prot, int flags, + int64_t direct_memory_start, size_t alignment) { + PRINT_NAME(); + + std::lock_guard memory_operation_lock(g_memory_operation_mutex); + + LOGF("\t type = %d\n", type); + + auto ret = KernelMapDirectMemory(addr, len, prot, flags, direct_memory_start, alignment); + if (ret == OK && addr != nullptr && *addr != nullptr) { + const auto out_addr = reinterpret_cast(*addr); + g_physical_memory->SetVirtualRangeMemoryType(out_addr, len, type); + g_virtual_ranges->SetMemoryType(out_addr, len, type); + } + + return ret; +} + +int KYTY_SYSV_ABI KernelMapNamedDirectMemory(void** addr, size_t len, int prot, int flags, + int64_t direct_memory_start, size_t alignment, + const char* name) { + PRINT_NAME(); + + std::lock_guard memory_operation_lock(g_memory_operation_mutex); + + LOGF("\t name = %s\n", name != nullptr ? name : "(null)"); + + if (name == nullptr) { + return KERNEL_ERROR_EFAULT; + } + + if (std::strlen(name) >= KERNEL_MAXIMUM_NAME_LENGTH) { + return KERNEL_ERROR_ENAMETOOLONG; + } + + auto ret = KernelMapDirectMemory(addr, len, prot, flags, direct_memory_start, alignment); + if (ret == OK && addr != nullptr) { + g_physical_memory->SetVirtualRangeName(reinterpret_cast(*addr), len, name); + g_virtual_ranges->Rename(reinterpret_cast(*addr), len, name); + } + + return ret; +} + +int KYTY_SYSV_ABI KernelIsAddressSanitizerEnabled() { + PRINT_NAME(); + + return 0; +} + +int KYTY_SYSV_ABI KernelQueryMemoryProtection(void* addr, void** start, void** end, int* prot) { + PRINT_NAME(); + + std::lock_guard memory_operation_lock(g_memory_operation_mutex); + + EXIT_NOT_IMPLEMENTED(addr == nullptr); + + VirtualRanges::Range range {}; + if (!g_virtual_ranges->Query(reinterpret_cast(addr), 0, &range)) { + return KERNEL_ERROR_EACCES; + } + + if (start != nullptr) { + *start = reinterpret_cast(range.start); + } + if (end != nullptr) { + *end = reinterpret_cast(range.start + range.size); + } + if (prot != nullptr) { + *prot = range.protection; + } + + return OK; +} + +static bool ReserveFixedHostRange(uint64_t start, uint64_t size) { + constexpr uint64_t PAGE_SIZE = 0x4000; + +#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS + SYSTEM_INFO system_info {}; + GetSystemInfo(&system_info); + const uint64_t granularity = system_info.dwAllocationGranularity; + if ((start & (granularity - 1u)) == 0 && (size & (granularity - 1u)) == 0) { + MEMORY_BASIC_INFORMATION info {}; + if (VirtualQuery(reinterpret_cast(start), &info, sizeof(info)) != 0 && + info.State == MEM_FREE && info.RegionSize >= size && + VirtualMemory::ReserveFixed(start, size)) { + return true; + } + } +#endif + + for (uint64_t addr = start; addr < start + size; addr += PAGE_SIZE) { +#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS + MEMORY_BASIC_INFORMATION info {}; + if (VirtualQuery(reinterpret_cast(addr), &info, sizeof(info)) != 0) { + if (info.State == MEM_COMMIT) { + if (!VirtualMemory::Decommit(addr, PAGE_SIZE)) { + LOGF_COLOR(Log::Color::Red, + "\t reserve-fixed replace: decommit failed at 0x%016" PRIx64 "\n", + addr); + return false; + } + continue; + } + if (info.State == MEM_RESERVE) { + continue; + } + } +#endif + if (!VirtualMemory::ReserveFixed(addr, PAGE_SIZE)) { + LOGF_COLOR(Log::Color::Red, + "\t reserve-fixed replace: reserve failed at 0x%016" PRIx64 "\n", addr); + return false; + } + } + + return true; +} + +static bool ReplaceFixedRangeWithReserved(uint64_t start, uint64_t size, bool* placeholder_backed) { + EXIT_IF(placeholder_backed == nullptr); + + *placeholder_backed = false; + + struct ReplacedChunk { + VirtualRanges::Range range {}; + VirtualMemory::Mode mode = VirtualMemory::Mode::NoAccess; + GpuAccessMode gpu_mode = GpuAccessMode::NoAccess; + bool shared_backing = false; + }; + + std::vector chunks; + const auto end = start + size; + auto current = start; + + while (current < end) { + VirtualRanges::Range range {}; + if (!g_virtual_ranges->Query(current, 1, &range)) { + break; + } + if (range.start >= end) { + break; + } + if (current < range.start) { + current = std::min(end, range.start); + continue; + } + + const auto range_end = range.start + range.size; + const auto chunk = std::min(end, range_end) - current; + + ReplacedChunk replaced {}; + replaced.range = range; + replaced.range.start = current; + replaced.range.size = chunk; + if (range.type == VirtualRangeType::Direct) { + replaced.range.offset += current - range.start; + } + DecodeMemoryProtection(replaced.range.protection, &replaced.mode, &replaced.gpu_mode); + replaced.shared_backing = (range.type == VirtualRangeType::Direct && + g_direct_memory_backing->Contains(current, chunk)); + chunks.push_back(replaced); + + current += chunk; + } + + auto restore_chunks = [&chunks]() -> bool { + bool ok = true; + for (auto it = chunks.rbegin(); it != chunks.rend(); ++it) { + const auto& chunk = *it; + bool host_restored = true; + bool backend_restored = true; + + if (chunk.range.type == VirtualRangeType::Direct) { + host_restored = + (chunk.shared_backing + ? g_direct_memory_backing->MapFixed(chunk.range.start, chunk.range.size, + chunk.range.offset, chunk.mode) + : CommitFixedHostRange(chunk.range.start, chunk.range.size, chunk.mode)); + backend_restored = + g_physical_memory->Map(chunk.range.start, chunk.range.offset, chunk.range.size, + chunk.range.protection, chunk.mode, chunk.gpu_mode); + } else if (chunk.range.type == VirtualRangeType::Flexible || + chunk.range.type == VirtualRangeType::Stack || + chunk.range.type == VirtualRangeType::Pooled) { + host_restored = + CommitFixedHostRange(chunk.range.start, chunk.range.size, chunk.mode); + backend_restored = g_flexible_memory->Map(chunk.range.start, chunk.range.size, + chunk.range.protection, chunk.mode, + chunk.gpu_mode, chunk.range.name); + } + + const bool range_restored = g_virtual_ranges->Add( + chunk.range.start, chunk.range.size, chunk.range.offset, chunk.range.protection, + chunk.range.memory_type, chunk.range.type, chunk.range.name, + chunk.range.committed_from_reserved, chunk.range.placeholder_backed); + ok = host_restored && backend_restored && range_restored && ok; + } + return ok; + }; + + for (const auto& chunk: chunks) { + if (chunk.range.type == VirtualRangeType::Pooled) { + EXIT("reserve-fixed replacement of pooled memory is unsupported: addr=0x%016" PRIx64 + " size=0x%016" PRIx64 "\n", + chunk.range.start, chunk.range.size); + } + } + const bool gpu_unmapped = std::any_of(chunks.begin(), chunks.end(), [](const auto& chunk) { + return IsCommittedRangeType(chunk.range.type) && + IsGpuAddressRange(chunk.range.start, chunk.range.size); + }); + for (const auto& chunk: chunks) { + if (IsCommittedRangeType(chunk.range.type)) { + UnmapGpuRange(chunk.range.start, chunk.range.size, chunk.gpu_mode); + } + } + + g_virtual_ranges->Remove(start, size); + + for (auto& chunk: chunks) { + GpuAccessMode gpu_mode = GpuAccessMode::NoAccess; + bool unmapped = true; + + if (chunk.range.type == VirtualRangeType::Direct) { + if (chunk.shared_backing) { + bool chunk_placeholder_restored = false; + if (!g_direct_memory_backing->Unmap(chunk.range.start, chunk.range.size, false, + &chunk_placeholder_restored)) { + unmapped = false; + } else if (chunk_placeholder_restored) { + g_placeholder_address_space->AddFree(chunk.range.start, chunk.range.size); + } + } + if (unmapped) { + unmapped = g_physical_memory->Unmap(chunk.range.start, chunk.range.size, &gpu_mode); + chunk.gpu_mode = gpu_mode; + } + } else if (chunk.range.type == VirtualRangeType::Flexible || + chunk.range.type == VirtualRangeType::Stack || + chunk.range.type == VirtualRangeType::Pooled) { + unmapped = g_flexible_memory->Unmap(chunk.range.start, chunk.range.size, &gpu_mode); + chunk.gpu_mode = gpu_mode; + } + + if (!unmapped) { + if (gpu_unmapped) { + EXIT("reserve-fixed backend unmap failed after GPU unmap: addr=0x%016" PRIx64 + " size=0x%016" PRIx64 "\n", + chunk.range.start, chunk.range.size); + } + LOGF_COLOR(Log::Color::Red, + "\t reserve-fixed replace: backend unmap failed at 0x%016" PRIx64 + ", size=0x%016" PRIx64 ", type=%s\n", + chunk.range.start, chunk.range.size, + Common::EnumName(chunk.range.type).c_str()); + restore_chunks(); + return false; + } + } + + if (g_placeholder_address_space->ReserveFixed(start, size)) { + *placeholder_backed = true; + } else if (!ReserveFixedHostRange(start, size)) { + if (gpu_unmapped) { + EXIT("reserve-fixed host reservation failed after GPU unmap: addr=0x%016" PRIx64 + " size=0x%016" PRIx64 "\n", + start, size); + } + restore_chunks(); + return false; + } + + if (!g_virtual_ranges->Add(start, size, 0, 0, 0, VirtualRangeType::Reserved, "anon", false, + *placeholder_backed)) { + if (gpu_unmapped) { + EXIT("reserve-fixed range registration failed after GPU unmap: addr=0x%016" PRIx64 + " size=0x%016" PRIx64 "\n", + start, size); + } + if (*placeholder_backed) { + g_placeholder_address_space->ReleaseFree(start, size); + } else { + VirtualMemory::Free(start); + } + LOGF_COLOR(Log::Color::Red, + "\t reserve-fixed replace: range add failed at 0x%016" PRIx64 + ", size=0x%016" PRIx64 "\n", + start, size); + restore_chunks(); + return false; + } + + for (const auto& chunk: chunks) { + if (g_free_callback != nullptr && IsCommittedRangeType(chunk.range.type)) { + g_free_callback(chunk.range.start, chunk.range.size); + } + } + + return true; +} + +int KYTY_SYSV_ABI KernelReserveVirtualRange(void** addr, size_t len, int flags, size_t alignment) { + PRINT_NAME(); + + std::lock_guard memory_operation_lock(g_memory_operation_mutex); + + const auto in_addr = (addr != nullptr ? reinterpret_cast(*addr) : 0); + + LOGF("\t in_addr = 0x%016" PRIx64 "\n" + "\t len = 0x%016" PRIx64 "\n" + "\t flags = 0x%08" PRIx32 "\n" + "\t alignment = 0x%016" PRIx64 "\n", + in_addr, len, flags, alignment); + + constexpr size_t PAGE_SIZE = 0x4000; + constexpr int MAP_FIXED = 0x10; + constexpr int MAP_NO_OVERWRITE = 0x80; + + if (addr == nullptr || len == 0 || (len & (PAGE_SIZE - 1)) != 0) { + return KERNEL_ERROR_EINVAL; + } + if (alignment != 0 && (alignment & (alignment - 1)) != 0) { + return KERNEL_ERROR_EINVAL; + } + + uint64_t out_addr = 0; + bool range_already_added = false; + bool placeholder_backed = false; + if ((flags & MAP_FIXED) != 0) { + if (in_addr == 0 || (in_addr & (PAGE_SIZE - 1)) != 0) { + return KERNEL_ERROR_EINVAL; + } + if ((flags & MAP_NO_OVERWRITE) != 0 && g_virtual_ranges->HasOverlap(in_addr, len)) { + return KERNEL_ERROR_ENOMEM; + } + if (ReplaceFixedRangeWithReserved(in_addr, len, &placeholder_backed)) { + out_addr = in_addr; + range_already_added = true; + } + } else { + alignment = (alignment != 0 ? alignment : PAGE_SIZE); + out_addr = g_placeholder_address_space->ReserveAligned(in_addr, len, alignment); + placeholder_backed = (out_addr != 0); + if (out_addr == 0) { + out_addr = VirtualMemory::ReserveAligned(in_addr, len, alignment); + placeholder_backed = false; + } + } + + if (out_addr == 0) { + return KERNEL_ERROR_ENOMEM; + } + + if (!range_already_added && + !g_virtual_ranges->Add(out_addr, len, 0, 0, 0, VirtualRangeType::Reserved, "anon", false, + placeholder_backed)) { + if (!placeholder_backed || !g_placeholder_address_space->ReleaseFree(out_addr, len)) { + VirtualMemory::Free(out_addr); + } + return KERNEL_ERROR_EBUSY; + } + + *addr = reinterpret_cast(out_addr); + + LOGF("\t out_addr = 0x%016" PRIx64 "\n" + "\t placeholder = %s\n", + out_addr, placeholder_backed ? "yes" : "no"); + + return OK; +} + +bool KernelHandleReservedRangeAccessViolation(uint64_t vaddr) { + std::lock_guard memory_operation_lock(g_memory_operation_mutex); + + VirtualRanges::Range range {}; + if (!g_virtual_ranges->Query(vaddr, 0, &range) || + std::strncmp(range.name, "AMM", KERNEL_MAXIMUM_NAME_LENGTH) != 0) { + return false; + } + EXIT("AMM virtual-memory unmap is unsupported: addr=0x%016" PRIx64 "\n", vaddr); +} + +int KYTY_SYSV_ABI KernelVirtualQuery(const void* addr, int flags, VirtualQueryInfo* info, + uint64_t info_size) { + PRINT_NAME(); + + std::lock_guard memory_operation_lock(g_memory_operation_mutex); + + auto vaddr = reinterpret_cast(addr); + + LOGF("\t addr = 0x%016" PRIx64 "\n" + "\t flags = 0x%08" PRIx32 "\n" + "\t info_size = 0x%016" PRIx64 "\n", + vaddr, flags, info_size); + + if (info == nullptr || info_size != sizeof(VirtualQueryInfo) || (flags != 0 && flags != 1)) { + return KERNEL_ERROR_EINVAL; + } + + VirtualRanges::Range candidate {}; + if (!g_virtual_ranges->Query(vaddr, flags, &candidate)) { + return KERNEL_ERROR_EACCES; + } + + std::memset(info, 0, sizeof(VirtualQueryInfo)); + info->start = candidate.start; + info->end = candidate.start + candidate.size; + info->offset = candidate.offset; + info->protection = candidate.protection; + info->memory_type = candidate.memory_type; + info->is_flexible = (candidate.type == VirtualRangeType::Flexible ? 1 : 0); + info->is_direct = (candidate.type == VirtualRangeType::Direct ? 1 : 0); + info->is_stack = (candidate.type == VirtualRangeType::Stack ? 1 : 0); + info->is_pooled = (IsPooledRangeType(candidate.type) ? 1 : 0); + info->is_committed = (IsCommittedRangeType(candidate.type) ? 1 : 0); + info->is_gpu_prt = IsInPrtAperture(vaddr); + CopyVirtualRangeName(info->name, candidate.name); + + static std::atomic log_count {0}; + if (log_count.fetch_add(1) < 64) { + LOGF("\t start = 0x%016" PRIx64 "\n" + "\t end = 0x%016" PRIx64 "\n" + "\t offset = 0x%016" PRIx64 "\n" + "\t protection = 0x%08" PRIx32 "\n" + "\t memory_type = %d\n" + "\t flexible = %d\n" + "\t direct = %d\n" + "\t name = %s\n", + static_cast(info->start), static_cast(info->end), info->offset, + info->protection, info->memory_type, static_cast(info->is_flexible), + static_cast(info->is_direct), info->name); + } + + return OK; +} + +int KYTY_SYSV_ABI KernelIsStack(void* addr, void** start, void** end) { + PRINT_NAME(); + + std::lock_guard memory_operation_lock(g_memory_operation_mutex); + + auto vaddr = reinterpret_cast(addr); + + LOGF("\t addr = 0x%016" PRIx64 "\n", vaddr); + + VirtualRanges::Range candidate {}; + if (!g_virtual_ranges->Query(vaddr, 0, &candidate)) { + return KERNEL_ERROR_EACCES; + } + + uint64_t stack_start = 0; + uint64_t stack_end = 0; + if (candidate.type == VirtualRangeType::Stack) { + stack_start = candidate.start; + stack_end = candidate.start + candidate.size; + } + + if (start != nullptr) { + *start = reinterpret_cast(stack_start); + } + + if (end != nullptr) { + *end = reinterpret_cast(stack_end); + } + + LOGF("\t start = 0x%016" PRIx64 "\n" + "\t end = 0x%016" PRIx64 "\n", + stack_start, stack_end); + + return OK; +} + +int KYTY_SYSV_ABI KernelAvailableFlexibleMemorySize(size_t* size) { + PRINT_NAME(); + + std::lock_guard memory_operation_lock(g_memory_operation_mutex); + + EXIT_NOT_IMPLEMENTED(size == nullptr); + + *size = g_flexible_memory->Available(); + + LOGF("\t *size = 0x%016" PRIx64 "\n", *size); + + return OK; +} + +int KYTY_SYSV_ABI KernelConfiguredFlexibleMemorySize(uint64_t* size) { + PRINT_NAME(); + + if (size == nullptr) { + return KERNEL_ERROR_EINVAL; + } + + *size = FlexibleMemory::Size(); + + LOGF("\t *size = 0x%016" PRIx64 "\n", *size); + + return OK; +} + +static int ProgramProtection(VirtualMemory::Mode mode) { + const auto protection = static_cast(mode); + if ((protection & ~(PROT_CPU_READ | PROT_CPU_WRITE | PROT_CPU_EXEC)) != 0) { + EXIT("unsupported program-memory protection: 0x%08x\n", protection); + } + return protection; +} + +static std::vector RequireProgramMemory(uint64_t vaddr, uint64_t size) { + std::vector ranges; + if (g_virtual_ranges == nullptr || !g_virtual_ranges->QuerySpan(vaddr, size, &ranges) || + std::any_of(ranges.begin(), ranges.end(), + [](const auto& range) { return range.type != VirtualRangeType::Code; })) { + EXIT("program-memory range is not fully mapped: addr=0x%016" PRIx64 " size=0x%016" PRIx64 + "\n", + vaddr, size); + } + return ranges; +} + +void RegisterProgramMemory(uint64_t vaddr, uint64_t size, VirtualMemory::Mode mode, + const char* name) { + std::lock_guard memory_operation_lock(g_memory_operation_mutex); + + if (g_virtual_ranges == nullptr || vaddr == 0 || size == 0 || size > UINT64_MAX - vaddr || + name == nullptr || + !g_virtual_ranges->Add(vaddr, size, 0, ProgramProtection(mode), 0, VirtualRangeType::Code, + name)) { + EXIT("failed to register program memory: addr=0x%016" PRIx64 " size=0x%016" PRIx64 "\n", + vaddr, size); + } +} + +void UpdateProgramMemoryProtection(uint64_t vaddr, uint64_t size, VirtualMemory::Mode mode) { + std::lock_guard memory_operation_lock(g_memory_operation_mutex); + RequireProgramMemory(vaddr, size); + g_virtual_ranges->Protect(vaddr, size, ProgramProtection(mode)); +} + +void UnregisterProgramMemory(uint64_t vaddr, uint64_t size) { + std::lock_guard memory_operation_lock(g_memory_operation_mutex); + + for (const auto& range: RequireProgramMemory(vaddr, size)) { + const auto gpu_mode = GetGpuAccessMode(range.protection); + if (gpu_mode != GpuAccessMode::NoAccess) { + if (!GetGpuResources().IsMapped(range.start, range.size)) { + EXIT("program GPU range is not tracked: addr=0x%016" PRIx64 " size=0x%016" PRIx64 + "\n", + range.start, range.size); + } + UnmapGpuRange(range.start, range.size, gpu_mode); + } + } + if (!g_virtual_ranges->Remove(vaddr, size)) { + EXIT("failed to unregister program memory: addr=0x%016" PRIx64 " size=0x%016" PRIx64 "\n", + vaddr, size); + } +} + +int KYTY_SYSV_ABI KernelMprotect(const void* addr, size_t len, int prot) { + PRINT_NAME(); + + std::lock_guard memory_operation_lock(g_memory_operation_mutex); + + auto vaddr = reinterpret_cast(addr); + + LOGF("\t addr = 0x%016" PRIx64 "\n" + "\t len = 0x%016" PRIx64 "\n", + vaddr, static_cast(len)); + + constexpr uint64_t PAGE_SIZE = 0x4000; + auto aligned_addr = vaddr & ~(PAGE_SIZE - 1); + const auto page_offset = vaddr - aligned_addr; + if (len > UINT64_MAX - page_offset || len + page_offset > UINT64_MAX - (PAGE_SIZE - 1)) { + EXIT("memory-protection range overflows: addr=0x%016" PRIx64 " size=0x%016" PRIx64 "\n", + vaddr, static_cast(len)); + } + auto aligned_len = (len + page_offset + PAGE_SIZE - 1) & ~(PAGE_SIZE - 1); + + if (aligned_len == 0) { + return OK; + } + + VirtualMemory::Mode mode = VirtualMemory::Mode::NoAccess; + GpuAccessMode gpu_mode = GpuAccessMode::NoAccess; + + if (!DecodeMemoryProtection(prot, &mode, &gpu_mode)) { + return KERNEL_ERROR_EINVAL; + } + std::vector old_ranges; + if (!g_virtual_ranges->QuerySpan(aligned_addr, aligned_len, &old_ranges) || + std::any_of(old_ranges.begin(), old_ranges.end(), + [](const auto& range) { return !IsCommittedRangeType(range.type); })) { + EXIT("memory-protection range is not fully mapped: addr=0x%016" PRIx64 " size=0x%016" PRIx64 + "\n", + aligned_addr, aligned_len); + } + for (const auto& old_range: old_ranges) { + const auto old_gpu_mode = GetGpuAccessMode(old_range.protection); + if (old_gpu_mode == GpuAccessMode::NoAccess) { + continue; + } + if ((old_range.type != VirtualRangeType::Stack && + old_range.type != VirtualRangeType::Code) || + !GetGpuResources().IsMapped(old_range.start, old_range.size)) { + EXIT("GPU protection transition requires tracked stack/code memory: addr=0x%016" PRIx64 + " size=0x%016" PRIx64 " type=%s\n", + old_range.start, old_range.size, Common::EnumName(old_range.type).c_str()); + } + UnmapGpuRange(old_range.start, old_range.size, old_gpu_mode); + } + + VirtualMemory::Mode old_mode {}; + bool ok = ProtectCommittedHostMemory(aligned_addr, aligned_len, mode, &old_mode); + + if (!ok) { + EXIT("host memory-protection update failed: addr=0x%016" PRIx64 " size=0x%016" PRIx64 + " prot=0x%08x\n", + aligned_addr, aligned_len, prot); + } + g_virtual_ranges->Protect(aligned_addr, aligned_len, prot); + g_direct_memory_backing->UpdateProtection(aligned_addr, aligned_len, mode); + MapGpuRange(aligned_addr, aligned_len, gpu_mode); + + LOGF("\t prot: %s -> %s\n", Common::EnumName(old_mode).c_str(), Common::EnumName(mode).c_str()); + + return OK; +} + +int KYTY_SYSV_ABI KernelMtypeprotect(const void* addr, size_t len, int type, int prot) { + PRINT_NAME(); + + std::lock_guard memory_operation_lock(g_memory_operation_mutex); + + LOGF("\t addr = 0x%016" PRIx64 "\n" + "\t len = 0x%016" PRIx64 "\n" + "\t type = 0x%08" PRIx32 "\n" + "\t prot = 0x%08" PRIx32 "\n", + reinterpret_cast(addr), static_cast(len), static_cast(type), + static_cast(prot)); + + return KernelMprotect(addr, len, prot); +} + +int KYTY_SYSV_ABI KernelBatchMap2(KernelBatchMapEntry* entries, int num_entries, + int* num_entries_out, int flags) { + PRINT_NAME(); + + std::lock_guard memory_operation_lock(g_memory_operation_mutex); + + LOGF("\t entries = %p\n" + "\t num_entries = %d\n" + "\t num_entries_out = %p\n" + "\t flags = 0x%08" PRIx32 "\n", + static_cast(entries), num_entries, static_cast(num_entries_out), + static_cast(flags)); + + enum MemoryOpTypes { + MAP_OP_MAP_DIRECT = 0, + MAP_OP_UNMAP = 1, + MAP_OP_PROTECT = 2, + MAP_OP_MAP_FLEXIBLE = 3, + MAP_OP_TYPE_PROTECT = 4, + }; + + if (entries == nullptr || num_entries < 0) { + return KERNEL_ERROR_EINVAL; + } + + int processed = 0; + + for (int i = 0; i < num_entries; i++, processed++) { + auto* entry = &entries[i]; + + LOGF("\t [%d] start = %p, offset = 0x%016" PRIx64 ", length = 0x%016" PRIx64 + ", prot = 0x%02" PRIx32 ", type = 0x%02" PRIx32 ", op = %d\n", + i, entry->start, entry->offset, entry->length, + static_cast(static_cast(entry->protection)), + static_cast(static_cast(entry->type)), entry->operation); + + if (entry->length == 0 || entry->operation < MAP_OP_MAP_DIRECT || + entry->operation > MAP_OP_TYPE_PROTECT) { + break; + } + + int result = OK; + switch (entry->operation) { + case MAP_OP_MAP_DIRECT: + result = KernelMapNamedDirectMemory(&entry->start, entry->length, entry->protection, + flags, static_cast(entry->offset), 0, + "anon"); + break; + case MAP_OP_UNMAP: + result = KernelMunmap(reinterpret_cast(entry->start), entry->length); + break; + case MAP_OP_PROTECT: + result = KernelMprotect(entry->start, entry->length, entry->protection); + break; + case MAP_OP_MAP_FLEXIBLE: + result = KernelMapNamedFlexibleMemory(&entry->start, entry->length, + entry->protection, flags, "anon"); + break; + case MAP_OP_TYPE_PROTECT: + result = + KernelMtypeprotect(entry->start, entry->length, entry->type, entry->protection); + break; + default: result = KERNEL_ERROR_EINVAL; break; + } + + if (result != OK) { + if (num_entries_out != nullptr) { + *num_entries_out = processed; + } + return result; + } + } + + if (num_entries_out != nullptr) { + *num_entries_out = processed; + } + + return (processed == num_entries ? OK : KERNEL_ERROR_EINVAL); +} + +int KYTY_SYSV_ABI KernelBatchMap(KernelBatchMapEntry* entries, int num_entries, + int* num_entries_out) { + constexpr int MAP_FIXED = 0x10; + return KernelBatchMap2(entries, num_entries, num_entries_out, MAP_FIXED); +} + +static bool IsAligned(uint64_t value, uint64_t alignment) { + return alignment == 0 || (value & (alignment - 1u)) == 0; +} + +static bool IsPowerOfTwo(uint64_t value) { + return value != 0 && (value & (value - 1u)) == 0; +} + +static void MemoryPoolSubtractCommitted(uint64_t len) { + auto current = g_memory_pool_committed.load(std::memory_order_relaxed); + while (current != 0) { + const auto next = (current > len ? current - len : 0); + if (g_memory_pool_committed.compare_exchange_weak(current, next, + std::memory_order_relaxed)) { + return; + } + } +} + +int KYTY_SYSV_ABI KernelMemoryPoolExpand(int64_t search_start, int64_t search_end, size_t len, + size_t alignment, int64_t* phys_addr_out) { + PRINT_NAME(); + + std::lock_guard memory_operation_lock(g_memory_operation_mutex); + + constexpr uint64_t POOL_PAGE_SIZE = 0x10000; + if (search_start < 0 || search_end <= search_start || len == 0 || + (len & (POOL_PAGE_SIZE - 1u)) != 0 || phys_addr_out == nullptr || + (alignment != 0 && + (!IsPowerOfTwo(alignment) || (alignment & (POOL_PAGE_SIZE - 1u)) != 0))) { + return KERNEL_ERROR_EINVAL; + } + if (static_cast(search_end - search_start) < len) { + return KERNEL_ERROR_ENOMEM; + } + + const auto effective_alignment = (alignment != 0 ? alignment : POOL_PAGE_SIZE); + uint64_t phys_addr = 0; + if (!g_physical_memory->Alloc(static_cast(search_start), + static_cast(search_end), len, effective_alignment, + &phys_addr, 0, true)) { + return KERNEL_ERROR_ENOMEM; + } + + g_pooled_memory->Expand(phys_addr, len); + *phys_addr_out = static_cast(phys_addr); + + LOGF("\t search_start = 0x%016" PRIx64 "\n" + "\t search_end = 0x%016" PRIx64 "\n" + "\t len = 0x%016" PRIx64 "\n" + "\t alignment = 0x%016" PRIx64 "\n" + "\t phys_addr = 0x%016" PRIx64 "\n", + search_start, search_end, static_cast(len), static_cast(alignment), + phys_addr); + return OK; +} + +int KYTY_SYSV_ABI KernelMemoryPoolReserve(void* addr_in, size_t len, size_t alignment, int flags, + void** addr_out) { + PRINT_NAME(); + + std::lock_guard memory_operation_lock(g_memory_operation_mutex); + + LOGF("\t addr_in = 0x%016" PRIx64 "\n" + "\t len = 0x%016" PRIx64 "\n" + "\t alignment = 0x%016" PRIx64 "\n" + "\t flags = 0x%08" PRIx32 "\n" + "\t addr_out = %p\n", + reinterpret_cast(addr_in), static_cast(len), + static_cast(alignment), static_cast(flags), + static_cast(addr_out)); + + constexpr uint64_t POOL_RESERVE_ALIGNMENT = 0x200000; + + if (addr_out == nullptr || len == 0 || !IsAligned(len, POOL_RESERVE_ALIGNMENT)) { + return KERNEL_ERROR_EINVAL; + } + if (alignment != 0 && + (!IsPowerOfTwo(alignment) || !IsAligned(alignment, POOL_RESERVE_ALIGNMENT))) { + return KERNEL_ERROR_EINVAL; + } + + void* out_addr = addr_in; + const auto reserve_alignment = (alignment != 0 ? alignment : POOL_RESERVE_ALIGNMENT); + const int ret = KernelReserveVirtualRange(&out_addr, len, flags, reserve_alignment); + if (ret == OK) { + const auto out_vaddr = reinterpret_cast(out_addr); + VirtualRanges::Range reserved_range {}; + if (!g_virtual_ranges->Query(out_vaddr, 0, &reserved_range) || + reserved_range.start != out_vaddr || reserved_range.size != len || + !IsReservedRangeType(reserved_range.type)) { + return KERNEL_ERROR_EBUSY; + } + g_virtual_ranges->Remove(out_vaddr, len); + if (!g_virtual_ranges->Add(out_vaddr, len, 0, 0, 0, VirtualRangeType::PoolReserved, + reserved_range.name, false, reserved_range.placeholder_backed)) { + g_virtual_ranges->Add(out_vaddr, len, reserved_range.offset, reserved_range.protection, + reserved_range.memory_type, reserved_range.type, + reserved_range.name, reserved_range.committed_from_reserved, + reserved_range.placeholder_backed); + return KERNEL_ERROR_EBUSY; + } + *addr_out = out_addr; + LOGF("\t out_addr = 0x%016" PRIx64 "\n", reinterpret_cast(out_addr)); + } + + return ret; +} + +int KYTY_SYSV_ABI KernelMemoryPoolCommit(void* addr, size_t len, int type, int prot, int flags) { + PRINT_NAME(); + + std::lock_guard memory_operation_lock(g_memory_operation_mutex); + + LOGF("\t addr = 0x%016" PRIx64 "\n" + "\t len = 0x%016" PRIx64 "\n" + "\t type = 0x%08" PRIx32 "\n" + "\t prot = 0x%08" PRIx32 "\n" + "\t flags = 0x%08" PRIx32 "\n", + reinterpret_cast(addr), static_cast(len), static_cast(type), + static_cast(prot), static_cast(flags)); + + constexpr uint64_t POOL_COMMIT_ALIGNMENT = 0x10000; + constexpr int PROT_CPU_EXEC = 0x04; + + if (addr == nullptr || len == 0 || !IsAligned(len, POOL_COMMIT_ALIGNMENT) || + (prot & PROT_CPU_EXEC) != 0) { + return KERNEL_ERROR_EINVAL; + } + + VirtualMemory::Mode mode = VirtualMemory::Mode::NoAccess; + GpuAccessMode gpu_mode = GpuAccessMode::NoAccess; + if (!DecodeMemoryProtection(prot, &mode, &gpu_mode)) { + return KERNEL_ERROR_EINVAL; + } + + const auto vaddr = reinterpret_cast(addr); + + VirtualRanges::Range old_range {}; + if (!g_virtual_ranges->Query(vaddr, 0, &old_range) || + old_range.type != VirtualRangeType::PoolReserved) { + return KERNEL_ERROR_EACCES; + } + + if (!g_virtual_ranges->ConsumeReserved(vaddr, len, VirtualRangeType::PoolReserved)) { + return KERNEL_ERROR_EACCES; + } + + std::vector mappings; + if (!g_pooled_memory->Allocate(vaddr, len, gpu_mode, &mappings)) { + g_virtual_ranges->Add(vaddr, len, 0, 0, 0, VirtualRangeType::PoolReserved, old_range.name, + false, old_range.placeholder_backed); + return KERNEL_ERROR_ENOMEM; + } + + std::vector mapped; + auto rollback = [&]() { + if (!UnmapPooledBackingTransactional(mapped, mode)) { + EXIT("failed to roll back pooled-memory backing maps\n"); + } + GpuAccessMode rollback_gpu_mode = GpuAccessMode::NoAccess; + if (!g_pooled_memory->Release(vaddr, len, &rollback_gpu_mode)) { + EXIT("failed to release pooled-memory rollback allocation\n"); + } + g_virtual_ranges->Add(vaddr, len, 0, 0, 0, VirtualRangeType::PoolReserved, old_range.name, + false, old_range.placeholder_backed); + }; + + for (const auto& mapping: mappings) { + auto failure_reason = DirectMemoryBacking::FailureReason::None; + const bool placeholder_ready = + g_placeholder_address_space->Consume(mapping.vaddr, mapping.size); + const bool ok = + placeholder_ready + ? g_direct_memory_backing->MapExistingPlaceholderFixed( + mapping.vaddr, mapping.size, mapping.phys_addr, mode, &failure_reason) + : g_direct_memory_backing->MapFixed(mapping.vaddr, mapping.size, mapping.phys_addr, + mode, &failure_reason); + if (!ok) { + if (placeholder_ready) { + g_placeholder_address_space->AddFree(mapping.vaddr, mapping.size); + } + LOGF_COLOR(Log::Color::Red, "\t pool backing map failed: %s\n", + DirectMemoryBacking::GetFailureReasonName(failure_reason)); + rollback(); + return KERNEL_ERROR_ENOMEM; + } + mapped.push_back(mapping); + } + + if (!g_virtual_ranges->Add(vaddr, len, 0, prot, type, VirtualRangeType::Pooled, old_range.name, + true, old_range.placeholder_backed)) { + rollback(); + return KERNEL_ERROR_EBUSY; + } + + MapGpuRange(vaddr, len, gpu_mode); + + if (g_alloc_callback != nullptr) { + g_alloc_callback(vaddr, len); + } + + g_memory_pool_committed.fetch_add(len, std::memory_order_relaxed); + + return OK; +} + +int KYTY_SYSV_ABI KernelMemoryPoolDecommit(void* addr, size_t len, int flags) { + PRINT_NAME(); + + std::lock_guard memory_operation_lock(g_memory_operation_mutex); + + LOGF("\t addr = 0x%016" PRIx64 "\n" + "\t len = 0x%016" PRIx64 "\n" + "\t flags = 0x%08" PRIx32 "\n", + reinterpret_cast(addr), static_cast(len), + static_cast(flags)); + + constexpr uint64_t POOL_COMMIT_ALIGNMENT = 0x10000; + + if (addr == nullptr || len == 0 || !IsAligned(len, POOL_COMMIT_ALIGNMENT)) { + return KERNEL_ERROR_EINVAL; + } + + const auto vaddr = reinterpret_cast(addr); + if (UINT64_MAX - vaddr < len) { + return KERNEL_ERROR_EINVAL; + } + const auto end = vaddr + len; + auto scan = vaddr; + while (scan < end) { + VirtualRanges::Range scan_range {}; + if (!g_virtual_ranges->Query(scan, 0, &scan_range) || + (scan_range.type != VirtualRangeType::Pooled && + scan_range.type != VirtualRangeType::PoolReserved)) { + return KERNEL_ERROR_EACCES; + } + const auto next = std::min(end, scan_range.start + scan_range.size); + if (next <= scan) { + return KERNEL_ERROR_EACCES; + } + scan = next; + } + + VirtualRanges::Range old_range {}; + if (!g_virtual_ranges->Query(vaddr, 0, &old_range)) { + return KERNEL_ERROR_EACCES; + } + const auto chunk_len = std::min(len, old_range.size - (vaddr - old_range.start)); + if (old_range.type == VirtualRangeType::PoolReserved) { + return chunk_len < len + ? KernelMemoryPoolDecommit(reinterpret_cast(vaddr + chunk_len), + len - chunk_len, flags) + : OK; + } + if (old_range.type != VirtualRangeType::Pooled) { + return KERNEL_ERROR_EACCES; + } + if (chunk_len < len) { + const int ret = KernelMemoryPoolDecommit(addr, chunk_len, flags); + return ret == OK ? KernelMemoryPoolDecommit(reinterpret_cast(vaddr + chunk_len), + len - chunk_len, flags) + : ret; + } + + std::vector mappings; + if (!g_pooled_memory->Query(vaddr, len, &mappings)) { + return KERNEL_ERROR_EACCES; + } + + VirtualMemory::Mode mode = VirtualMemory::Mode::NoAccess; + GpuAccessMode decoded_gpu = GpuAccessMode::NoAccess; + if (!DecodeMemoryProtection(old_range.protection, &mode, &decoded_gpu)) { + return KERNEL_ERROR_EACCES; + } + UnmapGpuRange(vaddr, len, decoded_gpu); + if (!UnmapPooledBackingTransactional(mappings, mode)) { + EXIT("pooled-memory backing transaction failed after GPU unmap: addr=0x%016" PRIx64 + " size=0x%016" PRIx64 "\n", + vaddr, len); + } + + GpuAccessMode gpu_mode = GpuAccessMode::NoAccess; + if (!g_pooled_memory->Release(vaddr, len, &gpu_mode)) { + EXIT("failed to release decommitted pooled-memory range\n"); + } + + g_virtual_ranges->Remove(vaddr, len); + old_range.placeholder_backed = true; + g_virtual_ranges->Add(vaddr, len, 0, 0, 0, VirtualRangeType::PoolReserved, old_range.name, + false, old_range.placeholder_backed); + + if (g_free_callback != nullptr) { + g_free_callback(vaddr, len); + } + + MemoryPoolSubtractCommitted(len); + + return OK; +} + +int KYTY_SYSV_ABI KernelMemoryPoolBatch(const KernelMemoryPoolBatchEntry* entries, int num_entries, + int* num_entries_out, int flags) { + PRINT_NAME(); + + std::lock_guard memory_operation_lock(g_memory_operation_mutex); + + if (entries == nullptr || num_entries < 0) { + return KERNEL_ERROR_EINVAL; + } + + enum MemoryPoolOp { + POOL_OP_COMMIT = 1, + POOL_OP_DECOMMIT = 2, + POOL_OP_PROTECT = 3, + POOL_OP_TYPE_PROTECT = 4, + POOL_OP_MOVE = 5, + }; + + int processed = 0; + int result = OK; + + for (int i = 0; i < num_entries; i++, processed++) { + const auto& entry = entries[i]; + switch (entry.op) { + case POOL_OP_COMMIT: + result = KernelMemoryPoolCommit(entry.commit.addr, entry.commit.len, + entry.commit.type, entry.commit.prot, entry.flags); + break; + case POOL_OP_DECOMMIT: + result = + KernelMemoryPoolDecommit(entry.decommit.addr, entry.decommit.len, entry.flags); + break; + case POOL_OP_PROTECT: + result = KernelMprotect(entry.protect.addr, entry.protect.len, entry.protect.prot); + break; + case POOL_OP_TYPE_PROTECT: + result = KernelMtypeprotect(entry.type_protect.addr, entry.type_protect.len, + entry.type_protect.type, entry.type_protect.prot); + break; + case POOL_OP_MOVE: + default: result = KERNEL_ERROR_EINVAL; break; + } + + if (result != OK) { + break; + } + } + + if (num_entries_out != nullptr) { + *num_entries_out = processed; + } + + (void)flags; + return result; +} + +int KYTY_SYSV_ABI KernelMemoryPoolGetBlockStats(KernelMemoryPoolBlockStats* output, + size_t output_size) { + PRINT_NAME(); + + std::lock_guard memory_operation_lock(g_memory_operation_mutex); + + if (output == nullptr && output_size != 0) { + return KERNEL_ERROR_EFAULT; + } + + KernelMemoryPoolBlockStats stats {}; + constexpr uint64_t BLOCK_SIZE = 0x10000; + const uint64_t committed = g_memory_pool_committed.load(std::memory_order_relaxed); + const uint64_t available = (g_pooled_memory != nullptr ? g_pooled_memory->Available() : 0); + + stats.available_flushed_blocks = static_cast(available / BLOCK_SIZE); + stats.available_cached_blocks = 0; + stats.allocated_flushed_blocks = static_cast(committed / BLOCK_SIZE); + stats.allocated_cached_blocks = 0; + + const auto copy_size = std::min(output_size, sizeof(stats)); + if (copy_size != 0) { + std::memcpy(output, &stats, copy_size); + } + + return OK; +} + +} // namespace Libs::LibKernel::Memory diff --git a/src/kernel/memory.h b/src/kernel/memory.h new file mode 100644 index 000000000..c39c37d78 --- /dev/null +++ b/src/kernel/memory.h @@ -0,0 +1,164 @@ +#ifndef EMULATOR_INCLUDE_EMULATOR_KERNEL_MEMORY_H_ +#define EMULATOR_INCLUDE_EMULATOR_KERNEL_MEMORY_H_ + +#include "common/abi.h" +#include "common/common.h" +#include "common/subsystems.h" +#include "common/virtualMemory.h" + +namespace Libs::LibKernel::Memory { + +KYTY_SUBSYSTEM_DEFINE(Memory); + +using callback_func_t = void (*)(uintptr_t addr, size_t size); + +constexpr uint32_t KERNEL_MAXIMUM_NAME_LENGTH = 32; + +struct VirtualQueryInfo { + uintptr_t start; + uintptr_t end; + uint64_t offset; + int32_t protection; + int32_t memory_type; + uint32_t is_flexible : 1; + uint32_t is_direct : 1; + uint32_t is_stack : 1; + uint32_t is_pooled : 1; + uint32_t is_committed : 1; + uint32_t is_gpu_prt : 1; + uint32_t amm_usage : 1; + uint32_t reserved : 1; + char name[KERNEL_MAXIMUM_NAME_LENGTH]; + uint8_t gpu_mask_id; + uint8_t reserved2; +}; + +static_assert(sizeof(VirtualQueryInfo) == 72, "VirtualQueryInfo struct size is incorrect"); + +struct KernelBatchMapEntry { + void* start; + uint64_t offset; + uint64_t length; + char protection; + char type; + int16_t reserved; + int32_t operation; +}; + +static_assert(sizeof(KernelBatchMapEntry) == 32, "KernelBatchMapEntry struct size is incorrect"); + +struct KernelMemoryPoolBatchEntry { + uint32_t op; + uint32_t flags; + union { + struct { + void* addr; + uint64_t len; + uint8_t prot; + uint8_t type; + } commit; + struct { + void* addr; + uint64_t len; + } decommit; + struct { + void* addr; + uint64_t len; + uint8_t prot; + } protect; + struct { + void* addr; + uint64_t len; + uint8_t prot; + uint8_t type; + } type_protect; + struct { + void* dst; + void* src; + uint64_t len; + } move; + uintptr_t padding[3]; + }; +}; + +static_assert(sizeof(KernelMemoryPoolBatchEntry) == 32, + "KernelMemoryPoolBatchEntry struct size is incorrect"); + +struct KernelMemoryPoolBlockStats { + int32_t available_flushed_blocks; + int32_t available_cached_blocks; + int32_t allocated_flushed_blocks; + int32_t allocated_cached_blocks; +}; + +static_assert(sizeof(KernelMemoryPoolBlockStats) == 16, + "KernelMemoryPoolBlockStats struct size is incorrect"); + +void RegisterCallbacks(callback_func_t alloc_func, callback_func_t free_func); +void SetFlexibleMemorySize(uint64_t size); +bool TryWriteBacking(uint64_t vaddr, const void* data, uint64_t size); +bool TryReadBacking(uint64_t vaddr, void* data, uint64_t size); +void WriteBacking(uint64_t vaddr, const void* data, uint64_t size) noexcept; + +int KYTY_SYSV_ABI KernelMapNamedFlexibleMemory(void** addr_in_out, size_t len, int prot, int flags, + const char* name); +int KYTY_SYSV_ABI KernelMapFlexibleMemory(void** addr_in_out, size_t len, int prot, int flags); +int KYTY_SYSV_ABI KernelSetVirtualRangeName(const void* addr, uint64_t len, const char* name); +int KYTY_SYSV_ABI KernelMunmap(uint64_t vaddr, size_t len); +size_t KYTY_SYSV_ABI KernelGetDirectMemorySize(); +int KYTY_SYSV_ABI KernelAvailableDirectMemorySize(int64_t search_start, int64_t search_end, + size_t alignment, int64_t* phys_addr_out, + size_t* size_out); +int KYTY_SYSV_ABI KernelGetPageTableStats(int* cpu_total, int* cpu_available, int* gpu_total, + int* gpu_available); +int KYTY_SYSV_ABI KernelAllocateDirectMemory(int64_t search_start, int64_t search_end, size_t len, + size_t alignment, int memory_type, + int64_t* phys_addr_out); +int KYTY_SYSV_ABI KernelAllocateMainDirectMemory(size_t len, size_t alignment, int memory_type, + int64_t* phys_addr_out); +int KYTY_SYSV_ABI KernelCheckedReleaseDirectMemory(int64_t start, size_t len); +int KYTY_SYSV_ABI KernelReleaseDirectMemory(int64_t start, size_t len); +int KYTY_SYSV_ABI KernelMapDirectMemory(void** addr, size_t len, int prot, int flags, + int64_t direct_memory_start, size_t alignment); +int KYTY_SYSV_ABI KernelMapDirectMemory2(void** addr, size_t len, int type, int prot, int flags, + int64_t direct_memory_start, size_t alignment); +int KYTY_SYSV_ABI KernelMapNamedDirectMemory(void** addr, size_t len, int prot, int flags, + int64_t direct_memory_start, size_t alignment, + const char* name); +int KYTY_SYSV_ABI KernelSetPrtAperture(int index, void* addr, size_t len); +int KYTY_SYSV_ABI KernelGetPrtAperture(int index, void** addr, size_t* len); +int KYTY_SYSV_ABI KernelIsAddressSanitizerEnabled(); +int KYTY_SYSV_ABI KernelQueryMemoryProtection(void* addr, void** start, void** end, int* prot); +int KYTY_SYSV_ABI KernelDirectMemoryQuery(int64_t offset, int flags, void* info, size_t info_size); +int KYTY_SYSV_ABI KernelVirtualQuery(const void* addr, int flags, VirtualQueryInfo* info, + uint64_t info_size); +int KYTY_SYSV_ABI KernelIsStack(void* addr, void** start, void** end); +int KYTY_SYSV_ABI KernelReserveVirtualRange(void** addr, size_t len, int flags, size_t alignment); +bool KernelHandleReservedRangeAccessViolation(uint64_t vaddr); +int KYTY_SYSV_ABI KernelAvailableFlexibleMemorySize(size_t* size); +int KYTY_SYSV_ABI KernelConfiguredFlexibleMemorySize(uint64_t* size); +int KYTY_SYSV_ABI KernelMprotect(const void* addr, size_t len, int prot); +int KYTY_SYSV_ABI KernelMtypeprotect(const void* addr, size_t len, int type, int prot); +int KYTY_SYSV_ABI KernelBatchMap(KernelBatchMapEntry* entries, int num_entries, + int* num_entries_out); +int KYTY_SYSV_ABI KernelBatchMap2(KernelBatchMapEntry* entries, int num_entries, + int* num_entries_out, int flags); +int KYTY_SYSV_ABI KernelMemoryPoolExpand(int64_t search_start, int64_t search_end, size_t len, + size_t alignment, int64_t* phys_addr_out); +int KYTY_SYSV_ABI KernelMemoryPoolReserve(void* addr_in, size_t len, size_t alignment, int flags, + void** addr_out); +int KYTY_SYSV_ABI KernelMemoryPoolCommit(void* addr, size_t len, int type, int prot, int flags); +int KYTY_SYSV_ABI KernelMemoryPoolDecommit(void* addr, size_t len, int flags); +int KYTY_SYSV_ABI KernelMemoryPoolBatch(const KernelMemoryPoolBatchEntry* entries, int num_entries, + int* num_entries_out, int flags); +int KYTY_SYSV_ABI KernelMemoryPoolGetBlockStats(KernelMemoryPoolBlockStats* output, + size_t output_size); + +void RegisterProgramMemory(uint64_t vaddr, uint64_t size, Common::VirtualMemory::Mode mode, + const char* name); +void UpdateProgramMemoryProtection(uint64_t vaddr, uint64_t size, Common::VirtualMemory::Mode mode); +void UnregisterProgramMemory(uint64_t vaddr, uint64_t size); + +} // namespace Libs::LibKernel::Memory + +#endif /* EMULATOR_INCLUDE_EMULATOR_KERNEL_MEMORY_H_ */ diff --git a/src/kernel/memoryAddressSpace.inc b/src/kernel/memoryAddressSpace.inc new file mode 100644 index 000000000..decac757c --- /dev/null +++ b/src/kernel/memoryAddressSpace.inc @@ -0,0 +1,1378 @@ +// Internal memory address-space helpers included by memory.cpp. +// Kept in the same translation unit because they depend on memory.cpp-local helpers. + +#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS +static constexpr uint64_t HOST_SYSTEM_MANAGED_MIN = 0x0000040000ull; +static constexpr uint64_t HOST_SYSTEM_MANAGED_MAX = 0x07fffeffffull; +static constexpr uint64_t HOST_USER_MIN = 0x1000000000ull; +static constexpr uint64_t HOST_USER_MAX = 0xfbffffffffull; +#endif + +class DirectMemoryBacking { +public: + enum class FailureReason { + None, + BackingUnavailable, + PhysicalRangeOutOfBounds, + HostAddressNotGranularityAligned, + SizeNotGranularityAligned, + ReserveAlignedFailed, + FixedMapViewFailed, + PlaceholderReserveFailed, + PlaceholderMapFailed, + ProtectFailed, + ReservedContainerHasCommittedPages, + ReservedContainerReleaseFailed, + ReservedContainerRestoreFailed, + ReservedContainerSplitNotGranularityAligned, + WrongHostAddress, + UnmapRollbackFailed, + }; + + explicit DirectMemoryBacking(uint64_t size): m_size(size) { +#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS + SYSTEM_INFO info {}; + GetSystemInfo(&info); + m_granularity = info.dwAllocationGranularity; + + m_handle = CreateFileMappingA(INVALID_HANDLE_VALUE, nullptr, PAGE_EXECUTE_READWRITE, + static_cast(m_size >> 32u), + static_cast(m_size & 0xffffffffu), nullptr); + if (m_handle == nullptr) { + LOGF_COLOR(Log::Color::Red, + "\t direct-memory backing: CreateFileMapping failed: 0x%08" PRIx32 "\n", + static_cast(GetLastError())); + return; + } + m_backing_base = static_cast( + MapViewOfFile(m_handle, FILE_MAP_READ | FILE_MAP_WRITE, 0, 0, m_size)); + if (m_backing_base == nullptr) { + LOGF_COLOR(Log::Color::Red, + "\t direct-memory backing: writable alias failed: 0x%08" PRIx32 "\n", + static_cast(GetLastError())); + CloseHandle(m_handle); + m_handle = nullptr; + } +#elif KYTY_PLATFORM == KYTY_PLATFORM_LINUX + m_fd = static_cast(syscall(SYS_memfd_create, "KytyDirectMemory", 0)); + if (m_fd < 0) { + LOGF_COLOR(Log::Color::Red, "\t direct-memory backing: memfd_create failed: %d\n", + errno); + return; + } + if (ftruncate(m_fd, static_cast(m_size)) != 0) { + LOGF_COLOR(Log::Color::Red, "\t direct-memory backing: ftruncate failed: %d\n", errno); + close(m_fd); + m_fd = -1; + return; + } + auto* backing = mmap(nullptr, m_size, PROT_READ | PROT_WRITE, MAP_SHARED, m_fd, 0); + if (backing == MAP_FAILED) { + LOGF_COLOR(Log::Color::Red, "\t direct-memory backing: writable alias failed: %d\n", + errno); + close(m_fd); + m_fd = -1; + return; + } + m_backing_base = static_cast(backing); +#endif + } + + ~DirectMemoryBacking() { +#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS + if (m_backing_base != nullptr) { + UnmapViewOfFile(m_backing_base); + } + if (m_handle != nullptr) { + CloseHandle(m_handle); + } +#elif KYTY_PLATFORM == KYTY_PLATFORM_LINUX + if (m_backing_base != nullptr) { + munmap(m_backing_base, m_size); + } + if (m_fd >= 0) { + close(m_fd); + } +#endif + } + + KYTY_CLASS_NO_COPY(DirectMemoryBacking); + + static const char* GetFailureReasonName(FailureReason reason) { + switch (reason) { + case FailureReason::None: return "n/a"; + case FailureReason::BackingUnavailable: return "backing-unavailable"; + case FailureReason::PhysicalRangeOutOfBounds: return "physical-range-out-of-bounds"; + case FailureReason::HostAddressNotGranularityAligned: + return "host-address-not-64k-aligned"; + case FailureReason::SizeNotGranularityAligned: return "size-not-64k-aligned"; + case FailureReason::ReserveAlignedFailed: return "reserve-aligned-failed"; + case FailureReason::FixedMapViewFailed: return "fixed-map-view-failed"; + case FailureReason::PlaceholderReserveFailed: return "placeholder-reserve-failed"; + case FailureReason::PlaceholderMapFailed: return "placeholder-map-failed"; + case FailureReason::ProtectFailed: return "protect-failed"; + case FailureReason::ReservedContainerHasCommittedPages: + return "reserved-container-has-committed-pages"; + case FailureReason::ReservedContainerReleaseFailed: + return "reserved-container-release-failed"; + case FailureReason::ReservedContainerRestoreFailed: + return "reserved-container-restore-failed"; + case FailureReason::ReservedContainerSplitNotGranularityAligned: + return "reserved-container-split-not-64k-aligned"; + case FailureReason::WrongHostAddress: return "wrong-host-address"; + case FailureReason::UnmapRollbackFailed: return "unmap-rollback-failed"; + default: return "unknown"; + } + } + + [[nodiscard]] bool IsAvailable() const { +#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS + return m_handle != nullptr && m_backing_base != nullptr; +#elif KYTY_PLATFORM == KYTY_PLATFORM_LINUX + return m_fd >= 0 && m_backing_base != nullptr; +#else + return false; +#endif + } + + bool TryWriteBacking(uint64_t vaddr, const void* data, uint64_t size) { + if (data == nullptr) { + return false; + } + const auto* source = static_cast(data); + return TryTransferBacking(vaddr, size, + [source](uint8_t* backing, uint64_t data_offset, size_t bytes) { + std::memcpy(backing, source + data_offset, bytes); + }); + } + + bool TryReadBacking(uint64_t vaddr, void* data, uint64_t size) { + if (data == nullptr) { + return false; + } + auto* destination = static_cast(data); + return TryTransferBacking(vaddr, size, + [destination](uint8_t* backing, uint64_t data_offset, size_t bytes) { + std::memcpy(destination + data_offset, backing, bytes); + }); + } + + uint64_t MapAligned(uint64_t search_addr, uint64_t size, uint64_t backing_offset, + VirtualMemory::Mode mode, uint64_t alignment, + FailureReason* failure_reason = nullptr) { + if (!IsAvailable() || !RangeOk(backing_offset, size)) { + SetFailure(failure_reason, !IsAvailable() ? FailureReason::BackingUnavailable + : FailureReason::PhysicalRangeOutOfBounds); + return 0; + } + + if (alignment == 0) { + alignment = 0x4000; + } +#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS + alignment = std::max(alignment, m_granularity); + return MapNewPlaceholderAligned(search_addr, size, backing_offset, mode, alignment, + failure_reason); +#else + auto reserved = VirtualMemory::ReserveAligned(search_addr, size, alignment); + if (reserved == 0) { + SetFailure(failure_reason, FailureReason::ReserveAlignedFailed); + return 0; + } + if (!ReleaseReservedProbe(reserved, failure_reason)) { + return 0; + } + if (MapFixed(reserved, size, backing_offset, mode, failure_reason)) { + return reserved; + } + + return 0; +#endif + } + + bool MapFixed(uint64_t vaddr, uint64_t size, uint64_t backing_offset, VirtualMemory::Mode mode, + FailureReason* failure_reason = nullptr) { +#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS + if (ShouldChunkMap(size) && IsGranularityAligned(vaddr) && + IsGranularityAligned(backing_offset)) { + return MapFixedChunked(vaddr, size, backing_offset, mode, failure_reason); + } +#endif + + if (!IsAvailable() || !RangeOk(backing_offset, size)) { + SetFailure(failure_reason, !IsAvailable() ? FailureReason::BackingUnavailable + : FailureReason::PhysicalRangeOutOfBounds); + return false; + } + + void* mapped = MapView(vaddr, size, backing_offset, mode, failure_reason); +#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS + if (mapped == nullptr && ReleaseExactHostAllocation(vaddr, size)) { + mapped = MapView(vaddr, size, backing_offset, mode, failure_reason); + } + if (mapped == nullptr) { + ReleasedReservedHostAllocation released {}; + if (ReleaseReservedHostAllocationForMap(vaddr, size, &released, failure_reason)) { + mapped = MapView(vaddr, size, backing_offset, mode, failure_reason); + if (mapped == nullptr || reinterpret_cast(mapped) != vaddr) { + if (mapped != nullptr) { + UnmapView(mapped, size); + mapped = nullptr; + } + RestoreReleasedReservedHostAllocation(released); + } else if (!ReserveReleasedReservedHostRemainders(released)) { + UnmapView(mapped, size); + mapped = nullptr; + RestoreReleasedReservedHostAllocation(released); + SetFailure(failure_reason, FailureReason::ReservedContainerRestoreFailed); + } + } + } + if (mapped == nullptr) { + std::vector released_ranges; + if (ReleaseReservedHostAllocationsForMap(vaddr, size, &released_ranges, + failure_reason)) { + mapped = MapView(vaddr, size, backing_offset, mode, failure_reason); + if (mapped == nullptr || reinterpret_cast(mapped) != vaddr) { + if (mapped != nullptr) { + UnmapView(mapped, size); + mapped = nullptr; + } + RestoreReleasedReservedHostAllocations(released_ranges); + } else if (!ReserveReleasedReservedHostRemainders(released_ranges)) { + UnmapView(mapped, size); + mapped = nullptr; + RestoreReleasedReservedHostAllocations(released_ranges); + SetFailure(failure_reason, FailureReason::ReservedContainerRestoreFailed); + } + } + } +#endif + if (mapped == nullptr || reinterpret_cast(mapped) != vaddr) { + if (mapped != nullptr) { + if (!UnmapView(mapped, size)) { + SetFailure(failure_reason, FailureReason::UnmapRollbackFailed); + } + } + if (mapped != nullptr) { + SetFailure(failure_reason, FailureReason::WrongHostAddress); + } + return false; + } + + { + std::lock_guard lock(m_mutex); + EXIT_IF(!m_maps.emplace(vaddr, MapEntry {vaddr, size, backing_offset, mode}).second); + } + SetFailure(failure_reason, FailureReason::None); + return true; + } + + bool MapExistingPlaceholderFixed(uint64_t vaddr, uint64_t size, uint64_t backing_offset, + VirtualMemory::Mode mode, + FailureReason* failure_reason = nullptr) { + if (!IsAvailable() || !RangeOk(backing_offset, size)) { + SetFailure(failure_reason, !IsAvailable() ? FailureReason::BackingUnavailable + : FailureReason::PhysicalRangeOutOfBounds); + return false; + } + +#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS + void* mapped = ReplacePlaceholderView(vaddr, size, backing_offset, mode, failure_reason); + if (mapped == nullptr || reinterpret_cast(mapped) != vaddr) { + if (mapped != nullptr) { + UnmapView(mapped, size); + SetFailure(failure_reason, FailureReason::WrongHostAddress); + } + return false; + } + + { + std::lock_guard lock(m_mutex); + EXIT_IF(!m_maps.emplace(vaddr, MapEntry {vaddr, size, backing_offset, mode}).second); + } + SetFailure(failure_reason, FailureReason::None); + return true; +#else + return MapFixed(vaddr, size, backing_offset, mode, failure_reason); +#endif + } + + bool Unmap(uint64_t vaddr, uint64_t size, bool preserve_placeholder = false, + bool* placeholder_preserved = nullptr) { + if (size == 0 || UINT64_MAX - vaddr < size) { + return false; + } + + if (placeholder_preserved != nullptr) { + *placeholder_preserved = false; + } + + if (!Contains(vaddr, size)) { + return false; + } + + const auto end = vaddr + size; + auto current = vaddr; + while (current < end) { + MapEntry entry {}; + { + std::lock_guard lock(m_mutex); + if (!FindContainingUnlocked(current, 1, &entry)) { + return false; + } + } + + const auto entry_end = entry.vaddr + entry.size; + const auto part_size = std::min(end, entry_end) - current; + const bool preserve_part_placeholder = + preserve_placeholder && current == vaddr && part_size == size; + if (!UnmapOne(current, part_size, preserve_part_placeholder, placeholder_preserved)) { + return false; + } + + current += part_size; + } + + return true; + } + + bool UnmapOne(uint64_t vaddr, uint64_t size, bool preserve_placeholder = false, + bool* placeholder_preserved = nullptr) { + MapEntry old {}; + { + std::lock_guard lock(m_mutex); + auto next = m_maps.upper_bound(vaddr); + if (next != m_maps.begin()) { + auto entry = std::prev(next); + if (vaddr + size <= entry->second.vaddr + entry->second.size) { + old = entry->second; + m_maps.erase(entry); + } + } + } + + if (old.size == 0) { + return false; + } + + const bool exact_unmap = (vaddr == old.vaddr && size == old.size); + if (exact_unmap) { + const bool unmapped = + preserve_placeholder + ? UnmapViewPreservePlaceholder(reinterpret_cast(old.vaddr), old.size) + : UnmapView(reinterpret_cast(old.vaddr), old.size); + if (!unmapped) { + std::lock_guard lock(m_mutex); + m_maps.emplace(old.vaddr, old); + return false; + } + if (preserve_placeholder && placeholder_preserved != nullptr) { + *placeholder_preserved = true; + } + return true; + } + + const uint64_t left_size = vaddr - old.vaddr; + const uint64_t right_addr = vaddr + size; + const uint64_t old_end = old.vaddr + old.size; + const uint64_t right_size = old_end - right_addr; + +#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS + if (UnmapViewPreservePlaceholder(reinterpret_cast(old.vaddr), old.size)) { + bool ok = true; + if (left_size != 0 && left_size != old.size) { + ok = SplitPlaceholder(old.vaddr, left_size) && ok; + } + if (right_size != 0) { + ok = SplitPlaceholder(vaddr, size) && ok; + } + if (left_size != 0) { + ok = MapExistingPlaceholderFixed(old.vaddr, left_size, old.backing_offset, + old.mode) && + ok; + } + if (right_size != 0) { + ok = MapExistingPlaceholderFixed(right_addr, right_size, + old.backing_offset + (right_addr - old.vaddr), + old.mode) && + ok; + } + if (ok) { + if (placeholder_preserved != nullptr) { + *placeholder_preserved = true; + } + return true; + } + } else +#endif + { + if (!UnmapView(reinterpret_cast(old.vaddr), old.size)) { + std::lock_guard lock(m_mutex); + m_maps.emplace(old.vaddr, old); + return false; + } + } + + bool ok = true; + if (left_size != 0) { + ok = MapFixed(old.vaddr, left_size, old.backing_offset, old.mode) && ok; + } + if (right_size != 0) { + ok = MapFixed(right_addr, right_size, old.backing_offset + (right_addr - old.vaddr), + old.mode) && + ok; + } + if (!ok) { + if (left_size != 0) { + UnmapOne(old.vaddr, left_size); + } + if (right_size != 0) { + UnmapOne(right_addr, right_size); + } + MapFixed(old.vaddr, old.size, old.backing_offset, old.mode); + return false; + } + + return ok; + } + + void UpdateProtection(uint64_t vaddr, uint64_t size, VirtualMemory::Mode mode) { + std::lock_guard lock(m_mutex); + for (auto& map_entry: m_maps) { + auto& entry = map_entry.second; + if (VirtualRangesOverlap(vaddr, size, entry.vaddr, entry.size)) { + entry.mode = mode; + } + } + } + + bool Contains(uint64_t vaddr, uint64_t size) { + if (size == 0 || UINT64_MAX - vaddr < size) { + return false; + } + + std::lock_guard lock(m_mutex); + const auto end = vaddr + size; + auto current = vaddr; + + while (current < end) { + MapEntry entry {}; + if (!FindContainingUnlocked(current, 1, &entry)) { + return false; + } + const auto entry_end = entry.vaddr + entry.size; + if (entry_end <= current) { + return false; + } + current = std::min(end, entry_end); + } + + return true; + } + + bool SelfTest() { + const uint64_t test_size = GetMappableTestSize(); + if (test_size == 0 || !RangeOk(0, test_size)) { + LOGF_COLOR(Log::Color::Yellow, "\t direct-memory backing self-test skipped\n"); + return false; + } + + const auto first = MapAligned(0, test_size, 0, VirtualMemory::Mode::ReadWrite, test_size); + if (first == 0) { + LOGF_COLOR(Log::Color::Yellow, + "\t direct-memory backing self-test skipped: first alias unavailable\n"); + return false; + } + + const auto second = MapAligned(0, test_size, 0, VirtualMemory::Mode::ReadWrite, test_size); + if (second == 0) { + Unmap(first, test_size); + LOGF_COLOR(Log::Color::Yellow, + "\t direct-memory backing self-test skipped: second alias unavailable\n"); + return false; + } + + auto* second_ptr = reinterpret_cast(second); + + constexpr uint64_t test_value = 0x4b595459414c4941ull; // "KYTYALIA" + EXIT_IF(!VirtualMemory::Protect(first, test_size, VirtualMemory::Mode::NoAccess)); + const bool ok = TryWriteBacking(first, &test_value, sizeof(test_value)) && + *second_ptr == test_value; + const uint64_t clear = 0; + EXIT_IF(!TryWriteBacking(first, &clear, sizeof(clear))); + + Unmap(second, test_size); + Unmap(first, test_size); + + LOGF_COLOR(ok ? Log::Color::Green : Log::Color::Red, + "\t direct-memory backing self-test: %s\n", ok ? "shared aliases ok" : "failed"); + return ok; + } + +private: + template + bool TryTransferBacking(uint64_t vaddr, uint64_t size, Transfer transfer) { + if (!IsAvailable() || size == 0 || UINT64_MAX - vaddr < size) { + return false; + } + + std::lock_guard lock(m_mutex); + const auto end = vaddr + size; + auto current = vaddr; + while (current < end) { + MapEntry entry {}; + if (!FindContainingUnlocked(current, 1, &entry)) { + return false; + } + const auto bytes = std::min(end, entry.vaddr + entry.size) - current; + if (!RangeOk(entry.backing_offset + current - entry.vaddr, bytes)) { + return false; + } + current += bytes; + } + + current = vaddr; + while (current < end) { + MapEntry entry {}; + EXIT_IF(!FindContainingUnlocked(current, 1, &entry)); + const auto bytes = std::min(end, entry.vaddr + entry.size) - current; + const auto offset = entry.backing_offset + current - entry.vaddr; + transfer(m_backing_base + offset, current - vaddr, static_cast(bytes)); + current += bytes; + } + return true; + } + + struct MapEntry { + uint64_t vaddr = 0; + uint64_t size = 0; + uint64_t backing_offset = 0; + VirtualMemory::Mode mode = VirtualMemory::Mode::NoAccess; + }; + + bool FindContainingUnlocked(uint64_t vaddr, uint64_t size, MapEntry* out) const { + EXIT_IF(out == nullptr); + + if (size == 0 || UINT64_MAX - vaddr < size) { + return false; + } + + auto next = m_maps.upper_bound(vaddr); + if (next == m_maps.begin()) { + return false; + } + const auto& entry = std::prev(next)->second; + if (vaddr + size <= entry.vaddr + entry.size) { + *out = entry; + return true; + } + + return false; + } + + [[nodiscard]] bool RangeOk(uint64_t backing_offset, uint64_t size) const { + return size != 0 && backing_offset < m_size && size <= m_size - backing_offset; + } + + static void SetFailure(FailureReason* out, FailureReason reason) { + if (out != nullptr) { + *out = reason; + } + } + +#if KYTY_PLATFORM != KYTY_PLATFORM_WINDOWS + static bool ReleaseReservedProbe(uint64_t vaddr, FailureReason* failure_reason) { + if (VirtualMemory::Free(vaddr)) { + return true; + } + SetFailure(failure_reason, FailureReason::ReservedContainerReleaseFailed); + return false; + } +#endif + +#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS + [[nodiscard]] uint64_t GetMappableTestSize() const { return m_granularity; } + static constexpr uint64_t PageSize() { return 0x4000; } + + [[nodiscard]] static bool IsPageAligned(uint64_t value) { + return (value & (PageSize() - 1u)) == 0; + } + + [[nodiscard]] bool IsGranularityAligned(uint64_t value) const { + return (value & (m_granularity - 1u)) == 0; + } + + [[nodiscard]] uint64_t GetMapChunkSize() const { + return std::max(m_granularity, 0x200000ull); + } + + [[nodiscard]] bool ShouldChunkMap(uint64_t size) const { return size > GetMapChunkSize(); } + + uint64_t MapNewPlaceholderAligned(uint64_t search_addr, uint64_t size, + uint64_t backing_offset, VirtualMemory::Mode mode, + uint64_t alignment, FailureReason* failure_reason) { + MEM_ADDRESS_REQUIREMENTS requirements {}; + MEM_EXTENDED_PARAMETER parameter {}; + const auto aligned_search = + (search_addr + alignment - 1u) & ~(alignment - 1u); + requirements.LowestStartingAddress = reinterpret_cast( + search_addr != 0 ? aligned_search : HOST_SYSTEM_MANAGED_MIN); + requirements.HighestEndingAddress = + reinterpret_cast(search_addr != 0 ? HOST_USER_MAX : HOST_SYSTEM_MANAGED_MAX); + requirements.Alignment = alignment; + parameter.Type = MemExtendedParameterAddressRequirements; + parameter.Pointer = &requirements; + + auto reserve = [&]() { + return VirtualAlloc2(GetCurrentProcess(), nullptr, size, + MEM_RESERVE | MEM_RESERVE_PLACEHOLDER, PAGE_NOACCESS, ¶meter, 1); + }; + auto* placeholder = reserve(); + if (placeholder == nullptr && search_addr == 0) { + requirements.LowestStartingAddress = reinterpret_cast(HOST_USER_MIN); + requirements.HighestEndingAddress = reinterpret_cast(HOST_USER_MAX); + placeholder = reserve(); + } + if (placeholder == nullptr) { + SetFailure(failure_reason, FailureReason::PlaceholderReserveFailed); + return 0; + } + + const auto vaddr = reinterpret_cast(placeholder); + std::vector entries; + uint64_t mapped = 0; + while (mapped < size) { + const auto current = vaddr + mapped; + const auto remaining = size - mapped; + const auto part_size = std::min(GetMapChunkSize(), remaining); + const bool split = part_size != remaining; + if (split && !SplitPlaceholder(current, part_size)) { + for (const auto& entry: entries) { + UnmapView(reinterpret_cast(entry.vaddr), entry.size); + } + VirtualFree(reinterpret_cast(current), 0, MEM_RELEASE); + SetFailure(failure_reason, FailureReason::PlaceholderReserveFailed); + return 0; + } + if (ReplacePlaceholderView(current, part_size, backing_offset + mapped, mode, + failure_reason) == nullptr) { + for (const auto& entry: entries) { + UnmapView(reinterpret_cast(entry.vaddr), entry.size); + } + VirtualFree(reinterpret_cast(current), 0, MEM_RELEASE); + if (split) { + VirtualFree(reinterpret_cast(current + part_size), 0, MEM_RELEASE); + } + return 0; + } + entries.push_back(MapEntry {current, part_size, backing_offset + mapped, mode}); + mapped += part_size; + } + + { + std::lock_guard lock(m_mutex); + for (const auto& entry: entries) { + EXIT_IF(!m_maps.emplace(entry.vaddr, entry).second); + } + } + SetFailure(failure_reason, FailureReason::None); + return vaddr; + } + + bool MapFixedChunked(uint64_t vaddr, uint64_t size, uint64_t backing_offset, + VirtualMemory::Mode mode, FailureReason* failure_reason) { + if (!IsAvailable() || !RangeOk(backing_offset, size)) { + SetFailure(failure_reason, !IsAvailable() ? FailureReason::BackingUnavailable + : FailureReason::PhysicalRangeOutOfBounds); + return false; + } + if (!IsGranularityAligned(vaddr) || !IsGranularityAligned(backing_offset)) { + SetFailure(failure_reason, FailureReason::HostAddressNotGranularityAligned); + return false; + } + + const auto chunk_size = GetMapChunkSize(); + uint64_t mapped = 0; + while (mapped < size) { + const auto part_size = std::min(chunk_size, size - mapped); + if (!MapFixed(vaddr + mapped, part_size, backing_offset + mapped, mode, + failure_reason)) { + uint64_t rollback = 0; + while (rollback < mapped) { + const auto rollback_size = std::min(chunk_size, mapped - rollback); + UnmapOne(vaddr + rollback, rollback_size); + rollback += rollback_size; + } + return false; + } + mapped += part_size; + } + + SetFailure(failure_reason, FailureReason::None); + return true; + } + + static DWORD GetProtect(VirtualMemory::Mode mode) { + switch (mode) { + case VirtualMemory::Mode::Read: return PAGE_READONLY; + case VirtualMemory::Mode::Write: + case VirtualMemory::Mode::ReadWrite: return PAGE_READWRITE; + case VirtualMemory::Mode::Execute: return PAGE_EXECUTE; + case VirtualMemory::Mode::ExecuteRead: return PAGE_EXECUTE_READ; + case VirtualMemory::Mode::ExecuteWrite: + case VirtualMemory::Mode::ExecuteReadWrite: return PAGE_EXECUTE_READWRITE; + case VirtualMemory::Mode::NoAccess: + default: return PAGE_NOACCESS; + } + } + + static bool ApplyProtection(void* ptr, uint64_t size, VirtualMemory::Mode mode) { + DWORD old_protect = 0; + return VirtualProtect(ptr, size, GetProtect(mode), &old_protect) != 0; + } + + static bool SplitPlaceholder(uint64_t vaddr, uint64_t size) { + if (size == 0) { + return true; + } + return VirtualFree(reinterpret_cast(vaddr), size, + MEM_RELEASE | MEM_PRESERVE_PLACEHOLDER) != 0; + } + + static bool ReleaseExactHostAllocation(uint64_t vaddr, uint64_t size) { + MEMORY_BASIC_INFORMATION info {}; + if (VirtualQuery(reinterpret_cast(vaddr), &info, sizeof(info)) == 0) { + return false; + } + + if ((info.State != MEM_RESERVE && info.State != MEM_COMMIT) || + reinterpret_cast(info.BaseAddress) != vaddr || + reinterpret_cast(info.AllocationBase) != vaddr || info.RegionSize != size) { + return false; + } + + return VirtualFree(reinterpret_cast(vaddr), 0, MEM_RELEASE) != 0; + } + + struct ReleasedReservedHostAllocation { + uint64_t base = 0; + uint64_t size = 0; + uint64_t map_start = 0; + uint64_t map_size = 0; + }; + + static bool ReserveHostRange(uint64_t vaddr, uint64_t size) { + if (size == 0) { + return true; + } + + auto* ptr = VirtualAlloc(reinterpret_cast(vaddr), size, MEM_RESERVE, PAGE_NOACCESS); + return ptr != nullptr && reinterpret_cast(ptr) == vaddr; + } + + static bool + RestoreReleasedReservedHostAllocation(const ReleasedReservedHostAllocation& released) { + return ReserveHostRange(released.base, released.size); + } + + static bool RestoreReleasedReservedHostAllocations( + const std::vector& released_ranges) { + bool ok = true; + for (const auto& released: released_ranges) { + ok = RestoreReleasedReservedHostAllocation(released) && ok; + } + return ok; + } + + void* ReplacePlaceholderView(uint64_t vaddr, uint64_t size, uint64_t backing_offset, + VirtualMemory::Mode mode, FailureReason* failure_reason) { + const DWORD map_protect = + mode == VirtualMemory::Mode::NoAccess ? PAGE_READWRITE : GetProtect(mode); + void* ptr = + MapViewOfFile3(m_handle, GetCurrentProcess(), reinterpret_cast(vaddr), + backing_offset, size, MEM_REPLACE_PLACEHOLDER, map_protect, nullptr, 0); + if (ptr == nullptr || reinterpret_cast(ptr) != vaddr) { + if (ptr != nullptr) { + UnmapViewOfFile(ptr); + } + SetFailure(failure_reason, FailureReason::PlaceholderMapFailed); + return nullptr; + } + + if (!ApplyProtection(ptr, size, mode)) { + UnmapViewOfFile(ptr); + SetFailure(failure_reason, FailureReason::ProtectFailed); + return nullptr; + } + return ptr; + } + + static bool + ReserveReleasedReservedHostRemainders(const ReleasedReservedHostAllocation& released) { + const auto map_end = released.map_start + released.map_size; + const auto released_end = released.base + released.size; + const auto left_size = released.map_start - released.base; + const auto right_size = released_end - map_end; + + if (!ReserveHostRange(released.base, left_size)) { + return false; + } + if (!ReserveHostRange(map_end, right_size)) { + if (left_size != 0) { + VirtualFree(reinterpret_cast(released.base), 0, MEM_RELEASE); + } + return false; + } + return true; + } + + static bool ReserveReleasedReservedHostRemainders( + const std::vector& released_ranges) { + bool ok = true; + for (const auto& released: released_ranges) { + ok = ReserveReleasedReservedHostRemainders(released) && ok; + } + return ok; + } + + bool QueryReservedHostAllocation(uint64_t vaddr, ReleasedReservedHostAllocation* out, + FailureReason* failure_reason) const { + EXIT_IF(out == nullptr); + + MEMORY_BASIC_INFORMATION info {}; + if (VirtualQuery(reinterpret_cast(vaddr), &info, sizeof(info)) == 0) { + SetFailure(failure_reason, FailureReason::PlaceholderReserveFailed); + return false; + } + + const auto allocation_base = reinterpret_cast(info.AllocationBase); + const auto query_base = reinterpret_cast(info.BaseAddress); + if (info.State != MEM_RESERVE || query_base > vaddr || allocation_base > vaddr) { + SetFailure(failure_reason, FailureReason::PlaceholderReserveFailed); + return false; + } + + uint64_t allocation_end = query_base + info.RegionSize; + uint64_t scan = allocation_end; + while (true) { + MEMORY_BASIC_INFORMATION next {}; + if (VirtualQuery(reinterpret_cast(scan), &next, sizeof(next)) == 0 || + reinterpret_cast(next.AllocationBase) != allocation_base || + next.State == MEM_FREE) { + break; + } + if (next.State != MEM_RESERVE) { + SetFailure(failure_reason, FailureReason::ReservedContainerHasCommittedPages); + return false; + } + allocation_end = reinterpret_cast(next.BaseAddress) + next.RegionSize; + scan = allocation_end; + } + + scan = allocation_base; + while (scan < allocation_end) { + MEMORY_BASIC_INFORMATION part {}; + if (VirtualQuery(reinterpret_cast(scan), &part, sizeof(part)) == 0 || + reinterpret_cast(part.AllocationBase) != allocation_base || + part.State != MEM_RESERVE) { + SetFailure(failure_reason, FailureReason::ReservedContainerHasCommittedPages); + return false; + } + scan = reinterpret_cast(part.BaseAddress) + part.RegionSize; + } + + out->base = allocation_base; + out->size = allocation_end - allocation_base; + out->map_start = 0; + out->map_size = 0; + return true; + } + + bool ReleaseReservedHostAllocationForMap(uint64_t vaddr, uint64_t size, + ReleasedReservedHostAllocation* out, + FailureReason* failure_reason) const { + if (!QueryReservedHostAllocation(vaddr, out, failure_reason)) { + return false; + } + + const auto allocation_end = out->base + out->size; + if (vaddr < out->base || vaddr + size > allocation_end) { + SetFailure(failure_reason, FailureReason::PlaceholderReserveFailed); + return false; + } + if (out->base == vaddr && out->size == size) { + return false; + } + if (!IsGranularityAligned(size)) { + SetFailure(failure_reason, FailureReason::ReservedContainerSplitNotGranularityAligned); + return false; + } + if (VirtualFree(reinterpret_cast(out->base), 0, MEM_RELEASE) == 0) { + SetFailure(failure_reason, FailureReason::ReservedContainerReleaseFailed); + return false; + } + + out->map_start = vaddr; + out->map_size = size; + return true; + } + + bool ReleaseReservedHostAllocationsForMap(uint64_t vaddr, uint64_t size, + std::vector* out, + FailureReason* failure_reason) const { + EXIT_IF(out == nullptr); + + out->clear(); + if (size == 0 || !IsPageAligned(vaddr) || !IsPageAligned(size)) { + SetFailure(failure_reason, FailureReason::ReservedContainerSplitNotGranularityAligned); + return false; + } + + const auto target_end = vaddr + size; + auto current = vaddr; + while (current < target_end) { + ReleasedReservedHostAllocation released {}; + if (!QueryReservedHostAllocation(current, &released, failure_reason)) { + out->clear(); + return false; + } + + const auto map_start = std::max(vaddr, released.base); + const auto map_end = std::min(target_end, released.base + released.size); + if (map_start >= map_end || !IsPageAligned(map_start) || + !IsPageAligned(map_end - map_start)) { + SetFailure(failure_reason, + FailureReason::ReservedContainerSplitNotGranularityAligned); + out->clear(); + return false; + } + + released.map_start = map_start; + released.map_size = map_end - map_start; + out->push_back(released); + current = map_end; + } + + for (size_t i = 0; i < out->size(); i++) { + const auto& released = (*out)[i]; + if (VirtualFree(reinterpret_cast(released.base), 0, MEM_RELEASE) == 0) { + for (size_t restore_index = 0; restore_index < i; restore_index++) { + RestoreReleasedReservedHostAllocation((*out)[restore_index]); + } + SetFailure(failure_reason, FailureReason::ReservedContainerReleaseFailed); + out->clear(); + return false; + } + } + + return !out->empty(); + } + + void* MapPlaceholderView(uint64_t vaddr, uint64_t size, uint64_t backing_offset, + VirtualMemory::Mode mode, FailureReason* failure_reason) { + void* placeholder = + VirtualAlloc2(GetCurrentProcess(), reinterpret_cast(vaddr), size, + MEM_RESERVE | MEM_RESERVE_PLACEHOLDER, PAGE_NOACCESS, nullptr, 0); + if (placeholder == nullptr || reinterpret_cast(placeholder) != vaddr) { + if (placeholder != nullptr) { + VirtualFree(placeholder, 0, MEM_RELEASE); + } + SetFailure(failure_reason, FailureReason::PlaceholderReserveFailed); + return nullptr; + } + + auto* ptr = ReplacePlaceholderView(vaddr, size, backing_offset, mode, failure_reason); + if (ptr == nullptr) { + VirtualFree(reinterpret_cast(vaddr), 0, MEM_RELEASE); + return nullptr; + } + return ptr; + } + + void* MapView(uint64_t vaddr, uint64_t size, uint64_t backing_offset, VirtualMemory::Mode mode, + FailureReason* failure_reason) { + constexpr DWORD access = FILE_MAP_READ | FILE_MAP_WRITE | FILE_MAP_EXECUTE; + void* ptr = MapViewOfFileEx(m_handle, access, static_cast(backing_offset >> 32u), + static_cast(backing_offset & 0xffffffffu), size, + reinterpret_cast(vaddr)); + if (ptr == nullptr) { + SetFailure(failure_reason, FailureReason::FixedMapViewFailed); + ptr = ReplacePlaceholderView(vaddr, size, backing_offset, mode, failure_reason); + if (ptr != nullptr) { + return ptr; + } + return MapPlaceholderView(vaddr, size, backing_offset, mode, failure_reason); + } + + if (!ApplyProtection(ptr, size, mode)) { + UnmapViewOfFile(ptr); + SetFailure(failure_reason, FailureReason::ProtectFailed); + return nullptr; + } + return ptr; + } + + static bool UnmapView(void* vaddr, uint64_t /*size*/) { return UnmapViewOfFile(vaddr) != 0; } + + static bool UnmapViewPreservePlaceholder(void* vaddr, uint64_t /*size*/) { + return UnmapViewOfFile2(GetCurrentProcess(), vaddr, MEM_PRESERVE_PLACEHOLDER) != 0; + } + + HANDLE m_handle = nullptr; + uint64_t m_granularity = 0x10000; +#elif KYTY_PLATFORM == KYTY_PLATFORM_LINUX + [[nodiscard]] static uint64_t GetMappableTestSize() { return 0x4000; } + + static int GetProtect(VirtualMemory::Mode mode) { + switch (mode) { + case VirtualMemory::Mode::Read: return PROT_READ; + case VirtualMemory::Mode::Write: + case VirtualMemory::Mode::ReadWrite: return PROT_READ | PROT_WRITE; + case VirtualMemory::Mode::Execute: return PROT_EXEC; + case VirtualMemory::Mode::ExecuteRead: return PROT_EXEC | PROT_READ; + case VirtualMemory::Mode::ExecuteWrite: + case VirtualMemory::Mode::ExecuteReadWrite: return PROT_EXEC | PROT_READ | PROT_WRITE; + case VirtualMemory::Mode::NoAccess: + default: return PROT_NONE; + } + } + + void* MapView(uint64_t vaddr, uint64_t size, uint64_t backing_offset, VirtualMemory::Mode mode, + FailureReason* failure_reason) { + void* ptr = mmap(reinterpret_cast(vaddr), size, GetProtect(mode), + MAP_SHARED | MAP_FIXED, m_fd, static_cast(backing_offset)); + if (ptr == MAP_FAILED) { + SetFailure(failure_reason, FailureReason::FixedMapViewFailed); + return nullptr; + } + return ptr; + } + + static bool UnmapView(void* vaddr, uint64_t size) { return munmap(vaddr, size) == 0; } + + static bool UnmapViewPreservePlaceholder(void* /*vaddr*/, uint64_t /*size*/) { return false; } + + int m_fd = -1; +#endif + + uint8_t* m_backing_base = nullptr; + uint64_t m_size = 0; + std::mutex m_mutex; + std::map m_maps; +}; + +class PlaceholderAddressSpace { +public: + PlaceholderAddressSpace() { +#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS + SYSTEM_INFO info {}; + GetSystemInfo(&info); + m_granularity = info.dwAllocationGranularity; +#endif + } + + KYTY_CLASS_NO_COPY(PlaceholderAddressSpace); + + [[nodiscard]] uint64_t GetGranularity() const { return m_granularity; } + + uint64_t ReserveAligned(uint64_t search_addr, uint64_t size, uint64_t alignment) { +#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS + if (size == 0) { + return 0; + } + alignment = std::max(alignment != 0 ? alignment : 0x4000, m_granularity); + + MEM_ADDRESS_REQUIREMENTS req {}; + MEM_EXTENDED_PARAMETER param {}; + req.LowestStartingAddress = reinterpret_cast( + AlignUp(search_addr != 0 ? search_addr : HOST_SYSTEM_MANAGED_MIN, alignment)); + req.HighestEndingAddress = + reinterpret_cast(search_addr != 0 ? HOST_USER_MAX : HOST_SYSTEM_MANAGED_MAX); + req.Alignment = alignment; + param.Type = MemExtendedParameterAddressRequirements; + param.Pointer = &req; + + auto* ptr = VirtualAlloc2(GetCurrentProcess(), nullptr, size, + MEM_RESERVE | MEM_RESERVE_PLACEHOLDER, PAGE_NOACCESS, ¶m, 1); + if (ptr == nullptr && search_addr == 0) { + req.LowestStartingAddress = reinterpret_cast(HOST_USER_MIN); + req.HighestEndingAddress = reinterpret_cast(HOST_USER_MAX); + ptr = VirtualAlloc2(GetCurrentProcess(), nullptr, size, + MEM_RESERVE | MEM_RESERVE_PLACEHOLDER, PAGE_NOACCESS, ¶m, 1); + } + if (ptr == nullptr) { + return 0; + } + + const auto addr = reinterpret_cast(ptr); + AddFree(addr, size); + return addr; +#else + auto addr = VirtualMemory::ReserveAligned(search_addr, size, alignment); + return addr; +#endif + } + + bool ReserveFixed(uint64_t vaddr, uint64_t size) { +#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS + if (vaddr == 0 || size == 0 || !IsPageAligned(vaddr) || !IsPageAligned(size)) { + return false; + } + { + std::lock_guard lock(m_mutex); + if (FindContaining(vaddr, size) != m_free.end()) { + return true; + } + } + auto* ptr = VirtualAlloc2(GetCurrentProcess(), reinterpret_cast(vaddr), size, + MEM_RESERVE | MEM_RESERVE_PLACEHOLDER, PAGE_NOACCESS, nullptr, 0); + if (ptr == nullptr || reinterpret_cast(ptr) != vaddr) { + if (ptr != nullptr) { + VirtualFree(ptr, 0, MEM_RELEASE); + } + return false; + } + AddFree(vaddr, size); + return true; +#else + return VirtualMemory::ReserveFixed(vaddr, size); +#endif + } + + bool Consume(uint64_t vaddr, uint64_t size) { +#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS + std::lock_guard lock(m_mutex); + auto it = FindContaining(vaddr, size); + if (it == m_free.end()) { + return false; + } + if (!SplitUnlocked(it, vaddr, size)) { + return false; + } + it = m_free.find(vaddr); + if (it == m_free.end() || it->second != size) { + return false; + } + m_free.erase(it); + return true; +#else + return false; +#endif + } + + bool ReleaseFree(uint64_t vaddr, uint64_t size) { +#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS + std::lock_guard lock(m_mutex); + auto it = FindContaining(vaddr, size); + if (it == m_free.end() || !SplitUnlocked(it, vaddr, size)) { + return false; + } + it = m_free.find(vaddr); + if (it == m_free.end() || it->second != size) { + return false; + } + m_free.erase(it); + if (VirtualFree(reinterpret_cast(vaddr), 0, MEM_RELEASE) != 0) { + return true; + } + AddFreeUnlocked(vaddr, size); + return false; +#else + return VirtualMemory::FreeRange(vaddr, size); +#endif + } + + bool Commit(uint64_t vaddr, uint64_t size, VirtualMemory::Mode mode) { +#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS + if (!Consume(vaddr, size)) { + return false; + } + + auto* ptr = VirtualAlloc2(GetCurrentProcess(), reinterpret_cast(vaddr), size, + MEM_RESERVE | MEM_COMMIT | MEM_REPLACE_PLACEHOLDER, + GetProtect(mode), nullptr, 0); + if (ptr == nullptr || reinterpret_cast(ptr) != vaddr) { + if (ptr != nullptr) { + VirtualFree(ptr, 0, MEM_RELEASE); + } + AddFree(vaddr, size); + return false; + } + return true; +#else + return false; +#endif + } + + bool ReleaseCommitted(uint64_t vaddr, uint64_t size) { +#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS + if (vaddr == 0 || size == 0 || !IsPageAligned(vaddr) || !IsPageAligned(size)) { + return false; + } + if (VirtualFree(reinterpret_cast(vaddr), size, + MEM_RELEASE | MEM_PRESERVE_PLACEHOLDER) == 0) { + return false; + } + AddFree(vaddr, size); + return true; +#else + return false; +#endif + } + + bool SelfTest() { +#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS + const auto addr = ReserveAligned(0, m_granularity, m_granularity); + if (addr == 0) { + LOGF_COLOR(Log::Color::Yellow, + "\t placeholder address-space self-test skipped: reserve unavailable\n"); + return false; + } + if (!Commit(addr, m_granularity, VirtualMemory::Mode::ReadWrite)) { + ReleaseFree(addr, m_granularity); + LOGF_COLOR(Log::Color::Red, "\t placeholder address-space self-test: commit failed\n"); + return false; + } + + auto* ptr = reinterpret_cast(addr); + *ptr = 0x4b59545950484f4cull; // "KYTYPHOL" + + if (!ReleaseCommitted(addr, m_granularity)) { + VirtualFree(reinterpret_cast(addr), 0, MEM_RELEASE); + LOGF_COLOR(Log::Color::Red, + "\t placeholder address-space self-test: release-committed failed\n"); + return false; + } + + const bool ok = ReleaseFree(addr, m_granularity); + LOGF_COLOR(ok ? Log::Color::Green : Log::Color::Red, + "\t placeholder address-space self-test: %s\n", + ok ? "ok" : "release-free failed"); + return ok; +#else + LOGF_COLOR(Log::Color::Yellow, "\t placeholder address-space self-test skipped\n"); + return false; +#endif + } + + void AddFree(uint64_t vaddr, uint64_t size) { +#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS + std::lock_guard lock(m_mutex); + AddFreeUnlocked(vaddr, size); +#endif + } + +private: +#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS + static constexpr uint64_t PageSize() { return 0x4000; } + + static uint64_t AlignUp(uint64_t value, uint64_t alignment) { + return (value + alignment - 1u) & ~(alignment - 1u); + } + + [[nodiscard]] static bool IsPageAligned(uint64_t value) { + return (value & (PageSize() - 1u)) == 0; + } + + [[nodiscard]] bool IsGranularityAligned(uint64_t value) const { + return (value & (m_granularity - 1u)) == 0; + } + + static DWORD GetProtect(VirtualMemory::Mode mode) { + switch (mode) { + case VirtualMemory::Mode::Read: return PAGE_READONLY; + case VirtualMemory::Mode::Write: + case VirtualMemory::Mode::ReadWrite: return PAGE_READWRITE; + case VirtualMemory::Mode::Execute: return PAGE_EXECUTE; + case VirtualMemory::Mode::ExecuteRead: return PAGE_EXECUTE_READ; + case VirtualMemory::Mode::ExecuteWrite: + case VirtualMemory::Mode::ExecuteReadWrite: return PAGE_EXECUTE_READWRITE; + case VirtualMemory::Mode::NoAccess: + default: return PAGE_NOACCESS; + } + } + + std::map::iterator FindContaining(uint64_t vaddr, uint64_t size) { + auto next = m_free.upper_bound(vaddr); + if (next == m_free.begin()) { + return m_free.end(); + } + auto it = std::prev(next); + auto end = it->first + it->second; + return (vaddr >= it->first && vaddr + size <= end ? it : m_free.end()); + } + + bool SplitUnlocked(std::map::iterator it, uint64_t vaddr, uint64_t size) { + const auto base = it->first; + const auto end = it->first + it->second; + if (base == vaddr && end == vaddr + size) { + return true; + } + + if (base != vaddr) { + const auto left_size = vaddr - base; + if (VirtualFree(reinterpret_cast(base), left_size, + MEM_RELEASE | MEM_PRESERVE_PLACEHOLDER) == 0) { + return false; + } + it->second = left_size; + it = m_free.emplace(vaddr, end - vaddr).first; + } + + if (it->second != size) { + if (VirtualFree(reinterpret_cast(it->first), size, + MEM_RELEASE | MEM_PRESERVE_PLACEHOLDER) == 0) { + return false; + } + const auto right_start = it->first + size; + const auto right_size = it->second - size; + it->second = size; + m_free.emplace(right_start, right_size); + } + + return true; + } + + void AddFreeUnlocked(uint64_t vaddr, uint64_t size) { + if (vaddr == 0 || size == 0) { + return; + } + auto [it, inserted] = m_free.emplace(vaddr, size); + if (!inserted) { + it->second = size; + } + + auto base = it->first; + auto end = it->first + it->second; + + auto first = it; + while (first != m_free.begin()) { + auto prev = std::prev(first); + if (prev->first + prev->second != base) { + break; + } + base = prev->first; + first = prev; + } + + auto last = std::next(it); + while (last != m_free.end() && end == last->first) { + end = last->first + last->second; + last = std::next(last); + } + + if (base != it->first || end != it->first + it->second) { + if (VirtualFree(reinterpret_cast(base), end - base, + MEM_RELEASE | MEM_COALESCE_PLACEHOLDERS) != 0) { + m_free.erase(first, last); + m_free.emplace(base, end - base); + } + } + } + + uint64_t m_granularity = 0x10000; + std::mutex m_mutex; + std::map m_free; +#endif +}; diff --git a/src/kernel/pthread.cpp b/src/kernel/pthread.cpp new file mode 100644 index 000000000..da7501315 --- /dev/null +++ b/src/kernel/pthread.cpp @@ -0,0 +1,4408 @@ +#include "kernel/pthread.h" + +#include "common/assert.h" +#include "common/common.h" +#include "common/dateTime.h" +#include "common/emulatorConfig.h" +#include "common/logging/log.h" +#include "common/singleton.h" +#include "common/stringUtils.h" +#include "common/threads.h" +#include "common/timer.h" +#include "kernel/memory.h" +#include "libs/errno.h" +#include "libs/libs.h" +#include "loader/runtimeLinker.h" +#include "loader/timer.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS +#ifndef NOMINMAX +#define NOMINMAX +#endif +#if defined(_M_X64) || defined(__x86_64__) +#include +#endif +#include +#endif + +#if KYTY_PLATFORM == KYTY_PLATFORM_LINUX +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wdeprecated-declarations" +#endif + +#include + +#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS +#include +#include +#endif + +#ifdef pthread_attr_getguardsize +#undef pthread_attr_getguardsize +#endif + +namespace Libs { + +namespace LibKernel { + +LIB_NAME("libkernel", "libkernel"); + +constexpr int KEYS_MAX = 256; +constexpr int DESTRUCTOR_ITERATIONS = 4; +constexpr size_t PTHREAD_STACK_DEFAULT = 0x100000; +constexpr size_t PTHREAD_STACK_MIN = 0x4000; +constexpr size_t PTHREAD_STACK_PAGE = 0x4000; +constexpr size_t PTHREAD_STACK_GRANULARITY = 0x10000; +constexpr size_t PTHREAD_STACK_INITIAL = 0x200000; +constexpr size_t PTHREAD_STACK_EXTRA = 0x100000; +constexpr uint64_t PTHREAD_STACK_TOP = 0x7efff8000ull; +constexpr uint32_t SIGNAL_APC_POLL_MICROS = 10000; + +static constexpr KernelClockid KERNEL_CLOCK_REALTIME = 0; +static constexpr KernelClockid KERNEL_CLOCK_VIRTUAL = 1; +static constexpr KernelClockid KERNEL_CLOCK_PROF = 2; +static constexpr KernelClockid KERNEL_CLOCK_MONOTONIC = 4; +static constexpr KernelClockid KERNEL_CLOCK_UPTIME = 5; +static constexpr KernelClockid KERNEL_CLOCK_UPTIME_PRECISE = 7; +static constexpr KernelClockid KERNEL_CLOCK_UPTIME_FAST = 8; +static constexpr KernelClockid KERNEL_CLOCK_REALTIME_PRECISE = 9; +static constexpr KernelClockid KERNEL_CLOCK_REALTIME_FAST = 10; +static constexpr KernelClockid KERNEL_CLOCK_MONOTONIC_PRECISE = 11; +static constexpr KernelClockid KERNEL_CLOCK_MONOTONIC_FAST = 12; +static constexpr KernelClockid KERNEL_CLOCK_SECOND = 13; +static constexpr KernelClockid KERNEL_CLOCK_THREAD_CPUTIME_ID = 14; +static constexpr KernelClockid KERNEL_CLOCK_PROCTIME = 15; +static constexpr KernelClockid KERNEL_CLOCK_EXT_NETWORK = 16; +static constexpr KernelClockid KERNEL_CLOCK_EXT_DEBUG_NETWORK = 17; +static constexpr KernelClockid KERNEL_CLOCK_EXT_AD_NETWORK = 18; +static constexpr KernelClockid KERNEL_CLOCK_EXT_RAW_NETWORK = 19; + +static constexpr int KERNEL_PTHREAD_MUTEX_ERRORCHECK = 1; +static constexpr int KERNEL_PTHREAD_MUTEX_RECURSIVE = 2; +static constexpr int KERNEL_PTHREAD_MUTEX_NORMAL = 3; +static constexpr int KERNEL_PTHREAD_MUTEX_ADAPTIVE = 4; + +static uint64_t KernelReadTscNative() { +#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS && (defined(_M_X64) || defined(__x86_64__)) + return __rdtsc(); +#else + return Common::Timer::QueryPerformanceCounter(); +#endif +} + +static uint64_t KernelGetTscFrequencyNative() { + static const uint64_t frequency = [] { +#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS && (defined(_M_X64) || defined(__x86_64__)) + const auto host_frequency = Common::Timer::QueryPerformanceFrequency(); + if (host_frequency == 0) { + return uint64_t {1000000000}; + } + + KernelReadTscNative(); + Common::Thread::Sleep(1); + KernelReadTscNative(); + + const auto host_start = Common::Timer::QueryPerformanceCounter(); + const auto tsc_start = KernelReadTscNative(); + std::this_thread::sleep_for(std::chrono::milliseconds(100)); + const auto host_end = Common::Timer::QueryPerformanceCounter(); + const auto tsc_end = KernelReadTscNative(); + + const auto host_delta = host_end - host_start; + const auto tsc_delta = tsc_end - tsc_start; + if (host_delta == 0 || tsc_delta == 0) { + return host_frequency; + } + + auto freq = static_cast( + (static_cast(tsc_delta) * static_cast(host_frequency)) / + static_cast(host_delta)); + static constexpr uint64_t ROUND_TO = 100000; + const auto mod = freq % ROUND_TO; + freq = (mod >= ROUND_TO / 2 ? freq - mod + ROUND_TO : freq - mod); + + return freq != 0 ? freq : host_frequency; +#else + return Common::Timer::QueryPerformanceFrequency(); +#endif + }(); + + return frequency; +} + +static uint64_t KernelGetInitialTsc() { + static const uint64_t initial_tsc = KernelReadTscNative(); + return initial_tsc; +} + +static uint64_t KernelGetElapsedTsc() { + const auto initial_tsc = KernelGetInitialTsc(); + const auto current_tsc = KernelReadTscNative(); + return current_tsc >= initial_tsc ? current_tsc - initial_tsc : 0; +} + +static uint64_t KernelGetProcessTimeUsNative() { + const auto frequency = KernelGetTscFrequencyNative(); + if (frequency == 0) { + return static_cast(Loader::Timer::GetTimeMs() * 1000.0); + } + + const auto elapsed = KernelGetElapsedTsc(); + return static_cast((static_cast(elapsed) * 1000000.0L) / + static_cast(frequency)); +} + +static void KernelUsToTimespec(uint64_t us, KernelTimespec* tp) { + EXIT_IF(tp == nullptr); + tp->tv_sec = static_cast(us / 1000000); + tp->tv_nsec = static_cast((us % 1000000) * 1000); +} + +#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS +static uint64_t KernelFiletimeTo100ns(FILETIME ft) { + ULARGE_INTEGER v {}; + v.LowPart = ft.dwLowDateTime; + v.HighPart = ft.dwHighDateTime; + return v.QuadPart; +} + +static void Kernel100nsToTimespec(uint64_t value, KernelTimespec* tp) { + EXIT_IF(tp == nullptr); + tp->tv_sec = static_cast(value / 10000000); + tp->tv_nsec = static_cast((value % 10000000) * 100); +} + +static bool KernelRealtimeToTimespec(bool precise, KernelTimespec* tp) { + EXIT_IF(tp == nullptr); + + FILETIME ft {}; + if (precise) { + GetSystemTimePreciseAsFileTime(&ft); + } else { + GetSystemTimeAsFileTime(&ft); + } + + static constexpr uint64_t WINDOWS_UNIX_EPOCH_DELTA_100NS = 116444736000000000ULL; + const auto value = KernelFiletimeTo100ns(ft); + if (value < WINDOWS_UNIX_EPOCH_DELTA_100NS) { + return false; + } + + Kernel100nsToTimespec(value - WINDOWS_UNIX_EPOCH_DELTA_100NS, tp); + return true; +} + +static bool KernelMonotonicToTimespec(KernelTimespec* tp) { + EXIT_IF(tp == nullptr); + + const auto frequency = Common::Timer::QueryPerformanceFrequency(); + if (frequency == 0) { + return false; + } + + const auto counter = Common::Timer::QueryPerformanceCounter(); + tp->tv_sec = static_cast(counter / frequency); + tp->tv_nsec = static_cast(((counter % frequency) * 1000000000ull) / frequency); + return true; +} + +static bool KernelClockGettimeSpecial(KernelClockid clock_id, KernelTimespec* tp, int* error) { + EXIT_IF(tp == nullptr); + EXIT_IF(error == nullptr); + + if (clock_id == KERNEL_CLOCK_PROCTIME) { + KernelUsToTimespec(KernelGetProcessTimeUsNative(), tp); + return true; + } + + FILETIME create_time {}; + FILETIME exit_time {}; + FILETIME kernel_time {}; + FILETIME user_time {}; + + switch (clock_id) { + case KERNEL_CLOCK_REALTIME: + case KERNEL_CLOCK_REALTIME_PRECISE: + if (!KernelRealtimeToTimespec(true, tp)) { + *error = KERNEL_ERROR_EFAULT; + } + return true; + case KERNEL_CLOCK_REALTIME_FAST: + case KERNEL_CLOCK_SECOND: + if (!KernelRealtimeToTimespec(false, tp)) { + *error = KERNEL_ERROR_EFAULT; + } + if (clock_id == KERNEL_CLOCK_SECOND) { + tp->tv_nsec = 0; + } + return true; + case KERNEL_CLOCK_MONOTONIC: + case KERNEL_CLOCK_UPTIME: + case KERNEL_CLOCK_UPTIME_PRECISE: + case KERNEL_CLOCK_UPTIME_FAST: + case KERNEL_CLOCK_MONOTONIC_PRECISE: + case KERNEL_CLOCK_MONOTONIC_FAST: + case KERNEL_CLOCK_EXT_NETWORK: + case KERNEL_CLOCK_EXT_DEBUG_NETWORK: + case KERNEL_CLOCK_EXT_AD_NETWORK: + case KERNEL_CLOCK_EXT_RAW_NETWORK: + if (!KernelMonotonicToTimespec(tp)) { + *error = KERNEL_ERROR_EFAULT; + } + return true; + case KERNEL_CLOCK_THREAD_CPUTIME_ID: + if (!GetThreadTimes(GetCurrentThread(), &create_time, &exit_time, &kernel_time, + &user_time)) { + *error = KERNEL_ERROR_EFAULT; + return true; + } + Kernel100nsToTimespec( + KernelFiletimeTo100ns(kernel_time) + KernelFiletimeTo100ns(user_time), tp); + return true; + case KERNEL_CLOCK_VIRTUAL: + if (!GetProcessTimes(GetCurrentProcess(), &create_time, &exit_time, &kernel_time, + &user_time)) { + *error = KERNEL_ERROR_EFAULT; + return true; + } + Kernel100nsToTimespec(KernelFiletimeTo100ns(user_time), tp); + return true; + case KERNEL_CLOCK_PROF: + if (!GetProcessTimes(GetCurrentProcess(), &create_time, &exit_time, &kernel_time, + &user_time)) { + *error = KERNEL_ERROR_EFAULT; + return true; + } + Kernel100nsToTimespec(KernelFiletimeTo100ns(kernel_time), tp); + return true; + default: return false; + } +} +#else +static bool kernel_clock_gettime_special(KernelClockid clock_id, KernelTimespec* tp, int* error) { + EXIT_IF(tp == nullptr); + EXIT_IF(error == nullptr); + + if (clock_id == KERNEL_CLOCK_PROCTIME) { + kernel_us_to_timespec(kernel_get_process_time_us_native(), tp); + return true; + } + + return false; +} +#endif + +struct PthreadMutexPrivate { + uint8_t reserved[256]; + std::string name; + std::mutex m; + std::condition_variable cv; + Pthread owner = nullptr; + uint32_t count = 0; + int type = 1; + int pprotocol = PTHREAD_PRIO_NONE; +}; + +struct PthreadMutexattrPrivate { + uint8_t reserved[64]; + pthread_mutexattr_t p; + int pprotocol; + int type; +}; + +struct PthreadAttrPrivate { + uint8_t reserved[64]; + KernelCpumask affinity; + size_t guard_size; + void* stack_addr; + size_t stack_size; + bool stack_user; + uint64_t stack_map_addr; + size_t stack_map_size; + int policy; + int inherit_sched; + int solosched; + bool detached; + pthread_attr_t p; +}; + +struct PthreadCondPrivate; + +struct PthreadGuestData { + int32_t thread_id; + uint8_t reserved[4092]; +}; + +static_assert(sizeof(PthreadGuestData) == 4096); + +struct PthreadPrivate { + PthreadGuestData guest; + std::string name; + pthread_t p; + PthreadAttr attr; + pthread_entry_func_t entry; + void* arg; + int unique_id; + std::atomic_bool detached; + std::atomic_bool almost_done; + std::atomic_bool free; + uint64_t host_thread_id; + uintptr_t guest_host_rbx; + uintptr_t guest_host_rsp; + uintptr_t guest_host_rbp; + uint64_t cond_sequence = 0; + PthreadCondPrivate* waiting_cond = nullptr; + std::atomic pending_signal_mask {0}; +#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS + uintptr_t guest_host_gs8; + uintptr_t guest_host_gs10; +#endif +}; + +static PthreadAttrPrivate* GetPthreadAttrValue(const PthreadAttr* attr, const char* func_name) { + if (attr == nullptr) { + return nullptr; + } + + auto* attr_value = *attr; + if (attr_value == nullptr || reinterpret_cast(attr_value) < 0x10000u) { + (void)func_name; + return nullptr; + } + + return attr_value; +} + +struct PthreadRwlockPrivate { + struct Reader { + Pthread thread = nullptr; + uint32_t count = 0; + }; + + uint8_t reserved[256]; + std::string name; + std::mutex m; + std::condition_variable cv; + Pthread writer = nullptr; + uint32_t writer_count = 0; + uint32_t reader_count = 0; + uint32_t waiting_writers = 0; + std::vector readers; +}; + +struct PthreadRwlockattrPrivate { + uint8_t reserved[64]; + int type; + pthread_rwlockattr_t p; +}; + +struct PthreadCondattrPrivate { + uint8_t reserved[64]; + pthread_condattr_t p; + KernelClockid clock_id = KERNEL_CLOCK_REALTIME; +}; + +struct PthreadCondPrivate { + uint8_t reserved[256]; + std::string name; + std::mutex m; + std::condition_variable cv; + uint64_t sequence = 0; + KernelClockid clock_id = KERNEL_CLOCK_REALTIME; + std::vector waiters; +}; + +static void CondAddWaiter(PthreadCondPrivate* cond, Pthread thread) { + EXIT_IF(cond == nullptr); + EXIT_IF(thread == nullptr); + + thread->waiting_cond = cond; + cond->waiters.push_back(thread); +} + +static bool CondRemoveWaiter(PthreadCondPrivate* cond, Pthread thread) { + EXIT_IF(cond == nullptr); + EXIT_IF(thread == nullptr); + + auto it = std::find(cond->waiters.begin(), cond->waiters.end(), thread); + if (it == cond->waiters.end()) { + return false; + } + + cond->waiters.erase(it); + if (thread->waiting_cond == cond) { + thread->waiting_cond = nullptr; + } + return true; +} + +static bool CondWakeWaiter(PthreadCondPrivate* cond, Pthread thread) { + EXIT_IF(cond == nullptr); + + if (thread == nullptr) { + if (cond->waiters.empty()) { + return false; + } + thread = cond->waiters.front(); + } + + auto it = std::find(cond->waiters.begin(), cond->waiters.end(), thread); + if (it == cond->waiters.end()) { + return false; + } + + cond->waiters.erase(it); + if (thread->waiting_cond == cond) { + thread->waiting_cond = nullptr; + } + thread->cond_sequence++; + return true; +} + +static void CondClearWaiters(PthreadCondPrivate* cond) { + EXIT_IF(cond == nullptr); + + for (auto* thread: cond->waiters) { + if (thread != nullptr && thread->waiting_cond == cond) { + thread->waiting_cond = nullptr; + } + } + cond->waiters.clear(); +} + +void PthreadWakeForSignal(Pthread thread) { + if (thread == nullptr) { + return; + } + + auto* cond = thread->waiting_cond; + if (cond == nullptr) { + return; + } + + bool notify = false; + { + std::lock_guard lock(cond->m); + notify = (thread->waiting_cond == cond); + } + if (notify) { + cond->cv.notify_all(); + } +} + +void KernelDispatchPendingSignalForCurrentThread(); + +static void SchedulerBackoffOnce() { +#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS + if (SwitchToThread() == 0) { + Sleep(0); + } +#else + std::this_thread::yield(); +#endif +} + +static bool SleepMicroSchedulerBackoff(uint64_t microseconds) { + if (microseconds > 1) { + return false; + } + + SchedulerBackoffOnce(); + return true; +} + +static void SleepMicroWithSignalPoll(uint64_t microseconds) { + if (microseconds == 0) { + KernelDispatchPendingSignalForCurrentThread(); + return; + } + + while (microseconds > 0) { + const auto step = std::min(microseconds, SIGNAL_APC_POLL_MICROS); + if (!SleepMicroSchedulerBackoff(step)) { + Common::Thread::SleepMicro(step); + } + microseconds -= step; + KernelDispatchPendingSignalForCurrentThread(); + } +} + +static void SleepNanoWithSignalPoll(uint64_t nanoseconds) { + if (nanoseconds == 0) { + KernelDispatchPendingSignalForCurrentThread(); + return; + } + + constexpr uint64_t poll_nanos = static_cast(SIGNAL_APC_POLL_MICROS) * 1000ull; + while (nanoseconds > 0) { + const auto step = std::min(nanoseconds, poll_nanos); + Common::Thread::SleepNano(step); + nanoseconds -= step; + KernelDispatchPendingSignalForCurrentThread(); + } +} + +struct PthreadStaticObject { + enum class Type { Mutex, Cond, Rwlock }; + + Type type; + uint64_t vaddr; + Loader::Program* program; +}; + +class PthreadStaticObjects { +public: + PthreadStaticObjects() { EXIT_NOT_IMPLEMENTED(!Common::Thread::IsMainThread()); } + virtual ~PthreadStaticObjects() { KYTY_NOT_IMPLEMENTED; } + + KYTY_CLASS_NO_COPY(PthreadStaticObjects); + + void* CreateObject(void* addr, PthreadStaticObject::Type type); + void DeleteObjects(Loader::Program* program); + +private: + std::vector m_objects; + Common::Mutex m_mutex; +}; + +class PthreadKeys { +public: + PthreadKeys() { EXIT_NOT_IMPLEMENTED(!Common::Thread::IsMainThread()); } + virtual ~PthreadKeys() { KYTY_NOT_IMPLEMENTED; } + + KYTY_CLASS_NO_COPY(PthreadKeys); + + bool Create(int* key, pthread_key_destructor_func_t destructor); + bool Delete(int key); + void Destruct(int thread_id); + bool Set(int key, int thread_id, void* data); + bool Get(int key, int thread_id, void** data); + +private: + struct Map { + int thread_id = -1; + void* data = nullptr; + }; + + struct Key { + bool used = false; + pthread_key_destructor_func_t destructor = nullptr; + std::vector specific_values; + }; + + Common::Mutex m_mutex; + Key m_keys[KEYS_MAX]; +}; + +class PthreadPool { +public: + PthreadPool() { EXIT_NOT_IMPLEMENTED(!Common::Thread::IsMainThread()); } + virtual ~PthreadPool() { KYTY_NOT_IMPLEMENTED; } + + KYTY_CLASS_NO_COPY(PthreadPool); + + Pthread Create(); + + void FreeDetachedThreads(); + +private: + std::vector m_threads; + Common::Mutex m_mutex; +}; + +class PThreadContext { +public: + PThreadContext() { EXIT_NOT_IMPLEMENTED(!Common::Thread::IsMainThread()); } + virtual ~PThreadContext() { KYTY_NOT_IMPLEMENTED; } + + KYTY_CLASS_NO_COPY(PThreadContext); + + PthreadAttr* GetDefaultAttr() { return &m_default_attr; } + void SetDefaultAttr(PthreadAttr attr) { m_default_attr = attr; } + PthreadCondattr* GetDefaultCondattr() { return &m_default_condattr; } + void SetDefaultCondattr(PthreadCondattr attr) { m_default_condattr = attr; } + PthreadMutexattr* GetDefaultMutexattr() { return &m_default_mutexattr; } + void SetDefaultMutexattr(PthreadMutexattr attr) { m_default_mutexattr = attr; } + PthreadRwlockattr* GetDefaultRwlockattr() { return &m_default_rwlockattr; } + void SetDefaultRwlockattr(PthreadRwlockattr attr) { m_default_rwlockattr = attr; } + PthreadPool* GetPthreadPool() { return m_pthread_pool; } + void SetPthreadPool(PthreadPool* pool) { m_pthread_pool = pool; } + PthreadStaticObjects* GetPthreadStaticObjects() { return m_pthread_static_objects; } + void SetPthreadStaticObjects(PthreadStaticObjects* objs) { m_pthread_static_objects = objs; } + PthreadKeys* GetPthreadKeys() { return m_pthread_keys; } + void SetPthreadKeys(PthreadKeys* keys) { m_pthread_keys = keys; } + + [[nodiscard]] thread_dtors_func_t GetThreadDtors() const { return m_thread_dtors; } + void SetThreadDtors(thread_dtors_func_t dtors) { m_thread_dtors = dtors; } + +private: + // Common::Mutex m_mutex; + PthreadMutexattr m_default_mutexattr = nullptr; + PthreadRwlockattr m_default_rwlockattr = nullptr; + PthreadCondattr m_default_condattr = nullptr; + PthreadAttr m_default_attr = nullptr; + PthreadPool* m_pthread_pool = nullptr; + PthreadStaticObjects* m_pthread_static_objects = nullptr; + PthreadKeys* m_pthread_keys = nullptr; + + std::atomic m_thread_dtors = nullptr; +}; + +thread_local Pthread g_pthread_self = nullptr; +static Pthread g_pthread_main = nullptr; +PThreadContext* g_pthread_context = nullptr; +thread_local uintptr_t g_guest_entry_return_rsp = 0; +static std::atomic g_pthread_thread_id = 0; + +static Common::Mutex g_guest_stack_mutex; +static uint64_t g_guest_stack_last = 0; + +static size_t RoundStackSize(size_t size) { + return ((size + PTHREAD_STACK_PAGE - 1) / PTHREAD_STACK_PAGE) * PTHREAD_STACK_PAGE; +} + +static size_t RoundStackMappingSize(size_t size) { + return ((size + PTHREAD_STACK_GRANULARITY - 1) / PTHREAD_STACK_GRANULARITY) * + PTHREAD_STACK_GRANULARITY; +} + +static int CreateGuestStack(PthreadAttr attr) { + if (attr == nullptr) { + return KERNEL_ERROR_EINVAL; + } + + if (attr->stack_addr != nullptr) { + attr->guard_size = 0; + attr->stack_user = true; + attr->stack_map_addr = 0; + attr->stack_map_size = 0; + return OK; + } + + const auto stack_size = RoundStackSize(attr->stack_size); + const auto guard_size = RoundStackSize(attr->guard_size); + const auto map_size = RoundStackMappingSize(stack_size + guard_size); + + uint64_t stack_addr = 0; + { + Common::LockGuard lock(g_guest_stack_mutex); + + if (g_guest_stack_last == 0) { + g_guest_stack_last = (PTHREAD_STACK_TOP - PTHREAD_STACK_INITIAL - PTHREAD_STACK_PAGE) & + ~(static_cast(PTHREAD_STACK_GRANULARITY) - 1); + } + + stack_addr = g_guest_stack_last - map_size; + g_guest_stack_last -= map_size; + } + + void* mapped_addr = reinterpret_cast(stack_addr); + + constexpr int PROT_READ_WRITE = 0x03; + constexpr int MAP_PRIVATE = 0x02; + constexpr int MAP_FIXED = 0x10; + constexpr int MAP_STACK = 0x400; + constexpr int MAP_ANON = 0x1000; + + int result = Memory::KernelMapNamedFlexibleMemory( + &mapped_addr, map_size, PROT_READ_WRITE, MAP_PRIVATE | MAP_FIXED | MAP_STACK | MAP_ANON, + "stack"); + if (result != OK) { + return KERNEL_ERROR_EAGAIN; + } + + if (guard_size != 0) { + result = Memory::KernelMprotect(reinterpret_cast(stack_addr), guard_size, 0); + if (result != OK) { + Memory::KernelMunmap(stack_addr, map_size); + return KERNEL_ERROR_EAGAIN; + } + } + + attr->stack_addr = reinterpret_cast(stack_addr + guard_size); + attr->stack_size = map_size - guard_size; + attr->stack_user = false; + attr->stack_map_addr = stack_addr; + attr->stack_map_size = map_size; + + std::memset(attr->stack_addr, 0, stack_size); + + return OK; +} + +static void FreeGuestStack(PthreadAttr attr) { + if (attr == nullptr || attr->stack_user || attr->stack_map_addr == 0 || + attr->stack_map_size == 0) { + return; + } + + Memory::KernelMunmap(attr->stack_map_addr, attr->stack_map_size); + + attr->stack_addr = nullptr; + attr->stack_map_addr = 0; + attr->stack_map_size = 0; +} + +static KYTY_SYSV_ABI void* RunOnGuestStack(void* arg, pthread_entry_func_t func, void* stack_top) { +#if defined(__x86_64__) || defined(_M_X64) + void* ret = nullptr; + const auto guest_rsp = reinterpret_cast(stack_top) & ~static_cast(0x0f); + const auto guest_rbp = guest_rsp - 4u * sizeof(uint64_t); + + auto* guest_root_frame = reinterpret_cast(guest_rbp); + guest_root_frame[0] = 0; + guest_root_frame[1] = 0; + + g_guest_entry_return_rsp = guest_rsp - sizeof(uint64_t); + + uintptr_t host_rsp = 0; + uintptr_t host_rbp = 0; + uintptr_t host_rbx = 0; + asm volatile("movq %%rsp, %0\n\t" + "movq %%rbp, %1\n\t" + "movq %%rbx, %2\n\t" + : "=r"(host_rsp), "=r"(host_rbp), "=r"(host_rbx) + : + : "memory"); + + if (g_pthread_self != nullptr) { + g_pthread_self->guest_host_rbx = host_rbx; + g_pthread_self->guest_host_rsp = host_rsp - (2u * sizeof(uint64_t)); + g_pthread_self->guest_host_rbp = host_rbp; +#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS + uintptr_t host_gs8 = 0; + uintptr_t host_gs10 = 0; + asm volatile("movq %%gs:0x08, %0\n\t" + "movq %%gs:0x10, %1\n\t" + : "=r"(host_gs8), "=r"(host_gs10) + : + : "memory"); + g_pthread_self->guest_host_rsp -= 2u * sizeof(uint64_t); + g_pthread_self->guest_host_gs8 = host_gs8; + g_pthread_self->guest_host_gs10 = host_gs10; +#endif + } + + // The guest ABI expects the entry argument in rdi and a 16-byte aligned stack before call. + asm volatile("pushq %%r12\n\t" + "pushq %%r13\n\t" +#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS + "pushq %%r14\n\t" + "pushq %%r15\n\t" + "movq %%gs:0x08, %%r14\n\t" + "movq %%gs:0x10, %%r15\n\t" + "xorq %%rcx, %%rcx\n\t" + "movq %%rcx, %%gs:0x08\n\t" + "movq %%rcx, %%gs:0x10\n\t" +#endif + "movq %%rsp, %%r12\n\t" + "movq %%rbp, %%r13\n\t" + "movq %[guest_rsp], %%rsp\n\t" + "movq %[guest_rbp], %%rbp\n\t" + "callq *%%rsi\n\t" + "movq %%r13, %%rbp\n\t" + "movq %%r12, %%rsp\n\t" +#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS + "movq %%r14, %%gs:0x08\n\t" + "movq %%r15, %%gs:0x10\n\t" + "popq %%r15\n\t" + "popq %%r14\n\t" +#endif + "popq %%r13\n\t" + "popq %%r12\n\t" + : "=a"(ret), "+D"(arg), "+S"(func) + : [guest_rsp] "r"(guest_rsp), [guest_rbp] "r"(guest_rbp) + : "cc", "memory", "rcx", "rdx", "r8", "r9", "r10", "r11", "xmm0", "xmm1", "xmm2", + "xmm3", "xmm4", "xmm5", "xmm6", "xmm7", "xmm8", "xmm9", "xmm10", "xmm11", + "xmm12", "xmm13", "xmm14", "xmm15"); + + g_guest_entry_return_rsp = 0; + if (g_pthread_self != nullptr) { + g_pthread_self->guest_host_rbx = 0; + g_pthread_self->guest_host_rsp = 0; + g_pthread_self->guest_host_rbp = 0; +#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS + g_pthread_self->guest_host_gs8 = 0; + g_pthread_self->guest_host_gs10 = 0; +#endif + } + + return ret; +#else + (void)stack_top; + return func(arg); +#endif +} + +static void UpdateCurrentThreadStackAttr(PthreadAttr* attr) { + if (attr == nullptr || *attr == nullptr) { + return; + } + +#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS + ULONG_PTR low = 0; + ULONG_PTR high = 0; + GetCurrentThreadStackLimits(&low, &high); + + if (low != 0 && high > low) { + (*attr)->stack_addr = reinterpret_cast(low); + (*attr)->stack_size = high - low; + (*attr)->stack_user = true; + } +#endif +} + +static void FreeDetachedThreads(void* /*arg*/) { + PRINT_NAME_ENABLE(false); + + auto* pthread_pool = g_pthread_context->GetPthreadPool(); + + while (true) { + Common::Thread::Sleep(10000); + pthread_pool->FreeDetachedThreads(); + } +} + +void PthreadDeleteStaticObjects(Loader::Program* program) { + + auto* pthread_static_objects = g_pthread_context->GetPthreadStaticObjects(); + + pthread_static_objects->DeleteObjects(program); +} + +void PthreadInitSelfForMainThread() { + EXIT_IF(g_pthread_self != nullptr); + + g_pthread_self = new PthreadPrivate {}; + PthreadAttrInit(&g_pthread_self->attr); + UpdateCurrentThreadStackAttr(&g_pthread_self->attr); + g_pthread_self->p = pthread_self(); + g_pthread_self->name = "MainThread"; + g_pthread_self->guest.thread_id = ++g_pthread_thread_id; + g_pthread_self->unique_id = Common::Thread::GetThreadIdUnique(); + g_pthread_self->free = false; + g_pthread_self->detached = false; + g_pthread_self->almost_done = false; + g_pthread_self->entry = nullptr; + g_pthread_self->arg = nullptr; + + uint64_t os_thread_id = 0; +#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS + os_thread_id = static_cast(GetCurrentThreadId()); +#endif + g_pthread_self->host_thread_id = os_thread_id; + g_pthread_main = g_pthread_self; + + LOGF("\tPthread main self: id = %d, os_thread_id = %" PRIu64 ", stack_addr = 0x%016" PRIx64 + ", stack_size = %" PRIu64 "\n", + g_pthread_self->unique_id, os_thread_id, + reinterpret_cast(g_pthread_self->attr->stack_addr), + static_cast(g_pthread_self->attr->stack_size)); +} + +void* PthreadCreateMainGuestStack() { + EXIT_IF(g_pthread_self == nullptr); + EXIT_IF(g_pthread_self->attr == nullptr); + + if (g_pthread_self->attr->stack_map_addr != 0) { + return reinterpret_cast( + (reinterpret_cast(g_pthread_self->attr->stack_addr) + + g_pthread_self->attr->stack_size) & + ~static_cast(0x0f)); + } + + g_pthread_self->attr->stack_addr = nullptr; + g_pthread_self->attr->stack_size = PTHREAD_STACK_DEFAULT + PTHREAD_STACK_EXTRA; + g_pthread_self->attr->stack_user = false; + g_pthread_self->attr->stack_map_addr = 0; + g_pthread_self->attr->stack_map_size = 0; + + const auto result = CreateGuestStack(g_pthread_self->attr); + EXIT_NOT_IMPLEMENTED(result != OK); + + auto* stack_top = + reinterpret_cast((reinterpret_cast(g_pthread_self->attr->stack_addr) + + g_pthread_self->attr->stack_size) & + ~static_cast(0x0f)); + + LOGF("\tPthread main guest stack: stack_addr = 0x%016" PRIx64 ", stack_size = %" PRIu64 + ", stack_top = 0x%016" PRIx64 "\n", + reinterpret_cast(g_pthread_self->attr->stack_addr), + static_cast(g_pthread_self->attr->stack_size), + reinterpret_cast(stack_top)); + + return stack_top; +} + +KYTY_SUBSYSTEM_INIT(Pthread) { + PRINT_NAME_ENABLE(false); + + EXIT_IF(g_pthread_context != nullptr); + + g_pthread_context = new PThreadContext; + + g_pthread_context->SetPthreadStaticObjects(new PthreadStaticObjects); + g_pthread_context->SetPthreadPool(new PthreadPool); + g_pthread_context->SetPthreadKeys(new PthreadKeys); + Common::CondVar::SetWaitPollCallback(KernelDispatchPendingSignalForCurrentThread); + + PthreadMutexattr default_mutexattr = nullptr; + PthreadRwlockattr default_rwlockattr = nullptr; + PthreadCondattr default_condattr = nullptr; + PthreadAttr default_attr = nullptr; + + PthreadAttrInit(&default_attr); + PthreadMutexattrInit(&default_mutexattr); + PthreadRwlockattrInit(&default_rwlockattr); + PthreadCondattrInit(&default_condattr); + + g_pthread_context->SetDefaultMutexattr(default_mutexattr); + g_pthread_context->SetDefaultRwlockattr(default_rwlockattr); + g_pthread_context->SetDefaultCondattr(default_condattr); + g_pthread_context->SetDefaultAttr(default_attr); + + PRINT_NAME_ENABLE(true); + + Common::Thread thread(FreeDetachedThreads, nullptr); + thread.Detach(); +} + +KYTY_SUBSYSTEM_UNEXPECTED_SHUTDOWN(Pthread) {} + +KYTY_SUBSYSTEM_DESTROY(Pthread) {} + +static int PthreadAttrCopy(PthreadAttr* dst, const PthreadAttr* src) { + if (dst == nullptr || *dst == nullptr || src == nullptr || *src == nullptr) { + return KERNEL_ERROR_EINVAL; + } + + KernelCpumask mask = 0; + int state = 0; + size_t guard_size = 0; + int inherit_sched = 0; + KernelSchedParam param = {}; + int policy = 0; + int solosched = 0; + void* stack_addr = nullptr; + size_t stack_size = 0; + + int result = 0; + + result = (result == 0 ? PthreadAttrGetaffinity(src, &mask) : result); + result = (result == 0 ? PthreadAttrGetdetachstate(src, &state) : result); + result = (result == 0 ? PthreadAttrGetguardsize(src, &guard_size) : result); + result = (result == 0 ? PthreadAttrGetinheritsched(src, &inherit_sched) : result); + result = (result == 0 ? PthreadAttrGetschedparam(src, ¶m) : result); + result = (result == 0 ? PthreadAttrGetschedpolicy(src, &policy) : result); + result = (result == 0 ? PthreadAttrGetsolosched(src, &solosched) : result); + result = (result == 0 ? PthreadAttrGetstackaddr(src, &stack_addr) : result); + result = (result == 0 ? PthreadAttrGetstacksize(src, &stack_size) : result); + + result = (result == 0 ? PthreadAttrSetaffinity(dst, mask) : result); + result = (result == 0 ? PthreadAttrSetdetachstate(dst, state) : result); + result = (result == 0 ? PthreadAttrSetguardsize(dst, guard_size) : result); + result = (result == 0 ? PthreadAttrSetinheritsched(dst, inherit_sched) : result); + result = (result == 0 ? PthreadAttrSetschedparam(dst, ¶m) : result); + result = (result == 0 ? PthreadAttrSetschedpolicy(dst, policy) : result); + result = (result == 0 ? PthreadAttrSetsolosched(dst, solosched) : result); + if (stack_addr != nullptr) { + result = (result == 0 ? PthreadAttrSetstackaddr(dst, stack_addr) : result); + } + if (stack_size != 0) { + result = (result == 0 ? PthreadAttrSetstacksize(dst, stack_size) : result); + } + return result; +} + +static void PthreadAttrDbgPrint(const PthreadAttr* src) { + KernelCpumask mask = 0; + int state = 0; + size_t guard_size = 0; + int inherit_sched = 0; + KernelSchedParam param = {}; + int policy = 0; + int solosched = 0; + void* stack_addr = nullptr; + size_t stack_size = 0; + + PthreadAttrGetaffinity(src, &mask); + PthreadAttrGetdetachstate(src, &state); + PthreadAttrGetguardsize(src, &guard_size); + PthreadAttrGetinheritsched(src, &inherit_sched); + PthreadAttrGetschedparam(src, ¶m); + PthreadAttrGetschedpolicy(src, &policy); + PthreadAttrGetsolosched(src, &solosched); + PthreadAttrGetstackaddr(src, &stack_addr); + PthreadAttrGetstacksize(src, &stack_size); + + LOGF("\tcpu_mask = 0x%" PRIx64 "\n" + "\tdetach_state = %d\n" + "\tguard_size = %" PRIu64 "\n" + "\tinherit_sched = %d\n" + "\tsched_priority = %d\n" + "\tpolicy = %d\n" + "\tsolosched = %d\n" + "\tstack_addr = 0x%016" PRIx64 "\n" + "\tstack_size = %" PRIu64 "\n", + mask, state, guard_size, inherit_sched, param.sched_priority, policy, solosched, + reinterpret_cast(stack_addr), reinterpret_cast(stack_size)); +} + +static constexpr int32_t DST_NONE = 0; +static constexpr int32_t DST_MET = 4; + +static int32_t GetDstSeconds() { +#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS + TIME_ZONE_INFORMATION tzi {}; + const DWORD result = GetTimeZoneInformation(&tzi); + return (result == TIME_ZONE_ID_DAYLIGHT ? -tzi.DaylightBias * 60 : 0); +#else + const std::time_t now = std::time(nullptr); + std::tm local_tm {}; + localtime_r(&now, &local_tm); + return (local_tm.tm_isdst > 0 ? 3600 : 0); +#endif +} + +#if KYTY_PLATFORM != KYTY_PLATFORM_WINDOWS +static void sec_to_timeval(KernelTimeval* ts, double sec) { + ts->tv_sec = static_cast(sec); + ts->tv_usec = static_cast((sec - static_cast(ts->tv_sec)) * 1000000.0); +} +#endif + +static bool GetPosixClockId(KernelClockid clock_id, clockid_t* out) { + EXIT_IF(out == nullptr); + + switch (clock_id) { + case KERNEL_CLOCK_REALTIME: + case KERNEL_CLOCK_REALTIME_PRECISE: + case KERNEL_CLOCK_REALTIME_FAST: + case KERNEL_CLOCK_SECOND: *out = CLOCK_REALTIME; return true; + case KERNEL_CLOCK_MONOTONIC: + case KERNEL_CLOCK_UPTIME: + case KERNEL_CLOCK_UPTIME_PRECISE: + case KERNEL_CLOCK_UPTIME_FAST: + case KERNEL_CLOCK_MONOTONIC_PRECISE: + case KERNEL_CLOCK_MONOTONIC_FAST: + case KERNEL_CLOCK_EXT_NETWORK: + case KERNEL_CLOCK_EXT_DEBUG_NETWORK: + case KERNEL_CLOCK_EXT_AD_NETWORK: + case KERNEL_CLOCK_EXT_RAW_NETWORK: *out = CLOCK_MONOTONIC; return true; + default: return false; + } +} + +static bool NativeClockGettime(KernelClockid clock_id, KernelTimespec* tp) { + EXIT_IF(tp == nullptr); + + int error = OK; + if (KernelClockGettimeSpecial(clock_id, tp, &error)) { + return error == OK; + } + + clockid_t native_clock_id {}; + if (!GetPosixClockId(clock_id, &native_clock_id)) { + return false; + } + + timespec native_time {}; + if (::clock_gettime(native_clock_id, &native_time) != 0) { + return false; + } + + tp->tv_sec = native_time.tv_sec; + tp->tv_nsec = native_time.tv_nsec; + return true; +} + +static int PthreadMutexInitNamed(PthreadMutex* mutex, const PthreadMutexattr* attr, + const char* name, int static_type = 0); +static int PthreadRwlockInitNamed(PthreadRwlock* rwlock, const PthreadRwlockattr* attr, + const char* name); +static int PthreadCondInitNamed(PthreadCond* cond, const PthreadCondattr* attr, const char* name); + +static int NativeMutexLock(PthreadMutexPrivate* mutex, KernelUseconds* timeout_us) { + EXIT_IF(mutex == nullptr); + + auto* self = g_pthread_self; + EXIT_NOT_IMPLEMENTED(self == nullptr); + + std::unique_lock lock(mutex->m); + + if (mutex->owner == self) { + if (mutex->type == 2) { + if (mutex->count == UINT32_MAX) { + return EAGAIN; + } + mutex->count++; + return OK; + } + if (timeout_us != nullptr) { + if (*timeout_us > 0) { + mutex->cv.wait_for(lock, std::chrono::microseconds(*timeout_us)); + } + return ETIMEDOUT; + } + return EDEADLK; + } + + if (timeout_us == nullptr) { + while (mutex->owner != nullptr) { + mutex->cv.wait_for(lock, std::chrono::microseconds(SIGNAL_APC_POLL_MICROS)); + if (mutex->owner != nullptr) { + lock.unlock(); + KernelDispatchPendingSignalForCurrentThread(); + lock.lock(); + } + } + } else if (*timeout_us == 0) { + if (mutex->owner != nullptr) { + return ETIMEDOUT; + } + } else { + const auto deadline = + std::chrono::steady_clock::now() + std::chrono::microseconds(*timeout_us); + while (mutex->owner != nullptr) { + const auto now = std::chrono::steady_clock::now(); + if (now >= deadline) { + return ETIMEDOUT; + } + + const auto remaining = deadline - now; + const auto poll = (remaining < std::chrono::microseconds(SIGNAL_APC_POLL_MICROS) + ? remaining + : std::chrono::steady_clock::duration( + std::chrono::microseconds(SIGNAL_APC_POLL_MICROS))); + mutex->cv.wait_for(lock, poll); + if (mutex->owner != nullptr) { + lock.unlock(); + KernelDispatchPendingSignalForCurrentThread(); + lock.lock(); + } + } + } + + mutex->owner = self; + mutex->count = 1; + return OK; +} + +static int NativeMutexTrylock(PthreadMutexPrivate* mutex) { + EXIT_IF(mutex == nullptr); + + auto* self = g_pthread_self; + EXIT_NOT_IMPLEMENTED(self == nullptr); + + std::unique_lock lock(mutex->m); + + if (mutex->owner == self) { + if (mutex->type == 2) { + if (mutex->count == UINT32_MAX) { + return EAGAIN; + } + mutex->count++; + return OK; + } + return EBUSY; + } + + if (mutex->owner != nullptr) { + return EBUSY; + } + + mutex->owner = self; + mutex->count = 1; + return OK; +} + +static int NativeMutexUnlock(PthreadMutexPrivate* mutex, uint32_t* recurse = nullptr) { + EXIT_IF(mutex == nullptr); + + auto* self = g_pthread_self; + EXIT_NOT_IMPLEMENTED(self == nullptr); + + std::unique_lock lock(mutex->m); + + if (mutex->owner != self) { + return EPERM; + } + + if (recurse != nullptr) { + *recurse = mutex->count; + mutex->count = 1; + } + + if (mutex->type == 2 && mutex->count > 1) { + mutex->count--; + return OK; + } + + mutex->owner = nullptr; + mutex->count = 0; + lock.unlock(); + mutex->cv.notify_one(); + + return OK; +} + +static int NativeMutexLockRecurse(PthreadMutexPrivate* mutex, uint32_t recurse) { + EXIT_IF(mutex == nullptr); + + int result = NativeMutexLock(mutex, nullptr); + if (result == OK) { + std::lock_guard lock(mutex->m); + mutex->count = std::max(recurse, 1); + } + return result; +} + +static bool NativeCondDeadlineFromAbs(KernelClockid clock_id, const KernelTimespec* abstime, + std::chrono::steady_clock::time_point* out) { + if (abstime == nullptr || out == nullptr || abstime->tv_sec < 0 || abstime->tv_nsec < 0 || + abstime->tv_nsec >= 1000000000) { + return false; + } + + KernelTimespec now {}; + if (!NativeClockGettime(clock_id, &now)) { + return false; + } + + auto delta = std::chrono::seconds(abstime->tv_sec - now.tv_sec) + + std::chrono::nanoseconds(abstime->tv_nsec - now.tv_nsec); + *out = std::chrono::steady_clock::now() + + std::chrono::duration_cast(delta); + return true; +} + +void* PthreadStaticObjects::CreateObject(void* addr, PthreadStaticObject::Type type) { + if (addr == nullptr) { + return addr; + } + + auto* current = static_cast(addr); + auto current_ref = std::atomic_ref(*current); + auto* current_value = current_ref.load(std::memory_order_acquire); + const bool adaptive_static_mutex = + (type == PthreadStaticObject::Type::Mutex && + current_value == reinterpret_cast(static_cast(1))); + + if (current_value != nullptr && !adaptive_static_mutex) { + return addr; + } + + Common::LockGuard lock(m_mutex); + + current_value = current_ref.load(std::memory_order_acquire); + const bool adaptive_static_mutex_after_lock = + (type == PthreadStaticObject::Type::Mutex && + current_value == reinterpret_cast(static_cast(1))); + + if (current_value != nullptr && !adaptive_static_mutex_after_lock) { + return addr; + } + + auto* rt = Common::Singleton::Instance(); + auto vaddr = reinterpret_cast(addr); + auto* program = rt->FindProgramByAddr(vaddr); + + EXIT_NOT_IMPLEMENTED(program == nullptr); + + auto* obj = new PthreadStaticObject; + obj->program = program; + obj->type = type; + obj->vaddr = vaddr; + + std::string name = fmt::format("Static{:016x}", vaddr); + + int result = OK; + switch (type) { + case PthreadStaticObject::Type::Mutex: + result = PthreadMutexInitNamed(static_cast(addr), nullptr, name.c_str(), + adaptive_static_mutex_after_lock ? 4 : 0); + break; + case PthreadStaticObject::Type::Cond: + result = PthreadCondInitNamed(static_cast(addr), nullptr, name.c_str()); + break; + case PthreadStaticObject::Type::Rwlock: + result = + PthreadRwlockInitNamed(static_cast(addr), nullptr, name.c_str()); + break; + default: EXIT("unknown type: %d\n", static_cast(type)); + } + + EXIT_NOT_IMPLEMENTED(result != OK); + + const auto it = std::find(m_objects.begin(), m_objects.end(), nullptr); + if (it != m_objects.end()) { + *it = obj; + } else { + m_objects.push_back(obj); + } + + return addr; +} + +void PthreadStaticObjects::DeleteObjects(Loader::Program* program) { + Common::LockGuard lock(m_mutex); + + for (auto& obj: m_objects) { + if (obj != nullptr && obj->program == program) { + int result = OK; + switch (obj->type) { + case PthreadStaticObject::Type::Mutex: + result = PthreadMutexDestroy(reinterpret_cast(obj->vaddr)); + break; + case PthreadStaticObject::Type::Cond: + result = PthreadCondDestroy(reinterpret_cast(obj->vaddr)); + break; + case PthreadStaticObject::Type::Rwlock: + result = PthreadRwlockDestroy(reinterpret_cast(obj->vaddr)); + break; + default: EXIT("unknown type: %d\n", static_cast(obj->type)); + } + + EXIT_NOT_IMPLEMENTED(result != OK); + + delete obj; + obj = nullptr; + } + } +} + +Pthread PthreadPool::Create() { + Common::LockGuard lock(m_mutex); + + for (auto* p: m_threads) { + if (p->free) { + p->free = false; + return p; + } + } + + auto* ret = new PthreadPrivate {}; + + ret->free = false; + ret->detached = false; + ret->almost_done = false; + ret->attr = nullptr; + + m_threads.push_back(ret); + + return ret; +} + +void PthreadPool::FreeDetachedThreads() { + Common::LockGuard lock(m_mutex); + + for (auto* p: m_threads) { + if (p->detached && p->almost_done && !p->free) { + PthreadJoin(p, nullptr); + } + } +} + +bool PthreadKeys::Create(int* key, pthread_key_destructor_func_t destructor) { + EXIT_IF(key == nullptr); + + Common::LockGuard lock(m_mutex); + + for (int index = 0; index < KEYS_MAX; index++) { + if (!m_keys[index].used) { + *key = index; + m_keys[index].used = true; + m_keys[index].destructor = destructor; + m_keys[index].specific_values.clear(); + return true; + } + } + + return false; +} + +bool PthreadKeys::Delete(int key) { + Common::LockGuard lock(m_mutex); + + if (key < 0 || key >= KEYS_MAX || !m_keys[key].used) { + return false; + } + + m_keys[key].used = false; + m_keys[key].destructor = nullptr; + m_keys[key].specific_values.clear(); + + return true; +} + +void PthreadKeys::Destruct(int thread_id) { + struct CallInfo { + pthread_key_destructor_func_t destructor; + void* data; + }; + + for (int iter = 0; iter < DESTRUCTOR_ITERATIONS; iter++) { + std::vector delete_list; + + { + Common::LockGuard lock(m_mutex); + + for (auto& key: m_keys) { + if (key.used && key.destructor != nullptr) { + for (auto& v: key.specific_values) { + if (v.thread_id == thread_id && v.data != nullptr) { + delete_list.push_back(CallInfo {key.destructor, v.data}); + v.data = nullptr; + } + } + } + } + } + + if (delete_list.empty()) { + return; + } + + for (auto& d: delete_list) { + d.destructor(d.data); + } + } + + Common::LockGuard lock(m_mutex); + + for (auto& key: m_keys) { + auto& values = key.specific_values; + values.erase(std::remove_if(values.begin(), values.end(), + [thread_id](const Map& v) { return v.thread_id == thread_id; }), + values.end()); + } +} + +bool PthreadKeys::Set(int key, int thread_id, void* data) { + Common::LockGuard lock(m_mutex); + + if (key < 0 || key >= KEYS_MAX || !m_keys[key].used) { + return false; + } + + for (auto& v: m_keys[key].specific_values) { + if (v.thread_id == thread_id) { + v.data = data; + return true; + } + } + + m_keys[key].specific_values.push_back(Map({thread_id, data})); + + return true; +} + +bool PthreadKeys::Get(int key, int thread_id, void** data) { + EXIT_IF(data == nullptr); + + Common::LockGuard lock(m_mutex); + + if (key < 0 || key >= KEYS_MAX || !m_keys[key].used) { + return false; + } + + for (auto& v: m_keys[key].specific_values) { + if (v.thread_id == thread_id) { + *data = v.data; + return true; + } + } + + *data = nullptr; + + return true; +} + +int KYTY_SYSV_ABI PthreadMutexattrInit(PthreadMutexattr* attr) { + // PRINT_NAME(); + + EXIT_NOT_IMPLEMENTED(attr == nullptr); + + *attr = new PthreadMutexattrPrivate {}; + + int result = pthread_mutexattr_init(&(*attr)->p); + + result = (result == 0 ? PthreadMutexattrSettype(attr, 1) : result); + result = (result == 0 ? PthreadMutexattrSetprotocol(attr, 0) : result); + + switch (result) { + case 0: return OK; + case ENOMEM: return KERNEL_ERROR_ENOMEM; + default: return KERNEL_ERROR_EINVAL; + } +} + +int KYTY_SYSV_ABI PthreadMutexattrDestroy(PthreadMutexattr* attr) { + // PRINT_NAME(); + + EXIT_NOT_IMPLEMENTED(attr == nullptr || *attr == nullptr); + + int result = pthread_mutexattr_destroy(&(*attr)->p); + + delete *attr; + *attr = nullptr; + + switch (result) { + case 0: return OK; + case ENOMEM: return KERNEL_ERROR_ENOMEM; + default: return KERNEL_ERROR_EINVAL; + } +} + +int KYTY_SYSV_ABI PthreadMutexattrSettype(PthreadMutexattr* attr, int type) { + // PRINT_NAME(); + + EXIT_NOT_IMPLEMENTED(attr == nullptr || *attr == nullptr); + + int ptype = PTHREAD_MUTEX_DEFAULT; + switch (type) { + case KERNEL_PTHREAD_MUTEX_ERRORCHECK: ptype = PTHREAD_MUTEX_ERRORCHECK; break; + case KERNEL_PTHREAD_MUTEX_RECURSIVE: ptype = PTHREAD_MUTEX_RECURSIVE; break; + case KERNEL_PTHREAD_MUTEX_NORMAL: + case KERNEL_PTHREAD_MUTEX_ADAPTIVE: ptype = PTHREAD_MUTEX_NORMAL; break; + default: return KERNEL_ERROR_EINVAL; + } + + int result = pthread_mutexattr_settype(&(*attr)->p, ptype); + + if (result == 0) { + (*attr)->type = type; + return OK; + } + + return KERNEL_ERROR_EINVAL; +} + +int KYTY_SYSV_ABI PthreadMutexattrSetprotocol([[maybe_unused]] PthreadMutexattr* attr, + int protocol) { + // PRINT_NAME(); + + EXIT_NOT_IMPLEMENTED(attr == nullptr || *attr == nullptr); + + [[maybe_unused]] int pprotocol = PTHREAD_PRIO_NONE; + switch (protocol) { + case 0: pprotocol = PTHREAD_PRIO_NONE; break; + case 1: pprotocol = PTHREAD_PRIO_INHERIT; break; + case 2: pprotocol = PTHREAD_PRIO_PROTECT; break; + default: EXIT("invalid protocol: %d\n", protocol); + } + + // protocol doesn't work in winpthreads + int result = 0; // pthread_mutexattr_setprotocol(&(*attr)->p, pprotocol); + (*attr)->pprotocol = pprotocol; + + if (result == 0) { + return OK; + } + + return KERNEL_ERROR_EINVAL; +} + +static int PthreadMutexInitNamed(PthreadMutex* mutex, const PthreadMutexattr* attr, + const char* name, int static_type) { + if (name != nullptr && name[0] != '\0') { + PRINT_NAME(); + } + + // EXIT_NOT_IMPLEMENTED(!Common::Thread::IsMainThread()); + if (mutex == nullptr) { + return KERNEL_ERROR_EINVAL; + } + + PthreadMutexattrPrivate static_attr {}; + PthreadMutexattr static_attr_ptr = nullptr; + + if (static_type != 0) { + static_attr_ptr = &static_attr; + pthread_mutexattr_init(&static_attr.p); + PthreadMutexattrSettype(&static_attr_ptr, static_type); + PthreadMutexattrSetprotocol(&static_attr_ptr, 0); + attr = &static_attr_ptr; + } else if (attr == nullptr) { + + attr = g_pthread_context->GetDefaultMutexattr(); + } + + auto* new_mutex = new PthreadMutexPrivate {}; + + new_mutex->name = (name != nullptr ? name : ""); + new_mutex->type = (*attr)->type; + new_mutex->pprotocol = (*attr)->pprotocol; + + int result = 0; + + if (static_attr_ptr != nullptr) { + pthread_mutexattr_destroy(&static_attr.p); + } + + if (name != nullptr && name[0] != '\0') { + LOGF("\tmutex init: %s, %d\n", new_mutex->name.c_str(), result); + } + + std::atomic_ref(*mutex).store(new_mutex, std::memory_order_release); + + switch (result) { + case 0: return OK; + case EAGAIN: return KERNEL_ERROR_EAGAIN; + case EINVAL: return KERNEL_ERROR_EINVAL; + case ENOMEM: return KERNEL_ERROR_ENOMEM; + default: return KERNEL_ERROR_EINVAL; + } +} + +int KYTY_SYSV_ABI PthreadMutexInit(PthreadMutex* mutex, const PthreadMutexattr* attr) { + return PthreadMutexInitNamed(mutex, attr, ""); +} + +int KYTY_SYSV_ABI PthreadMutexDestroy(PthreadMutex* mutex) { + // Hot path for Python/Ren'Py startup; keep this quiet unless it fails. + + if (mutex == nullptr || *mutex == nullptr) { + return KERNEL_ERROR_EINVAL; + } + + int result = ((*mutex)->owner == nullptr ? 0 : EBUSY); + + if (result != 0) { + LOGF("\tmutex destroy: %s, %d\n", (*mutex)->name.c_str(), result); + switch (result) { + case EBUSY: return KERNEL_ERROR_EBUSY; + case EINVAL: + default: return KERNEL_ERROR_EINVAL; + } + } + + delete *mutex; + *mutex = nullptr; + + switch (result) { + case 0: return OK; + case EBUSY: return KERNEL_ERROR_EBUSY; + case EINVAL: + default: return KERNEL_ERROR_EINVAL; + } +} + +int KYTY_SYSV_ABI PthreadMutexLock(PthreadMutex* mutex) { + // PRINT_NAME(); + + auto* pthread_static_objects = g_pthread_context->GetPthreadStaticObjects(); + + mutex = static_cast( + pthread_static_objects->CreateObject(mutex, PthreadStaticObject::Type::Mutex)); + + if (mutex == nullptr) { + return KERNEL_ERROR_EINVAL; + } + + EXIT_NOT_IMPLEMENTED(*mutex == nullptr); + + int result = NativeMutexLock(*mutex, nullptr); + + // LOGF("\tmutex lock: %s, %d\n", (*mutex)->name.c_str(), result); + + switch (result) { + case 0: return OK; + case EAGAIN: return KERNEL_ERROR_EAGAIN; + case EINVAL: return KERNEL_ERROR_EINVAL; + case EDEADLK: return KERNEL_ERROR_EDEADLK; + default: return KERNEL_ERROR_EINVAL; + } +} + +int KYTY_SYSV_ABI PthreadMutexTrylock(PthreadMutex* mutex) { + // PRINT_NAME(); + + auto* pthread_static_objects = g_pthread_context->GetPthreadStaticObjects(); + + mutex = static_cast( + pthread_static_objects->CreateObject(mutex, PthreadStaticObject::Type::Mutex)); + + if (mutex == nullptr) { + return KERNEL_ERROR_EINVAL; + } + + EXIT_NOT_IMPLEMENTED(*mutex == nullptr); + + int result = NativeMutexTrylock(*mutex); + + // LOGF("\tmutex trylock: %s, %d\n", (*mutex)->name.c_str(), result); + + switch (result) { + case 0: return OK; + case EAGAIN: return KERNEL_ERROR_EAGAIN; + case EBUSY: return KERNEL_ERROR_EBUSY; + case EINVAL: + default: return KERNEL_ERROR_EINVAL; + } +} + +int KYTY_SYSV_ABI PthreadMutexTimedlock(PthreadMutex* mutex, KernelUseconds usec) { + // PRINT_NAME(); + + auto* pthread_static_objects = g_pthread_context->GetPthreadStaticObjects(); + + mutex = static_cast( + pthread_static_objects->CreateObject(mutex, PthreadStaticObject::Type::Mutex)); + + if (mutex == nullptr) { + return KERNEL_ERROR_EINVAL; + } + + EXIT_NOT_IMPLEMENTED(*mutex == nullptr); + + int result = NativeMutexLock(*mutex, &usec); + + switch (result) { + case 0: return OK; + case ETIMEDOUT: return KERNEL_ERROR_ETIMEDOUT; + case EAGAIN: return KERNEL_ERROR_EAGAIN; + case EDEADLK: return KERNEL_ERROR_EDEADLK; + case EINVAL: + default: return KERNEL_ERROR_EINVAL; + } +} + +int KYTY_SYSV_ABI PthreadMutexUnlock(PthreadMutex* mutex) { + // PRINT_NAME(); + + auto* pthread_static_objects = g_pthread_context->GetPthreadStaticObjects(); + + mutex = static_cast( + pthread_static_objects->CreateObject(mutex, PthreadStaticObject::Type::Mutex)); + + if (mutex == nullptr) { + return KERNEL_ERROR_EINVAL; + } + + EXIT_NOT_IMPLEMENTED(*mutex == nullptr); + + int result = NativeMutexUnlock(*mutex); + + if (result != 0) { + LOGF("\tmutex unlock: %s, %d, thread_id = %d\n", (*mutex)->name.c_str(), result, + Common::Thread::GetThreadIdUnique()); + } + + switch (result) { + case 0: return OK; + + case EINVAL: return KERNEL_ERROR_EINVAL; + case EPERM: return KERNEL_ERROR_EPERM; + default: return KERNEL_ERROR_EINVAL; + } +} + +int KYTY_SYSV_ABI PthreadAttrInit(PthreadAttr* attr) { + // PRINT_NAME(); + + *attr = new PthreadAttrPrivate {}; + + int result = pthread_attr_init(&(*attr)->p); + + (*attr)->affinity = 0x7f; + (*attr)->guard_size = 0x1000; + (*attr)->stack_addr = nullptr; + (*attr)->stack_size = PTHREAD_STACK_DEFAULT; + (*attr)->stack_user = false; + (*attr)->stack_map_addr = 0; + (*attr)->stack_map_size = 0; + + KernelSchedParam param; + param.sched_priority = 700; + + result = (result == 0 ? PthreadAttrSetinheritsched(attr, 4) : result); + result = (result == 0 ? PthreadAttrSetschedparam(attr, ¶m) : result); + result = (result == 0 ? PthreadAttrSetschedpolicy(attr, 1) : result); + result = (result == 0 ? PthreadAttrSetdetachstate(attr, 0) : result); + result = (result == 0 ? PthreadAttrSetstacksize(attr, PTHREAD_STACK_DEFAULT) : result); + + if (false && PRINT_NAME_ENABLED) { + PthreadAttrDbgPrint(attr); + } + + switch (result) { + case 0: return OK; + case ENOMEM: return KERNEL_ERROR_ENOMEM; + default: return KERNEL_ERROR_EINVAL; + } +} + +int KYTY_SYSV_ABI PthreadAttrDestroy(PthreadAttr* attr) { + // PRINT_NAME(); + + auto* attr_value = GetPthreadAttrValue(attr, "PthreadAttrDestroy"); + if (attr_value == nullptr) { + return KERNEL_ERROR_EINVAL; + } + + int result = pthread_attr_destroy(&attr_value->p); + + delete *attr; + *attr = nullptr; + + if (result == 0) { + return OK; + } + return KERNEL_ERROR_EINVAL; +} + +int KYTY_SYSV_ABI PthreadAttrGet(Pthread thread, PthreadAttr* attr) { + // PRINT_NAME(); + + if (thread == nullptr || attr == nullptr || *attr == nullptr) { + return KERNEL_ERROR_EINVAL; + } + + return PthreadAttrCopy(attr, &thread->attr); +} + +int KYTY_SYSV_ABI PthreadAttrGetaffinity(const PthreadAttr* attr, KernelCpumask* mask) { + // PRINT_NAME(); + + if (mask == nullptr || attr == nullptr || *attr == nullptr) { + return KERNEL_ERROR_EINVAL; + } + + *mask = (*attr)->affinity; + + return OK; +} + +int KYTY_SYSV_ABI PthreadAttrGetdetachstate(const PthreadAttr* attr, int* state) { + // PRINT_NAME(); + + if (state == nullptr || attr == nullptr || *attr == nullptr) { + return KERNEL_ERROR_EINVAL; + } + + // int result = pthread_attr_getdetachstate(&(*attr)->p, state); + int result = 0; + + *state = ((*attr)->detached ? PTHREAD_CREATE_DETACHED : PTHREAD_CREATE_JOINABLE); + + switch (*state) { + case PTHREAD_CREATE_JOINABLE: *state = 0; break; + case PTHREAD_CREATE_DETACHED: *state = 1; break; + default: EXIT("unknown state: %d\n", *state); + } + + if (result == 0) { + return OK; + } + return KERNEL_ERROR_EINVAL; +} + +int KYTY_SYSV_ABI PthreadAttrGetguardsize(const PthreadAttr* attr, size_t* guard_size) { + // PRINT_NAME(); + + if (guard_size == nullptr || attr == nullptr || *attr == nullptr) { + return KERNEL_ERROR_EINVAL; + } + + *guard_size = (*attr)->guard_size; + + return OK; +} + +int KYTY_SYSV_ABI PthreadAttrGetinheritsched(const PthreadAttr* attr, int* inherit_sched) { + // PRINT_NAME(); + + if (inherit_sched == nullptr || attr == nullptr || *attr == nullptr) { + return KERNEL_ERROR_EINVAL; + } + + *inherit_sched = (*attr)->inherit_sched; + + switch (*inherit_sched) { + case 0: + case 4: break; + default: EXIT("unknown inherit_sched: %d\n", *inherit_sched); + } + + return OK; +} + +int KYTY_SYSV_ABI PthreadAttrGetschedparam(const PthreadAttr* attr, KernelSchedParam* param) { + // PRINT_NAME(); + + if (param == nullptr || attr == nullptr || *attr == nullptr) { + return KERNEL_ERROR_EINVAL; + } + + int result = pthread_attr_getschedparam(&(*attr)->p, param); + + if (param->sched_priority <= -2) { + param->sched_priority = 767; + } else if (param->sched_priority >= +2) { + param->sched_priority = 256; + } else { + param->sched_priority = 700; + } + + if (result == 0) { + return OK; + } + return KERNEL_ERROR_EINVAL; +} + +int KYTY_SYSV_ABI PthreadAttrGetschedpolicy(const PthreadAttr* attr, int* policy) { + // PRINT_NAME(); + + if (policy == nullptr || attr == nullptr || *attr == nullptr) { + return KERNEL_ERROR_EINVAL; + } + + int result = pthread_attr_getschedpolicy(&(*attr)->p, policy); + + switch (*policy) { + case SCHED_OTHER: *policy = (*attr)->policy; break; + case SCHED_FIFO: *policy = 1; break; + case SCHED_RR: *policy = 3; break; + default: EXIT("unknown policy: %d\n", *policy); + } + + if (result == 0) { + return OK; + } + return KERNEL_ERROR_EINVAL; +} + +int KYTY_SYSV_ABI PthreadAttrGetsolosched(const PthreadAttr* attr, int* solosched) { + // PRINT_NAME(); + + if (solosched == nullptr || attr == nullptr || *attr == nullptr) { + return KERNEL_ERROR_EINVAL; + } + + *solosched = (*attr)->solosched; + + return OK; +} + +int KYTY_SYSV_ABI PthreadAttrGetstack(const PthreadAttr* __restrict attr, + void** __restrict stack_addr, size_t* __restrict stack_size) { + // PRINT_NAME(); + + if (stack_size == nullptr || stack_addr == nullptr || attr == nullptr || *attr == nullptr) { + return KERNEL_ERROR_EINVAL; + } + + *stack_addr = (*attr)->stack_addr; + *stack_size = (*attr)->stack_size; + + return OK; +} + +int KYTY_SYSV_ABI PthreadAttrGetstackaddr(const PthreadAttr* attr, void** stack_addr) { + // PRINT_NAME(); + + if (stack_addr == nullptr || attr == nullptr || *attr == nullptr) { + return KERNEL_ERROR_EINVAL; + } + + *stack_addr = (*attr)->stack_addr; + + return OK; +} + +int KYTY_SYSV_ABI PthreadAttrGetstacksize(const PthreadAttr* attr, size_t* stack_size) { + // PRINT_NAME(); + + if (stack_size == nullptr || attr == nullptr || *attr == nullptr) { + return KERNEL_ERROR_EINVAL; + } + + *stack_size = (*attr)->stack_size; + + return OK; +} + +int KYTY_SYSV_ABI PthreadAttrSetaffinity(PthreadAttr* attr, KernelCpumask mask) { + // PRINT_NAME(); + + auto* attr_value = GetPthreadAttrValue(attr, "PthreadAttrSetaffinity"); + if (attr_value == nullptr) { + return KERNEL_ERROR_EINVAL; + } + + attr_value->affinity = mask; + + return OK; +} + +int KYTY_SYSV_ABI PthreadAttrSetdetachstate(PthreadAttr* attr, int state) { + // PRINT_NAME(); + + auto* attr_value = GetPthreadAttrValue(attr, "PthreadAttrSetdetachstate"); + if (attr_value == nullptr) { + return KERNEL_ERROR_EINVAL; + } + + int pstate = PTHREAD_CREATE_JOINABLE; + switch (state) { + case 0: pstate = PTHREAD_CREATE_JOINABLE; break; + case 1: pstate = PTHREAD_CREATE_DETACHED; break; + default: EXIT("unknown state: %d\n", state); + } + + // int result = pthread_attr_setdetachstate(&(*attr)->p, pstate); + int result = 0; + + attr_value->detached = (pstate == PTHREAD_CREATE_DETACHED); + + if (result == 0) { + return OK; + } + return KERNEL_ERROR_EINVAL; +} + +int KYTY_SYSV_ABI PthreadAttrSetguardsize(PthreadAttr* attr, size_t guard_size) { + // PRINT_NAME(); + + auto* attr_value = GetPthreadAttrValue(attr, "PthreadAttrSetguardsize"); + if (attr_value == nullptr) { + return KERNEL_ERROR_EINVAL; + } + + attr_value->guard_size = guard_size; + + return OK; +} + +int KYTY_SYSV_ABI PthreadAttrSetinheritsched(PthreadAttr* attr, int inherit_sched) { + // PRINT_NAME(); + + auto* attr_value = GetPthreadAttrValue(attr, "PthreadAttrSetinheritsched"); + if (attr_value == nullptr) { + return KERNEL_ERROR_EINVAL; + } + + int pinherit_sched = PTHREAD_INHERIT_SCHED; + switch (inherit_sched) { + case 0: pinherit_sched = PTHREAD_EXPLICIT_SCHED; break; + case 4: pinherit_sched = PTHREAD_INHERIT_SCHED; break; + default: EXIT("unknown inherit_sched: %d\n", inherit_sched); + } + + // Keep this in Kyty state. winpthreads' inheritsched support is not needed for guest-visible + // behavior, and some guest attr flows can leave the native attr in a shape that crashes inside + // winpthreads. + attr_value->inherit_sched = inherit_sched; + (void)pinherit_sched; + + return OK; +} + +int KYTY_SYSV_ABI PthreadAttrSetschedparam(PthreadAttr* attr, const KernelSchedParam* param) { + // PRINT_NAME(); + + auto* attr_value = GetPthreadAttrValue(attr, "PthreadAttrSetschedparam"); + if (param == nullptr || attr_value == nullptr) { + return KERNEL_ERROR_EINVAL; + } + + KernelSchedParam pparam {}; + if (param->sched_priority <= 478) { + pparam.sched_priority = +2; + } else if (param->sched_priority >= 733) { + pparam.sched_priority = -2; + } else { + pparam.sched_priority = 0; + } + + int result = pthread_attr_setschedparam(&attr_value->p, &pparam); + + if (result == 0) { + return OK; + } + return KERNEL_ERROR_EINVAL; +} + +int KYTY_SYSV_ABI PthreadAttrSetschedpolicy(PthreadAttr* attr, int policy) { + // PRINT_NAME(); + + auto* attr_value = GetPthreadAttrValue(attr, "PthreadAttrSetschedpolicy"); + if (attr_value == nullptr) { + return KERNEL_ERROR_EINVAL; + } + + // winpthreads supports only SCHED_OTHER policy + int ppolicy = SCHED_OTHER; + + attr_value->policy = policy; + + int result = pthread_attr_setschedpolicy(&attr_value->p, ppolicy); + + if (result == 0) { + return OK; + } + return KERNEL_ERROR_EINVAL; +} + +int KYTY_SYSV_ABI PthreadAttrSetsolosched(PthreadAttr* attr, int solosched) { + // PRINT_NAME(); + + auto* attr_value = GetPthreadAttrValue(attr, "PthreadAttrSetsolosched"); + if (attr_value == nullptr) { + return KERNEL_ERROR_EINVAL; + } + + attr_value->solosched = solosched; + + return OK; +} + +int KYTY_SYSV_ABI PthreadAttrSetstack(PthreadAttr* attr, void* addr, size_t size) { + // PRINT_NAME(); + + auto* attr_value = GetPthreadAttrValue(attr, "PthreadAttrSetstack"); + if (addr == nullptr || size < PTHREAD_STACK_MIN || attr_value == nullptr) { + return KERNEL_ERROR_EINVAL; + } + + attr_value->stack_addr = addr; + attr_value->stack_size = size; + attr_value->stack_user = true; + attr_value->stack_map_addr = 0; + attr_value->stack_map_size = 0; + + return OK; +} + +int KYTY_SYSV_ABI PthreadAttrSetstackaddr(PthreadAttr* attr, void* addr) { + // PRINT_NAME(); + + auto* attr_value = GetPthreadAttrValue(attr, "PthreadAttrSetstackaddr"); + if (addr == nullptr || attr_value == nullptr) { + return KERNEL_ERROR_EINVAL; + } + + attr_value->stack_addr = addr; + attr_value->stack_user = true; + attr_value->stack_map_addr = 0; + attr_value->stack_map_size = 0; + + return OK; +} + +int KYTY_SYSV_ABI PthreadAttrSetstacksize(PthreadAttr* attr, size_t stack_size) { + // PRINT_NAME(); + + auto* attr_value = GetPthreadAttrValue(attr, "PthreadAttrSetstacksize"); + if (stack_size < PTHREAD_STACK_MIN || attr_value == nullptr) { + return KERNEL_ERROR_EINVAL; + } + + attr_value->stack_size = stack_size; + + return OK; +} + +int KYTY_SYSV_ABI PthreadRwlockDestroy(PthreadRwlock* rwlock) { + PRINT_NAME(); + + if (rwlock == nullptr) { + return KERNEL_ERROR_EINVAL; + } + + EXIT_NOT_IMPLEMENTED(*rwlock == nullptr); + + { + std::lock_guard lock((*rwlock)->m); + if ((*rwlock)->writer != nullptr || (*rwlock)->reader_count != 0) { + return KERNEL_ERROR_EBUSY; + } + } + + LOGF("\trwlock destroy: %s, 0\n", (*rwlock)->name.c_str()); + + delete *rwlock; + *rwlock = nullptr; + + return OK; +} + +static int PthreadRwlockInitNamed(PthreadRwlock* rwlock, const PthreadRwlockattr* attr, + const char* name) { + PRINT_NAME(); + + if (rwlock == nullptr) { + return KERNEL_ERROR_EINVAL; + } + + if (attr == nullptr) { + + attr = g_pthread_context->GetDefaultRwlockattr(); + } + + auto* new_rwlock = new PthreadRwlockPrivate {}; + + new_rwlock->name = (name != nullptr ? name : ""); + + LOGF("\trwlock init: %s, 0\n", new_rwlock->name.c_str()); + + std::atomic_ref(*rwlock).store(new_rwlock, std::memory_order_release); + + return OK; +} + +int KYTY_SYSV_ABI PthreadRwlockInit(PthreadRwlock* rwlock, const PthreadRwlockattr* attr) { + return PthreadRwlockInitNamed(rwlock, attr, ""); +} + +static PthreadRwlockPrivate::Reader* RwlockFindReader(PthreadRwlock rwlock, Pthread thread) { + EXIT_IF(rwlock == nullptr); + + for (auto& reader: rwlock->readers) { + if (reader.thread == thread) { + return &reader; + } + } + + return nullptr; +} + +static void RwlockAddReader(PthreadRwlock rwlock, Pthread thread) { + if (auto* reader = RwlockFindReader(rwlock, thread); reader != nullptr) { + reader->count++; + } else { + rwlock->readers.push_back({thread, 1}); + } + rwlock->reader_count++; +} + +static bool RwlockRemoveReader(PthreadRwlock rwlock, Pthread thread) { + for (auto it = rwlock->readers.begin(); it != rwlock->readers.end(); ++it) { + if (it->thread == thread) { + it->count--; + rwlock->reader_count--; + if (it->count == 0) { + rwlock->readers.erase(it); + } + return true; + } + } + + return false; +} + +static int RwlockLockCooperative(PthreadRwlock rwlock, bool write, KernelUseconds* timeout_us) { + EXIT_IF(rwlock == nullptr); + + auto* self = g_pthread_self; + if (self == nullptr) { + return KERNEL_ERROR_EINVAL; + } + + const auto has_timeout = (timeout_us != nullptr); + const auto deadline = + (has_timeout ? std::chrono::steady_clock::now() + std::chrono::microseconds(*timeout_us) + : std::chrono::steady_clock::time_point {}); + bool writer_wait_registered = false; + + for (;;) { + { + std::unique_lock lock(rwlock->m); + + auto unregister_writer_wait = [&rwlock, &writer_wait_registered] { + if (writer_wait_registered) { + rwlock->waiting_writers--; + writer_wait_registered = false; + } + }; + + if (write) { + if (rwlock->writer == self || RwlockFindReader(rwlock, self) != nullptr) { + unregister_writer_wait(); + return KERNEL_ERROR_EDEADLK; + } + if (rwlock->writer == nullptr && rwlock->reader_count == 0) { + unregister_writer_wait(); + rwlock->writer = self; + rwlock->writer_count = 1; + return OK; + } + if (!writer_wait_registered) { + rwlock->waiting_writers++; + writer_wait_registered = true; + } + } else { + const bool already_reader = (RwlockFindReader(rwlock, self) != nullptr); + if ((rwlock->writer == nullptr && + (rwlock->waiting_writers == 0 || already_reader)) || + rwlock->writer == self) { + RwlockAddReader(rwlock, self); + return OK; + } + } + + if (has_timeout) { + const auto now = std::chrono::steady_clock::now(); + if (*timeout_us == 0 || now >= deadline) { + unregister_writer_wait(); + return KERNEL_ERROR_ETIMEDOUT; + } + + const auto remaining = deadline - now; + const auto poll = (remaining < std::chrono::microseconds(SIGNAL_APC_POLL_MICROS) + ? remaining + : std::chrono::steady_clock::duration( + std::chrono::microseconds(SIGNAL_APC_POLL_MICROS))); + rwlock->cv.wait_for(lock, poll); + } else { + rwlock->cv.wait_for(lock, std::chrono::microseconds(SIGNAL_APC_POLL_MICROS)); + } + } + + KernelDispatchPendingSignalForCurrentThread(); + } +} + +int KYTY_SYSV_ABI PthreadRwlockRdlock(PthreadRwlock* rwlock) { + // Hot path for some PS5 titles; per-call name logging can dominate runtime. + + auto* pthread_static_objects = g_pthread_context->GetPthreadStaticObjects(); + + rwlock = static_cast( + pthread_static_objects->CreateObject(rwlock, PthreadStaticObject::Type::Rwlock)); + + if (rwlock == nullptr) { + return KERNEL_ERROR_EINVAL; + } + + EXIT_NOT_IMPLEMENTED(*rwlock == nullptr); + + return RwlockLockCooperative(*rwlock, false, nullptr); +} + +int KYTY_SYSV_ABI PthreadRwlockTimedrdlock(PthreadRwlock* rwlock, KernelUseconds usec) { + PRINT_NAME(); + + if (rwlock == nullptr) { + return KERNEL_ERROR_EINVAL; + } + + EXIT_NOT_IMPLEMENTED(*rwlock == nullptr); + + return RwlockLockCooperative(*rwlock, false, &usec); +} + +int KYTY_SYSV_ABI PthreadRwlockTimedwrlock(PthreadRwlock* rwlock, KernelUseconds usec) { + PRINT_NAME(); + + if (rwlock == nullptr) { + return KERNEL_ERROR_EINVAL; + } + + EXIT_NOT_IMPLEMENTED(*rwlock == nullptr); + + return RwlockLockCooperative(*rwlock, true, &usec); +} + +int KYTY_SYSV_ABI PthreadRwlockTryrdlock(PthreadRwlock* rwlock) { + PRINT_NAME(); + + if (rwlock == nullptr) { + return KERNEL_ERROR_EINVAL; + } + + EXIT_NOT_IMPLEMENTED(*rwlock == nullptr); + + auto* self = g_pthread_self; + if (self == nullptr) { + return KERNEL_ERROR_EINVAL; + } + + { + std::lock_guard lock((*rwlock)->m); + const bool already_reader = (RwlockFindReader(*rwlock, self) != nullptr); + if (((*rwlock)->writer == nullptr && ((*rwlock)->waiting_writers == 0 || already_reader)) || + (*rwlock)->writer == self) { + RwlockAddReader(*rwlock, self); + return OK; + } + } + + return KERNEL_ERROR_EBUSY; +} + +int KYTY_SYSV_ABI PthreadRwlockTrywrlock(PthreadRwlock* rwlock) { + PRINT_NAME(); + + if (rwlock == nullptr) { + return KERNEL_ERROR_EINVAL; + } + + EXIT_NOT_IMPLEMENTED(*rwlock == nullptr); + + auto* self = g_pthread_self; + if (self == nullptr) { + return KERNEL_ERROR_EINVAL; + } + + { + std::lock_guard lock((*rwlock)->m); + if ((*rwlock)->writer == self || RwlockFindReader(*rwlock, self) != nullptr) { + return KERNEL_ERROR_EDEADLK; + } + if ((*rwlock)->writer == nullptr && (*rwlock)->reader_count == 0) { + (*rwlock)->writer = self; + (*rwlock)->writer_count = 1; + return OK; + } + } + + return KERNEL_ERROR_EBUSY; +} + +int KYTY_SYSV_ABI PthreadRwlockUnlock(PthreadRwlock* rwlock) { + // PRINT_NAME(); + + auto* pthread_static_objects = g_pthread_context->GetPthreadStaticObjects(); + + rwlock = static_cast( + pthread_static_objects->CreateObject(rwlock, PthreadStaticObject::Type::Rwlock)); + + if (rwlock == nullptr) { + return KERNEL_ERROR_EINVAL; + } + + EXIT_NOT_IMPLEMENTED(*rwlock == nullptr); + + auto* self = g_pthread_self; + if (self == nullptr) { + return KERNEL_ERROR_EINVAL; + } + + { + std::lock_guard lock((*rwlock)->m); + if (RwlockRemoveReader(*rwlock, self)) { + (*rwlock)->cv.notify_all(); + return OK; + } + if ((*rwlock)->writer == self) { + EXIT_IF((*rwlock)->writer_count == 0); + (*rwlock)->writer_count--; + if ((*rwlock)->writer_count == 0) { + (*rwlock)->writer = nullptr; + (*rwlock)->cv.notify_all(); + } + return OK; + } + + return KERNEL_ERROR_EPERM; + } +} + +int KYTY_SYSV_ABI PthreadRwlockWrlock(PthreadRwlock* rwlock) { + // PRINT_NAME(); + + auto* pthread_static_objects = g_pthread_context->GetPthreadStaticObjects(); + + rwlock = static_cast( + pthread_static_objects->CreateObject(rwlock, PthreadStaticObject::Type::Rwlock)); + + if (rwlock == nullptr) { + return KERNEL_ERROR_EINVAL; + } + + EXIT_NOT_IMPLEMENTED(*rwlock == nullptr); + + return RwlockLockCooperative(*rwlock, true, nullptr); +} + +int KYTY_SYSV_ABI PthreadRwlockattrDestroy(PthreadRwlockattr* attr) { + PRINT_NAME(); + + if (attr == nullptr) { + return KERNEL_ERROR_EINVAL; + } + + EXIT_NOT_IMPLEMENTED(*attr == nullptr); + + int result = pthread_rwlockattr_destroy(&(*attr)->p); + + delete *attr; + *attr = nullptr; + + if (result == 0) { + return OK; + } + return KERNEL_ERROR_EINVAL; +} + +int KYTY_SYSV_ABI PthreadRwlockattrInit(PthreadRwlockattr* attr) { + PRINT_NAME(); + + *attr = new PthreadRwlockattrPrivate {}; + + int result = pthread_rwlockattr_init(&(*attr)->p); + + result = (result == 0 ? PthreadRwlockattrSettype(attr, 1) : result); + + switch (result) { + case 0: return OK; + case ENOMEM: return KERNEL_ERROR_ENOMEM; + default: return KERNEL_ERROR_EINVAL; + } +} + +int KYTY_SYSV_ABI PthreadRwlockattrGettype(PthreadRwlockattr* attr, int* type) { + PRINT_NAME(); + + if (type == nullptr || attr == nullptr) { + return KERNEL_ERROR_EINVAL; + } + + *type = (*attr)->type; + + return OK; +} + +int KYTY_SYSV_ABI PthreadRwlockattrSettype(PthreadRwlockattr* attr, int type) { + PRINT_NAME(); + + if (attr == nullptr) { + return KERNEL_ERROR_EINVAL; + } + + (*attr)->type = type; + + return OK; +} + +int KYTY_SYSV_ABI PthreadCondattrDestroy(PthreadCondattr* attr) { + PRINT_NAME(); + + if (attr == nullptr) { + return KERNEL_ERROR_EINVAL; + } + + EXIT_NOT_IMPLEMENTED(*attr == nullptr); + + int result = pthread_condattr_destroy(&(*attr)->p); + + delete *attr; + *attr = nullptr; + + if (result == 0) { + return OK; + } + return KERNEL_ERROR_EINVAL; +} + +int KYTY_SYSV_ABI PthreadCondattrInit(PthreadCondattr* attr) { + PRINT_NAME(); + + *attr = new PthreadCondattrPrivate {}; + + int result = pthread_condattr_init(&(*attr)->p); + (*attr)->clock_id = KERNEL_CLOCK_REALTIME; + + switch (result) { + case 0: return OK; + case ENOMEM: return KERNEL_ERROR_ENOMEM; + default: return KERNEL_ERROR_EINVAL; + } +} + +int KYTY_SYSV_ABI PthreadCondattrSetclock(PthreadCondattr* attr, KernelClockid clock_id) { + PRINT_NAME(); + + if (attr == nullptr || *attr == nullptr) { + return KERNEL_ERROR_EINVAL; + } + + clockid_t pclock_id = {}; + if (!GetPosixClockId(clock_id, &pclock_id)) { + LOGF("\tunknown clock_id: %d\n", clock_id); + return KERNEL_ERROR_EINVAL; + } + + int result = pthread_condattr_setclock(&(*attr)->p, pclock_id); + (*attr)->clock_id = clock_id; + + LOGF("\tcondattr setclock: clock_id = %d, native = %d, result = %d\n", clock_id, + static_cast(pclock_id), result); + + if (result == EINVAL && pclock_id == CLOCK_MONOTONIC) { + LOGF("\tcondattr setclock: host rejected CLOCK_MONOTONIC, keeping emulated attribute\n"); + return OK; + } + + switch (result) { + case 0: return OK; + case EINVAL: + default: return KERNEL_ERROR_EINVAL; + } +} + +int KYTY_SYSV_ABI PthreadCondBroadcast(PthreadCond* cond) { + // PRINT_NAME(); + + auto* pthread_static_objects = g_pthread_context->GetPthreadStaticObjects(); + + cond = static_cast( + pthread_static_objects->CreateObject(cond, PthreadStaticObject::Type::Cond)); + + if (cond == nullptr) { + return KERNEL_ERROR_EINVAL; + } + + EXIT_NOT_IMPLEMENTED(*cond == nullptr); + + int result = 0; + bool notify = false; + { + std::lock_guard lock((*cond)->m); + if (!(*cond)->waiters.empty()) { + (*cond)->sequence++; + CondClearWaiters(*cond); + notify = true; + } + } + if (notify) { + (*cond)->cv.notify_all(); + } + + // LOGF("\tcond broadcast: %s(0x%016" PRIx64 "), %d\n", (*cond)->name.c_str(), + // reinterpret_cast(cond), result); + + if (result == 0) { + return OK; + } + return KERNEL_ERROR_EINVAL; +} + +int KYTY_SYSV_ABI PthreadCondDestroy(PthreadCond* cond) { + PRINT_NAME(); + + if (cond == nullptr) { + return KERNEL_ERROR_EINVAL; + } + + EXIT_NOT_IMPLEMENTED(*cond == nullptr); + + int result = 0; + + LOGF("\tcond destroy: %s, %d\n", (*cond)->name.c_str(), result); + + delete *cond; + *cond = nullptr; + + switch (result) { + case 0: return OK; + case EINVAL: return KERNEL_ERROR_EINVAL; + case EBUSY: return KERNEL_ERROR_EBUSY; + default: return KERNEL_ERROR_EINVAL; + } +} + +static int PthreadCondInitNamed(PthreadCond* cond, const PthreadCondattr* attr, const char* name) { + if (name != nullptr && name[0] != '\0') { + PRINT_NAME(); + } + + if (cond == nullptr) { + return KERNEL_ERROR_EINVAL; + } + + if (attr == nullptr) { + + attr = g_pthread_context->GetDefaultCondattr(); + } + + auto* new_cond = new PthreadCondPrivate {}; + + new_cond->name = (name != nullptr ? name : ""); + new_cond->clock_id = (*attr)->clock_id; + + int result = 0; + + if (name != nullptr && name[0] != '\0') { + LOGF("\tcond init: %s, cond=0x%016" PRIx64 ", caller=0x%016" PRIx64 ", %d\n", + new_cond->name.c_str(), reinterpret_cast(cond), + reinterpret_cast(__builtin_return_address(0)), result); + } + + std::atomic_ref(*cond).store(new_cond, std::memory_order_release); + + switch (result) { + case 0: return OK; + case EAGAIN: return KERNEL_ERROR_EAGAIN; + case EINVAL: return KERNEL_ERROR_EINVAL; + case ENOMEM: return KERNEL_ERROR_ENOMEM; + default: return KERNEL_ERROR_EINVAL; + } +} + +int KYTY_SYSV_ABI PthreadCondInit(PthreadCond* cond, const PthreadCondattr* attr) { + return PthreadCondInitNamed(cond, attr, ""); +} + +int KYTY_SYSV_ABI PthreadCondSignal(PthreadCond* cond) { + // PRINT_NAME(); + + auto* pthread_static_objects = g_pthread_context->GetPthreadStaticObjects(); + + cond = static_cast( + pthread_static_objects->CreateObject(cond, PthreadStaticObject::Type::Cond)); + + if (cond == nullptr) { + return KERNEL_ERROR_EINVAL; + } + + EXIT_NOT_IMPLEMENTED(*cond == nullptr); + + int result = 0; + bool notify = false; + { + std::lock_guard lock((*cond)->m); + notify = CondWakeWaiter(*cond, nullptr); + } + if (notify) { + (*cond)->cv.notify_all(); + } + + // LOGF("\tcond signal: %s(0x%016" PRIx64 "), %d\n", (*cond)->name.c_str(), + // reinterpret_cast(cond), result); + + if (result == 0) { + return OK; + } + return KERNEL_ERROR_EINVAL; +} + +int KYTY_SYSV_ABI PthreadCondSignalto(PthreadCond* cond, Pthread thread) { + // PRINT_NAME(); + + auto* pthread_static_objects = g_pthread_context->GetPthreadStaticObjects(); + + cond = static_cast( + pthread_static_objects->CreateObject(cond, PthreadStaticObject::Type::Cond)); + + if (cond == nullptr || thread == nullptr) { + return KERNEL_ERROR_EINVAL; + } + + EXIT_NOT_IMPLEMENTED(*cond == nullptr); + + int result = 0; + bool notify = false; + { + std::lock_guard lock((*cond)->m); + notify = CondWakeWaiter(*cond, thread); + } + if (notify) { + (*cond)->cv.notify_all(); + } + + // LOGF("\tcond signalto: %s(0x%016" PRIx64 "), %d\n", (*cond)->name.c_str(), + // reinterpret_cast(cond), result); + + if (result == 0) { + return OK; + } + return KERNEL_ERROR_EINVAL; +} + +int KYTY_SYSV_ABI PthreadCondTimedwait(PthreadCond* cond, PthreadMutex* mutex, + KernelUseconds usec) { + // PRINT_NAME(); + + auto* pthread_static_objects = g_pthread_context->GetPthreadStaticObjects(); + + cond = static_cast( + pthread_static_objects->CreateObject(cond, PthreadStaticObject::Type::Cond)); + mutex = static_cast( + pthread_static_objects->CreateObject(mutex, PthreadStaticObject::Type::Mutex)); + + if (cond == nullptr || mutex == nullptr) { + return KERNEL_ERROR_EINVAL; + } + + EXIT_NOT_IMPLEMENTED(*cond == nullptr); + EXIT_NOT_IMPLEMENTED(*mutex == nullptr); + + auto* cond_value = *cond; + auto* mutex_value = *mutex; + + if (mutex_value->owner != g_pthread_self) { + return KERNEL_ERROR_EPERM; + } + + std::unique_lock cond_lock(cond_value->m); + const auto sequence = cond_value->sequence; + auto* thread = g_pthread_self; + const auto thread_sequence = thread->cond_sequence; + CondAddWaiter(cond_value, thread); + + uint32_t recurse = 0; + int result = NativeMutexUnlock(mutex_value, &recurse); + if (result != OK) { + CondRemoveWaiter(cond_value, thread); + return (result == EPERM ? KERNEL_ERROR_EPERM : KERNEL_ERROR_EINVAL); + } + + const auto deadline = std::chrono::steady_clock::now() + std::chrono::microseconds(usec); + auto ready = [cond_value, thread, sequence, thread_sequence] { + return cond_value->sequence != sequence || thread->cond_sequence != thread_sequence; + }; + + if (usec == 0) { + result = ETIMEDOUT; + } else { + while (!ready()) { + const auto now = std::chrono::steady_clock::now(); + if (now >= deadline) { + break; + } + + const auto remaining = deadline - now; + const auto poll = (remaining < std::chrono::microseconds(SIGNAL_APC_POLL_MICROS) + ? remaining + : std::chrono::steady_clock::duration( + std::chrono::microseconds(SIGNAL_APC_POLL_MICROS))); + cond_value->cv.wait_for(cond_lock, poll); + + if (!ready()) { + cond_lock.unlock(); + KernelDispatchPendingSignalForCurrentThread(); + cond_lock.lock(); + } + } + + result = (ready() ? OK : ETIMEDOUT); + } + CondRemoveWaiter(cond_value, thread); + cond_lock.unlock(); + + int lock_result = NativeMutexLockRecurse(mutex_value, recurse); + if (result == OK) { + result = lock_result; + } + + // LOGF("\tcond timedwait: %s, %d\n", (*cond)->name.c_str(), result); + + switch (result) { + case 0: return OK; + case ETIMEDOUT: return KERNEL_ERROR_ETIMEDOUT; + case EPERM: return KERNEL_ERROR_EPERM; + case EINVAL: + default: return KERNEL_ERROR_EINVAL; + } +} + +int KYTY_SYSV_ABI PthreadCondTimedwaitAbs(PthreadCond* cond, PthreadMutex* mutex, + const KernelTimespec* abstime) { + // PRINT_NAME(); + + auto* pthread_static_objects = g_pthread_context->GetPthreadStaticObjects(); + + cond = static_cast( + pthread_static_objects->CreateObject(cond, PthreadStaticObject::Type::Cond)); + mutex = static_cast( + pthread_static_objects->CreateObject(mutex, PthreadStaticObject::Type::Mutex)); + + if (cond == nullptr || mutex == nullptr) { + return KERNEL_ERROR_EINVAL; + } + + EXIT_NOT_IMPLEMENTED(*cond == nullptr); + EXIT_NOT_IMPLEMENTED(*mutex == nullptr); + + auto* cond_value = *cond; + auto* mutex_value = *mutex; + + std::chrono::steady_clock::time_point deadline {}; + if (!NativeCondDeadlineFromAbs(cond_value->clock_id, abstime, &deadline)) { + return KERNEL_ERROR_EINVAL; + } + + if (mutex_value->owner != g_pthread_self) { + return KERNEL_ERROR_EPERM; + } + + std::unique_lock cond_lock(cond_value->m); + const auto sequence = cond_value->sequence; + auto* thread = g_pthread_self; + const auto thread_sequence = thread->cond_sequence; + CondAddWaiter(cond_value, thread); + + uint32_t recurse = 0; + int result = NativeMutexUnlock(mutex_value, &recurse); + if (result != OK) { + CondRemoveWaiter(cond_value, thread); + return (result == EPERM ? KERNEL_ERROR_EPERM : KERNEL_ERROR_EINVAL); + } + + auto ready = [cond_value, thread, sequence, thread_sequence] { + return cond_value->sequence != sequence || thread->cond_sequence != thread_sequence; + }; + + while (!ready()) { + const auto now = std::chrono::steady_clock::now(); + if (now >= deadline) { + break; + } + + const auto remaining = deadline - now; + const auto poll = (remaining < std::chrono::microseconds(SIGNAL_APC_POLL_MICROS) + ? remaining + : std::chrono::steady_clock::duration( + std::chrono::microseconds(SIGNAL_APC_POLL_MICROS))); + cond_value->cv.wait_for(cond_lock, poll); + + if (!ready()) { + cond_lock.unlock(); + KernelDispatchPendingSignalForCurrentThread(); + cond_lock.lock(); + } + } + + result = (ready() ? OK : ETIMEDOUT); + CondRemoveWaiter(cond_value, thread); + cond_lock.unlock(); + + int lock_result = NativeMutexLockRecurse(mutex_value, recurse); + if (result == OK) { + result = lock_result; + } + + switch (result) { + case 0: return OK; + case ETIMEDOUT: return KERNEL_ERROR_ETIMEDOUT; + case EPERM: return KERNEL_ERROR_EPERM; + case EINVAL: + default: return KERNEL_ERROR_EINVAL; + } +} + +int KYTY_SYSV_ABI PthreadCondWait(PthreadCond* cond, PthreadMutex* mutex) { + PRINT_NAME(); + + auto* pthread_static_objects = g_pthread_context->GetPthreadStaticObjects(); + + cond = static_cast( + pthread_static_objects->CreateObject(cond, PthreadStaticObject::Type::Cond)); + mutex = static_cast( + pthread_static_objects->CreateObject(mutex, PthreadStaticObject::Type::Mutex)); + + if (cond == nullptr || mutex == nullptr) { + return KERNEL_ERROR_EINVAL; + } + + EXIT_NOT_IMPLEMENTED(*cond == nullptr); + EXIT_NOT_IMPLEMENTED(*mutex == nullptr); + + auto* cond_value = *cond; + auto* mutex_value = *mutex; + + if (mutex_value->owner != g_pthread_self) { + return KERNEL_ERROR_EPERM; + } + + std::unique_lock cond_lock(cond_value->m); + const auto sequence = cond_value->sequence; + auto* thread = g_pthread_self; + const auto thread_sequence = thread->cond_sequence; + CondAddWaiter(cond_value, thread); + + uint32_t recurse = 0; + int result = NativeMutexUnlock(mutex_value, &recurse); + if (result != OK) { + CondRemoveWaiter(cond_value, thread); + return (result == EPERM ? KERNEL_ERROR_EPERM : KERNEL_ERROR_EINVAL); + } + + auto ready = [cond_value, thread, sequence, thread_sequence] { + return cond_value->sequence != sequence || thread->cond_sequence != thread_sequence; + }; + + while (!ready()) { + cond_value->cv.wait_for(cond_lock, std::chrono::microseconds(SIGNAL_APC_POLL_MICROS)); + if (!ready()) { + cond_lock.unlock(); + KernelDispatchPendingSignalForCurrentThread(); + cond_lock.lock(); + } + } + CondRemoveWaiter(cond_value, thread); + cond_lock.unlock(); + + result = NativeMutexLockRecurse(mutex_value, recurse); + + switch (result) { + case 0: return OK; + case EPERM: return KERNEL_ERROR_EPERM; + case EINVAL: + default: return KERNEL_ERROR_EINVAL; + } +} + +Pthread KYTY_SYSV_ABI PthreadSelf() { + // PRINT_NAME(); + + EXIT_NOT_IMPLEMENTED(g_pthread_self == nullptr); + + return g_pthread_self; +} + +Pthread PthreadSelfOrNull() { + return g_pthread_self; +} + +Pthread PthreadSwapSelfForSignal(Pthread thread) { + auto* previous = g_pthread_self; + g_pthread_self = thread; + return previous; +} + +int PthreadGetUniqueId(Pthread thread) { + return thread != nullptr ? thread->unique_id : 0; +} + +uint64_t PthreadGetHostThreadId(Pthread thread) { + return thread != nullptr ? thread->host_thread_id : 0; +} + +void PthreadQueuePendingSignal(Pthread thread, int signum) { + if (thread == nullptr || signum < 0 || signum >= 64) { + return; + } + + thread->pending_signal_mask.fetch_or(1ull << static_cast(signum), + std::memory_order_release); +} + +bool PthreadHasPendingSignal(Pthread thread, int signum) { + if (thread == nullptr || signum < 0 || signum >= 64) { + return false; + } + + const auto mask = 1ull << static_cast(signum); + return (thread->pending_signal_mask.load(std::memory_order_acquire) & mask) != 0; +} + +bool PthreadTakePendingSignal(Pthread thread, int signum) { + if (thread == nullptr || signum < 0 || signum >= 64) { + return false; + } + + const auto mask = 1ull << static_cast(signum); + return (thread->pending_signal_mask.fetch_and(~mask, std::memory_order_acq_rel) & mask) != 0; +} + +bool PthreadGetGuestStack(Pthread thread, uint64_t* stack_addr, uint64_t* stack_size) { + if (thread == nullptr || thread->attr == nullptr || stack_addr == nullptr || + stack_size == nullptr) { + return false; + } + + auto* addr = thread->attr->stack_addr; + auto size = thread->attr->stack_size; + if (addr == nullptr || size == 0) { + return false; + } + + *stack_addr = reinterpret_cast(addr); + *stack_size = static_cast(size); + return true; +} + +int PthreadGetPriorityForKernel(Pthread thread) { + if (thread == nullptr) { + return 700; + } + + sched_param param {}; + int pol = 0; + + if (pthread_getschedparam(thread->p, &pol, ¶m) != 0) { + return 700; + } + + if (param.sched_priority <= -2) { + return 767; + } + if (param.sched_priority >= +2) { + return 256; + } + return 700; +} + +int PthreadGetCurrentPriorityForKernel() { + return PthreadGetPriorityForKernel(g_pthread_self); +} + +static void CleanupThread(void* arg) { + auto* thread = static_cast(arg); + + auto thread_dtors = g_pthread_context->GetThreadDtors(); + + if (thread_dtors != nullptr) { + thread_dtors(); + } + + g_pthread_context->GetPthreadKeys()->Destruct(thread->unique_id); + + auto* rt = Common::Singleton::Instance(); + rt->DeleteTlss(thread->unique_id); + + thread->almost_done = true; +} + +static void* RunThread(void* arg) { + auto* thread = static_cast(arg); + void* ret = nullptr; + + thread->unique_id = Common::Thread::GetThreadIdUnique(); + + g_pthread_self = thread; + + uint64_t os_thread_id = 0; +#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS + os_thread_id = static_cast(GetCurrentThreadId()); +#endif + thread->host_thread_id = os_thread_id; + + LOGF("\tPthread run begin: %s, id = %d, os_thread_id = %" PRIu64 ", entry = 0x%016" PRIx64 + ", arg = 0x%016" PRIx64 ", stack_addr = 0x%016" PRIx64 ", stack_size = %" PRIu64 "\n", + thread->name.c_str(), thread->unique_id, os_thread_id, + reinterpret_cast(thread->entry), reinterpret_cast(thread->arg), + reinterpret_cast(thread->attr->stack_addr), + static_cast(thread->attr->stack_size)); + + // NOLINTNEXTLINE(cppcoreguidelines-pro-type-cstyle-cast) + pthread_cleanup_push(CleanupThread, thread); + + auto* stack_top = reinterpret_cast( + (reinterpret_cast(thread->attr->stack_addr) + thread->attr->stack_size) & + ~static_cast(0x0f)); + ret = RunOnGuestStack(thread->arg, thread->entry, stack_top); + + // NOLINTNEXTLINE(cppcoreguidelines-pro-type-cstyle-cast) + pthread_cleanup_pop(1); + + return ret; +} + +int KYTY_SYSV_ABI PthreadCreate(Pthread* thread, const PthreadAttr* attr, + pthread_entry_func_t entry, void* arg, const char* name) { + PRINT_NAME(); + + if (thread == nullptr) { + return KERNEL_ERROR_EINVAL; + } + + auto* pthread_pool = g_pthread_context->GetPthreadPool(); + + if (attr == nullptr) { + attr = g_pthread_context->GetDefaultAttr(); + } + if (GetPthreadAttrValue(attr, "PthreadCreate") == nullptr) { + attr = g_pthread_context->GetDefaultAttr(); + } + + PRINT_NAME_ENABLE(false); + + auto* created_thread = pthread_pool->Create(); + *thread = created_thread; + + if (created_thread->attr != nullptr) { + PthreadAttrDestroy(&created_thread->attr); + } + + PthreadAttrInit(&created_thread->attr); + + int result = PthreadAttrCopy(&created_thread->attr, attr); + + if (result == 0) { + EXIT_IF(created_thread->free); + + if (created_thread->attr->stack_addr == nullptr) { + created_thread->attr->stack_size += PTHREAD_STACK_EXTRA; + } + + result = CreateGuestStack(created_thread->attr); + } + + if (result == 0) { + created_thread->name = (name != nullptr ? name : ""); + created_thread->entry = entry; + created_thread->arg = arg; + created_thread->guest.thread_id = ++g_pthread_thread_id; + created_thread->almost_done = false; + created_thread->detached = created_thread->attr->detached; + created_thread->unique_id = -1; + + result = + pthread_create(&created_thread->p, &created_thread->attr->p, RunThread, created_thread); + + if (result != 0) { + FreeGuestStack(created_thread->attr); + } + } + + if (result != 0) { + created_thread->free = true; + } + + LOGF("\tthread create: %s, id = %d, %d\n", created_thread->name.c_str(), + created_thread->unique_id, result); + + PthreadAttrDbgPrint(&created_thread->attr); + + PRINT_NAME_ENABLE(true); + + if (result < 0) { + return result; + } + + switch (result) { + case 0: return OK; + case ENOMEM: return KERNEL_ERROR_ENOMEM; + case EAGAIN: return KERNEL_ERROR_EAGAIN; + case EDEADLK: return KERNEL_ERROR_EDEADLK; + case EPERM: return KERNEL_ERROR_EPERM; + default: return KERNEL_ERROR_EINVAL; + } +} + +int KYTY_SYSV_ABI PthreadDetach(Pthread thread) { + PRINT_NAME(); + + if (thread == nullptr) { + return KERNEL_ERROR_EINVAL; + } + + LOGF("\tthread detach: %s, %d\n", thread->name.c_str(), 0); + + thread->detached = true; + + return OK; +} + +int KYTY_SYSV_ABI PthreadJoin(Pthread thread, void** value) { + PRINT_NAME(); + + if (thread == nullptr) { + return KERNEL_ERROR_EINVAL; + } + + int result = pthread_join(thread->p, value); + + if (PRINT_NAME_ENABLED) { + LOGF("\tthread join: %s, %d\n", thread->name.c_str(), result); + } + + if (result == 0) { + FreeGuestStack(thread->attr); + + thread->almost_done = false; + thread->free = true; + } + + switch (result) { + case 0: return OK; + case ESRCH: return KERNEL_ERROR_ESRCH; + case EDEADLK: return KERNEL_ERROR_EDEADLK; + case EOPNOTSUPP: return KERNEL_ERROR_EOPNOTSUPP; + default: return KERNEL_ERROR_EINVAL; + } +} + +int KYTY_SYSV_ABI PthreadCancel(Pthread thread) { + PRINT_NAME(); + + if (thread == nullptr) { + return KERNEL_ERROR_EINVAL; + } + + int result = pthread_cancel(thread->p); + + LOGF("\tthread cancel: %s, %d\n", thread->name.c_str(), result); + + switch (result) { + case 0: return OK; + case ESRCH: return KERNEL_ERROR_ESRCH; + default: return KERNEL_ERROR_EINVAL; + } +} + +int KYTY_SYSV_ABI PthreadSetaffinity(Pthread thread, KernelCpumask mask) { + PRINT_NAME(); + + if (thread == nullptr) { + return KERNEL_ERROR_ESRCH; + } + + auto result = PthreadAttrSetaffinity(&thread->attr, mask); + + return result; +} + +int KYTY_SYSV_ABI PthreadGetaffinity(Pthread thread, KernelCpumask* mask) { + PRINT_NAME(); + + if (thread == nullptr) { + return KERNEL_ERROR_ESRCH; + } + if (mask == nullptr) { + return KERNEL_ERROR_EINVAL; + } + + return PthreadAttrGetaffinity(&thread->attr, mask); +} + +int KYTY_SYSV_ABI PthreadSetcancelstate(int state, int* old_state) { + PRINT_NAME(); + + int pstate = PTHREAD_CANCEL_DISABLE; + + switch (state) { + case 0: pstate = PTHREAD_CANCEL_ENABLE; break; + case 1: pstate = PTHREAD_CANCEL_DISABLE; break; + default: EXIT("unknown state: %d", state); + } + + int result = pthread_setcancelstate(pstate, old_state); + + LOGF("\tthread setcancelstate: %d\n", result); + + if (old_state != nullptr) { + switch (*old_state) { + case PTHREAD_CANCEL_ENABLE: *old_state = 0; break; + case PTHREAD_CANCEL_DISABLE: *old_state = 1; break; + default: EXIT("unknown old_state: %d", *old_state); + } + } + + if (result == 0) { + return OK; + } + return KERNEL_ERROR_EINVAL; +} + +int KYTY_SYSV_ABI PthreadSetcanceltype(int type, int* old_type) { + PRINT_NAME(); + + int ptype = PTHREAD_CANCEL_DEFERRED; + + switch (type) { + case 0: ptype = PTHREAD_CANCEL_DEFERRED; break; + case 2: ptype = PTHREAD_CANCEL_ASYNCHRONOUS; break; + default: EXIT("unknown type: %d", type); + } + + int result = pthread_setcanceltype(ptype, old_type); + + LOGF("\tthread setcanceltype: %d\n", result); + + if (old_type != nullptr) { + switch (*old_type) { + case PTHREAD_CANCEL_DEFERRED: *old_type = 0; break; + case PTHREAD_CANCEL_ASYNCHRONOUS: *old_type = 2; break; + default: EXIT("unknown type: %d", *old_type); + } + } + + if (result == 0) { + return OK; + } + return KERNEL_ERROR_EINVAL; +} + +int KYTY_SYSV_ABI PthreadGetprio(Pthread thread, int* prio) { + PRINT_NAME(); + + if (thread == nullptr) { + return KERNEL_ERROR_ESRCH; + } + + EXIT_NOT_IMPLEMENTED(prio == nullptr); + + sched_param param {}; + int pol = 0; + + int result = pthread_getschedparam(thread->p, &pol, ¶m); + + if (result == 0) { + if (param.sched_priority <= -2) { + *prio = 767; + } else if (param.sched_priority >= +2) { + *prio = 256; + } else { + *prio = 700; + } + + LOGF("\t PthreadGetprio: %d, %d\n", thread->unique_id, *prio); + + return OK; + } + + return KERNEL_ERROR_EINVAL; +} + +int KYTY_SYSV_ABI PthreadSetprio(Pthread thread, int prio) { + PRINT_NAME(); + + if (thread == nullptr) { + return KERNEL_ERROR_ESRCH; + } + + sched_param param {}; + int pol = 0; + + int result = pthread_getschedparam(thread->p, &pol, ¶m); + + if (result == 0) { + if (prio <= 478) { + param.sched_priority = +2; + } else if (prio >= 733) { + param.sched_priority = -2; + } else { + param.sched_priority = 0; + } + + result = pthread_setschedparam(thread->p, pol, ¶m); + + if (result == 0) { + LOGF("\t PthreadSetprio: %d, %d\n", thread->unique_id, prio); + + return OK; + } + } + + return KERNEL_ERROR_EINVAL; +} + +void KYTY_SYSV_ABI PthreadTestcancel() { + PRINT_NAME(); + + pthread_testcancel(); +} + +void KYTY_SYSV_ABI PthreadExit(void* value) { + PRINT_NAME(); + +#if defined(__x86_64__) || defined(_M_X64) + if (g_guest_entry_return_rsp != 0) { + const auto return_rsp = g_guest_entry_return_rsp; + g_guest_entry_return_rsp = 0; + + if (g_pthread_self != nullptr && g_pthread_self->guest_host_rsp != 0) { + const auto host_rbx = g_pthread_self->guest_host_rbx; + const auto host_rsp = g_pthread_self->guest_host_rsp; + const auto host_rbp = g_pthread_self->guest_host_rbp; +#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS + const auto host_gs8 = g_pthread_self->guest_host_gs8; + const auto host_gs10 = g_pthread_self->guest_host_gs10; +#endif + + asm volatile("movq %2, %%rbx\n\t" + "movq %3, %%r12\n\t" + "movq %4, %%r13\n\t" +#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS + "movq %5, %%r14\n\t" + "movq %6, %%r15\n\t" +#endif + "movq %0, %%rax\n\t" + "movq %1, %%rsp\n\t" + "retq\n\t" + : +#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS + : "m"(value), "m"(return_rsp), "m"(host_rbx), "m"(host_rsp), "m"(host_rbp), + "m"(host_gs8), "m"(host_gs10) +#else + : "m"(value), "m"(return_rsp), "m"(host_rbx), "m"(host_rsp), "m"(host_rbp) +#endif + : "rax", "memory"); + __builtin_unreachable(); + } + } +#endif + + pthread_exit(value); +} + +int KYTY_SYSV_ABI PthreadEqual(Pthread thread1, Pthread thread2) { + static std::atomic log_count {0}; + + const int result = (thread1 == thread2 ? 1 : 0); + + const auto count = log_count.fetch_add(1); + if (count < 32 || (result != 0 && count < 128)) { + LOGF("\tPthreadEqual: t1 = 0x%016" PRIx64 ", t2 = 0x%016" PRIx64 ", result = %d\n", + reinterpret_cast(thread1), reinterpret_cast(thread2), result); + } + + return result; +} + +int KYTY_SYSV_ABI PthreadGetname(Pthread thread, char* name) { + PRINT_NAME(); + + if (thread == nullptr) { + return KERNEL_ERROR_ESRCH; + } + + if (name == nullptr) { + return KERNEL_ERROR_EFAULT; + } + + strncpy(name, thread->name.c_str(), 32); + name[31] = '\0'; + + return OK; +} + +int KYTY_SYSV_ABI PthreadRename(Pthread thread, const char* name) { + PRINT_NAME(); + + if (thread == nullptr) { + return KERNEL_ERROR_EINVAL; + } + + if (name == nullptr) { + return OK; + } + + thread->name = std::string(name); + + return OK; +} + +void KYTY_SYSV_ABI PthreadYield() { + SchedulerBackoffOnce(); +} + +int KYTY_SYSV_ABI PthreadGetthreadid() { + // PRINT_NAME(); + + return g_pthread_self != nullptr ? g_pthread_self->guest.thread_id : 0; +} + +int KYTY_SYSV_ABI KernelClockGetres(KernelClockid clock_id, KernelTimespec* tp) { + PRINT_NAME(); + + if (tp == nullptr) { + return KERNEL_ERROR_EFAULT; + } + + if (clock_id == KERNEL_CLOCK_PROCTIME || clock_id == KERNEL_CLOCK_THREAD_CPUTIME_ID || + clock_id == KERNEL_CLOCK_VIRTUAL || clock_id == KERNEL_CLOCK_PROF) { + const auto frequency = KernelGetTscFrequencyNative(); + tp->tv_sec = 0; + tp->tv_nsec = static_cast( + frequency != 0 ? std::max((1000000000 + (frequency >> 1)) / frequency, 1) + : 1); + return OK; + } + + clockid_t pclock_id = {}; + if (!GetPosixClockId(clock_id, &pclock_id)) { + LOGF("\tunknown clock_id: %d\n", clock_id); + return KERNEL_ERROR_EINVAL; + } + + timespec t {}; + + int result = clock_getres(pclock_id, &t); + + tp->tv_sec = t.tv_sec; + tp->tv_nsec = t.tv_nsec; + + if (result == 0) { + return OK; + } + return KERNEL_ERROR_EINVAL; +} + +int KYTY_SYSV_ABI KernelClockGettime(KernelClockid clock_id, KernelTimespec* tp) { + // Called constantly by Python frame/timer code. + + if (tp == nullptr) { + return KERNEL_ERROR_EFAULT; + } + + int special_error = OK; + if (KernelClockGettimeSpecial(clock_id, tp, &special_error)) { + return special_error; + } + + clockid_t pclock_id = {}; + if (!GetPosixClockId(clock_id, &pclock_id)) { + LOGF("\tunknown clock_id: %d\n", clock_id); + return KERNEL_ERROR_EINVAL; + } + + timespec t {}; + + int result = clock_gettime(pclock_id, &t); + + tp->tv_sec = t.tv_sec; + tp->tv_nsec = t.tv_nsec; + + if (result == 0) { + return OK; + } + return KERNEL_ERROR_EINVAL; +} + +int KYTY_SYSV_ABI KernelGettimeofday(KernelTimeval* tp) { + // PRINT_NAME(); + + if (tp == nullptr) { + return KERNEL_ERROR_EFAULT; + } + + int result = 0; +#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS + FILETIME ft {}; + GetSystemTimePreciseAsFileTime(&ft); + uint64_t ticks = ft.dwHighDateTime; + ticks <<= 32; + ticks |= ft.dwLowDateTime; + ticks /= 10; + ticks -= 11644473600000000ULL; + tp->tv_sec = static_cast(ticks / 1000000); + tp->tv_usec = static_cast(ticks % 1000000); +#else + auto dt = Common::DateTime::FromSystemUTC(); + sec_to_timeval(tp, dt.ToUnix()); +#endif + + if (result == 0) { + return OK; + } + return KERNEL_ERROR_EINVAL; +} + +int KYTY_SYSV_ABI KernelGettimezone(KernelTimezone* tz) { + // Hot path in Unity's time/date code. + + if (tz == nullptr) { + return KERNEL_ERROR_EFAULT; + } + +#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS + TIME_ZONE_INFORMATION tzi {}; + const DWORD result = GetTimeZoneInformation(&tzi); + + tz->tz_minuteswest = tzi.Bias; + tz->tz_dsttime = (result == TIME_ZONE_ID_UNKNOWN ? DST_NONE : DST_MET); +#else + const std::time_t now = std::time(nullptr); + std::tm local_tm {}; + std::tm utc_tm {}; + localtime_r(&now, &local_tm); + gmtime_r(&now, &utc_tm); + const auto local = std::mktime(&local_tm); + const auto utc = std::mktime(&utc_tm); + + tz->tz_minuteswest = static_cast((utc - local) / 60); + tz->tz_dsttime = (local_tm.tm_isdst > 0 ? DST_MET : DST_NONE); +#endif + + return OK; +} + +int KYTY_SYSV_ABI KernelConvertLocaltimeToUtc(int64_t local_time, int64_t /*reserved*/, + int64_t* utc_time, KernelTimezone* timezone, + int32_t* dst_seconds) { + // Hot path in Unity's time/date code. + + if (timezone == nullptr) { + return KERNEL_ERROR_EINVAL; + } + + KernelTimezone local_timezone {}; + auto ret = KernelGettimezone(&local_timezone); + if (ret != OK) { + return ret; + } + + const auto dst_sec = GetDstSeconds(); + *timezone = local_timezone; + + if (utc_time != nullptr) { + *utc_time = local_time + static_cast(local_timezone.tz_minuteswest) * 60 - dst_sec; + } + if (dst_seconds != nullptr) { + *dst_seconds = dst_sec; + } + + return OK; +} + +int KYTY_SYSV_ABI KernelConvertUtcToLocaltime(int64_t utc_time, int64_t* local_time, + KernelTimesec* st, uint64_t* dst_sec) { + // Hot path in Unity's time/date code. + + KernelTimezone tz {}; + auto ret = KernelGettimezone(&tz); + if (ret != OK) { + return ret; + } + + const auto dst = static_cast(GetDstSeconds()); + const auto west = static_cast(tz.tz_minuteswest) * 60; + if (local_time != nullptr) { + *local_time = utc_time - west + dst; + } + if (st != nullptr) { + st->t = utc_time; + st->west_sec = static_cast(-west); + st->dst_sec = static_cast(dst); + } + if (dst_sec != nullptr) { + *dst_sec = static_cast(dst); + } + + return OK; +} + +uint64_t KYTY_SYSV_ABI KernelGetTscFrequency() { + return KernelGetTscFrequencyNative(); +} + +uint64_t KYTY_SYSV_ABI KernelReadTsc() { + return KernelReadTscNative(); +} + +uint64_t KYTY_SYSV_ABI KernelGetProcessTime() { + const auto frequency = KernelGetTscFrequencyNative(); + if (frequency == 0) { + return static_cast(Loader::Timer::GetTimeMs() * 1000.0); + } + + const auto elapsed = KernelGetElapsedTsc(); + return static_cast((static_cast(elapsed) * 1000000.0L) / + static_cast(frequency)); +} + +uint64_t KYTY_SYSV_ABI KernelGetProcessTimeCounter() { + return KernelGetElapsedTsc(); +} + +uint64_t KYTY_SYSV_ABI KernelGetProcessTimeCounterFrequency() { + return KernelGetTscFrequencyNative(); +} + +void KYTY_SYSV_ABI KernelSetThreadDtors(thread_dtors_func_t dtors) { + PRINT_NAME(); + + // EXIT_NOT_IMPLEMENTED(!Common::Thread::IsMainThread()); + EXIT_NOT_IMPLEMENTED(g_pthread_context->GetThreadDtors() != nullptr); + + g_pthread_context->SetThreadDtors(dtors); + // g_thread_dtors = dtors; +} + +int KYTY_SYSV_ABI KernelUsleep(KernelUseconds microseconds) { + Common::Timer t; + t.Start(); + SleepMicroWithSignalPoll(microseconds); + // double ts = t.GetTimeS(); + // LOGF("\tactual: %g microseconds\n", ts * 1000000.0); + return OK; +} + +unsigned int KYTY_SYSV_ABI KernelSleep(unsigned int seconds) { + PRINT_NAME(); + LOGF("\tsleep: %u\n", seconds); + Common::Timer t; + t.Start(); + SleepMicroWithSignalPoll(static_cast(seconds) * 1000000ull); + double ts = t.GetTimeS(); + LOGF("\tactual: %g seconds\n", ts); + return OK; +} + +int KYTY_SYSV_ABI KernelNanosleep(const KernelTimespec* rqtp, KernelTimespec* rmtp) { + PRINT_NAME(); + + if (rqtp == nullptr) { + return KERNEL_ERROR_EFAULT; + } + + if (rqtp->tv_sec < 0 || rqtp->tv_nsec < 0 || rqtp->tv_nsec >= 1000000000) { + return KERNEL_ERROR_EINVAL; + } + + if (static_cast(rqtp->tv_sec) > + (UINT64_MAX - static_cast(rqtp->tv_nsec)) / 1000000000ull) { + return KERNEL_ERROR_EINVAL; + } + + uint64_t nanos = + static_cast(rqtp->tv_sec) * 1000000000ull + static_cast(rqtp->tv_nsec); + + LOGF("\tnanosleep: %" PRIu64 "\n", nanos); + + Common::Timer t; + t.Start(); + SleepNanoWithSignalPoll(nanos); + double ts = t.GetTimeS(); + LOGF("\tactual: %g nanoseconds\n", ts * 1000000000.0); + + if (rmtp != nullptr) { + rmtp->tv_sec = 0; + rmtp->tv_nsec = 0; + } + + return OK; +} + +int KYTY_SYSV_ABI PthreadKeyCreate(PthreadKey* key, pthread_key_destructor_func_t destructor) { + PRINT_NAME(); + + if (key == nullptr) { + return KERNEL_ERROR_EINVAL; + } + + if (!g_pthread_context->GetPthreadKeys()->Create(key, destructor)) { + return KERNEL_ERROR_EAGAIN; + } + + LOGF("\t destructor = %016" PRIx64 "\n" + "\t key = %d\n", + reinterpret_cast(destructor), *key); + + return OK; +} + +int KYTY_SYSV_ABI PthreadKeyDelete(PthreadKey key) { + PRINT_NAME(); + + LOGF("\t key = %d\n", key); + + if (!g_pthread_context->GetPthreadKeys()->Delete(key)) { + return KERNEL_ERROR_EINVAL; + } + + return OK; +} + +int KYTY_SYSV_ABI PthreadSetspecific(PthreadKey key, void* value) { + // PRINT_NAME(); + + int thread_id = Common::Thread::GetThreadIdUnique(); + + LOGF("\t key = %d\n" + "\t thread_id = %d\n" + "\t value = %016" PRIx64 "\n", + key, thread_id, reinterpret_cast(value)); + + if (!g_pthread_context->GetPthreadKeys()->Set(key, thread_id, value)) { + return KERNEL_ERROR_EINVAL; + } + + return OK; +} + +void* KYTY_SYSV_ABI PthreadGetspecific(PthreadKey key) { + int thread_id = Common::Thread::GetThreadIdUnique(); + + void* value = nullptr; + + if (!g_pthread_context->GetPthreadKeys()->Get(key, thread_id, &value)) { + return nullptr; + } + + return value; +} + +} // namespace LibKernel + +namespace Posix { + +LIB_NAME("Posix", "libkernel"); + +struct PthreadOnce { + int state; + LibKernel::PthreadMutex mutex; +}; + +static_assert(sizeof(PthreadOnce) == 0x10); + +constexpr int PTHREAD_ONCE_NEEDS_INIT = 0; +constexpr int PTHREAD_ONCE_DONE_INIT = 1; +constexpr int PTHREAD_ONCE_IN_PROGRESS = 2; +constexpr int PTHREAD_ONCE_WAIT = 3; + +static std::mutex g_pthread_once_mutex; +static std::condition_variable g_pthread_once_cv; + +int KYTY_SYSV_ABI pthread_once(void* once_control, void(KYTY_SYSV_ABI* init_routine)()) { + PRINT_NAME(); + + if (once_control == nullptr || init_routine == nullptr) { + return POSIX_EINVAL; + } + + auto* once = static_cast(once_control); + + { + std::unique_lock lock(g_pthread_once_mutex); + + for (;;) { + switch (once->state) { + case PTHREAD_ONCE_DONE_INIT: return OK; + case PTHREAD_ONCE_NEEDS_INIT: once->state = PTHREAD_ONCE_IN_PROGRESS; goto run_init; + case PTHREAD_ONCE_IN_PROGRESS: + case PTHREAD_ONCE_WAIT: + once->state = PTHREAD_ONCE_WAIT; + while (once->state == PTHREAD_ONCE_WAIT) { + g_pthread_once_cv.wait_for( + lock, std::chrono::microseconds(LibKernel::SIGNAL_APC_POLL_MICROS)); + if (once->state == PTHREAD_ONCE_WAIT) { + lock.unlock(); + LibKernel::KernelDispatchPendingSignalForCurrentThread(); + lock.lock(); + } + } + break; + default: return POSIX_EINVAL; + } + } + } + +run_init: + init_routine(); + + { + std::lock_guard lock(g_pthread_once_mutex); + once->state = PTHREAD_ONCE_DONE_INIT; + } + g_pthread_once_cv.notify_all(); + + return OK; +} + +int KYTY_SYSV_ABI pthread_create(LibKernel::Pthread* thread, const LibKernel::PthreadAttr* attr, + LibKernel::pthread_entry_func_t entry, void* arg) { + PRINT_NAME(); + + return POSIX_PTHREAD_CALL(LibKernel::PthreadCreate(thread, attr, entry, arg, "")); +} + +int KYTY_SYSV_ABI pthread_create_name_np(LibKernel::Pthread* thread, + const LibKernel::PthreadAttr* attr, + LibKernel::pthread_entry_func_t entry, void* arg, + const char* name) { + PRINT_NAME(); + + return POSIX_PTHREAD_CALL( + LibKernel::PthreadCreate(thread, attr, entry, arg, (name != nullptr ? name : ""))); +} + +int KYTY_SYSV_ABI pthread_detach(LibKernel::Pthread thread) { + PRINT_NAME(); + + return POSIX_PTHREAD_CALL(LibKernel::PthreadDetach(thread)); +} + +void KYTY_SYSV_ABI pthread_exit(void* value) { + PRINT_NAME(); + + LibKernel::PthreadExit(value); +} + +LibKernel::Pthread KYTY_SYSV_ABI pthread_self() { + PRINT_NAME(); + + return LibKernel::PthreadSelf(); +} + +int KYTY_SYSV_ABI pthread_rename_np(LibKernel::Pthread thread, const char* name) { + PRINT_NAME(); + + return POSIX_PTHREAD_CALL(LibKernel::PthreadRename(thread, name)); +} + +int KYTY_SYSV_ABI pthread_setcancelstate(int state, int* old_state) { + PRINT_NAME(); + + return POSIX_PTHREAD_CALL(LibKernel::PthreadSetcancelstate(state, old_state)); +} + +int KYTY_SYSV_ABI pthread_setprio(LibKernel::Pthread thread, int prio) { + PRINT_NAME(); + + return POSIX_PTHREAD_CALL(LibKernel::PthreadSetprio(thread, prio)); +} + +int KYTY_SYSV_ABI pthread_getschedparam(LibKernel::Pthread thread, int* policy, + LibKernel::KernelSchedParam* param) { + PRINT_NAME(); + + if (thread == nullptr || policy == nullptr || param == nullptr) { + return POSIX_EINVAL; + } + + int result = LibKernel::PthreadAttrGetschedpolicy(&thread->attr, policy); + result = (result == OK ? LibKernel::PthreadAttrGetschedparam(&thread->attr, param) : result); + + return (result == OK ? 0 : LibKernel::KernelToPosix(result)); +} + +int KYTY_SYSV_ABI pthread_setschedparam(LibKernel::Pthread thread, int policy, + const LibKernel::KernelSchedParam* param) { + PRINT_NAME(); + + if (thread == nullptr || param == nullptr) { + return POSIX_EINVAL; + } + + int result = LibKernel::PthreadAttrSetschedpolicy(&thread->attr, policy); + result = (result == OK ? LibKernel::PthreadAttrSetschedparam(&thread->attr, param) : result); + + return (result == OK ? 0 : LibKernel::KernelToPosix(result)); +} + +void KYTY_SYSV_ABI pthread_yield() { + PRINT_NAME(); + + LibKernel::PthreadYield(); +} + +int KYTY_SYSV_ABI sched_get_priority_max(int policy) { + PRINT_NAME(); + + LOGF("\t policy = %d\n", policy); + + return 256; +} + +int KYTY_SYSV_ABI sched_get_priority_min(int policy) { + PRINT_NAME(); + + LOGF("\t policy = %d\n", policy); + + return 767; +} + +int KYTY_SYSV_ABI pthread_join(LibKernel::Pthread thread, void** value) { + PRINT_NAME(); + + return POSIX_PTHREAD_CALL(LibKernel::PthreadJoin(thread, value)); +} + +int KYTY_SYSV_ABI pthread_attr_init(LibKernel::PthreadAttr* attr) { + // PRINT_NAME(); + + return POSIX_PTHREAD_CALL(LibKernel::PthreadAttrInit(attr)); +} + +int KYTY_SYSV_ABI pthread_attr_destroy(LibKernel::PthreadAttr* attr) { + // PRINT_NAME(); + + return POSIX_PTHREAD_CALL(LibKernel::PthreadAttrDestroy(attr)); +} + +int KYTY_SYSV_ABI pthread_attr_get_np(LibKernel::Pthread thread, LibKernel::PthreadAttr* attr) { + // PRINT_NAME(); + + return POSIX_PTHREAD_CALL(LibKernel::PthreadAttrGet(thread, attr)); +} + +int KYTY_SYSV_ABI pthread_attr_getdetachstate(const LibKernel::PthreadAttr* attr, int* state) { + // PRINT_NAME(); + + return POSIX_PTHREAD_CALL(LibKernel::PthreadAttrGetdetachstate(attr, state)); +} + +int KYTY_SYSV_ABI pthread_attr_getguardsize(const LibKernel::PthreadAttr* attr, + size_t* guard_size) { + // PRINT_NAME(); + + return POSIX_PTHREAD_CALL(LibKernel::PthreadAttrGetguardsize(attr, guard_size)); +} + +int KYTY_SYSV_ABI pthread_attr_getinheritsched(const LibKernel::PthreadAttr* attr, + int* inherit_sched) { + // PRINT_NAME(); + + return POSIX_PTHREAD_CALL(LibKernel::PthreadAttrGetinheritsched(attr, inherit_sched)); +} + +int KYTY_SYSV_ABI pthread_attr_getschedparam(const LibKernel::PthreadAttr* attr, + LibKernel::KernelSchedParam* param) { + // PRINT_NAME(); + + return POSIX_PTHREAD_CALL(LibKernel::PthreadAttrGetschedparam(attr, param)); +} + +int KYTY_SYSV_ABI pthread_attr_getschedpolicy(const LibKernel::PthreadAttr* attr, int* policy) { + // PRINT_NAME(); + + return POSIX_PTHREAD_CALL(LibKernel::PthreadAttrGetschedpolicy(attr, policy)); +} + +int KYTY_SYSV_ABI pthread_attr_getstack(const LibKernel::PthreadAttr* __restrict attr, + void** __restrict stack_addr, + size_t* __restrict stack_size) { + // PRINT_NAME(); + + return POSIX_PTHREAD_CALL(LibKernel::PthreadAttrGetstack(attr, stack_addr, stack_size)); +} + +int KYTY_SYSV_ABI pthread_attr_getstacksize(const LibKernel::PthreadAttr* attr, + size_t* stack_size) { + // PRINT_NAME(); + + return POSIX_PTHREAD_CALL(LibKernel::PthreadAttrGetstacksize(attr, stack_size)); +} + +int KYTY_SYSV_ABI pthread_attr_setdetachstate(LibKernel::PthreadAttr* attr, int state) { + // PRINT_NAME(); + + return POSIX_PTHREAD_CALL(LibKernel::PthreadAttrSetdetachstate(attr, state)); +} + +int KYTY_SYSV_ABI pthread_attr_setguardsize(LibKernel::PthreadAttr* attr, size_t guard_size) { + // PRINT_NAME(); + + return POSIX_PTHREAD_CALL(LibKernel::PthreadAttrSetguardsize(attr, guard_size)); +} + +int KYTY_SYSV_ABI pthread_attr_setinheritsched(LibKernel::PthreadAttr* attr, int inherit_sched) { + // PRINT_NAME(); + + return POSIX_PTHREAD_CALL(LibKernel::PthreadAttrSetinheritsched(attr, inherit_sched)); +} + +int KYTY_SYSV_ABI pthread_attr_setschedparam(LibKernel::PthreadAttr* attr, + const LibKernel::KernelSchedParam* param) { + // PRINT_NAME(); + + return POSIX_PTHREAD_CALL(LibKernel::PthreadAttrSetschedparam(attr, param)); +} + +int KYTY_SYSV_ABI pthread_attr_setschedpolicy(LibKernel::PthreadAttr* attr, int policy) { + // PRINT_NAME(); + + return POSIX_PTHREAD_CALL(LibKernel::PthreadAttrSetschedpolicy(attr, policy)); +} + +int KYTY_SYSV_ABI pthread_attr_setstacksize(LibKernel::PthreadAttr* attr, size_t stack_size) { + // PRINT_NAME(); + + return POSIX_PTHREAD_CALL(LibKernel::PthreadAttrSetstacksize(attr, stack_size)); +} + +int KYTY_SYSV_ABI pthread_cond_broadcast(LibKernel::PthreadCond* cond) { + // PRINT_NAME(); + + return POSIX_PTHREAD_CALL(LibKernel::PthreadCondBroadcast(cond)); +} + +int KYTY_SYSV_ABI pthread_cond_signal(LibKernel::PthreadCond* cond) { + // PRINT_NAME(); + + return POSIX_PTHREAD_CALL(LibKernel::PthreadCondSignal(cond)); +} + +int KYTY_SYSV_ABI pthread_condattr_setclock(LibKernel::PthreadCondattr* attr, + LibKernel::KernelClockid clock_id) { + PRINT_NAME(); + + return POSIX_PTHREAD_CALL(LibKernel::PthreadCondattrSetclock(attr, clock_id)); +} + +int KYTY_SYSV_ABI pthread_condattr_init(LibKernel::PthreadCondattr* attr) { + PRINT_NAME(); + + return POSIX_PTHREAD_CALL(LibKernel::PthreadCondattrInit(attr)); +} + +int KYTY_SYSV_ABI pthread_condattr_destroy(LibKernel::PthreadCondattr* attr) { + PRINT_NAME(); + + return POSIX_PTHREAD_CALL(LibKernel::PthreadCondattrDestroy(attr)); +} + +int KYTY_SYSV_ABI pthread_cond_wait(LibKernel::PthreadCond* cond, LibKernel::PthreadMutex* mutex) { + // PRINT_NAME(); + + return POSIX_PTHREAD_CALL(LibKernel::PthreadCondWait(cond, mutex)); +} + +int KYTY_SYSV_ABI pthread_cond_timedwait(LibKernel::PthreadCond* cond, + LibKernel::PthreadMutex* mutex, + const LibKernel::KernelTimespec* abstime) { + PRINT_NAME(); + + return POSIX_PTHREAD_CALL(LibKernel::PthreadCondTimedwaitAbs(cond, mutex, abstime)); +} + +int KYTY_SYSV_ABI pthread_mutex_lock(LibKernel::PthreadMutex* mutex) { + // PRINT_NAME(); + + return POSIX_PTHREAD_CALL(LibKernel::PthreadMutexLock(mutex)); +} + +int KYTY_SYSV_ABI pthread_mutex_trylock(LibKernel::PthreadMutex* mutex) { + // PRINT_NAME(); + + return POSIX_PTHREAD_CALL(LibKernel::PthreadMutexTrylock(mutex)); +} + +int KYTY_SYSV_ABI pthread_mutex_timedlock(LibKernel::PthreadMutex* mutex, + const LibKernel::KernelTimespec* abstime) { + PRINT_NAME(); + + if (abstime == nullptr || abstime->tv_sec < 0 || abstime->tv_nsec < 0 || + abstime->tv_nsec >= 1000000000) { + return POSIX_PTHREAD_CALL(LibKernel::KERNEL_ERROR_EINVAL); + } + + LibKernel::KernelTimespec now {}; + if (LibKernel::KernelClockGettime(0, &now) != OK) { + return POSIX_PTHREAD_CALL(LibKernel::KERNEL_ERROR_EINVAL); + } + + int64_t sec_delta = abstime->tv_sec - now.tv_sec; + int64_t nsec_delta = abstime->tv_nsec - now.tv_nsec; + if (nsec_delta < 0) { + sec_delta--; + nsec_delta += 1000000000; + } + + uint64_t usec = 0; + if (sec_delta > 0 || (sec_delta == 0 && nsec_delta > 0)) { + usec = (static_cast(sec_delta) > UINT32_MAX / 1000000ull + ? UINT32_MAX + : static_cast(sec_delta) * 1000000ull); + usec = std::min(UINT32_MAX, usec + static_cast(nsec_delta / 1000)); + } + + return POSIX_PTHREAD_CALL( + LibKernel::PthreadMutexTimedlock(mutex, static_cast(usec))); +} + +int KYTY_SYSV_ABI pthread_mutex_unlock(LibKernel::PthreadMutex* mutex) { + // PRINT_NAME(); + + return POSIX_PTHREAD_CALL(LibKernel::PthreadMutexUnlock(mutex)); +} + +int KYTY_SYSV_ABI pthread_rwlock_rdlock(LibKernel::PthreadRwlock* rwlock) { + // Hot path for some PS5 titles; per-call name logging can dominate runtime. + + return POSIX_PTHREAD_CALL(LibKernel::PthreadRwlockRdlock(rwlock)); +} + +int KYTY_SYSV_ABI pthread_rwlock_unlock(LibKernel::PthreadRwlock* rwlock) { + // Hot path for some PS5 titles; per-call name logging can dominate runtime. + + return POSIX_PTHREAD_CALL(LibKernel::PthreadRwlockUnlock(rwlock)); +} + +int KYTY_SYSV_ABI pthread_rwlock_wrlock(LibKernel::PthreadRwlock* rwlock) { + // Hot path for some PS5 titles; per-call name logging can dominate runtime. + + return POSIX_PTHREAD_CALL(LibKernel::PthreadRwlockWrlock(rwlock)); +} + +int KYTY_SYSV_ABI pthread_rwlock_destroy(LibKernel::PthreadRwlock* rwlock) { + PRINT_NAME(); + + return POSIX_PTHREAD_CALL(LibKernel::PthreadRwlockDestroy(rwlock)); +} + +int KYTY_SYSV_ABI pthread_key_create(LibKernel::PthreadKey* key, + LibKernel::pthread_key_destructor_func_t destructor) { + PRINT_NAME(); + + return POSIX_PTHREAD_CALL(LibKernel::PthreadKeyCreate(key, destructor)); +} + +int KYTY_SYSV_ABI pthread_key_delete(LibKernel::PthreadKey key) { + PRINT_NAME(); + + return POSIX_PTHREAD_CALL(LibKernel::PthreadKeyDelete(key)); +} + +int KYTY_SYSV_ABI pthread_setspecific(LibKernel::PthreadKey key, void* value) { + PRINT_NAME(); + + return POSIX_PTHREAD_CALL(LibKernel::PthreadSetspecific(key, value)); +} + +void* KYTY_SYSV_ABI pthread_getspecific(LibKernel::PthreadKey key) { + PRINT_NAME(); + + return (LibKernel::PthreadGetspecific(key)); +} + +int KYTY_SYSV_ABI pthread_mutex_destroy(LibKernel::PthreadMutex* mutex) { + PRINT_NAME(); + + return POSIX_PTHREAD_CALL(LibKernel::PthreadMutexDestroy(mutex)); +} + +int KYTY_SYSV_ABI pthread_mutex_init(LibKernel::PthreadMutex* mutex, + const LibKernel::PthreadMutexattr* attr) { + PRINT_NAME(); + + return POSIX_PTHREAD_CALL(LibKernel::PthreadMutexInit(mutex, attr)); +} + +int KYTY_SYSV_ABI pthread_mutexattr_init(LibKernel::PthreadMutexattr* attr) { + PRINT_NAME(); + + return POSIX_PTHREAD_CALL(LibKernel::PthreadMutexattrInit(attr)); +} + +int KYTY_SYSV_ABI pthread_mutexattr_settype(LibKernel::PthreadMutexattr* attr, int type) { + PRINT_NAME(); + + return POSIX_PTHREAD_CALL(LibKernel::PthreadMutexattrSettype(attr, type)); +} + +int KYTY_SYSV_ABI pthread_mutexattr_setprotocol(LibKernel::PthreadMutexattr* attr, int protocol) { + PRINT_NAME(); + + return POSIX_PTHREAD_CALL(LibKernel::PthreadMutexattrSetprotocol(attr, protocol)); +} + +int KYTY_SYSV_ABI pthread_mutexattr_destroy(LibKernel::PthreadMutexattr* attr) { + PRINT_NAME(); + + return POSIX_PTHREAD_CALL(LibKernel::PthreadMutexattrDestroy(attr)); +} + +int KYTY_SYSV_ABI pthread_getstack(const LibKernel::PthreadAttr* __restrict attr, + void** __restrict stack_addr, size_t* __restrict stack_size) { + PRINT_NAME(); + + return pthread_attr_getstack(attr, stack_addr, stack_size); +} + +} // namespace Posix + +} // namespace Libs + +#if KYTY_PLATFORM == KYTY_PLATFORM_LINUX +#pragma GCC diagnostic pop +#endif diff --git a/src/kernel/pthread.h b/src/kernel/pthread.h new file mode 100644 index 000000000..8b1942ef6 --- /dev/null +++ b/src/kernel/pthread.h @@ -0,0 +1,283 @@ +#ifndef EMULATOR_INCLUDE_EMULATOR_KERNEL_PTHREAD_H_ +#define EMULATOR_INCLUDE_EMULATOR_KERNEL_PTHREAD_H_ + +#include "common/abi.h" +#include "common/common.h" +#include "common/subsystems.h" + +// IWYU pragma: no_include + +extern "C" { +struct sched_param; +} + +namespace Loader { +struct Program; +} // namespace Loader + +namespace Libs { + +namespace LibKernel { + +KYTY_SUBSYSTEM_DEFINE(Pthread); + +struct PthreadAttrPrivate; +struct PthreadPrivate; +struct PthreadMutexPrivate; +struct PthreadMutexattrPrivate; +struct PthreadRwlockPrivate; +struct PthreadRwlockattrPrivate; +struct PthreadCondattrPrivate; +struct PthreadCondPrivate; + +struct KernelTimespec { + int64_t tv_sec; + int64_t tv_nsec; +}; + +struct KernelTimeval { + int64_t tv_sec; + int64_t tv_usec; +}; + +struct KernelTimezone { + int32_t tz_minuteswest; + int32_t tz_dsttime; +}; + +struct KernelTimesec { + int64_t t; + uint32_t west_sec; + uint32_t dst_sec; +}; + +using PthreadAttr = PthreadAttrPrivate*; +using Pthread = PthreadPrivate*; +using KernelCpumask = uint64_t; +using PthreadMutex = PthreadMutexPrivate*; +using PthreadMutexattr = PthreadMutexattrPrivate*; +using KernelSchedParam = struct sched_param; +using PthreadRwlock = PthreadRwlockPrivate*; +using PthreadRwlockattr = PthreadRwlockattrPrivate*; +using KernelUseconds = unsigned int; +using PthreadCondattr = PthreadCondattrPrivate*; +using PthreadCond = PthreadCondPrivate*; +using KernelClockid = int32_t; +using PthreadKey = int; + +using pthread_entry_func_t = KYTY_SYSV_ABI void* (*)(void*); +using thread_dtors_func_t = KYTY_SYSV_ABI void (*)(); +using pthread_key_destructor_func_t = KYTY_SYSV_ABI void (*)(void*); + +void PthreadInitSelfForMainThread(); +void* PthreadCreateMainGuestStack(); +void PthreadDeleteStaticObjects(Loader::Program* program); + +int KYTY_SYSV_ABI PthreadMutexattrInit(PthreadMutexattr* attr); +int KYTY_SYSV_ABI PthreadMutexattrDestroy(PthreadMutexattr* attr); +int KYTY_SYSV_ABI PthreadMutexattrSettype(PthreadMutexattr* attr, int type); +int KYTY_SYSV_ABI PthreadMutexattrSetprotocol(PthreadMutexattr* attr, int protocol); +int KYTY_SYSV_ABI PthreadMutexInit(PthreadMutex* mutex, const PthreadMutexattr* attr); +int KYTY_SYSV_ABI PthreadMutexDestroy(PthreadMutex* mutex); +int KYTY_SYSV_ABI PthreadMutexLock(PthreadMutex* mutex); +int KYTY_SYSV_ABI PthreadMutexTrylock(PthreadMutex* mutex); +int KYTY_SYSV_ABI PthreadMutexTimedlock(PthreadMutex* mutex, KernelUseconds usec); +int KYTY_SYSV_ABI PthreadMutexUnlock(PthreadMutex* mutex); + +Pthread KYTY_SYSV_ABI PthreadSelf(); +Pthread PthreadSelfOrNull(); +Pthread PthreadSwapSelfForSignal(Pthread thread); +int KYTY_SYSV_ABI PthreadCreate(Pthread* thread, const PthreadAttr* attr, + pthread_entry_func_t entry, void* arg, const char* name); +int KYTY_SYSV_ABI PthreadDetach(Pthread thread); +int KYTY_SYSV_ABI PthreadJoin(Pthread thread, void** value); +int KYTY_SYSV_ABI PthreadCancel(Pthread thread); +int KYTY_SYSV_ABI PthreadSetcancelstate(int state, int* old_state); +int KYTY_SYSV_ABI PthreadSetcanceltype(int type, int* old_type); +int KYTY_SYSV_ABI PthreadGetprio(Pthread thread, int* prio); +int KYTY_SYSV_ABI PthreadSetprio(Pthread thread, int prio); +void KYTY_SYSV_ABI PthreadTestcancel(); +int KYTY_SYSV_ABI PthreadSetaffinity(Pthread thread, KernelCpumask mask); +void KYTY_SYSV_ABI PthreadExit(void* value); +int KYTY_SYSV_ABI PthreadEqual(Pthread thread1, Pthread thread2); +int KYTY_SYSV_ABI PthreadGetname(Pthread thread, char* name); +int KYTY_SYSV_ABI PthreadRename(Pthread thread, const char* name); +void KYTY_SYSV_ABI PthreadYield(); +int KYTY_SYSV_ABI PthreadGetthreadid(); +int KYTY_SYSV_ABI PthreadGetaffinity(Pthread thread, KernelCpumask* mask); +int PthreadGetUniqueId(Pthread thread); +uint64_t PthreadGetHostThreadId(Pthread thread); +void PthreadWakeForSignal(Pthread thread); +void PthreadQueuePendingSignal(Pthread thread, int signum); +bool PthreadHasPendingSignal(Pthread thread, int signum); +bool PthreadTakePendingSignal(Pthread thread, int signum); +bool PthreadGetGuestStack(Pthread thread, uint64_t* stack_addr, uint64_t* stack_size); +int PthreadGetPriorityForKernel(Pthread thread); +int PthreadGetCurrentPriorityForKernel(); + +int KYTY_SYSV_ABI KernelUsleep(KernelUseconds microseconds); +unsigned int KYTY_SYSV_ABI KernelSleep(unsigned int seconds); +int KYTY_SYSV_ABI KernelNanosleep(const KernelTimespec* rqtp, KernelTimespec* rmtp); + +int KYTY_SYSV_ABI PthreadKeyCreate(PthreadKey* key, pthread_key_destructor_func_t destructor); +int KYTY_SYSV_ABI PthreadKeyDelete(PthreadKey key); +int KYTY_SYSV_ABI PthreadSetspecific(PthreadKey key, void* value); +void* KYTY_SYSV_ABI PthreadGetspecific(PthreadKey key); + +void KYTY_SYSV_ABI KernelSetThreadDtors(thread_dtors_func_t dtors); + +int KYTY_SYSV_ABI PthreadAttrInit(PthreadAttr* attr); +int KYTY_SYSV_ABI PthreadAttrDestroy(PthreadAttr* attr); +int KYTY_SYSV_ABI PthreadAttrGet(Pthread thread, PthreadAttr* attr); +int KYTY_SYSV_ABI PthreadAttrGetaffinity(const PthreadAttr* attr, KernelCpumask* mask); +int KYTY_SYSV_ABI PthreadAttrGetdetachstate(const PthreadAttr* attr, int* state); +int KYTY_SYSV_ABI PthreadAttrGetguardsize(const PthreadAttr* attr, size_t* guard_size); +int KYTY_SYSV_ABI PthreadAttrGetinheritsched(const PthreadAttr* attr, int* inherit_sched); +int KYTY_SYSV_ABI PthreadAttrGetschedparam(const PthreadAttr* attr, KernelSchedParam* param); +int KYTY_SYSV_ABI PthreadAttrGetschedpolicy(const PthreadAttr* attr, int* policy); +int KYTY_SYSV_ABI PthreadAttrGetsolosched(const PthreadAttr* attr, int* solosched); +int KYTY_SYSV_ABI PthreadAttrGetstack(const PthreadAttr* __restrict attr, + void** __restrict stack_addr, size_t* __restrict stack_size); +int KYTY_SYSV_ABI PthreadAttrGetstackaddr(const PthreadAttr* attr, void** stack_addr); +int KYTY_SYSV_ABI PthreadAttrGetstacksize(const PthreadAttr* attr, size_t* stack_size); +int KYTY_SYSV_ABI PthreadAttrSetaffinity(PthreadAttr* attr, KernelCpumask mask); +int KYTY_SYSV_ABI PthreadAttrSetdetachstate(PthreadAttr* attr, int state); +int KYTY_SYSV_ABI PthreadAttrSetguardsize(PthreadAttr* attr, size_t guard_size); +int KYTY_SYSV_ABI PthreadAttrSetinheritsched(PthreadAttr* attr, int inherit_sched); +int KYTY_SYSV_ABI PthreadAttrSetschedparam(PthreadAttr* attr, const KernelSchedParam* param); +int KYTY_SYSV_ABI PthreadAttrSetschedpolicy(PthreadAttr* attr, int policy); +int KYTY_SYSV_ABI PthreadAttrSetsolosched(PthreadAttr* attr, int solosched); +int KYTY_SYSV_ABI PthreadAttrSetstack(PthreadAttr* attr, void* addr, size_t size); +int KYTY_SYSV_ABI PthreadAttrSetstackaddr(PthreadAttr* attr, void* addr); +int KYTY_SYSV_ABI PthreadAttrSetstacksize(PthreadAttr* attr, size_t stack_size); + +int KYTY_SYSV_ABI PthreadRwlockDestroy(PthreadRwlock* rwlock); +int KYTY_SYSV_ABI PthreadRwlockInit(PthreadRwlock* rwlock, const PthreadRwlockattr* attr); +int KYTY_SYSV_ABI PthreadRwlockRdlock(PthreadRwlock* rwlock); +int KYTY_SYSV_ABI PthreadRwlockTimedrdlock(PthreadRwlock* rwlock, KernelUseconds usec); +int KYTY_SYSV_ABI PthreadRwlockTimedwrlock(PthreadRwlock* rwlock, KernelUseconds usec); +int KYTY_SYSV_ABI PthreadRwlockTryrdlock(PthreadRwlock* rwlock); +int KYTY_SYSV_ABI PthreadRwlockTrywrlock(PthreadRwlock* rwlock); +int KYTY_SYSV_ABI PthreadRwlockUnlock(PthreadRwlock* rwlock); +int KYTY_SYSV_ABI PthreadRwlockWrlock(PthreadRwlock* rwlock); +int KYTY_SYSV_ABI PthreadRwlockattrDestroy(PthreadRwlockattr* attr); +int KYTY_SYSV_ABI PthreadRwlockattrInit(PthreadRwlockattr* attr); +int KYTY_SYSV_ABI PthreadRwlockattrGettype(PthreadRwlockattr* attr, int* type); +int KYTY_SYSV_ABI PthreadRwlockattrSettype(PthreadRwlockattr* attr, int type); + +int KYTY_SYSV_ABI PthreadCondattrDestroy(PthreadCondattr* attr); +int KYTY_SYSV_ABI PthreadCondattrInit(PthreadCondattr* attr); +int KYTY_SYSV_ABI PthreadCondattrSetclock(PthreadCondattr* attr, KernelClockid clock_id); +int KYTY_SYSV_ABI PthreadCondBroadcast(PthreadCond* cond); +int KYTY_SYSV_ABI PthreadCondDestroy(PthreadCond* cond); +int KYTY_SYSV_ABI PthreadCondInit(PthreadCond* cond, const PthreadCondattr* attr); +int KYTY_SYSV_ABI PthreadCondSignal(PthreadCond* cond); +int KYTY_SYSV_ABI PthreadCondSignalto(PthreadCond* cond, Pthread thread); +int KYTY_SYSV_ABI PthreadCondTimedwait(PthreadCond* cond, PthreadMutex* mutex, KernelUseconds usec); +int KYTY_SYSV_ABI PthreadCondTimedwaitAbs(PthreadCond* cond, PthreadMutex* mutex, + const KernelTimespec* abstime); +int KYTY_SYSV_ABI PthreadCondWait(PthreadCond* cond, PthreadMutex* mutex); + +int KYTY_SYSV_ABI KernelClockGetres(KernelClockid clock_id, KernelTimespec* tp); +int KYTY_SYSV_ABI KernelClockGettime(KernelClockid clock_id, KernelTimespec* tp); +int KYTY_SYSV_ABI KernelGettimeofday(KernelTimeval* tp); +int KYTY_SYSV_ABI KernelGettimezone(KernelTimezone* tz); +int KYTY_SYSV_ABI KernelConvertLocaltimeToUtc(int64_t local_time, int64_t reserved, + int64_t* utc_time, KernelTimezone* timezone, + int32_t* dst_seconds); +int KYTY_SYSV_ABI KernelConvertUtcToLocaltime(int64_t utc_time, int64_t* local_time, + KernelTimesec* st, uint64_t* dst_sec); +uint64_t KYTY_SYSV_ABI KernelGetTscFrequency(); +uint64_t KYTY_SYSV_ABI KernelReadTsc(); +uint64_t KYTY_SYSV_ABI KernelGetProcessTime(); +uint64_t KYTY_SYSV_ABI KernelGetProcessTimeCounter(); +uint64_t KYTY_SYSV_ABI KernelGetProcessTimeCounterFrequency(); + +} // namespace LibKernel + +namespace Posix { + +int KYTY_SYSV_ABI pthread_create(LibKernel::Pthread* thread, const LibKernel::PthreadAttr* attr, + LibKernel::pthread_entry_func_t entry, void* arg); +int KYTY_SYSV_ABI pthread_create_name_np(LibKernel::Pthread* thread, + const LibKernel::PthreadAttr* attr, + LibKernel::pthread_entry_func_t entry, void* arg, + const char* name); +int KYTY_SYSV_ABI pthread_detach(LibKernel::Pthread thread); +void KYTY_SYSV_ABI pthread_exit(void* value); +LibKernel::Pthread KYTY_SYSV_ABI pthread_self(); +int KYTY_SYSV_ABI pthread_rename_np(LibKernel::Pthread thread, const char* name); +int KYTY_SYSV_ABI pthread_setcancelstate(int state, int* old_state); +int KYTY_SYSV_ABI pthread_setprio(LibKernel::Pthread thread, int prio); +int KYTY_SYSV_ABI pthread_getschedparam(LibKernel::Pthread thread, int* policy, + LibKernel::KernelSchedParam* param); +int KYTY_SYSV_ABI pthread_setschedparam(LibKernel::Pthread thread, int policy, + const LibKernel::KernelSchedParam* param); +void KYTY_SYSV_ABI pthread_yield(); +int KYTY_SYSV_ABI sched_get_priority_max(int policy); +int KYTY_SYSV_ABI sched_get_priority_min(int policy); +int KYTY_SYSV_ABI pthread_join(LibKernel::Pthread thread, void** value); +int KYTY_SYSV_ABI pthread_attr_init(LibKernel::PthreadAttr* attr); +int KYTY_SYSV_ABI pthread_attr_destroy(LibKernel::PthreadAttr* attr); +int KYTY_SYSV_ABI pthread_attr_get_np(LibKernel::Pthread thread, LibKernel::PthreadAttr* attr); +int KYTY_SYSV_ABI pthread_attr_getdetachstate(const LibKernel::PthreadAttr* attr, int* state); +int KYTY_SYSV_ABI pthread_attr_getguardsize(const LibKernel::PthreadAttr* attr, size_t* guard_size); +int KYTY_SYSV_ABI pthread_attr_getinheritsched(const LibKernel::PthreadAttr* attr, + int* inherit_sched); +int KYTY_SYSV_ABI pthread_attr_getschedparam(const LibKernel::PthreadAttr* attr, + LibKernel::KernelSchedParam* param); +int KYTY_SYSV_ABI pthread_attr_getschedpolicy(const LibKernel::PthreadAttr* attr, int* policy); +int KYTY_SYSV_ABI pthread_attr_getstack(const LibKernel::PthreadAttr* __restrict attr, + void** __restrict stack_addr, + size_t* __restrict stack_size); +int KYTY_SYSV_ABI pthread_attr_getstacksize(const LibKernel::PthreadAttr* attr, size_t* stack_size); +int KYTY_SYSV_ABI pthread_attr_setdetachstate(LibKernel::PthreadAttr* attr, int state); +int KYTY_SYSV_ABI pthread_attr_setguardsize(LibKernel::PthreadAttr* attr, size_t guard_size); +int KYTY_SYSV_ABI pthread_attr_setinheritsched(LibKernel::PthreadAttr* attr, int inherit_sched); +int KYTY_SYSV_ABI pthread_attr_setschedparam(LibKernel::PthreadAttr* attr, + const LibKernel::KernelSchedParam* param); +int KYTY_SYSV_ABI pthread_attr_setschedpolicy(LibKernel::PthreadAttr* attr, int policy); +int KYTY_SYSV_ABI pthread_attr_setstacksize(LibKernel::PthreadAttr* attr, size_t stack_size); +int KYTY_SYSV_ABI pthread_cond_broadcast(LibKernel::PthreadCond* cond); +int KYTY_SYSV_ABI pthread_cond_signal(LibKernel::PthreadCond* cond); +int KYTY_SYSV_ABI pthread_cond_init(LibKernel::PthreadCond* cond, + const LibKernel::PthreadCondattr* attr); +int KYTY_SYSV_ABI pthread_condattr_init(LibKernel::PthreadCondattr* attr); +int KYTY_SYSV_ABI pthread_condattr_destroy(LibKernel::PthreadCondattr* attr); +int KYTY_SYSV_ABI pthread_condattr_setclock(LibKernel::PthreadCondattr* attr, + LibKernel::KernelClockid clock_id); +int KYTY_SYSV_ABI pthread_cond_wait(LibKernel::PthreadCond* cond, LibKernel::PthreadMutex* mutex); +int KYTY_SYSV_ABI pthread_cond_timedwait(LibKernel::PthreadCond* cond, + LibKernel::PthreadMutex* mutex, + const LibKernel::KernelTimespec* abstime); +int KYTY_SYSV_ABI pthread_once(void* once_control, void(KYTY_SYSV_ABI* init_routine)()); +int KYTY_SYSV_ABI pthread_mutex_lock(LibKernel::PthreadMutex* mutex); +int KYTY_SYSV_ABI pthread_mutex_trylock(LibKernel::PthreadMutex* mutex); +int KYTY_SYSV_ABI pthread_mutex_timedlock(LibKernel::PthreadMutex* mutex, + const LibKernel::KernelTimespec* abstime); +int KYTY_SYSV_ABI pthread_mutex_unlock(LibKernel::PthreadMutex* mutex); +int KYTY_SYSV_ABI pthread_rwlock_rdlock(LibKernel::PthreadRwlock* rwlock); +int KYTY_SYSV_ABI pthread_rwlock_unlock(LibKernel::PthreadRwlock* rwlock); +int KYTY_SYSV_ABI pthread_rwlock_wrlock(LibKernel::PthreadRwlock* rwlock); +int KYTY_SYSV_ABI pthread_rwlock_destroy(LibKernel::PthreadRwlock* rwlock); +int KYTY_SYSV_ABI pthread_key_create(LibKernel::PthreadKey* key, + LibKernel::pthread_key_destructor_func_t destructor); +int KYTY_SYSV_ABI pthread_key_delete(LibKernel::PthreadKey key); +int KYTY_SYSV_ABI pthread_setspecific(LibKernel::PthreadKey key, void* value); +void* KYTY_SYSV_ABI pthread_getspecific(LibKernel::PthreadKey key); +int KYTY_SYSV_ABI pthread_mutex_destroy(LibKernel::PthreadMutex* mutex); +int KYTY_SYSV_ABI pthread_mutex_init(LibKernel::PthreadMutex* mutex, + const LibKernel::PthreadMutexattr* attr); +int KYTY_SYSV_ABI pthread_mutexattr_init(LibKernel::PthreadMutexattr* attr); +int KYTY_SYSV_ABI pthread_mutexattr_settype(LibKernel::PthreadMutexattr* attr, int type); +int KYTY_SYSV_ABI pthread_mutexattr_setprotocol(LibKernel::PthreadMutexattr* attr, int protocol); +int KYTY_SYSV_ABI pthread_mutexattr_destroy(LibKernel::PthreadMutexattr* attr); +int KYTY_SYSV_ABI pthread_getstack(const LibKernel::PthreadAttr* __restrict attr, + void** __restrict stack_addr, size_t* __restrict stack_size); +int64_t KYTY_SYSV_ABI lseek(int d, int64_t offset, int whence); + +} // namespace Posix + +} // namespace Libs + +#endif /* EMULATOR_INCLUDE_EMULATOR_KERNEL_PTHREAD_H_ */ diff --git a/src/kernel/semaphore.cpp b/src/kernel/semaphore.cpp new file mode 100644 index 000000000..77da4ba2b --- /dev/null +++ b/src/kernel/semaphore.cpp @@ -0,0 +1,745 @@ +#include "kernel/semaphore.h" + +#include "common/assert.h" +#include "common/logging/log.h" +#include "common/stringUtils.h" +#include "common/threads.h" +#include "common/timer.h" +#include "libs/errno.h" +#include "libs/libs.h" + +#include +#include +#include +#include + +namespace Libs::LibKernel { +void KernelDispatchPendingSignalForCurrentThread(); +} // namespace Libs::LibKernel + +namespace Libs::LibKernel::Semaphore { + +LIB_NAME("libkernel", "libkernel"); + +constexpr uint32_t SIGNAL_APC_POLL_MICROS = 10000; + +class KernelSemaPrivate { +public: + enum class Result { Ok, TimedOut, Canceled, Deleted, InvalCount }; + + KernelSemaPrivate(const std::string& name, bool fifo, int init_count, int max_count) + : m_name(name), m_fifo_order(fifo), m_count(init_count), m_init_count(init_count), + m_max_count(max_count) {}; + virtual ~KernelSemaPrivate(); + + KYTY_CLASS_NO_COPY(KernelSemaPrivate); + + Result Cancel(int set_count, int* num_waiting_threads); + Result Signal(int signal_count); + Result Wait(int need_count, uint32_t* ptr_micros); + + Result Poll(int need_count) { + uint32_t micros = 0; + return Wait(need_count, µs); + } + + [[nodiscard]] const std::string& GetName() const { return m_name; } + +private: + enum class Status { Set, Deleted }; + + struct WaitingThread { + int id = 0; + int priority = 700; + int need_count = 0; + Result result = Result::TimedOut; + bool ready = false; + }; + + void AddWaiter(WaitingThread* waiter); + void RemoveWaiter(WaitingThread* waiter); + void WakeWaiters(); + + Common::Mutex m_mutex; + Common::CondVar m_cond_var; + std::string m_name; + Status m_status = Status::Set; + std::vector m_waiting_threads; + bool m_fifo_order; + int m_count; + int m_init_count; + int m_max_count; +}; + +KernelSemaPrivate::~KernelSemaPrivate() { + Common::LockGuard lock(m_mutex); + + while (m_status != Status::Set) { + m_mutex.Unlock(); + Common::Thread::SleepMicro(10); + m_mutex.Lock(); + } + + m_status = Status::Deleted; + + for (auto* waiter: m_waiting_threads) { + if (!waiter->ready) { + waiter->result = Result::Deleted; + waiter->ready = true; + } + } + + m_cond_var.SignalAll(); + + while (!m_waiting_threads.empty()) { + m_mutex.Unlock(); + Common::Thread::SleepMicro(10); + m_mutex.Lock(); + } +} + +KernelSemaPrivate::Result KernelSemaPrivate::Cancel(int set_count, int* num_waiting_threads) { + Common::LockGuard lock(m_mutex); + + if (set_count > m_max_count) { + return Result::InvalCount; + } + + if (m_status == Status::Deleted) { + return Result::Deleted; + } + + if (num_waiting_threads != nullptr) { + *num_waiting_threads = static_cast( + std::count_if(m_waiting_threads.begin(), m_waiting_threads.end(), + [](const WaitingThread* waiter) { return !waiter->ready; })); + } + + m_count = (set_count < 0 ? m_init_count : set_count); + + for (auto* waiter: m_waiting_threads) { + if (!waiter->ready) { + waiter->result = Result::Canceled; + waiter->ready = true; + } + } + + m_cond_var.SignalAll(); + + return Result::Ok; +} + +KernelSemaPrivate::Result KernelSemaPrivate::Signal(int signal_count) { + Common::LockGuard lock(m_mutex); + + if (m_status == Status::Deleted) { + return Result::Deleted; + } + + if (m_count + signal_count > m_max_count) { + return Result::InvalCount; + } + + m_count += signal_count; + + WakeWaiters(); + + m_cond_var.SignalAll(); + + return Result::Ok; +} + +void KernelSemaPrivate::AddWaiter(WaitingThread* waiter) { + if (m_fifo_order) { + m_waiting_threads.push_back(waiter); + return; + } + + auto it = m_waiting_threads.begin(); + while (it != m_waiting_threads.end() && (*it)->priority <= waiter->priority) { + ++it; + } + m_waiting_threads.insert(it, waiter); +} + +void KernelSemaPrivate::RemoveWaiter(WaitingThread* waiter) { + auto it = std::find(m_waiting_threads.begin(), m_waiting_threads.end(), waiter); + if (it != m_waiting_threads.end()) { + m_waiting_threads.erase(it); + } +} + +void KernelSemaPrivate::WakeWaiters() { + for (auto* waiter: m_waiting_threads) { + if (!waiter->ready && waiter->need_count <= m_count) { + m_count -= waiter->need_count; + waiter->result = Result::Ok; + waiter->ready = true; + } + } +} + +KernelSemaPrivate::Result KernelSemaPrivate::Wait(int need_count, uint32_t* ptr_micros) { + Common::LockGuard lock(m_mutex); + + if (need_count < 1 || need_count > m_max_count) { + return Result::InvalCount; + } + + if (m_status == Status::Deleted) { + return Result::Deleted; + } + + uint32_t micros = 0; + bool infinitely = true; + if (ptr_micros != nullptr) { + micros = *ptr_micros; + infinitely = false; + } + + uint32_t elapsed = 0; + Common::Timer t; + t.Start(); + + int id = Common::Thread::GetThreadIdUnique(); + + if (m_count >= need_count) { + m_count -= need_count; + if (ptr_micros != nullptr) { + *ptr_micros = micros; + } + return Result::Ok; + } + + if (!infinitely && micros == 0) { + return Result::TimedOut; + } + + WaitingThread waiter {}; + waiter.id = id; + waiter.priority = (m_fifo_order ? 0 : Libs::LibKernel::PthreadGetCurrentPriorityForKernel()); + waiter.need_count = need_count; + AddWaiter(&waiter); + + while (!waiter.ready) { + if ((elapsed >= micros && !infinitely)) { + RemoveWaiter(&waiter); + *ptr_micros = 0; + return Result::TimedOut; + } + + if (infinitely) { + m_cond_var.WaitFor(&m_mutex, SIGNAL_APC_POLL_MICROS); + } else { + m_cond_var.WaitFor(&m_mutex, micros - elapsed); + } + + m_mutex.Unlock(); + LibKernel::KernelDispatchPendingSignalForCurrentThread(); + m_mutex.Lock(); + + elapsed = static_cast(t.GetTimeS() * 1000000.0); + } + + RemoveWaiter(&waiter); + + if (ptr_micros != nullptr) { + *ptr_micros = (elapsed >= micros ? 0 : micros - elapsed); + } + + return waiter.result; +} + +int KYTY_SYSV_ABI KernelCreateSema(KernelSema* sem, const char* name, uint32_t attr, int init, + int max, void* opt) { + PRINT_NAME(); + + EXIT_NOT_IMPLEMENTED(sem == nullptr); + + if (name == nullptr || attr > 2 || init < 0 || max <= 0 || init > max) { + return KERNEL_ERROR_EINVAL; + } + + bool fifo = false; + + switch (attr) { + case 0x01: fifo = true; break; + case 0x02: + default: fifo = false; break; + } + + *sem = new KernelSemaPrivate(std::string(name), fifo, init, max); + + LOGF("\t Semaphore create: %s, ptr=0x%016" PRIx64 ", init=%d, max=%d\n", name, + reinterpret_cast(*sem), init, max); + + return OK; +} + +int KYTY_SYSV_ABI KernelDeleteSema(KernelSema sem) { + PRINT_NAME(); + + if (sem == nullptr) { + return KERNEL_ERROR_ESRCH; + } + + delete sem; + + return OK; +} + +int KYTY_SYSV_ABI KernelWaitSema(KernelSema sem, int need, KernelUseconds* time) { + if (sem == nullptr) { + return KERNEL_ERROR_ESRCH; + } + + auto result = sem->Wait(need, time); + + int ret = OK; + + switch (result) { + case KernelSemaPrivate::Result::Ok: ret = OK; break; + case KernelSemaPrivate::Result::InvalCount: ret = KERNEL_ERROR_EINVAL; break; + case KernelSemaPrivate::Result::TimedOut: ret = KERNEL_ERROR_ETIMEDOUT; break; + case KernelSemaPrivate::Result::Canceled: ret = KERNEL_ERROR_ECANCELED; break; + case KernelSemaPrivate::Result::Deleted: ret = KERNEL_ERROR_EACCES; break; + } + + return ret; +} + +int KYTY_SYSV_ABI KernelPollSema(KernelSema sem, int need) { + PRINT_NAME(); + + if (sem == nullptr) { + return KERNEL_ERROR_ESRCH; + } + + LOGF("\t Semaphore poll: %s, %d\n", sem->GetName().c_str(), need); + + auto result = sem->Poll(need); + + int ret = OK; + + switch (result) { + case KernelSemaPrivate::Result::Ok: ret = OK; break; + case KernelSemaPrivate::Result::InvalCount: ret = KERNEL_ERROR_EINVAL; break; + case KernelSemaPrivate::Result::TimedOut: + case KernelSemaPrivate::Result::Canceled: + case KernelSemaPrivate::Result::Deleted: ret = KERNEL_ERROR_EBUSY; break; + } + + return ret; +} + +int KYTY_SYSV_ABI KernelSignalSema(KernelSema sem, int count) { + if (sem == nullptr) { + return KERNEL_ERROR_ESRCH; + } + + auto result = sem->Signal(count); + + int ret = OK; + + switch (result) { + case KernelSemaPrivate::Result::Ok: ret = OK; break; + case KernelSemaPrivate::Result::InvalCount: + case KernelSemaPrivate::Result::TimedOut: + case KernelSemaPrivate::Result::Canceled: + case KernelSemaPrivate::Result::Deleted: ret = KERNEL_ERROR_EINVAL; break; + } + + return ret; +} + +int KYTY_SYSV_ABI KernelCancelSema(KernelSema sem, int count, int* threads) { + PRINT_NAME(); + + if (sem == nullptr) { + return KERNEL_ERROR_ESRCH; + } + + auto result = sem->Cancel(count, threads); + + int ret = OK; + + switch (result) { + case KernelSemaPrivate::Result::Ok: ret = OK; break; + case KernelSemaPrivate::Result::InvalCount: + case KernelSemaPrivate::Result::TimedOut: + case KernelSemaPrivate::Result::Canceled: + case KernelSemaPrivate::Result::Deleted: ret = KERNEL_ERROR_EINVAL; break; + } + + return ret; +} + +} // namespace Libs::LibKernel::Semaphore + +namespace Libs::Posix { + +LIB_NAME("Posix", "libkernel"); + +namespace { + +constexpr uint16_t POSIX_SEM_MAGIC = 0x09fa; +constexpr int POSIX_SEM_VALUE_MAX = 0x7fffffff; +constexpr uint32_t SIGNAL_APC_POLL_MICROS = 10000; + +struct PosixSemGuest { + uint16_t magic; + uint16_t nameid; + volatile uint32_t has_waiters; + volatile uint32_t count; + uint32_t flags; +}; + +static_assert(sizeof(PosixSemGuest) == 16); + +class PosixSemPrivate { +public: + PosixSemPrivate(PosixSemGuest* guest, unsigned int value, uint32_t flags) + : m_guest(guest), m_count(static_cast(value)) { + SyncGuest(flags); + } + + int Wait(uint32_t* micros) { + Common::LockGuard lock(m_mutex); + + uint32_t elapsed = 0; + const bool timed_wait = (micros != nullptr); + Common::Timer timer; + timer.Start(); + + while (m_count <= 0) { + if (timed_wait && elapsed >= *micros) { + SyncGuest(); + return POSIX_ETIMEDOUT; + } + + m_waiters++; + SyncGuest(); + + if (timed_wait) { + m_cond_var.WaitFor(&m_mutex, *micros - elapsed); + } else { + m_cond_var.WaitFor(&m_mutex, SIGNAL_APC_POLL_MICROS); + } + + m_mutex.Unlock(); + LibKernel::KernelDispatchPendingSignalForCurrentThread(); + m_mutex.Lock(); + + m_waiters--; + elapsed = static_cast(timer.GetTimeS() * 1000000.0); + } + + m_count--; + SyncGuest(); + + return OK; + } + + int TryWait() { + Common::LockGuard lock(m_mutex); + + if (m_count <= 0) { + SyncGuest(); + return POSIX_EAGAIN; + } + + m_count--; + SyncGuest(); + + return OK; + } + + int Post() { + Common::LockGuard lock(m_mutex); + + if (m_count == POSIX_SEM_VALUE_MAX) { + return POSIX_EOVERFLOW; + } + + m_count++; + SyncGuest(); + m_cond_var.Signal(); + + return OK; + } + + int GetValue() { + Common::LockGuard lock(m_mutex); + + return m_count; + } + +private: + void SyncGuest(uint32_t flags = std::numeric_limits::max()) { + if (m_guest != nullptr) { + m_guest->magic = POSIX_SEM_MAGIC; + m_guest->nameid = 0; + m_guest->has_waiters = (m_waiters > 0 ? 1u : 0u); + m_guest->count = static_cast(m_count); + if (flags != std::numeric_limits::max()) { + m_guest->flags = flags; + } + } + } + + Common::Mutex m_mutex; + Common::CondVar m_cond_var; + PosixSemGuest* m_guest = nullptr; + int m_count = 0; + uint32_t m_waiters = 0; +}; + +Common::Mutex g_posix_sem_mutex; +std::unordered_map g_posix_sems; + +static int PosixToKernel(int posix_errno) { + return (posix_errno >= 0 && posix_errno <= 102 ? posix_errno - 2147352576 + : LibKernel::KERNEL_ERROR_EINVAL); +} + +static PosixSemPrivate* GetSem(void* sem) { + if (sem == nullptr) { + return nullptr; + } + + Common::LockGuard lock(g_posix_sem_mutex); + + const auto it = g_posix_sems.find(static_cast(sem)); + return (it != g_posix_sems.end() ? it->second : nullptr); +} + +static int SetErrnoReturn(int posix_errno) { + *GetErrorAddr() = posix_errno; + return -1; +} + +static int SemInitImpl(void* sem, int pshared, unsigned int value, const char* name) { + if (sem == nullptr || value > POSIX_SEM_VALUE_MAX) { + return POSIX_EINVAL; + } + + auto* guest = static_cast(sem); + auto* obj = new PosixSemPrivate(guest, value, static_cast(pshared)); + + { + Common::LockGuard lock(g_posix_sem_mutex); + + if (auto it = g_posix_sems.find(guest); it != g_posix_sems.end()) { + delete it->second; + it->second = obj; + } else { + g_posix_sems.insert({guest, obj}); + } + } + + static std::atomic log_count {0}; + const auto count = log_count.fetch_add(1); + if (count < 32) { + LOGF("\t POSIX semaphore init: %s, value = %u, pshared = %d\n", + (name != nullptr ? name : "sem"), value, pshared); + } + + return OK; +} + +static int SemDestroyImpl(void* sem) { + if (sem == nullptr) { + return POSIX_EINVAL; + } + + auto* guest = static_cast(sem); + PosixSemPrivate* obj = nullptr; + + { + Common::LockGuard lock(g_posix_sem_mutex); + + const auto it = g_posix_sems.find(guest); + if (it == g_posix_sems.end()) { + return POSIX_EINVAL; + } + + obj = it->second; + g_posix_sems.erase(it); + } + + delete obj; + guest->magic = 0; + guest->has_waiters = 0; + guest->count = 0; + guest->flags = 0; + + return OK; +} + +static int SemTimedwaitImpl(void* sem, uint32_t* micros) { + auto* obj = GetSem(sem); + if (obj == nullptr) { + return POSIX_EINVAL; + } + + return obj->Wait(micros); +} + +static bool AbsTimespecToMicros(const LibKernel::KernelTimespec* abstime, uint32_t* micros) { + if (abstime == nullptr || micros == nullptr || abstime->tv_sec < 0 || abstime->tv_nsec < 0 || + abstime->tv_nsec >= 1000000000) { + return false; + } + + LibKernel::KernelTimespec now {}; + if (LibKernel::KernelClockGettime(0, &now) != OK) { + return false; + } + + const int64_t sec_delta = abstime->tv_sec - now.tv_sec; + const int64_t nsec_delta = abstime->tv_nsec - now.tv_nsec; + const int64_t usec_delta = sec_delta * 1000000 + nsec_delta / 1000; + + if (usec_delta <= 0) { + *micros = 0; + } else if (usec_delta > std::numeric_limits::max()) { + *micros = std::numeric_limits::max(); + } else { + *micros = static_cast(usec_delta); + } + + return true; +} + +} // namespace + +int KYTY_SYSV_ABI sem_init(void* sem, int pshared, unsigned int value) { + PRINT_NAME(); + + const int result = SemInitImpl(sem, pshared, value, nullptr); + return (result == OK ? 0 : SetErrnoReturn(result)); +} + +int KYTY_SYSV_ABI sem_destroy(void* sem) { + PRINT_NAME(); + + const int result = SemDestroyImpl(sem); + return (result == OK ? 0 : SetErrnoReturn(result)); +} + +int KYTY_SYSV_ABI sem_wait(void* sem) { + const int result = SemTimedwaitImpl(sem, nullptr); + return (result == OK ? 0 : SetErrnoReturn(result)); +} + +int KYTY_SYSV_ABI sem_trywait(void* sem) { + auto* obj = GetSem(sem); + if (obj == nullptr) { + return SetErrnoReturn(POSIX_EINVAL); + } + + const int result = obj->TryWait(); + return (result == OK ? 0 : SetErrnoReturn(result)); +} + +int KYTY_SYSV_ABI sem_reltimedwait_np(void* sem, uint32_t usec) { + const int result = SemTimedwaitImpl(sem, &usec); + return (result == OK ? 0 : SetErrnoReturn(result)); +} + +int KYTY_SYSV_ABI sem_timedwait(void* sem, const LibKernel::KernelTimespec* abstime) { + uint32_t micros = 0; + if (!AbsTimespecToMicros(abstime, µs)) { + return SetErrnoReturn(POSIX_EINVAL); + } + + const int result = SemTimedwaitImpl(sem, µs); + return (result == OK ? 0 : SetErrnoReturn(result)); +} + +int KYTY_SYSV_ABI sem_post(void* sem) { + auto* obj = GetSem(sem); + if (obj == nullptr) { + return SetErrnoReturn(POSIX_EINVAL); + } + + const int result = obj->Post(); + return (result == OK ? 0 : SetErrnoReturn(result)); +} + +int KYTY_SYSV_ABI sem_getvalue(void* sem, int* value) { + auto* obj = GetSem(sem); + if (obj == nullptr || value == nullptr) { + return SetErrnoReturn(POSIX_EINVAL); + } + + *value = obj->GetValue(); + + return 0; +} + +} // namespace Libs::Posix + +namespace Libs::LibKernel::Semaphore { + +int KYTY_SYSV_ABI PthreadSemInit(void* sem, int flag, unsigned int value, const char* name) { + PRINT_NAME(); + + if (flag != 0) { + return KERNEL_ERROR_EINVAL; + } + + const int result = Posix::SemInitImpl(sem, 0, value, name); + return (result == OK ? OK : Posix::PosixToKernel(result)); +} + +int KYTY_SYSV_ABI PthreadSemDestroy(void* sem) { + PRINT_NAME(); + + const int result = Posix::SemDestroyImpl(sem); + return (result == OK ? OK : Posix::PosixToKernel(result)); +} + +int KYTY_SYSV_ABI PthreadSemWait(void* sem) { + const int result = Posix::SemTimedwaitImpl(sem, nullptr); + return (result == OK ? OK : Posix::PosixToKernel(result)); +} + +int KYTY_SYSV_ABI PthreadSemTrywait(void* sem) { + auto* obj = Posix::GetSem(sem); + if (obj == nullptr) { + return KERNEL_ERROR_EINVAL; + } + + const int result = obj->TryWait(); + return (result == OK ? OK : Posix::PosixToKernel(result)); +} + +int KYTY_SYSV_ABI PthreadSemTimedwait(void* sem, KernelUseconds usec) { + uint32_t micros = usec; + const int result = Posix::SemTimedwaitImpl(sem, µs); + return (result == OK ? OK : Posix::PosixToKernel(result)); +} + +int KYTY_SYSV_ABI PthreadSemPost(void* sem) { + auto* obj = Posix::GetSem(sem); + if (obj == nullptr) { + return KERNEL_ERROR_EINVAL; + } + + const int result = obj->Post(); + return (result == OK ? OK : Posix::PosixToKernel(result)); +} + +int KYTY_SYSV_ABI PthreadSemGetvalue(void* sem, int* value) { + auto* obj = Posix::GetSem(sem); + if (obj == nullptr || value == nullptr) { + return KERNEL_ERROR_EINVAL; + } + + *value = obj->GetValue(); + + return OK; +} + +} // namespace Libs::LibKernel::Semaphore diff --git a/src/kernel/semaphore.h b/src/kernel/semaphore.h new file mode 100644 index 000000000..abdcc4687 --- /dev/null +++ b/src/kernel/semaphore.h @@ -0,0 +1,44 @@ +#ifndef EMULATOR_INCLUDE_EMULATOR_KERNEL_SEMAPHORE_H_ +#define EMULATOR_INCLUDE_EMULATOR_KERNEL_SEMAPHORE_H_ + +#include "common/abi.h" +#include "common/common.h" +#include "kernel/pthread.h" + +namespace Libs::LibKernel::Semaphore { + +class KernelSemaPrivate; + +using KernelSema = KernelSemaPrivate*; + +int KYTY_SYSV_ABI KernelCreateSema(KernelSema* sem, const char* name, uint32_t attr, int init, + int max, void* opt); +int KYTY_SYSV_ABI KernelDeleteSema(KernelSema sem); +int KYTY_SYSV_ABI KernelWaitSema(KernelSema sem, int need, KernelUseconds* time); +int KYTY_SYSV_ABI KernelPollSema(KernelSema sem, int need); +int KYTY_SYSV_ABI KernelSignalSema(KernelSema sem, int count); +int KYTY_SYSV_ABI KernelCancelSema(KernelSema sem, int count, int* threads); +int KYTY_SYSV_ABI PthreadSemInit(void* sem, int flag, unsigned int value, const char* name); +int KYTY_SYSV_ABI PthreadSemDestroy(void* sem); +int KYTY_SYSV_ABI PthreadSemWait(void* sem); +int KYTY_SYSV_ABI PthreadSemTrywait(void* sem); +int KYTY_SYSV_ABI PthreadSemTimedwait(void* sem, KernelUseconds usec); +int KYTY_SYSV_ABI PthreadSemPost(void* sem); +int KYTY_SYSV_ABI PthreadSemGetvalue(void* sem, int* value); + +} // namespace Libs::LibKernel::Semaphore + +namespace Libs::Posix { + +int KYTY_SYSV_ABI sem_init(void* sem, int pshared, unsigned int value); +int KYTY_SYSV_ABI sem_destroy(void* sem); +int KYTY_SYSV_ABI sem_wait(void* sem); +int KYTY_SYSV_ABI sem_trywait(void* sem); +int KYTY_SYSV_ABI sem_reltimedwait_np(void* sem, uint32_t usec); +int KYTY_SYSV_ABI sem_timedwait(void* sem, const LibKernel::KernelTimespec* abstime); +int KYTY_SYSV_ABI sem_post(void* sem); +int KYTY_SYSV_ABI sem_getvalue(void* sem, int* value); + +} // namespace Libs::Posix + +#endif /* EMULATOR_INCLUDE_EMULATOR_KERNEL_SEMAPHORE_H_ */ diff --git a/src/kytyGitVersion.h.in b/src/kytyGitVersion.h.in new file mode 100644 index 000000000..b04b3f8d2 --- /dev/null +++ b/src/kytyGitVersion.h.in @@ -0,0 +1 @@ +#define KYTY_GIT_VERSION "@KYTY_GIT_VERSION@" \ No newline at end of file diff --git a/src/launcher/CMakeLists.txt b/src/launcher/CMakeLists.txt new file mode 100644 index 000000000..ec2dcd2d5 --- /dev/null +++ b/src/launcher/CMakeLists.txt @@ -0,0 +1,103 @@ + +find_package(Qt6 COMPONENTS Concurrent Network Widgets REQUIRED) + +set(CMAKE_AUTOUIC_SEARCH_PATHS forms) +set(CMAKE_INCLUDE_CURRENT_DIR ON) + +file(GLOB launcher_src + "include/*.h" + "src/*.cpp" +) + +file(GLOB launcher_forms + "forms/*.ui" +) + +add_executable(launcher WIN32 + ${launcher_src} + ${launcher_forms} +) + + +target_link_libraries(launcher Qt6::Concurrent Qt6::Network Qt6::Widgets) +if(WIN32) +target_link_libraries(launcher setupapi) +endif() +if (CLANG AND NOT KYTY_CLANG_CL) +target_link_libraries(launcher pthread) +endif() +target_include_directories(launcher PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/include" "${CMAKE_CURRENT_SOURCE_DIR}/src") +set_target_properties(launcher PROPERTIES AUTOUIC ON AUTOMOC ON AUTORCC ON) + +if(WIN32 AND NOT KYTY_CLANG_CL) + set_target_properties(launcher PROPERTIES LINK_FLAGS "${KYTY_LD_OPTIONS}") +endif() + +add_dependencies(launcher KytyGitVersion) +add_dependencies(launcher kyty_emulator) + +if(KYTY_CLANG_CL) + add_custom_command(TARGET launcher POST_BUILD + COMMAND ${CMAKE_COMMAND} -E copy_if_different $ $/$ + COMMAND ${CMAKE_COMMAND} -E copy_if_different $/${KYTY_EMULATOR_MAP_NAME} $/${KYTY_EMULATOR_MAP_NAME} + COMMAND ${CMAKE_COMMAND} -E copy_if_different "${KYTY_THIRD_PARTY_DIR}/winpthread/bin/libwinpthread-1.dll" $/libwinpthread-1.dll + ) +endif() + +if(WIN32) + find_program(QT_WINDEPLOYQT NAMES windeployqt PATHS "${Qt6_DIR}/../../../bin") + if(NOT QT_WINDEPLOYQT) + message(FATAL_ERROR "Could not find windeployqt") + endif() + add_custom_command(TARGET launcher POST_BUILD + COMMAND ${QT_WINDEPLOYQT} $ + --no-compiler-runtime + --no-system-d3d-compiler + --no-system-dxc-compiler + --no-opengl-sw + --no-translations + COMMENT "Deploying Qt runtime and TLS plugins" + VERBATIM + ) +endif() + +set(launcher_name "launcher") + +set_target_properties(launcher PROPERTIES OUTPUT_NAME ${launcher_name}) + +install(TARGETS launcher DESTINATION .) + +if(WIN32) + install(CODE " + execute_process( + COMMAND ${QT_WINDEPLOYQT} \"\${CMAKE_INSTALL_PREFIX}/${launcher_name}.exe\" + --no-compiler-runtime + --no-system-d3d-compiler + --no-system-dxc-compiler + --no-opengl-sw + --no-translations + ) + ") +endif() + + +set(tidy_dirs "${CMAKE_SOURCE_DIR}/launcher/include") +set(iwyu_maps "${CMAKE_SOURCE_DIR}/launcher/utils/qt6_16.imp") + +get_property(inc_headers TARGET launcher PROPERTY INCLUDE_DIRECTORIES) + +list(APPEND inc_headers + ${CMAKE_SOURCE_DIR}/launcher + ${CMAKE_BINARY_DIR}/launcher/launcher_autogen/include + ${Qt6Widgets_INCLUDE_DIRS} + ${Qt6Core_INCLUDE_DIRS} + ${Qt6Gui_INCLUDE_DIRS} + #${Qt6_DIR}/../../../include + #${Qt6_DIR}/../../../include/QtWidgets + #${Qt6_DIR}/../../../include/QtCore + #${Qt6_DIR}/../../../include/QtGui +) + +include_what_you_use_with_mappings(launcher "${inc_headers}" "${iwyu_maps}") + +clang_tidy_check(launcher "" "${tidy_dirs}" "${inc_headers}") diff --git a/src/launcher/forms/configuration_edit_dialog.ui b/src/launcher/forms/configuration_edit_dialog.ui new file mode 100644 index 000000000..7b3d31b69 --- /dev/null +++ b/src/launcher/forms/configuration_edit_dialog.ui @@ -0,0 +1,338 @@ + + + ConfigurationEditDialog + + + + 0 + 0 + 560 + 420 + + + + Settings + + + + + + + + + QFrame::StyledPanel + + + + QFormLayout::AllNonFixedFieldsGrow + + + + + QFrame::StyledPanel + + + + + + Enable Vulkan validation layers + + + Vulkan validation + + + true + + + + + + + Validate SPIR-V binary + + + Shader validation + + + true + + + + + + + Dump command buffers + + + Command buffer dump + + + + + + + Enable RenderDoc capture + + + RenderDoc capture + + + + + + + Use the NGG 4-vertex path for rect-list DrawIndexAuto primitive 7 + + + Use NGG rect-list draw + + + true + + + + + + + + + + Screen resolution: + + + + + + + Window resolution + + + + + + + Vblank frequency: + + + + + + + Virtual display refresh rate used for frame pacing + + + 30 + + + 360 + + + 60 + + + + + + + Shader optimization type: + + + + + + + Optimize shaders for code size or performance + + + + + + + Shader log direction: + + + + + + + Dump shaders to file or console window. If enabled may decrease emulator performance + + + + + + + Shader log folder: + + + + + + + Specify directory to dump shaders + + + true + + + + + + + Command buffer dump folder: + + + + + + + Specify directory to dump command buffers + + + true + + + + + + + Printf direction: + + + + + + + Print logs to file or console window. If enabled may decrease emulator performance + + + + + + + Printf output file: + + + + + + + Specify file to dump logs + + + true + + + + + + + Profiler direction: + + + + + + + Enable or disable profiler. If enabled may decrease emulator performance + + + + + + + + + + + + + 80 + 25 + + + + Save + + + true + + + + + + + + 80 + 25 + + + + Cancel + + + + + + + Qt::Horizontal + + + + + + + + 80 + 25 + + + + Clear + + + + + + + Qt::Vertical + + + + 20 + 40 + + + + + + + + + + + MandatoryLineEdit + QLineEdit +
mandatoryLineEdit.h
+ + + + + + cancel_button + clicked() + ConfigurationEditDialog + reject() + + + 496 + 65 + + + 280 + 180 + + + + + diff --git a/src/launcher/forms/configuration_list_widget.ui b/src/launcher/forms/configuration_list_widget.ui new file mode 100644 index 000000000..6bed94c49 --- /dev/null +++ b/src/launcher/forms/configuration_list_widget.ui @@ -0,0 +1,223 @@ + + + ConfigurationListWidget + + + + 0 + 0 + 292 + 268 + + + + Configurations + + + + + + + + + 6 + + + 0 + + + 6 + + + 6 + + + + + 4 + + + 2 + + + + + 4 + + + + + + 36 + 36 + + + + <html><head/><body><p>Edit global settings</p></body></html> + + + + + + + 32 + 32 + + + + true + + + + + + + + 36 + 36 + + + + <html><head/><body><p>Edit configuration</p></body></html> + + + + + + + 32 + 32 + + + + true + + + + + + + + 36 + 36 + + + + <html><head/><body><p>Delete configuration</p></body></html> + + + + + + + + + + 32 + 32 + + + + true + + + + + + + Qt::Horizontal + + + + 40 + 20 + + + + + + + + + 0 + 0 + + + + + 220 + 30 + + + + + 320 + 30 + + + + Search game name or serial + + + Search name or serial + + + true + + + + + + + + + false + + + + Game + + + + + Serial + + + + + Firmware Version + + + + + Path + + + + + Status + + + + + Comments + + + + + + + + + + + + + GameListTreeWidget + QTreeWidget +
gameListTreeWidget.h
+ + + + + diff --git a/src/launcher/forms/main_dialog.ui b/src/launcher/forms/main_dialog.ui new file mode 100644 index 000000000..58c539d16 --- /dev/null +++ b/src/launcher/forms/main_dialog.ui @@ -0,0 +1,101 @@ + + + MainDialog + + + + 0 + 0 + 428 + 454 + + + + + 0 + 0 + + + + Kyty Launcher + + + + + + + + + + + + + + + + 0 + 0 + + + + + 400 + 300 + + + + + + + + + + + + + + TextLabel + + + + + + + QFrame::NoFrame + + + QFrame::Sunken + + + TextLabel + + + + + + + TextLabel + + + + + + + + + + + + + ConfigurationListWidget + QWidget +
configurationListWidget.h
+ 1 + + + + + + + + diff --git a/src/launcher/include/common.h b/src/launcher/include/common.h new file mode 100644 index 000000000..71f74cb2c --- /dev/null +++ b/src/launcher/include/common.h @@ -0,0 +1,20 @@ +#ifndef LAUNCHER_INCLUDE_COMMON_H_ +#define LAUNCHER_INCLUDE_COMMON_H_ + +// IWYU pragma: begin_exports +#include +#include + +#include +#include +#include +// IWYU pragma: end_exports + +#define KYTY_QT_CLASS_NO_COPY(name) \ +public: \ + name(const name&) = delete; /* NOLINT(bugprone-macro-parentheses) */ \ + name& operator=(const name&) = delete; /* NOLINT(bugprone-macro-parentheses) */ \ + name(name&&) noexcept = delete; /* NOLINT(bugprone-macro-parentheses) */ \ + name& operator=(name&&) noexcept = delete; /* NOLINT(bugprone-macro-parentheses) */ + +#endif /* LAUNCHER_INCLUDE_COMMON_H_ */ diff --git a/src/launcher/include/compatibilityDatabase.h b/src/launcher/include/compatibilityDatabase.h new file mode 100644 index 000000000..104f24970 --- /dev/null +++ b/src/launcher/include/compatibilityDatabase.h @@ -0,0 +1,40 @@ +#ifndef COMPATIBILITY_DATABASE_H +#define COMPATIBILITY_DATABASE_H + +#include "common.h" +#include "configuration.h" + +#include +#include +#include + +struct CompatibilityEntry { + Configuration::GameStatus status = Configuration::GameStatus::Unknown; + QString comment; +}; + +class CompatibilityDatabase: public QObject { + Q_OBJECT + KYTY_QT_CLASS_NO_COPY(CompatibilityDatabase); + +public: + explicit CompatibilityDatabase(bool local, QObject* parent = nullptr); + + [[nodiscard]] bool IsLocal() const { return m_local; } + [[nodiscard]] const CompatibilityEntry* Find(const QString& title_id) const; + + void Load(); + void SetStatus(const QString& title_id, Configuration::GameStatus status); + void SetComment(const QString& title_id, const QString& comment); + +signals: + void Updated(); + +private: + void Save() const; + + QMap m_entries; + bool m_local = false; +}; + +#endif // COMPATIBILITY_DATABASE_H diff --git a/src/launcher/include/configuration.h b/src/launcher/include/configuration.h new file mode 100644 index 000000000..74fb81e2f --- /dev/null +++ b/src/launcher/include/configuration.h @@ -0,0 +1,178 @@ +#ifndef LAUNCHER_INCLUDE_CONFIGURATION_H_ +#define LAUNCHER_INCLUDE_CONFIGURATION_H_ + +#include "common.h" + +#include +#include +#include +#include +#include +#include +#include +#include + +#define KYTY_CFG_SET(n) s->setValue(#n, QVariant::fromValue(n).toString()); +#define KYTY_CFG_GET(n) n = s->value(#n).value(); + +template +inline QStringList EnumToList() { + QStringList ret; + auto me = QMetaEnum::fromType(); + int count = me.keyCount(); + for (int i = 0; i < count; i++) { + auto key = QString(me.key(i)); + ret << (key.startsWith('R') && key.size() > 2 && key.at(1).isDigit() + ? key.remove('R').toLower() + : key); + } + return ret; +} + +template +T TextToEnum(const QString& text) { + auto me = QMetaEnum::fromType(); + return static_cast(me.keyToValue( + ((text.size() > 1 && text.at(0).isDigit()) ? 'R' + text.toUpper() : text).toUtf8().data())); +} + +template +QString EnumToText(T value) { + auto me = QMetaEnum::fromType(); + auto key = QString(me.valueToKey(static_cast(value))); + return (key.startsWith('R') && key.size() > 2 && key.at(1).isDigit() ? key.remove('R').toLower() + : key); +} + +class Configuration: public QObject { + Q_OBJECT + +public: + enum class Resolution { + R1280X720, + R1920X1080, + }; + Q_ENUM(Resolution) + + enum class ShaderOptimizationType { None, Size, Performance }; + Q_ENUM(ShaderOptimizationType) + + enum class ShaderLogDirection { Silent, Console, File }; + Q_ENUM(ShaderLogDirection) + + enum class ProfilerDirection { None, Network }; + Q_ENUM(ProfilerDirection) + + enum class LogDirection { Silent, Console, File }; + Q_ENUM(LogDirection) + + enum class GameStatus { Unknown, InGame, Logo, DoesntBoot, MainMenu }; + Q_ENUM(GameStatus) + + Configuration() = default; + + QString name; + QString title_id; /* Serial / title id from sce_sys/param.json */ + QString firmwareVer; /* requiredSystemSoftwareVersion from sce_sys/param.json */ + QString basedir; /* Game base directory */ + QString game_path; /* Launcher-unique game path */ + bool custom_settings = false; + GameStatus game_status = GameStatus::Unknown; + QString game_comment; + + Resolution screen_resolution = Resolution::R1280X720; + int vblank_frequency = 60; + bool vulkan_validation_enabled = true; + bool shader_validation_enabled = true; + ShaderOptimizationType shader_optimization_type = ShaderOptimizationType::Performance; + ShaderLogDirection shader_log_direction = ShaderLogDirection::Silent; + QString shader_log_folder = "_Shaders"; + bool command_buffer_dump_enabled = false; + QString command_buffer_dump_folder = "_Buffers"; + LogDirection printf_direction = LogDirection::Silent; + QString printf_output_file = "_kyty.txt"; + ProfilerDirection profiler_direction = ProfilerDirection::None; + bool renderdoc_enabled = false; + bool ngg_rectlist_draw_enabled = true; + + QString elf = QStringLiteral("eboot.bin"); + + void CopyEmulatorSettingsFrom(const Configuration& other) { + screen_resolution = other.screen_resolution; + vblank_frequency = other.vblank_frequency; + vulkan_validation_enabled = other.vulkan_validation_enabled; + shader_validation_enabled = other.shader_validation_enabled; + shader_optimization_type = other.shader_optimization_type; + shader_log_direction = other.shader_log_direction; + shader_log_folder = other.shader_log_folder; + command_buffer_dump_enabled = other.command_buffer_dump_enabled; + command_buffer_dump_folder = other.command_buffer_dump_folder; + printf_direction = other.printf_direction; + printf_output_file = other.printf_output_file; + profiler_direction = other.profiler_direction; + renderdoc_enabled = other.renderdoc_enabled; + ngg_rectlist_draw_enabled = other.ngg_rectlist_draw_enabled; + } + + void CopyFrom(const Configuration& other) { + name = other.name; + title_id = other.title_id; + firmwareVer = other.firmwareVer; + basedir = other.basedir; + game_path = other.game_path; + custom_settings = other.custom_settings; + game_status = other.game_status; + game_comment = other.game_comment; + CopyEmulatorSettingsFrom(other); + elf = other.elf; + } + + void WriteSettings(QSettings* s) const { + KYTY_CFG_SET(name); + KYTY_CFG_SET(basedir); + KYTY_CFG_SET(game_path); + KYTY_CFG_SET(custom_settings); + KYTY_CFG_SET(screen_resolution); + KYTY_CFG_SET(vblank_frequency); + KYTY_CFG_SET(vulkan_validation_enabled); + KYTY_CFG_SET(shader_validation_enabled); + KYTY_CFG_SET(shader_optimization_type); + KYTY_CFG_SET(shader_log_direction); + KYTY_CFG_SET(shader_log_folder); + KYTY_CFG_SET(command_buffer_dump_enabled); + KYTY_CFG_SET(command_buffer_dump_folder); + KYTY_CFG_SET(printf_direction); + KYTY_CFG_SET(printf_output_file); + KYTY_CFG_SET(profiler_direction); + KYTY_CFG_SET(renderdoc_enabled); + KYTY_CFG_SET(ngg_rectlist_draw_enabled); + KYTY_CFG_SET(elf); + } + + void ReadSettings(QSettings* s) { + KYTY_CFG_GET(name); + KYTY_CFG_GET(basedir); + KYTY_CFG_GET(game_path); + KYTY_CFG_GET(custom_settings); + KYTY_CFG_GET(screen_resolution); + vblank_frequency = s->value("vblank_frequency", vblank_frequency).toInt(); + KYTY_CFG_GET(vulkan_validation_enabled); + KYTY_CFG_GET(shader_validation_enabled); + KYTY_CFG_GET(shader_optimization_type); + KYTY_CFG_GET(shader_log_direction); + KYTY_CFG_GET(shader_log_folder); + KYTY_CFG_GET(command_buffer_dump_enabled); + KYTY_CFG_GET(command_buffer_dump_folder); + KYTY_CFG_GET(printf_direction); + KYTY_CFG_GET(printf_output_file); + KYTY_CFG_GET(profiler_direction); + KYTY_CFG_GET(renderdoc_enabled); + ngg_rectlist_draw_enabled = + s->value("ngg_rectlist_draw_enabled", ngg_rectlist_draw_enabled).toBool(); + elf = s->value("elf", elf).toString(); + } +}; + +Q_DECLARE_METATYPE(Configuration*) + +#endif /* LAUNCHER_INCLUDE_CONFIGURATION_H_ */ diff --git a/src/launcher/include/configurationEditDialog.h b/src/launcher/include/configurationEditDialog.h new file mode 100644 index 000000000..1ced2274b --- /dev/null +++ b/src/launcher/include/configurationEditDialog.h @@ -0,0 +1,65 @@ +#ifndef CONFIGURATION_EDIT_DIALOG_H +#define CONFIGURATION_EDIT_DIALOG_H + +#include "common.h" + +#include +#include +#include +class QByteArray; +class QGroupBox; +class QListWidget; +class QMoveEvent; +class QToolButton; +class QSettings; +class QWidget; + +class Configuration; +namespace Ui { +class ConfigurationEditDialog; +} // namespace Ui + +class ConfigurationEditDialog: public QDialog { + Q_OBJECT + KYTY_QT_CLASS_NO_COPY(ConfigurationEditDialog); + +public: + explicit ConfigurationEditDialog(Configuration& info, QWidget* parent = nullptr); + ~ConfigurationEditDialog() override; + + static void WriteSettings(QSettings& s); + static void ReadSettings(QSettings& s); + + void SetTitle(const QString& str); + void SetGameDirectories(const QStringList& dirs); + [[nodiscard]] QStringList GetGameDirectories() const; + +private: + Ui::ConfigurationEditDialog* m_ui = nullptr; + Configuration& m_info; + QGroupBox* m_game_dirs_group = nullptr; + QListWidget* m_game_dirs_list = nullptr; + QToolButton* m_remove_game_dir_button = nullptr; + bool m_show_game_dirs = false; + +protected: + void Init(const Configuration& info); + void InitGameDirectories(); + void AddGameDirectoryItem(const QString& dir); + + void moveEvent(QMoveEvent* event) override; + + static QByteArray g_last_geometry; + + /*slots:*/ + + void update_info(); + void adjust_size(); + void save(); + void clear(); + void add_game_directory(); + void remove_selected_game_directories(); + void update_game_directory_buttons(); +}; + +#endif // CONFIGURATION_EDIT_DIALOG_H diff --git a/src/launcher/include/configurationItem.h b/src/launcher/include/configurationItem.h new file mode 100644 index 000000000..219c56e25 --- /dev/null +++ b/src/launcher/include/configurationItem.h @@ -0,0 +1,48 @@ +#ifndef CONFIGURATION_ITEM_H +#define CONFIGURATION_ITEM_H + +#include "common.h" + +#include + +#include + +class QComboBox; +class QLabel; +class QLineEdit; +class QTreeWidget; +class QWidget; + +class Configuration; +class ConfigurationItem: public QTreeWidgetItem { +public: + explicit ConfigurationItem(std::unique_ptr info, QTreeWidget* parent); + ~ConfigurationItem() override; + + void Update(); + bool operator<(const QTreeWidgetItem& other) const override; + + KYTY_QT_CLASS_NO_COPY(ConfigurationItem); + + Configuration& GetInfo() { return *m_info; } + [[nodiscard]] const Configuration& GetInfo() const { return *m_info; } + // void SetInfo(Configuration* info); + void SetRunning(bool state); + void SetCompatibilityEditable(bool editable); + [[nodiscard]] bool IsRunning() const { return m_running; } + QComboBox* GetStatusCombo() { return m_status_combo; } + QLineEdit* GetCommentEdit() { return m_comment_edit; } + +private: + void UpdateIcon(); + void UpdateStatusIndicator(); + + std::unique_ptr m_info; + bool m_running = false; + QComboBox* m_status_combo = nullptr; + QLabel* m_status_indicator = nullptr; + QWidget* m_status_widget = nullptr; + QLineEdit* m_comment_edit = nullptr; +}; + +#endif // CONFIGURATION_ITEM_H diff --git a/src/launcher/include/configurationListWidget.h b/src/launcher/include/configurationListWidget.h new file mode 100644 index 000000000..9f28ac2e3 --- /dev/null +++ b/src/launcher/include/configurationListWidget.h @@ -0,0 +1,86 @@ +#ifndef CONFIGURATION_LIST_WIDGET_H +#define CONFIGURATION_LIST_WIDGET_H + +#include "common.h" +#include "configuration.h" + +#include +#include +#include +#include + +class ConfigurationItem; +class CompatibilityDatabase; +class QTreeWidgetItem; +class QPoint; +class MainDialog; + +namespace Ui { +class ConfigurationListWidget; +} // namespace Ui + +class ConfigurationListWidget: public QWidget { + Q_OBJECT + KYTY_QT_CLASS_NO_COPY(ConfigurationListWidget); + +public: + explicit ConfigurationListWidget(QWidget* parent = nullptr); + ~ConfigurationListWidget() override; + + void SetRunEnabled(bool flag) { m_run_enabled = flag; } + [[nodiscard]] bool IsRunEnabled() const { return m_run_enabled; } + + [[nodiscard]] const ConfigurationItem* GetSelectedItem() const { return m_selected_item; } + ConfigurationItem* GetSelectedItem() { return m_selected_item; } + + void SetMainDialog(MainDialog* main_dialog) { m_main_dialog = main_dialog; } + MainDialog* GetMainDialog() { return m_main_dialog; } + + [[nodiscard]] const QString& GetSettingsFile() const { return m_settings_file; } + [[nodiscard]] const QStringList& GetGameDirectories() const { return m_game_dirs; } + [[nodiscard]] bool CanViewSelectedTrophies() const; + + bool EnsureGameDirectory(); + void ScanGameDirectory(); + void ViewTrophies(); + +signals: + + void Run(); + void Select(); + +public slots: + void WriteSettings(); + void ReadSettings(); + +protected slots: + + void edit_configuration(); + void delete_configuartion(); + void edit_global_settings(); + void run_configuration(); + void list_currentItemChanged(QTreeWidgetItem* current, QTreeWidgetItem* previous); + void list_itemDoubleClicked(QTreeWidgetItem* witem, int column); + void show_context_menu(const QPoint& pos); + void open_game_folder(); + void remove_save_data(); + void filter_configurations(const QString& text); + +private: + void ClearCustomSettings(ConfigurationItem* item); + void SelectItem(QTreeWidgetItem* witem); + void ApplyCompatibility(); + [[nodiscard]] bool HasValidGameDirectory() const; + + ConfigurationItem* m_selected_item = nullptr; + bool m_run_enabled = true; + Ui::ConfigurationListWidget* m_ui = nullptr; + MainDialog* m_main_dialog = nullptr; + QString m_settings_file; + QStringList m_game_dirs; + Configuration m_global_info; + QMap m_custom_infos; + CompatibilityDatabase* m_compatibility = nullptr; +}; + +#endif // CONFIGURATION_LIST_WIDGET_H diff --git a/src/launcher/include/gameListTreeWidget.h b/src/launcher/include/gameListTreeWidget.h new file mode 100644 index 000000000..1ef44377a --- /dev/null +++ b/src/launcher/include/gameListTreeWidget.h @@ -0,0 +1,73 @@ +#ifndef GAME_LIST_TREE_WIDGET_H +#define GAME_LIST_TREE_WIDGET_H + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +class GameListTreeWidget: public QTreeWidget { +public: + explicit GameListTreeWidget(QWidget* parent = nullptr): QTreeWidget(parent) { + setAutoFillBackground(false); + viewport()->setAutoFillBackground(false); + } + + void SetBackgroundImage(const QString& path) { + if (m_path == path) { + return; + } + m_path = path; + m_source = QPixmap(); + if (!path.isEmpty()) { + m_source.load(path); + } + UpdateScaledBackground(); + viewport()->update(); + } + +protected: + void resizeEvent(QResizeEvent* event) override { + QTreeWidget::resizeEvent(event); + UpdateScaledBackground(); + } + + void paintEvent(QPaintEvent* event) override { + { + QPainter painter(viewport()); + if (m_scaled.isNull()) { + painter.fillRect(event->rect(), palette().brush(QPalette::Base)); + } else { + painter.drawPixmap(m_scaled_pos, m_scaled); + painter.fillRect(event->rect(), QColor(0, 0, 0, 115)); + } + } + + QTreeWidget::paintEvent(event); + } + +private: + void UpdateScaledBackground() { + m_scaled = QPixmap(); + if (m_source.isNull() || viewport()->size().isEmpty()) { + return; + } + + m_scaled = m_source.scaled(viewport()->size(), Qt::KeepAspectRatioByExpanding, + Qt::FastTransformation); + m_scaled_pos = QPoint((viewport()->width() - m_scaled.width()) / 2, + (viewport()->height() - m_scaled.height()) / 2); + } + + QString m_path; + QPixmap m_source; + QPixmap m_scaled; + QPoint m_scaled_pos; +}; + +#endif // GAME_LIST_TREE_WIDGET_H diff --git a/src/launcher/include/mainDialog.h b/src/launcher/include/mainDialog.h new file mode 100644 index 000000000..cb9031350 --- /dev/null +++ b/src/launcher/include/mainDialog.h @@ -0,0 +1,40 @@ +#ifndef MAIN_DIALOG_H +#define MAIN_DIALOG_H + +#include "common.h" + +#include +#include + +class QWidget; +class QProcess; +class MainDialogPrivate; +class QSettings; +class QResizeEvent; + +class Configuration; +class MainDialog: public QDialog { + Q_OBJECT + KYTY_QT_CLASS_NO_COPY(MainDialog); + +signals: + void Start(); + void Quit(); + void Resize(); + +public: + explicit MainDialog(QWidget* parent = nullptr); + ~MainDialog() override = default; + + void RunInterpreter(QProcess* process, const Configuration& info); + + static void WriteSettings(QSettings& s); + static void ReadSettings(QSettings& s); + + void resizeEvent(QResizeEvent* event) override; + +private: + MainDialogPrivate* m_p = nullptr; +}; + +#endif // MAIN_DIALOG_H diff --git a/src/launcher/include/mandatoryLineEdit.h b/src/launcher/include/mandatoryLineEdit.h new file mode 100644 index 000000000..e315ccd86 --- /dev/null +++ b/src/launcher/include/mandatoryLineEdit.h @@ -0,0 +1,29 @@ +#ifndef MANDATORY_LINEEDIT_H +#define MANDATORY_LINEEDIT_H + +#include "common.h" + +#include +#include + +class QPaintEvent; +class QWidget; + +constexpr char MANDATORY_AND_EMPTY[] = "mandatory_and_empty"; + +class MandatoryLineEdit: public QLineEdit { + Q_OBJECT + +public: + explicit MandatoryLineEdit(QWidget* parent = nullptr); + + void paintEvent(QPaintEvent* event) override; + + static bool FindEmpty(QObject* p); + +protected slots: + + void changed(const QString& text); +}; + +#endif // MANDATORY_LINEEDIT_H diff --git a/src/launcher/include/trophyViewerDialog.h b/src/launcher/include/trophyViewerDialog.h new file mode 100644 index 000000000..2f95571e1 --- /dev/null +++ b/src/launcher/include/trophyViewerDialog.h @@ -0,0 +1,27 @@ +#ifndef TROPHY_VIEWER_DIALOG_H +#define TROPHY_VIEWER_DIALOG_H + +#include "common.h" + +#include + +class QTabWidget; +class QWidget; + +class Configuration; +class TrophyViewerDialog: public QDialog { + KYTY_QT_CLASS_NO_COPY(TrophyViewerDialog); + +public: + explicit TrophyViewerDialog(QWidget* parent = nullptr); + + static bool HasTrophyData(const Configuration* info); + static void ShowForGame(const Configuration* info, QWidget* parent); + +private: + bool LoadGame(const Configuration& info, QString& error); + + QTabWidget* m_tabs = nullptr; +}; + +#endif // TROPHY_VIEWER_DIALOG_H diff --git a/src/launcher/src/compatibilityDatabase.cpp b/src/launcher/src/compatibilityDatabase.cpp new file mode 100644 index 000000000..4cd191b93 --- /dev/null +++ b/src/launcher/src/compatibilityDatabase.cpp @@ -0,0 +1,200 @@ +#include "compatibilityDatabase.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +namespace { + +constexpr char FILE_NAME[] = "compatibility_db.json"; +constexpr char URL[] = + "https://github.com/Nmzik/KytyPS5/releases/download/compat-db/compatibility_db.json"; +constexpr int RETRY_COUNT = 3; +constexpr int RETRY_DELAY_MS = 750; + +struct LoadResult { + QMap entries; + QString error; +}; + +QString TitleKey(const QString& title_id) { + return title_id.trimmed().toUpper(); +} + +Configuration::GameStatus StatusFromText(const QString& text) { + const auto value = text.trimmed(); + if (value == QStringLiteral("InGame") || value == QStringLiteral("In game")) { + return Configuration::GameStatus::InGame; + } + if (value == QStringLiteral("MainMenu") || value == QStringLiteral("Main menu")) { + return Configuration::GameStatus::MainMenu; + } + if (value == QStringLiteral("Logo")) { + return Configuration::GameStatus::Logo; + } + if (value == QStringLiteral("DoesntBoot") || value == QStringLiteral("Doesn't boot")) { + return Configuration::GameStatus::DoesntBoot; + } + return Configuration::GameStatus::Unknown; +} + +QString StatusToText(Configuration::GameStatus status) { + switch (status) { + case Configuration::GameStatus::InGame: return QStringLiteral("InGame"); + case Configuration::GameStatus::MainMenu: return QStringLiteral("MainMenu"); + case Configuration::GameStatus::Logo: return QStringLiteral("Logo"); + case Configuration::GameStatus::DoesntBoot: return QStringLiteral("DoesntBoot"); + case Configuration::GameStatus::Unknown: return QStringLiteral("Unknown"); + } + return QStringLiteral("Unknown"); +} + +LoadResult Parse(const QByteArray& data) { + LoadResult ret; + QJsonParseError parse_error; + const auto doc = QJsonDocument::fromJson(data, &parse_error); + if (parse_error.error != QJsonParseError::NoError || !doc.isObject()) { + ret.error = QStringLiteral("Invalid compatibility JSON: %1").arg(parse_error.errorString()); + return ret; + } + + const auto root = doc.object(); + for (auto it = root.constBegin(); it != root.constEnd(); ++it) { + const auto title_id = TitleKey(it.key()); + const auto object = it.value().toObject(); + if (title_id.isEmpty() || object.isEmpty()) { + continue; + } + ret.entries.insert(title_id, + {StatusFromText(object.value(QStringLiteral("status")).toString()), + object.value(QStringLiteral("comment")).toString()}); + } + return ret; +} + +LoadResult DownloadOnce() { + QNetworkAccessManager manager; + QNetworkRequest request(QUrl(QString::fromLatin1(URL))); + request.setAttribute(QNetworkRequest::RedirectPolicyAttribute, + QNetworkRequest::NoLessSafeRedirectPolicy); + request.setRawHeader("User-Agent", "Kyty-Launcher"); + + auto* reply = manager.get(request); + QEventLoop loop; + QTimer timeout; + timeout.setSingleShot(true); + QObject::connect(reply, &QNetworkReply::finished, &loop, &QEventLoop::quit); + QObject::connect(&timeout, &QTimer::timeout, reply, &QNetworkReply::abort); + timeout.start(15000); + loop.exec(); + + if (!timeout.isActive()) { + return {{}, QStringLiteral("Compatibility download timed out")}; + } + if (reply->error() != QNetworkReply::NoError) { + return {{}, reply->errorString()}; + } + return Parse(reply->readAll()); +} + +LoadResult Download() { + auto result = DownloadOnce(); + for (int retry = 1; retry <= RETRY_COUNT && !result.error.isEmpty(); retry++) { + QThread::msleep(static_cast(RETRY_DELAY_MS * retry)); + result = DownloadOnce(); + } + return result; +} + +} // namespace + +CompatibilityDatabase::CompatibilityDatabase(bool local, QObject* parent) + : QObject(parent), m_local(local) {} + +const CompatibilityEntry* CompatibilityDatabase::Find(const QString& title_id) const { + const auto entry = m_entries.constFind(TitleKey(title_id)); + return entry != m_entries.constEnd() ? &entry.value() : nullptr; +} + +void CompatibilityDatabase::Load() { + if (m_local) { + QFile file(QDir(".").absoluteFilePath(FILE_NAME)); + if (!file.exists()) { + emit Updated(); + return; + } + if (!file.open(QIODevice::ReadOnly)) { + qWarning() << "Could not open compatibility database:" << file.errorString(); + return; + } + auto result = Parse(file.readAll()); + if (!result.error.isEmpty()) { + qWarning() << result.error; + return; + } + m_entries = std::move(result.entries); + emit Updated(); + return; + } + + auto* watcher = new QFutureWatcher(this); + connect(watcher, &QFutureWatcher::finished, this, [this, watcher]() { + const auto result = watcher->result(); + watcher->deleteLater(); + if (!result.error.isEmpty()) { + qWarning() << "Could not refresh compatibility database:" << result.error; + return; + } + m_entries = result.entries; + emit Updated(); + }); + watcher->setFuture(QtConcurrent::run(Download)); +} + +void CompatibilityDatabase::SetStatus(const QString& title_id, Configuration::GameStatus status) { + const auto key = TitleKey(title_id); + if (!m_local || key.isEmpty()) { + return; + } + m_entries[key].status = status; + Save(); +} + +void CompatibilityDatabase::SetComment(const QString& title_id, const QString& comment) { + const auto key = TitleKey(title_id); + if (!m_local || key.isEmpty()) { + return; + } + m_entries[key].comment = comment; + Save(); +} + +void CompatibilityDatabase::Save() const { + QJsonObject root; + for (auto it = m_entries.constBegin(); it != m_entries.constEnd(); ++it) { + root.insert(it.key(), + QJsonObject {{QStringLiteral("status"), StatusToText(it.value().status)}, + {QStringLiteral("comment"), it.value().comment}}); + } + + QSaveFile file(QDir(".").absoluteFilePath(FILE_NAME)); + const auto data = QJsonDocument(root).toJson(QJsonDocument::Indented); + if (!file.open(QIODevice::WriteOnly) || file.write(data) != data.size() || !file.commit()) { + qWarning() << "Could not save compatibility database:" << file.errorString(); + } +} diff --git a/src/launcher/src/configurationEditDialog.cpp b/src/launcher/src/configurationEditDialog.cpp new file mode 100644 index 000000000..13141ee39 --- /dev/null +++ b/src/launcher/src/configurationEditDialog.cpp @@ -0,0 +1,331 @@ +#include "configurationEditDialog.h" + +#include "configuration.h" +#include "mandatoryLineEdit.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "ui_configuration_edit_dialog.h" + +constexpr char SETTINGS_CFG_DIALOG[] = "ConfigurationEditDialog"; +constexpr char SETTINGS_CFG_LAST_GEOMETRY[] = "geometry"; +constexpr int GLOBAL_SETTINGS_GAME_DIRS_MIN_WIDTH = 560; + +static QString NormalizeGameDirectory(const QString& dir) { + const auto trimmed = dir.trimmed(); + if (trimmed.isEmpty()) { + return {}; + } + + return QDir::cleanPath(QDir(trimmed).absolutePath()); +} + +static QString GameDirectoryKey(const QString& dir) { + const auto normalized = NormalizeGameDirectory(dir); + if (normalized.isEmpty()) { + return {}; + } + + auto canonical = QFileInfo(normalized).canonicalFilePath(); + if (canonical.isEmpty()) { + canonical = normalized; + } + canonical = QDir::cleanPath(canonical); + +#ifdef __linux__ + return canonical; +#else + return canonical.toCaseFolded(); +#endif +} + +ConfigurationEditDialog::ConfigurationEditDialog(Configuration& info, QWidget* parent) + : QDialog(parent, Qt::WindowCloseButtonHint), m_ui(new Ui::ConfigurationEditDialog), + m_info(info) { + m_ui->setupUi(this); + InitGameDirectories(); + + connect(m_ui->ok_button, &QPushButton::clicked, this, &ConfigurationEditDialog::save); + connect(m_ui->clear_button, &QPushButton::clicked, this, &ConfigurationEditDialog::clear); + connect(m_ui->comboBox_shader_log_direction, &QComboBox::currentTextChanged, this, + [this](const QString& text) { + auto log = TextToEnum(text); + m_ui->lineEdit_shader_log_folder->setEnabled( + log == Configuration::ShaderLogDirection::File); + }); + connect(m_ui->checkBox_cmd_dump, &QCheckBox::toggled, this, + [this](bool flag) { m_ui->lineEdit_cmd_dump_folder->setEnabled(flag); }); + connect(m_ui->comboBox_printf_direction, &QComboBox::currentTextChanged, this, + [this](const QString& text) { + auto log = TextToEnum(text); + m_ui->lineEdit_printf_file->setEnabled(log == Configuration::LogDirection::File); + }); + + layout()->setSizeConstraint(QLayout::SetFixedSize); + + restoreGeometry(g_last_geometry); + + Init(info); +} + +QByteArray ConfigurationEditDialog::g_last_geometry; + +ConfigurationEditDialog::~ConfigurationEditDialog() { + delete m_ui; +} + +void ConfigurationEditDialog::WriteSettings(QSettings& s) { + s.beginGroup(SETTINGS_CFG_DIALOG); + + if (!g_last_geometry.isEmpty()) { + s.setValue(SETTINGS_CFG_LAST_GEOMETRY, g_last_geometry); + } + + s.endGroup(); +} + +void ConfigurationEditDialog::ReadSettings(QSettings& s) { + s.beginGroup(SETTINGS_CFG_DIALOG); + + g_last_geometry = s.value(SETTINGS_CFG_LAST_GEOMETRY, g_last_geometry).toByteArray(); + + s.endGroup(); +} + +template +static void ListInit(QComboBox* combo, T value) { + combo->clear(); + combo->addItems(EnumToList()); + combo->setCurrentText(EnumToText(value)); +} + +void ConfigurationEditDialog::Init(const Configuration& info) { + ListInit(m_ui->comboBox_screen_resolution, info.screen_resolution); + m_ui->spinBox_vblank_frequency->setValue(info.vblank_frequency); + m_ui->checkBox_shader_validation->setChecked(info.shader_validation_enabled); + m_ui->checkBox_vulkan_validation->setChecked(info.vulkan_validation_enabled); + m_ui->checkBox_renderdoc_capture->setChecked(info.renderdoc_enabled); + m_ui->checkBox_ngg_rectlist_draw->setChecked(info.ngg_rectlist_draw_enabled); + ListInit(m_ui->comboBox_shader_optimization_type, info.shader_optimization_type); + ListInit(m_ui->comboBox_shader_log_direction, info.shader_log_direction); + m_ui->lineEdit_shader_log_folder->setText(info.shader_log_folder); + m_ui->lineEdit_shader_log_folder->setEnabled(info.shader_log_direction == + Configuration::ShaderLogDirection::File); + m_ui->checkBox_cmd_dump->setChecked(info.command_buffer_dump_enabled); + m_ui->lineEdit_cmd_dump_folder->setText(info.command_buffer_dump_folder); + m_ui->lineEdit_cmd_dump_folder->setEnabled(info.command_buffer_dump_enabled); + ListInit(m_ui->comboBox_printf_direction, info.printf_direction); + m_ui->lineEdit_printf_file->setText(info.printf_output_file); + m_ui->lineEdit_printf_file->setEnabled(info.printf_direction == + Configuration::LogDirection::File); + ListInit(m_ui->comboBox_profiler_direction, info.profiler_direction); +} + +void ConfigurationEditDialog::InitGameDirectories() { + m_game_dirs_group = new QGroupBox(tr("Game folders"), this); + + auto* group_layout = new QVBoxLayout(m_game_dirs_group); + group_layout->setContentsMargins(8, 8, 8, 8); + group_layout->setSpacing(6); + + m_game_dirs_list = new QListWidget(m_game_dirs_group); + m_game_dirs_list->setMinimumHeight(120); + m_game_dirs_list->setSelectionMode(QAbstractItemView::ExtendedSelection); + group_layout->addWidget(m_game_dirs_list); + + auto* button_layout = new QHBoxLayout; + button_layout->setContentsMargins(0, 0, 0, 0); + button_layout->setSpacing(4); + + auto* add_button = new QToolButton(m_game_dirs_group); + add_button->setIcon(style()->standardIcon(QStyle::SP_DirOpenIcon)); + add_button->setToolTip(tr("Add game folder")); + add_button->setAutoRaise(true); + button_layout->addWidget(add_button); + + m_remove_game_dir_button = new QToolButton(m_game_dirs_group); + m_remove_game_dir_button->setIcon(style()->standardIcon(QStyle::SP_DialogDiscardButton)); + m_remove_game_dir_button->setToolTip(tr("Remove selected game folders")); + m_remove_game_dir_button->setAutoRaise(true); + button_layout->addWidget(m_remove_game_dir_button); + + button_layout->addStretch(1); + group_layout->addLayout(button_layout); + + m_ui->gridLayout->addWidget(m_game_dirs_group, 1, 0, 1, 1); + m_game_dirs_group->setVisible(false); + + connect(add_button, &QToolButton::clicked, this, &ConfigurationEditDialog::add_game_directory); + connect(m_remove_game_dir_button, &QToolButton::clicked, this, + &ConfigurationEditDialog::remove_selected_game_directories); + connect(m_game_dirs_list, &QListWidget::itemSelectionChanged, this, + &ConfigurationEditDialog::update_game_directory_buttons); + + update_game_directory_buttons(); +} + +void ConfigurationEditDialog::SetTitle(const QString& str) { + setWindowTitle(str); +} + +void ConfigurationEditDialog::SetGameDirectories(const QStringList& dirs) { + m_show_game_dirs = true; + m_game_dirs_list->clear(); + m_game_dirs_group->setMinimumWidth(GLOBAL_SETTINGS_GAME_DIRS_MIN_WIDTH); + + for (const auto& dir: dirs) { + AddGameDirectoryItem(dir); + } + + m_game_dirs_group->setVisible(true); + update_game_directory_buttons(); + adjustSize(); +} + +QStringList ConfigurationEditDialog::GetGameDirectories() const { + QStringList dirs; + if (!m_show_game_dirs) { + return dirs; + } + + for (int index = 0; index < m_game_dirs_list->count(); index++) { + auto* item = m_game_dirs_list->item(index); + dirs.append(item->text()); + } + + return dirs; +} + +void ConfigurationEditDialog::AddGameDirectoryItem(const QString& dir) { + const auto normalized = NormalizeGameDirectory(dir); + const auto key = GameDirectoryKey(normalized); + if (normalized.isEmpty() || key.isEmpty()) { + return; + } + + for (int index = 0; index < m_game_dirs_list->count(); index++) { + auto* item = m_game_dirs_list->item(index); + if (GameDirectoryKey(item->text()) == key) { + return; + } + } + + auto* item = new QListWidgetItem(normalized, m_game_dirs_list); + item->setToolTip(normalized); +} + +void ConfigurationEditDialog::moveEvent(QMoveEvent* event) { + QDialog::moveEvent(event); + g_last_geometry = saveGeometry(); +} + +static void UpdateInfo(Configuration& info, Ui::ConfigurationEditDialog& ui) { + info.screen_resolution = + TextToEnum(ui.comboBox_screen_resolution->currentText()); + info.vblank_frequency = ui.spinBox_vblank_frequency->value(); + info.vulkan_validation_enabled = ui.checkBox_vulkan_validation->isChecked(); + info.shader_validation_enabled = ui.checkBox_shader_validation->isChecked(); + info.renderdoc_enabled = ui.checkBox_renderdoc_capture->isChecked(); + info.ngg_rectlist_draw_enabled = ui.checkBox_ngg_rectlist_draw->isChecked(); + info.shader_optimization_type = TextToEnum( + ui.comboBox_shader_optimization_type->currentText()); + info.shader_log_direction = TextToEnum( + ui.comboBox_shader_log_direction->currentText()); + info.shader_log_folder = ui.lineEdit_shader_log_folder->text(); + info.command_buffer_dump_enabled = ui.checkBox_cmd_dump->isChecked(); + info.command_buffer_dump_folder = ui.lineEdit_cmd_dump_folder->text(); + info.printf_direction = + TextToEnum(ui.comboBox_printf_direction->currentText()); + info.printf_output_file = ui.lineEdit_printf_file->text(); + info.profiler_direction = + TextToEnum(ui.comboBox_profiler_direction->currentText()); +} + +void ConfigurationEditDialog::update_info() { + UpdateInfo(m_info, *m_ui); +} + +void ConfigurationEditDialog::adjust_size() { + this->adjustSize(); +} + +void ConfigurationEditDialog::save() { + if (MandatoryLineEdit::FindEmpty(this)) { + QMessageBox::critical(this, tr("Save failed"), tr("Please fill all mandatory fields")); + return; + } + + update_info(); + + emit accept(); +} + +void ConfigurationEditDialog::clear() { + Configuration default_info; + Init(default_info); + + if (m_show_game_dirs) { + m_game_dirs_list->clear(); + update_game_directory_buttons(); + } +} + +void ConfigurationEditDialog::add_game_directory() { + QString start_dir = QDir::homePath(); + if (m_game_dirs_list->count() > 0) { + start_dir = m_game_dirs_list->item(m_game_dirs_list->count() - 1)->text(); + } + + QFileDialog dialog(this, tr("Select game folders"), start_dir); + dialog.setFileMode(QFileDialog::Directory); + dialog.setOption(QFileDialog::ShowDirsOnly, true); + dialog.setOption(QFileDialog::DontUseNativeDialog, true); + dialog.setLabelText(QFileDialog::Accept, tr("Add")); + + auto* list_view = dialog.findChild(QStringLiteral("listView")); + if (list_view != nullptr) { + list_view->setSelectionMode(QAbstractItemView::ExtendedSelection); + } + auto* tree_view = dialog.findChild(QStringLiteral("treeView")); + if (tree_view != nullptr) { + tree_view->setSelectionMode(QAbstractItemView::ExtendedSelection); + } + + if (dialog.exec() != QDialog::Accepted) { + return; + } + + for (const auto& dir: dialog.selectedFiles()) { + AddGameDirectoryItem(dir); + } + + update_game_directory_buttons(); +} + +void ConfigurationEditDialog::remove_selected_game_directories() { + qDeleteAll(m_game_dirs_list->selectedItems()); + update_game_directory_buttons(); +} + +void ConfigurationEditDialog::update_game_directory_buttons() { + m_remove_game_dir_button->setEnabled(!m_game_dirs_list->selectedItems().isEmpty()); +} diff --git a/src/launcher/src/configurationItem.cpp b/src/launcher/src/configurationItem.cpp new file mode 100644 index 000000000..8901c5b0d --- /dev/null +++ b/src/launcher/src/configurationItem.cpp @@ -0,0 +1,257 @@ +#include "configurationItem.h" + +#include "configuration.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace { + +enum Column { + NameColumn, + SerialColumn, + FirmwareVersionColumn, + PathColumn, + StatusColumn, + CommentsColumn, +}; + +class NoWheelComboBox: public QComboBox { +public: + explicit NoWheelComboBox(QWidget* parent = nullptr): QComboBox(parent) {} + +protected: + void wheelEvent(QWheelEvent* event) override { event->ignore(); } +}; + +QString GetStatusText(Configuration::GameStatus status) { + switch (status) { + case Configuration::GameStatus::Unknown: return QStringLiteral("Unknown"); + case Configuration::GameStatus::MainMenu: return QStringLiteral("Main menu"); + case Configuration::GameStatus::InGame: return QStringLiteral("In game"); + case Configuration::GameStatus::Logo: return QStringLiteral("Logo"); + case Configuration::GameStatus::DoesntBoot: return QStringLiteral("Doesn't boot"); + } + + return QStringLiteral("Unknown"); +} + +QString GetStatusColor(Configuration::GameStatus status) { + switch (status) { + case Configuration::GameStatus::InGame: return QStringLiteral("#2fb344"); + case Configuration::GameStatus::MainMenu: return QStringLiteral("#2f80ed"); + case Configuration::GameStatus::Logo: return QStringLiteral("#f2c94c"); + case Configuration::GameStatus::DoesntBoot: return QStringLiteral("#e55353"); + case Configuration::GameStatus::Unknown: return QStringLiteral("#8a8a8a"); + } + + return QStringLiteral("#8a8a8a"); +} + +void AddStatus(QComboBox* combo, Configuration::GameStatus status) { + combo->addItem(GetStatusText(status), static_cast(status)); +} + +void SetStatus(QComboBox* combo, Configuration::GameStatus status) { + const int index = combo->findData(static_cast(status)); + combo->setCurrentIndex(index >= 0 ? index : 0); +} + +QIcon StandardIcon(QStyle::StandardPixmap icon) { + return QApplication::style()->standardIcon(icon); +} + +QString GetPathText(const Configuration& info) { + return !info.game_path.isEmpty() ? info.game_path : info.basedir; +} + +QString GetDisplayText(const Configuration& info) { + QStringList lines({info.name}); + + if (!info.title_id.isEmpty()) { + lines.append(QStringLiteral("Serial ID: %1").arg(info.title_id)); + } + if (!info.firmwareVer.isEmpty()) { + lines.append(QStringLiteral("Firmware version: %1").arg(info.firmwareVer)); + } + + const auto path = GetPathText(info); + if (!path.isEmpty()) { + lines.append(QStringLiteral("Path: %1").arg(path)); + } + if (!info.game_comment.isEmpty()) { + lines.append(QStringLiteral("Comment: %1").arg(info.game_comment)); + } + + return lines.join(QLatin1Char('\n')); +} + +QString GetSortText(const Configuration& info) { + return QStringLiteral("%1\n%2").arg(info.name.toCaseFolded(), info.game_path.toCaseFolded()); +} + +void MakeTransparent(QWidget* widget) { + widget->setAutoFillBackground(false); + widget->setAttribute(Qt::WA_NoSystemBackground); + widget->setAttribute(Qt::WA_TranslucentBackground); +} + +} // namespace + +ConfigurationItem::ConfigurationItem(std::unique_ptr info, QTreeWidget* parent) + : QTreeWidgetItem(parent), m_info(std::move(info)) { + setSizeHint(NameColumn, QSize(0, 72)); + + m_status_widget = new QWidget(parent); + MakeTransparent(m_status_widget); + auto* layout = new QHBoxLayout(m_status_widget); + layout->setContentsMargins(4, 0, 4, 0); + layout->setSpacing(6); + + m_status_indicator = new QLabel(m_status_widget); + m_status_indicator->setFixedSize(QSize(12, 12)); + layout->addWidget(m_status_indicator); + + m_status_combo = new NoWheelComboBox(m_status_widget); + MakeTransparent(m_status_combo); + AddStatus(m_status_combo, Configuration::GameStatus::Unknown); + AddStatus(m_status_combo, Configuration::GameStatus::MainMenu); + AddStatus(m_status_combo, Configuration::GameStatus::InGame); + AddStatus(m_status_combo, Configuration::GameStatus::Logo); + AddStatus(m_status_combo, Configuration::GameStatus::DoesntBoot); + m_status_combo->setSizeAdjustPolicy(QComboBox::AdjustToContents); + m_status_combo->setFixedWidth(125); + m_status_combo->setStyleSheet(QStringLiteral( + "QComboBox { background: transparent; border: 0; color: #eef4ff; padding-left: 2px; }" + "QComboBox:focus { background: rgba(255,255,255,35); border-radius: 3px; }" + "QComboBox::drop-down { border: 0; }" + "QComboBox QAbstractItemView { background: #181c24; color: #eef4ff; " + "selection-background-color: #267bd8; }")); + layout->addWidget(m_status_combo); + layout->addStretch(1); + + parent->setItemWidget(this, StatusColumn, m_status_widget); + + m_comment_edit = new QLineEdit(parent); + MakeTransparent(m_comment_edit); + m_comment_edit->setClearButtonEnabled(true); + m_comment_edit->setFrame(false); + m_comment_edit->setStyleSheet(QStringLiteral( + "QLineEdit { background: transparent; border: 0; color: #eef4ff; padding-left: 2px; " + "selection-background-color: #267bd8; }" + "QLineEdit:focus { background: rgba(255,255,255,35); " + "border: 1px solid rgba(255,255,255,90); border-radius: 3px; }")); + parent->setItemWidget(this, CommentsColumn, m_comment_edit); + + Update(); + SetRunning(false); +} + +ConfigurationItem::~ConfigurationItem() = default; + +void ConfigurationItem::Update() { + const auto display_text = GetDisplayText(*m_info); + const auto path = GetPathText(*m_info); + + setText(NameColumn, m_info->name); + setText(SerialColumn, m_info->title_id); + setText(FirmwareVersionColumn, + m_info->firmwareVer.isEmpty() ? QStringLiteral("\u2014") : m_info->firmwareVer); + setText(PathColumn, path); + setText(StatusColumn, {}); + setText(CommentsColumn, {}); + for (int column = NameColumn; column <= CommentsColumn; column++) { + setToolTip(column, display_text); + } + SetStatus(m_status_combo, m_info->game_status); + if (m_comment_edit->text() != m_info->game_comment) { + m_comment_edit->setText(m_info->game_comment); + } + + UpdateIcon(); + UpdateStatusIndicator(); +} + +bool ConfigurationItem::operator<(const QTreeWidgetItem& other) const { + const auto* other_item = dynamic_cast(&other); + if (other_item == nullptr) { + return QTreeWidgetItem::operator<(other); + } + + const int column = treeWidget() != nullptr ? treeWidget()->sortColumn() : NameColumn; + switch (column) { + case NameColumn: return GetSortText(*m_info) < GetSortText(*other_item->m_info); + case StatusColumn: + return GetStatusText(m_info->game_status) < + GetStatusText(other_item->m_info->game_status); + case FirmwareVersionColumn: { + const auto& version = m_info->firmwareVer; + const auto& other_version = other_item->m_info->firmwareVer; + if (version.isEmpty() || other_version.isEmpty()) { + return version.isEmpty() && !other_version.isEmpty(); + } + return QVersionNumber::compare(QVersionNumber::fromString(version), + QVersionNumber::fromString(other_version)) < 0; + } + case CommentsColumn: + return m_info->game_comment.toCaseFolded() < + other_item->m_info->game_comment.toCaseFolded(); + default: return text(column).toCaseFolded() < other.text(column).toCaseFolded(); + } +} + +void ConfigurationItem::SetRunning(bool state) { + m_running = state; + + QFont f = font(NameColumn); + f.setBold(state); + for (int column = NameColumn; column <= CommentsColumn; column++) { + setFont(column, f); + } + + UpdateIcon(); +} + +void ConfigurationItem::SetCompatibilityEditable(bool editable) { + m_status_combo->setEnabled(editable); + m_comment_edit->setReadOnly(!editable); + m_comment_edit->setClearButtonEnabled(editable); +} + +void ConfigurationItem::UpdateIcon() { + const QString icon_file = QDir(m_info->basedir).filePath(QStringLiteral("sce_sys/icon0.png")); + if (QFileInfo::exists(icon_file)) { + setIcon(NameColumn, QIcon(icon_file)); + return; + } + + if (m_running) { + setIcon(NameColumn, StandardIcon(QStyle::SP_MediaPlay)); + } else if (m_info->custom_settings) { + setIcon(NameColumn, StandardIcon(QStyle::SP_FileIcon)); + } else { + setIcon(NameColumn, StandardIcon(QStyle::SP_ComputerIcon)); + } +} + +void ConfigurationItem::UpdateStatusIndicator() { + m_status_indicator->setStyleSheet( + QStringLiteral( + "background-color: %1; border: 1px solid rgba(255, 255, 255, 90); border-radius: 6px;") + .arg(GetStatusColor(m_info->game_status))); + m_status_indicator->setToolTip(GetStatusText(m_info->game_status)); +} diff --git a/src/launcher/src/configurationListWidget.cpp b/src/launcher/src/configurationListWidget.cpp new file mode 100644 index 000000000..1125424cf --- /dev/null +++ b/src/launcher/src/configurationListWidget.cpp @@ -0,0 +1,908 @@ +#include "configurationListWidget.h" + +#include "common.h" +#include "compatibilityDatabase.h" +#include "configuration.h" +#include "configurationEditDialog.h" +#include "configurationItem.h" +#include "gameListTreeWidget.h" +#include "mainDialog.h" +#include "trophyViewerDialog.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "ui_configuration_list_widget.h" + +#include + +constexpr char CONF_FILE_NAME[] = "Kyty.ini"; +constexpr char CONF_ORG_NAME[] = "Kyty"; +constexpr char CONF_APP_NAME[] = "Kyty"; +constexpr char CONF_SECTION_NAME[] = "GameConfigurations"; +constexpr char CONF_LAUNCHER[] = "Launcher"; +constexpr char CONF_GAME_DIR[] = "game_dir"; +constexpr char CONF_GAME_DIRS[] = "game_dirs"; +constexpr char CONF_GLOBAL[] = "GlobalConfiguration"; +constexpr char SAVE_DATA_DIR[] = "_SaveData"; + +constexpr int GAME_NAME_COLUMN = 0; +constexpr int GAME_SERIAL_COLUMN = 1; +constexpr int GAME_FIRMWARE_VERSION_COLUMN = 2; +constexpr int GAME_PATH_COLUMN = 3; +constexpr int GAME_STATUS_COLUMN = 4; +constexpr int GAME_COMMENT_COLUMN = 5; + +static QString NormalizeGameDirectory(const QString& dir) { + const auto trimmed = dir.trimmed(); + if (trimmed.isEmpty()) { + return {}; + } + + return QDir::cleanPath(QDir(trimmed).absolutePath()); +} + +static QString PathKey(const QString& path) { + auto normalized = NormalizeGameDirectory(path); + if (normalized.isEmpty()) { + normalized = QDir::cleanPath(path.trimmed()); + } + if (normalized.isEmpty()) { + return {}; + } + + auto canonical = QFileInfo(normalized).canonicalFilePath(); + if (canonical.isEmpty()) { + canonical = normalized; + } + canonical = QDir::cleanPath(canonical); + +#ifdef __linux__ + return canonical; +#else + return canonical.toCaseFolded(); +#endif +} + +static QStringList NormalizeGameDirectories(const QStringList& dirs) { + QStringList dirs_ret; + QSet seen; + + for (const auto& dir: dirs) { + const auto normalized = NormalizeGameDirectory(dir); + const auto key = PathKey(normalized); + if (normalized.isEmpty() || key.isEmpty() || seen.contains(key)) { + continue; + } + + seen.insert(key); + dirs_ret.append(normalized); + } + + return dirs_ret; +} + +static QStringList SettingsStringList(const QVariant& value) { + auto list = value.toStringList(); + if (!list.isEmpty()) { + return list; + } + + const auto text = value.toString(); + return text.isEmpty() ? QStringList() : QStringList({text}); +} + +static QString GetPic0Path(const Configuration& info) { + if (info.basedir.isEmpty()) { + return {}; + } + + const auto path = QDir(info.basedir).filePath(QStringLiteral("sce_sys/pic0.png")); + return QFileInfo::exists(path) ? QFileInfo(path).absoluteFilePath() : QString(); +} + +static void ApplyGameListStyle(Ui::ConfigurationListWidget* ui) { + ui->cfgs_list->setAlternatingRowColors(false); + ui->cfgs_list->setAllColumnsShowFocus(true); + ui->cfgs_list->setIndentation(0); + ui->cfgs_list->setMouseTracking(false); + ui->cfgs_list->setSelectionBehavior(QAbstractItemView::SelectRows); + ui->cfgs_list->setSelectionMode(QAbstractItemView::SingleSelection); + ui->cfgs_list->setTextElideMode(Qt::ElideMiddle); + ui->cfgs_list->header()->setDefaultAlignment(Qt::AlignLeft | Qt::AlignVCenter); + ui->cfgs_list->header()->setHighlightSections(false); + ui->cfgs_list->header()->setStretchLastSection(true); + + ui->cfgs_list->setStyleSheet(QStringLiteral( + "QTreeWidget { background: transparent; border: 1px solid rgba(255,255,255,42); " + "color: #eef4ff; outline: 0; }" + "QTreeWidget::item { background: rgba(8,10,16,92); " + "border-bottom: 1px solid rgba(255,255,255,24); padding: 4px; }" + "QTreeWidget::item:selected { background: rgba(32,118,210,150); color: #ffffff; }" + "QHeaderView::section { background: rgba(22,25,31,235); color: #dce6f6; border: 0; " + "border-right: 1px solid rgba(255,255,255,35); border-bottom: 1px solid " + "rgba(255,255,255,45); " + "padding: 5px 6px; }" + "QScrollBar:vertical { background: rgba(12,14,18,150); width: 14px; margin: 0; }" + "QScrollBar::handle:vertical { background: rgba(220,230,245,120); min-height: 36px; }" + "QScrollBar::add-line:vertical, QScrollBar::sub-line:vertical { height: 0; }")); + + ui->search_line_edit->setStyleSheet(QStringLiteral( + "QLineEdit { background: rgba(12,14,18,210); border: 1px solid rgba(255,255,255,42); " + "border-radius: 4px; color: #eef4ff; padding: 5px 8px; selection-background-color: " + "#267bd8; }" + "QLineEdit:focus { border-color: rgba(80,160,255,180); }")); + ui->global_settings_button->setStyleSheet(QStringLiteral( + "QToolButton { background: rgba(255,255,255,18); border: 1px solid transparent; " + "border-radius: 5px; padding: 3px; }" + "QToolButton:hover { background: rgba(255,255,255,45); border-color: rgba(255,255,255,70); " + "}" + "QToolButton:disabled { background: transparent; }")); + ui->edit_button->setStyleSheet(ui->global_settings_button->styleSheet()); + ui->delete_button->setStyleSheet(ui->global_settings_button->styleSheet()); +} + +static void AddSaveDataDir(QStringList* dirs, QSet* seen, const QString& root, + const QString& title_id) { + const auto path = QDir(root).filePath( + QStringLiteral("%1/%2").arg(QString::fromLatin1(SAVE_DATA_DIR), title_id)); + QDir dir(path); + if (!dir.exists()) { + return; + } + + const auto absolute = dir.absolutePath(); + auto canonical = QFileInfo(absolute).canonicalFilePath(); + if (canonical.isEmpty()) { + canonical = absolute; + } + canonical = QDir::cleanPath(canonical); + + if (!seen->contains(canonical)) { + seen->insert(canonical); + dirs->append(absolute); + } +} + +static QStringList GetSaveDataDirs(const Configuration& info) { + QStringList dirs; + if (info.title_id.trimmed().isEmpty()) { + return dirs; + } + + QSet seen; + QStringList roots({QDir::currentPath(), QCoreApplication::applicationDirPath()}); + + QDir current_parent(QDir::currentPath()); + if (current_parent.cdUp()) { + roots.append(current_parent.absolutePath()); + } + + QDir app_parent(QCoreApplication::applicationDirPath()); + if (app_parent.cdUp()) { + roots.append(app_parent.absolutePath()); + } + + for (const auto& root: roots) { + AddSaveDataDir(&dirs, &seen, root, info.title_id.trimmed()); + } + + return dirs; +} + +ConfigurationListWidget::ConfigurationListWidget(QWidget* parent) + : QWidget(parent), m_ui(new Ui::ConfigurationListWidget) { + m_compatibility = new CompatibilityDatabase( + QCoreApplication::arguments().contains(QStringLiteral("--local")), this); + m_ui->setupUi(this); + ApplyGameListStyle(m_ui); + + m_ui->global_settings_button->setIcon(style()->standardIcon(QStyle::SP_FileDialogDetailedView)); + m_ui->global_settings_button->setToolTip(tr("Edit global settings and game folders")); + m_ui->edit_button->setIcon(style()->standardIcon(QStyle::SP_FileIcon)); + m_ui->delete_button->setIcon(style()->standardIcon(QStyle::SP_DialogDiscardButton)); + + m_ui->delete_button->setEnabled(false); + m_ui->edit_button->setEnabled(false); + + m_ui->cfgs_list->setContextMenuPolicy(Qt::CustomContextMenu); + m_ui->cfgs_list->setIconSize(QSize(48, 48)); + m_ui->cfgs_list->setRootIsDecorated(false); + m_ui->cfgs_list->setUniformRowHeights(true); + m_ui->cfgs_list->setSortingEnabled(true); + m_ui->cfgs_list->setColumnWidth(GAME_NAME_COLUMN, 320); + m_ui->cfgs_list->setColumnWidth(GAME_SERIAL_COLUMN, 110); + m_ui->cfgs_list->setColumnWidth(GAME_FIRMWARE_VERSION_COLUMN, 150); + m_ui->cfgs_list->setColumnWidth(GAME_PATH_COLUMN, 320); + m_ui->cfgs_list->setColumnWidth(GAME_STATUS_COLUMN, 150); + m_ui->cfgs_list->setColumnWidth(GAME_COMMENT_COLUMN, 240); + + connect(m_ui->global_settings_button, &QToolButton::clicked, this, + &ConfigurationListWidget::edit_global_settings); + connect(m_ui->edit_button, &QToolButton::clicked, this, + &ConfigurationListWidget::edit_configuration); + connect(m_ui->delete_button, &QToolButton::clicked, this, + &ConfigurationListWidget::delete_configuartion); + connect(m_ui->cfgs_list, &QTreeWidget::currentItemChanged, this, + &ConfigurationListWidget::list_currentItemChanged); + connect(m_ui->cfgs_list, &QTreeWidget::itemDoubleClicked, this, + &ConfigurationListWidget::list_itemDoubleClicked); + connect(m_ui->cfgs_list, &QTreeWidget::customContextMenuRequested, this, + &ConfigurationListWidget::show_context_menu); + connect(m_ui->search_line_edit, &QLineEdit::textChanged, this, + &ConfigurationListWidget::filter_configurations); + connect(m_compatibility, &CompatibilityDatabase::Updated, this, + &ConfigurationListWidget::ApplyCompatibility); + + m_ui->cfgs_list->setDragDropMode(QAbstractItemView::NoDragDrop); + + ReadSettings(); + if (m_compatibility->IsLocal()) { + m_compatibility->Load(); + } + ScanGameDirectory(); + if (!m_compatibility->IsLocal()) { + m_compatibility->Load(); + } +} + +ConfigurationListWidget::~ConfigurationListWidget() { + qDeleteAll(m_custom_infos); + delete m_ui; +} + +void ConfigurationListWidget::WriteSettings() { + QFile file = QFile(QDir(".").absoluteFilePath(CONF_FILE_NAME)); + std::unique_ptr s; + if (file.exists()) { + s = std::make_unique(CONF_FILE_NAME, QSettings::IniFormat); + } else { +#ifdef __linux__ + s = std::make_unique(QSettings::IniFormat, QSettings::UserScope, CONF_ORG_NAME, + CONF_APP_NAME); +#else + s = std::make_unique(QSettings::IniFormat, QSettings::SystemScope, CONF_ORG_NAME, + CONF_APP_NAME); +#endif + } + + MainDialog::WriteSettings(*s); + ConfigurationEditDialog::WriteSettings(*s); + + s->beginGroup(CONF_LAUNCHER); + m_game_dirs = NormalizeGameDirectories(m_game_dirs); + s->setValue(CONF_GAME_DIRS, m_game_dirs); + s->remove(CONF_GAME_DIR); + s->endGroup(); + + s->remove(CONF_GLOBAL); + s->beginGroup(CONF_GLOBAL); + m_global_info.WriteSettings(s.get()); + s->endGroup(); + + s->remove(CONF_SECTION_NAME); + s->beginWriteArray(CONF_SECTION_NAME); + int i = 0; + for (auto it = m_custom_infos.constBegin(); it != m_custom_infos.constEnd(); ++it) { + s->setArrayIndex(i++); + it.value()->WriteSettings(s.get()); + } + s->endArray(); +} + +void ConfigurationListWidget::ReadSettings() { + QFile file = QFile(QDir(".").absoluteFilePath(CONF_FILE_NAME)); + std::unique_ptr s; + if (file.exists()) { + s = std::make_unique(CONF_FILE_NAME, QSettings::IniFormat); + } else { +#ifdef __linux__ + s = std::make_unique(QSettings::IniFormat, QSettings::UserScope, CONF_ORG_NAME, + CONF_APP_NAME); +#else + s = std::make_unique(QSettings::IniFormat, QSettings::SystemScope, CONF_ORG_NAME, + CONF_APP_NAME); +#endif + } + + m_settings_file = s->fileName(); + + MainDialog::ReadSettings(*s); + ConfigurationEditDialog::ReadSettings(*s); + + s->beginGroup(CONF_LAUNCHER); + m_game_dirs = NormalizeGameDirectories(SettingsStringList(s->value(CONF_GAME_DIRS))); + if (m_game_dirs.isEmpty()) { + m_game_dirs = NormalizeGameDirectories(SettingsStringList(s->value(CONF_GAME_DIR))); + } + s->endGroup(); + + s->beginGroup(CONF_GLOBAL); + if (!s->childKeys().isEmpty()) { + m_global_info.ReadSettings(s.get()); + } + s->endGroup(); + + qDeleteAll(m_custom_infos); + m_custom_infos.clear(); + + int size = s->beginReadArray(CONF_SECTION_NAME); + + for (int i = 0; i < size; i++) { + s->setArrayIndex(i); + auto* info = new Configuration; + info->ReadSettings(s.get()); + info->custom_settings = true; + if (!info->game_path.isEmpty()) { + m_custom_infos.insert(info->game_path, info); + } else { + delete info; + } + } + s->endArray(); +} + +void ConfigurationListWidget::ApplyCompatibility() { + const bool sorting_enabled = m_ui->cfgs_list->isSortingEnabled(); + const int sort_column = m_ui->cfgs_list->sortColumn(); + const auto sort_order = m_ui->cfgs_list->header()->sortIndicatorOrder(); + m_ui->cfgs_list->setSortingEnabled(false); + + for (int index = 0; index < m_ui->cfgs_list->topLevelItemCount(); index++) { + auto* item = static_cast(m_ui->cfgs_list->topLevelItem(index)); + const auto& title_id = item->GetInfo().title_id; + const auto* entry = m_compatibility->Find(title_id); + item->GetInfo().game_status = + entry != nullptr ? entry->status : Configuration::GameStatus::Unknown; + item->GetInfo().game_comment = entry != nullptr ? entry->comment : QString(); + + const QSignalBlocker blocker(item->GetStatusCombo()); + item->Update(); + item->SetCompatibilityEditable(m_compatibility->IsLocal() && !title_id.trimmed().isEmpty()); + } + + m_ui->cfgs_list->setSortingEnabled(sorting_enabled); + if (sorting_enabled) { + m_ui->cfgs_list->sortItems(sort_column, sort_order); + } +} + +static Configuration* CloneConfiguration(const Configuration& source) { + auto* ret = new Configuration; + ret->CopyFrom(source); + return ret; +} + +struct GameMetadata { + QString title_name; + QString title_id; + QString firmwareVer; +}; + +static QString GetJsonString(const QJsonObject& obj, const QString& key) { + return obj.value(key).toString().trimmed(); +} + +static QString GetLocalizedTitleName(const QJsonObject& root) { + const auto localized = root.value(QStringLiteral("localizedParameters")).toObject(); + if (localized.isEmpty()) { + return {}; + } + + const auto default_language = GetJsonString(localized, QStringLiteral("defaultLanguage")); + if (!default_language.isEmpty()) { + const auto title = GetJsonString(localized.value(default_language).toObject(), + QStringLiteral("titleName")); + if (!title.isEmpty()) { + return title; + } + } + + const auto english_title = GetJsonString(localized.value(QStringLiteral("en-US")).toObject(), + QStringLiteral("titleName")); + if (!english_title.isEmpty()) { + return english_title; + } + + for (auto it = localized.constBegin(); it != localized.constEnd(); ++it) { + const auto title = GetJsonString(it.value().toObject(), QStringLiteral("titleName")); + if (!title.isEmpty()) { + return title; + } + } + + return {}; +} + +static QString GetFirmwareVersion(const QJsonObject& root) { + const auto encoded = GetJsonString(root, QStringLiteral("requiredSystemSoftwareVersion")); + static const QRegularExpression version_pattern( + QStringLiteral("^0[xX]([0-9]{6})[0-9A-Fa-f]{10}$")); + const auto match = version_pattern.match(encoded); + if (!match.hasMatch()) { + return {}; + } + + const auto digits = match.captured(1); + bool valid = false; + const int major = digits.left(2).toInt(&valid, 10); + if (!valid) { + return {}; + } + + QString version = QStringLiteral("%1.%2").arg(major).arg(digits.mid(2, 2)); + const auto patch = digits.mid(4, 2); + if (patch != QStringLiteral("00")) { + version += QStringLiteral(".") + patch; + } + + return version; +} + +static GameMetadata GetGameMetadata(const QString& param_file, const QString& fallback) { + GameMetadata ret; + ret.title_name = fallback; + + QFile param(param_file); + if (!param.open(QIODevice::ReadOnly)) { + return ret; + } + + QJsonParseError parse_error; + const auto doc = QJsonDocument::fromJson(param.readAll(), &parse_error); + if (parse_error.error != QJsonParseError::NoError || !doc.isObject()) { + return ret; + } + + const auto root = doc.object(); + const auto title = GetLocalizedTitleName(root); + if (!title.isEmpty()) { + ret.title_name = title; + } + + ret.title_id = GetJsonString(root, QStringLiteral("titleId")); + ret.firmwareVer = GetFirmwareVersion(root); + + return ret; +} + +static void SetGameFiles(Configuration& info, const QString& game_dir, const QString& game_path, + const GameMetadata& metadata) { + QDir game(game_dir); + + info.game_path = game_path; + info.basedir = game.absolutePath(); + info.name = metadata.title_name; + info.title_id = metadata.title_id; + info.firmwareVer = metadata.firmwareVer; + + if (info.name.isEmpty()) { + info.name = game.dirName(); + } + if (info.elf.isEmpty()) { + info.elf = QStringLiteral("eboot.bin"); + } +} + +static Configuration* FindCustomInfo(QMap* custom_infos, + const QString& game_path, const QString& legacy_game_path) { + auto custom = custom_infos->find(game_path); + if (custom != custom_infos->end()) { + return custom.value(); + } + + custom = custom_infos->find(legacy_game_path); + if (custom == custom_infos->end()) { + return nullptr; + } + + auto* info = custom.value(); + custom_infos->erase(custom); + info->game_path = game_path; + custom_infos->insert(game_path, info); + + return info; +} + +bool ConfigurationListWidget::EnsureGameDirectory() { + if (HasValidGameDirectory()) { + return true; + } + + QMessageBox::information(this, tr("Game folders"), + tr("Add at least one game folder in global settings.")); + edit_global_settings(); + + return HasValidGameDirectory(); +} + +bool ConfigurationListWidget::HasValidGameDirectory() const { + for (const auto& dir: m_game_dirs) { + if (!dir.isEmpty() && QDir(dir).exists()) { + return true; + } + } + + return false; +} + +void ConfigurationListWidget::ScanGameDirectory() { + m_selected_item = nullptr; + m_ui->cfgs_list->clear(); + m_ui->cfgs_list->SetBackgroundImage({}); + m_ui->edit_button->setEnabled(false); + m_ui->delete_button->setEnabled(false); + + const QString eboot_name = QStringLiteral("eboot.bin"); + QSet found_games; + + for (const auto& root_path: m_game_dirs) { + QDir root(root_path); + if (root_path.isEmpty() || !root.exists()) { + continue; + } + + QList pending_dirs; + const auto root_subdirs = + root.entryInfoList(QDir::Dirs | QDir::NoDotAndDotDot | QDir::NoSymLinks); + for (const auto& subdir: root_subdirs) { + pending_dirs.append(QDir(subdir.absoluteFilePath())); + } + + while (!pending_dirs.isEmpty()) { + QDir game_dir = pending_dirs.takeFirst(); + + if (game_dir.exists(eboot_name)) { + const QString game_path = NormalizeGameDirectory(game_dir.absolutePath()); + const QString legacy_game_path = root.relativeFilePath(game_dir.absolutePath()); + const QString game_key = PathKey(game_path); + if (game_key.isEmpty() || found_games.contains(game_key)) { + continue; + } + found_games.insert(game_key); + + auto metadata = GetGameMetadata( + game_dir.filePath(QStringLiteral("sce_sys/param.json")), game_dir.dirName()); + + auto info = std::make_unique(); + auto* custom = FindCustomInfo(&m_custom_infos, game_path, legacy_game_path); + if (custom != nullptr) { + info->CopyFrom(*custom); + info->custom_settings = true; + } else { + info->CopyEmulatorSettingsFrom(m_global_info); + info->custom_settings = false; + } + + SetGameFiles(*info, game_dir.absolutePath(), game_path, metadata); + const auto* compatibility = m_compatibility->Find(info->title_id); + if (compatibility != nullptr) { + info->game_status = compatibility->status; + info->game_comment = compatibility->comment; + } + + auto* item = new ConfigurationItem(std::move(info), m_ui->cfgs_list); + item->SetCompatibilityEditable(m_compatibility->IsLocal() && + !item->GetInfo().title_id.trimmed().isEmpty()); + connect(item->GetStatusCombo(), &QComboBox::currentIndexChanged, this, + [this, item](int /*index*/) { + if (!m_compatibility->IsLocal()) { + return; + } + + const auto& title_id = item->GetInfo().title_id; + if (title_id.trimmed().isEmpty()) { + return; + } + item->GetInfo().game_status = static_cast( + item->GetStatusCombo()->currentData().toInt()); + item->Update(); + m_compatibility->SetStatus(title_id, item->GetInfo().game_status); + m_ui->cfgs_list->setCurrentItem(item); + SelectItem(item); + }); + connect(item->GetCommentEdit(), &QLineEdit::editingFinished, this, [this, item]() { + if (!m_compatibility->IsLocal()) { + return; + } + + const auto& title_id = item->GetInfo().title_id; + if (title_id.trimmed().isEmpty()) { + return; + } + item->GetInfo().game_comment = item->GetCommentEdit()->text(); + item->Update(); + m_compatibility->SetComment(title_id, item->GetInfo().game_comment); + m_ui->cfgs_list->setCurrentItem(item); + SelectItem(item); + }); + continue; + } + + const auto subdirs = + game_dir.entryInfoList(QDir::Dirs | QDir::NoDotAndDotDot | QDir::NoSymLinks); + for (const auto& subdir: subdirs) { + pending_dirs.append(QDir(subdir.absoluteFilePath())); + } + } + } + + m_ui->cfgs_list->sortItems(GAME_NAME_COLUMN, Qt::AscendingOrder); + filter_configurations(m_ui->search_line_edit->text()); +} + +void ConfigurationListWidget::edit_configuration() { + auto* item = static_cast(m_ui->cfgs_list->currentItem()); + if (item == nullptr) { + return; + } + + ConfigurationEditDialog dlg(item->GetInfo(), this); + dlg.SetTitle(tr("Edit game settings")); + + if (dlg.exec() == QDialog::Accepted) { + item->GetInfo().custom_settings = true; + auto game_path = item->GetInfo().game_path; + delete m_custom_infos.take(game_path); + m_custom_infos.insert(game_path, CloneConfiguration(item->GetInfo())); + WriteSettings(); + ScanGameDirectory(); + } +} + +void ConfigurationListWidget::delete_configuartion() { + auto* item = static_cast(m_ui->cfgs_list->currentItem()); + if (item == nullptr || !item->GetInfo().custom_settings) { + return; + } + + if (QMessageBox::Yes == QMessageBox::question(this, tr("Clear custom settings"), + tr("Do you want to clear custom settings?"))) { + ClearCustomSettings(item); + WriteSettings(); + ScanGameDirectory(); + } +} + +void ConfigurationListWidget::edit_global_settings() { + Configuration info; + info.CopyEmulatorSettingsFrom(m_global_info); + info.name = tr("Global settings"); + + ConfigurationEditDialog dlg(info, this); + dlg.SetTitle(tr("Global settings")); + dlg.SetGameDirectories(m_game_dirs); + + if (dlg.exec() == QDialog::Accepted) { + m_global_info.CopyEmulatorSettingsFrom(info); + m_game_dirs = NormalizeGameDirectories(dlg.GetGameDirectories()); + WriteSettings(); + ScanGameDirectory(); + } +} + +void ConfigurationListWidget::ClearCustomSettings(ConfigurationItem* item) { + delete m_custom_infos.take(item->GetInfo().game_path); +} + +void ConfigurationListWidget::run_configuration() { + emit Run(); +} + +bool ConfigurationListWidget::CanViewSelectedTrophies() const { + return m_selected_item != nullptr && + TrophyViewerDialog::HasTrophyData(&m_selected_item->GetInfo()); +} + +void ConfigurationListWidget::ViewTrophies() { + auto* current = static_cast(m_ui->cfgs_list->currentItem()); + auto* item = current != nullptr ? current : m_selected_item; + if (item == nullptr) { + return; + } + + TrophyViewerDialog::ShowForGame(&item->GetInfo(), this); +} + +void ConfigurationListWidget::open_game_folder() { + auto* item = static_cast(m_ui->cfgs_list->currentItem()); + if (item == nullptr) { + return; + } + + const QDir game_dir(item->GetInfo().basedir); + if (!game_dir.exists()) { + QMessageBox::warning(this, tr("Open game folder"), tr("Game folder does not exist.")); + return; + } + + if (!QDesktopServices::openUrl(QUrl::fromLocalFile(game_dir.absolutePath()))) { + QMessageBox::warning(this, tr("Open game folder"), tr("Could not open game folder.")); + } +} + +void ConfigurationListWidget::remove_save_data() { + auto* item = static_cast(m_ui->cfgs_list->currentItem()); + if (item == nullptr) { + return; + } + + const auto save_data_dirs = GetSaveDataDirs(item->GetInfo()); + if (save_data_dirs.isEmpty()) { + QMessageBox::information(this, tr("Remove save data"), + tr("No save data folder found for this game.")); + return; + } + + const auto title = + !item->GetInfo().name.isEmpty() ? item->GetInfo().name : item->GetInfo().title_id; + const auto text = + tr("Remove save data for \"%1\"?\n\nThis will delete:\n%2\n\nThis cannot be undone.") + .arg(title, save_data_dirs.join(QLatin1Char('\n'))); + + if (QMessageBox::Yes != QMessageBox::question(this, tr("Remove save data"), text)) { + return; + } + + QStringList failed_dirs; + for (const auto& path: save_data_dirs) { + QDir dir(path); + if (dir.exists() && !dir.removeRecursively()) { + failed_dirs.append(path); + } + } + + if (!failed_dirs.isEmpty()) { + QMessageBox::warning(this, tr("Remove save data"), + tr("Could not remove:\n%1").arg(failed_dirs.join(QLatin1Char('\n')))); + } +} + +void ConfigurationListWidget::filter_configurations(const QString& text) { + const auto query = text.trimmed(); + const bool has_query = !query.isEmpty(); + bool selection_is_shown = false; + + for (int item_index = 0; item_index < m_ui->cfgs_list->topLevelItemCount(); item_index++) { + auto* item = m_ui->cfgs_list->topLevelItem(item_index); + if (item == nullptr) { + continue; + } + + const bool match = !has_query || + item->text(GAME_NAME_COLUMN).contains(query, Qt::CaseInsensitive) || + item->text(GAME_SERIAL_COLUMN).contains(query, Qt::CaseInsensitive); + item->setHidden(!match); + + if (match && item == m_selected_item) { + selection_is_shown = true; + } + } + + if (m_selected_item != nullptr && !selection_is_shown) { + m_ui->cfgs_list->clearSelection(); + m_ui->cfgs_list->setCurrentItem(nullptr); + SelectItem(nullptr); + } +} + +void ConfigurationListWidget::SelectItem(QTreeWidgetItem* witem) { + auto* item = static_cast(witem); + if (item == nullptr) { + m_selected_item = nullptr; + m_ui->cfgs_list->SetBackgroundImage({}); + m_ui->edit_button->setEnabled(false); + m_ui->delete_button->setEnabled(false); + emit Select(); + return; + } + + m_ui->delete_button->setEnabled(!item->IsRunning() && item->GetInfo().custom_settings); + m_ui->edit_button->setEnabled(!item->IsRunning()); + + m_selected_item = item; + m_ui->cfgs_list->SetBackgroundImage(GetPic0Path(item->GetInfo())); + + emit Select(); +} + +void ConfigurationListWidget::list_currentItemChanged(QTreeWidgetItem* current, + QTreeWidgetItem* /*previous*/) { + SelectItem(current); +} + +void ConfigurationListWidget::list_itemDoubleClicked(QTreeWidgetItem* witem, int /*column*/) { + SelectItem(witem); + if (m_run_enabled) { + emit Run(); + } +} + +void ConfigurationListWidget::show_context_menu(const QPoint& pos) { + auto* item = static_cast(m_ui->cfgs_list->itemAt(pos)); + + if (item != nullptr) { + m_ui->cfgs_list->setCurrentItem(item); + SelectItem(item); + } + + QMenu menu; + const auto save_data_dirs = item != nullptr ? GetSaveDataDirs(item->GetInfo()) : QStringList(); + const bool has_trophy_data = + item != nullptr && TrophyViewerDialog::HasTrophyData(&item->GetInfo()); + + QAction* action_run = menu.addAction(tr("Run"), this, SLOT(run_configuration())); + QAction* action_open_folder = + menu.addAction(style()->standardIcon(QStyle::SP_DirOpenIcon), tr("Open game folder"), this, + SLOT(open_game_folder())); + QAction* action_view_trophies = menu.addAction( + style()->standardIcon(QStyle::SP_FileDialogContentsView), tr("View trophies...")); + connect(action_view_trophies, &QAction::triggered, this, + &ConfigurationListWidget::ViewTrophies); + QAction* action_remove_save_data = + menu.addAction(style()->standardIcon(QStyle::SP_DialogDiscardButton), + tr("Remove save data..."), this, SLOT(remove_save_data())); + menu.addSeparator(); + QAction* action_edit = + menu.addAction(style()->standardIcon(QStyle::SP_FileIcon), tr("Edit game settings..."), + this, SLOT(edit_configuration())); + QAction* action_delete = + menu.addAction(style()->standardIcon(QStyle::SP_DialogDiscardButton), + tr("Clear custom settings"), this, SLOT(delete_configuartion())); + + if (item == nullptr) { + menu.addSeparator(); + /*QAction *action_global = */ menu.addAction( + style()->standardIcon(QStyle::SP_FileDialogDetailedView), tr("Global settings..."), + this, SLOT(edit_global_settings())); + } + + if (item != nullptr) { + action_run->setDisabled(item->IsRunning()); + action_open_folder->setDisabled(!QDir(item->GetInfo().basedir).exists()); + action_view_trophies->setDisabled(!has_trophy_data); + action_remove_save_data->setDisabled(item->IsRunning() || save_data_dirs.isEmpty()); + action_edit->setDisabled(item->IsRunning()); + action_delete->setDisabled(item->IsRunning() || !item->GetInfo().custom_settings); + } else { + action_run->setDisabled(true); + action_open_folder->setDisabled(true); + action_view_trophies->setDisabled(true); + action_remove_save_data->setDisabled(true); + action_edit->setDisabled(true); + action_delete->setDisabled(true); + } + + if (!m_run_enabled) { + action_run->setDisabled(true); + } + + menu.exec(QCursor::pos()); +} diff --git a/src/launcher/src/main.cpp b/src/launcher/src/main.cpp new file mode 100644 index 000000000..98128a0a5 --- /dev/null +++ b/src/launcher/src/main.cpp @@ -0,0 +1,24 @@ +#include "mainDialog.h" + +#include +#include +#include +#include + +class QStyle; + +int main(int argc, char* argv[]) { + QApplication a(argc, argv); + MainDialog w; + + QStyle* s = QStyleFactory::create("fusion"); + QApplication::setStyle(s); + + QObject::connect(&a, &QApplication::aboutToQuit, &w, &MainDialog::Quit); + + w.emit Start(); + + w.show(); + + return QApplication::exec(); +} diff --git a/src/launcher/src/mainDialog.cpp b/src/launcher/src/mainDialog.cpp new file mode 100644 index 000000000..1d97a29f0 --- /dev/null +++ b/src/launcher/src/mainDialog.cpp @@ -0,0 +1,374 @@ +#include "mainDialog.h" + +#include "common.h" +#include "configuration.h" +#include "configurationItem.h" +#include "configurationListWidget.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "ui_main_dialog.h" + +#ifndef __linux__ +#include // IWYU pragma: keep +#endif + +// IWYU pragma: no_include +// IWYU pragma: no_include +// IWYU pragma: no_include + +class QWidget; + +#ifdef __linux__ +constexpr char EMULATOR_EXE[] = "kyty_emulator"; +#else +constexpr char EMULATOR_EXE[] = "kyty_emulator.exe"; +#endif + +#ifndef __linux__ +constexpr char CMD_EXE[] = "cmd.exe"; +#else +constexpr char GNOME[] = "gnome-terminal"; +constexpr char XTERM[] = "xterm"; +constexpr char KYTY_BASH_FILE[] = "kyty_run.sh"; +#endif +#ifndef __linux__ +constexpr DWORD CMD_X_CHARS = 175; +constexpr DWORD CMD_Y_CHARS = 1000; +#endif +constexpr char SETTINGS_MAIN_DIALOG[] = "MainDialog"; +constexpr char SETTINGS_MAIN_LAST_GEOMETRY[] = "geometry"; + +class DetachableProcess: public QProcess { + Q_OBJECT; + +public: + explicit DetachableProcess(QObject* parent = nullptr): QProcess(parent) {} + void Detach() { + this->waitForStarted(); + setProcessState(QProcess::NotRunning); + } +}; + +class MainDialogPrivate: public QObject { + Q_OBJECT + KYTY_QT_CLASS_NO_COPY(MainDialogPrivate); + +public: + explicit MainDialogPrivate(QObject* parent = nullptr): QObject(parent) {} + ~MainDialogPrivate() override; + + void Setup(MainDialog* main_dialog); + + /*slots:*/ + + void Update(); + void FindInterpreter(); + void Run(); + + [[nodiscard]] const QString& GetInterpreter() const { return m_interpreter; } + + static void WriteSettings(QSettings& s); + static void ReadSettings(QSettings& s); + +private: + static QByteArray g_last_geometry; + + Ui::MainDialog* m_ui = {nullptr}; + MainDialog* m_main_dialog = nullptr; + QString m_interpreter; + + /*DetachableProcess*/ QProcess m_process; + + ConfigurationItem* m_running_item = nullptr; +}; + +QByteArray MainDialogPrivate::g_last_geometry; + +MainDialog::MainDialog(QWidget* parent): QDialog(parent), m_p(new MainDialogPrivate(this)) { + m_p->Setup(this); +} + +MainDialogPrivate::~MainDialogPrivate() { + delete m_ui; +} + +void MainDialogPrivate::Setup(MainDialog* main_dialog) { + m_ui = new Ui::MainDialog; + m_ui->setupUi(main_dialog); + + m_main_dialog = main_dialog; + m_ui->widget->SetMainDialog(main_dialog); + + main_dialog->setWindowFlags(Qt::Dialog /*| Qt::MSWindowsFixedSizeDialogHint*/); + + connect(main_dialog, &MainDialog::Start, this, &MainDialogPrivate::FindInterpreter, + Qt::QueuedConnection); + connect(m_ui->widget, &ConfigurationListWidget::Select, this, &MainDialogPrivate::Update); + connect(m_ui->widget, &ConfigurationListWidget::Run, this, &MainDialogPrivate::Run); + connect(main_dialog, &MainDialog::Resize, [this]() { + g_last_geometry = m_main_dialog->saveGeometry(); + m_ui->widget->WriteSettings(); + }); + + connect(&m_process, + static_cast(&QProcess::finished), + [this](int /*exitCode*/, QProcess::ExitStatus /*exitStatus*/) { + if (m_running_item != nullptr) { + m_running_item->SetRunning(false); + } + Update(); + }); + + // connect(main_dialog, &MainDialog::Quit, [=]() { m_process.Detach(); }); + + m_ui->label_settings_file->setText(tr("Settings file: ") + m_ui->widget->GetSettingsFile()); + + m_main_dialog->restoreGeometry(g_last_geometry); + + Update(); +} + +void MainDialogPrivate::FindInterpreter() { + QDir search_dir(QApplication::applicationDirPath()); + m_interpreter = search_dir.absoluteFilePath(EMULATOR_EXE); + + if (!QFile::exists(m_interpreter)) { + search_dir.cdUp(); + m_interpreter = search_dir.absoluteFilePath(EMULATOR_EXE); + } + + bool found = QFile::exists(m_interpreter); + + if (found) { + m_ui->label_Interpreter->setText(tr("Emulator: ") + m_interpreter); + + QProcess test; + test.setProgram(m_interpreter); + test.start(); + test.waitForFinished(); + + auto output = QString(test.readAllStandardOutput()); + auto lines = output.split(QRegularExpression("[\r\n]"), Qt::SkipEmptyParts); + + if (lines.count() >= 2) { + m_ui->label_Version->setText( + tr("Version: ") + (lines.at(0).startsWith("exe_name") ? lines.at(1) : lines.at(0))); + } else { + found = false; + } + } + + if (!found) { + QMessageBox::critical(m_main_dialog, tr("Error"), tr("Can't find emulator")); + QApplication::quit(); + return; + } + + if (!m_ui->widget->EnsureGameDirectory()) { + QApplication::quit(); + return; + } + + m_ui->label_settings_file->setText(tr("Settings file: ") + m_ui->widget->GetSettingsFile()); + + Update(); +} + +static QString BoolArg(bool value) { + return value ? QStringLiteral("true") : QStringLiteral("false"); +} + +static QStringList CreateEmulatorArgs(const Configuration& info) { + QStringList args; + auto r = EnumToText(info.screen_resolution).split('x'); + + if (r.size() != 2) { + return {}; + } + + args << "--screen-width" << r.at(0); + args << "--screen-height" << r.at(1); + args << "--vblank-frequency" << QString::number(info.vblank_frequency); + args << "--vulkan-validation" << BoolArg(info.vulkan_validation_enabled); + args << "--shader-validation" << BoolArg(info.shader_validation_enabled); + args << "--shader-optimization-type" << EnumToText(info.shader_optimization_type); + args << "--shader-log-direction" << EnumToText(info.shader_log_direction); + args << "--shader-log-folder" << info.shader_log_folder; + args << "--command-buffer-dump" << BoolArg(info.command_buffer_dump_enabled); + args << "--command-buffer-dump-folder" << info.command_buffer_dump_folder; + args << "--printf-direction" << EnumToText(info.printf_direction); + args << "--printf-output-file" << info.printf_output_file; + args << "--profiler-direction" << EnumToText(info.profiler_direction); + args << "--spirv-debug-printf" << "false"; + args << "--ngg-rectlist-draw" << BoolArg(info.ngg_rectlist_draw_enabled); + if (info.renderdoc_enabled) { + args << "--rd"; + } + + QString game = info.basedir; + if (!info.elf.isEmpty()) { + game = QDir(info.basedir).filePath(info.elf); + } + args << "--game" << game; + + return args; +} + +#ifdef __linux__ +static QString BashQuote(QString value) { + value.replace('\'', "'\\''"); + return QStringLiteral("'") + value + QStringLiteral("'"); +} + +static bool CreateBashScript(const QString& interpreter, const QStringList& args, + const QString& file_name) { + QFile file(file_name); + if (file.open(QIODevice::WriteOnly | QIODevice::Text)) { + QTextStream s(&file); + + s << "#!/bin/bash\n"; + s << BashQuote(interpreter); + for (const auto& arg: args) { + s << " " << BashQuote(arg); + } + s << "\n"; + s << "echo Press any key...\n"; + s << "read -n1\n"; + + file.close(); + + return file.setPermissions(file.permissions() | QFile::ExeUser | QFile::ExeOwner | + QFile::ExeGroup); + } + return false; +} +#endif + +void MainDialog::RunInterpreter(QProcess* process, const Configuration& info) { + const auto& interpreter = m_p->GetInterpreter(); + + QFileInfo f(interpreter); + auto dir = f.absoluteDir(); + + auto args = CreateEmulatorArgs(info); + if (args.isEmpty()) { + QMessageBox::critical(this, tr("Error"), tr("Invalid emulator configuration")); + QApplication::quit(); + return; + } + +#ifdef __linux__ + auto bash_file_name = dir.filePath(KYTY_BASH_FILE); + if (!CreateBashScript(interpreter, args, bash_file_name)) { + QMessageBox::critical(this, tr("Error"), tr("Can't create file:\n") + bash_file_name); + QApplication::quit(); + return; + } + + { + process->setProgram(GNOME); + process->setArguments({"--", "bash", "-c", bash_file_name}); + } +#else + { + process->setProgram(CMD_EXE); + QStringList process_args; + process_args << QStringLiteral("/K") << interpreter; + process_args += args; + process->setArguments(process_args); + } +#endif + process->setWorkingDirectory(dir.path()); +#ifndef __linux__ + process->setCreateProcessArgumentsModifier([](QProcess::CreateProcessArguments* args) { + args->flags |= static_cast(CREATE_NEW_CONSOLE); + args->startupInfo->dwFlags &= ~static_cast(STARTF_USESTDHANDLES); + args->startupInfo->dwFlags |= static_cast(STARTF_USECOUNTCHARS); + args->startupInfo->dwXCountChars = CMD_X_CHARS; + args->startupInfo->dwYCountChars = CMD_Y_CHARS; + // args->startupInfo->dwFlags |= static_cast(STARTF_USEFILLATTRIBUTE); + // args->startupInfo->dwFillAttribute = + // static_cast(BACKGROUND_BLUE) | static_cast(FOREGROUND_RED) | + // static_cast(FOREGROUND_INTENSITY); + }); +#endif + process->start(); + process->waitForFinished(100); +} + +void MainDialog::WriteSettings(QSettings& s) { + MainDialogPrivate::WriteSettings(s); +} + +void MainDialog::ReadSettings(QSettings& s) { + MainDialogPrivate::ReadSettings(s); +} + +void MainDialog::resizeEvent(QResizeEvent* event) { + emit Resize(); + QDialog::resizeEvent(event); +} + +void MainDialogPrivate::WriteSettings(QSettings& s) { + s.beginGroup(SETTINGS_MAIN_DIALOG); + + if (!g_last_geometry.isEmpty()) { + s.setValue(SETTINGS_MAIN_LAST_GEOMETRY, g_last_geometry); + } + + s.endGroup(); +} + +void MainDialogPrivate::ReadSettings(QSettings& s) { + s.beginGroup(SETTINGS_MAIN_DIALOG); + + g_last_geometry = s.value(SETTINGS_MAIN_LAST_GEOMETRY, g_last_geometry).toByteArray(); + + s.endGroup(); +} + +void MainDialogPrivate::Run() { + m_running_item = m_ui->widget->GetSelectedItem(); + if (m_running_item == nullptr) { + return; + } + + m_running_item->SetRunning(true); + + m_main_dialog->RunInterpreter(&m_process, m_running_item->GetInfo()); + + Update(); +} + +void MainDialogPrivate::Update() { + const auto* item = m_ui->widget->GetSelectedItem(); + + bool run_enabled = (m_process.state() == QProcess::NotRunning && item != nullptr); + + if (run_enabled) { + const auto& info = item->GetInfo(); + auto dir = info.basedir; + run_enabled = !dir.isEmpty() && QDir(dir).exists(); + } + + m_ui->widget->SetRunEnabled(run_enabled); +} + +#include "mainDialog.moc" diff --git a/src/launcher/src/mandatoryLineEdit.cpp b/src/launcher/src/mandatoryLineEdit.cpp new file mode 100644 index 000000000..b4074a2f9 --- /dev/null +++ b/src/launcher/src/mandatoryLineEdit.cpp @@ -0,0 +1,44 @@ +#include "mandatoryLineEdit.h" + +#include +#include +#include +#include +#include +#include + +class QPaintEvent; +class QWidget; + +MandatoryLineEdit::MandatoryLineEdit(QWidget* parent): QLineEdit(parent) { + this->setProperty(MANDATORY_AND_EMPTY, true); + + connect(this, &QLineEdit::textChanged, this, &MandatoryLineEdit::changed); +} + +void MandatoryLineEdit::paintEvent(QPaintEvent* event) { + QLineEdit::paintEvent(event); + + if (this->property(MANDATORY_AND_EMPTY).toBool() && this->isEnabled()) { + QPainter painter(this); + QLine line(4, this->height() - 4, this->width() - 4, this->height() - 4); + painter.setPen(QPen(QColor(255, 0, 0), 1.0, Qt::DotLine)); + painter.drawLine(line); + } +} + +bool MandatoryLineEdit::FindEmpty(QObject* p) { + const QList& ch_list = p->findChildren(); + + foreach (QLineEdit* ch, ch_list) { + if (ch->isVisible() && ch->property(MANDATORY_AND_EMPTY).toBool() && ch->isEnabled()) { + return true; + } + } + + return false; +} + +void MandatoryLineEdit::changed(const QString& text) { + this->setProperty(MANDATORY_AND_EMPTY, text.isEmpty()); +} diff --git a/src/launcher/src/trophyViewerDialog.cpp b/src/launcher/src/trophyViewerDialog.cpp new file mode 100644 index 000000000..763fdc268 --- /dev/null +++ b/src/launcher/src/trophyViewerDialog.cpp @@ -0,0 +1,582 @@ +#include "trophyViewerDialog.h" + +#include "configuration.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +namespace { + +constexpr quint32 UCP_MAGIC = 0xb228c60a; +constexpr quint32 UCP_VERSION = 1; +constexpr int UCP_HEADER_LEN = 0x40; +constexpr int UCP_TOC_SKIP = 0x20; +constexpr int UCP_ENTRY_LEN = 0x40; +constexpr int UCP_NAME_LEN = 0x20; + +struct UcpEntry { + QString name; + quint64 offset = 0; + quint64 size = 0; +}; + +struct TrophyDefinition { + QString id; + QString grade; + bool hidden = false; + bool has_reward = false; +}; + +struct TrophyText { + QString name; + QString detail; + QString reward; +}; + +struct TrophyRow { + QString id; + QString name; + QString detail; + QString grade; + QString reward; + bool hidden = false; + bool has_reward = false; + QPixmap icon; +}; + +struct TrophySet { + QString tab_title; + QList trophies; +}; + +static bool CanRead(const QByteArray& data, qsizetype offset, qsizetype size) { + return offset >= 0 && size >= 0 && offset <= data.size() && size <= data.size() - offset; +} + +static quint32 ReadBe32(const QByteArray& data, qsizetype offset) { + const auto* p = reinterpret_cast(data.constData() + offset); + return (static_cast(p[0]) << 24u) | (static_cast(p[1]) << 16u) | + (static_cast(p[2]) << 8u) | static_cast(p[3]); +} + +static quint64 ReadBe64(const QByteArray& data, qsizetype offset) { + return (static_cast(ReadBe32(data, offset)) << 32u) | + static_cast(ReadBe32(data, offset + 4)); +} + +static bool FitsQSizeType(quint64 value) { + return value <= static_cast(std::numeric_limits::max()); +} + +static QString ReadFixedString(const QByteArray& data, qsizetype offset, qsizetype max_size) { + if (!CanRead(data, offset, max_size)) { + return {}; + } + + qsizetype len = 0; + while (len < max_size && data.at(offset + len) != '\0') { + len++; + } + + return QString::fromLatin1(data.constData() + offset, len); +} + +static bool ReadUcp(const QString& file_name, QMap& files, QString& error) { + QFile file(file_name); + if (!file.open(QIODevice::ReadOnly)) { + error = QObject::tr("Could not open %1").arg(QDir::toNativeSeparators(file_name)); + return false; + } + + const QByteArray data = file.readAll(); + if (data.size() < UCP_HEADER_LEN) { + error = QObject::tr("%1 is too small to be a trophy package.") + .arg(QFileInfo(file_name).fileName()); + return false; + } + + const auto magic = ReadBe32(data, 0x00); + if (magic != UCP_MAGIC) { + error = QObject::tr("%1 has an invalid trophy package magic.") + .arg(QFileInfo(file_name).fileName()); + return false; + } + + const auto version = ReadBe32(data, 0x04); + if (version != UCP_VERSION) { + error = QObject::tr("%1 uses unsupported trophy package version %2.") + .arg(QFileInfo(file_name).fileName(), QString::number(version)); + return false; + } + + const auto declared_size = ReadBe64(data, 0x08); + if (declared_size > static_cast(data.size())) { + error = QObject::tr("%1 is truncated.").arg(QFileInfo(file_name).fileName()); + return false; + } + + const auto file_count = ReadBe32(data, 0x10); + const auto toc_offset = static_cast(ReadBe32(data, 0x14)); + const auto data_size = static_cast(data.size()); + + const quint64 table_size = UCP_TOC_SKIP + static_cast(file_count) * UCP_ENTRY_LEN; + if (toc_offset > data_size || table_size > data_size - toc_offset || + !FitsQSizeType(toc_offset + table_size)) { + error = QObject::tr("%1 has an invalid table of contents.") + .arg(QFileInfo(file_name).fileName()); + return false; + } + + for (quint32 i = 0; i < file_count; i++) { + const auto entry_offset = static_cast(toc_offset + UCP_TOC_SKIP + + static_cast(i) * UCP_ENTRY_LEN); + UcpEntry entry; + entry.name = ReadFixedString(data, entry_offset, UCP_NAME_LEN).trimmed(); + entry.offset = ReadBe64(data, entry_offset + 0x20); + entry.size = ReadBe64(data, entry_offset + 0x28); + + if (entry.name.isEmpty()) { + continue; + } + if (entry.offset > data_size || entry.size > data_size - entry.offset || + !FitsQSizeType(entry.offset) || !FitsQSizeType(entry.size)) { + error = QObject::tr("%1 has an invalid entry for %2.") + .arg(QFileInfo(file_name).fileName(), entry.name); + return false; + } + + files.insert(entry.name.toCaseFolded(), data.mid(static_cast(entry.offset), + static_cast(entry.size))); + } + + return true; +} + +static const QByteArray* FindFile(const QMap& files, const QString& name) { + const auto it = files.constFind(name.toCaseFolded()); + return it == files.constEnd() ? nullptr : &it.value(); +} + +static const QByteArray* FindFirstTrophyMetadataFile(const QMap& files) { + for (auto it = files.constBegin(); it != files.constEnd(); ++it) { + if (it.key().startsWith(QStringLiteral("tropmeta_")) && + it.key().endsWith(QStringLiteral(".json"))) { + return &it.value(); + } + } + + return FindFile(files, QStringLiteral("tropmeta.json")); +} + +static bool ReadJsonObject(const QByteArray& data, const QString& file_name, QJsonObject& object, + QString& error) { + QJsonParseError parse_error; + const auto doc = QJsonDocument::fromJson(data, &parse_error); + if (parse_error.error != QJsonParseError::NoError || !doc.isObject()) { + error = QObject::tr("Could not read %1: %2").arg(file_name, parse_error.errorString()); + return false; + } + + object = doc.object(); + return true; +} + +static QString JsonString(const QJsonValue& value) { + if (value.isString()) { + return value.toString(); + } + if (value.isDouble()) { + const double number = value.toDouble(); + if (number == static_cast(number)) { + return QString::number(static_cast(number)); + } + return QString::number(number); + } + if (value.isBool()) { + return value.toBool() ? QStringLiteral("true") : QStringLiteral("false"); + } + + return {}; +} + +static QList ReadDefinitions(const QJsonObject& tropconf) { + QList ret; + + const auto trophies = tropconf.value(QStringLiteral("trophies")).toArray(); + for (const auto& value: trophies) { + const auto obj = value.toObject(); + TrophyDefinition def; + def.id = JsonString(obj.value(QStringLiteral("id"))).trimmed(); + def.grade = JsonString(obj.value(QStringLiteral("grade"))).trimmed(); + def.hidden = obj.value(QStringLiteral("hidden")).toBool(false); + def.has_reward = obj.value(QStringLiteral("hasReward")).toBool(false); + + if (!def.id.isEmpty()) { + ret.append(def); + } + } + + return ret; +} + +static QMap ReadMetadata(const QJsonObject& tropmeta) { + QMap ret; + + const auto metadata = tropmeta.value(QStringLiteral("metadata")).toObject(); + const auto trophies = metadata.value(QStringLiteral("trophyMetadata")).toArray(); + for (const auto& value: trophies) { + const auto obj = value.toObject(); + const auto id = JsonString(obj.value(QStringLiteral("id"))).trimmed(); + if (id.isEmpty()) { + continue; + } + + TrophyText text; + text.name = JsonString(obj.value(QStringLiteral("name"))).trimmed(); + text.detail = JsonString(obj.value(QStringLiteral("detail"))).trimmed(); + text.reward = JsonString(obj.value(QStringLiteral("reward"))).trimmed(); + ret.insert(id, text); + } + + return ret; +} + +static QString GradeToText(const QString& grade) { + if (grade == QStringLiteral("P")) { + return QObject::tr("Platinum"); + } + if (grade == QStringLiteral("G")) { + return QObject::tr("Gold"); + } + if (grade == QStringLiteral("S")) { + return QObject::tr("Silver"); + } + if (grade == QStringLiteral("B")) { + return QObject::tr("Bronze"); + } + + return grade; +} + +static QPixmap LoadTrophyIcon(const QMap& files, const QString& id) { + QStringList names; + names.append(QStringLiteral("trop%1.png").arg(id)); + + bool id_is_number = false; + const int id_num = id.toInt(&id_is_number); + if (id_is_number) { + names.append(QStringLiteral("trop%1.png").arg(id_num, 4, 10, QLatin1Char('0'))); + } + + for (const auto& name: names) { + const auto* data = FindFile(files, name); + if (data == nullptr) { + continue; + } + + QPixmap pixmap; + if (pixmap.loadFromData(*data)) { + return pixmap; + } + } + + return {}; +} + +static QString TrophyTabTitle(const QString& file_name) { + static const QRegularExpression trophy_file_re(QStringLiteral("^trophy(\\d+)\\.ucp$"), + QRegularExpression::CaseInsensitiveOption); + + const auto name = QFileInfo(file_name).fileName(); + const auto match = trophy_file_re.match(name); + if (match.hasMatch()) { + return QObject::tr("Trophy %1").arg(match.captured(1)); + } + + return QFileInfo(file_name).completeBaseName(); +} + +static bool BuildTrophySet(const QString& ucp_file, TrophySet& set, QString& error) { + QMap files; + if (!ReadUcp(ucp_file, files, error)) { + return false; + } + + const auto* conf_data = FindFile(files, QStringLiteral("tropconf.json")); + if (conf_data == nullptr) { + error = + QObject::tr("%1 does not contain tropconf.json.").arg(QFileInfo(ucp_file).fileName()); + return false; + } + + QJsonObject tropconf; + if (!ReadJsonObject(*conf_data, QStringLiteral("tropconf.json"), tropconf, error)) { + return false; + } + + const auto default_language = + JsonString(tropconf.value(QStringLiteral("defaultLanguage"))).trimmed(); + const QByteArray* meta_data = nullptr; + if (!default_language.isEmpty()) { + meta_data = FindFile(files, QStringLiteral("tropmeta_%1.json").arg(default_language)); + } + if (meta_data == nullptr) { + meta_data = FindFirstTrophyMetadataFile(files); + } + if (meta_data == nullptr) { + error = QObject::tr("%1 does not contain readable trophy metadata.") + .arg(QFileInfo(ucp_file).fileName()); + return false; + } + + QJsonObject tropmeta; + if (!ReadJsonObject(*meta_data, QStringLiteral("tropmeta.json"), tropmeta, error)) { + return false; + } + + const auto definitions = ReadDefinitions(tropconf); + if (definitions.isEmpty()) { + error = QObject::tr("%1 does not define any trophies.").arg(QFileInfo(ucp_file).fileName()); + return false; + } + + const auto texts = ReadMetadata(tropmeta); + + set.tab_title = TrophyTabTitle(ucp_file); + for (const auto& def: definitions) { + const auto text = texts.value(def.id); + + TrophyRow row; + row.id = def.id; + row.grade = def.grade; + row.hidden = def.hidden; + row.has_reward = def.has_reward; + row.reward = text.reward; + row.name = text.name; + row.detail = text.detail; + row.icon = LoadTrophyIcon(files, def.id); + + if (row.name.isEmpty()) { + row.name = + row.hidden ? QObject::tr("Hidden Trophy") : QObject::tr("Trophy %1").arg(def.id); + } + if (row.detail.isEmpty() && row.hidden) { + row.detail = QObject::tr("This trophy is hidden."); + } + + set.trophies.append(row); + } + + return true; +} + +static QStringList FindTrophyFiles(const Configuration* info) { + if (info == nullptr || info->basedir.isEmpty()) { + return {}; + } + + const QDir trophy_dir(QDir(info->basedir).filePath(QStringLiteral("sce_sys/trophy2"))); + if (!trophy_dir.exists()) { + return {}; + } + + const auto files = + trophy_dir.entryInfoList({QStringLiteral("Trophy*.ucp"), QStringLiteral("trophy*.ucp")}, + QDir::Files | QDir::NoSymLinks, QDir::Name | QDir::IgnoreCase); + + QStringList trophy_files; + QSet seen; + for (const auto& file: files) { + auto key = file.canonicalFilePath(); + if (key.isEmpty()) { + key = file.absoluteFilePath(); + } + key = QDir::cleanPath(key).toCaseFolded(); + + if (!seen.contains(key)) { + seen.insert(key); + trophy_files.append(file.absoluteFilePath()); + } + } + + return trophy_files; +} + +static void PrepareTable(QTableWidget* table) { + table->setColumnCount(4); + table->setHorizontalHeaderLabels({QObject::tr("Unlocked"), QObject::tr("Trophy"), + QObject::tr("Name"), QObject::tr("Description")}); + table->setAlternatingRowColors(true); + table->setEditTriggers(QAbstractItemView::NoEditTriggers); + table->setSelectionBehavior(QAbstractItemView::SelectRows); + table->setSelectionMode(QAbstractItemView::SingleSelection); + table->setShowGrid(false); + table->setWordWrap(true); + table->verticalHeader()->setVisible(false); + table->verticalHeader()->setDefaultSectionSize(112); + table->horizontalHeader()->setStretchLastSection(true); + table->horizontalHeader()->setHighlightSections(false); + table->setColumnWidth(0, 100); + table->setColumnWidth(1, 132); + table->setColumnWidth(2, 260); + table->setColumnWidth(3, 480); +} + +static QTableWidgetItem* CreateItem(const QString& text) { + auto* item = new QTableWidgetItem(text); + item->setFlags(item->flags() & ~Qt::ItemIsEditable); + return item; +} + +static QString TrophyTooltip(const TrophyRow& row) { + QStringList lines; + lines.append(row.name); + if (!row.detail.isEmpty()) { + lines.append(row.detail); + } + + const auto grade = GradeToText(row.grade); + if (!grade.isEmpty()) { + lines.append(QObject::tr("Grade: %1").arg(grade)); + } + if (row.hidden) { + lines.append(QObject::tr("Hidden trophy")); + } + if (row.has_reward && !row.reward.isEmpty()) { + lines.append(QObject::tr("Reward: %1").arg(row.reward)); + } + + return lines.join(QLatin1Char('\n')); +} + +} // namespace + +TrophyViewerDialog::TrophyViewerDialog(QWidget* parent): QDialog(parent) { + setWindowTitle(tr("Trophy Viewer")); + resize(1000, 640); + + auto* layout = new QVBoxLayout(this); + + m_tabs = new QTabWidget(this); + layout->addWidget(m_tabs, 1); + + auto* buttons = new QDialogButtonBox(QDialogButtonBox::Close, this); + connect(buttons, &QDialogButtonBox::rejected, this, &QDialog::reject); + layout->addWidget(buttons); +} + +bool TrophyViewerDialog::HasTrophyData(const Configuration* info) { + return !FindTrophyFiles(info).isEmpty(); +} + +void TrophyViewerDialog::ShowForGame(const Configuration* info, QWidget* parent) { + if (info == nullptr) { + return; + } + + TrophyViewerDialog dlg(parent); + if (!info->name.isEmpty()) { + dlg.setWindowTitle(tr("Trophy Viewer - %1").arg(info->name)); + } + + QString error; + if (!dlg.LoadGame(*info, error)) { + QMessageBox::warning(parent, tr("Trophy Viewer"), error); + return; + } + + dlg.exec(); +} + +bool TrophyViewerDialog::LoadGame(const Configuration& info, QString& error) { + const auto trophy_files = FindTrophyFiles(&info); + if (trophy_files.isEmpty()) { + error = tr("No trophy package found in sce_sys/trophy2."); + return false; + } + + QStringList errors; + for (const auto& file: trophy_files) { + TrophySet set; + QString set_error; + if (!BuildTrophySet(file, set, set_error)) { + errors.append(set_error); + continue; + } + + auto* table = new QTableWidget(set.trophies.size(), 4, m_tabs); + PrepareTable(table); + + for (int row_index = 0; row_index < set.trophies.size(); row_index++) { + const auto& row = set.trophies.at(row_index); + table->setRowHeight(row_index, 112); + + auto* status = CreateItem(tr("locked")); + status->setTextAlignment(Qt::AlignCenter); + if (row.hidden) { + status->setForeground(QBrush(Qt::gray)); + } + table->setItem(row_index, 0, status); + + auto* icon_label = new QLabel(table); + icon_label->setAlignment(Qt::AlignCenter); + icon_label->setMinimumSize(QSize(108, 108)); + if (!row.icon.isNull()) { + icon_label->setPixmap( + row.icon.scaled(QSize(96, 96), Qt::KeepAspectRatio, Qt::SmoothTransformation)); + } + table->setCellWidget(row_index, 1, icon_label); + + auto* name = CreateItem(row.name); + auto font = name->font(); + font.setBold(true); + name->setFont(font); + name->setToolTip(TrophyTooltip(row)); + table->setItem(row_index, 2, name); + + auto* detail = CreateItem(row.detail); + detail->setToolTip(TrophyTooltip(row)); + table->setItem(row_index, 3, detail); + } + + m_tabs->addTab(table, set.tab_title); + } + + if (m_tabs->count() == 0) { + error = errors.isEmpty() ? tr("No readable trophy data found.") + : errors.join(QLatin1Char('\n')); + return false; + } + + return true; +} diff --git a/src/libs/agc.cpp b/src/libs/agc.cpp new file mode 100644 index 000000000..9258c6aba --- /dev/null +++ b/src/libs/agc.cpp @@ -0,0 +1,4108 @@ +#include "libs/agc.h" + +#include "common/assert.h" +#include "common/emulatorConfig.h" +#include "common/file.h" +#include "common/logging/log.h" +#include "common/stringUtils.h" +#include "common/virtualMemory.h" +#include "graphics/guest_gpu/graphicsRun.h" +#include "graphics/guest_gpu/hardwareContext.h" +#include "graphics/guest_gpu/pm4.h" +#include "graphics/guest_gpu/gpu_defs.h" +#include "graphics/guest_gpu/tile.h" +#include "graphics/host_gpu/hostMemory.h" +#include "graphics/host_gpu/objects/label.h" +#include "graphics/host_gpu/renderer/render.h" +#include "graphics/presentation/renderDoc.h" +#include "graphics/presentation/videoOut.h" +#include "graphics/presentation/window.h" +#include "graphics/shader/shader.h" +#include "kernel/pthread.h" +#include "libs/errno.h" +#include "libs/libs.h" + +#include +#include +#include +#include +#include +#include + +#ifdef min +#undef min +#endif +#ifdef max +#undef max +#endif + +namespace Libs::Graphics { + +KYTY_SUBSYSTEM_INIT(Graphics) { + // Some games lock up if this is not called first + if (Config::RenderDocEnabled()) { + RenderDocInit(); + } + + auto width = Config::GetScreenWidth(); + auto height = Config::GetScreenHeight(); + + WindowInit(width, height); + VideoOut::VideoOutInit(width, height); + GraphicsRenderInit(); + GraphicsRunInit(); + LabelInit(); + TileInit(); + ShaderInit(); +} + +KYTY_SUBSYSTEM_UNEXPECTED_SHUTDOWN(Graphics) {} + +KYTY_SUBSYSTEM_DESTROY(Graphics) {} + +void GraphicsDbgDumpDcb(const char* type, uint32_t num_dw, uint32_t* cmd_buffer) { + EXIT_IF(type == nullptr); + + static std::atomic_int id = 0; + + if (Config::CommandBufferDumpEnabled() && num_dw > 0 && cmd_buffer != nullptr) { + Common::File f; + auto file_name = + Config::GetCommandBufferDumpFolder() / + fmt::format("{:04d}_{:04d}_buffer_{}.log", GraphicsRunGetFrameNum(), id++, type); + Common::File::CreateDirectories(file_name.parent_path()); + f.Create(file_name); + if (f.IsInvalid()) { + auto file_name_text = Common::PathToString(file_name); + LOGF_COLOR(Log::Color::BrightRed, "Can't create file: %s\n", file_name_text.c_str()); + return; + } + Pm4::DumpPm4PacketStream(&f, cmd_buffer, 0, num_dw); + f.Close(); + } +} + +namespace Gen5 { + +LIB_NAME("Graphics5", "Graphics5"); + +struct RegisterDefaults { + ShaderRegister** tbl0 = nullptr; + ShaderRegister** tbl1 = nullptr; + ShaderRegister** tbl2 = nullptr; + ShaderRegister** tbl3 = nullptr; + uint32_t tbl0_register_count = 0; + uint32_t tbl1_register_count = 0; + uint32_t tbl2_register_count = 0; + uint32_t tbl3_register_count = 0; + uint32_t* types = nullptr; + uint32_t count = 0; +}; + +static_assert(offsetof(RegisterDefaults, count) == 0x38); + +struct CompactRegisterDefaults { + ShaderRegister* tbl0_regs = nullptr; + uint32_t tbl0_register_count = 0; + const uint16_t* tbl0_pointer_offsets = nullptr; + uint32_t tbl0_pointer_count = 0; + ShaderRegister* tbl1_regs = nullptr; + uint32_t tbl1_register_count = 0; + const uint16_t* tbl1_pointer_offsets = nullptr; + uint32_t tbl1_pointer_count = 0; + ShaderRegister* tbl2_regs = nullptr; + uint32_t tbl2_register_count = 0; + const uint16_t* tbl2_pointer_offsets = nullptr; + uint32_t tbl2_pointer_count = 0; + ShaderRegister* tbl3_regs = nullptr; + uint32_t tbl3_register_count = 0; + const uint16_t* tbl3_pointer_offsets = nullptr; + uint32_t tbl3_pointer_count = 0; + uint32_t* types = nullptr; + uint32_t count = 0; +}; + +struct RegisterDefaultsStorage { + RegisterDefaults defaults; + std::vector tbl0; + std::vector tbl1; + std::vector tbl2; + std::vector tbl3; + bool initialized = false; +}; + +#include "libs/agcRegisterDefaults.inc" + +#include + +static constexpr uint32_t GRAPHICS_REGISTER_DEFAULTS_MAX_VERSION = 12; +static constexpr uint32_t GRAPHICS_REGISTER_DEFAULTS_FALLBACK_VERSION = 11; +static constexpr uint32_t GRAPHICS_INIT_NO_FEATURE_STATE = 0; + +static std::mutex g_register_defaults_mutex; + +static ShaderRegister** init_register_default_pointer_table(std::vector* table, + ShaderRegister* regs, + uint32_t register_count, + const uint16_t* offsets, + uint32_t pointer_count) { + EXIT_IF(table == nullptr); + + table->clear(); + if (pointer_count == 0) { + return nullptr; + } + + EXIT_IF(regs == nullptr); + EXIT_IF(offsets == nullptr); + + table->resize(pointer_count); + for (uint32_t i = 0; i < pointer_count; i++) { + EXIT_IF(offsets[i] >= register_count); + (*table)[i] = regs + offsets[i]; + } + + return table->data(); +} + +static RegisterDefaults* get_register_defaults(CompactRegisterDefaults* compact, + RegisterDefaultsStorage* storage) { + EXIT_IF(compact == nullptr); + EXIT_IF(storage == nullptr); + + std::lock_guard lock(g_register_defaults_mutex); + if (!storage->initialized) { + storage->defaults.tbl0 = init_register_default_pointer_table( + &storage->tbl0, compact->tbl0_regs, compact->tbl0_register_count, + compact->tbl0_pointer_offsets, compact->tbl0_pointer_count); + storage->defaults.tbl1 = init_register_default_pointer_table( + &storage->tbl1, compact->tbl1_regs, compact->tbl1_register_count, + compact->tbl1_pointer_offsets, compact->tbl1_pointer_count); + storage->defaults.tbl2 = init_register_default_pointer_table( + &storage->tbl2, compact->tbl2_regs, compact->tbl2_register_count, + compact->tbl2_pointer_offsets, compact->tbl2_pointer_count); + storage->defaults.tbl3 = init_register_default_pointer_table( + &storage->tbl3, compact->tbl3_regs, compact->tbl3_register_count, + compact->tbl3_pointer_offsets, compact->tbl3_pointer_count); + storage->defaults.tbl0_register_count = compact->tbl0_register_count; + storage->defaults.tbl1_register_count = compact->tbl1_register_count; + storage->defaults.tbl2_register_count = compact->tbl2_register_count; + storage->defaults.tbl3_register_count = compact->tbl3_register_count; + storage->defaults.types = compact->types; + storage->defaults.count = compact->count; + storage->initialized = true; + } + + return &storage->defaults; +} + +static uint32_t normalize_register_defaults_version(uint32_t ver) { + return ver <= GRAPHICS_REGISTER_DEFAULTS_MAX_VERSION + ? ver + : GRAPHICS_REGISTER_DEFAULTS_FALLBACK_VERSION; +} + +static RegisterDefaults* get_public_register_defaults(uint32_t ver) { + static RegisterDefaultsStorage storage[GRAPHICS_REGISTER_DEFAULTS_MAX_VERSION + 1] = {}; + const auto index = normalize_register_defaults_version(ver); + return get_register_defaults(g_agc_public_reg_defaults_by_version[index], &storage[index]); +} + +static RegisterDefaults* get_internal_register_defaults(uint32_t ver) { + static RegisterDefaultsStorage storage[GRAPHICS_REGISTER_DEFAULTS_MAX_VERSION + 1] = {}; + const auto index = normalize_register_defaults_version(ver); + return get_register_defaults(g_agc_internal_reg_defaults_by_version[index], &storage[index]); +} + +struct PendingGraphicsSegment { + uint32_t* start = nullptr; + uint32_t* end = nullptr; + uint32_t* range_end = nullptr; +}; + +static std::mutex g_pending_graphics_segment_mutex; +static PendingGraphicsSegment g_pending_graphics_segment; + +static void track_pending_graphics_segment_after_submit(uint32_t* dcb, uint32_t size_in_dwords) { + if (dcb == nullptr || size_in_dwords == 0) { + return; + } + + auto* segment_start = dcb + size_in_dwords; + auto* range_end = segment_start + 0xfffffu; + + std::lock_guard lock(g_pending_graphics_segment_mutex); + g_pending_graphics_segment.start = segment_start; + g_pending_graphics_segment.end = segment_start; + g_pending_graphics_segment.range_end = range_end; +} + +static void track_pending_graphics_allocation(uint32_t* cmd, uint32_t size_dw) { + if (cmd == nullptr || size_dw == 0) { + return; + } + + std::lock_guard lock(g_pending_graphics_segment_mutex); + auto* range_start = g_pending_graphics_segment.start; + auto* range_end = g_pending_graphics_segment.range_end; + if (range_start == nullptr || range_end == nullptr || cmd < range_start || cmd >= range_end) { + return; + } + + auto* cmd_end = cmd + size_dw; + if (cmd > g_pending_graphics_segment.end) { + static std::atomic log_count {0}; + if (log_count.fetch_add(1) < 64) { + LOGF("\t pending graphics segment: ignoring non-contiguous allocation cmd = " + "0x%016" PRIx64 ", tracked_end = 0x%016" PRIx64 "\n", + reinterpret_cast(cmd), + reinterpret_cast(g_pending_graphics_segment.end)); + } + return; + } + if (cmd_end > g_pending_graphics_segment.end && cmd_end <= range_end) { + g_pending_graphics_segment.end = cmd_end; + } +} + +struct CommandBuffer { + using Callback = KYTY_SYSV_ABI bool (*)(CommandBuffer*, uint32_t, void*); + + uint32_t* bottom; + uint32_t* top; + uint32_t* cursor_up; + uint32_t* cursor_down; + Callback callback; + void* user_data; + uint32_t reserved_dw; + + void DbgDump() const { + if (!Config::GraphicsDebugDumpEnabled() || + Config::GetPrintfDirection() == Config::OutputDirection::Silent) { + return; + } + static std::atomic log_count {0}; + if (log_count.fetch_add(1) >= 64) { + return; + } + + LOGF("\t bottom = 0x%016" PRIx64 "\n" + "\t top = 0x%016" PRIx64 "\n" + "\t cursor_up = 0x%016" PRIx64 "\n" + "\t cursor_down = 0x%016" PRIx64 "\n" + "\t callback = 0x%016" PRIx64 "\n" + "\t user_data = 0x%016" PRIx64 "\n" + "\t reserved_dw = %" PRIu32 "\n", + reinterpret_cast(bottom), reinterpret_cast(top), + reinterpret_cast(cursor_up), reinterpret_cast(cursor_down), + reinterpret_cast(callback), reinterpret_cast(user_data), + reserved_dw); + } + + [[nodiscard]] KYTY_SYSV_ABI uint32_t GetAvailableSizeDW() const { + if (cursor_up == nullptr || cursor_down == nullptr || cursor_down <= cursor_up) { + return 0; + } + + auto available = static_cast(cursor_down - cursor_up); + if (available <= reserved_dw) { + return 0; + } + if (available - reserved_dw > UINT32_MAX) { + LOGF_COLOR( + Log::Color::Red, + "\t command buffer has suspiciously large free space: cursor_up = 0x%016" PRIx64 + ", cursor_down = 0x%016" PRIx64 ", reserved_dw = %" PRIu32 "\n", + reinterpret_cast(cursor_up), reinterpret_cast(cursor_down), + reserved_dw); + return UINT32_MAX; + } + return static_cast(available - reserved_dw); + } + + KYTY_SYSV_ABI bool ReserveDW(uint32_t num_dw) { + uint32_t remaining = GetAvailableSizeDW(); + if (num_dw > remaining) { + if (callback == nullptr) { + LOGF_COLOR( + Log::Color::Red, + "\t command buffer exhausted and has no grow callback: requested = %" PRIu32 + ", remaining = %" PRIu32 ", reserved_dw = %" PRIu32 "\n", + num_dw, remaining, reserved_dw); + DbgDump(); + return false; + } + + bool result = callback(this, num_dw + reserved_dw, user_data); + if (!result) { + LOGF_COLOR(Log::Color::Red, + "\t command buffer grow callback failed: requested = %" PRIu32 + ", remaining = %" PRIu32 ", reserved_dw = %" PRIu32 "\n", + num_dw, remaining, reserved_dw); + DbgDump(); + return false; + } + if (GetAvailableSizeDW() < num_dw) { + LOGF_COLOR(Log::Color::Red, + "\t command buffer grow callback did not provide enough space: " + "requested = %" PRIu32 ", remaining = %" PRIu32 + ", reserved_dw = %" PRIu32 "\n", + num_dw, GetAvailableSizeDW(), reserved_dw); + DbgDump(); + return false; + } + } + return true; + } + + KYTY_SYSV_ABI uint32_t* AllocateDW(uint32_t size_dw) { + if (size_dw == 0 || !ReserveDW(size_dw)) { + LOGF_COLOR(Log::Color::Red, + "\t command buffer AllocateDW failed: size_dw = %" PRIu32 "\n", size_dw); + return nullptr; + } + auto* ret_ptr = cursor_up; + cursor_up += size_dw; + track_pending_graphics_allocation(ret_ptr, size_dw); + return ret_ptr; + } +}; + +struct Label { + volatile uint64_t m_value; + uint64_t m_reserved[3]; +}; + +static bool is_readable_command_packet_range(const uint32_t* cmd, uint32_t num_dw) { + return HostMemoryRangeIsReadable(reinterpret_cast(cmd), + static_cast(num_dw) * sizeof(uint32_t)); +} + +int KYTY_SYSV_ABI GraphicsInit(uint32_t* state, uint32_t ver) { + PRINT_NAME(); + + LOGF("\t state = 0x%016" PRIx64 "\n" + "\t ver = %u\n", + reinterpret_cast(state), ver); + + if (state == nullptr) { + LOGF_COLOR(Log::Color::Red, "\t state is null\n"); + return OK; + } + + if (ver > GRAPHICS_REGISTER_DEFAULTS_MAX_VERSION) { + LOGF_COLOR(Log::Color::Red, "\t unsupported version %u\n", ver); + } + + printf("version = %u\n", ver); + + state[0] = ver; + state[1] = GRAPHICS_INIT_NO_FEATURE_STATE; + + return OK; +} + +void* KYTY_SYSV_ABI GraphicsGetRegisterDefaults2(uint32_t ver) { + PRINT_NAME(); + + if (ver > GRAPHICS_REGISTER_DEFAULTS_MAX_VERSION) { + LOGF_COLOR(Log::Color::Red, "\t unsupported version %u\n", ver); + } + return get_public_register_defaults(ver); +} + +void* KYTY_SYSV_ABI GraphicsGetRegisterDefaults2Internal(uint32_t ver) { + PRINT_NAME(); + + if (ver > GRAPHICS_REGISTER_DEFAULTS_MAX_VERSION) { + LOGF_COLOR(Log::Color::Red, "\t unsupported version %u\n", ver); + } + return get_internal_register_defaults(ver); +} + +static void dbg_dump_shader(const Shader* h) { + LOGF("\t file_header = 0x%08" PRIx32 "\n" + "\t version = 0x%08" PRIx32 "\n" + "\t user_data = 0x%016" PRIx64 "\n", + h->file_header, h->version, reinterpret_cast(h->user_data)); + if (h->user_data != nullptr) { + LOGF("\t\t direct_resource_count = 0x%04" PRIx16 "\n" + "\t\t direct_resource_offset = 0x%016" PRIx64 "\n", + h->user_data->direct_resource_count, + reinterpret_cast(h->user_data->direct_resource_offset)); + for (int i = 0; i < static_cast(h->user_data->direct_resource_count); i++) { + LOGF("\t\t\t offset[%02d] = %04" PRIx16 "\n", i, + h->user_data->direct_resource_offset[i]); + } + for (int imm = 0; imm < 4; imm++) { + LOGF("\t\t sharp_resource_count [%d] = 0x%04" PRIx16 "\n", imm, + h->user_data->sharp_resource_count[imm]); + LOGF("\t\t sharp_resource_offset [%d] = 0x%016" PRIx64 "\n", imm, + reinterpret_cast(h->user_data->sharp_resource_offset[imm])); + for (int i = 0; i < static_cast(h->user_data->sharp_resource_count[imm]); i++) { + LOGF("\t\t\t offset_dw[%d] = %04" PRIx16 ", size = %" PRIu16 "\n", i, + static_cast(h->user_data->sharp_resource_offset[imm][i].offset_dw), + static_cast(h->user_data->sharp_resource_offset[imm][i].size)); + } + } + LOGF("\t\t eud_size_dw = 0x%04" PRIx16 "\n" + "\t\t srt_size_dw = 0x%04" PRIx16 "\n", + h->user_data->eud_size_dw, h->user_data->srt_size_dw); + } + LOGF("\t code = 0x%016" PRIx64 "\n" + "\t num_cx_registers = 0x%02" PRIx8 "\n" + "\t cx_registers = 0x%016" PRIx64 "\n", + reinterpret_cast(h->code), h->num_cx_registers, + reinterpret_cast(h->cx_registers)); + for (int i = 0; i < static_cast(h->num_cx_registers); i++) { + LOGF("\t\t cx[%d]: offset = %08" PRIx32 ", value = %08" PRIx32 "\n", i, + h->cx_registers[i].offset, h->cx_registers[i].value); + } + LOGF("\t num_sh_registers = 0x%02" PRIx8 "\n" + "\t sh_registers = 0x%016" PRIx64 "\n", + h->num_sh_registers, reinterpret_cast(h->sh_registers)); + for (int i = 0; i < static_cast(h->num_sh_registers); i++) { + LOGF("\t\t sh[%d]: offset = %08" PRIx32 ", value = %08" PRIx32 "\n", i, + h->sh_registers[i].offset, h->sh_registers[i].value); + } + LOGF("\t specials = 0x%016" PRIx64 "\n", + reinterpret_cast(h->specials)); + LOGF("\t\t ge_cntl: offset = %08" PRIx32 ", value = %08" PRIx32 "\n", + h->specials->ge_cntl.offset, h->specials->ge_cntl.value); + LOGF("\t\t vgt_shader_stages_en: offset = %08" PRIx32 ", value = %08" PRIx32 "\n", + h->specials->vgt_shader_stages_en.offset, h->specials->vgt_shader_stages_en.value); + LOGF("\t\t vgt_gs_out_prim_type: offset = %08" PRIx32 ", value = %08" PRIx32 "\n", + h->specials->vgt_gs_out_prim_type.offset, h->specials->vgt_gs_out_prim_type.value); + LOGF("\t\t ge_user_vgpr_en: offset = %08" PRIx32 ", value = %08" PRIx32 "\n", + h->specials->ge_user_vgpr_en.offset, h->specials->ge_user_vgpr_en.value); + LOGF("\t\t dispatch_modifier = %08" PRIx32 "\n", h->specials->dispatch_modifier); + LOGF("\t\t user_data_range: start = %08" PRIx32 ", end = %08" PRIx32 "\n", + h->specials->user_data_range.start, h->specials->user_data_range.end); + LOGF("\t\t draw_modifier: enbl_start_vertex_offset = %08" PRIx32 "\n" + "\t\t draw_modifier: enbl_start_index_offset = %08" PRIx32 "\n" + "\t\t draw_modifier: enbl_start_instance_offset = %08" PRIx32 "\n" + "\t\t draw_modifier: enbl_draw_index = %08" PRIx32 "\n" + "\t\t draw_modifier: enbl_user_vgprs = %08" PRIx32 "\n" + "\t\t draw_modifier: render_target_slice_offset = %08" PRIx32 "\n" + "\t\t draw_modifier: fuse_draws = %08" PRIx32 "\n" + "\t\t draw_modifier: compiler_flags = %08" PRIx32 "\n" + "\t\t draw_modifier: is_default = %08" PRIx32 "\n" + "\t\t draw_modifier: reserved = %08" PRIx32 "\n" + "\t num_input_semantics = 0x%08" PRIx32 "\n" + "\t input_semantics = 0x%016" PRIx64 "\n", + static_cast(h->specials->draw_modifier.enbl_start_vertex_offset), + static_cast(h->specials->draw_modifier.enbl_start_index_offset), + static_cast(h->specials->draw_modifier.enbl_start_instance_offset), + static_cast(h->specials->draw_modifier.enbl_draw_index), + static_cast(h->specials->draw_modifier.enbl_user_vgprs), + static_cast(h->specials->draw_modifier.render_target_slice_offset), + static_cast(h->specials->draw_modifier.fuse_draws), + static_cast(h->specials->draw_modifier.compiler_flags), + static_cast(h->specials->draw_modifier.is_default), + static_cast(h->specials->draw_modifier.reserved), h->num_input_semantics, + reinterpret_cast(h->input_semantics)); + for (int i = 0; i < static_cast(h->num_input_semantics); i++) { + LOGF("\t\t input_semantics[%d]: semantic = %08" PRIx32 "\n" + "\t\t input_semantics[%d]: hardware_mapping = %08" PRIx32 "\n" + "\t\t input_semantics[%d]: size_in_elements = %08" PRIx32 "\n" + "\t\t input_semantics[%d]: is_f16 = %08" PRIx32 "\n" + "\t\t input_semantics[%d]: is_flat_shaded = %08" PRIx32 "\n" + "\t\t input_semantics[%d]: is_linear = %08" PRIx32 "\n" + "\t\t input_semantics[%d]: is_custom = %08" PRIx32 "\n" + "\t\t input_semantics[%d]: static_vb_index = %08" PRIx32 "\n" + "\t\t input_semantics[%d]: static_attribute = %08" PRIx32 "\n" + "\t\t input_semantics[%d]: reserved = %08" PRIx32 "\n" + "\t\t input_semantics[%d]: default_value = %08" PRIx32 "\n" + "\t\t input_semantics[%d]: default_value_hi = %08" PRIx32 "\n", + i, static_cast(h->input_semantics[i].semantic), i, + static_cast(h->input_semantics[i].hardware_mapping), i, + static_cast(h->input_semantics[i].size_in_elements), i, + static_cast(h->input_semantics[i].is_f16), i, + static_cast(h->input_semantics[i].is_flat_shaded), i, + static_cast(h->input_semantics[i].is_linear), i, + static_cast(h->input_semantics[i].is_custom), i, + static_cast(h->input_semantics[i].static_vb_index), i, + static_cast(h->input_semantics[i].static_attribute), i, + static_cast(h->input_semantics[i].reserved), i, + static_cast(h->input_semantics[i].default_value), i, + static_cast(h->input_semantics[i].default_value_hi)); + } + LOGF("\t num_output_semantics = 0x%04" PRIx16 "\n" + "\t output_semantics = 0x%016" PRIx64 "\n", + h->num_output_semantics, reinterpret_cast(h->output_semantics)); + for (int i = 0; i < static_cast(h->num_output_semantics); i++) { + LOGF("\t\t output_semantics[%d]: semantic = %08" PRIx32 "\n" + "\t\t output_semantics[%d]: hardware_mapping = %08" PRIx32 "\n" + "\t\t output_semantics[%d]: size_in_elements = %08" PRIx32 "\n" + "\t\t output_semantics[%d]: is_f16 = %08" PRIx32 "\n" + "\t\t output_semantics[%d]: is_flat_shaded = %08" PRIx32 "\n" + "\t\t output_semantics[%d]: is_linear = %08" PRIx32 "\n" + "\t\t output_semantics[%d]: is_custom = %08" PRIx32 "\n" + "\t\t output_semantics[%d]: static_vb_index = %08" PRIx32 "\n" + "\t\t output_semantics[%d]: static_attribute = %08" PRIx32 "\n" + "\t\t output_semantics[%d]: reserved = %08" PRIx32 "\n" + "\t\t output_semantics[%d]: default_value = %08" PRIx32 "\n" + "\t\t output_semantics[%d]: default_value_hi = %08" PRIx32 "\n", + i, static_cast(h->output_semantics[i].semantic), i, + static_cast(h->output_semantics[i].hardware_mapping), i, + static_cast(h->output_semantics[i].size_in_elements), i, + static_cast(h->output_semantics[i].is_f16), i, + static_cast(h->output_semantics[i].is_flat_shaded), i, + static_cast(h->output_semantics[i].is_linear), i, + static_cast(h->output_semantics[i].is_custom), i, + static_cast(h->output_semantics[i].static_vb_index), i, + static_cast(h->output_semantics[i].static_attribute), i, + static_cast(h->output_semantics[i].reserved), i, + static_cast(h->output_semantics[i].default_value), i, + static_cast(h->output_semantics[i].default_value_hi)); + } + LOGF("\t header_size = 0x%08" PRIx32 "\n" + "\t shader_size = 0x%08" PRIx32 "\n" + "\t embedded_constant_buffer_size_dqw = 0x%08" PRIx32 "\n" + "\t target = 0x%08" PRIx32 "\n" + "\t scratch_size_dw_per_thread = 0x%04" PRIx16 "\n" + "\t special_sizes_bytes = 0x%04" PRIx16 "\n" + "\t type = 0x%02" PRIx8 "\n", + h->header_size, h->shader_size, h->embedded_constant_buffer_size_dqw, h->target, + h->scratch_size_dw_per_thread, h->special_sizes_bytes, h->type); +} + +static constexpr int GRAPHICS5_ERROR_INVALID_SHADER_PROGRAM = static_cast(0x8a6c0005u); +static constexpr int GRAPHICS5_ERROR_INVALID_SHADER_HALVES = static_cast(0x8a6c0008u); + +static bool get_shader_program_address_register(uint8_t type, uint32_t* lo_offset) { + if (lo_offset == nullptr) { + return false; + } + + switch (static_cast(type)) { + case Prospero::ShaderBinaryType::kCs: *lo_offset = Pm4::COMPUTE_PGM_LO; return true; + case Prospero::ShaderBinaryType::kPs: *lo_offset = Pm4::SPI_SHADER_PGM_LO_PS; return true; + case Prospero::ShaderBinaryType::kGs: *lo_offset = Pm4::SPI_SHADER_PGM_LO_ES; return true; + case Prospero::ShaderBinaryType::kHs: *lo_offset = Pm4::SPI_SHADER_PGM_LO_LS; return true; + case Prospero::ShaderBinaryType::kGsBack: *lo_offset = Pm4::SPI_SHADER_PGM_LO_GS; return true; + case Prospero::ShaderBinaryType::kHsBack: *lo_offset = Pm4::SPI_SHADER_PGM_LO_HS; return true; + case Prospero::ShaderBinaryType::kGsFront: + case Prospero::ShaderBinaryType::kHsFront: + case Prospero::ShaderBinaryType::kFs: return false; + } + + return false; +} + +static int patch_program_address_register(ShaderRegister* regs, uint32_t num_regs, uint8_t type, + uint64_t base) { + uint32_t lo_offset = 0; + if (!get_shader_program_address_register(type, &lo_offset)) { + return OK; + } + + if (regs == nullptr || num_regs == 0) { + return GRAPHICS5_ERROR_INVALID_SHADER_PROGRAM; + } + + for (auto lo_index = 0u; lo_index < num_regs; lo_index++) { + if (regs[lo_index].offset != lo_offset) { + continue; + } + + const auto hi_index = lo_index + 1u; + const auto hi_offset = lo_offset + 1u; + if (hi_index >= num_regs || regs[hi_index].offset != hi_offset) { + return GRAPHICS5_ERROR_INVALID_SHADER_PROGRAM; + } + + const auto shader_offset = (static_cast(regs[lo_index].value) << 8u) | + ((static_cast(regs[hi_index].value) & 0xffu) << 40u); + const auto addr = base + shader_offset; + + regs[lo_index].value = static_cast((addr >> 8u) & 0xffffffffu); + regs[hi_index].value &= 0xffffff00u; + regs[hi_index].value |= static_cast((addr >> 40u) & 0xffu); + return OK; + } + + return GRAPHICS5_ERROR_INVALID_SHADER_PROGRAM; +} + +static ShaderRegister* find_shader_register(ShaderRegister* regs, uint32_t num_regs, + uint32_t offset, uint32_t occurrence = 0) { + if (regs == nullptr) { + return nullptr; + } + + for (uint32_t i = 0; i < num_regs; i++) { + if (regs[i].offset != offset) { + continue; + } + if (occurrence == 0) { + return regs + i; + } + occurrence--; + } + + return nullptr; +} + +static void patch_shader_register_address(ShaderRegister* regs, uint32_t num_regs, + uint32_t lo_offset, uint64_t address) { + auto* lo = find_shader_register(regs, num_regs, lo_offset); + if (lo == nullptr) { + return; + } + + auto* hi = (lo + 1 < regs + num_regs && (lo + 1)->offset == lo_offset + 1u ? lo + 1 : nullptr); + if (hi == nullptr) { + return; + } + + lo->value = static_cast((address >> 8u) & 0xffffffffu); + hi->value &= 0xffffff00u; + hi->value |= static_cast((address >> 40u) & 0xffu); +} + +int KYTY_SYSV_ABI GraphicsCreateShader(Shader** dst, void* header, const volatile void* code) { + PRINT_NAME(); + + EXIT_NOT_IMPLEMENTED(dst == nullptr); + EXIT_NOT_IMPLEMENTED(header == nullptr); + EXIT_NOT_IMPLEMENTED(code == nullptr); + + LOGF("\t header = 0x%016" PRIx64 "\n" + "\t code = 0x%016" PRIx64 "\n", + reinterpret_cast(header), reinterpret_cast(code)); + + auto* h = static_cast(header); + + auto update_addr = [](auto& m) { + if (m != nullptr) { + m = reinterpret_cast::type>( + reinterpret_cast(m) + reinterpret_cast(&m)); + } + }; + + update_addr(h->cx_registers); + update_addr(h->sh_registers); + update_addr(h->user_data); + update_addr(h->specials); + update_addr(h->input_semantics); + update_addr(h->output_semantics); + if (h->user_data != nullptr) { + update_addr(h->user_data->direct_resource_offset); + update_addr(h->user_data->sharp_resource_offset[0]); + update_addr(h->user_data->sharp_resource_offset[1]); + update_addr(h->user_data->sharp_resource_offset[2]); + update_addr(h->user_data->sharp_resource_offset[3]); + } + + h->code = code; + + EXIT_NOT_IMPLEMENTED(h->file_header != 0x34333231); + EXIT_NOT_IMPLEMENTED(h->version != 0x00000018); + + auto base = reinterpret_cast(code); + + LOGF("\t base = 0x%016" PRIx64 "\n", base); + + ShaderMappedData map; + map.user_data = h->user_data; + map.input_semantics = h->input_semantics; + map.num_input_semantics = h->num_input_semantics; + map.code_size_bytes = h->shader_size; + + ShaderMapUserData(base, map); + + EXIT_NOT_IMPLEMENTED((base & 0xFFFF0000000000FFull) != 0); + + auto patch_result = + patch_program_address_register(h->sh_registers, h->num_sh_registers, h->type, base); + if (patch_result != OK) { + return patch_result; + } + + *dst = h; + + dbg_dump_shader(h); + + return OK; +} + +int KYTY_SYSV_ABI GraphicsUnknownGetFusedShaderSize(SizeAlign* dst, const Shader* front, + const Shader* back) { + PRINT_NAME(); + + LOGF("\t dst = 0x%016" PRIx64 "\n" + "\t front = 0x%016" PRIx64 "\n" + "\t back = 0x%016" PRIx64 "\n", + reinterpret_cast(dst), reinterpret_cast(front), + reinterpret_cast(back)); + + EXIT_NOT_IMPLEMENTED(dst == nullptr); + EXIT_NOT_IMPLEMENTED(front == nullptr); + EXIT_NOT_IMPLEMENTED(back == nullptr); + + const auto front_type = static_cast(front->type); + const auto back_type = static_cast(back->type); + if (!((front_type == Prospero::ShaderBinaryType::kGsFront && + back_type == Prospero::ShaderBinaryType::kGsBack) || + (front_type == Prospero::ShaderBinaryType::kHsFront && + back_type == Prospero::ShaderBinaryType::kHsBack))) { + return GRAPHICS5_ERROR_INVALID_SHADER_HALVES; + } + + dst->m_size = static_cast(back->num_sh_registers) * sizeof(ShaderRegister); + dst->m_align = 4; + + return OK; +} + +int KYTY_SYSV_ABI GraphicsUnknownFuseShaderHalves(Shader* fused_result, const Shader* front, + const Shader* back, void* scratch_mem) { + PRINT_NAME(); + + LOGF("\t fused_result = 0x%016" PRIx64 "\n" + "\t front = 0x%016" PRIx64 "\n" + "\t back = 0x%016" PRIx64 "\n" + "\t scratch_mem = 0x%016" PRIx64 "\n", + reinterpret_cast(fused_result), reinterpret_cast(front), + reinterpret_cast(back), reinterpret_cast(scratch_mem)); + + EXIT_NOT_IMPLEMENTED(fused_result == nullptr); + EXIT_NOT_IMPLEMENTED(front == nullptr); + EXIT_NOT_IMPLEMENTED(back == nullptr); + + const auto front_type = static_cast(front->type); + const auto back_type = static_cast(back->type); + if (!((front_type == Prospero::ShaderBinaryType::kGsFront && + back_type == Prospero::ShaderBinaryType::kGsBack) || + (front_type == Prospero::ShaderBinaryType::kHsFront && + back_type == Prospero::ShaderBinaryType::kHsBack))) { + return GRAPHICS5_ERROR_INVALID_SHADER_HALVES; + } + + *fused_result = *back; + fused_result->type = static_cast(front_type == Prospero::ShaderBinaryType::kGsFront + ? Prospero::ShaderBinaryType::kGs + : Prospero::ShaderBinaryType::kHs); + + if (front->specials != nullptr && back->specials != nullptr) { + const auto front_stages = front->specials->vgt_shader_stages_en.value; + const auto back_stages = back->specials->vgt_shader_stages_en.value; + const auto mismatch_bit = + (front_type == Prospero::ShaderBinaryType::kGsFront ? (1u << 22u) : (1u << 21u)); + if (((front_stages ^ back_stages) & mismatch_bit) != 0) { + return GRAPHICS5_ERROR_INVALID_SHADER_HALVES; + } + } + + if (scratch_mem != nullptr && back->sh_registers != nullptr && back->num_sh_registers != 0) { + auto* sh_registers = static_cast(scratch_mem); + memcpy(sh_registers, back->sh_registers, + static_cast(back->num_sh_registers) * sizeof(ShaderRegister)); + fused_result->sh_registers = sh_registers; + } + + auto* fused_regs = fused_result->sh_registers; + const auto fused_reg_count = static_cast(fused_result->num_sh_registers); + const auto front_reg_count = static_cast(front->num_sh_registers); + + if (front_type == Prospero::ShaderBinaryType::kGsFront) { + for (uint32_t occurrence = 0; occurrence < 2; occurrence++) { + auto* dst = find_shader_register(fused_regs, fused_reg_count, + Pm4::SPI_SHADER_PGM_CHKSUM_GS, occurrence); + const auto* src = find_shader_register(front->sh_registers, front_reg_count, + Pm4::SPI_SHADER_PGM_CHKSUM_GS, occurrence); + if (dst != nullptr && src != nullptr) { + dst->value = src->value; + } + } + patch_shader_register_address(fused_regs, fused_reg_count, Pm4::SPI_SHADER_PGM_LO_ES, + reinterpret_cast(front->code)); + } else { + patch_shader_register_address(fused_regs, fused_reg_count, Pm4::SPI_SHADER_PGM_LO_LS, + reinterpret_cast(front->code)); + } + + fused_result->user_data = nullptr; + + return OK; +} + +static constexpr int GRAPHICS5_ERROR_INVALID_PACKET = static_cast(0x8a6c000cu); + +enum class RegIndirectPacket : uint32_t { + Cx, + Sh, + Uc, +}; + +static uint32_t reg_indirect_native_op(RegIndirectPacket type) { + switch (type) { + case RegIndirectPacket::Cx: return Pm4::IT_SET_CONTEXT_REG_INDIRECT; + case RegIndirectPacket::Sh: return Pm4::IT_SET_SH_REG_INDIRECT; + case RegIndirectPacket::Uc: return Pm4::IT_SET_UCONFIG_REG_INDIRECT; + } + return 0; +} + +static bool is_native_reg_indirect_packet(const uint32_t* cmd, RegIndirectPacket type) { + return ((cmd[0] >> 8u) & 0xffu) == reg_indirect_native_op(type); +} + +static uint32_t reg_indirect_pm4_r(RegIndirectPacket type) { + return (type == RegIndirectPacket::Uc && GraphicsGetIsTrinityMode() != 0 ? 1u : 0u); +} + +static void reg_indirect_write_packet(uint32_t* cmd, uint64_t vaddr, uint32_t num_regs, + RegIndirectPacket type) { + cmd[0] = KYTY_PM4(5, reg_indirect_native_op(type), reg_indirect_pm4_r(type)); + cmd[1] = static_cast(vaddr) & 0xfffffffcu; + cmd[2] = static_cast(vaddr >> 32u); + cmd[3] = 0x80000000u; + cmd[4] = num_regs & 0x3fffu; +} + +static int reg_indirect_patch_set_address(uint32_t* cmd, uint64_t vaddr, RegIndirectPacket type) { + EXIT_NOT_IMPLEMENTED(cmd == nullptr); + + if (is_native_reg_indirect_packet(cmd, type)) { + cmd[1] = (cmd[1] & 0x3u) | (static_cast(vaddr) & 0xfffffffcu); + cmd[2] = static_cast(vaddr >> 32u); + return OK; + } + + return GRAPHICS5_ERROR_INVALID_PACKET; +} + +static int reg_indirect_patch_set_num_registers(uint32_t* cmd, uint32_t num_regs, + RegIndirectPacket type) { + EXIT_NOT_IMPLEMENTED(cmd == nullptr); + + if (is_native_reg_indirect_packet(cmd, type)) { + cmd[4] = (cmd[4] & ~0x3fffu) | (num_regs & 0x3fffu); + return OK; + } + + return GRAPHICS5_ERROR_INVALID_PACKET; +} + +static int reg_indirect_patch_add_registers(uint32_t* cmd, uint32_t num_regs, + RegIndirectPacket type) { + EXIT_NOT_IMPLEMENTED(cmd == nullptr); + + if (is_native_reg_indirect_packet(cmd, type)) { + auto new_num_regs = ((cmd[4] & 0x3fffu) + num_regs) & 0x3fffu; + cmd[4] = (cmd[4] & ~0x3fffu) | new_num_regs; + return OK; + } + + return GRAPHICS5_ERROR_INVALID_PACKET; +} + +int KYTY_SYSV_ABI GraphicsSetCxRegIndirectPatchSetAddress(uint32_t* cmd, + const volatile ShaderRegister* regs) { + PRINT_NAME(); + + LOGF("\t cmd = 0x%016" PRIx64 "\n" + "\t regs = 0x%016" PRIx64 "\n", + reinterpret_cast(cmd), reinterpret_cast(regs)); + + EXIT_NOT_IMPLEMENTED(cmd == nullptr); + EXIT_NOT_IMPLEMENTED(regs == nullptr); + + auto vaddr = reinterpret_cast(regs); + + return reg_indirect_patch_set_address(cmd, vaddr, RegIndirectPacket::Cx); +} + +int KYTY_SYSV_ABI GraphicsSetShRegIndirectPatchSetAddress(uint32_t* cmd, + const volatile ShaderRegister* regs) { + PRINT_NAME(); + + LOGF("\t cmd = 0x%016" PRIx64 "\n" + "\t regs = 0x%016" PRIx64 "\n", + reinterpret_cast(cmd), reinterpret_cast(regs)); + + EXIT_NOT_IMPLEMENTED(cmd == nullptr); + EXIT_NOT_IMPLEMENTED(regs == nullptr); + + auto vaddr = reinterpret_cast(regs); + + return reg_indirect_patch_set_address(cmd, vaddr, RegIndirectPacket::Sh); +} + +int KYTY_SYSV_ABI GraphicsSetUcRegIndirectPatchSetAddress(uint32_t* cmd, + const volatile ShaderRegister* regs) { + PRINT_NAME(); + + LOGF("\t cmd = 0x%016" PRIx64 "\n" + "\t regs = 0x%016" PRIx64 "\n", + reinterpret_cast(cmd), reinterpret_cast(regs)); + + EXIT_NOT_IMPLEMENTED(cmd == nullptr); + EXIT_NOT_IMPLEMENTED(regs == nullptr); + + auto vaddr = reinterpret_cast(regs); + + return reg_indirect_patch_set_address(cmd, vaddr, RegIndirectPacket::Uc); +} + +int KYTY_SYSV_ABI GraphicsSetCxRegIndirectPatchSetNumRegisters(uint32_t* cmd, uint32_t num_regs) { + PRINT_NAME(); + + LOGF("\t cmd = 0x%016" PRIx64 "\n" + "\t num_regs = %" PRIu32 "\n", + reinterpret_cast(cmd), num_regs); + + EXIT_NOT_IMPLEMENTED(cmd == nullptr); + + return reg_indirect_patch_set_num_registers(cmd, num_regs, RegIndirectPacket::Cx); +} + +int KYTY_SYSV_ABI GraphicsSetShRegIndirectPatchSetNumRegisters(uint32_t* cmd, uint32_t num_regs) { + PRINT_NAME(); + + LOGF("\t cmd = 0x%016" PRIx64 "\n" + "\t num_regs = %" PRIu32 "\n", + reinterpret_cast(cmd), num_regs); + + EXIT_NOT_IMPLEMENTED(cmd == nullptr); + + return reg_indirect_patch_set_num_registers(cmd, num_regs, RegIndirectPacket::Sh); +} + +int KYTY_SYSV_ABI GraphicsSetUcRegIndirectPatchSetNumRegisters(uint32_t* cmd, uint32_t num_regs) { + PRINT_NAME(); + + LOGF("\t cmd = 0x%016" PRIx64 "\n" + "\t num_regs = %" PRIu32 "\n", + reinterpret_cast(cmd), num_regs); + + EXIT_NOT_IMPLEMENTED(cmd == nullptr); + + return reg_indirect_patch_set_num_registers(cmd, num_regs, RegIndirectPacket::Uc); +} + +int KYTY_SYSV_ABI GraphicsSetCxRegIndirectPatchAddRegisters(uint32_t* cmd, uint32_t num_regs) { + PRINT_NAME(); + + LOGF("\t cmd = 0x%016" PRIx64 "\n" + "\t num_regs = %" PRIu32 "\n", + reinterpret_cast(cmd), num_regs); + + EXIT_NOT_IMPLEMENTED(cmd == nullptr); + + return reg_indirect_patch_add_registers(cmd, num_regs, RegIndirectPacket::Cx); +} + +int KYTY_SYSV_ABI GraphicsSetShRegIndirectPatchAddRegisters(uint32_t* cmd, uint32_t num_regs) { + PRINT_NAME(); + + LOGF("\t cmd = 0x%016" PRIx64 "\n" + "\t num_regs = %" PRIu32 "\n", + reinterpret_cast(cmd), num_regs); + + EXIT_NOT_IMPLEMENTED(cmd == nullptr); + + return reg_indirect_patch_add_registers(cmd, num_regs, RegIndirectPacket::Sh); +} + +int KYTY_SYSV_ABI GraphicsSetUcRegIndirectPatchAddRegisters(uint32_t* cmd, uint32_t num_regs) { + PRINT_NAME(); + + LOGF("\t cmd = 0x%016" PRIx64 "\n" + "\t num_regs = %" PRIu32 "\n", + reinterpret_cast(cmd), num_regs); + + EXIT_NOT_IMPLEMENTED(cmd == nullptr); + + return reg_indirect_patch_add_registers(cmd, num_regs, RegIndirectPacket::Uc); +} + +static uint32_t GraphicsPrimitiveTypeToGsOut(uint32_t prim_type) { + switch (static_cast(prim_type)) { + case Prospero::PrimitiveType::kPointList: return Prospero::GpuEnumValue(Prospero::GsOutputPrimitiveType::kPoints); + case Prospero::PrimitiveType::kLineList: + case Prospero::PrimitiveType::kLineStrip: + case Prospero::PrimitiveType::kLineListAdjacency: + case Prospero::PrimitiveType::kLineStripAdjacency: + case Prospero::PrimitiveType::kLineLoop: return Prospero::GpuEnumValue(Prospero::GsOutputPrimitiveType::kLines); + case Prospero::PrimitiveType::kRectList: + return Prospero::GpuEnumValue(Prospero::GsOutputPrimitiveType::k2dRectangle); + case Prospero::PrimitiveType::kRectListLegacy: + return Prospero::GpuEnumValue(Prospero::GsOutputPrimitiveType::kRectList); + default: return Prospero::GpuEnumValue(Prospero::GsOutputPrimitiveType::kTriangles); + } +} + +int KYTY_SYSV_ABI GraphicsCreatePrimState(ShaderRegister* cx_regs, ShaderRegister* uc_regs, + const Shader* hs, const Shader* gs, uint32_t prim_type) { + PRINT_NAME(); + + LOGF("\t cx_regs = 0x%016" PRIx64 "\n" + "\t uc_regs = 0x%016" PRIx64 "\n" + "\t hs = 0x%016" PRIx64 "\n" + "\t gs = 0x%016" PRIx64 "\n" + "\t prim_type = %" PRIu32 "\n", + reinterpret_cast(cx_regs), reinterpret_cast(uc_regs), + reinterpret_cast(hs), reinterpret_cast(gs), prim_type); + + if (cx_regs == nullptr && uc_regs == nullptr) { + return OK; + } + + EXIT_NOT_IMPLEMENTED(gs == nullptr); + + EXIT_NOT_IMPLEMENTED(hs != nullptr && + static_cast(hs->type) != Prospero::ShaderBinaryType::kHs); + EXIT_NOT_IMPLEMENTED(static_cast(gs->type) != Prospero::ShaderBinaryType::kGs); + + if (cx_regs != nullptr) { + EXIT_NOT_IMPLEMENTED(hs != nullptr && hs->specials->vgt_shader_stages_en.offset != + Pm4::VGT_SHADER_STAGES_EN); + EXIT_NOT_IMPLEMENTED(hs != nullptr && hs->specials->vgt_gs_out_prim_type.offset != + Pm4::VGT_GS_OUT_PRIM_TYPE); + EXIT_NOT_IMPLEMENTED(gs->specials->vgt_shader_stages_en.offset != + Pm4::VGT_SHADER_STAGES_EN); + EXIT_NOT_IMPLEMENTED(gs->specials->vgt_gs_out_prim_type.offset != + Pm4::VGT_GS_OUT_PRIM_TYPE); + + cx_regs[0] = gs->specials->vgt_shader_stages_en; + if ((cx_regs[0].value & 0x20u) != 0) { + cx_regs[1] = gs->specials->vgt_gs_out_prim_type; + } else { + cx_regs[1].offset = Pm4::VGT_GS_OUT_PRIM_TYPE; + cx_regs[1].value = GraphicsPrimitiveTypeToGsOut(prim_type); + } + + if (hs != nullptr) { + cx_regs[0].value |= hs->specials->vgt_shader_stages_en.value; + if ((cx_regs[0].value & 0x20u) == 0) { + cx_regs[1] = hs->specials->vgt_gs_out_prim_type; + } + } + } + + if (uc_regs != nullptr) { + EXIT_NOT_IMPLEMENTED(gs->specials->ge_cntl.offset != Pm4::GE_CNTL); + EXIT_NOT_IMPLEMENTED(gs->specials->ge_user_vgpr_en.offset != Pm4::GE_USER_VGPR_EN); + EXIT_NOT_IMPLEMENTED(hs != nullptr && + hs->specials->ge_user_vgpr_en.offset != Pm4::GE_USER_VGPR_EN); + + uc_regs[0] = gs->specials->ge_cntl; + uc_regs[1] = gs->specials->ge_user_vgpr_en; + uc_regs[2].offset = Pm4::VGT_PRIMITIVE_TYPE; + uc_regs[2].value = prim_type; + + if (hs != nullptr) { + uc_regs[1] = hs->specials->ge_user_vgpr_en; + } + } + + return OK; +} + +int KYTY_SYSV_ABI GraphicsUpdatePrimState(ShaderRegister* cx_regs, ShaderRegister* uc_regs, + uint32_t prim_type) { + PRINT_NAME(); + + LOGF("\t cx_regs = 0x%016" PRIx64 "\n" + "\t uc_regs = 0x%016" PRIx64 "\n" + "\t prim_type = %" PRIu32 "\n", + reinterpret_cast(cx_regs), reinterpret_cast(uc_regs), prim_type); + + if (cx_regs != nullptr && (cx_regs[0].value & 0x24u) == 0) { + cx_regs[1].value &= ~0x7u; + cx_regs[1].value |= GraphicsPrimitiveTypeToGsOut(prim_type); + } + + if (uc_regs != nullptr) { + uc_regs[2].value &= ~0x1fu; + uc_regs[2].value |= prim_type; + } + + return OK; +} + +static uint32_t shader_semantic_word(const ShaderSemantic& semantic) { + return ((semantic.semantic & 0xffu) << 0u) | ((semantic.hardware_mapping & 0xffu) << 8u) | + ((semantic.size_in_elements & 0xfu) << 16u) | ((semantic.is_f16 & 0x3u) << 20u) | + ((semantic.is_flat_shaded & 0x1u) << 22u) | ((semantic.is_linear & 0x1u) << 23u) | + ((semantic.is_custom & 0x1u) << 24u) | ((semantic.static_vb_index & 0x1u) << 25u) | + ((semantic.static_attribute & 0x1u) << 26u) | ((semantic.reserved & 0x1u) << 27u) | + ((semantic.default_value & 0x3u) << 28u) | ((semantic.default_value_hi & 0x3u) << 30u); +} + +static uint32_t apply_interpolant_default_value(uint32_t value, uint32_t ps_word) { + value &= ~0x00000300u; + value |= ((ps_word >> 28u) & 0x3u) << 8u; + return value; +} + +static uint32_t apply_interpolant_default_value_hi(uint32_t value, uint32_t ps_word) { + value &= ~0x00600000u; + value |= ((ps_word >> 30u) & 0x3u) << 21u; + return value; +} + +static uint32_t create_interpolant_mapping_value(uint32_t value, uint32_t ps_word, + uint32_t gs_word) { + const uint32_t flat_shade = + ((ps_word & 0x00400000u) != 0 || (ps_word & 0x01000000u) != 0 ? 0x00000400u : 0u); + + value &= ~0x0000001fu; + value |= (gs_word >> 8u) & 0x1fu; + value &= ~0x00000400u; + value |= flat_shade; + + return apply_interpolant_default_value(value, ps_word); +} + +static uint32_t create_interpolant_default_value(uint32_t value, uint32_t ps_word) { + value &= ~0x0000001fu; + value &= ~0x00000400u; + + return apply_interpolant_default_value(value, ps_word); +} + +static uint32_t create_interpolant_f16_value(uint32_t ps_word, const ShaderSemantic* gs_semantic) { + uint32_t value = (ps_word << 4u) & 0x03000000u; + + if (gs_semantic == nullptr) { + value |= 0x00180020u; + } else { + const uint32_t common_word = ps_word & shader_semantic_word(*gs_semantic); + + value &= 0xfff7ffdfu; + value |= (common_word >> 15u) & 0x20u; + value ^= 0x00080020u; + value &= ~0x00100000u; + value |= (~common_word >> 1u) & 0x00100000u; + } + + return apply_interpolant_default_value_hi(value, ps_word); +} + +static uint32_t create_interpolant_non_f16_value(uint32_t ps_word, + const ShaderSemantic* gs_semantic) { + uint32_t value = 0; + if ((ps_word & 0x01000000u) != 0 || gs_semantic == nullptr) { + value |= 0x20u; + } + + return value; +} + +static const ShaderSemantic* find_interpolant_output_semantic(const Shader* gs, uint32_t semantic) { + if (gs == nullptr || gs->output_semantics == nullptr) { + return nullptr; + } + + for (uint16_t i = 0; i < gs->num_output_semantics; i++) { + if (gs->output_semantics[i].semantic == semantic) { + return &gs->output_semantics[i]; + } + } + + return nullptr; +} + +static void set_interpolant_register(ShaderRegister* regs, uint32_t index, uint32_t value) { + // The native implementation writes a descriptor offset (0x10000000 + index). Kyty stores + // decoded PM4 register offsets here because direct/custom indirect paths consume them. + regs[index].offset = Pm4::SPI_PS_INPUT_CNTL_0 + index; + regs[index].value = value; +} + +static void set_identity_interpolant_registers(ShaderRegister* regs, uint32_t first_index) { + for (uint32_t i = first_index; i < 32u; i++) { + set_interpolant_register(regs, i, i); + } +} + +int KYTY_SYSV_ABI GraphicsCreateInterpolantMapping(ShaderRegister* regs, const Shader* gs, + const Shader* ps) { + PRINT_NAME(); + + LOGF("\t regs = 0x%016" PRIx64 "\n" + "\t gs = 0x%016" PRIx64 "\n" + "\t ps = 0x%016" PRIx64 "\n", + reinterpret_cast(regs), reinterpret_cast(gs), + reinterpret_cast(ps)); + + EXIT_NOT_IMPLEMENTED(regs == nullptr); + EXIT_NOT_IMPLEMENTED(ps != nullptr && ps->num_input_semantics != 0 && + ps->input_semantics == nullptr); + + if (ps == nullptr || ps->num_input_semantics == 0) { + set_identity_interpolant_registers(regs, 0); + return OK; + } + + EXIT_NOT_IMPLEMENTED(gs == nullptr); + EXIT_NOT_IMPLEMENTED(gs->num_output_semantics != 0 && gs->output_semantics == nullptr); + + for (uint32_t ps_index = 0; ps_index < ps->num_input_semantics; ps_index++) { + const auto& ps_semantic = ps->input_semantics[ps_index]; + const auto* gs_semantic = find_interpolant_output_semantic(gs, ps_semantic.semantic); + const auto ps_word = shader_semantic_word(ps_semantic); + + auto value = + ((ps_word & 0x00300000u) != 0 ? create_interpolant_f16_value(ps_word, gs_semantic) + : create_interpolant_non_f16_value(ps_word, gs_semantic)); + value = (gs_semantic == nullptr ? create_interpolant_default_value(value, ps_word) + : create_interpolant_mapping_value( + value, ps_word, shader_semantic_word(*gs_semantic))); + + set_interpolant_register(regs, ps_index, value); + } + + set_identity_interpolant_registers(regs, ps->num_input_semantics); + + return OK; +} + +int KYTY_SYSV_ABI GraphicsGetDataPacketPayloadAddress(uint32_t** addr, uint32_t* cmd, int type) { + PRINT_NAME(); + + static std::atomic log_count {0}; + if (log_count.fetch_add(1) < 64) { + LOGF("\t addr = 0x%016" PRIx64 "\n" + "\t cmd = 0x%016" PRIx64 "\n" + "\t type = %d\n", + reinterpret_cast(addr), reinterpret_cast(cmd), type); + } + + EXIT_NOT_IMPLEMENTED(addr == nullptr); + + if (type != 0) { + *addr = cmd + 2; + return OK; + } + + auto cmd_id = cmd[0]; + *addr = ((~cmd_id & 0x3fff0000u) != 0 ? cmd + 1 : nullptr); + + return OK; +} + +int KYTY_SYSV_ABI GraphicsGetDataPacketPayloadRange(MemoryRange* range, uint32_t* cmd, int type) { + PRINT_NAME(); + + static std::atomic log_count {0}; + const bool should_log = (log_count.fetch_add(1) < 64); + if (should_log) { + LOGF("\t range = 0x%016" PRIx64 "\n" + "\t cmd = 0x%016" PRIx64 "\n" + "\t type = %d\n", + reinterpret_cast(range), reinterpret_cast(cmd), type); + } + + EXIT_NOT_IMPLEMENTED(range == nullptr); + EXIT_NOT_IMPLEMENTED(cmd == nullptr); + + const auto cmd_id = cmd[0]; + const auto len = KYTY_PM4_LEN(cmd_id); + + if (type != 0) { + range->m_base = cmd + 2; + range->m_size = static_cast((cmd_id >> 14u) & 0xfffcu); + } else { + if ((~cmd_id & 0x3fff0000u) == 0) { + range->m_base = nullptr; + range->m_size = 0; + } else { + range->m_base = cmd + 1; + range->m_size = static_cast((cmd_id >> 14u) & 0xfffcu) + sizeof(uint32_t); + } + } + + if (should_log) { + LOGF("\t cmd_id = 0x%08" PRIx32 ", len = %" PRIu32 "\n" + "\t base = 0x%016" PRIx64 "\n" + "\t size = 0x%016" PRIx64 "\n", + cmd_id, len, reinterpret_cast(range->m_base), range->m_size); + } + + return OK; +} + +int KYTY_SYSV_ABI GraphicsWriteDataPatchSetAddressOrOffset(uint32_t* cmd, + uint64_t address_or_offset) { + EXIT_NOT_IMPLEMENTED(cmd == nullptr); + + auto op = (cmd[0] >> 8u) & 0xffu; + if (op == Pm4::IT_WRITE_DATA) { + cmd[2] = static_cast(address_or_offset & 0xffffffffu); + cmd[3] = static_cast((address_or_offset >> 32u) & 0xffffffffu); + return OK; + } + + return static_cast(0x8a6c000cu); +} + +int KYTY_SYSV_ABI GraphicsUnknownJumpPatchSetTarget(uint32_t* cmd, const volatile uint32_t* target, + uint32_t size_in_dwords) { + PRINT_NAME(); + + LOGF("Calling unknown stub"); + + return GraphicsJumpPatchSetTarget(cmd, target, size_in_dwords); +} + +int KYTY_SYSV_ABI GraphicsJumpPatchSetTarget(uint32_t* cmd, const volatile uint32_t* target, + uint32_t size_in_dwords) { + PRINT_NAME(); + + LOGF("\t cmd = 0x%016" PRIx64 "\n" + "\t target = 0x%016" PRIx64 "\n" + "\t size_in_dwords = %" PRIu32 "\n", + reinterpret_cast(cmd), reinterpret_cast(target), size_in_dwords); + + EXIT_NOT_IMPLEMENTED(cmd == nullptr); + + auto cmd_id = cmd[0]; + auto op = (cmd_id >> 8u) & 0xffu; + + LOGF("\t cmd_id = 0x%08" PRIx32 ", op = 0x%02" PRIx32 ", len = %" PRIu32 "\n", cmd_id, + op, KYTY_PM4_LEN(cmd_id)); + + if (op != Pm4::IT_INDIRECT_BUFFER) { + return GRAPHICS5_ERROR_INVALID_PACKET; + } + + auto vaddr = reinterpret_cast(target); + + cmd[1] = static_cast(vaddr & 0xffffffffu); + cmd[2] = (cmd[2] & 0xffff0000u) | static_cast((vaddr >> 32u) & 0xffffu); + cmd[3] = (cmd[3] & 0xfff00000u) | (size_in_dwords & 0xfffffu); + + return OK; +} + +int KYTY_SYSV_ABI GraphicsSuspendPoint() { + PRINT_NAME(); + + GraphicsRunDone(); + + return OK; +} + +uint32_t* KYTY_SYSV_ABI GraphicsUnknownQj7QZpgr9Uw(CommandBuffer* buf, uint32_t mode, + uint32_t value) { + PRINT_NAME(); + // ResetContextState? + LOGF("\t mode = 0x%08" PRIx32 "\n" + "\t value = 0x%08" PRIx32 "\n", + mode, value); + + if (buf == nullptr) { + return nullptr; + } + + buf->DbgDump(); + + auto* cmd = buf->AllocateDW(1); + + if (cmd == nullptr) { + return nullptr; + } + + cmd[0] = 0x80000000u; + + return cmd; +} + +uint64_t KYTY_SYSV_ABI GraphicsGetIsTrinityMode() { + // PS5 Pro? + return 0; +} + +static constexpr int GRAPHICS5_DRIVER_ERROR_INVALID_VALUE = static_cast(0x8a6c0033u); +static constexpr int GRAPHICS5_DRIVER_ERROR_INVALID_ARGUMENT = static_cast(0x8a6c0035u); +static constexpr uint32_t WORKLOAD_STREAM_RECORD_SIZE = 32; +static constexpr uint32_t WORKLOAD_ACTIVE_PACKET_SIZE_DW = 18; +static constexpr uint32_t WORKLOAD_COMPLETE_PACKET_SIZE_DW = 12; +static constexpr uint32_t WORKLOAD_STREAM_MIN_ID = 1; +static constexpr uint32_t WORKLOAD_STREAM_MAX_ID = 31; +static constexpr uint32_t WORKLOAD_ID_MAX = 63; +static constexpr uint32_t WORKLOAD_ACTIVE_COUNT_MAX = 63; + +static std::mutex g_workload_stream_mutex; +static uint32_t g_workload_stream_mask = 0; +static uint8_t g_workload_streams[WORKLOAD_STREAM_MAX_ID + 1][WORKLOAD_STREAM_RECORD_SIZE] {}; + +uint32_t KYTY_SYSV_ABI GraphicsDriverGetDefaultOwner() { + PRINT_NAME(); + + return 0x8a6c9018u; +} + +uint32_t KYTY_SYSV_ABI GraphicsDriverGetResourceRegistrationMaxNameLength() { + PRINT_NAME(); + + return 0x8a6c9018u; +} + +uint32_t KYTY_SYSV_ABI GraphicsDriverInitResourceRegistration() { + PRINT_NAME(); + + return 0x8a6c9018u; +} + +uint32_t KYTY_SYSV_ABI +GraphicsDriverQueryResourceRegistrationUserMemoryRequirements(uint64_t* size_in_bytes) { + PRINT_NAME(); + + if (size_in_bytes != nullptr) { + *size_in_bytes = 0; + } + + return 0x8a6c9018u; +} + +int KYTY_SYSV_ABI GraphicsDriverRegisterOwner() { + PRINT_NAME(); + + return static_cast(0x8a6c9018u); +} + +int KYTY_SYSV_ABI GraphicsDriverRegisterResource() { + PRINT_NAME(); + + return static_cast(0x8a6c9018u); +} + +int KYTY_SYSV_ABI GraphicsDriverUnregisterResource() { + PRINT_NAME(); + + return static_cast(0x8a6c9018u); +} + +int KYTY_SYSV_ABI GraphicsDriverRegisterWorkloadStream(uint32_t stream_id, const void* stream) { + PRINT_NAME(); + + LOGF("\t stream_id = %" PRIu32 "\n" + "\t stream = 0x%016" PRIx64 "\n", + stream_id, reinterpret_cast(stream)); + + if (stream_id < WORKLOAD_STREAM_MIN_ID || stream_id > WORKLOAD_STREAM_MAX_ID) { + return GRAPHICS5_DRIVER_ERROR_INVALID_VALUE; + } + + if (stream == nullptr) { + return GRAPHICS5_DRIVER_ERROR_INVALID_ARGUMENT; + } + + std::lock_guard lock(g_workload_stream_mutex); + + const uint32_t stream_bit = 1u << stream_id; + if ((g_workload_stream_mask & stream_bit) != 0) { + return GRAPHICS5_DRIVER_ERROR_INVALID_VALUE; + } + + std::memset(g_workload_streams[stream_id], 0, WORKLOAD_STREAM_RECORD_SIZE); + std::memcpy(g_workload_streams[stream_id], stream, WORKLOAD_STREAM_RECORD_SIZE); + g_workload_stream_mask |= stream_bit; + + return OK; +} + +uint32_t* KYTY_SYSV_ABI GraphicsCbNop(CommandBuffer* buf, uint32_t size_in_dwords) { + if (buf == nullptr || size_in_dwords < 2) { + return nullptr; + } + + auto* cmd = buf->AllocateDW(size_in_dwords); + + if (cmd == nullptr) { + return nullptr; + } + + cmd[0] = KYTY_PM4(size_in_dwords, Pm4::IT_NOP, Pm4::R_ZERO); + if (size_in_dwords > 1) { + memset(cmd + 1, 0, static_cast(size_in_dwords - 1) * 4); + } + + return cmd; +} + +uint32_t KYTY_SYSV_ABI GraphicsCbNopGetSize(uint32_t size_in_dwords) { + return 4u * size_in_dwords; +} + +uint32_t* KYTY_SYSV_ABI GraphicsCbDispatch(CommandBuffer* buf, uint32_t thread_group_x, + uint32_t thread_group_y, uint32_t thread_group_z, + uint32_t modifier) { + PRINT_NAME(); + + static std::atomic log_count {0}; + if (log_count.fetch_add(1) < 64) { + LOGF("\t thread_group_x = %" PRIu32 "\n" + "\t thread_group_y = %" PRIu32 "\n" + "\t thread_group_z = %" PRIu32 "\n" + "\t modifier = 0x%08" PRIx32 "\n", + thread_group_x, thread_group_y, thread_group_z, modifier); + } + + if (buf == nullptr) { + return nullptr; + } + + auto* cmd = buf->AllocateDW(5); + + if (cmd == nullptr) { + return nullptr; + } + + cmd[0] = KYTY_PM4(5, Pm4::IT_DISPATCH_DIRECT, 0u); + cmd[1] = thread_group_x; + cmd[2] = thread_group_y; + cmd[3] = thread_group_z; + cmd[4] = (modifier & 0xa038u) | 0x41u; + + return cmd; +} + +uint32_t KYTY_SYSV_ABI GraphicsCbDispatchGetSize() { + PRINT_NAME(); + + return 20; +} + +uint32_t* KYTY_SYSV_ABI GraphicsCbBranch(CommandBuffer* buf, uint8_t mode, uint8_t compare_function, + const volatile uint64_t* compare_addr, uint64_t mask, + uint64_t reference, uint8_t cache_policy1, + const volatile uint32_t* buffer1, uint32_t size_in_dwords1, + uint8_t cache_policy2, const volatile uint32_t* buffer2, + uint32_t size_in_dwords2) { + PRINT_NAME(); + + LOGF("\t mode = 0x%02" PRIx8 "\n" + "\t compare_function = 0x%02" PRIx8 "\n" + "\t compare_addr = 0x%016" PRIx64 "\n" + "\t mask = 0x%016" PRIx64 "\n" + "\t reference = 0x%016" PRIx64 "\n" + "\t cache_policy1 = 0x%02" PRIx8 "\n" + "\t buffer1 = 0x%016" PRIx64 "\n" + "\t size_in_dwords1 = %" PRIu32 "\n" + "\t cache_policy2 = 0x%02" PRIx8 "\n" + "\t buffer2 = 0x%016" PRIx64 "\n" + "\t size_in_dwords2 = %" PRIu32 "\n", + mode, compare_function, reinterpret_cast(compare_addr), mask, reference, + cache_policy1, reinterpret_cast(buffer1), size_in_dwords1, cache_policy2, + reinterpret_cast(buffer2), size_in_dwords2); + + EXIT_NOT_IMPLEMENTED(buf == nullptr); + + auto* cmd = buf->AllocateDW(14); + + if (cmd == nullptr) { + LOGF_COLOR(Log::Color::Red, "\t failed to allocate branch packet\n"); + return nullptr; + } + + const auto compare_vaddr = reinterpret_cast(compare_addr); + const auto then_vaddr = reinterpret_cast(buffer1); + const auto else_vaddr = reinterpret_cast(buffer2); + + cmd[0] = KYTY_PM4(14, Pm4::IT_INDIRECT_BUFFER, 0u); + cmd[1] = (mode & 0x3u) | ((static_cast(compare_function) & 0x7u) << 8u); + cmd[2] = static_cast(compare_vaddr & 0xfffffff8u); + cmd[3] = static_cast((compare_vaddr >> 32u) & 0xffffffffu); + cmd[4] = static_cast(mask & 0xffffffffu); + cmd[5] = static_cast((mask >> 32u) & 0xffffffffu); + cmd[6] = static_cast(reference & 0xffffffffu); + cmd[7] = static_cast((reference >> 32u) & 0xffffffffu); + cmd[8] = static_cast(then_vaddr & 0xfffffffcu); + cmd[9] = static_cast((then_vaddr >> 32u) & 0xffffffffu); + cmd[10] = (size_in_dwords1 & 0xfffffu) | ((static_cast(cache_policy1) & 0x3u) << 28u); + cmd[11] = static_cast(else_vaddr & 0xfffffffcu); + cmd[12] = static_cast((else_vaddr >> 32u) & 0xffffffffu); + cmd[13] = (size_in_dwords2 & 0xfffffu) | ((static_cast(cache_policy2) & 0x3u) << 28u); + + return cmd; +} + +uint32_t* KYTY_SYSV_ABI GraphicsCbSetShRegisterRangeDirect(CommandBuffer* buf, uint32_t offset, + const uint32_t* values, + uint32_t num_values) { + PRINT_NAME(); + + static std::atomic log_count {0}; + if (log_count.fetch_add(1) < 64) { + LOGF("\t buf = 0x%016" PRIx64 "\n" + "\t offset = %" PRIx32 "\n" + "\t values = 0x%016" PRIx64 "\n" + "\t num_values = %" PRIu32 "\n", + reinterpret_cast(buf), offset, reinterpret_cast(values), + num_values); + } + + EXIT_NOT_IMPLEMENTED(buf == nullptr); + + buf->DbgDump(); + + auto* cmd = buf->AllocateDW(num_values + 2); + + if (cmd == nullptr) { + LOGF_COLOR(Log::Color::Red, "\t failed to allocate set-register-range command\n"); + return nullptr; + } + + cmd[0] = KYTY_PM4(num_values + 2, Pm4::IT_SET_SH_REG, 0u); + cmd[1] = offset & 0xffffu; + + if (values != nullptr) { + memcpy(cmd + 2, values, static_cast(num_values) * 4); + } + + return cmd; +} + +uint32_t KYTY_SYSV_ABI GraphicsCbSetShRegisterRangeDirectGetSize(uint32_t num_values) { + PRINT_NAME(); + + LOGF("\t num_values = %" PRIu32 "\n", num_values); + + return 4u * num_values + 8u; +} + +uint32_t* KYTY_SYSV_ABI GraphicsCbSetShRegistersDirect(CommandBuffer* buf, + const volatile ShaderRegister* regs, + uint32_t num_regs) { + PRINT_NAME(); + + LOGF("\t regs = 0x%016" PRIx64 "\n" + "\t num_regs = %" PRIu32 "\n", + reinterpret_cast(regs), num_regs); + + if (num_regs == 0) { + return nullptr; + } + + EXIT_NOT_IMPLEMENTED(buf == nullptr); + EXIT_NOT_IMPLEMENTED(regs == nullptr); + + buf->DbgDump(); + + std::vector local_regs(num_regs); + for (uint32_t i = 0; i < num_regs; i++) { + local_regs[i].offset = regs[i].offset; + local_regs[i].value = regs[i].value; + } + + uint32_t* first_cmd = nullptr; + uint32_t run_start_index = 0; + uint32_t run_start_offset = local_regs[0].offset; + uint32_t prev_offset = run_start_offset; + + for (uint32_t i = 1; i <= num_regs; i++) { + const bool flush = (i == num_regs || local_regs[i].offset != prev_offset + 1u); + + if (flush) { + const auto run_count = i - run_start_index; + auto* cmd = buf->AllocateDW(run_count + 2u); + + if (cmd == nullptr) { + LOGF_COLOR(Log::Color::Red, + "\t failed to allocate set-sh-registers-direct command\n"); + return first_cmd; + } + + if (first_cmd == nullptr) { + first_cmd = cmd; + } + + cmd[0] = KYTY_PM4(run_count + 2u, Pm4::IT_SET_SH_REG, 0u); + cmd[1] = run_start_offset & 0xffffu; + + for (uint32_t j = 0; j < run_count; j++) { + cmd[j + 2u] = local_regs[run_start_index + j].value; + } + + if (i < num_regs) { + run_start_index = i; + run_start_offset = local_regs[i].offset; + } + } + + if (i < num_regs) { + prev_offset = local_regs[i].offset; + } + } + + return first_cmd; +} + +int KYTY_SYSV_ABI GraphicsDebugRaiseException(uint32_t exception_id) { + PRINT_NAME(); + + LOGF("\t exception_id = 0x%08" PRIx32 "\n", exception_id); + + return OK; +} + +uint32_t* KYTY_SYSV_ABI GraphicsCbReleaseMem(CommandBuffer* buf, uint8_t action, uint16_t gcr_cntl, + uint8_t dst, uint8_t cache_policy, + const volatile Label* address, uint8_t data_sel, + uint64_t data, uint16_t gds_offset, uint16_t gds_size, + uint8_t interrupt, uint32_t interrupt_ctx_id) { + PRINT_NAME(); + + static std::atomic log_count {0}; + if (log_count.fetch_add(1) < 64) { + LOGF("\t action = 0x%02" PRIx8 "\n" + "\t gcr_cntl = 0x%04" PRIx16 "\n" + "\t dst = %" PRIu8 "\n" + "\t cache_policy = 0x%02" PRIx8 "\n" + "\t address = 0x%016" PRIx64 "\n" + "\t data_sel = 0x%02" PRIx8 "\n" + "\t data = 0x%016" PRIx64 "\n" + "\t gds_offset = %" PRIu16 "\n" + "\t gds_size = %" PRIu16 "\n" + "\t interrupt = 0x%02" PRIx8 "\n" + "\t interrupt_ctx_id = %" PRIu32 "\n", + action, gcr_cntl, dst, cache_policy, reinterpret_cast(address), data_sel, + data, gds_offset, gds_size, interrupt, interrupt_ctx_id); + } + + EXIT_NOT_IMPLEMENTED(buf == nullptr); + EXIT_NOT_IMPLEMENTED(dst > 1); + EXIT_NOT_IMPLEMENTED(data_sel != 0 && data_sel != 1 && data_sel != 2 && data_sel != 3 && + data_sel != 5); + EXIT_NOT_IMPLEMENTED(interrupt > 4); + + buf->DbgDump(); + + auto* cmd = buf->AllocateDW(8); + + if (cmd == nullptr) { + LOGF_COLOR(Log::Color::Red, "\t failed to allocate release_mem packet\n"); + return nullptr; + } + + auto packet_gcr_cntl = static_cast(gcr_cntl); + if ((packet_gcr_cntl & 0x300u) == 0x100u) { + packet_gcr_cntl |= 0x200u; + } + + auto address_value = reinterpret_cast(address); + auto packet_data = data; + if ((interrupt & 0x7u) == 4u) { + address_value = 0; + packet_data = 0; + } else if ((data_sel & 0x7u) == 5u) { + packet_data = static_cast(gds_offset) | (static_cast(gds_size) << 16u); + } + + const uint32_t packet_action = static_cast(action) & 0x3fu; + const uint32_t packet_event_index = (action >= 0x2fu ? 6u : 5u); + + cmd[0] = KYTY_PM4(8, Pm4::IT_NOP, Pm4::R_RELEASE_MEM); + cmd[1] = packet_action | (packet_event_index << 8u) | ((packet_gcr_cntl & 0xfffu) << 12u) | + ((static_cast(cache_policy) & 0x3u) << 25u); + cmd[2] = ((static_cast(dst) & 0x3u) << 16u) | + ((static_cast(interrupt) & 0x7u) << 24u) | + ((static_cast(data_sel) & 0x7u) << 29u); + cmd[3] = static_cast(address_value & 0xfffffffcu); + cmd[4] = static_cast((address_value >> 32u) & 0xffffffffu); + cmd[5] = static_cast(packet_data & 0xffffffffu); + cmd[6] = static_cast((packet_data >> 32u) & 0xffffffffu); + cmd[7] = interrupt_ctx_id & 0x07ffffffu; + + return cmd; +} + +uint32_t KYTY_SYSV_ABI GraphicsCbQueueEndOfPipeActionGetSize() { + PRINT_NAME(); + + return 32; +} + +uint32_t* KYTY_SYSV_ABI GraphicsAcbResetQueue(CommandBuffer* buf, uint32_t op) { + PRINT_NAME(); + + LOGF("\t op = 0x%08" PRIx32 "\n", op); + + EXIT_NOT_IMPLEMENTED(buf == nullptr); + EXIT_NOT_IMPLEMENTED((op & ~0x1c2u) != 0); + + buf->DbgDump(); + + auto* cmd = buf->AllocateDW(2); + + EXIT_NOT_IMPLEMENTED(cmd == nullptr); + + cmd[0] = KYTY_PM4(2, Pm4::IT_NOP, Pm4::R_DISPATCH_RESET); + cmd[1] = 0; + + return cmd; +} + +uint32_t* KYTY_SYSV_ABI GraphicsDcbResetQueue(CommandBuffer* buf, uint32_t op, uint32_t state) { + PRINT_NAME(); + + LOGF("\t op = 0x%08" PRIx32 "\n" + "\t state = 0x%08" PRIx32 "\n", + op, state); + + EXIT_NOT_IMPLEMENTED(buf == nullptr); + EXIT_NOT_IMPLEMENTED((op & ~0xfffu) != 0); + + buf->DbgDump(); + + auto* cmd = buf->AllocateDW(2); + + EXIT_NOT_IMPLEMENTED(cmd == nullptr); + + cmd[0] = KYTY_PM4(2, Pm4::IT_CLEAR_STATE, 0u); + cmd[1] = state & 0xfu; + + return cmd; +} + +uint32_t* KYTY_SYSV_ABI GraphicsDcbWaitUntilSafeForRendering(CommandBuffer* buf, + uint32_t video_out_handle, + uint32_t display_buffer_index) { + PRINT_NAME(); + + LOGF("\t video_out_handle = %" PRIu32 "\n" + "\t display_buffer_index = %" PRIu32 "\n", + video_out_handle, display_buffer_index); + + EXIT_NOT_IMPLEMENTED(buf == nullptr); + + buf->DbgDump(); + + auto* cmd = buf->AllocateDW(7); + + EXIT_NOT_IMPLEMENTED(cmd == nullptr); + + cmd[0] = KYTY_PM4(7, Pm4::IT_NOP, Pm4::R_WAIT_FLIP_DONE); + cmd[1] = video_out_handle; + cmd[2] = display_buffer_index; + cmd[3] = 0; + cmd[4] = 0; + cmd[5] = 0; + cmd[6] = 0; + + return cmd; +} + +uint32_t* KYTY_SYSV_ABI GraphicsDcbSetWorkloadsActive(CommandBuffer* buf, uint32_t stream_id, + const uint32_t* workload_ids, + uint32_t workload_count) { + PRINT_NAME(); + + LOGF("\t stream_id = %" PRIu32 "\n" + "\t workload_ids = 0x%016" PRIx64 "\n" + "\t workload_count = %" PRIu32 "\n", + stream_id, reinterpret_cast(workload_ids), workload_count); + + if (buf == nullptr || workload_ids == nullptr || workload_count == 0 || + workload_count > WORKLOAD_ACTIVE_COUNT_MAX || stream_id < WORKLOAD_STREAM_MIN_ID || + stream_id > WORKLOAD_STREAM_MAX_ID) { + return nullptr; + } + + uint64_t workload_mask = 0; + for (uint32_t i = 0; i < workload_count; i++) { + const auto workload_id = workload_ids[i]; + if (workload_id > WORKLOAD_ID_MAX) { + return nullptr; + } + + const uint64_t workload_bit = 1ull << workload_id; + if ((workload_mask & workload_bit) != 0) { + return nullptr; + } + workload_mask |= workload_bit; + } + + { + std::lock_guard lock(g_workload_stream_mutex); + if ((g_workload_stream_mask & (1u << stream_id)) == 0) { + return nullptr; + } + } + + buf->DbgDump(); + + auto* cmd = buf->AllocateDW(WORKLOAD_ACTIVE_PACKET_SIZE_DW); + if (cmd == nullptr) { + return nullptr; + } + + cmd[0] = KYTY_PM4(WORKLOAD_ACTIVE_PACKET_SIZE_DW, Pm4::IT_NOP, Pm4::R_ZERO); + std::memset(cmd + 1, 0, + static_cast(WORKLOAD_ACTIVE_PACKET_SIZE_DW - 1) * sizeof(uint32_t)); + cmd[1] = stream_id; + cmd[2] = static_cast(workload_mask & 0xffffffffu); + cmd[3] = static_cast((workload_mask >> 32u) & 0xffffffffu); + + return cmd; +} + +uint32_t* KYTY_SYSV_ABI GraphicsDcbSetWorkloadComplete(CommandBuffer* buf, uint32_t stream_id, + uint32_t workload_id) { + PRINT_NAME(); + + LOGF("\t stream_id = %" PRIu32 "\n" + "\t workload_id = %" PRIu32 "\n", + stream_id, workload_id); + + if (buf == nullptr || stream_id < WORKLOAD_STREAM_MIN_ID || + stream_id > WORKLOAD_STREAM_MAX_ID || workload_id > WORKLOAD_ID_MAX) { + return nullptr; + } + + { + std::lock_guard lock(g_workload_stream_mutex); + if ((g_workload_stream_mask & (1u << stream_id)) == 0) { + return nullptr; + } + } + + buf->DbgDump(); + + auto* cmd = buf->AllocateDW(WORKLOAD_COMPLETE_PACKET_SIZE_DW); + if (cmd == nullptr) { + return nullptr; + } + + const uint64_t workload_clear_mask = ~(1ull << workload_id); + + cmd[0] = KYTY_PM4(WORKLOAD_COMPLETE_PACKET_SIZE_DW, Pm4::IT_NOP, Pm4::R_ZERO); + std::memset(cmd + 1, 0, + static_cast(WORKLOAD_COMPLETE_PACKET_SIZE_DW - 1) * sizeof(uint32_t)); + cmd[1] = stream_id; + cmd[2] = workload_id; + cmd[3] = static_cast(workload_clear_mask & 0xffffffffu); + cmd[4] = static_cast((workload_clear_mask >> 32u) & 0xffffffffu); + + return cmd; +} + +uint32_t* KYTY_SYSV_ABI GraphicsDcbSetShRegisterDirect(CommandBuffer* buf, ShaderRegister reg) { + PRINT_NAME(); + + EXIT_NOT_IMPLEMENTED(buf == nullptr); + + buf->DbgDump(); + + auto* cmd = buf->AllocateDW(3); + + if (cmd == nullptr) { + LOGF_COLOR(Log::Color::Red, "\t failed to allocate set-sh-register-direct command\n"); + return nullptr; + } + + cmd[0] = KYTY_PM4(3, Pm4::IT_SET_SH_REG, 0u); + cmd[1] = reg.offset & 0xffffu; + cmd[2] = reg.value; + + return cmd; +} + +uint32_t* KYTY_SYSV_ABI GraphicsDcbSetCxRegisterDirect(CommandBuffer* buf, ShaderRegister reg) { + PRINT_NAME(); + + EXIT_NOT_IMPLEMENTED(buf == nullptr); + + buf->DbgDump(); + + auto* cmd = buf->AllocateDW(3); + + if (cmd == nullptr) { + LOGF_COLOR(Log::Color::Red, "\t failed to allocate set-cx-register-direct command\n"); + return nullptr; + } + + cmd[0] = KYTY_PM4(3, Pm4::IT_SET_CONTEXT_REG, 0u); + cmd[1] = reg.offset & 0xffffu; + cmd[2] = reg.value; + + return cmd; +} + +uint32_t KYTY_SYSV_ABI GraphicsDcbSetCxRegisterDirectGetSize() { + PRINT_NAME(); + + return 12; +} + +uint32_t* KYTY_SYSV_ABI GraphicsDcbSetUcRegisterDirect(CommandBuffer* buf, ShaderRegister reg) { + PRINT_NAME(); + + EXIT_NOT_IMPLEMENTED(buf == nullptr); + + buf->DbgDump(); + + auto* cmd = buf->AllocateDW(3); + + if (cmd == nullptr) { + LOGF_COLOR(Log::Color::Red, "\t failed to allocate set-uc-register-direct command\n"); + return nullptr; + } + + cmd[0] = KYTY_PM4(3, Pm4::IT_SET_UCONFIG_REG, 0u); + cmd[1] = reg.offset & 0xffffu; + cmd[2] = reg.value; + + return cmd; +} + +uint32_t* KYTY_SYSV_ABI GraphicsDcbSetCxRegistersIndirect(CommandBuffer* buf, + const volatile ShaderRegister* regs, + uint32_t num_regs) { + PRINT_NAME(); + + LOGF("\t regs = 0x%016" PRIx64 "\n" + "\t num_regs = %" PRIu32 "\n", + reinterpret_cast(regs), num_regs); + + EXIT_NOT_IMPLEMENTED(buf == nullptr); + + buf->DbgDump(); + + auto* cmd = buf->AllocateDW(5); + + EXIT_NOT_IMPLEMENTED(cmd == nullptr); + + auto vaddr = reinterpret_cast(regs); + + reg_indirect_write_packet(cmd, vaddr, num_regs, RegIndirectPacket::Cx); + + return cmd; +} + +uint32_t* KYTY_SYSV_ABI GraphicsDcbSetShRegistersIndirect(CommandBuffer* buf, + const volatile ShaderRegister* regs, + uint32_t num_regs) { + PRINT_NAME(); + + LOGF("\t regs = 0x%016" PRIx64 "\n" + "\t num_regs = %" PRIu32 "\n", + reinterpret_cast(regs), num_regs); + + EXIT_NOT_IMPLEMENTED(buf == nullptr); + + buf->DbgDump(); + + auto* cmd = buf->AllocateDW(5); + + EXIT_NOT_IMPLEMENTED(cmd == nullptr); + + auto vaddr = reinterpret_cast(regs); + + reg_indirect_write_packet(cmd, vaddr, num_regs, RegIndirectPacket::Sh); + + return cmd; +} + +uint32_t* KYTY_SYSV_ABI GraphicsDcbSetUcRegistersIndirect(CommandBuffer* buf, + const volatile ShaderRegister* regs, + uint32_t num_regs) { + PRINT_NAME(); + + LOGF("\t regs = 0x%016" PRIx64 "\n" + "\t num_regs = %" PRIu32 "\n", + reinterpret_cast(regs), num_regs); + + EXIT_NOT_IMPLEMENTED(buf == nullptr); + + buf->DbgDump(); + + auto* cmd = buf->AllocateDW(5); + + EXIT_NOT_IMPLEMENTED(cmd == nullptr); + + auto vaddr = reinterpret_cast(regs); + + reg_indirect_write_packet(cmd, vaddr, num_regs, RegIndirectPacket::Uc); + + return cmd; +} + +uint32_t* KYTY_SYSV_ABI GraphicsDcbSetIndexSize(CommandBuffer* buf, uint8_t index_size, + uint8_t cache_policy) { + PRINT_NAME(); + + LOGF("\t index_size = 0x%" PRIx8 "\n" + "\t cache_policy = 0x%" PRIx8 "\n", + index_size, cache_policy); + + EXIT_NOT_IMPLEMENTED(buf == nullptr); + buf->DbgDump(); + + auto* cmd = buf->AllocateDW(3); + + EXIT_NOT_IMPLEMENTED(cmd == nullptr); + + cmd[0] = KYTY_PM4(3, Pm4::IT_SET_UCONFIG_REG_INDEX, 0u); + cmd[1] = 0x20000000u | Pm4::VGT_INDEX_TYPE; + cmd[2] = 0x400u | (index_size & 0x3u) | ((cache_policy & 0x3u) << 6u); + + return cmd; +} + +uint32_t* KYTY_SYSV_ABI GraphicsDcbSetIndexBuffer(CommandBuffer* buf, uint64_t index_addr) { + PRINT_NAME(); + + LOGF("\t index_addr = 0x%016" PRIx64 "\n", index_addr); + + EXIT_NOT_IMPLEMENTED(buf == nullptr); + EXIT_NOT_IMPLEMENTED((index_addr & 1u) != 0); + + buf->DbgDump(); + + auto* cmd = buf->AllocateDW(3); + + EXIT_NOT_IMPLEMENTED(cmd == nullptr); + + cmd[0] = KYTY_PM4(3, Pm4::IT_INDEX_BASE, 0u); + cmd[1] = static_cast(index_addr & 0xffffffffu); + cmd[2] = static_cast((index_addr >> 32u) & 0xffffffffu); + + return cmd; +} + +uint32_t* KYTY_SYSV_ABI GraphicsDcbSetIndexCount(CommandBuffer* buf, uint32_t index_count) { + PRINT_NAME(); + + LOGF("\t index_count = 0x%" PRIx32 "\n", index_count); + + EXIT_NOT_IMPLEMENTED(buf == nullptr); + + buf->DbgDump(); + + auto* cmd = buf->AllocateDW(2); + + EXIT_NOT_IMPLEMENTED(cmd == nullptr); + + cmd[0] = KYTY_PM4(2, Pm4::IT_INDEX_BUFFER_SIZE, 0u); + cmd[1] = index_count; + + return cmd; +} + +uint32_t* KYTY_SYSV_ABI GraphicsDcbSetNumInstances(CommandBuffer* buf, uint32_t num_instances) { + PRINT_NAME(); + + LOGF("\t num_instances = 0x%" PRIx32 "\n", num_instances); + + if (buf == nullptr) { + return nullptr; + } + + buf->DbgDump(); + + auto* cmd = buf->AllocateDW(2); + + if (cmd == nullptr) { + return nullptr; + } + + cmd[0] = KYTY_PM4(2, Pm4::IT_NUM_INSTANCES, 0u); + cmd[1] = num_instances; + + return cmd; +} + +uint32_t KYTY_SYSV_ABI GraphicsDcbSetNumInstancesGetSize() { + PRINT_NAME(); + + return 8; +} + +static uint32_t extract_modifier_bits(uint32_t modifier, uint32_t start, uint32_t count) { + return (modifier >> start) & ((1u << count) - 1u); +} + +static uint32_t indirect_modifier_sgpr_base(uint32_t modifier) { + const auto stage = modifier >> 29u; + return ((stage == 3u || stage == 5u) ? 0x80u : 0u) + 0x8cu; +} + +static uint64_t decode_indirect_modifier_patch_offsets(uint64_t modifier, bool indexed) { + const auto low = static_cast(modifier); + const auto sgpr_base = indirect_modifier_sgpr_base(low); + + uint64_t base_vtx_loc = 0x280u; + if ((low & 0x1u) != 0) { + base_vtx_loc = sgpr_base + extract_modifier_bits(low, 9u, 5u); + } + + uint64_t start_inst_loc = 0x280u; + if ((low & 0x4u) != 0) { + start_inst_loc = sgpr_base + extract_modifier_bits(low, 19u, 5u); + } + + if (indexed && (low & 0x2u) != 0) { + base_vtx_loc |= static_cast(sgpr_base + extract_modifier_bits(low, 14u, 5u)) + << 16u; + base_vtx_loc |= 1ull << 59u; + } + + return base_vtx_loc | (start_inst_loc << 32u); +} + +static uint32_t decode_indirect_draw_initiator(uint64_t modifier) { + const auto low = static_cast(modifier); + uint32_t initiator = 2u; + + if ((modifier & (1ull << 32u)) == 0) { + initiator = ((low >> 3u) & 0x20u) | 2u; + } + + return initiator; +} + +static uint32_t decode_draw_index_initiator(uint64_t modifier) { + if ((modifier & (1ull << 32u)) != 0) { + return 0; + } + + return (static_cast(modifier) >> 3u) & 0x20u; +} + +uint32_t* KYTY_SYSV_ABI GraphicsDcbDrawIndex(CommandBuffer* buf, uint32_t index_count, + const volatile void* index_addr, uint64_t modifier) { + PRINT_NAME(); + + LOGF("\t index_count = 0x%" PRIx32 "\n" + "\t index_addr = 0x%016" PRIx64 "\n" + "\t modifier = 0x%016" PRIx64 "\n", + index_count, reinterpret_cast(index_addr), modifier); + + EXIT_NOT_IMPLEMENTED(buf == nullptr); + EXIT_NOT_IMPLEMENTED(index_addr == nullptr); + EXIT_NOT_IMPLEMENTED((reinterpret_cast(index_addr) & 1u) != 0); + + buf->DbgDump(); + + auto* cmd = buf->AllocateDW(6); + + EXIT_NOT_IMPLEMENTED(cmd == nullptr); + + auto vaddr = reinterpret_cast(index_addr); + + cmd[0] = KYTY_PM4(6, Pm4::IT_DRAW_INDEX_2, 0u); + cmd[1] = (index_count == 0 ? 1u : index_count); + cmd[2] = static_cast(vaddr & 0xffffffffu); + cmd[3] = static_cast((vaddr >> 32u) & 0xffffffffu); + cmd[4] = index_count; + cmd[5] = decode_draw_index_initiator(modifier); + + return cmd; +} + +uint32_t KYTY_SYSV_ABI GraphicsDcbDrawIndexGetSize() { + PRINT_NAME(); + + return 6u * 4u; +} + +uint32_t* KYTY_SYSV_ABI GraphicsDcbDrawIndexMultiInstanced(CommandBuffer* buf, uint32_t index_count, + const volatile void* index_addr, + const volatile void* object_ids, + uint32_t instance_count, + uint64_t modifier) { + PRINT_NAME(); + + LOGF("\t index_count = 0x%" PRIx32 "\n" + "\t index_addr = 0x%016" PRIx64 "\n" + "\t instance_count = 0x%" PRIx32 "\n" + "\t object_ids = 0x%016" PRIx64 "\n" + "\t modifier = 0x%016" PRIx64 "\n", + index_count, reinterpret_cast(index_addr), instance_count, + reinterpret_cast(object_ids), modifier); + + EXIT_NOT_IMPLEMENTED(buf == nullptr); + EXIT_NOT_IMPLEMENTED(index_addr == nullptr); + EXIT_NOT_IMPLEMENTED(object_ids == nullptr); + EXIT_NOT_IMPLEMENTED((reinterpret_cast(index_addr) & 1u) != 0); + + buf->DbgDump(); + + auto* cmd = buf->AllocateDW(9); + + EXIT_NOT_IMPLEMENTED(cmd == nullptr); + + auto index_vaddr = reinterpret_cast(index_addr); + auto object_vaddr = reinterpret_cast(object_ids); + + cmd[0] = KYTY_PM4(9, Pm4::IT_DISPATCH_DRAW_PREAMBLE, 0u); + cmd[1] = index_count; + cmd[2] = static_cast(index_vaddr & 0xffffffffu); + cmd[3] = static_cast((index_vaddr >> 32u) & 0xffffffffu); + cmd[4] = (instance_count == 0 ? 1u : instance_count); + cmd[5] = static_cast(object_vaddr & 0xffffffffu); + cmd[6] = static_cast((object_vaddr >> 32u) & 0xffffffffu); + cmd[7] = instance_count; + cmd[8] = decode_draw_index_initiator(modifier) | 0x80u; + + return cmd; +} + +uint32_t KYTY_SYSV_ABI GraphicsDcbDrawIndexMultiInstancedGetSize() { + PRINT_NAME(); + + return 9u * 4u; +} + +int KYTY_SYSV_ABI GraphicsUnknownIkfdtRIqCE(uint32_t* cmd, uint64_t arg1, + const volatile uint32_t* target, + uint32_t size_in_dwords, uint64_t arg4, uint64_t arg5) { + PRINT_NAME(); + + LOGF("\t cmd = 0x%016" PRIx64 "\n" + "\t arg1 = 0x%016" PRIx64 "\n" + "\t target = 0x%016" PRIx64 "\n" + "\t size_in_dwords = %" PRIu32 "\n" + "\t arg4 = 0x%016" PRIx64 "\n" + "\t arg5 = 0x%016" PRIx64 "\n", + reinterpret_cast(cmd), arg1, reinterpret_cast(target), size_in_dwords, + arg4, arg5); + + EXIT_NOT_IMPLEMENTED(cmd == nullptr); + + auto op = (cmd[0] >> 8u) & 0xffu; + if (op != Pm4::IT_INDIRECT_BUFFER) { + return 0x8a6c000c; + } + + auto vaddr = reinterpret_cast(target); + auto cache_policy = static_cast(arg1) & 0x3u; + + cmd[1] = (cmd[1] & 0x3u) | (static_cast(vaddr) & 0xfffffffcu); + cmd[2] = static_cast(vaddr >> 32u); + cmd[3] = (cmd[3] & 0xcff00000u) | (cache_policy << 28u) | (size_in_dwords & 0xfffffu); + + return OK; +} + +uint32_t* KYTY_SYSV_ABI GraphicsDcbDrawIndexAuto(CommandBuffer* buf, uint32_t index_count, + uint64_t modifier) { + PRINT_NAME(); + + LOGF("\t index_count = 0x%" PRIx32 "\n" + "\t modifier = 0x%016" PRIx64 "\n", + index_count, modifier); + + EXIT_NOT_IMPLEMENTED(buf == nullptr); + + buf->DbgDump(); + + auto* cmd = buf->AllocateDW(3); + + EXIT_NOT_IMPLEMENTED(cmd == nullptr); + + cmd[0] = KYTY_PM4(3, Pm4::IT_DRAW_INDEX_AUTO, 0u); + cmd[1] = index_count; + cmd[2] = decode_draw_index_initiator(modifier) | 0x2u; + + return cmd; +} + +uint32_t KYTY_SYSV_ABI GraphicsDcbDrawIndexAutoGetSize() { + PRINT_NAME(); + + return 3u * 4u; +} + +uint32_t* KYTY_SYSV_ABI GraphicsDcbDrawIndexOffset(CommandBuffer* buf, uint32_t index_offset, + uint32_t index_count, uint64_t modifier) { + PRINT_NAME(); + + LOGF("\t index_offset = 0x%" PRIx32 "\n" + "\t index_count = 0x%" PRIx32 "\n" + "\t modifier = 0x%016" PRIx64 "\n", + index_offset, index_count, modifier); + + EXIT_NOT_IMPLEMENTED(buf == nullptr); + + buf->DbgDump(); + + auto* cmd = buf->AllocateDW(5); + + EXIT_NOT_IMPLEMENTED(cmd == nullptr); + + cmd[0] = KYTY_PM4(5, Pm4::IT_DRAW_INDEX_OFFSET_2, 0u); + cmd[1] = (index_count == 0 ? 1u : index_count); + cmd[2] = index_offset; + cmd[3] = index_count; + cmd[4] = decode_draw_index_initiator(modifier); + + return cmd; +} + +uint32_t KYTY_SYSV_ABI GraphicsDcbDrawIndexOffsetGetSize() { + PRINT_NAME(); + + return 5u * 4u; +} + +uint32_t* KYTY_SYSV_ABI GraphicsDcbSetBaseIndirectArgs(CommandBuffer* buf, uint32_t shader_type, + const volatile void* indirect_base_addr) { + PRINT_NAME(); + + LOGF("\t shader_type = %" PRIu32 "\n" + "\t indirect_base_addr = 0x%016" PRIx64 "\n", + shader_type, reinterpret_cast(indirect_base_addr)); + + EXIT_NOT_IMPLEMENTED(buf == nullptr); + + buf->DbgDump(); + + auto* cmd = buf->AllocateDW(4); + + EXIT_NOT_IMPLEMENTED(cmd == nullptr); + + const auto addr = reinterpret_cast(indirect_base_addr); + + cmd[0] = KYTY_PM4(4, Pm4::IT_SET_BASE, 0u) | (shader_type << 1u); + cmd[1] = 1u; + cmd[2] = static_cast(addr) & ~0x7u; + cmd[3] = static_cast(addr >> 32u); + + return cmd; +} + +uint32_t* KYTY_SYSV_ABI GraphicsDcbDrawIndexIndirect(CommandBuffer* buf, + uint32_t data_offset_in_bytes, + uint64_t modifier) { + PRINT_NAME(); + + LOGF("\t data_offset = 0x%" PRIx32 "\n" + "\t modifier = 0x%016" PRIx64 "\n", + data_offset_in_bytes, modifier); + + EXIT_NOT_IMPLEMENTED(buf == nullptr); + + buf->DbgDump(); + + auto* cmd = buf->AllocateDW(5); + + EXIT_NOT_IMPLEMENTED(cmd == nullptr); + + const auto patch_offsets = decode_indirect_modifier_patch_offsets(modifier, true); + + cmd[0] = KYTY_PM4(5, Pm4::IT_DRAW_INDEX_INDIRECT, 0u); + cmd[1] = data_offset_in_bytes; + cmd[2] = static_cast(patch_offsets); + cmd[3] = static_cast(patch_offsets >> 32u); + cmd[4] = decode_indirect_draw_initiator(modifier); + + return cmd; +} + +uint32_t* KYTY_SYSV_ABI GraphicsDcbDrawIndirect(CommandBuffer* buf, uint32_t data_offset_in_bytes, + uint64_t modifier) { + PRINT_NAME(); + + LOGF("\t data_offset = 0x%" PRIx32 "\n" + "\t modifier = 0x%016" PRIx64 "\n", + data_offset_in_bytes, modifier); + + EXIT_NOT_IMPLEMENTED(buf == nullptr); + + buf->DbgDump(); + + auto* cmd = buf->AllocateDW(5); + + EXIT_NOT_IMPLEMENTED(cmd == nullptr); + + const auto patch_offsets = decode_indirect_modifier_patch_offsets(modifier, false); + + cmd[0] = KYTY_PM4(5, Pm4::IT_DRAW_INDIRECT, 0u); + cmd[1] = data_offset_in_bytes; + cmd[2] = static_cast(patch_offsets); + cmd[3] = static_cast(patch_offsets >> 32u); + cmd[4] = decode_indirect_draw_initiator(modifier); + + return cmd; +} + +uint32_t* KYTY_SYSV_ABI GraphicsDcbDrawIndexIndirectMulti( + CommandBuffer* buf, uint32_t data_offset_in_bytes, uint32_t count_indirect, + uint32_t max_count_or_count, const volatile void* count_addr, uint32_t stride_in_bytes, + uint64_t modifier) { + PRINT_NAME(); + + LOGF("\t data_offset = 0x%" PRIx32 "\n" + "\t count_indirect = 0x%" PRIx32 "\n" + "\t max_count_or_count = 0x%" PRIx32 "\n" + "\t count_addr = 0x%016" PRIx64 "\n" + "\t stride_in_bytes = 0x%" PRIx32 "\n" + "\t modifier = 0x%016" PRIx64 "\n", + data_offset_in_bytes, count_indirect, max_count_or_count, + reinterpret_cast(count_addr), stride_in_bytes, modifier); + + EXIT_NOT_IMPLEMENTED(buf == nullptr); + EXIT_NOT_IMPLEMENTED((count_indirect & ~1u) != 0); + + buf->DbgDump(); + + auto* cmd = buf->AllocateDW(10); + + EXIT_NOT_IMPLEMENTED(cmd == nullptr); + + const auto patch_offsets = decode_indirect_modifier_patch_offsets(modifier, true); + const auto count_vaddr = reinterpret_cast(count_addr); + + cmd[0] = KYTY_PM4(10, Pm4::IT_DRAW_INDEX_INDIRECT_MULTI, 0u); + cmd[1] = data_offset_in_bytes; + cmd[2] = static_cast(patch_offsets); + cmd[3] = static_cast(patch_offsets >> 32u); + cmd[4] = (count_indirect & 1u) << 30u; + cmd[5] = max_count_or_count; + cmd[6] = static_cast(count_vaddr) & ~0x3u; + cmd[7] = static_cast(count_vaddr >> 32u); + cmd[8] = stride_in_bytes; + cmd[9] = decode_indirect_draw_initiator(modifier); + + return cmd; +} + +uint32_t KYTY_SYSV_ABI GraphicsDcbDrawIndirectGetSize() { + PRINT_NAME(); + + return 5u * 4u; +} + +uint32_t* KYTY_SYSV_ABI GraphicsDcbDispatchIndirect(CommandBuffer* buf, + uint32_t data_offset_in_bytes, uint32_t flags) { + PRINT_NAME(); + + LOGF("\t data_offset = 0x%" PRIx32 "\n" + "\t flags = 0x%08" PRIx32 "\n", + data_offset_in_bytes, flags); + + EXIT_NOT_IMPLEMENTED(buf == nullptr); + + buf->DbgDump(); + + auto* cmd = buf->AllocateDW(3); + + EXIT_NOT_IMPLEMENTED(cmd == nullptr); + + cmd[0] = KYTY_PM4(3, Pm4::IT_DISPATCH_INDIRECT, 0u); + cmd[1] = data_offset_in_bytes; + cmd[2] = (flags & 0xa038u) | 0x41u; + + return cmd; +} + +uint32_t KYTY_SYSV_ABI GraphicsDcbDispatchIndirectGetSize() { + PRINT_NAME(); + + return 3u * 4u; +} + +uint32_t* KYTY_SYSV_ABI GraphicsDcbEventWrite(CommandBuffer* buf, uint8_t event_type, + const volatile void* address) { + PRINT_NAME(); + + static std::atomic log_count {0}; + if (log_count.fetch_add(1) < 64) { + LOGF("\t event_type = 0x%02" PRIx8 "\n" + "\t address = 0x%016" PRIx64 "\n", + event_type, reinterpret_cast(address)); + } + + EXIT_NOT_IMPLEMENTED(buf == nullptr); + EXIT_NOT_IMPLEMENTED(event_type > 0x3fu); + + buf->DbgDump(); + + const bool addressed_event = ((event_type & 0xfeu) == 0x38u); + auto* cmd = buf->AllocateDW(addressed_event ? 4u : 2u); + + EXIT_NOT_IMPLEMENTED(cmd == nullptr); + + cmd[0] = KYTY_PM4(addressed_event ? 4u : 2u, Pm4::IT_EVENT_WRITE, 0u); + + if (event_type == 7u || event_type == 15u || event_type == 16u) { + cmd[1] = 0x400u | event_type; + } else if (addressed_event) { + auto addr = reinterpret_cast(address); + + cmd[1] = 0x100u | event_type; + cmd[2] = static_cast(addr) & 0xfffffff8u; + cmd[3] = static_cast(addr >> 32u); + } else { + cmd[1] = event_type & 0x3fu; + } + + return cmd; +} + +uint32_t* KYTY_SYSV_ABI GraphicsAcbEventWrite(CommandBuffer* buf, uint8_t event_type, + const volatile void* address) { + return GraphicsDcbEventWrite(buf, event_type, address); +} + +uint32_t* KYTY_SYSV_ABI GraphicsDcbAcquireMem(CommandBuffer* buf, uint8_t engine, uint32_t cb_db_op, + uint32_t gcr_cntl, const volatile void* base, + uint64_t size_bytes, uint32_t poll_cycles) { + PRINT_NAME(); + + static std::atomic log_count {0}; + if (log_count.fetch_add(1) < 64) { + LOGF("\t engine = 0x%02" PRIx8 "\n" + "\t cb_db_op = 0x%08" PRIx32 "\n" + "\t gcr_cntl = 0x%08" PRIx32 "\n" + "\t base = 0x%016" PRIx64 "\n" + "\t size_bytes = 0x%016" PRIx64 "\n" + "\t poll_cycles = %" PRIu32 "\n", + engine, cb_db_op, gcr_cntl, reinterpret_cast(base), size_bytes, poll_cycles); + } + + bool no_size = (static_cast(size_bytes) == -1); + auto vaddr = reinterpret_cast(base); + + EXIT_NOT_IMPLEMENTED(buf == nullptr); + + static std::atomic warning_log_count {0}; + const bool log_warning = (warning_log_count.fetch_add(1) < 64); + if (!no_size && (size_bytes & 0xffu) != 0) { + if (log_warning) { + LOGF_COLOR(Log::Color::Red, "\t warning: size_bytes is not 256-byte aligned\n"); + } + } + if (!no_size && (size_bytes >> 40u) != 0) { + if (log_warning) { + LOGF_COLOR(Log::Color::Red, "\t warning: size_bytes is too large\n"); + } + } + if ((vaddr & 0xffu) != 0) { + if (log_warning) { + LOGF_COLOR(Log::Color::Red, "\t warning: base is not 256-byte aligned\n"); + } + } + if ((vaddr >> 40u) != 0) { + if (log_warning) { + LOGF_COLOR(Log::Color::Red, "\t warning: base is too high\n"); + } + } + if (engine > 1) { + if (log_warning) { + LOGF_COLOR(Log::Color::Red, "\t warning: unsupported engine 0x%02" PRIx8 "\n", engine); + } + } + + buf->DbgDump(); + + auto* cmd = buf->AllocateDW(8); + + if (cmd == nullptr) { + LOGF_COLOR(Log::Color::Red, "\t failed to allocate acquire_mem packet\n"); + return nullptr; + } + + cmd[0] = KYTY_PM4(8, Pm4::IT_NOP, Pm4::R_ACQUIRE_MEM); + cmd[1] = (static_cast(engine & 1u) << 31u) | cb_db_op; + cmd[2] = (no_size ? 0 : static_cast(size_bytes >> 8u)); + cmd[3] = 0; + cmd[4] = static_cast(vaddr >> 8u); + cmd[5] = 0; + cmd[6] = poll_cycles / 40; + cmd[7] = gcr_cntl; + + return cmd; +} + +uint32_t KYTY_SYSV_ABI GraphicsDcbAcquireMemGetSize() { + PRINT_NAME(); + + return 8u * 4u; +} + +uint32_t* KYTY_SYSV_ABI GraphicsAcbAcquireMem(CommandBuffer* buf, uint32_t gcr_cntl, + const volatile void* base, uint64_t size_bytes, + uint32_t poll_cycles) { + return GraphicsDcbAcquireMem(buf, 1, 0, gcr_cntl, base, size_bytes, poll_cycles); +} + +uint32_t KYTY_SYSV_ABI GraphicsAcbAcquireMemGetSize() { + PRINT_NAME(); + + return GraphicsDcbAcquireMemGetSize(); +} + +uint32_t* KYTY_SYSV_ABI GraphicsDcbCondExec(CommandBuffer* buf, const volatile uint32_t* address, + uint32_t num_dwords) { + PRINT_NAME(); + + LOGF("\t address = 0x%016" PRIx64 "\n" + "\t num_dwords = %" PRIu32 "\n", + reinterpret_cast(address), num_dwords); + + if (buf == nullptr || address == nullptr) { + LOGF_COLOR(Log::Color::Red, "\t invalid arguments\n"); + return nullptr; + } + if ((reinterpret_cast(address) & 0x3u) != 0) { + LOGF_COLOR(Log::Color::Red, "\t condExec command address is not 4-byte aligned\n"); + return nullptr; + } + if (num_dwords > 0x3fffu) { + LOGF_COLOR(Log::Color::Red, "\t condExec range is too large: %" PRIu32 "\n", num_dwords); + return nullptr; + } + + buf->DbgDump(); + + auto* cmd = buf->AllocateDW(5); + + if (cmd == nullptr) { + LOGF_COLOR(Log::Color::Red, "\t failed to allocate condExec packet\n"); + return nullptr; + } + + auto addr = reinterpret_cast(address); + + cmd[0] = KYTY_PM4(5, Pm4::IT_COND_EXEC, 0u); + cmd[1] = static_cast(addr) & 0xfffffffcu; + cmd[2] = static_cast(addr >> 32u); + cmd[3] = 0; + cmd[4] = num_dwords & 0x3fffu; + + return cmd; +} + +uint32_t KYTY_SYSV_ABI GraphicsDcbCondExecGetSize() { + PRINT_NAME(); + + return 20; +} + +uint32_t* KYTY_SYSV_ABI GraphicsAcbCondExec(CommandBuffer* buf, const volatile uint32_t* address, + uint32_t num_dwords) { + return GraphicsDcbCondExec(buf, address, num_dwords); +} + +uint32_t KYTY_SYSV_ABI GraphicsAcbCondExecGetSize() { + PRINT_NAME(); + + return GraphicsDcbCondExecGetSize(); +} + +uint32_t* KYTY_SYSV_ABI GraphicsAcbWaitRegMem(CommandBuffer* buf, uint8_t size, + uint8_t compare_function, uint8_t cache_policy, + const volatile void* address, uint64_t reference, + uint64_t mask, uint32_t poll_cycles) { + return GraphicsDcbWaitRegMem(buf, size, compare_function, 0, cache_policy, address, reference, + mask, poll_cycles); +} + +uint32_t* KYTY_SYSV_ABI GraphicsAcbDmaData(CommandBuffer* buf, uint8_t engine, uint8_t dst, + uint8_t dst_cache_policy, uint64_t dst_address_or_offset, + uint8_t src, uint8_t src_cache_policy, + uint64_t src_address_or_offset_or_immediate, + uint32_t num_bytes, uint8_t wait_for_previous, + uint8_t write_confirm, uint8_t block_engine) { + return GraphicsDcbDmaData(buf, engine, dst, dst_cache_policy, dst_address_or_offset, src, + src_cache_policy, src_address_or_offset_or_immediate, num_bytes, + wait_for_previous, write_confirm, block_engine); +} + +uint32_t* KYTY_SYSV_ABI GraphicsAcbCopyData(CommandBuffer* buf, uint8_t dst, + uint8_t dst_cache_policy, uint64_t dst_address, + uint8_t src, uint8_t src_cache_policy, + uint64_t src_address_or_immediate, uint8_t item_size, + uint8_t write_confirm) { + const auto dcb_src = (src == 5 ? static_cast(5u << 1u) : src); + return GraphicsDcbCopyData(buf, dst, dst_cache_policy, dst_address, dcb_src, src_cache_policy, + src_address_or_immediate, item_size, write_confirm); +} + +uint32_t* KYTY_SYSV_ABI GraphicsAcbDispatchIndirect(CommandBuffer* buf, + const volatile void* indirect_args, + uint32_t modifier) { + PRINT_NAME(); + + LOGF("\t indirect_args = 0x%016" PRIx64 "\n" + "\t modifier = 0x%08" PRIx32 "\n", + reinterpret_cast(indirect_args), modifier); + + if (buf == nullptr) { + return nullptr; + } + + buf->DbgDump(); + + auto* cmd = buf->AllocateDW(4); + + if (cmd == nullptr) { + return nullptr; + } + + const auto args_addr = reinterpret_cast(indirect_args); + + cmd[0] = KYTY_PM4(4, Pm4::IT_DISPATCH_INDIRECT, 0u); + cmd[1] = static_cast(args_addr & 0xffffffffu); + cmd[2] = static_cast((args_addr >> 32u) & 0xffffffffu); + cmd[3] = (modifier & 0xa038u) | 0x41u; + + return cmd; +} + +uint32_t* KYTY_SYSV_ABI GraphicsAcbWriteData(CommandBuffer* buf, uint8_t dst, uint8_t cache_policy, + uint64_t address_or_offset, const void* data, + uint32_t num_dwords, uint8_t increment, + uint8_t write_confirm) { + return GraphicsDcbWriteData(buf, dst, cache_policy, address_or_offset, data, num_dwords, + increment, write_confirm); +} + +uint32_t* KYTY_SYSV_ABI GraphicsDcbStallCommandBufferParser(CommandBuffer* buf) { + PRINT_NAME(); + + EXIT_NOT_IMPLEMENTED(buf == nullptr); + + buf->DbgDump(); + + auto* cmd = buf->AllocateDW(2); + + EXIT_NOT_IMPLEMENTED(cmd == nullptr); + + cmd[0] = KYTY_PM4(2, Pm4::IT_PFP_SYNC_ME, 0u); + cmd[1] = 0; + + return cmd; +} + +uint32_t* KYTY_SYSV_ABI GraphicsDcbCopyData(CommandBuffer* buf, uint8_t dst, + uint8_t dst_cache_policy, uint64_t dst_address, + uint8_t src, uint8_t src_cache_policy, + uint64_t src_address_or_immediate, uint8_t item_size, + uint8_t write_confirm) { + PRINT_NAME(); + + LOGF("\t dst = 0x%02" PRIx8 "\n" + "\t dst_cache_policy = 0x%02" PRIx8 "\n" + "\t dst_address = 0x%016" PRIx64 "\n" + "\t src = 0x%02" PRIx8 "\n" + "\t src_cache_policy = 0x%02" PRIx8 "\n" + "\t src_address_or_immediate = 0x%016" PRIx64 "\n" + "\t item_size = 0x%02" PRIx8 "\n" + "\t write_confirm = 0x%02" PRIx8 "\n", + dst, dst_cache_policy, dst_address, src, src_cache_policy, src_address_or_immediate, + item_size, write_confirm); + + if (buf == nullptr) { + return nullptr; + } + + auto* cmd = buf->AllocateDW(6); + + if (cmd == nullptr) { + return nullptr; + } + + cmd[0] = KYTY_PM4(6, Pm4::IT_COPY_DATA, 0u); + cmd[1] = ((static_cast(src) >> 1u) & 0xfu) | + (((static_cast(dst) >> 1u) & 0xfu) << 8u) | + ((static_cast(src_cache_policy) & 0x3u) << 13u) | + ((static_cast(item_size) & 0x1u) << 16u) | + ((static_cast(write_confirm) & 0x1u) << 20u) | + ((static_cast(dst_cache_policy) & 0x3u) << 25u) | + ((static_cast(src) & 0x1u) << 30u); + cmd[2] = static_cast(src_address_or_immediate & 0xffffffffu); + cmd[3] = static_cast((src_address_or_immediate >> 32u) & 0xffffffffu); + cmd[4] = static_cast(dst_address & 0xffffffffu); + cmd[5] = static_cast((dst_address >> 32u) & 0xffffffffu); + + return cmd; +} + +uint32_t* KYTY_SYSV_ABI GraphicsDcbDmaData(CommandBuffer* buf, uint8_t engine, uint8_t dst, + uint8_t dst_cache_policy, uint64_t dst_address_or_offset, + uint8_t src, uint8_t src_cache_policy, + uint64_t src_address_or_offset_or_immediate, + uint32_t num_bytes, uint8_t wait_for_previous, + uint8_t write_confirm, uint8_t block_engine) { + if (buf == nullptr) { + return nullptr; + } + + auto src_address = src_address_or_offset_or_immediate; + switch (src) { + case 0x14: src_address = (engine == 1u ? 0x30148u : 0x30174u); break; + case 0x24: src_address = (engine == 1u ? 0x30150u : 0x3017cu); break; + case 0x64: src_address = (engine == 1u ? 0x30158u : 0x30184u); break; + default: break; + } + + auto* cmd = buf->AllocateDW(7); + + if (cmd == nullptr) { + return nullptr; + } + + cmd[0] = KYTY_PM4(7, Pm4::IT_DMA_DATA, 0u); + cmd[1] = (static_cast(engine) & 0x1u) | + ((static_cast(src_cache_policy) & 0x3u) << 13u) | + ((static_cast(dst) & 0x3u) << 20u) | + ((static_cast(dst_cache_policy) & 0x3u) << 25u) | + ((static_cast(src) & 0x3u) << 29u) | + ((static_cast(block_engine) & 0x1u) << 31u); + cmd[2] = static_cast(src_address & 0xffffffffu); + cmd[3] = static_cast((src_address >> 32u) & 0xffffffffu); + cmd[4] = static_cast(dst_address_or_offset & 0xffffffffu); + cmd[5] = static_cast((dst_address_or_offset >> 32u) & 0xffffffffu); + cmd[6] = (num_bytes & 0x03ffffffu) | ((static_cast(src) & 0x4u) << 24u) | + ((static_cast(dst) & 0x4u) << 25u) | + ((static_cast(src) & 0x8u) << 25u) | + ((static_cast(dst) & 0x8u) << 26u) | + ((static_cast(wait_for_previous) & 0x1u) << 30u) | + ((static_cast(write_confirm) & 0x1u) << 31u); + + return cmd; +} + +uint32_t* KYTY_SYSV_ABI GraphicsDcbJump(CommandBuffer* buf, uint8_t mode, uint8_t cache_policy, + const uint32_t* target, uint32_t size_in_dwords) { + PRINT_NAME(); + + LOGF("\t mode = 0x%02" PRIx8 "\n" + "\t cache_policy = 0x%02" PRIx8 "\n" + "\t target = 0x%016" PRIx64 "\n" + "\t size_in_dwords = %" PRIu32 "\n", + mode, cache_policy, reinterpret_cast(target), size_in_dwords); + + if (buf == nullptr) { + return nullptr; + } + + auto* cmd = buf->AllocateDW(4); + + if (cmd == nullptr) { + return nullptr; + } + + auto vaddr = reinterpret_cast(target); + + cmd[0] = KYTY_PM4(4, Pm4::IT_INDIRECT_BUFFER, 0u); + cmd[1] = static_cast(vaddr) & ~0x3u; + cmd[2] = static_cast(vaddr >> 32u); + cmd[3] = 0x0f200000u | ((static_cast(cache_policy) & 0x3u) << 28u) | + ((static_cast(mode) & 0x1u) << 20u) | (size_in_dwords & 0xfffffu); + + return cmd; +} + +uint32_t KYTY_SYSV_ABI GraphicsDcbJumpGetSize() { + PRINT_NAME(); + + return 16; +} + +uint32_t KYTY_SYSV_ABI GraphicsDcbRewindGetSize() { + PRINT_NAME(); + + return 8; +} + +uint32_t* KYTY_SYSV_ABI GraphicsDcbRewind(CommandBuffer* buf, uint32_t initial_state) { + PRINT_NAME(); + + LOGF("\t initial_state = 0x%08" PRIx32 "\n", initial_state); + + if (buf == nullptr) { + return nullptr; + } + + auto* cmd = buf->AllocateDW(2); + + if (cmd == nullptr) { + return nullptr; + } + + cmd[0] = KYTY_PM4(2, Pm4::IT_REWIND, 0u); + cmd[1] = (initial_state & 0x1u) << 31u; + + return cmd; +} + +uint32_t* KYTY_SYSV_ABI GraphicsDcbSetPredication(CommandBuffer* buf, uint8_t condition, uint8_t op, + uint8_t wait_op, const volatile void* address, + uint32_t count_in_dwords) { + PRINT_NAME(); + + LOGF("\t condition = 0x%02" PRIx8 "\n" + "\t op = 0x%02" PRIx8 "\n" + "\t wait_op = 0x%02" PRIx8 "\n" + "\t address = 0x%016" PRIx64 "\n" + "\t count_in_dwords = %" PRIu32 "\n", + condition, op, wait_op, reinterpret_cast(address), count_in_dwords); + + if (buf == nullptr) { + return nullptr; + } + + auto* cmd = buf->AllocateDW(4); + + if (cmd == nullptr) { + return nullptr; + } + + auto addr = reinterpret_cast(address); + + cmd[0] = KYTY_PM4(4, Pm4::IT_SET_PREDICATION, 0u); + cmd[1] = ((static_cast(condition) & 0x1u) << 8u) | + ((static_cast(wait_op) & 0x1u) << 12u) | + ((static_cast(op) & 0x7u) << 16u); + cmd[2] = static_cast(addr) & ~0xfu; + cmd[3] = static_cast(addr >> 32u); + + (void)count_in_dwords; + + return cmd; +} + +uint32_t* KYTY_SYSV_ABI GraphicsUnknownKRzWekV120(CommandBuffer* buf, uint32_t arg1, uint32_t arg2, + uint32_t arg3) { + PRINT_NAME(); + + LOGF("\t argc = 4\n" + "\t arg0 = 0x%016" PRIx64 "\n" + "\t arg1 = 0x%08" PRIx32 "\n" + "\t arg2 = 0x%08" PRIx32 "\n" + "\t arg3 = 0x%08" PRIx32 "\n", + reinterpret_cast(buf), arg1, arg2, arg3); + + if (buf == nullptr) { + return nullptr; + } + + auto* cmd = buf->AllocateDW(3); + + if (cmd == nullptr) { + return nullptr; + } + + cmd[0] = 0xc0017a00u; + cmd[1] = 0x20000243u; + + uint32_t word8 = 0x400u; + word8 |= (arg1 & 0x3u); + word8 |= (arg2 & 0x3u) << 6u; + word8 |= (arg3 & 0x1u) << 14u; + + cmd[2] = word8; + + return cmd; +} + +int KYTY_SYSV_ABI GraphicsDmaDataPatchSetDstAddressOrOffset(uint32_t* cmd, + uint64_t dst_address_or_offset) { + EXIT_NOT_IMPLEMENTED(cmd == nullptr); + + auto op = (cmd[0] >> 8u) & 0xffu; + if (op == Pm4::IT_DMA_DATA) { + cmd[4] = static_cast(dst_address_or_offset & 0xffffffffu); + cmd[5] = static_cast((dst_address_or_offset >> 32u) & 0xffffffffu); + return OK; + } + + return static_cast(0x8a6c000cu); +} + +int KYTY_SYSV_ABI GraphicsDmaDataPatchSetSrcAddressOrOffsetOrImmediate( + uint32_t* cmd, uint64_t src_address_or_offset_or_immediate) { + EXIT_NOT_IMPLEMENTED(cmd == nullptr); + + auto op = (cmd[0] >> 8u) & 0xffu; + if (op == Pm4::IT_DMA_DATA) { + cmd[2] = static_cast(src_address_or_offset_or_immediate & 0xffffffffu); + cmd[3] = static_cast((src_address_or_offset_or_immediate >> 32u) & 0xffffffffu); + return OK; + } + + return static_cast(0x8a6c000cu); +} + +uint32_t KYTY_SYSV_ABI GraphicsGetPacketSize(uint32_t* packet) { + const auto cmd_id = packet[0]; + if ((cmd_id & 0x3fffff00u) == 0x3fff1000u) { + return 1; + } + + return KYTY_PM4_LEN(cmd_id); +} + +int KYTY_SYSV_ABI GraphicsSetPacketPredication(uint32_t* packet, uint32_t predication) { + PRINT_NAME(); + + LOGF("\t packet = 0x%016" PRIx64 "\n" + "\t predication = 0x%08" PRIx32 "\n", + reinterpret_cast(packet), predication); + + LOGF("\t packet0 = 0x%08" PRIx32 ", op = 0x%02" PRIx32 ", r = 0x%02" PRIx32 + ", len = %" PRIu32 "\n", + packet[0], (packet[0] >> 8u) & 0xffu, KYTY_PM4_R(packet[0]), + GraphicsGetPacketSize(packet)); + + packet[0] = (packet[0] & ~1u) | (static_cast(predication) == 1 ? 1u : 0u); + + return OK; +} + +int KYTY_SYSV_ABI GraphicsSetRangePredication(uint32_t* start, const volatile uint32_t* end, + uint32_t predication) { + PRINT_NAME(); + + LOGF("\t start = 0x%016" PRIx64 "\n" + "\t end = 0x%016" PRIx64 "\n" + "\t predication = 0x%08" PRIx32 "\n", + reinterpret_cast(start), reinterpret_cast(end), predication); + + auto* packet = start; + auto* range_end = const_cast(reinterpret_cast(end)); + auto packet_va = reinterpret_cast(packet); + auto end_va = reinterpret_cast(range_end); + + if (packet_va >= end_va) { + return OK; + } + + const uint32_t predication_bit = (static_cast(predication) == 1 ? 1u : 0u); + while (packet_va < end_va) { + const auto cmd_id = packet[0]; + packet[0] = (cmd_id & ~1u) | predication_bit; + + auto size = KYTY_PM4_LEN(cmd_id); + if ((cmd_id & 0x3fffff00u) == 0x3fff1000u) { + size = 1; + } + + packet_va += size * sizeof(uint32_t); + packet = reinterpret_cast(packet_va); + } + + return OK; +} + +int KYTY_SYSV_ABI GraphicsCondExecPatchSetEnd(uint32_t* cmd, const volatile uint32_t* buffer) { + PRINT_NAME(); + + LOGF("\t cmd = 0x%016" PRIx64 "\n" + "\t buffer = 0x%016" PRIx64 "\n", + reinterpret_cast(cmd), reinterpret_cast(buffer)); + + if (cmd == nullptr || buffer == nullptr) { + return static_cast(0x8a6c000cu); + } + + auto op = (cmd[0] >> 8u) & 0xffu; + if (op != Pm4::IT_COND_EXEC) { + return static_cast(0x8a6c000cu); + } + + auto* packet_end = cmd + 5; + auto* range_end = const_cast(reinterpret_cast(buffer)); + if (range_end < packet_end) { + return static_cast(0x8a6c000cu); + } + + auto num_dwords = static_cast(range_end - packet_end); + if (num_dwords > 0x3fffu) { + return static_cast(0x8a6c000cu); + } + + cmd[4] = (cmd[4] & ~0x3fffu) | static_cast(num_dwords); + return OK; +} + +int KYTY_SYSV_ABI GraphicsCondExecPatchSetCommandAddress(uint32_t* cmd, + const volatile uint32_t* command) { + PRINT_NAME(); + + LOGF("\t cmd = 0x%016" PRIx64 "\n" + "\t command = 0x%016" PRIx64 "\n", + reinterpret_cast(cmd), reinterpret_cast(command)); + + if (cmd == nullptr || command == nullptr) { + return static_cast(0x8a6c000cu); + } + + auto op = (cmd[0] >> 8u) & 0xffu; + if (op != Pm4::IT_COND_EXEC || (reinterpret_cast(command) & 0x3u) != 0) { + return static_cast(0x8a6c000cu); + } + + auto addr = reinterpret_cast(command); + cmd[1] = (cmd[1] & 0x3u) | (static_cast(addr) & 0xfffffffcu); + cmd[2] = static_cast(addr >> 32u); + + return OK; +} + +uint32_t* KYTY_SYSV_ABI GraphicsDcbWriteData(CommandBuffer* buf, uint8_t dst, uint8_t cache_policy, + uint64_t address_or_offset, const void* data, + uint32_t num_dwords, uint8_t increment, + uint8_t write_confirm) { + PRINT_NAME(); + + LOGF("\t dst = 0x%02" PRIx8 "\n" + "\t cache_policy = 0x%02" PRIx8 "\n" + "\t address_or_offset = 0x%016" PRIx64 "\n" + "\t data = 0x%016" PRIx64 "\n" + "\t num_dwords = %" PRIu32 "\n" + "\t increment = 0x%02" PRIx8 "\n" + "\t write_confirm = 0x%02" PRIx8 "\n", + dst, cache_policy, address_or_offset, reinterpret_cast(data), num_dwords, + increment, write_confirm); + + if (buf == nullptr || data == nullptr) { + LOGF_COLOR(Log::Color::Red, "\t invalid arguments\n"); + return nullptr; + } + if ((4 + num_dwords - 2u) > 0x3fffu) { + LOGF_COLOR(Log::Color::Red, "\t packet is too large\n"); + return nullptr; + } + + buf->DbgDump(); + + auto* cmd = buf->AllocateDW(4 + num_dwords); + + if (cmd == nullptr) { + LOGF_COLOR(Log::Color::Red, "\t failed to allocate command\n"); + return nullptr; + } + + const auto dst_value = static_cast(dst); + const auto write_confirm_value = + (dst == 0u ? 0u : (static_cast(write_confirm) & 0x1u)); + + cmd[0] = KYTY_PM4(4 + num_dwords, Pm4::IT_WRITE_DATA, 0u); + cmd[1] = ((dst_value & 0x1u) << 30u) | ((dst_value & 0x1eu) << 7u) | + ((static_cast(increment) & 0x1u) << 16u) | (write_confirm_value << 20u) | + ((static_cast(cache_policy) & 0x3u) << 25u); + cmd[2] = static_cast(address_or_offset) & ~0x3u; + cmd[3] = static_cast(address_or_offset >> 32u); + + memcpy(cmd + 4, data, static_cast(num_dwords) * 4); + + return cmd; +} + +uint32_t KYTY_SYSV_ABI GraphicsDcbWriteDataGetSize(uint32_t num_dwords) { + PRINT_NAME(); + + LOGF("\t num_dwords = %" PRIu32 "\n", num_dwords); + + return 4u * num_dwords + 16u; +} + +uint32_t* KYTY_SYSV_ABI GraphicsDcbGetLodStats(CommandBuffer* buf, uint8_t cache_policy, + const volatile void* buffer, + uint32_t buffer_size_in_bytes, uint32_t reset_count, + uint8_t force_reset, uint8_t report_and_reset, + uint32_t reporting_interval_in_100k_clocks) { + PRINT_NAME(); + + LOGF("\t cache_policy = 0x%02" PRIx8 "\n" + "\t buffer = 0x%016" PRIx64 "\n" + "\t buffer_size_in_bytes = %" PRIu32 "\n" + "\t reset_count = %" PRIu32 "\n" + "\t force_reset = 0x%02" PRIx8 "\n" + "\t report_and_reset = 0x%02" PRIx8 "\n" + "\t reporting_interval_in_100k_clocks = %" PRIu32 "\n", + cache_policy, reinterpret_cast(buffer), buffer_size_in_bytes, reset_count, + force_reset, report_and_reset, reporting_interval_in_100k_clocks); + + if (buf == nullptr) { + return nullptr; + } + + buf->DbgDump(); + + auto* cmd = buf->AllocateDW(5); + + if (cmd == nullptr) { + return nullptr; + } + + const auto buffer_addr = reinterpret_cast(buffer); + + cmd[0] = KYTY_PM4(5, Pm4::IT_GET_LOD_STATS, 0u); + cmd[1] = buffer_size_in_bytes; + cmd[2] = static_cast(buffer_addr & 0xffffffc0u); + cmd[3] = static_cast((buffer_addr >> 32u) & 0xffffffffu); + cmd[4] = ((static_cast(cache_policy) & 0x3u) << 28u) | + ((static_cast(report_and_reset) & 0x1u) << 19u) | + ((static_cast(force_reset) & 0x1u) << 18u) | ((reset_count & 0xffu) << 10u) | + ((reporting_interval_in_100k_clocks & 0xffu) << 2u); + + return cmd; +} + +int KYTY_SYSV_ABI GraphicsWaitRegMemPatchAddress(uint32_t* cmd, const volatile void* address) { + PRINT_NAME(); + + LOGF("\t cmd = 0x%016" PRIx64 "\n" + "\t address = 0x%016" PRIx64 "\n", + reinterpret_cast(cmd), reinterpret_cast(address)); + + EXIT_NOT_IMPLEMENTED(cmd == nullptr); + + auto vaddr = reinterpret_cast(address); + auto op = (cmd[0] >> 8u) & 0xffu; + + if (op == Pm4::IT_NOP && KYTY_PM4_R(cmd[0]) == Pm4::R_WAIT_MEM_32) { + cmd[1] = static_cast(vaddr) & ~0x3u; + cmd[2] = static_cast(vaddr >> 32u) & 0x3ffffu; + } else if (op == Pm4::IT_NOP && KYTY_PM4_R(cmd[0]) == Pm4::R_WAIT_MEM_64) { + cmd[1] = static_cast(vaddr) & ~0x7u; + cmd[2] = static_cast(vaddr >> 32u) & 0x3ffffu; + } else { + EXIT("unsupported waitOnAddress packet for address patch: 0x%08" PRIx32 "\n", cmd[0]); + } + + return OK; +} + +int KYTY_SYSV_ABI GraphicsWaitRegMemPatchReference(uint32_t* cmd, uint64_t reference) { + PRINT_NAME(); + + LOGF("\t cmd = 0x%016" PRIx64 "\n" + "\t reference = 0x%016" PRIx64 "\n", + reinterpret_cast(cmd), reference); + + EXIT_NOT_IMPLEMENTED(cmd == nullptr); + + auto op = (cmd[0] >> 8u) & 0xffu; + + if (op == Pm4::IT_NOP && KYTY_PM4_R(cmd[0]) == Pm4::R_WAIT_MEM_32) { + cmd[4] = static_cast(reference & 0xffffffffu); + } else if (op == Pm4::IT_NOP && KYTY_PM4_R(cmd[0]) == Pm4::R_WAIT_MEM_64) { + cmd[5] = static_cast(reference & 0xffffffffu); + cmd[6] = static_cast((reference >> 32u) & 0xffffffffu); + } else { + EXIT("unsupported waitOnAddress packet for reference patch: 0x%08" PRIx32 "\n", cmd[0]); + } + + return OK; +} + +int KYTY_SYSV_ABI GraphicsQueueEndOfPipeActionPatchAddress(uint32_t* cmd, + const volatile Label* address) { + PRINT_NAME(); + + // Not sure + + LOGF("\t cmd = 0x%016" PRIx64 "\n" + "\t address = 0x%016" PRIx64 "\n", + reinterpret_cast(cmd), reinterpret_cast(address)); + + EXIT_NOT_IMPLEMENTED(cmd == nullptr); + + auto vaddr = reinterpret_cast(address); + auto op = (cmd[0] >> 8u) & 0xffu; + + if ((op == Pm4::IT_NOP && KYTY_PM4_R(cmd[0]) == Pm4::R_RELEASE_MEM) || + op == Pm4::IT_RELEASE_MEM) { + cmd[3] = static_cast(vaddr & 0xffffffffu); + cmd[4] = static_cast((vaddr >> 32u) & 0xffffffffu); + } else if (op == Pm4::IT_EVENT_WRITE_EOP) { + cmd[2] = static_cast(vaddr & 0xffffffffu); + cmd[3] = (cmd[3] & 0xffff0000u) | static_cast((vaddr >> 32u) & 0xffffu); + } else { + EXIT("unsupported queueEndOfPipeAction packet for address patch: 0x%08" PRIx32 "\n", + cmd[0]); + } + + return OK; +} + +int KYTY_SYSV_ABI GraphicsQueueEndOfPipeActionPatchData(uint32_t* cmd, uint32_t context_id, + uint32_t data_sel, uint64_t data) { + PRINT_NAME(); + + LOGF("\t cmd = 0x%016" PRIx64 "\n" + "\t context_id = 0x%08" PRIx32 "\n" + "\t data_sel = 0x%08" PRIx32 "\n" + "\t data = 0x%016" PRIx64 "\n", + reinterpret_cast(cmd), context_id, data_sel, data); + + EXIT_NOT_IMPLEMENTED(cmd == nullptr); + + auto op = (cmd[0] >> 8u) & 0xffu; + + if ((op == Pm4::IT_NOP && KYTY_PM4_R(cmd[0]) == Pm4::R_RELEASE_MEM) || + op == Pm4::IT_RELEASE_MEM) { + uint64_t packet_data = data; + if (op == Pm4::IT_NOP && KYTY_PM4_R(cmd[0]) == Pm4::R_RELEASE_MEM && context_id > 1 && + data_sel == 1) { + // Agc Core ring-buffer release packets pack the segment generation + // into bits 24..31 and wrap that byte every 256 submissions. The + // patch context carries the monotonic generation, so expand the + // packed value before it reaches the command processor. + packet_data = (static_cast(context_id - 2u) << 24u) | (data & 0x00ffffffull); + } + + cmd[5] = static_cast(packet_data & 0xffffffffu); + cmd[6] = static_cast((packet_data >> 32u) & 0xffffffffu); + } else if (op == Pm4::IT_EVENT_WRITE_EOP) { + cmd[4] = static_cast(data & 0xffffffffu); + cmd[5] = static_cast((data >> 32u) & 0xffffffffu); + } else { + EXIT("unsupported queueEndOfPipeAction packet for data patch: 0x%08" PRIx32 "\n", cmd[0]); + } + + return OK; +} + +static uint32_t wait_reg_mem_poll_cycles_to_packet(uint32_t poll_cycles) { + return std::min(poll_cycles >> 4u, 0xffffu); +} + +static uint32_t wait_reg_mem32_control(uint8_t compare_function, uint8_t op, uint8_t cache_policy) { + return 0x10u | (static_cast(compare_function) & 0x7u) | + ((static_cast(op) & 0x3u) << 8u) | ((static_cast(op) & 0xcu) << 4u) | + ((static_cast(cache_policy) & 0x3u) << 25u); +} + +static uint32_t wait_reg_mem64_control(uint8_t compare_function, uint8_t op, uint8_t cache_policy) { + return 0x10u | (static_cast(compare_function) & 0x7u) | + ((static_cast(op) & 0x1u) << 8u) | ((static_cast(op) & 0x6u) << 5u) | + ((static_cast(cache_policy) & 0x3u) << 25u); +} + +uint32_t* KYTY_SYSV_ABI GraphicsDcbWaitRegMem(CommandBuffer* buf, uint8_t size, + uint8_t compare_function, uint8_t op, + uint8_t cache_policy, const volatile void* address, + uint64_t reference, uint64_t mask, + uint32_t poll_cycles) { + PRINT_NAME(); + + LOGF("\t size = 0x%02" PRIx8 "\n" + "\t compare_function = 0x%02" PRIx8 "\n" + "\t op = 0x%02" PRIx8 "\n" + "\t cache_policy = 0x%02" PRIx8 "\n" + "\t address = 0x%016" PRIx64 "\n" + "\t reference = 0x%016" PRIx64 "\n" + "\t mask = 0x%016" PRIx64 "\n" + "\t poll_cycles = %" PRIu32 "\n", + size, compare_function, op, cache_policy, reinterpret_cast(address), reference, + mask, poll_cycles); + + EXIT_NOT_IMPLEMENTED(buf == nullptr); + if (size != 0 && size != 1) { + LOGF_COLOR(Log::Color::Red, + "\t warning: unsupported size 0x%02" PRIx8 ", expected 0 (32b) or 1 (64b)\n", + size); + return nullptr; + } + if (cache_policy > 3) { + LOGF_COLOR(Log::Color::Red, + "\t warning: unsupported cache_policy 0x%02" PRIx8 ", expected 0..3\n", + cache_policy); + } + + buf->DbgDump(); + + auto address_value = reinterpret_cast(address); + bool wait32 = (size == 0); + auto poll = wait_reg_mem_poll_cycles_to_packet(poll_cycles); + + auto* cmd = buf->AllocateDW(wait32 ? 7 : 9); + + if (cmd == nullptr) { + return nullptr; + } + + if (wait32) { + cmd[0] = KYTY_PM4(7, Pm4::IT_NOP, Pm4::R_WAIT_MEM_32); + cmd[1] = static_cast(address_value) & ~0x3u; + cmd[2] = static_cast(address_value >> 32u) & 0x3ffffu; + cmd[3] = static_cast(mask & 0xffffffffu); + cmd[4] = static_cast(reference & 0xffffffffu); + cmd[5] = wait_reg_mem32_control(compare_function, op, cache_policy); + cmd[6] = poll; + + return cmd; + } + + cmd[0] = KYTY_PM4(9, Pm4::IT_NOP, Pm4::R_WAIT_MEM_64); + cmd[1] = static_cast(address_value) & ~0x7u; + cmd[2] = static_cast(address_value >> 32u) & 0x3ffffu; + cmd[3] = static_cast(reinterpret_cast(mask) & 0xffffffffu); + cmd[4] = static_cast((reinterpret_cast(mask) >> 32u) & 0xffffffffu); + cmd[5] = static_cast(reinterpret_cast(reference) & 0xffffffffu); + cmd[6] = static_cast((reinterpret_cast(reference) >> 32u) & 0xffffffffu); + cmd[7] = wait_reg_mem64_control(compare_function, op, cache_policy); + cmd[8] = poll; + + return cmd; +} + +uint32_t KYTY_SYSV_ABI GraphicsDcbWaitOnAddressGetSize(uint32_t size) { + PRINT_NAME(); + + switch (size) { + case 0: return 14u * 4u; + case 1: return 16u * 4u; + default: return 0; + } +} + +uint32_t* KYTY_SYSV_ABI GraphicsDcbPushMarker(CommandBuffer* buf, const char* str, + uint32_t /*color*/) { + if (buf == nullptr) { + return nullptr; + } + + if (str == nullptr) { + str = ""; + } + + const auto len = strlen(str) + 1; + const auto payload_dwords = static_cast((len + 3) / 4); + const auto size = 1u + (payload_dwords == 0 ? 1u : payload_dwords); + auto* cmd = buf->AllocateDW(size); + + if (cmd == nullptr) { + return nullptr; + } + + cmd[0] = KYTY_PM4(size, Pm4::IT_NOP, Pm4::R_PUSH_MARKER); + memset(cmd + 1, 0, static_cast(size - 1) * sizeof(uint32_t)); + memcpy(cmd + 1, str, len); + + return cmd; +} + +uint32_t* KYTY_SYSV_ABI GraphicsDcbSetMarker(CommandBuffer* buf, const char* str, uint32_t color) { + return GraphicsDcbPushMarker(buf, str, color); +} + +uint32_t* KYTY_SYSV_ABI GraphicsAcbSetMarker(CommandBuffer* buf, const char* str, uint32_t color) { + return GraphicsDcbSetMarker(buf, str, color); +} + +uint32_t* KYTY_SYSV_ABI GraphicsAcbPushMarker(CommandBuffer* buf, const char* str, uint32_t color) { + return GraphicsDcbPushMarker(buf, str, color); +} + +uint32_t* KYTY_SYSV_ABI GraphicsDcbPopMarker(CommandBuffer* buf) { + if (buf == nullptr) { + return nullptr; + } + + auto* cmd = buf->AllocateDW(2); + + if (cmd == nullptr) { + return nullptr; + } + + cmd[0] = KYTY_PM4(2, Pm4::IT_NOP, Pm4::R_POP_MARKER); + cmd[1] = 0; + + return cmd; +} + +uint32_t* KYTY_SYSV_ABI GraphicsAcbPopMarker(CommandBuffer* buf) { + return GraphicsDcbPopMarker(buf); +} + +uint32_t* KYTY_SYSV_ABI GraphicsDcbSetFlip(CommandBuffer* buf, uint32_t video_out_handle, + int32_t display_buffer_index, uint32_t flip_mode, + int64_t flip_arg) { + PRINT_NAME(); + + LOGF("\t video_out_handle = %" PRIu32 "\n" + "\t display_buffer_index = %" PRId32 "\n" + "\t flip_mode = %" PRIu32 "\n" + "\t flip_arg = %" PRId64 "\n", + video_out_handle, display_buffer_index, flip_mode, flip_arg); + + EXIT_NOT_IMPLEMENTED(buf == nullptr); + + buf->DbgDump(); + + auto* cmd = buf->AllocateDW(6); + + EXIT_NOT_IMPLEMENTED(cmd == nullptr); + + cmd[0] = KYTY_PM4(6, Pm4::IT_NOP, Pm4::R_FLIP); + cmd[1] = video_out_handle; + cmd[2] = display_buffer_index; + cmd[3] = flip_mode; + cmd[4] = static_cast(static_cast(flip_arg) & 0xffffffffu); + cmd[5] = static_cast((static_cast(flip_arg) >> 32u) & 0xffffffffu); + + return cmd; +} + +} // namespace Gen5 + +namespace Gen5Driver { + +LIB_NAME("Graphics5Driver", "Graphics5Driver"); + +struct TessellationDriverState { + uint64_t tf_ring_base = 0; + uint32_t tf_ring_size = 0; + uint64_t hs_offchip_value0 = 0; + uint64_t hs_offchip_value1 = 0; + uint64_t hs_offchip_value2 = 0; +}; + +static TessellationDriverState g_tessellation_driver_state {}; + +struct Packet { + uint32_t* addr; + uint32_t dw_num; + uint8_t pad[4]; +}; + +static bool dcb_has_queued_interrupt(const uint32_t* dcb, uint32_t size_in_dwords) { + if (dcb == nullptr) { + return false; + } + + for (uint32_t offset = 0; offset < size_in_dwords;) { + auto cmd_id = dcb[offset]; + auto len = KYTY_PM4_LEN(cmd_id); + if (len == 0 || len > size_in_dwords - offset) { + return false; + } + + auto op = (cmd_id >> 8u) & 0xffu; + if (op == Pm4::IT_NOP && KYTY_PM4_R(cmd_id) == Pm4::R_RELEASE_MEM && len >= 6) { + auto interrupt = (dcb[offset + 2] >> 24u) & 0x7u; + if (interrupt == 1 || interrupt == 2 || interrupt >= 4) { + return true; + } + } + + offset += len; + } + + return false; +} + +static void submit_dcb(uint32_t* dcb, uint32_t size_in_dwords) { + GraphicsDbgDumpDcb("d", size_in_dwords, dcb); + GraphicsRunSubmit(dcb, size_in_dwords, nullptr, 0, + !dcb_has_queued_interrupt(dcb, size_in_dwords)); + Gen5::track_pending_graphics_segment_after_submit(dcb, size_in_dwords); +} + +static std::vector collect_acb_wait_addresses(const uint32_t* acb, + uint32_t size_in_dwords) { + std::vector addresses; + + for (uint32_t offset = 0; offset < size_in_dwords;) { + auto cmd_id = acb[offset]; + auto len = KYTY_PM4_LEN(cmd_id); + if (len == 0 || len > size_in_dwords - offset) { + return addresses; + } + + auto op = (cmd_id >> 8u) & 0xffu; + if (op == Pm4::IT_NOP && KYTY_PM4_R(cmd_id) == Pm4::R_WAIT_MEM_32 && len >= 7) { + auto address = static_cast(acb[offset + 1]) | + (static_cast(acb[offset + 2]) << 32u); + if (address != 0) { + addresses.push_back(address); + } + } else if (op == Pm4::IT_NOP && KYTY_PM4_R(cmd_id) == Pm4::R_WAIT_MEM_64 && len >= 9) { + auto address = static_cast(acb[offset + 1]) | + (static_cast(acb[offset + 2]) << 32u); + if (address != 0) { + addresses.push_back(address); + } + } + + offset += len; + } + + return addresses; +} + +static bool acb_waits_for_address(const std::vector& wait_addresses, + uint64_t release_address) { + for (auto address: wait_addresses) { + if (address == release_address) { + return true; + } + } + + return false; +} + +static void flush_pending_graphics_segment_before_acb(const uint32_t* acb, + uint32_t acb_size_in_dwords) { + uint32_t* dcb = nullptr; + uint32_t size_in_dwords = 0; + auto wait_addresses = collect_acb_wait_addresses(acb, acb_size_in_dwords); + + { + std::lock_guard lock(Gen5::g_pending_graphics_segment_mutex); + if (!wait_addresses.empty() && Gen5::g_pending_graphics_segment.start != nullptr) { + auto* scan = Gen5::g_pending_graphics_segment.start; + auto* matched_end = Gen5::g_pending_graphics_segment.start; + while (scan < Gen5::g_pending_graphics_segment.end && + Gen5::is_readable_command_packet_range(scan, 1)) { + auto cmd_id = *scan; + if (cmd_id == 0x80000000u) { + scan++; + continue; + } + if ((cmd_id & 0xC0000000u) != 0xC0000000u) { + break; + } + + auto len = KYTY_PM4_LEN(cmd_id); + if (len == 0 || + len > static_cast(Gen5::g_pending_graphics_segment.end - scan) || + !Gen5::is_readable_command_packet_range(scan, len)) { + break; + } + + if (((cmd_id >> 8u) & 0xffu) == Pm4::IT_NOP && + KYTY_PM4_R(cmd_id) == Pm4::R_RELEASE_MEM && len >= 7) { + auto release_addr = + static_cast(scan[3]) | (static_cast(scan[4]) << 32u); + if (acb_waits_for_address(wait_addresses, release_addr)) { + matched_end = scan + len; + } + } + + scan += len; + } + + if (matched_end > Gen5::g_pending_graphics_segment.start) { + Gen5::g_pending_graphics_segment.end = matched_end; + } + } + + if (Gen5::g_pending_graphics_segment.start != nullptr && + Gen5::g_pending_graphics_segment.end > Gen5::g_pending_graphics_segment.start) { + auto* scan = Gen5::g_pending_graphics_segment.start; + auto* valid_end = Gen5::g_pending_graphics_segment.start; + while (scan < Gen5::g_pending_graphics_segment.end && + Gen5::is_readable_command_packet_range(scan, 1)) { + auto cmd_id = *scan; + if (cmd_id == 0x80000000u) { + scan++; + valid_end = scan; + continue; + } + if ((cmd_id & 0xC0000000u) != 0xC0000000u) { + break; + } + + auto len = KYTY_PM4_LEN(cmd_id); + if (len == 0 || + len > static_cast(Gen5::g_pending_graphics_segment.end - scan) || + !Gen5::is_readable_command_packet_range(scan, len)) { + break; + } + + scan += len; + valid_end = scan; + } + + if (valid_end < Gen5::g_pending_graphics_segment.end) { + static std::atomic log_count {0}; + if (log_count.fetch_add(1) < 64) { + LOGF("\t trimming pending graphics segment: addr = 0x%016" PRIx64 + ", old_dw = 0x%08" PRIx32 ", new_dw = 0x%08" PRIx32 "\n", + reinterpret_cast(Gen5::g_pending_graphics_segment.start), + static_cast(Gen5::g_pending_graphics_segment.end - + Gen5::g_pending_graphics_segment.start), + static_cast(valid_end - Gen5::g_pending_graphics_segment.start)); + } + Gen5::g_pending_graphics_segment.end = valid_end; + } + } + + if (Gen5::g_pending_graphics_segment.start == nullptr || + Gen5::g_pending_graphics_segment.end <= Gen5::g_pending_graphics_segment.start) { + return; + } + + dcb = Gen5::g_pending_graphics_segment.start; + size_in_dwords = static_cast(Gen5::g_pending_graphics_segment.end - + Gen5::g_pending_graphics_segment.start); + } + + LOGF("\t flushing pending graphics segment before ACB: addr = 0x%016" PRIx64 + ", dw_num = 0x%08" PRIx32 "\n", + reinterpret_cast(dcb), size_in_dwords); + + submit_dcb(dcb, size_in_dwords); +} + +int KYTY_SYSV_ABI GraphicsDriverSubmitDcb(const Packet* packet) { + PRINT_NAME(); + + EXIT_NOT_IMPLEMENTED(packet == nullptr); + EXIT_NOT_IMPLEMENTED(packet->pad[0] != 0); + + LOGF("\t addr = 0x%016" PRIx64 "\n" + "\t dw_num = 0x%08" PRIx32 "\n", + reinterpret_cast(packet->addr), packet->dw_num); + + submit_dcb(packet->addr, packet->dw_num); + + return OK; +} + +int KYTY_SYSV_ABI GraphicsDriverSubmitMultiDcbs(uint32_t* const* dcb_gpu_addrs, + const uint32_t* dcb_sizes_in_dwords, + uint32_t count) { + PRINT_NAME(); + + LOGF("\t count = %" PRIu32 "\n", count); + + if (count == 0) { + return OK; + } + if (dcb_gpu_addrs == nullptr || dcb_sizes_in_dwords == nullptr) { + return LibKernel::KERNEL_ERROR_EINVAL; + } + + for (uint32_t i = 0; i < count; i++) { + auto* dcb = dcb_gpu_addrs[i]; + uint32_t size_in_dwords = dcb_sizes_in_dwords[i]; + + LOGF("\t dcb[%" PRIu32 "] = 0x%016" PRIx64 "\n" + "\t size[%" PRIu32 "] = 0x%08" PRIx32 "\n", + i, reinterpret_cast(dcb), i, size_in_dwords); + + if (dcb == nullptr) { + continue; + } + submit_dcb(dcb, size_in_dwords); + } + + return OK; +} + +int KYTY_SYSV_ABI GraphicsDriverSubmitCommandBuffer(uint32_t queue, uint32_t* dcb, + uint32_t size_in_dwords) { + PRINT_NAME(); + + LOGF("\t queue = 0x%08" PRIx32 "\n" + "\t dcb = 0x%016" PRIx64 "\n" + "\t size = 0x%08" PRIx32 "\n", + queue, reinterpret_cast(dcb), size_in_dwords); + + if (dcb == nullptr || size_in_dwords == 0) { + return OK; + } + + submit_dcb(dcb, size_in_dwords); + + return OK; +} + +int KYTY_SYSV_ABI GraphicsDriverSubmitMultiCommandBuffers(uint32_t queue, uint32_t* const* dcbs, + const uint32_t* sizes_in_dwords, + uint32_t count) { + PRINT_NAME(); + + LOGF("\t queue = 0x%08" PRIx32 "\n" + "\t count = %" PRIu32 "\n", + queue, count); + + if (count == 0) { + return OK; + } + if (dcbs == nullptr || sizes_in_dwords == nullptr) { + return LibKernel::KERNEL_ERROR_EINVAL; + } + + for (uint32_t i = 0; i < count; i++) { + GraphicsDriverSubmitCommandBuffer(queue, dcbs[i], sizes_in_dwords[i]); + } + + return OK; +} + +static void submit_acb(uint32_t queue, uint32_t* acb, uint32_t size_in_dwords) { + if (acb != nullptr && size_in_dwords >= 5 && Gen5::is_readable_command_packet_range(acb, 5)) { + auto descriptor_addr = + static_cast(acb[0]) | (static_cast(acb[1]) << 32u); + auto descriptor_size = acb[2]; + auto descriptor_flags = acb[3]; + auto descriptor_magic = acb[4]; + if (descriptor_addr != 0 && descriptor_size != 0 && descriptor_flags == 0 && + descriptor_magic == 0x5533ccaau && + Gen5::is_readable_command_packet_range(reinterpret_cast(descriptor_addr), + descriptor_size)) { + LOGF("\t descriptor addr = 0x%016" PRIx64 "\n" + "\t descriptor size = 0x%08" PRIx32 "\n" + "\t descriptor magic = 0x%08" PRIx32 "\n", + descriptor_addr, descriptor_size, descriptor_magic); + acb = reinterpret_cast(descriptor_addr); + size_in_dwords = descriptor_size; + } + } + + if (acb == nullptr || size_in_dwords == 0) { + return; + } + + for (uint32_t i = 0; i < std::min(size_in_dwords, 8); i++) { + LOGF("\t acb[%u] = 0x%08" PRIx32 "\n", i, acb[i]); + } + + flush_pending_graphics_segment_before_acb(acb, size_in_dwords); + + GraphicsDbgDumpDcb("a", size_in_dwords, acb); + + const bool trigger_interrupt_on_done = !dcb_has_queued_interrupt(acb, size_in_dwords); + GraphicsRunSubmitCompute(queue, acb, size_in_dwords, trigger_interrupt_on_done); +} + +int KYTY_SYSV_ABI GraphicsDriverSubmitAcb(uint32_t queue, const Packet* packet) { + PRINT_NAME(); + + LOGF("\t queue = 0x%08" PRIx32 "\n" + "\t packet = 0x%016" PRIx64 "\n", + queue, reinterpret_cast(packet)); + + if (packet == nullptr) { + return OK; + } + if (!Gen5::is_readable_command_packet_range(reinterpret_cast(packet), + sizeof(Packet) / sizeof(uint32_t))) { + return OK; + } + + LOGF("\t acb = 0x%016" PRIx64 "\n" + "\t size = 0x%08" PRIx32 "\n" + "\t flags = 0x%02" PRIx8 "\n", + reinterpret_cast(packet->addr), packet->dw_num, packet->pad[0]); + + submit_acb(queue, packet->addr, packet->dw_num); + + return OK; +} + +int KYTY_SYSV_ABI GraphicsDriverSubmitMultiAcbs(uint32_t queue, uint32_t* const* acbs, + const uint32_t* sizes_in_dwords, uint32_t count) { + PRINT_NAME(); + + LOGF("\t queue = 0x%08" PRIx32 "\n" + "\t count = %" PRIu32 "\n", + queue, count); + + if (count == 0) { + return OK; + } + if (acbs == nullptr || sizes_in_dwords == nullptr) { + return LibKernel::KERNEL_ERROR_EINVAL; + } + + for (uint32_t i = 0; i < count; i++) { + submit_acb(queue, acbs[i], sizes_in_dwords[i]); + } + + return OK; +} + +int KYTY_SYSV_ABI GraphicsDriverAddEqEvent(LibKernel::EventQueue::KernelEqueue eq, int id, + void* udata) { + PRINT_NAME(); + + if (eq == nullptr) { + return LibKernel::KERNEL_ERROR_EBADF; + } + + return GraphicsRenderAddEqEvent(eq, id, udata); +} + +int KYTY_SYSV_ABI GraphicsDriverDeleteEqEvent(LibKernel::EventQueue::KernelEqueue eq, int id) { + PRINT_NAME(); + + if (eq == nullptr) { + return LibKernel::KERNEL_ERROR_EBADF; + } + + return GraphicsRenderDeleteEqEvent(eq, id); +} + +int KYTY_SYSV_ABI GraphicsDriverGetEqEventType(const LibKernel::EventQueue::KernelEvent* ev) { + PRINT_NAME(); + + if (ev == nullptr) { + return 0; + } + + if (ev->filter == LibKernel::EventQueue::KERNEL_EVFILT_GRAPHICS) { + return static_cast(ev->ident); + } + + return static_cast(ev->data); +} + +uint32_t KYTY_SYSV_ABI GraphicsDriverGetEqContextId(const LibKernel::EventQueue::KernelEvent* ev) { + PRINT_NAME(); + + if (ev == nullptr) { + return 0; + } + + if (ev->filter == LibKernel::EventQueue::KERNEL_EVFILT_GRAPHICS) { + return static_cast(ev->data); + } + + return static_cast(ev->ident); +} + +int KYTY_SYSV_ABI GraphicsDriverSetTFRing(const volatile void* base, uint32_t size) { + PRINT_NAME(); + + g_tessellation_driver_state.tf_ring_base = reinterpret_cast(base); + g_tessellation_driver_state.tf_ring_size = size; + + LOGF("\t base = 0x%016" PRIx64 "\n" + "\t size = 0x%08" PRIx32 "\n", + g_tessellation_driver_state.tf_ring_base, g_tessellation_driver_state.tf_ring_size); + + return OK; +} + +int KYTY_SYSV_ABI GraphicsDriverSetHsOffchipParam(uint64_t value0, uint64_t value1, + uint64_t value2) { + PRINT_NAME(); + + g_tessellation_driver_state.hs_offchip_value0 = value0; + g_tessellation_driver_state.hs_offchip_value1 = value1; + g_tessellation_driver_state.hs_offchip_value2 = value2; + + LOGF("\t value0 = 0x%016" PRIx64 "\n" + "\t value1 = 0x%016" PRIx64 "\n" + "\t value2 = 0x%016" PRIx64 "\n", + value0, value1, value2); + + return OK; +} + +bool KYTY_SYSV_ABI GraphicsDriverIsCaptureInProgress() { + PRINT_NAME(); + + return false; +} + +int KYTY_SYSV_ABI GraphicsDriverUnknownU9ueyEhSkF4() { + PRINT_NAME(); + + return static_cast(0x8a6c9018u); +} + +} // namespace Gen5Driver + +} // namespace Libs::Graphics diff --git a/src/libs/agc.h b/src/libs/agc.h new file mode 100644 index 000000000..45cfe9f07 --- /dev/null +++ b/src/libs/agc.h @@ -0,0 +1,311 @@ +#ifndef EMULATOR_INCLUDE_EMULATOR_LIBS_AGC_H_ +#define EMULATOR_INCLUDE_EMULATOR_LIBS_AGC_H_ + +#include "common/abi.h" +#include "common/common.h" +#include "common/subsystems.h" +#include "kernel/eventQueue.h" + +namespace Libs::Graphics { + +struct Shader; +struct ShaderRegister; + +struct SizeAlign { + uint64_t m_size; + size_t m_align; +}; + +struct MemoryRange { + void* m_base; + uint64_t m_size; +}; + +KYTY_SUBSYSTEM_DEFINE(Graphics); + +void GraphicsDbgDumpDcb(const char* type, uint32_t num_dw, uint32_t* cmd_buffer); + +namespace Gen5 { + +struct CommandBuffer; +struct Label; + +int KYTY_SYSV_ABI GraphicsInit(uint32_t* state, uint32_t ver); +void* KYTY_SYSV_ABI GraphicsGetRegisterDefaults2(uint32_t ver); +void* KYTY_SYSV_ABI GraphicsGetRegisterDefaults2Internal(uint32_t ver); +int KYTY_SYSV_ABI GraphicsCreateShader(Shader** dst, void* header, const volatile void* code); +int KYTY_SYSV_ABI GraphicsUnknownGetFusedShaderSize(SizeAlign* dst, const Shader* front, + const Shader* back); +int KYTY_SYSV_ABI GraphicsUnknownFuseShaderHalves(Shader* fused_result, const Shader* front, + const Shader* back, void* scratch_mem); +int KYTY_SYSV_ABI GraphicsSetCxRegIndirectPatchSetAddress(uint32_t* cmd, + const volatile ShaderRegister* regs); +int KYTY_SYSV_ABI GraphicsSetShRegIndirectPatchSetAddress(uint32_t* cmd, + const volatile ShaderRegister* regs); +int KYTY_SYSV_ABI GraphicsSetUcRegIndirectPatchSetAddress(uint32_t* cmd, + const volatile ShaderRegister* regs); +int KYTY_SYSV_ABI GraphicsSetCxRegIndirectPatchSetNumRegisters(uint32_t* cmd, uint32_t num_regs); +int KYTY_SYSV_ABI GraphicsSetShRegIndirectPatchSetNumRegisters(uint32_t* cmd, uint32_t num_regs); +int KYTY_SYSV_ABI GraphicsSetUcRegIndirectPatchSetNumRegisters(uint32_t* cmd, uint32_t num_regs); +int KYTY_SYSV_ABI GraphicsSetCxRegIndirectPatchAddRegisters(uint32_t* cmd, uint32_t num_regs); +int KYTY_SYSV_ABI GraphicsSetShRegIndirectPatchAddRegisters(uint32_t* cmd, uint32_t num_regs); +int KYTY_SYSV_ABI GraphicsSetUcRegIndirectPatchAddRegisters(uint32_t* cmd, uint32_t num_regs); +int KYTY_SYSV_ABI GraphicsCreatePrimState(ShaderRegister* cx_regs, ShaderRegister* uc_regs, + const Shader* hs, const Shader* gs, uint32_t prim_type); +int KYTY_SYSV_ABI GraphicsUpdatePrimState(ShaderRegister* cx_regs, ShaderRegister* uc_regs, + uint32_t prim_type); +int KYTY_SYSV_ABI GraphicsCreateInterpolantMapping(ShaderRegister* regs, const Shader* gs, + const Shader* ps); +int KYTY_SYSV_ABI GraphicsGetDataPacketPayloadAddress(uint32_t** addr, uint32_t* cmd, int type); +int KYTY_SYSV_ABI GraphicsGetDataPacketPayloadRange(MemoryRange* range, uint32_t* cmd, int type); +int KYTY_SYSV_ABI GraphicsWriteDataPatchSetAddressOrOffset(uint32_t* cmd, + uint64_t address_or_offset); +int KYTY_SYSV_ABI GraphicsUnknownJumpPatchSetTarget(uint32_t* cmd, const volatile uint32_t* target, + uint32_t size_in_dwords); +int KYTY_SYSV_ABI GraphicsJumpPatchSetTarget(uint32_t* cmd, const volatile uint32_t* target, + uint32_t size_in_dwords); +int KYTY_SYSV_ABI GraphicsSuspendPoint(); +uint32_t* KYTY_SYSV_ABI GraphicsUnknownQj7QZpgr9Uw(CommandBuffer* buf, uint32_t mode, + uint32_t value); +uint64_t KYTY_SYSV_ABI GraphicsGetIsTrinityMode(); + +uint32_t KYTY_SYSV_ABI GraphicsDriverGetDefaultOwner(); +uint32_t KYTY_SYSV_ABI GraphicsDriverGetResourceRegistrationMaxNameLength(); +uint32_t KYTY_SYSV_ABI GraphicsDriverInitResourceRegistration(); +uint32_t KYTY_SYSV_ABI +GraphicsDriverQueryResourceRegistrationUserMemoryRequirements(uint64_t* size_in_bytes); +int KYTY_SYSV_ABI GraphicsDriverRegisterOwner(); +int KYTY_SYSV_ABI GraphicsDriverRegisterResource(); +int KYTY_SYSV_ABI GraphicsDriverUnregisterResource(); +int KYTY_SYSV_ABI GraphicsDriverRegisterWorkloadStream(uint32_t stream_id, const void* stream); + +uint32_t* KYTY_SYSV_ABI GraphicsCbNop(CommandBuffer* buf, uint32_t size_in_dwords); +uint32_t KYTY_SYSV_ABI GraphicsCbNopGetSize(uint32_t size_in_dwords); +uint32_t* KYTY_SYSV_ABI GraphicsCbDispatch(CommandBuffer* buf, uint32_t thread_group_x, + uint32_t thread_group_y, uint32_t thread_group_z, + uint32_t modifier); +uint32_t KYTY_SYSV_ABI GraphicsCbDispatchGetSize(); +uint32_t* KYTY_SYSV_ABI GraphicsCbBranch(CommandBuffer* buf, uint8_t mode, uint8_t compare_function, + const volatile uint64_t* compare_addr, uint64_t mask, + uint64_t reference, uint8_t cache_policy1, + const volatile uint32_t* buffer1, uint32_t size_in_dwords1, + uint8_t cache_policy2, const volatile uint32_t* buffer2, + uint32_t size_in_dwords2); +uint32_t* KYTY_SYSV_ABI GraphicsCbSetShRegisterRangeDirect(CommandBuffer* buf, uint32_t offset, + const uint32_t* values, + uint32_t num_values); +uint32_t KYTY_SYSV_ABI GraphicsCbSetShRegisterRangeDirectGetSize(uint32_t num_values); +uint32_t* KYTY_SYSV_ABI GraphicsCbSetShRegistersDirect(CommandBuffer* buf, + const volatile ShaderRegister* regs, + uint32_t num_regs); +uint32_t* KYTY_SYSV_ABI GraphicsCbReleaseMem(CommandBuffer* buf, uint8_t action, uint16_t gcr_cntl, + uint8_t dst, uint8_t cache_policy, + const volatile Label* address, uint8_t data_sel, + uint64_t data, uint16_t gds_offset, uint16_t gds_size, + uint8_t interrupt, uint32_t interrupt_ctx_id); +uint32_t KYTY_SYSV_ABI GraphicsCbQueueEndOfPipeActionGetSize(); +int KYTY_SYSV_ABI GraphicsDebugRaiseException(uint32_t exception_id); +uint32_t* KYTY_SYSV_ABI GraphicsAcbResetQueue(CommandBuffer* buf, uint32_t op); +uint32_t* KYTY_SYSV_ABI GraphicsDcbResetQueue(CommandBuffer* buf, uint32_t op, uint32_t state); +uint32_t* KYTY_SYSV_ABI GraphicsDcbWaitUntilSafeForRendering(CommandBuffer* buf, + uint32_t video_out_handle, + uint32_t display_buffer_index); +uint32_t* KYTY_SYSV_ABI GraphicsDcbSetWorkloadsActive(CommandBuffer* buf, uint32_t stream_id, + const uint32_t* workload_ids, + uint32_t workload_count); +uint32_t* KYTY_SYSV_ABI GraphicsDcbSetWorkloadComplete(CommandBuffer* buf, uint32_t stream_id, + uint32_t workload_id); +uint32_t* KYTY_SYSV_ABI GraphicsDcbSetShRegisterDirect(CommandBuffer* buf, ShaderRegister reg); +uint32_t* KYTY_SYSV_ABI GraphicsDcbSetCxRegisterDirect(CommandBuffer* buf, ShaderRegister reg); +uint32_t KYTY_SYSV_ABI GraphicsDcbSetCxRegisterDirectGetSize(); +uint32_t* KYTY_SYSV_ABI GraphicsDcbSetUcRegisterDirect(CommandBuffer* buf, ShaderRegister reg); +uint32_t* KYTY_SYSV_ABI GraphicsDcbSetCxRegistersIndirect(CommandBuffer* buf, + const volatile ShaderRegister* regs, + uint32_t num_regs); +uint32_t* KYTY_SYSV_ABI GraphicsDcbSetShRegistersIndirect(CommandBuffer* buf, + const volatile ShaderRegister* regs, + uint32_t num_regs); +uint32_t* KYTY_SYSV_ABI GraphicsDcbSetUcRegistersIndirect(CommandBuffer* buf, + const volatile ShaderRegister* regs, + uint32_t num_regs); +uint32_t* KYTY_SYSV_ABI GraphicsDcbSetIndexSize(CommandBuffer* buf, uint8_t index_size, + uint8_t cache_policy); +uint32_t* KYTY_SYSV_ABI GraphicsDcbSetIndexBuffer(CommandBuffer* buf, uint64_t index_addr); +uint32_t* KYTY_SYSV_ABI GraphicsDcbSetIndexCount(CommandBuffer* buf, uint32_t index_count); +uint32_t* KYTY_SYSV_ABI GraphicsDcbSetNumInstances(CommandBuffer* buf, uint32_t num_instances); +uint32_t KYTY_SYSV_ABI GraphicsDcbSetNumInstancesGetSize(); +uint32_t* KYTY_SYSV_ABI GraphicsDcbDrawIndex(CommandBuffer* buf, uint32_t index_count, + const volatile void* index_addr, uint64_t modifier); +uint32_t KYTY_SYSV_ABI GraphicsDcbDrawIndexGetSize(); + +uint32_t* KYTY_SYSV_ABI GraphicsDcbDrawIndexMultiInstanced(CommandBuffer* buf, uint32_t index_count, + const volatile void* index_addr, + const volatile void* object_ids, + uint32_t instance_count, + uint64_t modifier); +uint32_t KYTY_SYSV_ABI GraphicsDcbDrawIndexMultiInstancedGetSize(); +int KYTY_SYSV_ABI GraphicsUnknownIkfdtRIqCE(uint32_t* cmd, uint64_t arg1, + const volatile uint32_t* target, + uint32_t size_in_dwords, uint64_t arg4, uint64_t arg5); +uint32_t* KYTY_SYSV_ABI GraphicsDcbDrawIndexAuto(CommandBuffer* buf, uint32_t index_count, + uint64_t modifier); +uint32_t KYTY_SYSV_ABI GraphicsDcbDrawIndexAutoGetSize(); +uint32_t* KYTY_SYSV_ABI GraphicsDcbDrawIndexOffset(CommandBuffer* buf, uint32_t index_offset, + uint32_t index_count, uint64_t modifier); +uint32_t KYTY_SYSV_ABI GraphicsDcbDrawIndexOffsetGetSize(); +uint32_t* KYTY_SYSV_ABI GraphicsDcbSetBaseIndirectArgs(CommandBuffer* buf, uint32_t shader_type, + const volatile void* indirect_base_addr); +uint32_t* KYTY_SYSV_ABI GraphicsDcbDrawIndirect(CommandBuffer* buf, uint32_t data_offset_in_bytes, + uint64_t modifier); +uint32_t KYTY_SYSV_ABI GraphicsDcbDrawIndirectGetSize(); +uint32_t* KYTY_SYSV_ABI GraphicsDcbDrawIndexIndirect(CommandBuffer* buf, + uint32_t data_offset_in_bytes, + uint64_t modifier); +uint32_t* KYTY_SYSV_ABI GraphicsDcbDrawIndexIndirectMulti( + CommandBuffer* buf, uint32_t data_offset_in_bytes, uint32_t count_indirect, + uint32_t max_count_or_count, const volatile void* count_addr, uint32_t stride_in_bytes, + uint64_t modifier); +uint32_t* KYTY_SYSV_ABI GraphicsDcbDispatchIndirect(CommandBuffer* buf, + uint32_t data_offset_in_bytes, uint32_t flags); +uint32_t KYTY_SYSV_ABI GraphicsDcbDispatchIndirectGetSize(); +uint32_t* KYTY_SYSV_ABI GraphicsDcbEventWrite(CommandBuffer* buf, uint8_t event_type, + const volatile void* address); +uint32_t* KYTY_SYSV_ABI GraphicsDcbAcquireMem(CommandBuffer* buf, uint8_t engine, uint32_t cb_db_op, + uint32_t gcr_cntl, const volatile void* base, + uint64_t size_bytes, uint32_t poll_cycles); +uint32_t KYTY_SYSV_ABI GraphicsDcbAcquireMemGetSize(); +uint32_t* KYTY_SYSV_ABI GraphicsAcbEventWrite(CommandBuffer* buf, uint8_t event_type, + const volatile void* address); +uint32_t* KYTY_SYSV_ABI GraphicsAcbAcquireMem(CommandBuffer* buf, uint32_t gcr_cntl, + const volatile void* base, uint64_t size_bytes, + uint32_t poll_cycles); +uint32_t KYTY_SYSV_ABI GraphicsAcbAcquireMemGetSize(); +uint32_t* KYTY_SYSV_ABI GraphicsAcbCondExec(CommandBuffer* buf, const volatile uint32_t* address, + uint32_t num_dwords); +uint32_t KYTY_SYSV_ABI GraphicsAcbCondExecGetSize(); +uint32_t* KYTY_SYSV_ABI GraphicsAcbWaitRegMem(CommandBuffer* buf, uint8_t size, + uint8_t compare_function, uint8_t cache_policy, + const volatile void* address, uint64_t reference, + uint64_t mask, uint32_t poll_cycles); +uint32_t* KYTY_SYSV_ABI GraphicsAcbDmaData(CommandBuffer* buf, uint8_t engine, uint8_t dst, + uint8_t dst_cache_policy, uint64_t dst_address_or_offset, + uint8_t src, uint8_t src_cache_policy, + uint64_t src_address_or_offset_or_immediate, + uint32_t num_bytes, uint8_t wait_for_previous, + uint8_t write_confirm, uint8_t block_engine); +uint32_t* KYTY_SYSV_ABI GraphicsAcbCopyData(CommandBuffer* buf, uint8_t dst, + uint8_t dst_cache_policy, uint64_t dst_address, + uint8_t src, uint8_t src_cache_policy, + uint64_t src_address_or_immediate, uint8_t item_size, + uint8_t write_confirm); +uint32_t* KYTY_SYSV_ABI GraphicsAcbDispatchIndirect(CommandBuffer* buf, + const volatile void* indirect_args, + uint32_t modifier); +uint32_t* KYTY_SYSV_ABI GraphicsAcbWriteData(CommandBuffer* buf, uint8_t dst, uint8_t cache_policy, + uint64_t address_or_offset, const void* data, + uint32_t num_dwords, uint8_t increment, + uint8_t write_confirm); +uint32_t* KYTY_SYSV_ABI GraphicsAcbSetMarker(CommandBuffer* buf, const char* str, uint32_t color); +uint32_t* KYTY_SYSV_ABI GraphicsAcbPushMarker(CommandBuffer* buf, const char* str, uint32_t color); +uint32_t* KYTY_SYSV_ABI GraphicsAcbPopMarker(CommandBuffer* buf); +uint32_t* KYTY_SYSV_ABI GraphicsDcbStallCommandBufferParser(CommandBuffer* buf); +uint32_t* KYTY_SYSV_ABI GraphicsDcbCopyData(CommandBuffer* buf, uint8_t dst, + uint8_t dst_cache_policy, uint64_t dst_address, + uint8_t src, uint8_t src_cache_policy, + uint64_t src_address_or_immediate, uint8_t item_size, + uint8_t write_confirm); +uint32_t* KYTY_SYSV_ABI GraphicsDcbDmaData(CommandBuffer* buf, uint8_t engine, uint8_t dst, + uint8_t dst_cache_policy, uint64_t dst_address_or_offset, + uint8_t src, uint8_t src_cache_policy, + uint64_t src_address_or_offset_or_immediate, + uint32_t num_bytes, uint8_t wait_for_previous, + uint8_t write_confirm, uint8_t block_engine); +uint32_t* KYTY_SYSV_ABI GraphicsDcbJump(CommandBuffer* buf, uint8_t mode, uint8_t cache_policy, + const uint32_t* target, uint32_t size_in_dwords); +uint32_t KYTY_SYSV_ABI GraphicsDcbJumpGetSize(); +uint32_t KYTY_SYSV_ABI GraphicsDcbRewindGetSize(); +uint32_t* KYTY_SYSV_ABI GraphicsDcbRewind(CommandBuffer* buf, uint32_t initial_state); +uint32_t* KYTY_SYSV_ABI GraphicsDcbCondExec(CommandBuffer* buf, const volatile uint32_t* address, + uint32_t num_dwords); +uint32_t KYTY_SYSV_ABI GraphicsDcbCondExecGetSize(); +uint32_t* KYTY_SYSV_ABI GraphicsDcbSetPredication(CommandBuffer* buf, uint8_t condition, uint8_t op, + uint8_t wait_op, const volatile void* address, + uint32_t count_in_dwords); +uint32_t* KYTY_SYSV_ABI GraphicsUnknownKRzWekV120(CommandBuffer* buf, uint32_t arg1, uint32_t arg2, + uint32_t arg3); +int KYTY_SYSV_ABI GraphicsDmaDataPatchSetDstAddressOrOffset(uint32_t* cmd, + uint64_t dst_address_or_offset); +int KYTY_SYSV_ABI GraphicsDmaDataPatchSetSrcAddressOrOffsetOrImmediate( + uint32_t* cmd, uint64_t src_address_or_offset_or_immediate); +uint32_t KYTY_SYSV_ABI GraphicsGetPacketSize(uint32_t* packet); +int KYTY_SYSV_ABI GraphicsSetPacketPredication(uint32_t* packet, uint32_t predication); +int KYTY_SYSV_ABI GraphicsSetRangePredication(uint32_t* start, const volatile uint32_t* end, + uint32_t predication); +int KYTY_SYSV_ABI GraphicsCondExecPatchSetEnd(uint32_t* cmd, const volatile uint32_t* buffer); +int KYTY_SYSV_ABI GraphicsCondExecPatchSetCommandAddress(uint32_t* cmd, + const volatile uint32_t* command); +uint32_t* KYTY_SYSV_ABI GraphicsDcbWriteData(CommandBuffer* buf, uint8_t dst, uint8_t cache_policy, + uint64_t address_or_offset, const void* data, + uint32_t num_dwords, uint8_t increment, + uint8_t write_confirm); +uint32_t KYTY_SYSV_ABI GraphicsDcbWriteDataGetSize(uint32_t num_dwords); +uint32_t* KYTY_SYSV_ABI GraphicsDcbGetLodStats(CommandBuffer* buf, uint8_t cache_policy, + const volatile void* buffer, + uint32_t buffer_size_in_bytes, uint32_t reset_count, + uint8_t force_reset, uint8_t report_and_reset, + uint32_t reporting_interval_in_100k_clocks); +uint32_t* KYTY_SYSV_ABI GraphicsDcbWaitRegMem(CommandBuffer* buf, uint8_t size, + uint8_t compare_function, uint8_t op, + uint8_t cache_policy, const volatile void* address, + uint64_t reference, uint64_t mask, + uint32_t poll_cycles); +uint32_t KYTY_SYSV_ABI GraphicsDcbWaitOnAddressGetSize(uint32_t size); +uint32_t* KYTY_SYSV_ABI GraphicsDcbSetMarker(CommandBuffer* buf, const char* str, uint32_t color); +uint32_t* KYTY_SYSV_ABI GraphicsDcbPushMarker(CommandBuffer* buf, const char* str, uint32_t color); +uint32_t* KYTY_SYSV_ABI GraphicsDcbPopMarker(CommandBuffer* buf); + +int KYTY_SYSV_ABI GraphicsWaitRegMemPatchAddress(uint32_t* cmd, const volatile void* address); +int KYTY_SYSV_ABI GraphicsWaitRegMemPatchReference(uint32_t* cmd, uint64_t reference); +int KYTY_SYSV_ABI GraphicsQueueEndOfPipeActionPatchAddress(uint32_t* cmd, + const volatile Label* address); +int KYTY_SYSV_ABI GraphicsQueueEndOfPipeActionPatchData(uint32_t* cmd, uint32_t context_id, + uint32_t data_sel, uint64_t data); + +uint32_t* KYTY_SYSV_ABI GraphicsDcbSetFlip(CommandBuffer* buf, uint32_t video_out_handle, + int32_t display_buffer_index, uint32_t flip_mode, + int64_t flip_arg); + +} // namespace Gen5 + +namespace Gen5Driver { + +struct Packet; + +int KYTY_SYSV_ABI GraphicsDriverSubmitDcb(const Packet* packet); +int KYTY_SYSV_ABI GraphicsDriverSubmitMultiDcbs(uint32_t* const* dcb_gpu_addrs, + const uint32_t* dcb_sizes_in_dwords, + uint32_t count); +int KYTY_SYSV_ABI GraphicsDriverSubmitCommandBuffer(uint32_t queue, uint32_t* dcb, + uint32_t size_in_dwords); +int KYTY_SYSV_ABI GraphicsDriverSubmitMultiCommandBuffers(uint32_t queue, uint32_t* const* dcbs, + const uint32_t* sizes_in_dwords, + uint32_t count); +int KYTY_SYSV_ABI GraphicsDriverSubmitAcb(uint32_t queue, const Packet* packet); +int KYTY_SYSV_ABI GraphicsDriverSubmitMultiAcbs(uint32_t queue, uint32_t* const* acbs, + const uint32_t* sizes_in_dwords, uint32_t count); +int KYTY_SYSV_ABI GraphicsDriverAddEqEvent(LibKernel::EventQueue::KernelEqueue eq, int id, + void* udata); +int KYTY_SYSV_ABI GraphicsDriverDeleteEqEvent(LibKernel::EventQueue::KernelEqueue eq, int id); +int KYTY_SYSV_ABI GraphicsDriverGetEqEventType(const LibKernel::EventQueue::KernelEvent* ev); +uint32_t KYTY_SYSV_ABI GraphicsDriverGetEqContextId(const LibKernel::EventQueue::KernelEvent* ev); +int KYTY_SYSV_ABI GraphicsDriverSetTFRing(const volatile void* base, uint32_t size); +int KYTY_SYSV_ABI GraphicsDriverSetHsOffchipParam(uint64_t value0, uint64_t value1, + uint64_t value2); +bool KYTY_SYSV_ABI GraphicsDriverIsCaptureInProgress(); +int KYTY_SYSV_ABI GraphicsDriverUnknownU9ueyEhSkF4(); + +} // namespace Gen5Driver + +} // namespace Libs::Graphics + +#endif /* EMULATOR_INCLUDE_EMULATOR_LIBS_AGC_H_ */ diff --git a/src/libs/agcRegisterDefaults.inc b/src/libs/agcRegisterDefaults.inc new file mode 100644 index 000000000..709f73af8 --- /dev/null +++ b/src/libs/agcRegisterDefaults.inc @@ -0,0 +1,2787 @@ +// No-feature, non-0x840f00 GraphicsGetRegisterDefaults2 dispatch, versions 0..12. +// Type entries keep the pointer index in bits 0..9; higher bits are AGC flags. + +// g_agc_internal_reg_defaults_v0: regs=(1, 12, 0, 4), ptrs=(1, 12, 0, 2), types=15. +static ShaderRegister g_agc_internal_reg_defaults_v0_tbl0_regs[] = { + {0x01b5, 0x00000001}, +}; + +static const uint16_t g_agc_internal_reg_defaults_v0_tbl0_ptrs[] = { + 0x0000, +}; + +static ShaderRegister g_agc_internal_reg_defaults_v0_tbl1_regs[] = { + {0x0216, 0xffffffff}, {0x0217, 0xffffffff}, {0x0219, 0xffffffff}, {0x021a, 0xffffffff}, + {0x027d, 0x000004ff}, {0x022a, 0x00000000}, {0x0204, 0x00000000}, {0x0205, 0x00000000}, + {0x0206, 0x00000000}, {0x0080, 0x00000000}, {0x0100, 0x00000000}, {0x0006, 0x00000000}, +}; + +static const uint16_t g_agc_internal_reg_defaults_v0_tbl1_ptrs[] = { + 0x0000, 0x0001, 0x0002, 0x0003, 0x0004, 0x0005, 0x0006, 0x0007, 0x0008, 0x0009, 0x000a, 0x000b, +}; + +static ShaderRegister g_agc_internal_reg_defaults_v0_tbl3_regs[] = { + {0x026c, 0x00000000}, {0x026e, 0x00000000}, {0x0278, 0x00000000}, {0x0262, 0x00000000}, +}; + +static const uint16_t g_agc_internal_reg_defaults_v0_tbl3_ptrs[] = { + 0x0000, 0x0001, +}; + +static uint32_t g_agc_internal_reg_defaults_v0_types[] = { + 0xf58fea31, 0x00040400, 0x00000000, 0x6ac156ef, 0x00040401, 0x00000000, + 0x6ac15610, 0x00040405, 0x00000000, 0x6ac15009, 0x00040409, 0x00000000, + 0x6ac153ba, 0x0004040d, 0x00000000, 0xbe7dcd73, 0x00040411, 0x00000000, + 0x0c4b1438, 0x00040415, 0x00000000, 0xdb00d71a, 0x00040419, 0x00000000, + 0xdb00d249, 0x0004041d, 0x00000000, 0xdb00ec60, 0x00040421, 0x00000000, + 0x0c4d6fe4, 0x00040425, 0x00000000, 0x0c4a80ef, 0x00040429, 0x00000000, + 0x0dd283e7, 0x0004042d, 0x00000000, 0x31f34b9f, 0x00040403, 0x00000000, + 0x929fd95d, 0x00040c07, 0x00000000, +}; + +static CompactRegisterDefaults g_agc_internal_reg_defaults_v0 = { + g_agc_internal_reg_defaults_v0_tbl0_regs, 1, g_agc_internal_reg_defaults_v0_tbl0_ptrs, 1, + g_agc_internal_reg_defaults_v0_tbl1_regs, 12, g_agc_internal_reg_defaults_v0_tbl1_ptrs, 12, + nullptr, 0, nullptr, 0, + g_agc_internal_reg_defaults_v0_tbl3_regs, 4, g_agc_internal_reg_defaults_v0_tbl3_ptrs, 2, + g_agc_internal_reg_defaults_v0_types, 15, +}; + +// g_agc_internal_reg_defaults_v10: regs=(9, 15, 1, 6), ptrs=(9, 15, 1, 3), types=28. +static ShaderRegister g_agc_internal_reg_defaults_v10_tbl0_regs[] = { + {0x000e, 0x00000002}, {0x02af, 0x00040000}, {0x02d3, 0x00000000}, {0x0207, 0x00000000}, + {0x0314, 0x00000202}, {0x01b5, 0x00000001}, {0x01b1, 0x00000000}, {0x02e4, 0x00000000}, + {0x0291, 0x00000000}, +}; + +static const uint16_t g_agc_internal_reg_defaults_v10_tbl0_ptrs[] = { + 0x0000, 0x0001, 0x0002, 0x0003, 0x0004, 0x0005, 0x0006, 0x0007, 0x0008, +}; + +static ShaderRegister g_agc_internal_reg_defaults_v10_tbl1_regs[] = { + {0x0216, 0xffffffff}, {0x0217, 0xffffffff}, {0x0219, 0xffffffff}, {0x021a, 0xffffffff}, + {0x027d, 0x000004ff}, {0x022a, 0x00000000}, {0x0204, 0x00000000}, {0x0205, 0x00000000}, + {0x0206, 0x00000000}, {0x0080, 0x00000000}, {0x0100, 0x00000000}, {0x0006, 0x00000000}, + {0x0081, 0x00000000}, {0x0101, 0x00000000}, {0x0001, 0x00000003}, +}; + +static const uint16_t g_agc_internal_reg_defaults_v10_tbl1_ptrs[] = { + 0x0000, 0x0001, 0x0002, 0x0003, 0x0004, 0x0005, 0x0006, 0x0007, 0x0008, 0x0009, 0x000a, 0x000b, + 0x000c, 0x000d, 0x000e, +}; + +static ShaderRegister g_agc_internal_reg_defaults_v10_tbl2_regs[] = { + {0x0260, 0x00000000}, +}; + +static const uint16_t g_agc_internal_reg_defaults_v10_tbl2_ptrs[] = { + 0x0000, +}; + +static ShaderRegister g_agc_internal_reg_defaults_v10_tbl3_regs[] = { + {0x026c, 0x00000000}, {0x0094, 0x00000000}, {0x0095, 0x00000000}, {0x026e, 0x00000000}, + {0x0278, 0x00000000}, {0x0262, 0x00000000}, +}; + +static const uint16_t g_agc_internal_reg_defaults_v10_tbl3_ptrs[] = { + 0x0000, 0x0001, 0x0003, +}; + +static uint32_t g_agc_internal_reg_defaults_v10_types[] = { + 0x8fb4edb5, 0x00040400, 0x00000000, 0xb994ad29, 0x00040404, 0x00000000, + 0x700e23c3, 0x00040408, 0x00000000, 0x51a1d306, 0x0004040c, 0x00000000, + 0xd427322f, 0x00040410, 0x00000000, 0xf58fea31, 0x00040414, 0x00000000, + 0xa7fbef3f, 0x00040418, 0x00000000, 0xe7d3ab13, 0x0004041c, 0x00000000, + 0x1b1658df, 0x00040420, 0x00000000, 0x6ac156ef, 0x00040401, 0x00000000, + 0x6ac15610, 0x00040405, 0x00000000, 0x6ac15009, 0x00040409, 0x00000000, + 0x6ac153ba, 0x0004040d, 0x00000000, 0xbe7dcd73, 0x00040411, 0x00000000, + 0x0c4b1438, 0x00040415, 0x00000000, 0xdb00d71a, 0x00040419, 0x00000000, + 0xdb00d249, 0x0004041d, 0x00000000, 0xdb00ec60, 0x00040421, 0x00000000, + 0x0c4d6fe4, 0x00040425, 0x00000000, 0x0c4a80ef, 0x00040429, 0x00000000, + 0x0dd283e7, 0x0004042d, 0x00000000, 0xe27f8fe6, 0x00040431, 0x00000000, + 0xc67efacf, 0x00040435, 0x00000000, 0xd9e6d9f7, 0x00040439, 0x00000000, + 0x60289246, 0x00040402, 0x00000000, 0x31f34b9f, 0x00040403, 0x00000000, + 0xac0f9e76, 0x00080407, 0x00000000, 0x929fd95d, 0x00040c0b, 0x00000000, +}; + +static CompactRegisterDefaults g_agc_internal_reg_defaults_v10 = { + g_agc_internal_reg_defaults_v10_tbl0_regs, 9, g_agc_internal_reg_defaults_v10_tbl0_ptrs, 9, + g_agc_internal_reg_defaults_v10_tbl1_regs, 15, g_agc_internal_reg_defaults_v10_tbl1_ptrs, 15, + g_agc_internal_reg_defaults_v10_tbl2_regs, 1, g_agc_internal_reg_defaults_v10_tbl2_ptrs, 1, + g_agc_internal_reg_defaults_v10_tbl3_regs, 6, g_agc_internal_reg_defaults_v10_tbl3_ptrs, 3, + g_agc_internal_reg_defaults_v10_types, 28, +}; + +// g_agc_internal_reg_defaults_v11: regs=(8, 12, 1, 6), ptrs=(8, 12, 1, 3), types=24. +static ShaderRegister g_agc_internal_reg_defaults_v11_tbl0_regs[] = { + {0x000e, 0x00000002}, {0x02d3, 0x00000000}, {0x0207, 0x00000000}, {0x0314, 0x00000202}, + {0x01b5, 0x00000001}, {0x01b1, 0x00000000}, {0x02e4, 0x00000000}, {0x0291, 0x00000000}, +}; + +static const uint16_t g_agc_internal_reg_defaults_v11_tbl0_ptrs[] = { + 0x0000, 0x0001, 0x0002, 0x0003, 0x0004, 0x0005, 0x0006, 0x0007, +}; + +static ShaderRegister g_agc_internal_reg_defaults_v11_tbl1_regs[] = { + {0x0216, 0xffffffff}, {0x0217, 0xffffffff}, {0x0219, 0xffffffff}, {0x021a, 0xffffffff}, + {0x027d, 0x000004ff}, {0x0204, 0x00000000}, {0x0205, 0x00000000}, {0x0206, 0x00000000}, + {0x0100, 0x00ffffff}, {0x0081, 0x00000000}, {0x0101, 0x00000000}, {0x0001, 0x00000003}, +}; + +static const uint16_t g_agc_internal_reg_defaults_v11_tbl1_ptrs[] = { + 0x0000, 0x0001, 0x0002, 0x0003, 0x0004, 0x0005, 0x0006, 0x0007, 0x0008, 0x0009, 0x000a, 0x000b, +}; + +static ShaderRegister g_agc_internal_reg_defaults_v11_tbl2_regs[] = { + {0x0260, 0x00000000}, +}; + +static const uint16_t g_agc_internal_reg_defaults_v11_tbl2_ptrs[] = { + 0x0000, +}; + +static ShaderRegister g_agc_internal_reg_defaults_v11_tbl3_regs[] = { + {0x026c, 0x00000000}, {0x0094, 0x00000000}, {0x0095, 0x00000000}, {0x026e, 0x00000000}, + {0x0278, 0x00000000}, {0x0262, 0x00000000}, +}; + +static const uint16_t g_agc_internal_reg_defaults_v11_tbl3_ptrs[] = { + 0x0000, 0x0001, 0x0003, +}; + +static uint32_t g_agc_internal_reg_defaults_v11_types[] = { + 0x8fb4edb5, 0x00040400, 0x00000000, 0x700e23c3, 0x00040404, 0x00000000, + 0x51a1d306, 0x00040408, 0x00000000, 0xd427322f, 0x0004040c, 0x00000000, + 0xf58fea31, 0x00040410, 0x00000000, 0xa7fbef3f, 0x00040414, 0x00000000, + 0xe7d3ab13, 0x00040418, 0x00000000, 0x1b1658df, 0x0004041c, 0x00000000, + 0x6ac156ef, 0x00040401, 0x00000000, 0x6ac15610, 0x00040405, 0x00000000, + 0x6ac15009, 0x00040409, 0x00000000, 0x6ac153ba, 0x0004040d, 0x00000000, + 0xbe7dcd73, 0x00040411, 0x00000000, 0xdb00d71a, 0x00040415, 0x00000000, + 0xdb00d249, 0x00040419, 0x00000000, 0xdb00ec60, 0x0004041d, 0x00000000, + 0x0c4a80ef, 0x00040421, 0x00000000, 0xe27f8fe6, 0x00040425, 0x00000000, + 0xc67efacf, 0x00040429, 0x00000000, 0xd9e6d9f7, 0x0004042d, 0x00000000, + 0x60289246, 0x00040402, 0x00000000, 0x31f34b9f, 0x00040403, 0x00000000, + 0xac0f9e76, 0x00080407, 0x00000000, 0x929fd95d, 0x00040c0b, 0x00000000, +}; + +static CompactRegisterDefaults g_agc_internal_reg_defaults_v11 = { + g_agc_internal_reg_defaults_v11_tbl0_regs, 8, g_agc_internal_reg_defaults_v11_tbl0_ptrs, 8, + g_agc_internal_reg_defaults_v11_tbl1_regs, 12, g_agc_internal_reg_defaults_v11_tbl1_ptrs, 12, + g_agc_internal_reg_defaults_v11_tbl2_regs, 1, g_agc_internal_reg_defaults_v11_tbl2_ptrs, 1, + g_agc_internal_reg_defaults_v11_tbl3_regs, 6, g_agc_internal_reg_defaults_v11_tbl3_ptrs, 3, + g_agc_internal_reg_defaults_v11_types, 24, +}; + +// g_agc_internal_reg_defaults_v4: regs=(2, 15, 0, 4), ptrs=(2, 15, 0, 2), types=19. +static ShaderRegister g_agc_internal_reg_defaults_v4_tbl0_regs[] = { + {0x02af, 0x00040000}, {0x01b5, 0x00000001}, +}; + +static const uint16_t g_agc_internal_reg_defaults_v4_tbl0_ptrs[] = { + 0x0000, 0x0001, +}; + +static ShaderRegister g_agc_internal_reg_defaults_v4_tbl1_regs[] = { + {0x0216, 0xffffffff}, {0x0217, 0xffffffff}, {0x0219, 0xffffffff}, {0x021a, 0xffffffff}, + {0x027d, 0x000004ff}, {0x022a, 0x00000000}, {0x0204, 0x00000000}, {0x0205, 0x00000000}, + {0x0206, 0x00000000}, {0x0080, 0x00000000}, {0x0100, 0x00000000}, {0x0006, 0x00000000}, + {0x0081, 0x00000000}, {0x0101, 0x00000000}, {0x0001, 0x00000003}, +}; + +static const uint16_t g_agc_internal_reg_defaults_v4_tbl1_ptrs[] = { + 0x0000, 0x0001, 0x0002, 0x0003, 0x0004, 0x0005, 0x0006, 0x0007, 0x0008, 0x0009, 0x000a, 0x000b, + 0x000c, 0x000d, 0x000e, +}; + +static ShaderRegister g_agc_internal_reg_defaults_v4_tbl3_regs[] = { + {0x026c, 0x00000000}, {0x026e, 0x00000000}, {0x0278, 0x00000000}, {0x0262, 0x00000000}, +}; + +static const uint16_t g_agc_internal_reg_defaults_v4_tbl3_ptrs[] = { + 0x0000, 0x0001, +}; + +static uint32_t g_agc_internal_reg_defaults_v4_types[] = { + 0xb994ad29, 0x00040400, 0x00000000, 0xf58fea31, 0x00040404, 0x00000000, + 0x6ac156ef, 0x00040401, 0x00000000, 0x6ac15610, 0x00040405, 0x00000000, + 0x6ac15009, 0x00040409, 0x00000000, 0x6ac153ba, 0x0004040d, 0x00000000, + 0xbe7dcd73, 0x00040411, 0x00000000, 0x0c4b1438, 0x00040415, 0x00000000, + 0xdb00d71a, 0x00040419, 0x00000000, 0xdb00d249, 0x0004041d, 0x00000000, + 0xdb00ec60, 0x00040421, 0x00000000, 0x0c4d6fe4, 0x00040425, 0x00000000, + 0x0c4a80ef, 0x00040429, 0x00000000, 0x0dd283e7, 0x0004042d, 0x00000000, + 0xc620e68c, 0x00040431, 0x00000000, 0xc67efacf, 0x00040435, 0x00000000, + 0xd9e6d9f7, 0x00040439, 0x00000000, 0x31f34b9f, 0x00040403, 0x00000000, + 0x929fd95d, 0x00040c07, 0x00000000, +}; + +static CompactRegisterDefaults g_agc_internal_reg_defaults_v4 = { + g_agc_internal_reg_defaults_v4_tbl0_regs, 2, g_agc_internal_reg_defaults_v4_tbl0_ptrs, 2, + g_agc_internal_reg_defaults_v4_tbl1_regs, 15, g_agc_internal_reg_defaults_v4_tbl1_ptrs, 15, + nullptr, 0, nullptr, 0, + g_agc_internal_reg_defaults_v4_tbl3_regs, 4, g_agc_internal_reg_defaults_v4_tbl3_ptrs, 2, + g_agc_internal_reg_defaults_v4_types, 19, +}; + +// g_agc_internal_reg_defaults_v5: regs=(3, 15, 0, 6), ptrs=(3, 15, 0, 3), types=21. +static ShaderRegister g_agc_internal_reg_defaults_v5_tbl0_regs[] = { + {0x000e, 0x00000002}, {0x02af, 0x00040000}, {0x01b5, 0x00000001}, +}; + +static const uint16_t g_agc_internal_reg_defaults_v5_tbl0_ptrs[] = { + 0x0000, 0x0001, 0x0002, +}; + +static ShaderRegister g_agc_internal_reg_defaults_v5_tbl1_regs[] = { + {0x0216, 0xffffffff}, {0x0217, 0xffffffff}, {0x0219, 0xffffffff}, {0x021a, 0xffffffff}, + {0x027d, 0x000004ff}, {0x022a, 0x00000000}, {0x0204, 0x00000000}, {0x0205, 0x00000000}, + {0x0206, 0x00000000}, {0x0080, 0x00000000}, {0x0100, 0x00000000}, {0x0006, 0x00000000}, + {0x0081, 0x00000000}, {0x0101, 0x00000000}, {0x0001, 0x00000003}, +}; + +static const uint16_t g_agc_internal_reg_defaults_v5_tbl1_ptrs[] = { + 0x0000, 0x0001, 0x0002, 0x0003, 0x0004, 0x0005, 0x0006, 0x0007, 0x0008, 0x0009, 0x000a, 0x000b, + 0x000c, 0x000d, 0x000e, +}; + +static ShaderRegister g_agc_internal_reg_defaults_v5_tbl3_regs[] = { + {0x026c, 0x00000000}, {0x0094, 0x00000000}, {0x0095, 0x00000000}, {0x026e, 0x00000000}, + {0x0278, 0x00000000}, {0x0262, 0x00000000}, +}; + +static const uint16_t g_agc_internal_reg_defaults_v5_tbl3_ptrs[] = { + 0x0000, 0x0001, 0x0003, +}; + +static uint32_t g_agc_internal_reg_defaults_v5_types[] = { + 0x8fb4edb5, 0x00040400, 0x00000000, 0xb994ad29, 0x00040404, 0x00000000, + 0xf58fea31, 0x00040408, 0x00000000, 0x6ac156ef, 0x00040401, 0x00000000, + 0x6ac15610, 0x00040405, 0x00000000, 0x6ac15009, 0x00040409, 0x00000000, + 0x6ac153ba, 0x0004040d, 0x00000000, 0xbe7dcd73, 0x00040411, 0x00000000, + 0x0c4b1438, 0x00040415, 0x00000000, 0xdb00d71a, 0x00040419, 0x00000000, + 0xdb00d249, 0x0004041d, 0x00000000, 0xdb00ec60, 0x00040421, 0x00000000, + 0x0c4d6fe4, 0x00040425, 0x00000000, 0x0c4a80ef, 0x00040429, 0x00000000, + 0x0dd283e7, 0x0004042d, 0x00000000, 0xc620e68c, 0x00040431, 0x00000000, + 0xc67efacf, 0x00040435, 0x00000000, 0xd9e6d9f7, 0x00040439, 0x00000000, + 0x31f34b9f, 0x00040403, 0x00000000, 0xac0f9e76, 0x00080407, 0x00000000, + 0x929fd95d, 0x00040c0b, 0x00000000, +}; + +static CompactRegisterDefaults g_agc_internal_reg_defaults_v5 = { + g_agc_internal_reg_defaults_v5_tbl0_regs, 3, g_agc_internal_reg_defaults_v5_tbl0_ptrs, 3, + g_agc_internal_reg_defaults_v5_tbl1_regs, 15, g_agc_internal_reg_defaults_v5_tbl1_ptrs, 15, + nullptr, 0, nullptr, 0, + g_agc_internal_reg_defaults_v5_tbl3_regs, 6, g_agc_internal_reg_defaults_v5_tbl3_ptrs, 3, + g_agc_internal_reg_defaults_v5_types, 21, +}; + +// g_agc_internal_reg_defaults_v7: regs=(4, 15, 0, 6), ptrs=(4, 15, 0, 3), types=22. +static ShaderRegister g_agc_internal_reg_defaults_v7_tbl0_regs[] = { + {0x000e, 0x00000002}, {0x02af, 0x00040000}, {0x0314, 0x00000202}, {0x01b5, 0x00000001}, +}; + +static const uint16_t g_agc_internal_reg_defaults_v7_tbl0_ptrs[] = { + 0x0000, 0x0001, 0x0002, 0x0003, +}; + +static ShaderRegister g_agc_internal_reg_defaults_v7_tbl1_regs[] = { + {0x0216, 0xffffffff}, {0x0217, 0xffffffff}, {0x0219, 0xffffffff}, {0x021a, 0xffffffff}, + {0x027d, 0x000004ff}, {0x022a, 0x00000000}, {0x0204, 0x00000000}, {0x0205, 0x00000000}, + {0x0206, 0x00000000}, {0x0080, 0x00000000}, {0x0100, 0x00000000}, {0x0006, 0x00000000}, + {0x0081, 0x00000000}, {0x0101, 0x00000000}, {0x0001, 0x00000003}, +}; + +static const uint16_t g_agc_internal_reg_defaults_v7_tbl1_ptrs[] = { + 0x0000, 0x0001, 0x0002, 0x0003, 0x0004, 0x0005, 0x0006, 0x0007, 0x0008, 0x0009, 0x000a, 0x000b, + 0x000c, 0x000d, 0x000e, +}; + +static ShaderRegister g_agc_internal_reg_defaults_v7_tbl3_regs[] = { + {0x026c, 0x00000000}, {0x0094, 0x00000000}, {0x0095, 0x00000000}, {0x026e, 0x00000000}, + {0x0278, 0x00000000}, {0x0262, 0x00000000}, +}; + +static const uint16_t g_agc_internal_reg_defaults_v7_tbl3_ptrs[] = { + 0x0000, 0x0001, 0x0003, +}; + +static uint32_t g_agc_internal_reg_defaults_v7_types[] = { + 0x8fb4edb5, 0x00040400, 0x00000000, 0xb994ad29, 0x00040404, 0x00000000, + 0xd427322f, 0x00040408, 0x00000000, 0xf58fea31, 0x0004040c, 0x00000000, + 0x6ac156ef, 0x00040401, 0x00000000, 0x6ac15610, 0x00040405, 0x00000000, + 0x6ac15009, 0x00040409, 0x00000000, 0x6ac153ba, 0x0004040d, 0x00000000, + 0xbe7dcd73, 0x00040411, 0x00000000, 0x0c4b1438, 0x00040415, 0x00000000, + 0xdb00d71a, 0x00040419, 0x00000000, 0xdb00d249, 0x0004041d, 0x00000000, + 0xdb00ec60, 0x00040421, 0x00000000, 0x0c4d6fe4, 0x00040425, 0x00000000, + 0x0c4a80ef, 0x00040429, 0x00000000, 0x0dd283e7, 0x0004042d, 0x00000000, + 0xc620e68c, 0x00040431, 0x00000000, 0xc67efacf, 0x00040435, 0x00000000, + 0xd9e6d9f7, 0x00040439, 0x00000000, 0x31f34b9f, 0x00040403, 0x00000000, + 0xac0f9e76, 0x00080407, 0x00000000, 0x929fd95d, 0x00040c0b, 0x00000000, +}; + +static CompactRegisterDefaults g_agc_internal_reg_defaults_v7 = { + g_agc_internal_reg_defaults_v7_tbl0_regs, 4, g_agc_internal_reg_defaults_v7_tbl0_ptrs, 4, + g_agc_internal_reg_defaults_v7_tbl1_regs, 15, g_agc_internal_reg_defaults_v7_tbl1_ptrs, 15, + nullptr, 0, nullptr, 0, + g_agc_internal_reg_defaults_v7_tbl3_regs, 6, g_agc_internal_reg_defaults_v7_tbl3_ptrs, 3, + g_agc_internal_reg_defaults_v7_types, 22, +}; + +// g_agc_internal_reg_defaults_v8: regs=(4, 15, 0, 6), ptrs=(4, 15, 0, 3), types=22. +static ShaderRegister g_agc_internal_reg_defaults_v8_tbl0_regs[] = { + {0x000e, 0x00000002}, {0x02af, 0x00040000}, {0x0314, 0x00000202}, {0x01b5, 0x00000001}, +}; + +static const uint16_t g_agc_internal_reg_defaults_v8_tbl0_ptrs[] = { + 0x0000, 0x0001, 0x0002, 0x0003, +}; + +static ShaderRegister g_agc_internal_reg_defaults_v8_tbl1_regs[] = { + {0x0216, 0xffffffff}, {0x0217, 0xffffffff}, {0x0219, 0xffffffff}, {0x021a, 0xffffffff}, + {0x027d, 0x000004ff}, {0x022a, 0x00000000}, {0x0204, 0x00000000}, {0x0205, 0x00000000}, + {0x0206, 0x00000000}, {0x0080, 0x00000000}, {0x0100, 0x00000000}, {0x0006, 0x00000000}, + {0x0081, 0x00000000}, {0x0101, 0x00000000}, {0x0001, 0x00000003}, +}; + +static const uint16_t g_agc_internal_reg_defaults_v8_tbl1_ptrs[] = { + 0x0000, 0x0001, 0x0002, 0x0003, 0x0004, 0x0005, 0x0006, 0x0007, 0x0008, 0x0009, 0x000a, 0x000b, + 0x000c, 0x000d, 0x000e, +}; + +static ShaderRegister g_agc_internal_reg_defaults_v8_tbl3_regs[] = { + {0x026c, 0x00000000}, {0x0094, 0x00000000}, {0x0095, 0x00000000}, {0x026e, 0x00000000}, + {0x0278, 0x00000000}, {0x0262, 0x00000000}, +}; + +static const uint16_t g_agc_internal_reg_defaults_v8_tbl3_ptrs[] = { + 0x0000, 0x0001, 0x0003, +}; + +static uint32_t g_agc_internal_reg_defaults_v8_types[] = { + 0x8fb4edb5, 0x00040400, 0x00000000, 0xb994ad29, 0x00040404, 0x00000000, + 0xd427322f, 0x00040408, 0x00000000, 0xf58fea31, 0x0004040c, 0x00000000, + 0x6ac156ef, 0x00040401, 0x00000000, 0x6ac15610, 0x00040405, 0x00000000, + 0x6ac15009, 0x00040409, 0x00000000, 0x6ac153ba, 0x0004040d, 0x00000000, + 0xbe7dcd73, 0x00040411, 0x00000000, 0x0c4b1438, 0x00040415, 0x00000000, + 0xdb00d71a, 0x00040419, 0x00000000, 0xdb00d249, 0x0004041d, 0x00000000, + 0xdb00ec60, 0x00040421, 0x00000000, 0x0c4d6fe4, 0x00040425, 0x00000000, + 0x0c4a80ef, 0x00040429, 0x00000000, 0x0dd283e7, 0x0004042d, 0x00000000, + 0xc620e68c, 0x00040431, 0x00000000, 0xc67efacf, 0x00040435, 0x00000000, + 0xd9e6d9f7, 0x00040439, 0x00000000, 0x31f34b9f, 0x00040403, 0x00000000, + 0xac0f9e76, 0x00080407, 0x00000000, 0x929fd95d, 0x00040c0b, 0x00000000, +}; + +static CompactRegisterDefaults g_agc_internal_reg_defaults_v8 = { + g_agc_internal_reg_defaults_v8_tbl0_regs, 4, g_agc_internal_reg_defaults_v8_tbl0_ptrs, 4, + g_agc_internal_reg_defaults_v8_tbl1_regs, 15, g_agc_internal_reg_defaults_v8_tbl1_ptrs, 15, + nullptr, 0, nullptr, 0, + g_agc_internal_reg_defaults_v8_tbl3_regs, 6, g_agc_internal_reg_defaults_v8_tbl3_ptrs, 3, + g_agc_internal_reg_defaults_v8_types, 22, +}; + +// g_agc_internal_reg_defaults_v9: regs=(4, 14, 1, 6), ptrs=(4, 14, 1, 3), types=22. +static ShaderRegister g_agc_internal_reg_defaults_v9_tbl0_regs[] = { + {0x000e, 0x00000002}, {0x02af, 0x00040000}, {0x0314, 0x00000202}, {0x01b5, 0x00000001}, +}; + +static const uint16_t g_agc_internal_reg_defaults_v9_tbl0_ptrs[] = { + 0x0000, 0x0001, 0x0002, 0x0003, +}; + +static ShaderRegister g_agc_internal_reg_defaults_v9_tbl1_regs[] = { + {0x0216, 0xffffffff}, {0x0217, 0xffffffff}, {0x0219, 0xffffffff}, {0x021a, 0xffffffff}, + {0x027d, 0x000004ff}, {0x022a, 0x00000000}, {0x0204, 0x00000000}, {0x0205, 0x00000000}, + {0x0206, 0x00000000}, {0x0080, 0x00000000}, {0x0100, 0x00000000}, {0x0006, 0x00000000}, + {0x0101, 0x00000000}, {0x0001, 0x00000003}, +}; + +static const uint16_t g_agc_internal_reg_defaults_v9_tbl1_ptrs[] = { + 0x0000, 0x0001, 0x0002, 0x0003, 0x0004, 0x0005, 0x0006, 0x0007, 0x0008, 0x0009, 0x000a, 0x000b, + 0x000c, 0x000d, +}; + +static ShaderRegister g_agc_internal_reg_defaults_v9_tbl2_regs[] = { + {0x0260, 0x00000000}, +}; + +static const uint16_t g_agc_internal_reg_defaults_v9_tbl2_ptrs[] = { + 0x0000, +}; + +static ShaderRegister g_agc_internal_reg_defaults_v9_tbl3_regs[] = { + {0x026c, 0x00000000}, {0x0094, 0x00000000}, {0x0095, 0x00000000}, {0x026e, 0x00000000}, + {0x0278, 0x00000000}, {0x0262, 0x00000000}, +}; + +static const uint16_t g_agc_internal_reg_defaults_v9_tbl3_ptrs[] = { + 0x0000, 0x0001, 0x0003, +}; + +static uint32_t g_agc_internal_reg_defaults_v9_types[] = { + 0x8fb4edb5, 0x00040400, 0x00000000, 0xb994ad29, 0x00040404, 0x00000000, + 0xd427322f, 0x00040408, 0x00000000, 0xf58fea31, 0x0004040c, 0x00000000, + 0x6ac156ef, 0x00040401, 0x00000000, 0x6ac15610, 0x00040405, 0x00000000, + 0x6ac15009, 0x00040409, 0x00000000, 0x6ac153ba, 0x0004040d, 0x00000000, + 0xbe7dcd73, 0x00040411, 0x00000000, 0x0c4b1438, 0x00040415, 0x00000000, + 0xdb00d71a, 0x00040419, 0x00000000, 0xdb00d249, 0x0004041d, 0x00000000, + 0xdb00ec60, 0x00040421, 0x00000000, 0x0c4d6fe4, 0x00040425, 0x00000000, + 0x0c4a80ef, 0x00040429, 0x00000000, 0x0dd283e7, 0x0004042d, 0x00000000, + 0xc67efacf, 0x00040431, 0x00000000, 0xd9e6d9f7, 0x00040435, 0x00000000, + 0x60289246, 0x00040402, 0x00000000, 0x31f34b9f, 0x00040403, 0x00000000, + 0xac0f9e76, 0x00080407, 0x00000000, 0x929fd95d, 0x00040c0b, 0x00000000, +}; + +static CompactRegisterDefaults g_agc_internal_reg_defaults_v9 = { + g_agc_internal_reg_defaults_v9_tbl0_regs, 4, g_agc_internal_reg_defaults_v9_tbl0_ptrs, 4, + g_agc_internal_reg_defaults_v9_tbl1_regs, 14, g_agc_internal_reg_defaults_v9_tbl1_ptrs, 14, + g_agc_internal_reg_defaults_v9_tbl2_regs, 1, g_agc_internal_reg_defaults_v9_tbl2_ptrs, 1, + g_agc_internal_reg_defaults_v9_tbl3_regs, 6, g_agc_internal_reg_defaults_v9_tbl3_ptrs, 3, + g_agc_internal_reg_defaults_v9_types, 22, +}; + +// g_agc_public_reg_defaults_v0: regs=(523, 161, 43, 0), ptrs=(80, 28, 42, 0), types=150. +static ShaderRegister g_agc_public_reg_defaults_v0_tbl0_regs[] = { + {0x0202, 0x00cc0010}, {0x0109, 0x00000008}, {0x0104, 0x00000000}, {0x008f, 0x00000000}, + {0x008e, 0x00000000}, {0x02dc, 0x0000aa00}, {0x0001, 0x11000100}, {0x000b, 0x00000000}, + {0x0200, 0x00000000}, {0x0201, 0x00000000}, {0x0000, 0x00000000}, {0x0006, 0x00000000}, + {0x001f, 0x00000000}, {0x0203, 0x00000000}, {0x02b0, 0x00000000}, {0x02b1, 0x00000000}, + {0x010c, 0x00000000}, {0x010d, 0x00000000}, {0x000a, 0x00000000}, {0x010b, 0x00000000}, + {0x01ff, 0x00000000}, {0x0204, 0x00000000}, {0x020d, 0x00000000}, {0x0206, 0x0000043f}, + {0x02f8, 0x00000000}, {0x0083, 0x0000ffff}, {0x0313, 0x00006000}, {0x00f0, 0x00000000}, + {0x00f1, 0x00000000}, {0x00f2, 0x00000000}, {0x00ea, 0x00000000}, {0x00e9, 0x00000000}, + {0x0292, 0x00000002}, {0x0293, 0x06020000}, {0x00e8, 0x00000000}, {0x0080, 0x00000000}, + {0x0211, 0x00000000}, {0x0210, 0x00000000}, {0x008d, 0x00000000}, {0x0282, 0x00000008}, + {0x0281, 0xffff0000}, {0x0280, 0x00080008}, {0x02df, 0x00000000}, {0x02de, 0x000001e9}, + {0x0205, 0x00000240}, {0x020c, 0x00000021}, {0x02f9, 0x0000002d}, {0x01ba, 0x00000000}, + {0x02a6, 0x00000000}, {0x02ce, 0x00000400}, {0x029b, 0x00000002}, {0x02d6, 0x00000000}, + {0x0103, 0xffffffff}, {0x02a1, 0x00000000}, {0x02ad, 0x00000000}, {0x02d5, 0x00002000}, + {0x02d4, 0x88101010}, {0x02db, 0x00000000}, {0x02f5, 0x00000000}, {0x02f6, 0x00000000}, + {0x02fe, 0x00000000}, {0x02ff, 0x00000000}, {0x0300, 0x00000000}, {0x0301, 0x00000000}, + {0x0302, 0x00000000}, {0x0303, 0x00000000}, {0x0304, 0x00000000}, {0x0305, 0x00000000}, + {0x0306, 0x00000000}, {0x0307, 0x00000000}, {0x0308, 0x00000000}, {0x0309, 0x00000000}, + {0x030a, 0x00000000}, {0x030b, 0x00000000}, {0x030c, 0x00000000}, {0x030d, 0x00000000}, + {0x030e, 0xffffffff}, {0x030f, 0xffffffff}, {0x0311, 0x01f92000}, {0x0312, 0x03ff0080}, + {0x0105, 0x00000000}, {0x0106, 0x00000000}, {0x0107, 0x00000000}, {0x0108, 0x00000000}, + {0x01e0, 0x20010001}, {0x01e1, 0x20010001}, {0x01e2, 0x20010001}, {0x01e3, 0x20010001}, + {0x01e4, 0x20010001}, {0x01e5, 0x20010001}, {0x01e6, 0x20010001}, {0x01e7, 0x20010001}, + {0x0020, 0x00000000}, {0x0021, 0x00000000}, {0x0084, 0x00000000}, {0x0085, 0x20002000}, + {0x0086, 0x00000000}, {0x0087, 0x20002000}, {0x0088, 0x00000000}, {0x0089, 0x20002000}, + {0x008a, 0x00000000}, {0x008b, 0x20002000}, {0x0008, 0x00000000}, {0x0009, 0x00000000}, + {0x0010, 0x80000180}, {0x0011, 0x20000180}, {0x0012, 0x00000000}, {0x0013, 0x00000000}, + {0x0014, 0x00000000}, {0x0015, 0x00000000}, {0x0016, 0x00000000}, {0x0017, 0x00000000}, + {0x001a, 0x00000000}, {0x001b, 0x00000000}, {0x001c, 0x00000000}, {0x001d, 0x00000000}, + {0x001e, 0x00000000}, {0x0002, 0x00000000}, {0x0005, 0x00000000}, {0x02af, 0x00040000}, + {0x0007, 0x00000000}, {0x00eb, 0xff00ff00}, {0x00ec, 0x00010000}, {0x0090, 0x80000000}, + {0x0091, 0x40004000}, {0x02fa, 0x3f800000}, {0x02fb, 0x3f800000}, {0x02fc, 0x3f800000}, + {0x02fd, 0x3f800000}, {0x02e2, 0x00000000}, {0x02e3, 0x00000000}, {0x02e0, 0x00000000}, + {0x02e1, 0x00000000}, {0x0003, 0x00000000}, {0x0004, 0x00000000}, {0x0318, 0x00000000}, + {0x0319, 0x00000000}, {0x031a, 0x00000000}, {0x031b, 0x00000000}, {0x031c, 0x00000000}, + {0x031d, 0x00000000}, {0x031e, 0x00000048}, {0x031f, 0x00000000}, {0x0320, 0x00000000}, + {0x0321, 0x00000000}, {0x0322, 0x00000000}, {0x0323, 0x00000000}, {0x0324, 0x00000000}, + {0x0325, 0x00000000}, {0x0390, 0x00000000}, {0x0398, 0x00000000}, {0x03a0, 0x00000000}, + {0x03a8, 0x00000000}, {0x03b0, 0x00000000}, {0x03b8, 0x08c6c000}, {0x0327, 0x00000000}, + {0x0328, 0x00000000}, {0x0329, 0x00000000}, {0x032a, 0x00000000}, {0x032b, 0x00000000}, + {0x032c, 0x00000000}, {0x032d, 0x00000048}, {0x032e, 0x00000000}, {0x032f, 0x00000000}, + {0x0330, 0x00000000}, {0x0331, 0x00000000}, {0x0332, 0x00000000}, {0x0333, 0x00000000}, + {0x0334, 0x00000000}, {0x0391, 0x00000000}, {0x0399, 0x00000000}, {0x03a1, 0x00000000}, + {0x03a9, 0x00000000}, {0x03b1, 0x00000000}, {0x03b9, 0x08c6c000}, {0x0336, 0x00000000}, + {0x0337, 0x00000000}, {0x0338, 0x00000000}, {0x0339, 0x00000000}, {0x033a, 0x00000000}, + {0x033b, 0x00000000}, {0x033c, 0x00000048}, {0x033d, 0x00000000}, {0x033e, 0x00000000}, + {0x033f, 0x00000000}, {0x0340, 0x00000000}, {0x0341, 0x00000000}, {0x0342, 0x00000000}, + {0x0343, 0x00000000}, {0x0392, 0x00000000}, {0x039a, 0x00000000}, {0x03a2, 0x00000000}, + {0x03aa, 0x00000000}, {0x03b2, 0x00000000}, {0x03ba, 0x08c6c000}, {0x0345, 0x00000000}, + {0x0346, 0x00000000}, {0x0347, 0x00000000}, {0x0348, 0x00000000}, {0x0349, 0x00000000}, + {0x034a, 0x00000000}, {0x034b, 0x00000048}, {0x034c, 0x00000000}, {0x034d, 0x00000000}, + {0x034e, 0x00000000}, {0x034f, 0x00000000}, {0x0350, 0x00000000}, {0x0351, 0x00000000}, + {0x0352, 0x00000000}, {0x0393, 0x00000000}, {0x039b, 0x00000000}, {0x03a3, 0x00000000}, + {0x03ab, 0x00000000}, {0x03b3, 0x00000000}, {0x03bb, 0x08c6c000}, {0x0354, 0x00000000}, + {0x0355, 0x00000000}, {0x0356, 0x00000000}, {0x0357, 0x00000000}, {0x0358, 0x00000000}, + {0x0359, 0x00000000}, {0x035a, 0x00000048}, {0x035b, 0x00000000}, {0x035c, 0x00000000}, + {0x035d, 0x00000000}, {0x035e, 0x00000000}, {0x035f, 0x00000000}, {0x0360, 0x00000000}, + {0x0361, 0x00000000}, {0x0394, 0x00000000}, {0x039c, 0x00000000}, {0x03a4, 0x00000000}, + {0x03ac, 0x00000000}, {0x03b4, 0x00000000}, {0x03bc, 0x08c6c000}, {0x0363, 0x00000000}, + {0x0364, 0x00000000}, {0x0365, 0x00000000}, {0x0366, 0x00000000}, {0x0367, 0x00000000}, + {0x0368, 0x00000000}, {0x0369, 0x00000048}, {0x036a, 0x00000000}, {0x036b, 0x00000000}, + {0x036c, 0x00000000}, {0x036d, 0x00000000}, {0x036e, 0x00000000}, {0x036f, 0x00000000}, + {0x0370, 0x00000000}, {0x0395, 0x00000000}, {0x039d, 0x00000000}, {0x03a5, 0x00000000}, + {0x03ad, 0x00000000}, {0x03b5, 0x00000000}, {0x03bd, 0x08c6c000}, {0x0372, 0x00000000}, + {0x0373, 0x00000000}, {0x0374, 0x00000000}, {0x0375, 0x00000000}, {0x0376, 0x00000000}, + {0x0377, 0x00000000}, {0x0378, 0x00000048}, {0x0379, 0x00000000}, {0x037a, 0x00000000}, + {0x037b, 0x00000000}, {0x037c, 0x00000000}, {0x037d, 0x00000000}, {0x037e, 0x00000000}, + {0x037f, 0x00000000}, {0x0396, 0x00000000}, {0x039e, 0x00000000}, {0x03a6, 0x00000000}, + {0x03ae, 0x00000000}, {0x03b6, 0x00000000}, {0x03be, 0x08c6c000}, {0x0381, 0x00000000}, + {0x0382, 0x00000000}, {0x0383, 0x00000000}, {0x0384, 0x00000000}, {0x0385, 0x00000000}, + {0x0386, 0x00000000}, {0x0387, 0x00000048}, {0x0388, 0x00000000}, {0x0389, 0x00000000}, + {0x038a, 0x00000000}, {0x038b, 0x00000000}, {0x038c, 0x00000000}, {0x038d, 0x00000000}, + {0x038e, 0x00000000}, {0x0397, 0x00000000}, {0x039f, 0x00000000}, {0x03a7, 0x00000000}, + {0x03af, 0x00000000}, {0x03b7, 0x00000000}, {0x03bf, 0x08c6c000}, {0x000c, 0x00000000}, + {0x000d, 0x40004000}, {0x0191, 0x00000000}, {0x0192, 0x00000000}, {0x0193, 0x00000000}, + {0x0194, 0x00000000}, {0x0195, 0x00000000}, {0x0196, 0x00000000}, {0x0197, 0x00000000}, + {0x0198, 0x00000000}, {0x0199, 0x00000000}, {0x019a, 0x00000000}, {0x019b, 0x00000000}, + {0x019c, 0x00000000}, {0x019d, 0x00000000}, {0x019e, 0x00000000}, {0x019f, 0x00000000}, + {0x01a0, 0x00000000}, {0x01a1, 0x00000000}, {0x01a2, 0x00000000}, {0x01a3, 0x00000000}, + {0x01a4, 0x00000000}, {0x01a5, 0x00000000}, {0x01a6, 0x00000000}, {0x01a7, 0x00000000}, + {0x01a8, 0x00000000}, {0x01a9, 0x00000000}, {0x01aa, 0x00000000}, {0x01ab, 0x00000000}, + {0x01ac, 0x00000000}, {0x01ad, 0x00000000}, {0x01ae, 0x00000000}, {0x01af, 0x00000000}, + {0x01b0, 0x00000000}, {0x016f, 0x00000000}, {0x0170, 0x00000000}, {0x0171, 0x00000000}, + {0x0172, 0x00000000}, {0x0173, 0x00000000}, {0x0174, 0x00000000}, {0x0175, 0x00000000}, + {0x0176, 0x00000000}, {0x0177, 0x00000000}, {0x0178, 0x00000000}, {0x0179, 0x00000000}, + {0x017a, 0x00000000}, {0x017b, 0x00000000}, {0x017c, 0x00000000}, {0x017d, 0x00000000}, + {0x017e, 0x00000000}, {0x017f, 0x00000000}, {0x0180, 0x00000000}, {0x0181, 0x00000000}, + {0x0182, 0x00000000}, {0x0183, 0x00000000}, {0x0184, 0x00000000}, {0x0185, 0x00000000}, + {0x0186, 0x00000000}, {0x0187, 0x00000000}, {0x0188, 0x00000000}, {0x0189, 0x00000000}, + {0x018a, 0x00000000}, {0x018b, 0x00000000}, {0x018c, 0x00000000}, {0x018d, 0x00000000}, + {0x018e, 0x00000000}, {0x010f, 0x3f800000}, {0x0111, 0x3f800000}, {0x0113, 0x3f800000}, + {0x0110, 0x00000000}, {0x0112, 0x00000000}, {0x0114, 0x00000000}, {0x0094, 0x80000000}, + {0x0095, 0x40004000}, {0x00b4, 0x00000000}, {0x00b5, 0x00000000}, {0x0115, 0x3f800000}, + {0x0117, 0x3f800000}, {0x0119, 0x3f800000}, {0x0116, 0x00000000}, {0x0118, 0x00000000}, + {0x011a, 0x00000000}, {0x0096, 0x80000000}, {0x0097, 0x40004000}, {0x00b6, 0x00000000}, + {0x00b7, 0x00000000}, {0x011b, 0x3f800000}, {0x011d, 0x3f800000}, {0x011f, 0x3f800000}, + {0x011c, 0x00000000}, {0x011e, 0x00000000}, {0x0120, 0x00000000}, {0x0098, 0x80000000}, + {0x0099, 0x40004000}, {0x00b8, 0x00000000}, {0x00b9, 0x00000000}, {0x0121, 0x3f800000}, + {0x0123, 0x3f800000}, {0x0125, 0x3f800000}, {0x0122, 0x00000000}, {0x0124, 0x00000000}, + {0x0126, 0x00000000}, {0x009a, 0x80000000}, {0x009b, 0x40004000}, {0x00ba, 0x00000000}, + {0x00bb, 0x00000000}, {0x0127, 0x3f800000}, {0x0129, 0x3f800000}, {0x012b, 0x3f800000}, + {0x0128, 0x00000000}, {0x012a, 0x00000000}, {0x012c, 0x00000000}, {0x009c, 0x80000000}, + {0x009d, 0x40004000}, {0x00bc, 0x00000000}, {0x00bd, 0x00000000}, {0x012d, 0x3f800000}, + {0x012f, 0x3f800000}, {0x0131, 0x3f800000}, {0x012e, 0x00000000}, {0x0130, 0x00000000}, + {0x0132, 0x00000000}, {0x009e, 0x80000000}, {0x009f, 0x40004000}, {0x00be, 0x00000000}, + {0x00bf, 0x00000000}, {0x0133, 0x3f800000}, {0x0135, 0x3f800000}, {0x0137, 0x3f800000}, + {0x0134, 0x00000000}, {0x0136, 0x00000000}, {0x0138, 0x00000000}, {0x00a0, 0x80000000}, + {0x00a1, 0x40004000}, {0x00c0, 0x00000000}, {0x00c1, 0x00000000}, {0x0139, 0x3f800000}, + {0x013b, 0x3f800000}, {0x013d, 0x3f800000}, {0x013a, 0x00000000}, {0x013c, 0x00000000}, + {0x013e, 0x00000000}, {0x00a2, 0x80000000}, {0x00a3, 0x40004000}, {0x00c2, 0x00000000}, + {0x00c3, 0x00000000}, {0x013f, 0x3f800000}, {0x0141, 0x3f800000}, {0x0143, 0x3f800000}, + {0x0140, 0x00000000}, {0x0142, 0x00000000}, {0x0144, 0x00000000}, {0x00a4, 0x80000000}, + {0x00a5, 0x40004000}, {0x00c4, 0x00000000}, {0x00c5, 0x00000000}, {0x0145, 0x3f800000}, + {0x0147, 0x3f800000}, {0x0149, 0x3f800000}, {0x0146, 0x00000000}, {0x0148, 0x00000000}, + {0x014a, 0x00000000}, {0x00a6, 0x80000000}, {0x00a7, 0x40004000}, {0x00c6, 0x00000000}, + {0x00c7, 0x00000000}, {0x014b, 0x3f800000}, {0x014d, 0x3f800000}, {0x014f, 0x3f800000}, + {0x014c, 0x00000000}, {0x014e, 0x00000000}, {0x0150, 0x00000000}, {0x00a8, 0x80000000}, + {0x00a9, 0x40004000}, {0x00c8, 0x00000000}, {0x00c9, 0x00000000}, {0x0151, 0x3f800000}, + {0x0153, 0x3f800000}, {0x0155, 0x3f800000}, {0x0152, 0x00000000}, {0x0154, 0x00000000}, + {0x0156, 0x00000000}, {0x00aa, 0x80000000}, {0x00ab, 0x40004000}, {0x00ca, 0x00000000}, + {0x00cb, 0x00000000}, {0x0157, 0x3f800000}, {0x0159, 0x3f800000}, {0x015b, 0x3f800000}, + {0x0158, 0x00000000}, {0x015a, 0x00000000}, {0x015c, 0x00000000}, {0x00ac, 0x80000000}, + {0x00ad, 0x40004000}, {0x00cc, 0x00000000}, {0x00cd, 0x00000000}, {0x015d, 0x3f800000}, + {0x015f, 0x3f800000}, {0x0161, 0x3f800000}, {0x015e, 0x00000000}, {0x0160, 0x00000000}, + {0x0162, 0x00000000}, {0x00ae, 0x80000000}, {0x00af, 0x40004000}, {0x00ce, 0x00000000}, + {0x00cf, 0x00000000}, {0x0163, 0x3f800000}, {0x0165, 0x3f800000}, {0x0167, 0x3f800000}, + {0x0164, 0x00000000}, {0x0166, 0x00000000}, {0x0168, 0x00000000}, {0x00b0, 0x80000000}, + {0x00b1, 0x40004000}, {0x00d0, 0x00000000}, {0x00d1, 0x00000000}, {0x0169, 0x3f800000}, + {0x016b, 0x3f800000}, {0x016d, 0x3f800000}, {0x016a, 0x00000000}, {0x016c, 0x00000000}, + {0x016e, 0x00000000}, {0x00b2, 0x80000000}, {0x00b3, 0x40004000}, {0x00d2, 0x00000000}, + {0x00d3, 0x00000000}, {0x0081, 0x80000000}, {0x0082, 0x40004000}, +}; + +static const uint16_t g_agc_public_reg_defaults_v0_tbl0_ptrs[] = { + 0x0000, 0x0001, 0x0002, 0x0003, 0x0004, 0x0005, 0x0006, 0x0007, 0x0008, 0x0009, 0x000a, 0x000b, + 0x000c, 0x000d, 0x000e, 0x000f, 0x0010, 0x0011, 0x0012, 0x0013, 0x0014, 0x0015, 0x0016, 0x0017, + 0x0018, 0x0019, 0x001a, 0x001b, 0x001c, 0x001d, 0x001e, 0x001f, 0x0020, 0x0021, 0x0022, 0x0023, + 0x0024, 0x0025, 0x0026, 0x0027, 0x0028, 0x0029, 0x002a, 0x002b, 0x002c, 0x002d, 0x002e, 0x002f, + 0x0030, 0x0031, 0x0032, 0x0033, 0x0034, 0x0035, 0x0036, 0x0037, 0x0038, 0x0039, 0x003a, 0x003c, + 0x004c, 0x004e, 0x0050, 0x0054, 0x005c, 0x005e, 0x0066, 0x0068, 0x0079, 0x007b, 0x007d, 0x0081, + 0x0083, 0x0085, 0x0087, 0x0127, 0x0129, 0x0149, 0x0169, 0x0209, +}; + +static ShaderRegister g_agc_public_reg_defaults_v0_tbl1_regs[] = { + {0x0212, 0x00000000}, {0x0213, 0x00000000}, {0x0228, 0x00000000}, {0x0215, 0x00000000}, + {0x0218, 0x00000000}, {0x008a, 0x00000000}, {0x010a, 0x00000000}, {0x000a, 0x00000000}, + {0x008b, 0x00000000}, {0x010b, 0x00000000}, {0x000b, 0x00000000}, {0x0224, 0x00000000}, + {0x0225, 0x00000000}, {0x0226, 0x00000000}, {0x0227, 0x00000000}, {0x0107, 0xffff0000}, + {0x0101, 0x00000000}, {0x0087, 0x0000ffff}, {0x0081, 0x00000000}, {0x0007, 0x0000ffff}, + {0x0001, 0x00000003}, {0x020c, 0x00000000}, {0x020d, 0x00000000}, {0x00c8, 0x00000000}, + {0x00c9, 0x00000000}, {0x0088, 0x00000000}, {0x0089, 0x00000000}, {0x0108, 0x00000000}, + {0x0109, 0x00000000}, {0x0148, 0x00000000}, {0x0149, 0x00000000}, {0x0008, 0x00000000}, + {0x0009, 0x00000000}, {0x00b2, 0x00000000}, {0x00b3, 0x00000000}, {0x00b4, 0x00000000}, + {0x00b5, 0x00000000}, {0x0132, 0x00000000}, {0x0133, 0x00000000}, {0x0134, 0x00000000}, + {0x0135, 0x00000000}, {0x0032, 0x00000000}, {0x0033, 0x00000000}, {0x0034, 0x00000000}, + {0x0035, 0x00000000}, {0x0082, 0x00000000}, {0x0083, 0x00000000}, {0x0102, 0x00000000}, + {0x0103, 0x00000000}, {0x0240, 0x00000000}, {0x0241, 0x00000000}, {0x0242, 0x00000000}, + {0x0243, 0x00000000}, {0x0244, 0x00000000}, {0x0245, 0x00000000}, {0x0246, 0x00000000}, + {0x0247, 0x00000000}, {0x0248, 0x00000000}, {0x0249, 0x00000000}, {0x024a, 0x00000000}, + {0x024b, 0x00000000}, {0x024c, 0x00000000}, {0x024d, 0x00000000}, {0x024e, 0x00000000}, + {0x024f, 0x00000000}, {0x008c, 0x00000000}, {0x008d, 0x00000000}, {0x008e, 0x00000000}, + {0x008f, 0x00000000}, {0x0090, 0x00000000}, {0x0091, 0x00000000}, {0x0092, 0x00000000}, + {0x0093, 0x00000000}, {0x0094, 0x00000000}, {0x0095, 0x00000000}, {0x0096, 0x00000000}, + {0x0097, 0x00000000}, {0x0098, 0x00000000}, {0x0099, 0x00000000}, {0x009a, 0x00000000}, + {0x009b, 0x00000000}, {0x009c, 0x00000000}, {0x009d, 0x00000000}, {0x009e, 0x00000000}, + {0x009f, 0x00000000}, {0x00a0, 0x00000000}, {0x00a1, 0x00000000}, {0x00a2, 0x00000000}, + {0x00a3, 0x00000000}, {0x00a4, 0x00000000}, {0x00a5, 0x00000000}, {0x00a6, 0x00000000}, + {0x00a7, 0x00000000}, {0x00a8, 0x00000000}, {0x00a9, 0x00000000}, {0x00aa, 0x00000000}, + {0x00ab, 0x00000000}, {0x010c, 0x00000000}, {0x010d, 0x00000000}, {0x010e, 0x00000000}, + {0x010f, 0x00000000}, {0x0110, 0x00000000}, {0x0111, 0x00000000}, {0x0112, 0x00000000}, + {0x0113, 0x00000000}, {0x0114, 0x00000000}, {0x0115, 0x00000000}, {0x0116, 0x00000000}, + {0x0117, 0x00000000}, {0x0118, 0x00000000}, {0x0119, 0x00000000}, {0x011a, 0x00000000}, + {0x011b, 0x00000000}, {0x011c, 0x00000000}, {0x011d, 0x00000000}, {0x011e, 0x00000000}, + {0x011f, 0x00000000}, {0x0120, 0x00000000}, {0x0121, 0x00000000}, {0x0122, 0x00000000}, + {0x0123, 0x00000000}, {0x0124, 0x00000000}, {0x0125, 0x00000000}, {0x0126, 0x00000000}, + {0x0127, 0x00000000}, {0x0128, 0x00000000}, {0x0129, 0x00000000}, {0x012a, 0x00000000}, + {0x012b, 0x00000000}, {0x000c, 0x00000000}, {0x000d, 0x00000000}, {0x000e, 0x00000000}, + {0x000f, 0x00000000}, {0x0010, 0x00000000}, {0x0011, 0x00000000}, {0x0012, 0x00000000}, + {0x0013, 0x00000000}, {0x0014, 0x00000000}, {0x0015, 0x00000000}, {0x0016, 0x00000000}, + {0x0017, 0x00000000}, {0x0018, 0x00000000}, {0x0019, 0x00000000}, {0x001a, 0x00000000}, + {0x001b, 0x00000000}, {0x001c, 0x00000000}, {0x001d, 0x00000000}, {0x001e, 0x00000000}, + {0x001f, 0x00000000}, {0x0020, 0x00000000}, {0x0021, 0x00000000}, {0x0022, 0x00000000}, + {0x0023, 0x00000000}, {0x0024, 0x00000000}, {0x0025, 0x00000000}, {0x0026, 0x00000000}, + {0x0027, 0x00000000}, {0x0028, 0x00000000}, {0x0029, 0x00000000}, {0x002a, 0x00000000}, + {0x002b, 0x00000000}, +}; + +static const uint16_t g_agc_public_reg_defaults_v0_tbl1_ptrs[] = { + 0x0000, 0x0001, 0x0002, 0x0003, 0x0004, 0x0005, 0x0006, 0x0007, 0x0008, 0x0009, 0x000a, 0x000b, + 0x000f, 0x0015, 0x0017, 0x0019, 0x001b, 0x001d, 0x001f, 0x0021, 0x0025, 0x0029, 0x002d, 0x002f, + 0x0031, 0x0041, 0x0061, 0x0081, +}; + +static ShaderRegister g_agc_public_reg_defaults_v0_tbl2_regs[] = { + {0x041f, 0x00000000}, {0x041d, 0x00000000}, {0x041e, 0x00000000}, {0x025b, 0x00000000}, + {0x024a, 0x00000000}, {0x024b, 0x00000000}, {0x025f, 0x00000000}, {0x0262, 0x00000000}, + {0x02bb, 0x00000000}, {0x02e4, 0x00000000}, {0x02e5, 0x00000000}, {0x02c4, 0x00000000}, + {0x02c5, 0x00000000}, {0x02c6, 0x00000000}, {0x02c7, 0x00000000}, {0x02e8, 0x00000000}, + {0x02e9, 0x00000000}, {0x02cc, 0x00000000}, {0x02cd, 0x00000000}, {0x02ce, 0x00000000}, + {0x02cf, 0x00000000}, {0x02bd, 0x00000000}, {0x02bc, 0x00000000}, {0x02e6, 0x00000000}, + {0x02e7, 0x00000000}, {0x02c8, 0x00000000}, {0x02c9, 0x00000000}, {0x02ca, 0x00000000}, + {0x02cb, 0x00000000}, {0x02ea, 0x00000000}, {0x02eb, 0x00000000}, {0x02d0, 0x00000000}, + {0x02d1, 0x00000000}, {0x02d2, 0x00000000}, {0x02d3, 0x00000000}, {0x02bf, 0x00000000}, + {0x02be, 0x00000000}, {0x0384, 0x00000000}, {0x0383, 0x40000040}, {0x0248, 0x00000000}, + {0x0242, 0x00000000}, {0x0380, 0x00000000}, {0x0381, 0x00000000}, +}; + +static const uint16_t g_agc_public_reg_defaults_v0_tbl2_ptrs[] = { + 0x0000, 0x0001, 0x0002, 0x0003, 0x0004, 0x0005, 0x0006, 0x0007, 0x0008, 0x0009, 0x000a, 0x000b, + 0x000c, 0x000d, 0x000e, 0x000f, 0x0010, 0x0011, 0x0012, 0x0013, 0x0014, 0x0015, 0x0016, 0x0017, + 0x0018, 0x0019, 0x001a, 0x001b, 0x001c, 0x001d, 0x001e, 0x001f, 0x0020, 0x0021, 0x0022, 0x0023, + 0x0024, 0x0025, 0x0026, 0x0027, 0x0028, 0x0029, +}; + +static uint32_t g_agc_public_reg_defaults_v0_types[] = { + 0xe24f806d, 0x00040400, 0x00000000, 0xf6c28182, 0x00040404, 0x00000000, + 0x6f6e55a5, 0x00040408, 0x00000000, 0x0bc65da4, 0x0004040c, 0x00000000, + 0x9e5ad592, 0x00040410, 0x00000000, 0xbb513b98, 0x00040414, 0x00000000, + 0xab64b23b, 0x00040418, 0x00000000, 0xac198e0b, 0x0004041c, 0x00000000, + 0x53c39964, 0x00040420, 0x00000000, 0x01396b11, 0x00040424, 0x00000000, + 0x7d42019a, 0x00040428, 0x00000000, 0x3548f523, 0x0004042c, 0x00000000, + 0xf43ad28a, 0x00040430, 0x00000000, 0x6de4c312, 0x00040434, 0x00000000, + 0x00a77ae0, 0x00040438, 0x00000000, 0x00a779b7, 0x0004043c, 0x00000000, + 0x5100100c, 0x00040440, 0x00000000, 0x59958bba, 0x00040444, 0x00000000, + 0x7f7bd53e, 0x00040448, 0x00000000, 0x0c06f17c, 0x0004044c, 0x00000000, + 0x6f104b72, 0x00040450, 0x00000000, 0x25c70d9c, 0x00040454, 0x00000000, + 0x3881201e, 0x00040458, 0x00000000, 0x09afddaf, 0x0004045c, 0x00000000, + 0x367d63cf, 0x00040460, 0x00000000, 0x43707db8, 0x00040464, 0x00000000, + 0xf6ae26ba, 0x00040468, 0x00000000, 0x1b917652, 0x0004046c, 0x00000000, + 0x2cac726e, 0x00040470, 0x00000000, 0x2cac7111, 0x00040474, 0x00000000, + 0x94b1e4f7, 0x00040478, 0x00000000, 0xe3661b6c, 0x0004047c, 0x00000000, + 0x1eb8d73a, 0x00040480, 0x00000000, 0x15051fa3, 0x00040484, 0x00000000, + 0x9c51a7f1, 0x00040488, 0x00000000, 0xa20efc70, 0x0004048c, 0x00000000, + 0x0ec09f6e, 0x00040490, 0x00000000, 0x34a7d6d3, 0x00040494, 0x00000000, + 0xce831b94, 0x00040498, 0x00000000, 0x5cc72a74, 0x0004049c, 0x00000000, + 0x3b77713c, 0x000404a0, 0x00000000, 0x40f64410, 0x000404a4, 0x00000000, + 0x69441268, 0x000404a8, 0x00000000, 0x2e418b83, 0x000404ac, 0x00000000, + 0xa00d0c8d, 0x000404b0, 0x00000000, 0xb1289fb3, 0x000404b4, 0x00000000, + 0x144832fb, 0x000404b8, 0x00000000, 0x9890d9fa, 0x000404bc, 0x00000000, + 0x9016faf1, 0x000404c0, 0x00000000, 0x4b73ce27, 0x000404c4, 0x00000000, + 0x5f5a3e7b, 0x000404c8, 0x00000000, 0xd4af3a51, 0x000404cc, 0x00000000, + 0x6cf4f543, 0x000404d0, 0x00000000, 0x5fb86ccb, 0x000404d4, 0x00000000, + 0xedefa188, 0x000404d8, 0x00000000, 0xd0de9ee6, 0x000404dc, 0x00000000, + 0xc5831803, 0x000404e0, 0x00000000, 0x8e6de84b, 0x000404e4, 0x00000000, + 0xd0771662, 0x000408e8, 0x00000000, 0x569f7444, 0x004004ec, 0x00000000, + 0x5c6637cd, 0x000408f0, 0x00000000, 0xcae3e690, 0x000408f4, 0x00000000, + 0x43fbd769, 0x000410f8, 0x00000000, 0xef550356, 0x002004fc, 0x00000000, + 0x8f52e279, 0x00040900, 0x00000000, 0x1f2d8149, 0x00100904, 0x00000000, + 0x4413c6f9, 0x00040908, 0x00000000, 0x67096014, 0x0004450c, 0x00000000, + 0x88f5e915, 0x00040910, 0x00000000, 0x918106bb, 0x00040914, 0x00000000, + 0x95f0e7ac, 0x00041118, 0x00000000, 0xb48cbab2, 0x0004091c, 0x00000000, + 0x05bb3bc6, 0x00040920, 0x00000000, 0x94faba07, 0x00040924, 0x00000000, + 0x38e92c91, 0x00205128, 0x00000000, 0x0b177b43, 0x0004092c, 0x00000000, + 0x48531062, 0x00800530, 0x00000000, 0xaaa964b9, 0x00201134, 0x00000000, + 0x7690af6f, 0x00402938, 0x00000000, 0x078d7060, 0x0004093c, 0x00000000, + 0x5d6e3ec7, 0x00040401, 0x00000000, 0x57e7079a, 0x00040405, 0x00000000, + 0x7467fafd, 0x00040409, 0x00000000, 0x9e826b50, 0x0004040d, 0x00000000, + 0xdc484f18, 0x00040411, 0x00000000, 0x5da8bca3, 0x00040415, 0x00000000, + 0x5ca726d8, 0x00040419, 0x00000000, 0x5dd28360, 0x0004041d, 0x00000000, + 0x57efa0be, 0x00040421, 0x00000000, 0x502363d5, 0x00040425, 0x00000000, + 0x506d14bd, 0x00040429, 0x00000000, 0xb2609506, 0x0010042d, 0x00000000, + 0x9e5cfb8a, 0x00041831, 0x00000000, 0xc918df3e, 0x00040835, 0x00000000, + 0xc9751c9c, 0x00040839, 0x00000000, 0xc97ef77a, 0x0004083d, 0x00000000, + 0xc927c6b9, 0x00040841, 0x00000000, 0xc92a1ec5, 0x00040845, 0x00000000, + 0xc9e01b31, 0x00040849, 0x00000000, 0xb26219ca, 0x0010044d, 0x00000000, + 0xb25b6cf9, 0x00100451, 0x00000000, 0xb2f86101, 0x00100455, 0x00000000, + 0x07e3b155, 0x00040859, 0x00000000, 0x07e383c6, 0x0004085d, 0x00000000, + 0xbda98653, 0x00400461, 0x00000000, 0xbdbd1d0f, 0x00800465, 0x00000000, + 0xbd946fd4, 0x00800469, 0x00000000, 0xbdf02a4c, 0x0080046d, 0x00000000, + 0x19e93e85, 0x00040402, 0x00000000, 0x3b5c2af3, 0x00040406, 0x00000000, + 0x47974a35, 0x0004040a, 0x00000000, 0x105971c2, 0x0004040e, 0x00000000, + 0x7d137765, 0x00040412, 0x00000000, 0xd187febc, 0x00040416, 0x00000000, + 0x12f854ac, 0x0004041a, 0x00000000, 0x40d49ad1, 0x0004041e, 0x00000000, + 0x8c0923da, 0x00040422, 0x00000000, 0xf6be6f25, 0x00040426, 0x00000000, + 0xf6908bad, 0x0004042a, 0x00000000, 0x862ee1be, 0x0004042e, 0x00000000, + 0x8631d396, 0x00040432, 0x00000000, 0x89d7d386, 0x00040436, 0x00000000, + 0xd1006f62, 0x0004043a, 0x00000000, 0x7e51d354, 0x0004043e, 0x00000000, + 0x7e712edc, 0x00040442, 0x00000000, 0x141f6002, 0x00040446, 0x00000000, + 0x140c761a, 0x0004044a, 0x00000000, 0x14f68dfa, 0x0004044e, 0x00000000, + 0x6fea32c3, 0x00040452, 0x00000000, 0x8f378499, 0x00040456, 0x00000000, + 0x5d9ed616, 0x0004045a, 0x00000000, 0x2bbfa784, 0x0004045e, 0x00000000, + 0x2b45f28c, 0x00040462, 0x00000000, 0x5f4aaed9, 0x00040466, 0x00000000, + 0x5f477465, 0x0004046a, 0x00000000, 0x5ec84729, 0x0004046e, 0x00000000, + 0x48ca8407, 0x00040472, 0x00000000, 0x5f355d79, 0x00040476, 0x00000000, + 0x5f0f3449, 0x0004047a, 0x00000000, 0xee4a4919, 0x0004047e, 0x00000000, + 0xee49852d, 0x00040482, 0x00000000, 0xee4f8201, 0x00040486, 0x00000000, + 0x8a3f99a1, 0x0004048a, 0x00000000, 0x381fc91f, 0x0004048e, 0x00000000, + 0x19d8287b, 0x00040492, 0x00000000, 0xbb8df494, 0x00040496, 0x00000000, + 0xf6d8a76e, 0x0004049a, 0x00000000, 0x7620f1e9, 0x0004049e, 0x00000000, + 0x9ebfab10, 0x000404a2, 0x00000000, 0x98a09d0e, 0x000408a6, 0x00000000, +}; + +static CompactRegisterDefaults g_agc_public_reg_defaults_v0 = { + g_agc_public_reg_defaults_v0_tbl0_regs, 523, g_agc_public_reg_defaults_v0_tbl0_ptrs, 80, + g_agc_public_reg_defaults_v0_tbl1_regs, 161, g_agc_public_reg_defaults_v0_tbl1_ptrs, 28, + g_agc_public_reg_defaults_v0_tbl2_regs, 43, g_agc_public_reg_defaults_v0_tbl2_ptrs, 42, + nullptr, 0, nullptr, 0, + g_agc_public_reg_defaults_v0_types, 150, +}; + +// g_agc_public_reg_defaults_v10: regs=(482, 159, 55, 0), ptrs=(79, 29, 20, 0), types=128. +static ShaderRegister g_agc_public_reg_defaults_v10_tbl0_regs[] = { + {0x0202, 0x00cc0010}, {0x0109, 0x00000010}, {0x01e0, 0x20010001}, {0x0104, 0x00000000}, + {0x008f, 0x00000000}, {0x008e, 0x00000000}, {0x02dc, 0x0000aa00}, {0x0001, 0x11000100}, + {0x0200, 0x00000000}, {0x0201, 0x00000000}, {0x0000, 0x00000000}, {0x0006, 0x00000000}, + {0x001f, 0x00000000}, {0x0203, 0x00000000}, {0x02b0, 0x00000000}, {0x02b1, 0x00000000}, + {0x010c, 0x00000000}, {0x010d, 0x00000000}, {0x010b, 0x00000000}, {0x01ff, 0x00000000}, + {0x0204, 0x00000000}, {0x020d, 0x00000000}, {0x0206, 0x0000043f}, {0x02f8, 0x00000000}, + {0x0083, 0x0000ffff}, {0x0313, 0x00006000}, {0x00f0, 0x00000000}, {0x00ea, 0x00000000}, + {0x00e9, 0x00000000}, {0x0292, 0x00000002}, {0x0293, 0x06020000}, {0x00e8, 0x00000000}, + {0x0080, 0x00000000}, {0x0211, 0x00000000}, {0x0210, 0x04000000}, {0x008d, 0x00000000}, + {0x0282, 0x00000008}, {0x0281, 0xffff0000}, {0x0280, 0x00080008}, {0x02df, 0x00000000}, + {0x02de, 0x000001e9}, {0x0205, 0x00000240}, {0x020c, 0x00000021}, {0x02f9, 0x0000002d}, + {0x01ba, 0x00000000}, {0x02a6, 0x00000000}, {0x02ce, 0x00000400}, {0x029b, 0x00000002}, + {0x02d6, 0x00000000}, {0x0103, 0xffffffff}, {0x02a1, 0x00000000}, {0x02ad, 0x00000000}, + {0x02d5, 0x00002000}, {0x02d4, 0x88101000}, {0x02db, 0x00000000}, {0x02f5, 0x00000000}, + {0x02f6, 0x00000000}, {0x02fe, 0x00000000}, {0x02ff, 0x00000000}, {0x0300, 0x00000000}, + {0x0301, 0x00000000}, {0x0302, 0x00000000}, {0x0303, 0x00000000}, {0x0304, 0x00000000}, + {0x0305, 0x00000000}, {0x0306, 0x00000000}, {0x0307, 0x00000000}, {0x0308, 0x00000000}, + {0x0309, 0x00000000}, {0x030a, 0x00000000}, {0x030b, 0x00000000}, {0x030c, 0x00000000}, + {0x030d, 0x00000000}, {0x030e, 0xffffffff}, {0x030f, 0xffffffff}, {0x0311, 0x01fd2002}, + {0x0312, 0x03ff0080}, {0x0105, 0x00000000}, {0x0106, 0x00000000}, {0x0107, 0x00000000}, + {0x0108, 0x00000000}, {0x01e0, 0x20010001}, {0x01e1, 0x20010001}, {0x01e2, 0x20010001}, + {0x01e3, 0x20010001}, {0x01e4, 0x20010001}, {0x01e5, 0x20010001}, {0x01e6, 0x20010001}, + {0x01e7, 0x20010001}, {0x0020, 0x00000000}, {0x0021, 0x00000000}, {0x0084, 0x00000000}, + {0x0085, 0x20002000}, {0x0086, 0x00000000}, {0x0087, 0x20002000}, {0x0088, 0x00000000}, + {0x0089, 0x20002000}, {0x008a, 0x00000000}, {0x008b, 0x20002000}, {0x00db, 0x00000000}, + {0x0008, 0x00000000}, {0x0009, 0x00000000}, {0x0010, 0x80000180}, {0x0011, 0x20000180}, + {0x0012, 0x00000000}, {0x0013, 0x00000000}, {0x0014, 0x00000000}, {0x0015, 0x00000000}, + {0x001a, 0x00000000}, {0x001b, 0x00000000}, {0x001c, 0x00000000}, {0x001d, 0x00000000}, + {0x001e, 0x00000000}, {0x0002, 0x00000000}, {0x0005, 0x00000000}, {0x0007, 0x00000000}, + {0x000b, 0x00000000}, {0x000a, 0x00000000}, {0x00eb, 0xff00ff00}, {0x00ec, 0x00010000}, + {0x00f1, 0x00000000}, {0x00f2, 0x00000000}, {0x0090, 0x80000000}, {0x0091, 0x40004000}, + {0x02fa, 0x3f800000}, {0x02fb, 0x3f800000}, {0x02fc, 0x3f800000}, {0x02fd, 0x3f800000}, + {0x02e2, 0x00000000}, {0x02e3, 0x00000000}, {0x02e0, 0x00000000}, {0x02e1, 0x00000000}, + {0x0003, 0x00000000}, {0x0004, 0x00000000}, {0x0318, 0x00000000}, {0x031b, 0x00000000}, + {0x031c, 0x00000000}, {0x031d, 0x00000000}, {0x031e, 0x00000048}, {0x031f, 0x00000000}, + {0x0321, 0x00000000}, {0x0323, 0x00000000}, {0x0324, 0x00000000}, {0x0325, 0x00000000}, + {0x0390, 0x00000000}, {0x0398, 0x00000000}, {0x03a0, 0x00000000}, {0x03a8, 0x00000000}, + {0x03b0, 0x00000000}, {0x03b8, 0x08c6c000}, {0x0327, 0x00000000}, {0x032a, 0x00000000}, + {0x032b, 0x00000000}, {0x032c, 0x00000000}, {0x032d, 0x00000048}, {0x032e, 0x00000000}, + {0x0330, 0x00000000}, {0x0332, 0x00000000}, {0x0333, 0x00000000}, {0x0334, 0x00000000}, + {0x0391, 0x00000000}, {0x0399, 0x00000000}, {0x03a1, 0x00000000}, {0x03a9, 0x00000000}, + {0x03b1, 0x00000000}, {0x03b9, 0x08c6c000}, {0x0336, 0x00000000}, {0x0339, 0x00000000}, + {0x033a, 0x00000000}, {0x033b, 0x00000000}, {0x033c, 0x00000048}, {0x033d, 0x00000000}, + {0x033f, 0x00000000}, {0x0341, 0x00000000}, {0x0342, 0x00000000}, {0x0343, 0x00000000}, + {0x0392, 0x00000000}, {0x039a, 0x00000000}, {0x03a2, 0x00000000}, {0x03aa, 0x00000000}, + {0x03b2, 0x00000000}, {0x03ba, 0x08c6c000}, {0x0345, 0x00000000}, {0x0348, 0x00000000}, + {0x0349, 0x00000000}, {0x034a, 0x00000000}, {0x034b, 0x00000048}, {0x034c, 0x00000000}, + {0x034e, 0x00000000}, {0x0350, 0x00000000}, {0x0351, 0x00000000}, {0x0352, 0x00000000}, + {0x0393, 0x00000000}, {0x039b, 0x00000000}, {0x03a3, 0x00000000}, {0x03ab, 0x00000000}, + {0x03b3, 0x00000000}, {0x03bb, 0x08c6c000}, {0x0354, 0x00000000}, {0x0357, 0x00000000}, + {0x0358, 0x00000000}, {0x0359, 0x00000000}, {0x035a, 0x00000048}, {0x035b, 0x00000000}, + {0x035d, 0x00000000}, {0x035f, 0x00000000}, {0x0360, 0x00000000}, {0x0361, 0x00000000}, + {0x0394, 0x00000000}, {0x039c, 0x00000000}, {0x03a4, 0x00000000}, {0x03ac, 0x00000000}, + {0x03b4, 0x00000000}, {0x03bc, 0x08c6c000}, {0x0363, 0x00000000}, {0x0366, 0x00000000}, + {0x0367, 0x00000000}, {0x0368, 0x00000000}, {0x0369, 0x00000048}, {0x036a, 0x00000000}, + {0x036c, 0x00000000}, {0x036e, 0x00000000}, {0x036f, 0x00000000}, {0x0370, 0x00000000}, + {0x0395, 0x00000000}, {0x039d, 0x00000000}, {0x03a5, 0x00000000}, {0x03ad, 0x00000000}, + {0x03b5, 0x00000000}, {0x03bd, 0x08c6c000}, {0x0372, 0x00000000}, {0x0375, 0x00000000}, + {0x0376, 0x00000000}, {0x0377, 0x00000000}, {0x0378, 0x00000048}, {0x0379, 0x00000000}, + {0x037b, 0x00000000}, {0x037d, 0x00000000}, {0x037e, 0x00000000}, {0x037f, 0x00000000}, + {0x0396, 0x00000000}, {0x039e, 0x00000000}, {0x03a6, 0x00000000}, {0x03ae, 0x00000000}, + {0x03b6, 0x00000000}, {0x03be, 0x08c6c000}, {0x0381, 0x00000000}, {0x0384, 0x00000000}, + {0x0385, 0x00000000}, {0x0386, 0x00000000}, {0x0387, 0x00000048}, {0x0388, 0x00000000}, + {0x038a, 0x00000000}, {0x038c, 0x00000000}, {0x038d, 0x00000000}, {0x038e, 0x00000000}, + {0x0397, 0x00000000}, {0x039f, 0x00000000}, {0x03a7, 0x00000000}, {0x03af, 0x00000000}, + {0x03b7, 0x00000000}, {0x03bf, 0x08c6c000}, {0x000c, 0x00000000}, {0x000d, 0x40004000}, + {0x0191, 0x00000000}, {0x0192, 0x00000000}, {0x0193, 0x00000000}, {0x0194, 0x00000000}, + {0x0195, 0x00000000}, {0x0196, 0x00000000}, {0x0197, 0x00000000}, {0x0198, 0x00000000}, + {0x0199, 0x00000000}, {0x019a, 0x00000000}, {0x019b, 0x00000000}, {0x019c, 0x00000000}, + {0x019d, 0x00000000}, {0x019e, 0x00000000}, {0x019f, 0x00000000}, {0x01a0, 0x00000000}, + {0x01a1, 0x00000000}, {0x01a2, 0x00000000}, {0x01a3, 0x00000000}, {0x01a4, 0x00000000}, + {0x01a5, 0x00000000}, {0x01a6, 0x00000000}, {0x01a7, 0x00000000}, {0x01a8, 0x00000000}, + {0x01a9, 0x00000000}, {0x01aa, 0x00000000}, {0x01ab, 0x00000000}, {0x01ac, 0x00000000}, + {0x01ad, 0x00000000}, {0x01ae, 0x00000000}, {0x01af, 0x00000000}, {0x01b0, 0x00000000}, + {0x016f, 0x00000000}, {0x0170, 0x00000000}, {0x0171, 0x00000000}, {0x0172, 0x00000000}, + {0x0173, 0x00000000}, {0x0174, 0x00000000}, {0x0175, 0x00000000}, {0x0176, 0x00000000}, + {0x0177, 0x00000000}, {0x0178, 0x00000000}, {0x0179, 0x00000000}, {0x017a, 0x00000000}, + {0x017b, 0x00000000}, {0x017c, 0x00000000}, {0x017d, 0x00000000}, {0x017e, 0x00000000}, + {0x017f, 0x00000000}, {0x0180, 0x00000000}, {0x0181, 0x00000000}, {0x0182, 0x00000000}, + {0x0183, 0x00000000}, {0x0184, 0x00000000}, {0x0185, 0x00000000}, {0x0186, 0x00000000}, + {0x010f, 0x3f800000}, {0x0111, 0x3f800000}, {0x0113, 0x3f800000}, {0x0110, 0x00000000}, + {0x0112, 0x00000000}, {0x0114, 0x00000000}, {0x0094, 0x80000000}, {0x0095, 0x40004000}, + {0x00b4, 0x00000000}, {0x00b5, 0x00000000}, {0x0115, 0x3f800000}, {0x0117, 0x3f800000}, + {0x0119, 0x3f800000}, {0x0116, 0x00000000}, {0x0118, 0x00000000}, {0x011a, 0x00000000}, + {0x0096, 0x80000000}, {0x0097, 0x40004000}, {0x00b6, 0x00000000}, {0x00b7, 0x00000000}, + {0x011b, 0x3f800000}, {0x011d, 0x3f800000}, {0x011f, 0x3f800000}, {0x011c, 0x00000000}, + {0x011e, 0x00000000}, {0x0120, 0x00000000}, {0x0098, 0x80000000}, {0x0099, 0x40004000}, + {0x00b8, 0x00000000}, {0x00b9, 0x00000000}, {0x0121, 0x3f800000}, {0x0123, 0x3f800000}, + {0x0125, 0x3f800000}, {0x0122, 0x00000000}, {0x0124, 0x00000000}, {0x0126, 0x00000000}, + {0x009a, 0x80000000}, {0x009b, 0x40004000}, {0x00ba, 0x00000000}, {0x00bb, 0x00000000}, + {0x0127, 0x3f800000}, {0x0129, 0x3f800000}, {0x012b, 0x3f800000}, {0x0128, 0x00000000}, + {0x012a, 0x00000000}, {0x012c, 0x00000000}, {0x009c, 0x80000000}, {0x009d, 0x40004000}, + {0x00bc, 0x00000000}, {0x00bd, 0x00000000}, {0x012d, 0x3f800000}, {0x012f, 0x3f800000}, + {0x0131, 0x3f800000}, {0x012e, 0x00000000}, {0x0130, 0x00000000}, {0x0132, 0x00000000}, + {0x009e, 0x80000000}, {0x009f, 0x40004000}, {0x00be, 0x00000000}, {0x00bf, 0x00000000}, + {0x0133, 0x3f800000}, {0x0135, 0x3f800000}, {0x0137, 0x3f800000}, {0x0134, 0x00000000}, + {0x0136, 0x00000000}, {0x0138, 0x00000000}, {0x00a0, 0x80000000}, {0x00a1, 0x40004000}, + {0x00c0, 0x00000000}, {0x00c1, 0x00000000}, {0x0139, 0x3f800000}, {0x013b, 0x3f800000}, + {0x013d, 0x3f800000}, {0x013a, 0x00000000}, {0x013c, 0x00000000}, {0x013e, 0x00000000}, + {0x00a2, 0x80000000}, {0x00a3, 0x40004000}, {0x00c2, 0x00000000}, {0x00c3, 0x00000000}, + {0x013f, 0x3f800000}, {0x0141, 0x3f800000}, {0x0143, 0x3f800000}, {0x0140, 0x00000000}, + {0x0142, 0x00000000}, {0x0144, 0x00000000}, {0x00a4, 0x80000000}, {0x00a5, 0x40004000}, + {0x00c4, 0x00000000}, {0x00c5, 0x00000000}, {0x0145, 0x3f800000}, {0x0147, 0x3f800000}, + {0x0149, 0x3f800000}, {0x0146, 0x00000000}, {0x0148, 0x00000000}, {0x014a, 0x00000000}, + {0x00a6, 0x80000000}, {0x00a7, 0x40004000}, {0x00c6, 0x00000000}, {0x00c7, 0x00000000}, + {0x014b, 0x3f800000}, {0x014d, 0x3f800000}, {0x014f, 0x3f800000}, {0x014c, 0x00000000}, + {0x014e, 0x00000000}, {0x0150, 0x00000000}, {0x00a8, 0x80000000}, {0x00a9, 0x40004000}, + {0x00c8, 0x00000000}, {0x00c9, 0x00000000}, {0x0151, 0x3f800000}, {0x0153, 0x3f800000}, + {0x0155, 0x3f800000}, {0x0152, 0x00000000}, {0x0154, 0x00000000}, {0x0156, 0x00000000}, + {0x00aa, 0x80000000}, {0x00ab, 0x40004000}, {0x00ca, 0x00000000}, {0x00cb, 0x00000000}, + {0x0157, 0x3f800000}, {0x0159, 0x3f800000}, {0x015b, 0x3f800000}, {0x0158, 0x00000000}, + {0x015a, 0x00000000}, {0x015c, 0x00000000}, {0x00ac, 0x80000000}, {0x00ad, 0x40004000}, + {0x00cc, 0x00000000}, {0x00cd, 0x00000000}, {0x015d, 0x3f800000}, {0x015f, 0x3f800000}, + {0x0161, 0x3f800000}, {0x015e, 0x00000000}, {0x0160, 0x00000000}, {0x0162, 0x00000000}, + {0x00ae, 0x80000000}, {0x00af, 0x40004000}, {0x00ce, 0x00000000}, {0x00cf, 0x00000000}, + {0x0163, 0x3f800000}, {0x0165, 0x3f800000}, {0x0167, 0x3f800000}, {0x0164, 0x00000000}, + {0x0166, 0x00000000}, {0x0168, 0x00000000}, {0x00b0, 0x80000000}, {0x00b1, 0x40004000}, + {0x00d0, 0x00000000}, {0x00d1, 0x00000000}, {0x0169, 0x3f800000}, {0x016b, 0x3f800000}, + {0x016d, 0x3f800000}, {0x016a, 0x00000000}, {0x016c, 0x00000000}, {0x016e, 0x00000000}, + {0x00b2, 0x80000000}, {0x00b3, 0x40004000}, {0x00d2, 0x00000000}, {0x00d3, 0x00000000}, + {0x0081, 0x80000000}, {0x0082, 0x40004000}, +}; + +static const uint16_t g_agc_public_reg_defaults_v10_tbl0_ptrs[] = { + 0x0000, 0x0001, 0x0002, 0x0003, 0x0004, 0x0005, 0x0006, 0x0007, 0x0008, 0x0009, 0x000a, 0x000b, + 0x000c, 0x000d, 0x000e, 0x000f, 0x0010, 0x0011, 0x0012, 0x0013, 0x0014, 0x0015, 0x0016, 0x0017, + 0x0018, 0x0019, 0x001a, 0x001b, 0x001c, 0x001d, 0x001e, 0x001f, 0x0020, 0x0021, 0x0022, 0x0023, + 0x0024, 0x0025, 0x0026, 0x0027, 0x0028, 0x0029, 0x002a, 0x002b, 0x002c, 0x002d, 0x002e, 0x002f, + 0x0030, 0x0031, 0x0032, 0x0033, 0x0034, 0x0035, 0x0036, 0x0037, 0x0039, 0x0049, 0x004b, 0x004d, + 0x0051, 0x0059, 0x005b, 0x0063, 0x0064, 0x0066, 0x0076, 0x0078, 0x007a, 0x007c, 0x0080, 0x0082, + 0x0084, 0x0086, 0x0106, 0x0108, 0x0128, 0x0140, 0x01e0, +}; + +static ShaderRegister g_agc_public_reg_defaults_v10_tbl1_regs[] = { + {0x0212, 0x00000000}, {0x0213, 0x00000000}, {0x0228, 0x00000000}, {0x0215, 0x00000000}, + {0x0218, 0x00000000}, {0x008a, 0x00000000}, {0x010a, 0x00000000}, {0x000a, 0x00000000}, + {0x008b, 0x00000000}, {0x010b, 0x00000000}, {0x000b, 0x00000000}, {0x0224, 0x00000000}, + {0x0225, 0x00000000}, {0x0226, 0x00000000}, {0x0227, 0x00000000}, {0x0107, 0xffff0000}, + {0x0087, 0x0000ffff}, {0x0007, 0x0000ffff}, {0x020c, 0x00000000}, {0x020d, 0x00000000}, + {0x00c8, 0x00000000}, {0x00c9, 0x00000000}, {0x0088, 0x00000000}, {0x0089, 0x00000000}, + {0x0108, 0x00000000}, {0x0109, 0x00000000}, {0x0148, 0x00000000}, {0x0149, 0x00000000}, + {0x0008, 0x00000000}, {0x0009, 0x00000000}, {0x0280, 0x00000000}, {0x00b2, 0x00000000}, + {0x00b3, 0x00000000}, {0x00b4, 0x00000000}, {0x00b5, 0x00000000}, {0x0132, 0x00000000}, + {0x0133, 0x00000000}, {0x0134, 0x00000000}, {0x0135, 0x00000000}, {0x0032, 0x00000000}, + {0x0033, 0x00000000}, {0x0034, 0x00000000}, {0x0035, 0x00000000}, {0x0082, 0x00000000}, + {0x0083, 0x00000000}, {0x0102, 0x00000000}, {0x0103, 0x00000000}, {0x0240, 0x00000000}, + {0x0241, 0x00000000}, {0x0242, 0x00000000}, {0x0243, 0x00000000}, {0x0244, 0x00000000}, + {0x0245, 0x00000000}, {0x0246, 0x00000000}, {0x0247, 0x00000000}, {0x0248, 0x00000000}, + {0x0249, 0x00000000}, {0x024a, 0x00000000}, {0x024b, 0x00000000}, {0x024c, 0x00000000}, + {0x024d, 0x00000000}, {0x024e, 0x00000000}, {0x024f, 0x00000000}, {0x008c, 0x00000000}, + {0x008d, 0x00000000}, {0x008e, 0x00000000}, {0x008f, 0x00000000}, {0x0090, 0x00000000}, + {0x0091, 0x00000000}, {0x0092, 0x00000000}, {0x0093, 0x00000000}, {0x0094, 0x00000000}, + {0x0095, 0x00000000}, {0x0096, 0x00000000}, {0x0097, 0x00000000}, {0x0098, 0x00000000}, + {0x0099, 0x00000000}, {0x009a, 0x00000000}, {0x009b, 0x00000000}, {0x009c, 0x00000000}, + {0x009d, 0x00000000}, {0x009e, 0x00000000}, {0x009f, 0x00000000}, {0x00a0, 0x00000000}, + {0x00a1, 0x00000000}, {0x00a2, 0x00000000}, {0x00a3, 0x00000000}, {0x00a4, 0x00000000}, + {0x00a5, 0x00000000}, {0x00a6, 0x00000000}, {0x00a7, 0x00000000}, {0x00a8, 0x00000000}, + {0x00a9, 0x00000000}, {0x00aa, 0x00000000}, {0x00ab, 0x00000000}, {0x010c, 0x00000000}, + {0x010d, 0x00000000}, {0x010e, 0x00000000}, {0x010f, 0x00000000}, {0x0110, 0x00000000}, + {0x0111, 0x00000000}, {0x0112, 0x00000000}, {0x0113, 0x00000000}, {0x0114, 0x00000000}, + {0x0115, 0x00000000}, {0x0116, 0x00000000}, {0x0117, 0x00000000}, {0x0118, 0x00000000}, + {0x0119, 0x00000000}, {0x011a, 0x00000000}, {0x011b, 0x00000000}, {0x011c, 0x00000000}, + {0x011d, 0x00000000}, {0x011e, 0x00000000}, {0x011f, 0x00000000}, {0x0120, 0x00000000}, + {0x0121, 0x00000000}, {0x0122, 0x00000000}, {0x0123, 0x00000000}, {0x0124, 0x00000000}, + {0x0125, 0x00000000}, {0x0126, 0x00000000}, {0x0127, 0x00000000}, {0x0128, 0x00000000}, + {0x0129, 0x00000000}, {0x012a, 0x00000000}, {0x012b, 0x00000000}, {0x000c, 0x00000000}, + {0x000d, 0x00000000}, {0x000e, 0x00000000}, {0x000f, 0x00000000}, {0x0010, 0x00000000}, + {0x0011, 0x00000000}, {0x0012, 0x00000000}, {0x0013, 0x00000000}, {0x0014, 0x00000000}, + {0x0015, 0x00000000}, {0x0016, 0x00000000}, {0x0017, 0x00000000}, {0x0018, 0x00000000}, + {0x0019, 0x00000000}, {0x001a, 0x00000000}, {0x001b, 0x00000000}, {0x001c, 0x00000000}, + {0x001d, 0x00000000}, {0x001e, 0x00000000}, {0x001f, 0x00000000}, {0x0020, 0x00000000}, + {0x0021, 0x00000000}, {0x0022, 0x00000000}, {0x0023, 0x00000000}, {0x0024, 0x00000000}, + {0x0025, 0x00000000}, {0x0026, 0x00000000}, {0x0027, 0x00000000}, {0x0028, 0x00000000}, + {0x0029, 0x00000000}, {0x002a, 0x00000000}, {0x002b, 0x00000000}, +}; + +static const uint16_t g_agc_public_reg_defaults_v10_tbl1_ptrs[] = { + 0x0000, 0x0001, 0x0002, 0x0003, 0x0004, 0x0005, 0x0006, 0x0007, 0x0008, 0x0009, 0x000a, 0x000b, + 0x000f, 0x0012, 0x0014, 0x0016, 0x0018, 0x001a, 0x001c, 0x001e, 0x001f, 0x0023, 0x0027, 0x002b, + 0x002d, 0x002f, 0x003f, 0x005f, 0x007f, +}; + +static ShaderRegister g_agc_public_reg_defaults_v10_tbl2_regs[] = { + {0x041f, 0x00000000}, {0x041d, 0x00000000}, {0x041e, 0x00000000}, {0x025b, 0x00000000}, + {0x024a, 0x00000000}, {0x024b, 0x00000000}, {0x025f, 0x00000000}, {0x0262, 0x00000000}, + {0x02bb, 0x00000000}, {0x0384, 0x00000000}, {0x0383, 0x40000040}, {0x0248, 0x00000000}, + {0x0242, 0x00000000}, {0x0380, 0x00000000}, {0x0381, 0x00000000}, {0x02e4, 0x00000000}, + {0x02e5, 0x00000000}, {0x02e6, 0x00000000}, {0x02e7, 0x00000000}, {0x02e8, 0x00000000}, + {0x02e9, 0x00000000}, {0x02ea, 0x00000000}, {0x02eb, 0x00000000}, {0x02c4, 0x00000000}, + {0x02c5, 0x00000000}, {0x02c6, 0x00000000}, {0x02c7, 0x00000000}, {0x02c8, 0x00000000}, + {0x02c9, 0x00000000}, {0x02ca, 0x00000000}, {0x02cb, 0x00000000}, {0x02cc, 0x00000000}, + {0x02cd, 0x00000000}, {0x02ce, 0x00000000}, {0x02cf, 0x00000000}, {0x02d0, 0x00000000}, + {0x02d1, 0x00000000}, {0x02d2, 0x00000000}, {0x02d3, 0x00000000}, {0x02bc, 0x00000000}, + {0x02bd, 0x00000000}, {0x02be, 0x00000000}, {0x02bf, 0x00000000}, {0x0035, 0x00000000}, + {0x0036, 0x00000000}, {0x0037, 0x00000000}, {0x0038, 0x00000000}, {0x0039, 0x00000000}, + {0x003a, 0x00000000}, {0x003b, 0x00000000}, {0x003c, 0x00000000}, {0x00a2, 0x00000000}, + {0x025c, 0x00000000}, {0x025d, 0x00000000}, {0x025e, 0x00000000}, +}; + +static const uint16_t g_agc_public_reg_defaults_v10_tbl2_ptrs[] = { + 0x0000, 0x0001, 0x0002, 0x0003, 0x0004, 0x0005, 0x0006, 0x0007, 0x0008, 0x0009, 0x000a, 0x000b, + 0x000c, 0x000d, 0x000f, 0x0017, 0x0027, 0x002b, 0x0033, 0x0034, +}; + +static uint32_t g_agc_public_reg_defaults_v10_types[] = { + 0xe24f806d, 0x00040400, 0x00000000, 0xf6c28182, 0x00040404, 0x00000000, + 0xa6d12629, 0x00040408, 0x00000000, 0x6f6e55a5, 0x0004040c, 0x00000000, + 0x0bc65da4, 0x00040410, 0x00000000, 0x9e5ad592, 0x00040414, 0x00000000, + 0xbb513b98, 0x00040418, 0x00000000, 0xab64b23b, 0x0004041c, 0x00000000, + 0x53c39964, 0x00040420, 0x00000000, 0x01396b11, 0x00040424, 0x00000000, + 0x7d42019a, 0x00040428, 0x00000000, 0x3548f523, 0x0004042c, 0x00000000, + 0xf43ad28a, 0x00040430, 0x00000000, 0x6de4c312, 0x00040434, 0x00000000, + 0x00a77ae0, 0x00040438, 0x00000000, 0x00a779b7, 0x0004043c, 0x00000000, + 0x5100100c, 0x00040440, 0x00000000, 0x59958bba, 0x00040444, 0x00000000, + 0x0c06f17c, 0x00040448, 0x00000000, 0x6f104b72, 0x0004044c, 0x00000000, + 0x25c70d9c, 0x00040450, 0x00000000, 0x3881201e, 0x00040454, 0x00000000, + 0x09afddaf, 0x00040458, 0x00000000, 0x367d63cf, 0x0004045c, 0x00000000, + 0x43707db8, 0x00040460, 0x00000000, 0xf6ae26ba, 0x00040464, 0x00000000, + 0x1b917652, 0x00040468, 0x00000000, 0x94b1e4f7, 0x0004046c, 0x00000000, + 0xe3661b6c, 0x00040470, 0x00000000, 0x1eb8d73a, 0x00040474, 0x00000000, + 0x15051fa3, 0x00040478, 0x00000000, 0x9c51a7f1, 0x0004047c, 0x00000000, + 0xa20efc70, 0x00040480, 0x00000000, 0x0ec09f6e, 0x00040484, 0x00000000, + 0x34a7d6d3, 0x00040488, 0x00000000, 0xce831b94, 0x0004048c, 0x00000000, + 0x5cc72a74, 0x00040490, 0x00000000, 0x3b77713c, 0x00040494, 0x00000000, + 0x40f64410, 0x00040498, 0x00000000, 0x69441268, 0x0004049c, 0x00000000, + 0x2e418b83, 0x000404a0, 0x00000000, 0xa00d0c8d, 0x000404a4, 0x00000000, + 0xb1289fb3, 0x000404a8, 0x00000000, 0x144832fb, 0x000404ac, 0x00000000, + 0x9890d9fa, 0x000404b0, 0x00000000, 0x9016faf1, 0x000404b4, 0x00000000, + 0x4b73ce27, 0x000404b8, 0x00000000, 0x5f5a3e7b, 0x000404bc, 0x00000000, + 0xd4af3a51, 0x000404c0, 0x00000000, 0x6cf4f543, 0x000404c4, 0x00000000, + 0x5fb86ccb, 0x000404c8, 0x00000000, 0xedefa188, 0x000404cc, 0x00000000, + 0xd0de9ee6, 0x000404d0, 0x00000000, 0xc5831803, 0x000404d4, 0x00000000, + 0x8e6de84b, 0x000404d8, 0x00000000, 0xd0771662, 0x000408dc, 0x00000000, + 0x569f7444, 0x004004e0, 0x00000000, 0x5c6637cd, 0x000408e4, 0x00000000, + 0xcae3e690, 0x000408e8, 0x00000000, 0x43fbd769, 0x000410ec, 0x00000000, + 0xef550356, 0x002004f0, 0x00000000, 0x8f52e279, 0x000408f4, 0x00000000, + 0x1f2d8149, 0x001008f8, 0x00000000, 0x853d0614, 0x000404fc, 0x00000000, + 0x4413c6f9, 0x00040900, 0x00000000, 0x67096014, 0x00044104, 0x00000000, + 0x88f5e915, 0x00040908, 0x00000000, 0x033f1eff, 0x0004090c, 0x00000000, + 0x918106bb, 0x00040910, 0x00000000, 0x95f0e7ac, 0x00041114, 0x00000000, + 0xb48cbab2, 0x00040918, 0x00000000, 0x05bb3bc6, 0x0004091c, 0x00000000, + 0x94faba07, 0x00040920, 0x00000000, 0x38e92c91, 0x00204124, 0x00000000, + 0x0b177b43, 0x00040928, 0x00000000, 0x48531062, 0x0080052c, 0x00000000, + 0xaaa964b9, 0x00181130, 0x00000000, 0x7690af6f, 0x00402934, 0x00000000, + 0x078d7060, 0x00040938, 0x00000000, 0x5d6e3ec7, 0x00040401, 0x00000000, + 0x57e7079a, 0x00040405, 0x00000000, 0x7467fafd, 0x00040409, 0x00000000, + 0x9e826b50, 0x0004040d, 0x00000000, 0xdc484f18, 0x00040411, 0x00000000, + 0x5da8bca3, 0x00040415, 0x00000000, 0x5ca726d8, 0x00040419, 0x00000000, + 0x5dd28360, 0x0004041d, 0x00000000, 0x57efa0be, 0x00040421, 0x00000000, + 0x502363d5, 0x00040425, 0x00000000, 0x506d14bd, 0x00040429, 0x00000000, + 0xb2609506, 0x0010042d, 0x00000000, 0x9e5cfb8a, 0x00040c31, 0x00000000, + 0xc918df3e, 0x00040835, 0x00000000, 0xc9751c9c, 0x00040839, 0x00000000, + 0xc97ef77a, 0x0004083d, 0x00000000, 0xc927c6b9, 0x00040841, 0x00000000, + 0xc92a1ec5, 0x00040845, 0x00000000, 0xc9e01b31, 0x00040849, 0x00000000, + 0x50685f29, 0x0004044d, 0x00000000, 0xb26219ca, 0x00100451, 0x00000000, + 0xb25b6cf9, 0x00100455, 0x00000000, 0xb2f86101, 0x00100459, 0x00000000, + 0x07e3b155, 0x0004085d, 0x00000000, 0x07e383c6, 0x00040861, 0x00000000, + 0xbda98653, 0x00400465, 0x00000000, 0xbdbd1d0f, 0x00800469, 0x00000000, + 0xbd946fd4, 0x0080046d, 0x00000000, 0xbdf02a4c, 0x00800471, 0x00000000, + 0x19e93e85, 0x00040402, 0x00000000, 0x3b5c2af3, 0x00040406, 0x00000000, + 0x47974a35, 0x0004040a, 0x00000000, 0x105971c2, 0x0004040e, 0x00000000, + 0x7d137765, 0x00040412, 0x00000000, 0xd187febc, 0x00040416, 0x00000000, + 0x12f854ac, 0x0004041a, 0x00000000, 0x40d49ad1, 0x0004041e, 0x00000000, + 0x8c0923da, 0x00040422, 0x00000000, 0xbb8df494, 0x00040426, 0x00000000, + 0xf6d8a76e, 0x0004042a, 0x00000000, 0x7620f1e9, 0x0004042e, 0x00000000, + 0x9ebfab10, 0x00040432, 0x00000000, 0x98a09d0e, 0x00040836, 0x00000000, + 0x195d37d2, 0x0010083a, 0x00000000, 0xf9ec4f85, 0x0010103e, 0x00000000, + 0x4626b750, 0x00080842, 0x00000000, 0x4cc673a0, 0x00200446, 0x00000000, + 0xde5b3431, 0x0004044a, 0x00000000, 0x036ac8a6, 0x000c044e, 0x00000000, +}; + +static CompactRegisterDefaults g_agc_public_reg_defaults_v10 = { + g_agc_public_reg_defaults_v10_tbl0_regs, 482, g_agc_public_reg_defaults_v10_tbl0_ptrs, 79, + g_agc_public_reg_defaults_v10_tbl1_regs, 159, g_agc_public_reg_defaults_v10_tbl1_ptrs, 29, + g_agc_public_reg_defaults_v10_tbl2_regs, 55, g_agc_public_reg_defaults_v10_tbl2_ptrs, 20, + nullptr, 0, nullptr, 0, + g_agc_public_reg_defaults_v10_types, 128, +}; + +// g_agc_public_reg_defaults_v11: regs=(488, 162, 56, 0), ptrs=(84, 32, 21, 0), types=137. +static ShaderRegister g_agc_public_reg_defaults_v11_tbl0_regs[] = { + {0x0202, 0x00cc0010}, {0x0109, 0x00000010}, {0x01e0, 0x20010001}, {0x0104, 0x00000000}, + {0x008f, 0x00000000}, {0x008e, 0x00000000}, {0x02dc, 0x0000aa00}, {0x0001, 0x11000100}, + {0x0200, 0x00000000}, {0x0201, 0x00000000}, {0x02af, 0x00040000}, {0x0000, 0x00000000}, + {0x0006, 0x00000000}, {0x001f, 0x00000000}, {0x0203, 0x00000000}, {0x00db, 0x00000000}, + {0x02b0, 0x00000000}, {0x02b1, 0x00000000}, {0x010c, 0x00000000}, {0x010d, 0x00000000}, + {0x010b, 0x00000000}, {0x00db, 0x00000000}, {0x01ff, 0x00000000}, {0x0204, 0x00000000}, + {0x020d, 0x00000000}, {0x00db, 0x00002000}, {0x0206, 0x0000043f}, {0x02f8, 0x00000000}, + {0x0083, 0x0000ffff}, {0x0313, 0x00006000}, {0x00f0, 0x00000000}, {0x00ea, 0x00000000}, + {0x00e9, 0x00000000}, {0x0292, 0x00000002}, {0x0293, 0x06020000}, {0x00e8, 0x00000000}, + {0x0080, 0x00000000}, {0x0211, 0x00000000}, {0x0210, 0x04000000}, {0x008d, 0x00000000}, + {0x0282, 0x00000008}, {0x0281, 0xffff0000}, {0x0280, 0x00080008}, {0x02df, 0x00000000}, + {0x02de, 0x000001e9}, {0x0205, 0x00000240}, {0x020c, 0x00000021}, {0x02f9, 0x0000002d}, + {0x00db, 0x00000000}, {0x01ba, 0x00000000}, {0x02a6, 0x00000000}, {0x02ce, 0x00000400}, + {0x029b, 0x00000002}, {0x02d6, 0x00000000}, {0x0103, 0xffffffff}, {0x02a1, 0x00000000}, + {0x02ad, 0x00000000}, {0x02d5, 0x00002000}, {0x02d4, 0x88101000}, {0x02db, 0x00000000}, + {0x02f5, 0x00000000}, {0x02f6, 0x00000000}, {0x02fe, 0x00000000}, {0x02ff, 0x00000000}, + {0x0300, 0x00000000}, {0x0301, 0x00000000}, {0x0302, 0x00000000}, {0x0303, 0x00000000}, + {0x0304, 0x00000000}, {0x0305, 0x00000000}, {0x0306, 0x00000000}, {0x0307, 0x00000000}, + {0x0308, 0x00000000}, {0x0309, 0x00000000}, {0x030a, 0x00000000}, {0x030b, 0x00000000}, + {0x030c, 0x00000000}, {0x030d, 0x00000000}, {0x030e, 0xffffffff}, {0x030f, 0xffffffff}, + {0x0311, 0x01fd2002}, {0x0312, 0x03ff0080}, {0x0105, 0x00000000}, {0x0106, 0x00000000}, + {0x0107, 0x00000000}, {0x0108, 0x00000000}, {0x01e0, 0x20010001}, {0x01e1, 0x20010001}, + {0x01e2, 0x20010001}, {0x01e3, 0x20010001}, {0x01e4, 0x20010001}, {0x01e5, 0x20010001}, + {0x01e6, 0x20010001}, {0x01e7, 0x20010001}, {0x0020, 0x00000000}, {0x0021, 0x00000000}, + {0x0084, 0x00000000}, {0x0085, 0x20002000}, {0x0086, 0x00000000}, {0x0087, 0x20002000}, + {0x0088, 0x00000000}, {0x0089, 0x20002000}, {0x008a, 0x00000000}, {0x008b, 0x20002000}, + {0x00db, 0x00000000}, {0x0008, 0x00000000}, {0x0009, 0x00000000}, {0x0010, 0x80000180}, + {0x0011, 0x20000180}, {0x0012, 0x00000000}, {0x0013, 0x00000000}, {0x0014, 0x00000000}, + {0x0015, 0x00000000}, {0x001a, 0x00000000}, {0x001b, 0x00000000}, {0x001c, 0x00000000}, + {0x001d, 0x00000000}, {0x001e, 0x00000000}, {0x0002, 0x00000000}, {0x0005, 0x00000000}, + {0x02af, 0x00040000}, {0x0007, 0x00000000}, {0x000b, 0x00000000}, {0x000a, 0x00000000}, + {0x00eb, 0xff00ff00}, {0x00ec, 0x00010000}, {0x00f1, 0x00000000}, {0x00f2, 0x00000000}, + {0x0090, 0x80000000}, {0x0091, 0x40004000}, {0x02fa, 0x3f800000}, {0x02fb, 0x3f800000}, + {0x02fc, 0x3f800000}, {0x02fd, 0x3f800000}, {0x02e2, 0x00000000}, {0x02e3, 0x00000000}, + {0x02e0, 0x00000000}, {0x02e1, 0x00000000}, {0x0003, 0x00000000}, {0x0004, 0x00000000}, + {0x0318, 0x00000000}, {0x031b, 0x00000000}, {0x031c, 0x00000000}, {0x031d, 0x00000000}, + {0x031e, 0x00000048}, {0x031f, 0x00000000}, {0x0321, 0x00000000}, {0x0323, 0x00000000}, + {0x0324, 0x00000000}, {0x0325, 0x00000000}, {0x0390, 0x00000000}, {0x0398, 0x00000000}, + {0x03a0, 0x00000000}, {0x03a8, 0x00000000}, {0x03b0, 0x00000000}, {0x03b8, 0x08c6c000}, + {0x0327, 0x00000000}, {0x032a, 0x00000000}, {0x032b, 0x00000000}, {0x032c, 0x00000000}, + {0x032d, 0x00000048}, {0x032e, 0x00000000}, {0x0330, 0x00000000}, {0x0332, 0x00000000}, + {0x0333, 0x00000000}, {0x0334, 0x00000000}, {0x0391, 0x00000000}, {0x0399, 0x00000000}, + {0x03a1, 0x00000000}, {0x03a9, 0x00000000}, {0x03b1, 0x00000000}, {0x03b9, 0x08c6c000}, + {0x0336, 0x00000000}, {0x0339, 0x00000000}, {0x033a, 0x00000000}, {0x033b, 0x00000000}, + {0x033c, 0x00000048}, {0x033d, 0x00000000}, {0x033f, 0x00000000}, {0x0341, 0x00000000}, + {0x0342, 0x00000000}, {0x0343, 0x00000000}, {0x0392, 0x00000000}, {0x039a, 0x00000000}, + {0x03a2, 0x00000000}, {0x03aa, 0x00000000}, {0x03b2, 0x00000000}, {0x03ba, 0x08c6c000}, + {0x0345, 0x00000000}, {0x0348, 0x00000000}, {0x0349, 0x00000000}, {0x034a, 0x00000000}, + {0x034b, 0x00000048}, {0x034c, 0x00000000}, {0x034e, 0x00000000}, {0x0350, 0x00000000}, + {0x0351, 0x00000000}, {0x0352, 0x00000000}, {0x0393, 0x00000000}, {0x039b, 0x00000000}, + {0x03a3, 0x00000000}, {0x03ab, 0x00000000}, {0x03b3, 0x00000000}, {0x03bb, 0x08c6c000}, + {0x0354, 0x00000000}, {0x0357, 0x00000000}, {0x0358, 0x00000000}, {0x0359, 0x00000000}, + {0x035a, 0x00000048}, {0x035b, 0x00000000}, {0x035d, 0x00000000}, {0x035f, 0x00000000}, + {0x0360, 0x00000000}, {0x0361, 0x00000000}, {0x0394, 0x00000000}, {0x039c, 0x00000000}, + {0x03a4, 0x00000000}, {0x03ac, 0x00000000}, {0x03b4, 0x00000000}, {0x03bc, 0x08c6c000}, + {0x0363, 0x00000000}, {0x0366, 0x00000000}, {0x0367, 0x00000000}, {0x0368, 0x00000000}, + {0x0369, 0x00000048}, {0x036a, 0x00000000}, {0x036c, 0x00000000}, {0x036e, 0x00000000}, + {0x036f, 0x00000000}, {0x0370, 0x00000000}, {0x0395, 0x00000000}, {0x039d, 0x00000000}, + {0x03a5, 0x00000000}, {0x03ad, 0x00000000}, {0x03b5, 0x00000000}, {0x03bd, 0x08c6c000}, + {0x0372, 0x00000000}, {0x0375, 0x00000000}, {0x0376, 0x00000000}, {0x0377, 0x00000000}, + {0x0378, 0x00000048}, {0x0379, 0x00000000}, {0x037b, 0x00000000}, {0x037d, 0x00000000}, + {0x037e, 0x00000000}, {0x037f, 0x00000000}, {0x0396, 0x00000000}, {0x039e, 0x00000000}, + {0x03a6, 0x00000000}, {0x03ae, 0x00000000}, {0x03b6, 0x00000000}, {0x03be, 0x08c6c000}, + {0x0381, 0x00000000}, {0x0384, 0x00000000}, {0x0385, 0x00000000}, {0x0386, 0x00000000}, + {0x0387, 0x00000048}, {0x0388, 0x00000000}, {0x038a, 0x00000000}, {0x038c, 0x00000000}, + {0x038d, 0x00000000}, {0x038e, 0x00000000}, {0x0397, 0x00000000}, {0x039f, 0x00000000}, + {0x03a7, 0x00000000}, {0x03af, 0x00000000}, {0x03b7, 0x00000000}, {0x03bf, 0x08c6c000}, + {0x000c, 0x00000000}, {0x000d, 0x40004000}, {0x0191, 0x00000000}, {0x0192, 0x00000000}, + {0x0193, 0x00000000}, {0x0194, 0x00000000}, {0x0195, 0x00000000}, {0x0196, 0x00000000}, + {0x0197, 0x00000000}, {0x0198, 0x00000000}, {0x0199, 0x00000000}, {0x019a, 0x00000000}, + {0x019b, 0x00000000}, {0x019c, 0x00000000}, {0x019d, 0x00000000}, {0x019e, 0x00000000}, + {0x019f, 0x00000000}, {0x01a0, 0x00000000}, {0x01a1, 0x00000000}, {0x01a2, 0x00000000}, + {0x01a3, 0x00000000}, {0x01a4, 0x00000000}, {0x01a5, 0x00000000}, {0x01a6, 0x00000000}, + {0x01a7, 0x00000000}, {0x01a8, 0x00000000}, {0x01a9, 0x00000000}, {0x01aa, 0x00000000}, + {0x01ab, 0x00000000}, {0x01ac, 0x00000000}, {0x01ad, 0x00000000}, {0x01ae, 0x00000000}, + {0x01af, 0x00000000}, {0x01b0, 0x00000000}, {0x016f, 0x00000000}, {0x0170, 0x00000000}, + {0x0171, 0x00000000}, {0x0172, 0x00000000}, {0x0173, 0x00000000}, {0x0174, 0x00000000}, + {0x0175, 0x00000000}, {0x0176, 0x00000000}, {0x0177, 0x00000000}, {0x0178, 0x00000000}, + {0x0179, 0x00000000}, {0x017a, 0x00000000}, {0x017b, 0x00000000}, {0x017c, 0x00000000}, + {0x017d, 0x00000000}, {0x017e, 0x00000000}, {0x017f, 0x00000000}, {0x0180, 0x00000000}, + {0x0181, 0x00000000}, {0x0182, 0x00000000}, {0x0183, 0x00000000}, {0x0184, 0x00000000}, + {0x0185, 0x00000000}, {0x0186, 0x00000000}, {0x010f, 0x3f800000}, {0x0111, 0x3f800000}, + {0x0113, 0x3f800000}, {0x0110, 0x00000000}, {0x0112, 0x00000000}, {0x0114, 0x00000000}, + {0x0094, 0x80000000}, {0x0095, 0x40004000}, {0x00b4, 0x00000000}, {0x00b5, 0x00000000}, + {0x0115, 0x3f800000}, {0x0117, 0x3f800000}, {0x0119, 0x3f800000}, {0x0116, 0x00000000}, + {0x0118, 0x00000000}, {0x011a, 0x00000000}, {0x0096, 0x80000000}, {0x0097, 0x40004000}, + {0x00b6, 0x00000000}, {0x00b7, 0x00000000}, {0x011b, 0x3f800000}, {0x011d, 0x3f800000}, + {0x011f, 0x3f800000}, {0x011c, 0x00000000}, {0x011e, 0x00000000}, {0x0120, 0x00000000}, + {0x0098, 0x80000000}, {0x0099, 0x40004000}, {0x00b8, 0x00000000}, {0x00b9, 0x00000000}, + {0x0121, 0x3f800000}, {0x0123, 0x3f800000}, {0x0125, 0x3f800000}, {0x0122, 0x00000000}, + {0x0124, 0x00000000}, {0x0126, 0x00000000}, {0x009a, 0x80000000}, {0x009b, 0x40004000}, + {0x00ba, 0x00000000}, {0x00bb, 0x00000000}, {0x0127, 0x3f800000}, {0x0129, 0x3f800000}, + {0x012b, 0x3f800000}, {0x0128, 0x00000000}, {0x012a, 0x00000000}, {0x012c, 0x00000000}, + {0x009c, 0x80000000}, {0x009d, 0x40004000}, {0x00bc, 0x00000000}, {0x00bd, 0x00000000}, + {0x012d, 0x3f800000}, {0x012f, 0x3f800000}, {0x0131, 0x3f800000}, {0x012e, 0x00000000}, + {0x0130, 0x00000000}, {0x0132, 0x00000000}, {0x009e, 0x80000000}, {0x009f, 0x40004000}, + {0x00be, 0x00000000}, {0x00bf, 0x00000000}, {0x0133, 0x3f800000}, {0x0135, 0x3f800000}, + {0x0137, 0x3f800000}, {0x0134, 0x00000000}, {0x0136, 0x00000000}, {0x0138, 0x00000000}, + {0x00a0, 0x80000000}, {0x00a1, 0x40004000}, {0x00c0, 0x00000000}, {0x00c1, 0x00000000}, + {0x0139, 0x3f800000}, {0x013b, 0x3f800000}, {0x013d, 0x3f800000}, {0x013a, 0x00000000}, + {0x013c, 0x00000000}, {0x013e, 0x00000000}, {0x00a2, 0x80000000}, {0x00a3, 0x40004000}, + {0x00c2, 0x00000000}, {0x00c3, 0x00000000}, {0x013f, 0x3f800000}, {0x0141, 0x3f800000}, + {0x0143, 0x3f800000}, {0x0140, 0x00000000}, {0x0142, 0x00000000}, {0x0144, 0x00000000}, + {0x00a4, 0x80000000}, {0x00a5, 0x40004000}, {0x00c4, 0x00000000}, {0x00c5, 0x00000000}, + {0x0145, 0x3f800000}, {0x0147, 0x3f800000}, {0x0149, 0x3f800000}, {0x0146, 0x00000000}, + {0x0148, 0x00000000}, {0x014a, 0x00000000}, {0x00a6, 0x80000000}, {0x00a7, 0x40004000}, + {0x00c6, 0x00000000}, {0x00c7, 0x00000000}, {0x014b, 0x3f800000}, {0x014d, 0x3f800000}, + {0x014f, 0x3f800000}, {0x014c, 0x00000000}, {0x014e, 0x00000000}, {0x0150, 0x00000000}, + {0x00a8, 0x80000000}, {0x00a9, 0x40004000}, {0x00c8, 0x00000000}, {0x00c9, 0x00000000}, + {0x0151, 0x3f800000}, {0x0153, 0x3f800000}, {0x0155, 0x3f800000}, {0x0152, 0x00000000}, + {0x0154, 0x00000000}, {0x0156, 0x00000000}, {0x00aa, 0x80000000}, {0x00ab, 0x40004000}, + {0x00ca, 0x00000000}, {0x00cb, 0x00000000}, {0x0157, 0x3f800000}, {0x0159, 0x3f800000}, + {0x015b, 0x3f800000}, {0x0158, 0x00000000}, {0x015a, 0x00000000}, {0x015c, 0x00000000}, + {0x00ac, 0x80000000}, {0x00ad, 0x40004000}, {0x00cc, 0x00000000}, {0x00cd, 0x00000000}, + {0x015d, 0x3f800000}, {0x015f, 0x3f800000}, {0x0161, 0x3f800000}, {0x015e, 0x00000000}, + {0x0160, 0x00000000}, {0x0162, 0x00000000}, {0x00ae, 0x80000000}, {0x00af, 0x40004000}, + {0x00ce, 0x00000000}, {0x00cf, 0x00000000}, {0x0163, 0x3f800000}, {0x0165, 0x3f800000}, + {0x0167, 0x3f800000}, {0x0164, 0x00000000}, {0x0166, 0x00000000}, {0x0168, 0x00000000}, + {0x00b0, 0x80000000}, {0x00b1, 0x40004000}, {0x00d0, 0x00000000}, {0x00d1, 0x00000000}, + {0x0169, 0x3f800000}, {0x016b, 0x3f800000}, {0x016d, 0x3f800000}, {0x016a, 0x00000000}, + {0x016c, 0x00000000}, {0x016e, 0x00000000}, {0x00b2, 0x80000000}, {0x00b3, 0x40004000}, + {0x00d2, 0x00000000}, {0x00d3, 0x00000000}, {0x0081, 0x80000000}, {0x0082, 0x40004000}, +}; + +static const uint16_t g_agc_public_reg_defaults_v11_tbl0_ptrs[] = { + 0x0000, 0x0001, 0x0002, 0x0003, 0x0004, 0x0005, 0x0006, 0x0007, 0x0008, 0x0009, 0x000a, 0x000b, + 0x000c, 0x000d, 0x000e, 0x000f, 0x0010, 0x0011, 0x0012, 0x0013, 0x0014, 0x0015, 0x0016, 0x0017, + 0x0018, 0x0019, 0x001a, 0x001b, 0x001c, 0x001d, 0x001e, 0x001f, 0x0020, 0x0021, 0x0022, 0x0023, + 0x0024, 0x0025, 0x0026, 0x0027, 0x0028, 0x0029, 0x002a, 0x002b, 0x002c, 0x002d, 0x002e, 0x002f, + 0x0030, 0x0031, 0x0032, 0x0033, 0x0034, 0x0035, 0x0036, 0x0037, 0x0038, 0x0039, 0x003a, 0x003b, + 0x003c, 0x003e, 0x004e, 0x0050, 0x0052, 0x0056, 0x005e, 0x0060, 0x0068, 0x0069, 0x006b, 0x007c, + 0x007e, 0x0080, 0x0082, 0x0086, 0x0088, 0x008a, 0x008c, 0x010c, 0x010e, 0x012e, 0x0146, 0x01e6, +}; + +static ShaderRegister g_agc_public_reg_defaults_v11_tbl1_regs[] = { + {0x0212, 0x00000000}, {0x0213, 0x00000000}, {0x0228, 0x00000000}, {0x0215, 0x00000000}, + {0x022a, 0x00ffffff}, {0x0218, 0x00000000}, {0x0080, 0x00ffffff}, {0x0006, 0x00ffffff}, + {0x008a, 0x00000000}, {0x010a, 0x00000000}, {0x000a, 0x00000000}, {0x008b, 0x00000000}, + {0x010b, 0x00000000}, {0x000b, 0x00000000}, {0x0224, 0x00000000}, {0x0225, 0x00000000}, + {0x0226, 0x00000000}, {0x0227, 0x00000000}, {0x0107, 0xffff0000}, {0x0087, 0x0000ffff}, + {0x0007, 0x0000ffff}, {0x020c, 0x00000000}, {0x020d, 0x00000000}, {0x00c8, 0x00000000}, + {0x00c9, 0x00000000}, {0x0088, 0x00000000}, {0x0089, 0x00000000}, {0x0108, 0x00000000}, + {0x0109, 0x00000000}, {0x0148, 0x00000000}, {0x0149, 0x00000000}, {0x0008, 0x00000000}, + {0x0009, 0x00000000}, {0x0280, 0x00000000}, {0x00b2, 0x00000000}, {0x00b3, 0x00000000}, + {0x00b4, 0x00000000}, {0x00b5, 0x00000000}, {0x0132, 0x00000000}, {0x0133, 0x00000000}, + {0x0134, 0x00000000}, {0x0135, 0x00000000}, {0x0032, 0x00000000}, {0x0033, 0x00000000}, + {0x0034, 0x00000000}, {0x0035, 0x00000000}, {0x0082, 0x00000000}, {0x0083, 0x00000000}, + {0x0102, 0x00000000}, {0x0103, 0x00000000}, {0x0240, 0x00000000}, {0x0241, 0x00000000}, + {0x0242, 0x00000000}, {0x0243, 0x00000000}, {0x0244, 0x00000000}, {0x0245, 0x00000000}, + {0x0246, 0x00000000}, {0x0247, 0x00000000}, {0x0248, 0x00000000}, {0x0249, 0x00000000}, + {0x024a, 0x00000000}, {0x024b, 0x00000000}, {0x024c, 0x00000000}, {0x024d, 0x00000000}, + {0x024e, 0x00000000}, {0x024f, 0x00000000}, {0x008c, 0x00000000}, {0x008d, 0x00000000}, + {0x008e, 0x00000000}, {0x008f, 0x00000000}, {0x0090, 0x00000000}, {0x0091, 0x00000000}, + {0x0092, 0x00000000}, {0x0093, 0x00000000}, {0x0094, 0x00000000}, {0x0095, 0x00000000}, + {0x0096, 0x00000000}, {0x0097, 0x00000000}, {0x0098, 0x00000000}, {0x0099, 0x00000000}, + {0x009a, 0x00000000}, {0x009b, 0x00000000}, {0x009c, 0x00000000}, {0x009d, 0x00000000}, + {0x009e, 0x00000000}, {0x009f, 0x00000000}, {0x00a0, 0x00000000}, {0x00a1, 0x00000000}, + {0x00a2, 0x00000000}, {0x00a3, 0x00000000}, {0x00a4, 0x00000000}, {0x00a5, 0x00000000}, + {0x00a6, 0x00000000}, {0x00a7, 0x00000000}, {0x00a8, 0x00000000}, {0x00a9, 0x00000000}, + {0x00aa, 0x00000000}, {0x00ab, 0x00000000}, {0x010c, 0x00000000}, {0x010d, 0x00000000}, + {0x010e, 0x00000000}, {0x010f, 0x00000000}, {0x0110, 0x00000000}, {0x0111, 0x00000000}, + {0x0112, 0x00000000}, {0x0113, 0x00000000}, {0x0114, 0x00000000}, {0x0115, 0x00000000}, + {0x0116, 0x00000000}, {0x0117, 0x00000000}, {0x0118, 0x00000000}, {0x0119, 0x00000000}, + {0x011a, 0x00000000}, {0x011b, 0x00000000}, {0x011c, 0x00000000}, {0x011d, 0x00000000}, + {0x011e, 0x00000000}, {0x011f, 0x00000000}, {0x0120, 0x00000000}, {0x0121, 0x00000000}, + {0x0122, 0x00000000}, {0x0123, 0x00000000}, {0x0124, 0x00000000}, {0x0125, 0x00000000}, + {0x0126, 0x00000000}, {0x0127, 0x00000000}, {0x0128, 0x00000000}, {0x0129, 0x00000000}, + {0x012a, 0x00000000}, {0x012b, 0x00000000}, {0x000c, 0x00000000}, {0x000d, 0x00000000}, + {0x000e, 0x00000000}, {0x000f, 0x00000000}, {0x0010, 0x00000000}, {0x0011, 0x00000000}, + {0x0012, 0x00000000}, {0x0013, 0x00000000}, {0x0014, 0x00000000}, {0x0015, 0x00000000}, + {0x0016, 0x00000000}, {0x0017, 0x00000000}, {0x0018, 0x00000000}, {0x0019, 0x00000000}, + {0x001a, 0x00000000}, {0x001b, 0x00000000}, {0x001c, 0x00000000}, {0x001d, 0x00000000}, + {0x001e, 0x00000000}, {0x001f, 0x00000000}, {0x0020, 0x00000000}, {0x0021, 0x00000000}, + {0x0022, 0x00000000}, {0x0023, 0x00000000}, {0x0024, 0x00000000}, {0x0025, 0x00000000}, + {0x0026, 0x00000000}, {0x0027, 0x00000000}, {0x0028, 0x00000000}, {0x0029, 0x00000000}, + {0x002a, 0x00000000}, {0x002b, 0x00000000}, +}; + +static const uint16_t g_agc_public_reg_defaults_v11_tbl1_ptrs[] = { + 0x0000, 0x0001, 0x0002, 0x0003, 0x0004, 0x0005, 0x0006, 0x0007, 0x0008, 0x0009, 0x000a, 0x000b, + 0x000c, 0x000d, 0x000e, 0x0012, 0x0015, 0x0017, 0x0019, 0x001b, 0x001d, 0x001f, 0x0021, 0x0022, + 0x0026, 0x002a, 0x002e, 0x0030, 0x0032, 0x0042, 0x0062, 0x0082, +}; + +static ShaderRegister g_agc_public_reg_defaults_v11_tbl2_regs[] = { + {0x041f, 0x00000000}, {0x041d, 0x00000000}, {0x041e, 0x00000000}, {0x025b, 0x00000000}, + {0x024a, 0x00000000}, {0x024b, 0x00000000}, {0x025f, 0x00000000}, {0x0262, 0x00000000}, + {0x00a2, 0x00000000}, {0x02bb, 0x00000000}, {0x0384, 0x00000000}, {0x0383, 0x40000040}, + {0x0248, 0x00000000}, {0x0242, 0x00000000}, {0x0380, 0x00000000}, {0x0381, 0x00000000}, + {0x02e4, 0x00000000}, {0x02e5, 0x00000000}, {0x02e6, 0x00000000}, {0x02e7, 0x00000000}, + {0x02e8, 0x00000000}, {0x02e9, 0x00000000}, {0x02ea, 0x00000000}, {0x02eb, 0x00000000}, + {0x02c4, 0x00000000}, {0x02c5, 0x00000000}, {0x02c6, 0x00000000}, {0x02c7, 0x00000000}, + {0x02c8, 0x00000000}, {0x02c9, 0x00000000}, {0x02ca, 0x00000000}, {0x02cb, 0x00000000}, + {0x02cc, 0x00000000}, {0x02cd, 0x00000000}, {0x02ce, 0x00000000}, {0x02cf, 0x00000000}, + {0x02d0, 0x00000000}, {0x02d1, 0x00000000}, {0x02d2, 0x00000000}, {0x02d3, 0x00000000}, + {0x02bc, 0x00000000}, {0x02bd, 0x00000000}, {0x02be, 0x00000000}, {0x02bf, 0x00000000}, + {0x0035, 0x00000000}, {0x0036, 0x00000000}, {0x0037, 0x00000000}, {0x0038, 0x00000000}, + {0x0039, 0x00000000}, {0x003a, 0x00000000}, {0x003b, 0x00000000}, {0x003c, 0x00000000}, + {0x00a2, 0x00000000}, {0x025c, 0x00000000}, {0x025d, 0x00000000}, {0x025e, 0x00000000}, +}; + +static const uint16_t g_agc_public_reg_defaults_v11_tbl2_ptrs[] = { + 0x0000, 0x0001, 0x0002, 0x0003, 0x0004, 0x0005, 0x0006, 0x0007, 0x0008, 0x0009, 0x000a, 0x000b, + 0x000c, 0x000d, 0x000e, 0x0010, 0x0018, 0x0028, 0x002c, 0x0034, 0x0035, +}; + +static uint32_t g_agc_public_reg_defaults_v11_types[] = { + 0xe24f806d, 0x00040400, 0x00000000, 0xf6c28182, 0x00040404, 0x00000000, + 0xa6d12629, 0x00040408, 0x00000000, 0x6f6e55a5, 0x0004040c, 0x00000000, + 0x0bc65da4, 0x00040410, 0x00000000, 0x9e5ad592, 0x00040414, 0x00000000, + 0xbb513b98, 0x00040418, 0x00000000, 0xab64b23b, 0x0004041c, 0x00000000, + 0x53c39964, 0x00040420, 0x00000000, 0x01396b11, 0x00040424, 0x00000000, + 0xb994ad29, 0x00040428, 0x00000000, 0x7d42019a, 0x0004042c, 0x00000000, + 0x3548f523, 0x00040430, 0x00000000, 0xf43ad28a, 0x00040434, 0x00000000, + 0x6de4c312, 0x00040438, 0x00000000, 0x26ae2080, 0x0004043c, 0x00000000, + 0x00a77ae0, 0x00040440, 0x00000000, 0x00a779b7, 0x00040444, 0x00000000, + 0x5100100c, 0x00040448, 0x00000000, 0x59958bba, 0x0004044c, 0x00000000, + 0x0c06f17c, 0x00040450, 0x00000000, 0x9e0feb34, 0x00040454, 0x00000000, + 0x6f104b72, 0x00040458, 0x00000000, 0x25c70d9c, 0x0004045c, 0x00000000, + 0x3881201e, 0x00040460, 0x00000000, 0x381c673c, 0x00040464, 0x00000000, + 0x09afddaf, 0x00040468, 0x00000000, 0x367d63cf, 0x0004046c, 0x00000000, + 0x43707db8, 0x00040470, 0x00000000, 0xf6ae26ba, 0x00040474, 0x00000000, + 0x1b917652, 0x00040478, 0x00000000, 0x94b1e4f7, 0x0004047c, 0x00000000, + 0xe3661b6c, 0x00040480, 0x00000000, 0x1eb8d73a, 0x00040484, 0x00000000, + 0x15051fa3, 0x00040488, 0x00000000, 0x9c51a7f1, 0x0004048c, 0x00000000, + 0xa20efc70, 0x00040490, 0x00000000, 0x0ec09f6e, 0x00040494, 0x00000000, + 0x34a7d6d3, 0x00040498, 0x00000000, 0xce831b94, 0x0004049c, 0x00000000, + 0x5cc72a74, 0x000404a0, 0x00000000, 0x3b77713c, 0x000404a4, 0x00000000, + 0x40f64410, 0x000404a8, 0x00000000, 0x69441268, 0x000404ac, 0x00000000, + 0x2e418b83, 0x000404b0, 0x00000000, 0xa00d0c8d, 0x000404b4, 0x00000000, + 0xb1289fb3, 0x000404b8, 0x00000000, 0x144832fb, 0x000404bc, 0x00000000, + 0xa14657c5, 0x000404c0, 0x00000000, 0x9890d9fa, 0x000404c4, 0x00000000, + 0x9016faf1, 0x000404c8, 0x00000000, 0x4b73ce27, 0x000404cc, 0x00000000, + 0x5f5a3e7b, 0x000404d0, 0x00000000, 0xd4af3a51, 0x000404d4, 0x00000000, + 0x6cf4f543, 0x000404d8, 0x00000000, 0x5fb86ccb, 0x000404dc, 0x00000000, + 0xedefa188, 0x000404e0, 0x00000000, 0xd0de9ee6, 0x000404e4, 0x00000000, + 0xc5831803, 0x000404e8, 0x00000000, 0x8e6de84b, 0x000404ec, 0x00000000, + 0xd0771662, 0x000408f0, 0x00000000, 0x569f7444, 0x004004f4, 0x00000000, + 0x5c6637cd, 0x000408f8, 0x00000000, 0xcae3e690, 0x000408fc, 0x00000000, + 0x43fbd769, 0x00041100, 0x00000000, 0xef550356, 0x00200504, 0x00000000, + 0x8f52e279, 0x00040908, 0x00000000, 0x1f2d8149, 0x0010090c, 0x00000000, + 0x853d0614, 0x00040510, 0x00000000, 0x4413c6f9, 0x00040914, 0x00000000, + 0x67096014, 0x00044518, 0x00000000, 0x88f5e915, 0x0004091c, 0x00000000, + 0x033f1eff, 0x00040920, 0x00000000, 0x918106bb, 0x00040924, 0x00000000, + 0x95f0e7ac, 0x00041128, 0x00000000, 0xb48cbab2, 0x0004092c, 0x00000000, + 0x05bb3bc6, 0x00040930, 0x00000000, 0x94faba07, 0x00040934, 0x00000000, + 0x38e92c91, 0x00204138, 0x00000000, 0x0b177b43, 0x0004093c, 0x00000000, + 0x48531062, 0x00800540, 0x00000000, 0xaaa964b9, 0x00181144, 0x00000000, + 0x7690af6f, 0x00402948, 0x00000000, 0x078d7060, 0x0004094c, 0x00000000, + 0x5d6e3ec7, 0x00040401, 0x00000000, 0x57e7079a, 0x00040405, 0x00000000, + 0x7467fafd, 0x00040409, 0x00000000, 0x9e826b50, 0x0004040d, 0x00000000, + 0xf899653a, 0x00040411, 0x00000000, 0xdc484f18, 0x00040415, 0x00000000, + 0xf9754e16, 0x00040419, 0x00000000, 0xf8e274f5, 0x0004041d, 0x00000000, + 0x5da8bca3, 0x00040421, 0x00000000, 0x5ca726d8, 0x00040425, 0x00000000, + 0x5dd28360, 0x00040429, 0x00000000, 0x57efa0be, 0x0004042d, 0x00000000, + 0x502363d5, 0x00040431, 0x00000000, 0x506d14bd, 0x00040435, 0x00000000, + 0xb2609506, 0x00100439, 0x00000000, 0x9e5cfb8a, 0x00040c3d, 0x00000000, + 0xc918df3e, 0x00040841, 0x00000000, 0xc9751c9c, 0x00040845, 0x00000000, + 0xc97ef77a, 0x00040849, 0x00000000, 0xc927c6b9, 0x0004084d, 0x00000000, + 0xc92a1ec5, 0x00040851, 0x00000000, 0xc9e01b31, 0x00040855, 0x00000000, + 0x50685f29, 0x00040459, 0x00000000, 0xb26219ca, 0x0010045d, 0x00000000, + 0xb25b6cf9, 0x00100461, 0x00000000, 0xb2f86101, 0x00100465, 0x00000000, + 0x07e3b155, 0x00040869, 0x00000000, 0x07e383c6, 0x0004086d, 0x00000000, + 0xbda98653, 0x00400471, 0x00000000, 0xbdbd1d0f, 0x00800475, 0x00000000, + 0xbd946fd4, 0x00800479, 0x00000000, 0xbdf02a4c, 0x0080047d, 0x00000000, + 0x19e93e85, 0x00040402, 0x00000000, 0x3b5c2af3, 0x00040406, 0x00000000, + 0x47974a35, 0x0004040a, 0x00000000, 0x105971c2, 0x0004040e, 0x00000000, + 0x7d137765, 0x00040412, 0x00000000, 0xd187febc, 0x00040416, 0x00000000, + 0x12f854ac, 0x0004041a, 0x00000000, 0x40d49ad1, 0x0004041e, 0x00000000, + 0xe3a37123, 0x00040422, 0x00000000, 0x8c0923da, 0x00040426, 0x00000000, + 0xbb8df494, 0x0004042a, 0x00000000, 0xf6d8a76e, 0x0004042e, 0x00000000, + 0x7620f1e9, 0x00040432, 0x00000000, 0x9ebfab10, 0x00040436, 0x00000000, + 0x98a09d0e, 0x0004083a, 0x00000000, 0x195d37d2, 0x0010083e, 0x00000000, + 0xf9ec4f85, 0x00101042, 0x00000000, 0x4626b750, 0x00080846, 0x00000000, + 0x4cc673a0, 0x0020044a, 0x00000000, 0xde5b3431, 0x0004044e, 0x00000000, + 0x036ac8a6, 0x000c0452, 0x00000000, +}; + +static CompactRegisterDefaults g_agc_public_reg_defaults_v11 = { + g_agc_public_reg_defaults_v11_tbl0_regs, 488, g_agc_public_reg_defaults_v11_tbl0_ptrs, 84, + g_agc_public_reg_defaults_v11_tbl1_regs, 162, g_agc_public_reg_defaults_v11_tbl1_ptrs, 32, + g_agc_public_reg_defaults_v11_tbl2_regs, 56, g_agc_public_reg_defaults_v11_tbl2_ptrs, 21, + nullptr, 0, nullptr, 0, + g_agc_public_reg_defaults_v11_types, 137, +}; + +// g_agc_public_reg_defaults_v4: regs=(488, 158, 43, 0), ptrs=(80, 28, 42, 0), types=150. +static ShaderRegister g_agc_public_reg_defaults_v4_tbl0_regs[] = { + {0x0202, 0x00cc0010}, {0x0109, 0x00000008}, {0x0104, 0x00000000}, {0x008f, 0x00000000}, + {0x008e, 0x00000000}, {0x02dc, 0x0000aa00}, {0x0001, 0x11000100}, {0x000b, 0x00000000}, + {0x0200, 0x00000000}, {0x0201, 0x00000000}, {0x0000, 0x00000000}, {0x0006, 0x00000000}, + {0x001f, 0x00000000}, {0x0203, 0x00000000}, {0x02b0, 0x00000000}, {0x02b1, 0x00000000}, + {0x010c, 0x00000000}, {0x010d, 0x00000000}, {0x000a, 0x00000000}, {0x010b, 0x00000000}, + {0x01ff, 0x00000000}, {0x0204, 0x00000000}, {0x020d, 0x00000000}, {0x0206, 0x0000043f}, + {0x02f8, 0x00000000}, {0x0083, 0x0000ffff}, {0x0313, 0x00006000}, {0x00f0, 0x00000000}, + {0x00f1, 0x00000000}, {0x00f2, 0x00000000}, {0x00ea, 0x00000000}, {0x00e9, 0x00000000}, + {0x0292, 0x00000002}, {0x0293, 0x06020000}, {0x00e8, 0x00000000}, {0x0080, 0x00000000}, + {0x0211, 0x00000000}, {0x0210, 0x00000000}, {0x008d, 0x00000000}, {0x0282, 0x00000008}, + {0x0281, 0xffff0000}, {0x0280, 0x00080008}, {0x02df, 0x00000000}, {0x02de, 0x000001e9}, + {0x0205, 0x00000240}, {0x020c, 0x00000021}, {0x02f9, 0x0000002d}, {0x01ba, 0x00000000}, + {0x02a6, 0x00000000}, {0x02ce, 0x00000400}, {0x029b, 0x00000002}, {0x02d6, 0x00000000}, + {0x0103, 0xffffffff}, {0x02a1, 0x00000000}, {0x02ad, 0x00000000}, {0x02d5, 0x00002000}, + {0x02d4, 0x88101010}, {0x02db, 0x00000000}, {0x02f5, 0x00000000}, {0x02f6, 0x00000000}, + {0x02fe, 0x00000000}, {0x02ff, 0x00000000}, {0x0300, 0x00000000}, {0x0301, 0x00000000}, + {0x0302, 0x00000000}, {0x0303, 0x00000000}, {0x0304, 0x00000000}, {0x0305, 0x00000000}, + {0x0306, 0x00000000}, {0x0307, 0x00000000}, {0x0308, 0x00000000}, {0x0309, 0x00000000}, + {0x030a, 0x00000000}, {0x030b, 0x00000000}, {0x030c, 0x00000000}, {0x030d, 0x00000000}, + {0x030e, 0xffffffff}, {0x030f, 0xffffffff}, {0x0311, 0x01f92000}, {0x0312, 0x03ff0080}, + {0x0105, 0x00000000}, {0x0106, 0x00000000}, {0x0107, 0x00000000}, {0x0108, 0x00000000}, + {0x01e0, 0x20010001}, {0x01e1, 0x20010001}, {0x01e2, 0x20010001}, {0x01e3, 0x20010001}, + {0x01e4, 0x20010001}, {0x01e5, 0x20010001}, {0x01e6, 0x20010001}, {0x01e7, 0x20010001}, + {0x0020, 0x00000000}, {0x0021, 0x00000000}, {0x0084, 0x00000000}, {0x0085, 0x20002000}, + {0x0086, 0x00000000}, {0x0087, 0x20002000}, {0x0088, 0x00000000}, {0x0089, 0x20002000}, + {0x008a, 0x00000000}, {0x008b, 0x20002000}, {0x0008, 0x00000000}, {0x0009, 0x00000000}, + {0x0010, 0x80000180}, {0x0011, 0x20000180}, {0x0012, 0x00000000}, {0x0013, 0x00000000}, + {0x0014, 0x00000000}, {0x0015, 0x00000000}, {0x001a, 0x00000000}, {0x001b, 0x00000000}, + {0x001c, 0x00000000}, {0x001d, 0x00000000}, {0x001e, 0x00000000}, {0x0002, 0x00000000}, + {0x0005, 0x00000000}, {0x0007, 0x00000000}, {0x00eb, 0xff00ff00}, {0x00ec, 0x00010000}, + {0x0090, 0x80000000}, {0x0091, 0x40004000}, {0x02fa, 0x3f800000}, {0x02fb, 0x3f800000}, + {0x02fc, 0x3f800000}, {0x02fd, 0x3f800000}, {0x02e2, 0x00000000}, {0x02e3, 0x00000000}, + {0x02e0, 0x00000000}, {0x02e1, 0x00000000}, {0x0003, 0x00000000}, {0x0004, 0x00000000}, + {0x0318, 0x00000000}, {0x031b, 0x00000000}, {0x031c, 0x00000000}, {0x031d, 0x00000000}, + {0x031e, 0x00000048}, {0x031f, 0x00000000}, {0x0321, 0x00000000}, {0x0323, 0x00000000}, + {0x0324, 0x00000000}, {0x0325, 0x00000000}, {0x0390, 0x00000000}, {0x0398, 0x00000000}, + {0x03a0, 0x00000000}, {0x03a8, 0x00000000}, {0x03b0, 0x00000000}, {0x03b8, 0x08c6c000}, + {0x0327, 0x00000000}, {0x032a, 0x00000000}, {0x032b, 0x00000000}, {0x032c, 0x00000000}, + {0x032d, 0x00000048}, {0x032e, 0x00000000}, {0x0330, 0x00000000}, {0x0332, 0x00000000}, + {0x0333, 0x00000000}, {0x0334, 0x00000000}, {0x0391, 0x00000000}, {0x0399, 0x00000000}, + {0x03a1, 0x00000000}, {0x03a9, 0x00000000}, {0x03b1, 0x00000000}, {0x03b9, 0x08c6c000}, + {0x0336, 0x00000000}, {0x0339, 0x00000000}, {0x033a, 0x00000000}, {0x033b, 0x00000000}, + {0x033c, 0x00000048}, {0x033d, 0x00000000}, {0x033f, 0x00000000}, {0x0341, 0x00000000}, + {0x0342, 0x00000000}, {0x0343, 0x00000000}, {0x0392, 0x00000000}, {0x039a, 0x00000000}, + {0x03a2, 0x00000000}, {0x03aa, 0x00000000}, {0x03b2, 0x00000000}, {0x03ba, 0x08c6c000}, + {0x0345, 0x00000000}, {0x0348, 0x00000000}, {0x0349, 0x00000000}, {0x034a, 0x00000000}, + {0x034b, 0x00000048}, {0x034c, 0x00000000}, {0x034e, 0x00000000}, {0x0350, 0x00000000}, + {0x0351, 0x00000000}, {0x0352, 0x00000000}, {0x0393, 0x00000000}, {0x039b, 0x00000000}, + {0x03a3, 0x00000000}, {0x03ab, 0x00000000}, {0x03b3, 0x00000000}, {0x03bb, 0x08c6c000}, + {0x0354, 0x00000000}, {0x0357, 0x00000000}, {0x0358, 0x00000000}, {0x0359, 0x00000000}, + {0x035a, 0x00000048}, {0x035b, 0x00000000}, {0x035d, 0x00000000}, {0x035f, 0x00000000}, + {0x0360, 0x00000000}, {0x0361, 0x00000000}, {0x0394, 0x00000000}, {0x039c, 0x00000000}, + {0x03a4, 0x00000000}, {0x03ac, 0x00000000}, {0x03b4, 0x00000000}, {0x03bc, 0x08c6c000}, + {0x0363, 0x00000000}, {0x0366, 0x00000000}, {0x0367, 0x00000000}, {0x0368, 0x00000000}, + {0x0369, 0x00000048}, {0x036a, 0x00000000}, {0x036c, 0x00000000}, {0x036e, 0x00000000}, + {0x036f, 0x00000000}, {0x0370, 0x00000000}, {0x0395, 0x00000000}, {0x039d, 0x00000000}, + {0x03a5, 0x00000000}, {0x03ad, 0x00000000}, {0x03b5, 0x00000000}, {0x03bd, 0x08c6c000}, + {0x0372, 0x00000000}, {0x0375, 0x00000000}, {0x0376, 0x00000000}, {0x0377, 0x00000000}, + {0x0378, 0x00000048}, {0x0379, 0x00000000}, {0x037b, 0x00000000}, {0x037d, 0x00000000}, + {0x037e, 0x00000000}, {0x037f, 0x00000000}, {0x0396, 0x00000000}, {0x039e, 0x00000000}, + {0x03a6, 0x00000000}, {0x03ae, 0x00000000}, {0x03b6, 0x00000000}, {0x03be, 0x08c6c000}, + {0x0381, 0x00000000}, {0x0384, 0x00000000}, {0x0385, 0x00000000}, {0x0386, 0x00000000}, + {0x0387, 0x00000048}, {0x0388, 0x00000000}, {0x038a, 0x00000000}, {0x038c, 0x00000000}, + {0x038d, 0x00000000}, {0x038e, 0x00000000}, {0x0397, 0x00000000}, {0x039f, 0x00000000}, + {0x03a7, 0x00000000}, {0x03af, 0x00000000}, {0x03b7, 0x00000000}, {0x03bf, 0x08c6c000}, + {0x000c, 0x00000000}, {0x000d, 0x40004000}, {0x0191, 0x00000000}, {0x0192, 0x00000000}, + {0x0193, 0x00000000}, {0x0194, 0x00000000}, {0x0195, 0x00000000}, {0x0196, 0x00000000}, + {0x0197, 0x00000000}, {0x0198, 0x00000000}, {0x0199, 0x00000000}, {0x019a, 0x00000000}, + {0x019b, 0x00000000}, {0x019c, 0x00000000}, {0x019d, 0x00000000}, {0x019e, 0x00000000}, + {0x019f, 0x00000000}, {0x01a0, 0x00000000}, {0x01a1, 0x00000000}, {0x01a2, 0x00000000}, + {0x01a3, 0x00000000}, {0x01a4, 0x00000000}, {0x01a5, 0x00000000}, {0x01a6, 0x00000000}, + {0x01a7, 0x00000000}, {0x01a8, 0x00000000}, {0x01a9, 0x00000000}, {0x01aa, 0x00000000}, + {0x01ab, 0x00000000}, {0x01ac, 0x00000000}, {0x01ad, 0x00000000}, {0x01ae, 0x00000000}, + {0x01af, 0x00000000}, {0x01b0, 0x00000000}, {0x016f, 0x00000000}, {0x0170, 0x00000000}, + {0x0171, 0x00000000}, {0x0172, 0x00000000}, {0x0173, 0x00000000}, {0x0174, 0x00000000}, + {0x0175, 0x00000000}, {0x0176, 0x00000000}, {0x0177, 0x00000000}, {0x0178, 0x00000000}, + {0x0179, 0x00000000}, {0x017a, 0x00000000}, {0x017b, 0x00000000}, {0x017c, 0x00000000}, + {0x017d, 0x00000000}, {0x017e, 0x00000000}, {0x017f, 0x00000000}, {0x0180, 0x00000000}, + {0x0181, 0x00000000}, {0x0182, 0x00000000}, {0x0183, 0x00000000}, {0x0184, 0x00000000}, + {0x0185, 0x00000000}, {0x0186, 0x00000000}, {0x0187, 0x00000000}, {0x0188, 0x00000000}, + {0x0189, 0x00000000}, {0x018a, 0x00000000}, {0x018b, 0x00000000}, {0x018c, 0x00000000}, + {0x018d, 0x00000000}, {0x018e, 0x00000000}, {0x010f, 0x3f800000}, {0x0111, 0x3f800000}, + {0x0113, 0x3f800000}, {0x0110, 0x00000000}, {0x0112, 0x00000000}, {0x0114, 0x00000000}, + {0x0094, 0x80000000}, {0x0095, 0x40004000}, {0x00b4, 0x00000000}, {0x00b5, 0x00000000}, + {0x0115, 0x3f800000}, {0x0117, 0x3f800000}, {0x0119, 0x3f800000}, {0x0116, 0x00000000}, + {0x0118, 0x00000000}, {0x011a, 0x00000000}, {0x0096, 0x80000000}, {0x0097, 0x40004000}, + {0x00b6, 0x00000000}, {0x00b7, 0x00000000}, {0x011b, 0x3f800000}, {0x011d, 0x3f800000}, + {0x011f, 0x3f800000}, {0x011c, 0x00000000}, {0x011e, 0x00000000}, {0x0120, 0x00000000}, + {0x0098, 0x80000000}, {0x0099, 0x40004000}, {0x00b8, 0x00000000}, {0x00b9, 0x00000000}, + {0x0121, 0x3f800000}, {0x0123, 0x3f800000}, {0x0125, 0x3f800000}, {0x0122, 0x00000000}, + {0x0124, 0x00000000}, {0x0126, 0x00000000}, {0x009a, 0x80000000}, {0x009b, 0x40004000}, + {0x00ba, 0x00000000}, {0x00bb, 0x00000000}, {0x0127, 0x3f800000}, {0x0129, 0x3f800000}, + {0x012b, 0x3f800000}, {0x0128, 0x00000000}, {0x012a, 0x00000000}, {0x012c, 0x00000000}, + {0x009c, 0x80000000}, {0x009d, 0x40004000}, {0x00bc, 0x00000000}, {0x00bd, 0x00000000}, + {0x012d, 0x3f800000}, {0x012f, 0x3f800000}, {0x0131, 0x3f800000}, {0x012e, 0x00000000}, + {0x0130, 0x00000000}, {0x0132, 0x00000000}, {0x009e, 0x80000000}, {0x009f, 0x40004000}, + {0x00be, 0x00000000}, {0x00bf, 0x00000000}, {0x0133, 0x3f800000}, {0x0135, 0x3f800000}, + {0x0137, 0x3f800000}, {0x0134, 0x00000000}, {0x0136, 0x00000000}, {0x0138, 0x00000000}, + {0x00a0, 0x80000000}, {0x00a1, 0x40004000}, {0x00c0, 0x00000000}, {0x00c1, 0x00000000}, + {0x0139, 0x3f800000}, {0x013b, 0x3f800000}, {0x013d, 0x3f800000}, {0x013a, 0x00000000}, + {0x013c, 0x00000000}, {0x013e, 0x00000000}, {0x00a2, 0x80000000}, {0x00a3, 0x40004000}, + {0x00c2, 0x00000000}, {0x00c3, 0x00000000}, {0x013f, 0x3f800000}, {0x0141, 0x3f800000}, + {0x0143, 0x3f800000}, {0x0140, 0x00000000}, {0x0142, 0x00000000}, {0x0144, 0x00000000}, + {0x00a4, 0x80000000}, {0x00a5, 0x40004000}, {0x00c4, 0x00000000}, {0x00c5, 0x00000000}, + {0x0145, 0x3f800000}, {0x0147, 0x3f800000}, {0x0149, 0x3f800000}, {0x0146, 0x00000000}, + {0x0148, 0x00000000}, {0x014a, 0x00000000}, {0x00a6, 0x80000000}, {0x00a7, 0x40004000}, + {0x00c6, 0x00000000}, {0x00c7, 0x00000000}, {0x014b, 0x3f800000}, {0x014d, 0x3f800000}, + {0x014f, 0x3f800000}, {0x014c, 0x00000000}, {0x014e, 0x00000000}, {0x0150, 0x00000000}, + {0x00a8, 0x80000000}, {0x00a9, 0x40004000}, {0x00c8, 0x00000000}, {0x00c9, 0x00000000}, + {0x0151, 0x3f800000}, {0x0153, 0x3f800000}, {0x0155, 0x3f800000}, {0x0152, 0x00000000}, + {0x0154, 0x00000000}, {0x0156, 0x00000000}, {0x00aa, 0x80000000}, {0x00ab, 0x40004000}, + {0x00ca, 0x00000000}, {0x00cb, 0x00000000}, {0x0157, 0x3f800000}, {0x0159, 0x3f800000}, + {0x015b, 0x3f800000}, {0x0158, 0x00000000}, {0x015a, 0x00000000}, {0x015c, 0x00000000}, + {0x00ac, 0x80000000}, {0x00ad, 0x40004000}, {0x00cc, 0x00000000}, {0x00cd, 0x00000000}, + {0x015d, 0x3f800000}, {0x015f, 0x3f800000}, {0x0161, 0x3f800000}, {0x015e, 0x00000000}, + {0x0160, 0x00000000}, {0x0162, 0x00000000}, {0x00ae, 0x80000000}, {0x00af, 0x40004000}, + {0x00ce, 0x00000000}, {0x00cf, 0x00000000}, {0x0163, 0x3f800000}, {0x0165, 0x3f800000}, + {0x0167, 0x3f800000}, {0x0164, 0x00000000}, {0x0166, 0x00000000}, {0x0168, 0x00000000}, + {0x00b0, 0x80000000}, {0x00b1, 0x40004000}, {0x00d0, 0x00000000}, {0x00d1, 0x00000000}, + {0x0169, 0x3f800000}, {0x016b, 0x3f800000}, {0x016d, 0x3f800000}, {0x016a, 0x00000000}, + {0x016c, 0x00000000}, {0x016e, 0x00000000}, {0x00b2, 0x80000000}, {0x00b3, 0x40004000}, + {0x00d2, 0x00000000}, {0x00d3, 0x00000000}, {0x0081, 0x80000000}, {0x0082, 0x40004000}, +}; + +static const uint16_t g_agc_public_reg_defaults_v4_tbl0_ptrs[] = { + 0x0000, 0x0001, 0x0002, 0x0003, 0x0004, 0x0005, 0x0006, 0x0007, 0x0008, 0x0009, 0x000a, 0x000b, + 0x000c, 0x000d, 0x000e, 0x000f, 0x0010, 0x0011, 0x0012, 0x0013, 0x0014, 0x0015, 0x0016, 0x0017, + 0x0018, 0x0019, 0x001a, 0x001b, 0x001c, 0x001d, 0x001e, 0x001f, 0x0020, 0x0021, 0x0022, 0x0023, + 0x0024, 0x0025, 0x0026, 0x0027, 0x0028, 0x0029, 0x002a, 0x002b, 0x002c, 0x002d, 0x002e, 0x002f, + 0x0030, 0x0031, 0x0032, 0x0033, 0x0034, 0x0035, 0x0036, 0x0037, 0x0038, 0x0039, 0x003a, 0x003c, + 0x004c, 0x004e, 0x0050, 0x0054, 0x005c, 0x005e, 0x0066, 0x0068, 0x0076, 0x0078, 0x007a, 0x007e, + 0x0080, 0x0082, 0x0084, 0x0104, 0x0106, 0x0126, 0x0146, 0x01e6, +}; + +static ShaderRegister g_agc_public_reg_defaults_v4_tbl1_regs[] = { + {0x0212, 0x00000000}, {0x0213, 0x00000000}, {0x0228, 0x00000000}, {0x0215, 0x00000000}, + {0x0218, 0x00000000}, {0x008a, 0x00000000}, {0x010a, 0x00000000}, {0x000a, 0x00000000}, + {0x008b, 0x00000000}, {0x010b, 0x00000000}, {0x000b, 0x00000000}, {0x0224, 0x00000000}, + {0x0225, 0x00000000}, {0x0226, 0x00000000}, {0x0227, 0x00000000}, {0x0107, 0xffff0000}, + {0x0087, 0x0000ffff}, {0x0007, 0x0000ffff}, {0x020c, 0x00000000}, {0x020d, 0x00000000}, + {0x00c8, 0x00000000}, {0x00c9, 0x00000000}, {0x0088, 0x00000000}, {0x0089, 0x00000000}, + {0x0108, 0x00000000}, {0x0109, 0x00000000}, {0x0148, 0x00000000}, {0x0149, 0x00000000}, + {0x0008, 0x00000000}, {0x0009, 0x00000000}, {0x00b2, 0x00000000}, {0x00b3, 0x00000000}, + {0x00b4, 0x00000000}, {0x00b5, 0x00000000}, {0x0132, 0x00000000}, {0x0133, 0x00000000}, + {0x0134, 0x00000000}, {0x0135, 0x00000000}, {0x0032, 0x00000000}, {0x0033, 0x00000000}, + {0x0034, 0x00000000}, {0x0035, 0x00000000}, {0x0082, 0x00000000}, {0x0083, 0x00000000}, + {0x0102, 0x00000000}, {0x0103, 0x00000000}, {0x0240, 0x00000000}, {0x0241, 0x00000000}, + {0x0242, 0x00000000}, {0x0243, 0x00000000}, {0x0244, 0x00000000}, {0x0245, 0x00000000}, + {0x0246, 0x00000000}, {0x0247, 0x00000000}, {0x0248, 0x00000000}, {0x0249, 0x00000000}, + {0x024a, 0x00000000}, {0x024b, 0x00000000}, {0x024c, 0x00000000}, {0x024d, 0x00000000}, + {0x024e, 0x00000000}, {0x024f, 0x00000000}, {0x008c, 0x00000000}, {0x008d, 0x00000000}, + {0x008e, 0x00000000}, {0x008f, 0x00000000}, {0x0090, 0x00000000}, {0x0091, 0x00000000}, + {0x0092, 0x00000000}, {0x0093, 0x00000000}, {0x0094, 0x00000000}, {0x0095, 0x00000000}, + {0x0096, 0x00000000}, {0x0097, 0x00000000}, {0x0098, 0x00000000}, {0x0099, 0x00000000}, + {0x009a, 0x00000000}, {0x009b, 0x00000000}, {0x009c, 0x00000000}, {0x009d, 0x00000000}, + {0x009e, 0x00000000}, {0x009f, 0x00000000}, {0x00a0, 0x00000000}, {0x00a1, 0x00000000}, + {0x00a2, 0x00000000}, {0x00a3, 0x00000000}, {0x00a4, 0x00000000}, {0x00a5, 0x00000000}, + {0x00a6, 0x00000000}, {0x00a7, 0x00000000}, {0x00a8, 0x00000000}, {0x00a9, 0x00000000}, + {0x00aa, 0x00000000}, {0x00ab, 0x00000000}, {0x010c, 0x00000000}, {0x010d, 0x00000000}, + {0x010e, 0x00000000}, {0x010f, 0x00000000}, {0x0110, 0x00000000}, {0x0111, 0x00000000}, + {0x0112, 0x00000000}, {0x0113, 0x00000000}, {0x0114, 0x00000000}, {0x0115, 0x00000000}, + {0x0116, 0x00000000}, {0x0117, 0x00000000}, {0x0118, 0x00000000}, {0x0119, 0x00000000}, + {0x011a, 0x00000000}, {0x011b, 0x00000000}, {0x011c, 0x00000000}, {0x011d, 0x00000000}, + {0x011e, 0x00000000}, {0x011f, 0x00000000}, {0x0120, 0x00000000}, {0x0121, 0x00000000}, + {0x0122, 0x00000000}, {0x0123, 0x00000000}, {0x0124, 0x00000000}, {0x0125, 0x00000000}, + {0x0126, 0x00000000}, {0x0127, 0x00000000}, {0x0128, 0x00000000}, {0x0129, 0x00000000}, + {0x012a, 0x00000000}, {0x012b, 0x00000000}, {0x000c, 0x00000000}, {0x000d, 0x00000000}, + {0x000e, 0x00000000}, {0x000f, 0x00000000}, {0x0010, 0x00000000}, {0x0011, 0x00000000}, + {0x0012, 0x00000000}, {0x0013, 0x00000000}, {0x0014, 0x00000000}, {0x0015, 0x00000000}, + {0x0016, 0x00000000}, {0x0017, 0x00000000}, {0x0018, 0x00000000}, {0x0019, 0x00000000}, + {0x001a, 0x00000000}, {0x001b, 0x00000000}, {0x001c, 0x00000000}, {0x001d, 0x00000000}, + {0x001e, 0x00000000}, {0x001f, 0x00000000}, {0x0020, 0x00000000}, {0x0021, 0x00000000}, + {0x0022, 0x00000000}, {0x0023, 0x00000000}, {0x0024, 0x00000000}, {0x0025, 0x00000000}, + {0x0026, 0x00000000}, {0x0027, 0x00000000}, {0x0028, 0x00000000}, {0x0029, 0x00000000}, + {0x002a, 0x00000000}, {0x002b, 0x00000000}, +}; + +static const uint16_t g_agc_public_reg_defaults_v4_tbl1_ptrs[] = { + 0x0000, 0x0001, 0x0002, 0x0003, 0x0004, 0x0005, 0x0006, 0x0007, 0x0008, 0x0009, 0x000a, 0x000b, + 0x000f, 0x0012, 0x0014, 0x0016, 0x0018, 0x001a, 0x001c, 0x001e, 0x0022, 0x0026, 0x002a, 0x002c, + 0x002e, 0x003e, 0x005e, 0x007e, +}; + +static ShaderRegister g_agc_public_reg_defaults_v4_tbl2_regs[] = { + {0x041f, 0x00000000}, {0x041d, 0x00000000}, {0x041e, 0x00000000}, {0x025b, 0x00000000}, + {0x024a, 0x00000000}, {0x024b, 0x00000000}, {0x025f, 0x00000000}, {0x0262, 0x00000000}, + {0x02bb, 0x00000000}, {0x02e4, 0x00000000}, {0x02e5, 0x00000000}, {0x02c4, 0x00000000}, + {0x02c5, 0x00000000}, {0x02c6, 0x00000000}, {0x02c7, 0x00000000}, {0x02e8, 0x00000000}, + {0x02e9, 0x00000000}, {0x02cc, 0x00000000}, {0x02cd, 0x00000000}, {0x02ce, 0x00000000}, + {0x02cf, 0x00000000}, {0x02bd, 0x00000000}, {0x02bc, 0x00000000}, {0x02e6, 0x00000000}, + {0x02e7, 0x00000000}, {0x02c8, 0x00000000}, {0x02c9, 0x00000000}, {0x02ca, 0x00000000}, + {0x02cb, 0x00000000}, {0x02ea, 0x00000000}, {0x02eb, 0x00000000}, {0x02d0, 0x00000000}, + {0x02d1, 0x00000000}, {0x02d2, 0x00000000}, {0x02d3, 0x00000000}, {0x02bf, 0x00000000}, + {0x02be, 0x00000000}, {0x0384, 0x00000000}, {0x0383, 0x40000040}, {0x0248, 0x00000000}, + {0x0242, 0x00000000}, {0x0380, 0x00000000}, {0x0381, 0x00000000}, +}; + +static const uint16_t g_agc_public_reg_defaults_v4_tbl2_ptrs[] = { + 0x0000, 0x0001, 0x0002, 0x0003, 0x0004, 0x0005, 0x0006, 0x0007, 0x0008, 0x0009, 0x000a, 0x000b, + 0x000c, 0x000d, 0x000e, 0x000f, 0x0010, 0x0011, 0x0012, 0x0013, 0x0014, 0x0015, 0x0016, 0x0017, + 0x0018, 0x0019, 0x001a, 0x001b, 0x001c, 0x001d, 0x001e, 0x001f, 0x0020, 0x0021, 0x0022, 0x0023, + 0x0024, 0x0025, 0x0026, 0x0027, 0x0028, 0x0029, +}; + +static uint32_t g_agc_public_reg_defaults_v4_types[] = { + 0xe24f806d, 0x00040400, 0x00000000, 0xf6c28182, 0x00040404, 0x00000000, + 0x6f6e55a5, 0x00040408, 0x00000000, 0x0bc65da4, 0x0004040c, 0x00000000, + 0x9e5ad592, 0x00040410, 0x00000000, 0xbb513b98, 0x00040414, 0x00000000, + 0xab64b23b, 0x00040418, 0x00000000, 0xac198e0b, 0x0004041c, 0x00000000, + 0x53c39964, 0x00040420, 0x00000000, 0x01396b11, 0x00040424, 0x00000000, + 0x7d42019a, 0x00040428, 0x00000000, 0x3548f523, 0x0004042c, 0x00000000, + 0xf43ad28a, 0x00040430, 0x00000000, 0x6de4c312, 0x00040434, 0x00000000, + 0x00a77ae0, 0x00040438, 0x00000000, 0x00a779b7, 0x0004043c, 0x00000000, + 0x5100100c, 0x00040440, 0x00000000, 0x59958bba, 0x00040444, 0x00000000, + 0x7f7bd53e, 0x00040448, 0x00000000, 0x0c06f17c, 0x0004044c, 0x00000000, + 0x6f104b72, 0x00040450, 0x00000000, 0x25c70d9c, 0x00040454, 0x00000000, + 0x3881201e, 0x00040458, 0x00000000, 0x09afddaf, 0x0004045c, 0x00000000, + 0x367d63cf, 0x00040460, 0x00000000, 0x43707db8, 0x00040464, 0x00000000, + 0xf6ae26ba, 0x00040468, 0x00000000, 0x1b917652, 0x0004046c, 0x00000000, + 0x2cac726e, 0x00040470, 0x00000000, 0x2cac7111, 0x00040474, 0x00000000, + 0x94b1e4f7, 0x00040478, 0x00000000, 0xe3661b6c, 0x0004047c, 0x00000000, + 0x1eb8d73a, 0x00040480, 0x00000000, 0x15051fa3, 0x00040484, 0x00000000, + 0x9c51a7f1, 0x00040488, 0x00000000, 0xa20efc70, 0x0004048c, 0x00000000, + 0x0ec09f6e, 0x00040490, 0x00000000, 0x34a7d6d3, 0x00040494, 0x00000000, + 0xce831b94, 0x00040498, 0x00000000, 0x5cc72a74, 0x0004049c, 0x00000000, + 0x3b77713c, 0x000404a0, 0x00000000, 0x40f64410, 0x000404a4, 0x00000000, + 0x69441268, 0x000404a8, 0x00000000, 0x2e418b83, 0x000404ac, 0x00000000, + 0xa00d0c8d, 0x000404b0, 0x00000000, 0xb1289fb3, 0x000404b4, 0x00000000, + 0x144832fb, 0x000404b8, 0x00000000, 0x9890d9fa, 0x000404bc, 0x00000000, + 0x9016faf1, 0x000404c0, 0x00000000, 0x4b73ce27, 0x000404c4, 0x00000000, + 0x5f5a3e7b, 0x000404c8, 0x00000000, 0xd4af3a51, 0x000404cc, 0x00000000, + 0x6cf4f543, 0x000404d0, 0x00000000, 0x5fb86ccb, 0x000404d4, 0x00000000, + 0xedefa188, 0x000404d8, 0x00000000, 0xd0de9ee6, 0x000404dc, 0x00000000, + 0xc5831803, 0x000404e0, 0x00000000, 0x8e6de84b, 0x000404e4, 0x00000000, + 0xd0771662, 0x000408e8, 0x00000000, 0x569f7444, 0x004004ec, 0x00000000, + 0x5c6637cd, 0x000408f0, 0x00000000, 0xcae3e690, 0x000408f4, 0x00000000, + 0x43fbd769, 0x000410f8, 0x00000000, 0xef550356, 0x002004fc, 0x00000000, + 0x8f52e279, 0x00040900, 0x00000000, 0x1f2d8149, 0x00100904, 0x00000000, + 0x4413c6f9, 0x00040908, 0x00000000, 0x67096014, 0x0004390c, 0x00000000, + 0x88f5e915, 0x00040910, 0x00000000, 0x918106bb, 0x00040914, 0x00000000, + 0x95f0e7ac, 0x00041118, 0x00000000, 0xb48cbab2, 0x0004091c, 0x00000000, + 0x05bb3bc6, 0x00040920, 0x00000000, 0x94faba07, 0x00040924, 0x00000000, + 0x38e92c91, 0x00204128, 0x00000000, 0x0b177b43, 0x0004092c, 0x00000000, + 0x48531062, 0x00800530, 0x00000000, 0xaaa964b9, 0x00201134, 0x00000000, + 0x7690af6f, 0x00402938, 0x00000000, 0x078d7060, 0x0004093c, 0x00000000, + 0x5d6e3ec7, 0x00040401, 0x00000000, 0x57e7079a, 0x00040405, 0x00000000, + 0x7467fafd, 0x00040409, 0x00000000, 0x9e826b50, 0x0004040d, 0x00000000, + 0xdc484f18, 0x00040411, 0x00000000, 0x5da8bca3, 0x00040415, 0x00000000, + 0x5ca726d8, 0x00040419, 0x00000000, 0x5dd28360, 0x0004041d, 0x00000000, + 0x57efa0be, 0x00040421, 0x00000000, 0x502363d5, 0x00040425, 0x00000000, + 0x506d14bd, 0x00040429, 0x00000000, 0xb2609506, 0x0010042d, 0x00000000, + 0x9e5cfb8a, 0x00040c31, 0x00000000, 0xc918df3e, 0x00040835, 0x00000000, + 0xc9751c9c, 0x00040839, 0x00000000, 0xc97ef77a, 0x0004083d, 0x00000000, + 0xc927c6b9, 0x00040841, 0x00000000, 0xc92a1ec5, 0x00040845, 0x00000000, + 0xc9e01b31, 0x00040849, 0x00000000, 0xb26219ca, 0x0010044d, 0x00000000, + 0xb25b6cf9, 0x00100451, 0x00000000, 0xb2f86101, 0x00100455, 0x00000000, + 0x07e3b155, 0x00040859, 0x00000000, 0x07e383c6, 0x0004085d, 0x00000000, + 0xbda98653, 0x00400461, 0x00000000, 0xbdbd1d0f, 0x00800465, 0x00000000, + 0xbd946fd4, 0x00800469, 0x00000000, 0xbdf02a4c, 0x0080046d, 0x00000000, + 0x19e93e85, 0x00040402, 0x00000000, 0x3b5c2af3, 0x00040406, 0x00000000, + 0x47974a35, 0x0004040a, 0x00000000, 0x105971c2, 0x0004040e, 0x00000000, + 0x7d137765, 0x00040412, 0x00000000, 0xd187febc, 0x00040416, 0x00000000, + 0x12f854ac, 0x0004041a, 0x00000000, 0x40d49ad1, 0x0004041e, 0x00000000, + 0x8c0923da, 0x00040422, 0x00000000, 0xf6be6f25, 0x00040426, 0x00000000, + 0xf6908bad, 0x0004042a, 0x00000000, 0x862ee1be, 0x0004042e, 0x00000000, + 0x8631d396, 0x00040432, 0x00000000, 0x89d7d386, 0x00040436, 0x00000000, + 0xd1006f62, 0x0004043a, 0x00000000, 0x7e51d354, 0x0004043e, 0x00000000, + 0x7e712edc, 0x00040442, 0x00000000, 0x141f6002, 0x00040446, 0x00000000, + 0x140c761a, 0x0004044a, 0x00000000, 0x14f68dfa, 0x0004044e, 0x00000000, + 0x6fea32c3, 0x00040452, 0x00000000, 0x8f378499, 0x00040456, 0x00000000, + 0x5d9ed616, 0x0004045a, 0x00000000, 0x2bbfa784, 0x0004045e, 0x00000000, + 0x2b45f28c, 0x00040462, 0x00000000, 0x5f4aaed9, 0x00040466, 0x00000000, + 0x5f477465, 0x0004046a, 0x00000000, 0x5ec84729, 0x0004046e, 0x00000000, + 0x48ca8407, 0x00040472, 0x00000000, 0x5f355d79, 0x00040476, 0x00000000, + 0x5f0f3449, 0x0004047a, 0x00000000, 0xee4a4919, 0x0004047e, 0x00000000, + 0xee49852d, 0x00040482, 0x00000000, 0xee4f8201, 0x00040486, 0x00000000, + 0x8a3f99a1, 0x0004048a, 0x00000000, 0x381fc91f, 0x0004048e, 0x00000000, + 0x19d8287b, 0x00040492, 0x00000000, 0xbb8df494, 0x00040496, 0x00000000, + 0xf6d8a76e, 0x0004049a, 0x00000000, 0x7620f1e9, 0x0004049e, 0x00000000, + 0x9ebfab10, 0x000404a2, 0x00000000, 0x98a09d0e, 0x000408a6, 0x00000000, +}; + +static CompactRegisterDefaults g_agc_public_reg_defaults_v4 = { + g_agc_public_reg_defaults_v4_tbl0_regs, 488, g_agc_public_reg_defaults_v4_tbl0_ptrs, 80, + g_agc_public_reg_defaults_v4_tbl1_regs, 158, g_agc_public_reg_defaults_v4_tbl1_ptrs, 28, + g_agc_public_reg_defaults_v4_tbl2_regs, 43, g_agc_public_reg_defaults_v4_tbl2_ptrs, 42, + nullptr, 0, nullptr, 0, + g_agc_public_reg_defaults_v4_types, 150, +}; + +// g_agc_public_reg_defaults_v5: regs=(491, 159, 75, 0), ptrs=(80, 29, 47, 0), types=156. +static ShaderRegister g_agc_public_reg_defaults_v5_tbl0_regs[] = { + {0x0202, 0x00cc0010}, {0x0109, 0x00000008}, {0x0104, 0x00000000}, {0x008f, 0x00000000}, + {0x008e, 0x00000000}, {0x02dc, 0x0000aa00}, {0x0001, 0x11000100}, {0x0200, 0x00000000}, + {0x0201, 0x00000000}, {0x0000, 0x00000000}, {0x0006, 0x00000000}, {0x001f, 0x00000000}, + {0x0203, 0x00000000}, {0x02b0, 0x00000000}, {0x02b1, 0x00000000}, {0x010c, 0x00000000}, + {0x010d, 0x00000000}, {0x010b, 0x00000000}, {0x01ff, 0x00000000}, {0x0204, 0x00000000}, + {0x020d, 0x00000000}, {0x0206, 0x0000043f}, {0x02f8, 0x00000000}, {0x0083, 0x0000ffff}, + {0x0313, 0x00006000}, {0x00f0, 0x00000000}, {0x00f1, 0x00000000}, {0x00f2, 0x00000000}, + {0x00ea, 0x00000000}, {0x00e9, 0x00000000}, {0x0292, 0x00000002}, {0x0293, 0x06020000}, + {0x00e8, 0x00000000}, {0x0080, 0x00000000}, {0x0211, 0x00000000}, {0x0210, 0x04000000}, + {0x008d, 0x00000000}, {0x0282, 0x00000008}, {0x0281, 0xffff0000}, {0x0280, 0x00080008}, + {0x02df, 0x00000000}, {0x02de, 0x000001e9}, {0x0205, 0x00000240}, {0x020c, 0x00000021}, + {0x02f9, 0x0000002d}, {0x01ba, 0x00000000}, {0x02a6, 0x00000000}, {0x02ce, 0x00000400}, + {0x029b, 0x00000002}, {0x02d6, 0x00000000}, {0x0103, 0xffffffff}, {0x02a1, 0x00000000}, + {0x02ad, 0x00000000}, {0x02d5, 0x00002000}, {0x02d4, 0x88101010}, {0x02db, 0x00000000}, + {0x02f5, 0x00000000}, {0x02f6, 0x00000000}, {0x02fe, 0x00000000}, {0x02ff, 0x00000000}, + {0x0300, 0x00000000}, {0x0301, 0x00000000}, {0x0302, 0x00000000}, {0x0303, 0x00000000}, + {0x0304, 0x00000000}, {0x0305, 0x00000000}, {0x0306, 0x00000000}, {0x0307, 0x00000000}, + {0x0308, 0x00000000}, {0x0309, 0x00000000}, {0x030a, 0x00000000}, {0x030b, 0x00000000}, + {0x030c, 0x00000000}, {0x030d, 0x00000000}, {0x030e, 0xffffffff}, {0x030f, 0xffffffff}, + {0x0311, 0x01fd2002}, {0x0312, 0x03ff0080}, {0x0105, 0x00000000}, {0x0106, 0x00000000}, + {0x0107, 0x00000000}, {0x0108, 0x00000000}, {0x01e0, 0x20010001}, {0x01e1, 0x20010001}, + {0x01e2, 0x20010001}, {0x01e3, 0x20010001}, {0x01e4, 0x20010001}, {0x01e5, 0x20010001}, + {0x01e6, 0x20010001}, {0x01e7, 0x20010001}, {0x0020, 0x00000000}, {0x0021, 0x00000000}, + {0x0084, 0x00000000}, {0x0085, 0x20002000}, {0x0086, 0x00000000}, {0x0087, 0x20002000}, + {0x0088, 0x00000000}, {0x0089, 0x20002000}, {0x008a, 0x00000000}, {0x008b, 0x20002000}, + {0x00db, 0x00000000}, {0x0008, 0x00000000}, {0x0009, 0x00000000}, {0x0010, 0x80000180}, + {0x0011, 0x20000180}, {0x0012, 0x00000000}, {0x0013, 0x00000000}, {0x0014, 0x00000000}, + {0x0015, 0x00000000}, {0x001a, 0x00000000}, {0x001b, 0x00000000}, {0x001c, 0x00000000}, + {0x001d, 0x00000000}, {0x001e, 0x00000000}, {0x0002, 0x00000000}, {0x0005, 0x00000000}, + {0x0007, 0x00000000}, {0x000b, 0x00000000}, {0x000a, 0x00000000}, {0x00eb, 0xff00ff00}, + {0x00ec, 0x00010000}, {0x00f1, 0x00000000}, {0x00f2, 0x00000000}, {0x0090, 0x80000000}, + {0x0091, 0x40004000}, {0x02fa, 0x3f800000}, {0x02fb, 0x3f800000}, {0x02fc, 0x3f800000}, + {0x02fd, 0x3f800000}, {0x02e2, 0x00000000}, {0x02e3, 0x00000000}, {0x02e0, 0x00000000}, + {0x02e1, 0x00000000}, {0x0003, 0x00000000}, {0x0004, 0x00000000}, {0x0318, 0x00000000}, + {0x031b, 0x00000000}, {0x031c, 0x00000000}, {0x031d, 0x00000000}, {0x031e, 0x00000048}, + {0x031f, 0x00000000}, {0x0321, 0x00000000}, {0x0323, 0x00000000}, {0x0324, 0x00000000}, + {0x0325, 0x00000000}, {0x0390, 0x00000000}, {0x0398, 0x00000000}, {0x03a0, 0x00000000}, + {0x03a8, 0x00000000}, {0x03b0, 0x00000000}, {0x03b8, 0x08c6c000}, {0x0327, 0x00000000}, + {0x032a, 0x00000000}, {0x032b, 0x00000000}, {0x032c, 0x00000000}, {0x032d, 0x00000048}, + {0x032e, 0x00000000}, {0x0330, 0x00000000}, {0x0332, 0x00000000}, {0x0333, 0x00000000}, + {0x0334, 0x00000000}, {0x0391, 0x00000000}, {0x0399, 0x00000000}, {0x03a1, 0x00000000}, + {0x03a9, 0x00000000}, {0x03b1, 0x00000000}, {0x03b9, 0x08c6c000}, {0x0336, 0x00000000}, + {0x0339, 0x00000000}, {0x033a, 0x00000000}, {0x033b, 0x00000000}, {0x033c, 0x00000048}, + {0x033d, 0x00000000}, {0x033f, 0x00000000}, {0x0341, 0x00000000}, {0x0342, 0x00000000}, + {0x0343, 0x00000000}, {0x0392, 0x00000000}, {0x039a, 0x00000000}, {0x03a2, 0x00000000}, + {0x03aa, 0x00000000}, {0x03b2, 0x00000000}, {0x03ba, 0x08c6c000}, {0x0345, 0x00000000}, + {0x0348, 0x00000000}, {0x0349, 0x00000000}, {0x034a, 0x00000000}, {0x034b, 0x00000048}, + {0x034c, 0x00000000}, {0x034e, 0x00000000}, {0x0350, 0x00000000}, {0x0351, 0x00000000}, + {0x0352, 0x00000000}, {0x0393, 0x00000000}, {0x039b, 0x00000000}, {0x03a3, 0x00000000}, + {0x03ab, 0x00000000}, {0x03b3, 0x00000000}, {0x03bb, 0x08c6c000}, {0x0354, 0x00000000}, + {0x0357, 0x00000000}, {0x0358, 0x00000000}, {0x0359, 0x00000000}, {0x035a, 0x00000048}, + {0x035b, 0x00000000}, {0x035d, 0x00000000}, {0x035f, 0x00000000}, {0x0360, 0x00000000}, + {0x0361, 0x00000000}, {0x0394, 0x00000000}, {0x039c, 0x00000000}, {0x03a4, 0x00000000}, + {0x03ac, 0x00000000}, {0x03b4, 0x00000000}, {0x03bc, 0x08c6c000}, {0x0363, 0x00000000}, + {0x0366, 0x00000000}, {0x0367, 0x00000000}, {0x0368, 0x00000000}, {0x0369, 0x00000048}, + {0x036a, 0x00000000}, {0x036c, 0x00000000}, {0x036e, 0x00000000}, {0x036f, 0x00000000}, + {0x0370, 0x00000000}, {0x0395, 0x00000000}, {0x039d, 0x00000000}, {0x03a5, 0x00000000}, + {0x03ad, 0x00000000}, {0x03b5, 0x00000000}, {0x03bd, 0x08c6c000}, {0x0372, 0x00000000}, + {0x0375, 0x00000000}, {0x0376, 0x00000000}, {0x0377, 0x00000000}, {0x0378, 0x00000048}, + {0x0379, 0x00000000}, {0x037b, 0x00000000}, {0x037d, 0x00000000}, {0x037e, 0x00000000}, + {0x037f, 0x00000000}, {0x0396, 0x00000000}, {0x039e, 0x00000000}, {0x03a6, 0x00000000}, + {0x03ae, 0x00000000}, {0x03b6, 0x00000000}, {0x03be, 0x08c6c000}, {0x0381, 0x00000000}, + {0x0384, 0x00000000}, {0x0385, 0x00000000}, {0x0386, 0x00000000}, {0x0387, 0x00000048}, + {0x0388, 0x00000000}, {0x038a, 0x00000000}, {0x038c, 0x00000000}, {0x038d, 0x00000000}, + {0x038e, 0x00000000}, {0x0397, 0x00000000}, {0x039f, 0x00000000}, {0x03a7, 0x00000000}, + {0x03af, 0x00000000}, {0x03b7, 0x00000000}, {0x03bf, 0x08c6c000}, {0x000c, 0x00000000}, + {0x000d, 0x40004000}, {0x0191, 0x00000000}, {0x0192, 0x00000000}, {0x0193, 0x00000000}, + {0x0194, 0x00000000}, {0x0195, 0x00000000}, {0x0196, 0x00000000}, {0x0197, 0x00000000}, + {0x0198, 0x00000000}, {0x0199, 0x00000000}, {0x019a, 0x00000000}, {0x019b, 0x00000000}, + {0x019c, 0x00000000}, {0x019d, 0x00000000}, {0x019e, 0x00000000}, {0x019f, 0x00000000}, + {0x01a0, 0x00000000}, {0x01a1, 0x00000000}, {0x01a2, 0x00000000}, {0x01a3, 0x00000000}, + {0x01a4, 0x00000000}, {0x01a5, 0x00000000}, {0x01a6, 0x00000000}, {0x01a7, 0x00000000}, + {0x01a8, 0x00000000}, {0x01a9, 0x00000000}, {0x01aa, 0x00000000}, {0x01ab, 0x00000000}, + {0x01ac, 0x00000000}, {0x01ad, 0x00000000}, {0x01ae, 0x00000000}, {0x01af, 0x00000000}, + {0x01b0, 0x00000000}, {0x016f, 0x00000000}, {0x0170, 0x00000000}, {0x0171, 0x00000000}, + {0x0172, 0x00000000}, {0x0173, 0x00000000}, {0x0174, 0x00000000}, {0x0175, 0x00000000}, + {0x0176, 0x00000000}, {0x0177, 0x00000000}, {0x0178, 0x00000000}, {0x0179, 0x00000000}, + {0x017a, 0x00000000}, {0x017b, 0x00000000}, {0x017c, 0x00000000}, {0x017d, 0x00000000}, + {0x017e, 0x00000000}, {0x017f, 0x00000000}, {0x0180, 0x00000000}, {0x0181, 0x00000000}, + {0x0182, 0x00000000}, {0x0183, 0x00000000}, {0x0184, 0x00000000}, {0x0185, 0x00000000}, + {0x0186, 0x00000000}, {0x0187, 0x00000000}, {0x0188, 0x00000000}, {0x0189, 0x00000000}, + {0x018a, 0x00000000}, {0x018b, 0x00000000}, {0x018c, 0x00000000}, {0x018d, 0x00000000}, + {0x018e, 0x00000000}, {0x010f, 0x3f800000}, {0x0111, 0x3f800000}, {0x0113, 0x3f800000}, + {0x0110, 0x00000000}, {0x0112, 0x00000000}, {0x0114, 0x00000000}, {0x0094, 0x80000000}, + {0x0095, 0x40004000}, {0x00b4, 0x00000000}, {0x00b5, 0x00000000}, {0x0115, 0x3f800000}, + {0x0117, 0x3f800000}, {0x0119, 0x3f800000}, {0x0116, 0x00000000}, {0x0118, 0x00000000}, + {0x011a, 0x00000000}, {0x0096, 0x80000000}, {0x0097, 0x40004000}, {0x00b6, 0x00000000}, + {0x00b7, 0x00000000}, {0x011b, 0x3f800000}, {0x011d, 0x3f800000}, {0x011f, 0x3f800000}, + {0x011c, 0x00000000}, {0x011e, 0x00000000}, {0x0120, 0x00000000}, {0x0098, 0x80000000}, + {0x0099, 0x40004000}, {0x00b8, 0x00000000}, {0x00b9, 0x00000000}, {0x0121, 0x3f800000}, + {0x0123, 0x3f800000}, {0x0125, 0x3f800000}, {0x0122, 0x00000000}, {0x0124, 0x00000000}, + {0x0126, 0x00000000}, {0x009a, 0x80000000}, {0x009b, 0x40004000}, {0x00ba, 0x00000000}, + {0x00bb, 0x00000000}, {0x0127, 0x3f800000}, {0x0129, 0x3f800000}, {0x012b, 0x3f800000}, + {0x0128, 0x00000000}, {0x012a, 0x00000000}, {0x012c, 0x00000000}, {0x009c, 0x80000000}, + {0x009d, 0x40004000}, {0x00bc, 0x00000000}, {0x00bd, 0x00000000}, {0x012d, 0x3f800000}, + {0x012f, 0x3f800000}, {0x0131, 0x3f800000}, {0x012e, 0x00000000}, {0x0130, 0x00000000}, + {0x0132, 0x00000000}, {0x009e, 0x80000000}, {0x009f, 0x40004000}, {0x00be, 0x00000000}, + {0x00bf, 0x00000000}, {0x0133, 0x3f800000}, {0x0135, 0x3f800000}, {0x0137, 0x3f800000}, + {0x0134, 0x00000000}, {0x0136, 0x00000000}, {0x0138, 0x00000000}, {0x00a0, 0x80000000}, + {0x00a1, 0x40004000}, {0x00c0, 0x00000000}, {0x00c1, 0x00000000}, {0x0139, 0x3f800000}, + {0x013b, 0x3f800000}, {0x013d, 0x3f800000}, {0x013a, 0x00000000}, {0x013c, 0x00000000}, + {0x013e, 0x00000000}, {0x00a2, 0x80000000}, {0x00a3, 0x40004000}, {0x00c2, 0x00000000}, + {0x00c3, 0x00000000}, {0x013f, 0x3f800000}, {0x0141, 0x3f800000}, {0x0143, 0x3f800000}, + {0x0140, 0x00000000}, {0x0142, 0x00000000}, {0x0144, 0x00000000}, {0x00a4, 0x80000000}, + {0x00a5, 0x40004000}, {0x00c4, 0x00000000}, {0x00c5, 0x00000000}, {0x0145, 0x3f800000}, + {0x0147, 0x3f800000}, {0x0149, 0x3f800000}, {0x0146, 0x00000000}, {0x0148, 0x00000000}, + {0x014a, 0x00000000}, {0x00a6, 0x80000000}, {0x00a7, 0x40004000}, {0x00c6, 0x00000000}, + {0x00c7, 0x00000000}, {0x014b, 0x3f800000}, {0x014d, 0x3f800000}, {0x014f, 0x3f800000}, + {0x014c, 0x00000000}, {0x014e, 0x00000000}, {0x0150, 0x00000000}, {0x00a8, 0x80000000}, + {0x00a9, 0x40004000}, {0x00c8, 0x00000000}, {0x00c9, 0x00000000}, {0x0151, 0x3f800000}, + {0x0153, 0x3f800000}, {0x0155, 0x3f800000}, {0x0152, 0x00000000}, {0x0154, 0x00000000}, + {0x0156, 0x00000000}, {0x00aa, 0x80000000}, {0x00ab, 0x40004000}, {0x00ca, 0x00000000}, + {0x00cb, 0x00000000}, {0x0157, 0x3f800000}, {0x0159, 0x3f800000}, {0x015b, 0x3f800000}, + {0x0158, 0x00000000}, {0x015a, 0x00000000}, {0x015c, 0x00000000}, {0x00ac, 0x80000000}, + {0x00ad, 0x40004000}, {0x00cc, 0x00000000}, {0x00cd, 0x00000000}, {0x015d, 0x3f800000}, + {0x015f, 0x3f800000}, {0x0161, 0x3f800000}, {0x015e, 0x00000000}, {0x0160, 0x00000000}, + {0x0162, 0x00000000}, {0x00ae, 0x80000000}, {0x00af, 0x40004000}, {0x00ce, 0x00000000}, + {0x00cf, 0x00000000}, {0x0163, 0x3f800000}, {0x0165, 0x3f800000}, {0x0167, 0x3f800000}, + {0x0164, 0x00000000}, {0x0166, 0x00000000}, {0x0168, 0x00000000}, {0x00b0, 0x80000000}, + {0x00b1, 0x40004000}, {0x00d0, 0x00000000}, {0x00d1, 0x00000000}, {0x0169, 0x3f800000}, + {0x016b, 0x3f800000}, {0x016d, 0x3f800000}, {0x016a, 0x00000000}, {0x016c, 0x00000000}, + {0x016e, 0x00000000}, {0x00b2, 0x80000000}, {0x00b3, 0x40004000}, {0x00d2, 0x00000000}, + {0x00d3, 0x00000000}, {0x0081, 0x80000000}, {0x0082, 0x40004000}, +}; + +static const uint16_t g_agc_public_reg_defaults_v5_tbl0_ptrs[] = { + 0x0000, 0x0001, 0x0002, 0x0003, 0x0004, 0x0005, 0x0006, 0x0007, 0x0008, 0x0009, 0x000a, 0x000b, + 0x000c, 0x000d, 0x000e, 0x000f, 0x0010, 0x0011, 0x0012, 0x0013, 0x0014, 0x0015, 0x0016, 0x0017, + 0x0018, 0x0019, 0x001a, 0x001b, 0x001c, 0x001d, 0x001e, 0x001f, 0x0020, 0x0021, 0x0022, 0x0023, + 0x0024, 0x0025, 0x0026, 0x0027, 0x0028, 0x0029, 0x002a, 0x002b, 0x002c, 0x002d, 0x002e, 0x002f, + 0x0030, 0x0031, 0x0032, 0x0033, 0x0034, 0x0035, 0x0036, 0x0037, 0x0038, 0x003a, 0x004a, 0x004c, + 0x004e, 0x0052, 0x005a, 0x005c, 0x0064, 0x0065, 0x0067, 0x0077, 0x0079, 0x007b, 0x007d, 0x0081, + 0x0083, 0x0085, 0x0087, 0x0107, 0x0109, 0x0129, 0x0149, 0x01e9, +}; + +static ShaderRegister g_agc_public_reg_defaults_v5_tbl1_regs[] = { + {0x0212, 0x00000000}, {0x0213, 0x00000000}, {0x0228, 0x00000000}, {0x0215, 0x00000000}, + {0x0218, 0x00000000}, {0x008a, 0x00000000}, {0x010a, 0x00000000}, {0x000a, 0x00000000}, + {0x008b, 0x00000000}, {0x010b, 0x00000000}, {0x000b, 0x00000000}, {0x0224, 0x00000000}, + {0x0225, 0x00000000}, {0x0226, 0x00000000}, {0x0227, 0x00000000}, {0x0107, 0xffff0000}, + {0x0087, 0x0000ffff}, {0x0007, 0x0000ffff}, {0x020c, 0x00000000}, {0x020d, 0x00000000}, + {0x00c8, 0x00000000}, {0x00c9, 0x00000000}, {0x0088, 0x00000000}, {0x0089, 0x00000000}, + {0x0108, 0x00000000}, {0x0109, 0x00000000}, {0x0148, 0x00000000}, {0x0149, 0x00000000}, + {0x0008, 0x00000000}, {0x0009, 0x00000000}, {0x0280, 0x00000000}, {0x00b2, 0x00000000}, + {0x00b3, 0x00000000}, {0x00b4, 0x00000000}, {0x00b5, 0x00000000}, {0x0132, 0x00000000}, + {0x0133, 0x00000000}, {0x0134, 0x00000000}, {0x0135, 0x00000000}, {0x0032, 0x00000000}, + {0x0033, 0x00000000}, {0x0034, 0x00000000}, {0x0035, 0x00000000}, {0x0082, 0x00000000}, + {0x0083, 0x00000000}, {0x0102, 0x00000000}, {0x0103, 0x00000000}, {0x0240, 0x00000000}, + {0x0241, 0x00000000}, {0x0242, 0x00000000}, {0x0243, 0x00000000}, {0x0244, 0x00000000}, + {0x0245, 0x00000000}, {0x0246, 0x00000000}, {0x0247, 0x00000000}, {0x0248, 0x00000000}, + {0x0249, 0x00000000}, {0x024a, 0x00000000}, {0x024b, 0x00000000}, {0x024c, 0x00000000}, + {0x024d, 0x00000000}, {0x024e, 0x00000000}, {0x024f, 0x00000000}, {0x008c, 0x00000000}, + {0x008d, 0x00000000}, {0x008e, 0x00000000}, {0x008f, 0x00000000}, {0x0090, 0x00000000}, + {0x0091, 0x00000000}, {0x0092, 0x00000000}, {0x0093, 0x00000000}, {0x0094, 0x00000000}, + {0x0095, 0x00000000}, {0x0096, 0x00000000}, {0x0097, 0x00000000}, {0x0098, 0x00000000}, + {0x0099, 0x00000000}, {0x009a, 0x00000000}, {0x009b, 0x00000000}, {0x009c, 0x00000000}, + {0x009d, 0x00000000}, {0x009e, 0x00000000}, {0x009f, 0x00000000}, {0x00a0, 0x00000000}, + {0x00a1, 0x00000000}, {0x00a2, 0x00000000}, {0x00a3, 0x00000000}, {0x00a4, 0x00000000}, + {0x00a5, 0x00000000}, {0x00a6, 0x00000000}, {0x00a7, 0x00000000}, {0x00a8, 0x00000000}, + {0x00a9, 0x00000000}, {0x00aa, 0x00000000}, {0x00ab, 0x00000000}, {0x010c, 0x00000000}, + {0x010d, 0x00000000}, {0x010e, 0x00000000}, {0x010f, 0x00000000}, {0x0110, 0x00000000}, + {0x0111, 0x00000000}, {0x0112, 0x00000000}, {0x0113, 0x00000000}, {0x0114, 0x00000000}, + {0x0115, 0x00000000}, {0x0116, 0x00000000}, {0x0117, 0x00000000}, {0x0118, 0x00000000}, + {0x0119, 0x00000000}, {0x011a, 0x00000000}, {0x011b, 0x00000000}, {0x011c, 0x00000000}, + {0x011d, 0x00000000}, {0x011e, 0x00000000}, {0x011f, 0x00000000}, {0x0120, 0x00000000}, + {0x0121, 0x00000000}, {0x0122, 0x00000000}, {0x0123, 0x00000000}, {0x0124, 0x00000000}, + {0x0125, 0x00000000}, {0x0126, 0x00000000}, {0x0127, 0x00000000}, {0x0128, 0x00000000}, + {0x0129, 0x00000000}, {0x012a, 0x00000000}, {0x012b, 0x00000000}, {0x000c, 0x00000000}, + {0x000d, 0x00000000}, {0x000e, 0x00000000}, {0x000f, 0x00000000}, {0x0010, 0x00000000}, + {0x0011, 0x00000000}, {0x0012, 0x00000000}, {0x0013, 0x00000000}, {0x0014, 0x00000000}, + {0x0015, 0x00000000}, {0x0016, 0x00000000}, {0x0017, 0x00000000}, {0x0018, 0x00000000}, + {0x0019, 0x00000000}, {0x001a, 0x00000000}, {0x001b, 0x00000000}, {0x001c, 0x00000000}, + {0x001d, 0x00000000}, {0x001e, 0x00000000}, {0x001f, 0x00000000}, {0x0020, 0x00000000}, + {0x0021, 0x00000000}, {0x0022, 0x00000000}, {0x0023, 0x00000000}, {0x0024, 0x00000000}, + {0x0025, 0x00000000}, {0x0026, 0x00000000}, {0x0027, 0x00000000}, {0x0028, 0x00000000}, + {0x0029, 0x00000000}, {0x002a, 0x00000000}, {0x002b, 0x00000000}, +}; + +static const uint16_t g_agc_public_reg_defaults_v5_tbl1_ptrs[] = { + 0x0000, 0x0001, 0x0002, 0x0003, 0x0004, 0x0005, 0x0006, 0x0007, 0x0008, 0x0009, 0x000a, 0x000b, + 0x000f, 0x0012, 0x0014, 0x0016, 0x0018, 0x001a, 0x001c, 0x001e, 0x001f, 0x0023, 0x0027, 0x002b, + 0x002d, 0x002f, 0x003f, 0x005f, 0x007f, +}; + +static ShaderRegister g_agc_public_reg_defaults_v5_tbl2_regs[] = { + {0x041f, 0x00000000}, {0x041d, 0x00000000}, {0x041e, 0x00000000}, {0x025b, 0x00000000}, + {0x024a, 0x00000000}, {0x024b, 0x00000000}, {0x025f, 0x00000000}, {0x0262, 0x00000000}, + {0x02bb, 0x00000000}, {0x02e4, 0x00000000}, {0x02e5, 0x00000000}, {0x02c4, 0x00000000}, + {0x02c5, 0x00000000}, {0x02c6, 0x00000000}, {0x02c7, 0x00000000}, {0x02e8, 0x00000000}, + {0x02e9, 0x00000000}, {0x02cc, 0x00000000}, {0x02cd, 0x00000000}, {0x02ce, 0x00000000}, + {0x02cf, 0x00000000}, {0x02bd, 0x00000000}, {0x02bc, 0x00000000}, {0x02e6, 0x00000000}, + {0x02e7, 0x00000000}, {0x02c8, 0x00000000}, {0x02c9, 0x00000000}, {0x02ca, 0x00000000}, + {0x02cb, 0x00000000}, {0x02ea, 0x00000000}, {0x02eb, 0x00000000}, {0x02d0, 0x00000000}, + {0x02d1, 0x00000000}, {0x02d2, 0x00000000}, {0x02d3, 0x00000000}, {0x02bf, 0x00000000}, + {0x02be, 0x00000000}, {0x0384, 0x00000000}, {0x0383, 0x40000040}, {0x0248, 0x00000000}, + {0x0242, 0x00000000}, {0x0380, 0x00000000}, {0x0381, 0x00000000}, {0x02e4, 0x00000000}, + {0x02e5, 0x00000000}, {0x02e6, 0x00000000}, {0x02e7, 0x00000000}, {0x02e8, 0x00000000}, + {0x02e9, 0x00000000}, {0x02ea, 0x00000000}, {0x02eb, 0x00000000}, {0x02c4, 0x00000000}, + {0x02c5, 0x00000000}, {0x02c6, 0x00000000}, {0x02c7, 0x00000000}, {0x02c8, 0x00000000}, + {0x02c9, 0x00000000}, {0x02ca, 0x00000000}, {0x02cb, 0x00000000}, {0x02cc, 0x00000000}, + {0x02cd, 0x00000000}, {0x02ce, 0x00000000}, {0x02cf, 0x00000000}, {0x02d0, 0x00000000}, + {0x02d1, 0x00000000}, {0x02d2, 0x00000000}, {0x02d3, 0x00000000}, {0x02bc, 0x00000000}, + {0x02bd, 0x00000000}, {0x02be, 0x00000000}, {0x02bf, 0x00000000}, {0x00a2, 0x00000000}, + {0x025c, 0x00000000}, {0x025d, 0x00000000}, {0x025e, 0x00000000}, +}; + +static const uint16_t g_agc_public_reg_defaults_v5_tbl2_ptrs[] = { + 0x0000, 0x0001, 0x0002, 0x0003, 0x0004, 0x0005, 0x0006, 0x0007, 0x0008, 0x0009, 0x000a, 0x000b, + 0x000c, 0x000d, 0x000e, 0x000f, 0x0010, 0x0011, 0x0012, 0x0013, 0x0014, 0x0015, 0x0016, 0x0017, + 0x0018, 0x0019, 0x001a, 0x001b, 0x001c, 0x001d, 0x001e, 0x001f, 0x0020, 0x0021, 0x0022, 0x0023, + 0x0024, 0x0025, 0x0026, 0x0027, 0x0028, 0x0029, 0x002b, 0x0033, 0x0043, 0x0047, 0x0048, +}; + +static uint32_t g_agc_public_reg_defaults_v5_types[] = { + 0xe24f806d, 0x00040400, 0x00000000, 0xf6c28182, 0x00040404, 0x00000000, + 0x6f6e55a5, 0x00040408, 0x00000000, 0x0bc65da4, 0x0004040c, 0x00000000, + 0x9e5ad592, 0x00040410, 0x00000000, 0xbb513b98, 0x00040414, 0x00000000, + 0xab64b23b, 0x00040418, 0x00000000, 0x53c39964, 0x0004041c, 0x00000000, + 0x01396b11, 0x00040420, 0x00000000, 0x7d42019a, 0x00040424, 0x00000000, + 0x3548f523, 0x00040428, 0x00000000, 0xf43ad28a, 0x0004042c, 0x00000000, + 0x6de4c312, 0x00040430, 0x00000000, 0x00a77ae0, 0x00040434, 0x00000000, + 0x00a779b7, 0x00040438, 0x00000000, 0x5100100c, 0x0004043c, 0x00000000, + 0x59958bba, 0x00040440, 0x00000000, 0x0c06f17c, 0x00040444, 0x00000000, + 0x6f104b72, 0x00040448, 0x00000000, 0x25c70d9c, 0x0004044c, 0x00000000, + 0x3881201e, 0x00040450, 0x00000000, 0x09afddaf, 0x00040454, 0x00000000, + 0x367d63cf, 0x00040458, 0x00000000, 0x43707db8, 0x0004045c, 0x00000000, + 0xf6ae26ba, 0x00040460, 0x00000000, 0x1b917652, 0x00040464, 0x00000000, + 0x2cac726e, 0x00040468, 0x00000000, 0x2cac7111, 0x0004046c, 0x00000000, + 0x94b1e4f7, 0x00040470, 0x00000000, 0xe3661b6c, 0x00040474, 0x00000000, + 0x1eb8d73a, 0x00040478, 0x00000000, 0x15051fa3, 0x0004047c, 0x00000000, + 0x9c51a7f1, 0x00040480, 0x00000000, 0xa20efc70, 0x00040484, 0x00000000, + 0x0ec09f6e, 0x00040488, 0x00000000, 0x34a7d6d3, 0x0004048c, 0x00000000, + 0xce831b94, 0x00040490, 0x00000000, 0x5cc72a74, 0x00040494, 0x00000000, + 0x3b77713c, 0x00040498, 0x00000000, 0x40f64410, 0x0004049c, 0x00000000, + 0x69441268, 0x000404a0, 0x00000000, 0x2e418b83, 0x000404a4, 0x00000000, + 0xa00d0c8d, 0x000404a8, 0x00000000, 0xb1289fb3, 0x000404ac, 0x00000000, + 0x144832fb, 0x000404b0, 0x00000000, 0x9890d9fa, 0x000404b4, 0x00000000, + 0x9016faf1, 0x000404b8, 0x00000000, 0x4b73ce27, 0x000404bc, 0x00000000, + 0x5f5a3e7b, 0x000404c0, 0x00000000, 0xd4af3a51, 0x000404c4, 0x00000000, + 0x6cf4f543, 0x000404c8, 0x00000000, 0x5fb86ccb, 0x000404cc, 0x00000000, + 0xedefa188, 0x000404d0, 0x00000000, 0xd0de9ee6, 0x000404d4, 0x00000000, + 0xc5831803, 0x000404d8, 0x00000000, 0x8e6de84b, 0x000404dc, 0x00000000, + 0xd0771662, 0x000408e0, 0x00000000, 0x569f7444, 0x004004e4, 0x00000000, + 0x5c6637cd, 0x000408e8, 0x00000000, 0xcae3e690, 0x000408ec, 0x00000000, + 0x43fbd769, 0x000410f0, 0x00000000, 0xef550356, 0x002004f4, 0x00000000, + 0x8f52e279, 0x000408f8, 0x00000000, 0x1f2d8149, 0x001008fc, 0x00000000, + 0x853d0614, 0x00040500, 0x00000000, 0x4413c6f9, 0x00040904, 0x00000000, + 0x67096014, 0x00044108, 0x00000000, 0x88f5e915, 0x0004090c, 0x00000000, + 0x033f1eff, 0x00040910, 0x00000000, 0x918106bb, 0x00040914, 0x00000000, + 0x95f0e7ac, 0x00041118, 0x00000000, 0xb48cbab2, 0x0004091c, 0x00000000, + 0x05bb3bc6, 0x00040920, 0x00000000, 0x94faba07, 0x00040924, 0x00000000, + 0x38e92c91, 0x00204128, 0x00000000, 0x0b177b43, 0x0004092c, 0x00000000, + 0x48531062, 0x00800530, 0x00000000, 0xaaa964b9, 0x00201134, 0x00000000, + 0x7690af6f, 0x00402938, 0x00000000, 0x078d7060, 0x0004093c, 0x00000000, + 0x5d6e3ec7, 0x00040401, 0x00000000, 0x57e7079a, 0x00040405, 0x00000000, + 0x7467fafd, 0x00040409, 0x00000000, 0x9e826b50, 0x0004040d, 0x00000000, + 0xdc484f18, 0x00040411, 0x00000000, 0x5da8bca3, 0x00040415, 0x00000000, + 0x5ca726d8, 0x00040419, 0x00000000, 0x5dd28360, 0x0004041d, 0x00000000, + 0x57efa0be, 0x00040421, 0x00000000, 0x502363d5, 0x00040425, 0x00000000, + 0x506d14bd, 0x00040429, 0x00000000, 0xb2609506, 0x0010042d, 0x00000000, + 0x9e5cfb8a, 0x00040c31, 0x00000000, 0xc918df3e, 0x00040835, 0x00000000, + 0xc9751c9c, 0x00040839, 0x00000000, 0xc97ef77a, 0x0004083d, 0x00000000, + 0xc927c6b9, 0x00040841, 0x00000000, 0xc92a1ec5, 0x00040845, 0x00000000, + 0xc9e01b31, 0x00040849, 0x00000000, 0x50685f29, 0x0004044d, 0x00000000, + 0xb26219ca, 0x00100451, 0x00000000, 0xb25b6cf9, 0x00100455, 0x00000000, + 0xb2f86101, 0x00100459, 0x00000000, 0x07e3b155, 0x0004085d, 0x00000000, + 0x07e383c6, 0x00040861, 0x00000000, 0xbda98653, 0x00400465, 0x00000000, + 0xbdbd1d0f, 0x00800469, 0x00000000, 0xbd946fd4, 0x0080046d, 0x00000000, + 0xbdf02a4c, 0x00800471, 0x00000000, 0x19e93e85, 0x00040402, 0x00000000, + 0x3b5c2af3, 0x00040406, 0x00000000, 0x47974a35, 0x0004040a, 0x00000000, + 0x105971c2, 0x0004040e, 0x00000000, 0x7d137765, 0x00040412, 0x00000000, + 0xd187febc, 0x00040416, 0x00000000, 0x12f854ac, 0x0004041a, 0x00000000, + 0x40d49ad1, 0x0004041e, 0x00000000, 0x8c0923da, 0x00040422, 0x00000000, + 0xf6be6f25, 0x00040426, 0x00000000, 0xf6908bad, 0x0004042a, 0x00000000, + 0x862ee1be, 0x0004042e, 0x00000000, 0x8631d396, 0x00040432, 0x00000000, + 0x89d7d386, 0x00040436, 0x00000000, 0xd1006f62, 0x0004043a, 0x00000000, + 0x7e51d354, 0x0004043e, 0x00000000, 0x7e712edc, 0x00040442, 0x00000000, + 0x141f6002, 0x00040446, 0x00000000, 0x140c761a, 0x0004044a, 0x00000000, + 0x14f68dfa, 0x0004044e, 0x00000000, 0x6fea32c3, 0x00040452, 0x00000000, + 0x8f378499, 0x00040456, 0x00000000, 0x5d9ed616, 0x0004045a, 0x00000000, + 0x2bbfa784, 0x0004045e, 0x00000000, 0x2b45f28c, 0x00040462, 0x00000000, + 0x5f4aaed9, 0x00040466, 0x00000000, 0x5f477465, 0x0004046a, 0x00000000, + 0x5ec84729, 0x0004046e, 0x00000000, 0x48ca8407, 0x00040472, 0x00000000, + 0x5f355d79, 0x00040476, 0x00000000, 0x5f0f3449, 0x0004047a, 0x00000000, + 0xee4a4919, 0x0004047e, 0x00000000, 0xee49852d, 0x00040482, 0x00000000, + 0xee4f8201, 0x00040486, 0x00000000, 0x8a3f99a1, 0x0004048a, 0x00000000, + 0x381fc91f, 0x0004048e, 0x00000000, 0x19d8287b, 0x00040492, 0x00000000, + 0xbb8df494, 0x00040496, 0x00000000, 0xf6d8a76e, 0x0004049a, 0x00000000, + 0x7620f1e9, 0x0004049e, 0x00000000, 0x9ebfab10, 0x000404a2, 0x00000000, + 0x98a09d0e, 0x000408a6, 0x00000000, 0x195d37d2, 0x001008aa, 0x00000000, + 0xf9ec4f85, 0x001010ae, 0x00000000, 0x4626b750, 0x000808b2, 0x00000000, + 0xde5b3431, 0x000404b6, 0x00000000, 0x036ac8a6, 0x000c04ba, 0x00000000, +}; + +static CompactRegisterDefaults g_agc_public_reg_defaults_v5 = { + g_agc_public_reg_defaults_v5_tbl0_regs, 491, g_agc_public_reg_defaults_v5_tbl0_ptrs, 80, + g_agc_public_reg_defaults_v5_tbl1_regs, 159, g_agc_public_reg_defaults_v5_tbl1_ptrs, 29, + g_agc_public_reg_defaults_v5_tbl2_regs, 75, g_agc_public_reg_defaults_v5_tbl2_ptrs, 47, + nullptr, 0, nullptr, 0, + g_agc_public_reg_defaults_v5_types, 156, +}; + +// g_agc_public_reg_defaults_v7: regs=(489, 159, 55, 0), ptrs=(78, 29, 20, 0), types=127. +static ShaderRegister g_agc_public_reg_defaults_v7_tbl0_regs[] = { + {0x0202, 0x00cc0010}, {0x0109, 0x00000008}, {0x0104, 0x00000000}, {0x008f, 0x00000000}, + {0x008e, 0x00000000}, {0x02dc, 0x0000aa00}, {0x0001, 0x11000100}, {0x0200, 0x00000000}, + {0x0201, 0x00000000}, {0x0000, 0x00000000}, {0x0006, 0x00000000}, {0x001f, 0x00000000}, + {0x0203, 0x00000000}, {0x02b0, 0x00000000}, {0x02b1, 0x00000000}, {0x010c, 0x00000000}, + {0x010d, 0x00000000}, {0x010b, 0x00000000}, {0x01ff, 0x00000000}, {0x0204, 0x00000000}, + {0x020d, 0x00000000}, {0x0206, 0x0000043f}, {0x02f8, 0x00000000}, {0x0083, 0x0000ffff}, + {0x0313, 0x00006000}, {0x00f0, 0x00000000}, {0x00ea, 0x00000000}, {0x00e9, 0x00000000}, + {0x0292, 0x00000002}, {0x0293, 0x06020000}, {0x00e8, 0x00000000}, {0x0080, 0x00000000}, + {0x0211, 0x00000000}, {0x0210, 0x04000000}, {0x008d, 0x00000000}, {0x0282, 0x00000008}, + {0x0281, 0xffff0000}, {0x0280, 0x00080008}, {0x02df, 0x00000000}, {0x02de, 0x000001e9}, + {0x0205, 0x00000240}, {0x020c, 0x00000021}, {0x02f9, 0x0000002d}, {0x01ba, 0x00000000}, + {0x02a6, 0x00000000}, {0x02ce, 0x00000400}, {0x029b, 0x00000002}, {0x02d6, 0x00000000}, + {0x0103, 0xffffffff}, {0x02a1, 0x00000000}, {0x02ad, 0x00000000}, {0x02d5, 0x00002000}, + {0x02d4, 0x88101010}, {0x02db, 0x00000000}, {0x02f5, 0x00000000}, {0x02f6, 0x00000000}, + {0x02fe, 0x00000000}, {0x02ff, 0x00000000}, {0x0300, 0x00000000}, {0x0301, 0x00000000}, + {0x0302, 0x00000000}, {0x0303, 0x00000000}, {0x0304, 0x00000000}, {0x0305, 0x00000000}, + {0x0306, 0x00000000}, {0x0307, 0x00000000}, {0x0308, 0x00000000}, {0x0309, 0x00000000}, + {0x030a, 0x00000000}, {0x030b, 0x00000000}, {0x030c, 0x00000000}, {0x030d, 0x00000000}, + {0x030e, 0xffffffff}, {0x030f, 0xffffffff}, {0x0311, 0x01fd2002}, {0x0312, 0x03ff0080}, + {0x0105, 0x00000000}, {0x0106, 0x00000000}, {0x0107, 0x00000000}, {0x0108, 0x00000000}, + {0x01e0, 0x20010001}, {0x01e1, 0x20010001}, {0x01e2, 0x20010001}, {0x01e3, 0x20010001}, + {0x01e4, 0x20010001}, {0x01e5, 0x20010001}, {0x01e6, 0x20010001}, {0x01e7, 0x20010001}, + {0x0020, 0x00000000}, {0x0021, 0x00000000}, {0x0084, 0x00000000}, {0x0085, 0x20002000}, + {0x0086, 0x00000000}, {0x0087, 0x20002000}, {0x0088, 0x00000000}, {0x0089, 0x20002000}, + {0x008a, 0x00000000}, {0x008b, 0x20002000}, {0x00db, 0x00000000}, {0x0008, 0x00000000}, + {0x0009, 0x00000000}, {0x0010, 0x80000180}, {0x0011, 0x20000180}, {0x0012, 0x00000000}, + {0x0013, 0x00000000}, {0x0014, 0x00000000}, {0x0015, 0x00000000}, {0x001a, 0x00000000}, + {0x001b, 0x00000000}, {0x001c, 0x00000000}, {0x001d, 0x00000000}, {0x001e, 0x00000000}, + {0x0002, 0x00000000}, {0x0005, 0x00000000}, {0x0007, 0x00000000}, {0x000b, 0x00000000}, + {0x000a, 0x00000000}, {0x00eb, 0xff00ff00}, {0x00ec, 0x00010000}, {0x00f1, 0x00000000}, + {0x00f2, 0x00000000}, {0x0090, 0x80000000}, {0x0091, 0x40004000}, {0x02fa, 0x3f800000}, + {0x02fb, 0x3f800000}, {0x02fc, 0x3f800000}, {0x02fd, 0x3f800000}, {0x02e2, 0x00000000}, + {0x02e3, 0x00000000}, {0x02e0, 0x00000000}, {0x02e1, 0x00000000}, {0x0003, 0x00000000}, + {0x0004, 0x00000000}, {0x0318, 0x00000000}, {0x031b, 0x00000000}, {0x031c, 0x00000000}, + {0x031d, 0x00000000}, {0x031e, 0x00000048}, {0x031f, 0x00000000}, {0x0321, 0x00000000}, + {0x0323, 0x00000000}, {0x0324, 0x00000000}, {0x0325, 0x00000000}, {0x0390, 0x00000000}, + {0x0398, 0x00000000}, {0x03a0, 0x00000000}, {0x03a8, 0x00000000}, {0x03b0, 0x00000000}, + {0x03b8, 0x08c6c000}, {0x0327, 0x00000000}, {0x032a, 0x00000000}, {0x032b, 0x00000000}, + {0x032c, 0x00000000}, {0x032d, 0x00000048}, {0x032e, 0x00000000}, {0x0330, 0x00000000}, + {0x0332, 0x00000000}, {0x0333, 0x00000000}, {0x0334, 0x00000000}, {0x0391, 0x00000000}, + {0x0399, 0x00000000}, {0x03a1, 0x00000000}, {0x03a9, 0x00000000}, {0x03b1, 0x00000000}, + {0x03b9, 0x08c6c000}, {0x0336, 0x00000000}, {0x0339, 0x00000000}, {0x033a, 0x00000000}, + {0x033b, 0x00000000}, {0x033c, 0x00000048}, {0x033d, 0x00000000}, {0x033f, 0x00000000}, + {0x0341, 0x00000000}, {0x0342, 0x00000000}, {0x0343, 0x00000000}, {0x0392, 0x00000000}, + {0x039a, 0x00000000}, {0x03a2, 0x00000000}, {0x03aa, 0x00000000}, {0x03b2, 0x00000000}, + {0x03ba, 0x08c6c000}, {0x0345, 0x00000000}, {0x0348, 0x00000000}, {0x0349, 0x00000000}, + {0x034a, 0x00000000}, {0x034b, 0x00000048}, {0x034c, 0x00000000}, {0x034e, 0x00000000}, + {0x0350, 0x00000000}, {0x0351, 0x00000000}, {0x0352, 0x00000000}, {0x0393, 0x00000000}, + {0x039b, 0x00000000}, {0x03a3, 0x00000000}, {0x03ab, 0x00000000}, {0x03b3, 0x00000000}, + {0x03bb, 0x08c6c000}, {0x0354, 0x00000000}, {0x0357, 0x00000000}, {0x0358, 0x00000000}, + {0x0359, 0x00000000}, {0x035a, 0x00000048}, {0x035b, 0x00000000}, {0x035d, 0x00000000}, + {0x035f, 0x00000000}, {0x0360, 0x00000000}, {0x0361, 0x00000000}, {0x0394, 0x00000000}, + {0x039c, 0x00000000}, {0x03a4, 0x00000000}, {0x03ac, 0x00000000}, {0x03b4, 0x00000000}, + {0x03bc, 0x08c6c000}, {0x0363, 0x00000000}, {0x0366, 0x00000000}, {0x0367, 0x00000000}, + {0x0368, 0x00000000}, {0x0369, 0x00000048}, {0x036a, 0x00000000}, {0x036c, 0x00000000}, + {0x036e, 0x00000000}, {0x036f, 0x00000000}, {0x0370, 0x00000000}, {0x0395, 0x00000000}, + {0x039d, 0x00000000}, {0x03a5, 0x00000000}, {0x03ad, 0x00000000}, {0x03b5, 0x00000000}, + {0x03bd, 0x08c6c000}, {0x0372, 0x00000000}, {0x0375, 0x00000000}, {0x0376, 0x00000000}, + {0x0377, 0x00000000}, {0x0378, 0x00000048}, {0x0379, 0x00000000}, {0x037b, 0x00000000}, + {0x037d, 0x00000000}, {0x037e, 0x00000000}, {0x037f, 0x00000000}, {0x0396, 0x00000000}, + {0x039e, 0x00000000}, {0x03a6, 0x00000000}, {0x03ae, 0x00000000}, {0x03b6, 0x00000000}, + {0x03be, 0x08c6c000}, {0x0381, 0x00000000}, {0x0384, 0x00000000}, {0x0385, 0x00000000}, + {0x0386, 0x00000000}, {0x0387, 0x00000048}, {0x0388, 0x00000000}, {0x038a, 0x00000000}, + {0x038c, 0x00000000}, {0x038d, 0x00000000}, {0x038e, 0x00000000}, {0x0397, 0x00000000}, + {0x039f, 0x00000000}, {0x03a7, 0x00000000}, {0x03af, 0x00000000}, {0x03b7, 0x00000000}, + {0x03bf, 0x08c6c000}, {0x000c, 0x00000000}, {0x000d, 0x40004000}, {0x0191, 0x00000000}, + {0x0192, 0x00000000}, {0x0193, 0x00000000}, {0x0194, 0x00000000}, {0x0195, 0x00000000}, + {0x0196, 0x00000000}, {0x0197, 0x00000000}, {0x0198, 0x00000000}, {0x0199, 0x00000000}, + {0x019a, 0x00000000}, {0x019b, 0x00000000}, {0x019c, 0x00000000}, {0x019d, 0x00000000}, + {0x019e, 0x00000000}, {0x019f, 0x00000000}, {0x01a0, 0x00000000}, {0x01a1, 0x00000000}, + {0x01a2, 0x00000000}, {0x01a3, 0x00000000}, {0x01a4, 0x00000000}, {0x01a5, 0x00000000}, + {0x01a6, 0x00000000}, {0x01a7, 0x00000000}, {0x01a8, 0x00000000}, {0x01a9, 0x00000000}, + {0x01aa, 0x00000000}, {0x01ab, 0x00000000}, {0x01ac, 0x00000000}, {0x01ad, 0x00000000}, + {0x01ae, 0x00000000}, {0x01af, 0x00000000}, {0x01b0, 0x00000000}, {0x016f, 0x00000000}, + {0x0170, 0x00000000}, {0x0171, 0x00000000}, {0x0172, 0x00000000}, {0x0173, 0x00000000}, + {0x0174, 0x00000000}, {0x0175, 0x00000000}, {0x0176, 0x00000000}, {0x0177, 0x00000000}, + {0x0178, 0x00000000}, {0x0179, 0x00000000}, {0x017a, 0x00000000}, {0x017b, 0x00000000}, + {0x017c, 0x00000000}, {0x017d, 0x00000000}, {0x017e, 0x00000000}, {0x017f, 0x00000000}, + {0x0180, 0x00000000}, {0x0181, 0x00000000}, {0x0182, 0x00000000}, {0x0183, 0x00000000}, + {0x0184, 0x00000000}, {0x0185, 0x00000000}, {0x0186, 0x00000000}, {0x0187, 0x00000000}, + {0x0188, 0x00000000}, {0x0189, 0x00000000}, {0x018a, 0x00000000}, {0x018b, 0x00000000}, + {0x018c, 0x00000000}, {0x018d, 0x00000000}, {0x018e, 0x00000000}, {0x010f, 0x3f800000}, + {0x0111, 0x3f800000}, {0x0113, 0x3f800000}, {0x0110, 0x00000000}, {0x0112, 0x00000000}, + {0x0114, 0x00000000}, {0x0094, 0x80000000}, {0x0095, 0x40004000}, {0x00b4, 0x00000000}, + {0x00b5, 0x00000000}, {0x0115, 0x3f800000}, {0x0117, 0x3f800000}, {0x0119, 0x3f800000}, + {0x0116, 0x00000000}, {0x0118, 0x00000000}, {0x011a, 0x00000000}, {0x0096, 0x80000000}, + {0x0097, 0x40004000}, {0x00b6, 0x00000000}, {0x00b7, 0x00000000}, {0x011b, 0x3f800000}, + {0x011d, 0x3f800000}, {0x011f, 0x3f800000}, {0x011c, 0x00000000}, {0x011e, 0x00000000}, + {0x0120, 0x00000000}, {0x0098, 0x80000000}, {0x0099, 0x40004000}, {0x00b8, 0x00000000}, + {0x00b9, 0x00000000}, {0x0121, 0x3f800000}, {0x0123, 0x3f800000}, {0x0125, 0x3f800000}, + {0x0122, 0x00000000}, {0x0124, 0x00000000}, {0x0126, 0x00000000}, {0x009a, 0x80000000}, + {0x009b, 0x40004000}, {0x00ba, 0x00000000}, {0x00bb, 0x00000000}, {0x0127, 0x3f800000}, + {0x0129, 0x3f800000}, {0x012b, 0x3f800000}, {0x0128, 0x00000000}, {0x012a, 0x00000000}, + {0x012c, 0x00000000}, {0x009c, 0x80000000}, {0x009d, 0x40004000}, {0x00bc, 0x00000000}, + {0x00bd, 0x00000000}, {0x012d, 0x3f800000}, {0x012f, 0x3f800000}, {0x0131, 0x3f800000}, + {0x012e, 0x00000000}, {0x0130, 0x00000000}, {0x0132, 0x00000000}, {0x009e, 0x80000000}, + {0x009f, 0x40004000}, {0x00be, 0x00000000}, {0x00bf, 0x00000000}, {0x0133, 0x3f800000}, + {0x0135, 0x3f800000}, {0x0137, 0x3f800000}, {0x0134, 0x00000000}, {0x0136, 0x00000000}, + {0x0138, 0x00000000}, {0x00a0, 0x80000000}, {0x00a1, 0x40004000}, {0x00c0, 0x00000000}, + {0x00c1, 0x00000000}, {0x0139, 0x3f800000}, {0x013b, 0x3f800000}, {0x013d, 0x3f800000}, + {0x013a, 0x00000000}, {0x013c, 0x00000000}, {0x013e, 0x00000000}, {0x00a2, 0x80000000}, + {0x00a3, 0x40004000}, {0x00c2, 0x00000000}, {0x00c3, 0x00000000}, {0x013f, 0x3f800000}, + {0x0141, 0x3f800000}, {0x0143, 0x3f800000}, {0x0140, 0x00000000}, {0x0142, 0x00000000}, + {0x0144, 0x00000000}, {0x00a4, 0x80000000}, {0x00a5, 0x40004000}, {0x00c4, 0x00000000}, + {0x00c5, 0x00000000}, {0x0145, 0x3f800000}, {0x0147, 0x3f800000}, {0x0149, 0x3f800000}, + {0x0146, 0x00000000}, {0x0148, 0x00000000}, {0x014a, 0x00000000}, {0x00a6, 0x80000000}, + {0x00a7, 0x40004000}, {0x00c6, 0x00000000}, {0x00c7, 0x00000000}, {0x014b, 0x3f800000}, + {0x014d, 0x3f800000}, {0x014f, 0x3f800000}, {0x014c, 0x00000000}, {0x014e, 0x00000000}, + {0x0150, 0x00000000}, {0x00a8, 0x80000000}, {0x00a9, 0x40004000}, {0x00c8, 0x00000000}, + {0x00c9, 0x00000000}, {0x0151, 0x3f800000}, {0x0153, 0x3f800000}, {0x0155, 0x3f800000}, + {0x0152, 0x00000000}, {0x0154, 0x00000000}, {0x0156, 0x00000000}, {0x00aa, 0x80000000}, + {0x00ab, 0x40004000}, {0x00ca, 0x00000000}, {0x00cb, 0x00000000}, {0x0157, 0x3f800000}, + {0x0159, 0x3f800000}, {0x015b, 0x3f800000}, {0x0158, 0x00000000}, {0x015a, 0x00000000}, + {0x015c, 0x00000000}, {0x00ac, 0x80000000}, {0x00ad, 0x40004000}, {0x00cc, 0x00000000}, + {0x00cd, 0x00000000}, {0x015d, 0x3f800000}, {0x015f, 0x3f800000}, {0x0161, 0x3f800000}, + {0x015e, 0x00000000}, {0x0160, 0x00000000}, {0x0162, 0x00000000}, {0x00ae, 0x80000000}, + {0x00af, 0x40004000}, {0x00ce, 0x00000000}, {0x00cf, 0x00000000}, {0x0163, 0x3f800000}, + {0x0165, 0x3f800000}, {0x0167, 0x3f800000}, {0x0164, 0x00000000}, {0x0166, 0x00000000}, + {0x0168, 0x00000000}, {0x00b0, 0x80000000}, {0x00b1, 0x40004000}, {0x00d0, 0x00000000}, + {0x00d1, 0x00000000}, {0x0169, 0x3f800000}, {0x016b, 0x3f800000}, {0x016d, 0x3f800000}, + {0x016a, 0x00000000}, {0x016c, 0x00000000}, {0x016e, 0x00000000}, {0x00b2, 0x80000000}, + {0x00b3, 0x40004000}, {0x00d2, 0x00000000}, {0x00d3, 0x00000000}, {0x0081, 0x80000000}, + {0x0082, 0x40004000}, +}; + +static const uint16_t g_agc_public_reg_defaults_v7_tbl0_ptrs[] = { + 0x0000, 0x0001, 0x0002, 0x0003, 0x0004, 0x0005, 0x0006, 0x0007, 0x0008, 0x0009, 0x000a, 0x000b, + 0x000c, 0x000d, 0x000e, 0x000f, 0x0010, 0x0011, 0x0012, 0x0013, 0x0014, 0x0015, 0x0016, 0x0017, + 0x0018, 0x0019, 0x001a, 0x001b, 0x001c, 0x001d, 0x001e, 0x001f, 0x0020, 0x0021, 0x0022, 0x0023, + 0x0024, 0x0025, 0x0026, 0x0027, 0x0028, 0x0029, 0x002a, 0x002b, 0x002c, 0x002d, 0x002e, 0x002f, + 0x0030, 0x0031, 0x0032, 0x0033, 0x0034, 0x0035, 0x0036, 0x0038, 0x0048, 0x004a, 0x004c, 0x0050, + 0x0058, 0x005a, 0x0062, 0x0063, 0x0065, 0x0075, 0x0077, 0x0079, 0x007b, 0x007f, 0x0081, 0x0083, + 0x0085, 0x0105, 0x0107, 0x0127, 0x0147, 0x01e7, +}; + +static ShaderRegister g_agc_public_reg_defaults_v7_tbl1_regs[] = { + {0x0212, 0x00000000}, {0x0213, 0x00000000}, {0x0228, 0x00000000}, {0x0215, 0x00000000}, + {0x0218, 0x00000000}, {0x008a, 0x00000000}, {0x010a, 0x00000000}, {0x000a, 0x00000000}, + {0x008b, 0x00000000}, {0x010b, 0x00000000}, {0x000b, 0x00000000}, {0x0224, 0x00000000}, + {0x0225, 0x00000000}, {0x0226, 0x00000000}, {0x0227, 0x00000000}, {0x0107, 0xffff0000}, + {0x0087, 0x0000ffff}, {0x0007, 0x0000ffff}, {0x020c, 0x00000000}, {0x020d, 0x00000000}, + {0x00c8, 0x00000000}, {0x00c9, 0x00000000}, {0x0088, 0x00000000}, {0x0089, 0x00000000}, + {0x0108, 0x00000000}, {0x0109, 0x00000000}, {0x0148, 0x00000000}, {0x0149, 0x00000000}, + {0x0008, 0x00000000}, {0x0009, 0x00000000}, {0x0280, 0x00000000}, {0x00b2, 0x00000000}, + {0x00b3, 0x00000000}, {0x00b4, 0x00000000}, {0x00b5, 0x00000000}, {0x0132, 0x00000000}, + {0x0133, 0x00000000}, {0x0134, 0x00000000}, {0x0135, 0x00000000}, {0x0032, 0x00000000}, + {0x0033, 0x00000000}, {0x0034, 0x00000000}, {0x0035, 0x00000000}, {0x0082, 0x00000000}, + {0x0083, 0x00000000}, {0x0102, 0x00000000}, {0x0103, 0x00000000}, {0x0240, 0x00000000}, + {0x0241, 0x00000000}, {0x0242, 0x00000000}, {0x0243, 0x00000000}, {0x0244, 0x00000000}, + {0x0245, 0x00000000}, {0x0246, 0x00000000}, {0x0247, 0x00000000}, {0x0248, 0x00000000}, + {0x0249, 0x00000000}, {0x024a, 0x00000000}, {0x024b, 0x00000000}, {0x024c, 0x00000000}, + {0x024d, 0x00000000}, {0x024e, 0x00000000}, {0x024f, 0x00000000}, {0x008c, 0x00000000}, + {0x008d, 0x00000000}, {0x008e, 0x00000000}, {0x008f, 0x00000000}, {0x0090, 0x00000000}, + {0x0091, 0x00000000}, {0x0092, 0x00000000}, {0x0093, 0x00000000}, {0x0094, 0x00000000}, + {0x0095, 0x00000000}, {0x0096, 0x00000000}, {0x0097, 0x00000000}, {0x0098, 0x00000000}, + {0x0099, 0x00000000}, {0x009a, 0x00000000}, {0x009b, 0x00000000}, {0x009c, 0x00000000}, + {0x009d, 0x00000000}, {0x009e, 0x00000000}, {0x009f, 0x00000000}, {0x00a0, 0x00000000}, + {0x00a1, 0x00000000}, {0x00a2, 0x00000000}, {0x00a3, 0x00000000}, {0x00a4, 0x00000000}, + {0x00a5, 0x00000000}, {0x00a6, 0x00000000}, {0x00a7, 0x00000000}, {0x00a8, 0x00000000}, + {0x00a9, 0x00000000}, {0x00aa, 0x00000000}, {0x00ab, 0x00000000}, {0x010c, 0x00000000}, + {0x010d, 0x00000000}, {0x010e, 0x00000000}, {0x010f, 0x00000000}, {0x0110, 0x00000000}, + {0x0111, 0x00000000}, {0x0112, 0x00000000}, {0x0113, 0x00000000}, {0x0114, 0x00000000}, + {0x0115, 0x00000000}, {0x0116, 0x00000000}, {0x0117, 0x00000000}, {0x0118, 0x00000000}, + {0x0119, 0x00000000}, {0x011a, 0x00000000}, {0x011b, 0x00000000}, {0x011c, 0x00000000}, + {0x011d, 0x00000000}, {0x011e, 0x00000000}, {0x011f, 0x00000000}, {0x0120, 0x00000000}, + {0x0121, 0x00000000}, {0x0122, 0x00000000}, {0x0123, 0x00000000}, {0x0124, 0x00000000}, + {0x0125, 0x00000000}, {0x0126, 0x00000000}, {0x0127, 0x00000000}, {0x0128, 0x00000000}, + {0x0129, 0x00000000}, {0x012a, 0x00000000}, {0x012b, 0x00000000}, {0x000c, 0x00000000}, + {0x000d, 0x00000000}, {0x000e, 0x00000000}, {0x000f, 0x00000000}, {0x0010, 0x00000000}, + {0x0011, 0x00000000}, {0x0012, 0x00000000}, {0x0013, 0x00000000}, {0x0014, 0x00000000}, + {0x0015, 0x00000000}, {0x0016, 0x00000000}, {0x0017, 0x00000000}, {0x0018, 0x00000000}, + {0x0019, 0x00000000}, {0x001a, 0x00000000}, {0x001b, 0x00000000}, {0x001c, 0x00000000}, + {0x001d, 0x00000000}, {0x001e, 0x00000000}, {0x001f, 0x00000000}, {0x0020, 0x00000000}, + {0x0021, 0x00000000}, {0x0022, 0x00000000}, {0x0023, 0x00000000}, {0x0024, 0x00000000}, + {0x0025, 0x00000000}, {0x0026, 0x00000000}, {0x0027, 0x00000000}, {0x0028, 0x00000000}, + {0x0029, 0x00000000}, {0x002a, 0x00000000}, {0x002b, 0x00000000}, +}; + +static const uint16_t g_agc_public_reg_defaults_v7_tbl1_ptrs[] = { + 0x0000, 0x0001, 0x0002, 0x0003, 0x0004, 0x0005, 0x0006, 0x0007, 0x0008, 0x0009, 0x000a, 0x000b, + 0x000f, 0x0012, 0x0014, 0x0016, 0x0018, 0x001a, 0x001c, 0x001e, 0x001f, 0x0023, 0x0027, 0x002b, + 0x002d, 0x002f, 0x003f, 0x005f, 0x007f, +}; + +static ShaderRegister g_agc_public_reg_defaults_v7_tbl2_regs[] = { + {0x041f, 0x00000000}, {0x041d, 0x00000000}, {0x041e, 0x00000000}, {0x025b, 0x00000000}, + {0x024a, 0x00000000}, {0x024b, 0x00000000}, {0x025f, 0x00000000}, {0x0262, 0x00000000}, + {0x02bb, 0x00000000}, {0x0384, 0x00000000}, {0x0383, 0x40000040}, {0x0248, 0x00000000}, + {0x0242, 0x00000000}, {0x0380, 0x00000000}, {0x0381, 0x00000000}, {0x02e4, 0x00000000}, + {0x02e5, 0x00000000}, {0x02e6, 0x00000000}, {0x02e7, 0x00000000}, {0x02e8, 0x00000000}, + {0x02e9, 0x00000000}, {0x02ea, 0x00000000}, {0x02eb, 0x00000000}, {0x02c4, 0x00000000}, + {0x02c5, 0x00000000}, {0x02c6, 0x00000000}, {0x02c7, 0x00000000}, {0x02c8, 0x00000000}, + {0x02c9, 0x00000000}, {0x02ca, 0x00000000}, {0x02cb, 0x00000000}, {0x02cc, 0x00000000}, + {0x02cd, 0x00000000}, {0x02ce, 0x00000000}, {0x02cf, 0x00000000}, {0x02d0, 0x00000000}, + {0x02d1, 0x00000000}, {0x02d2, 0x00000000}, {0x02d3, 0x00000000}, {0x02bc, 0x00000000}, + {0x02bd, 0x00000000}, {0x02be, 0x00000000}, {0x02bf, 0x00000000}, {0x0035, 0x00000000}, + {0x0036, 0x00000000}, {0x0037, 0x00000000}, {0x0038, 0x00000000}, {0x0039, 0x00000000}, + {0x003a, 0x00000000}, {0x003b, 0x00000000}, {0x003c, 0x00000000}, {0x00a2, 0x00000000}, + {0x025c, 0x00000000}, {0x025d, 0x00000000}, {0x025e, 0x00000000}, +}; + +static const uint16_t g_agc_public_reg_defaults_v7_tbl2_ptrs[] = { + 0x0000, 0x0001, 0x0002, 0x0003, 0x0004, 0x0005, 0x0006, 0x0007, 0x0008, 0x0009, 0x000a, 0x000b, + 0x000c, 0x000d, 0x000f, 0x0017, 0x0027, 0x002b, 0x0033, 0x0034, +}; + +static uint32_t g_agc_public_reg_defaults_v7_types[] = { + 0xe24f806d, 0x00040400, 0x00000000, 0xf6c28182, 0x00040404, 0x00000000, + 0x6f6e55a5, 0x00040408, 0x00000000, 0x0bc65da4, 0x0004040c, 0x00000000, + 0x9e5ad592, 0x00040410, 0x00000000, 0xbb513b98, 0x00040414, 0x00000000, + 0xab64b23b, 0x00040418, 0x00000000, 0x53c39964, 0x0004041c, 0x00000000, + 0x01396b11, 0x00040420, 0x00000000, 0x7d42019a, 0x00040424, 0x00000000, + 0x3548f523, 0x00040428, 0x00000000, 0xf43ad28a, 0x0004042c, 0x00000000, + 0x6de4c312, 0x00040430, 0x00000000, 0x00a77ae0, 0x00040434, 0x00000000, + 0x00a779b7, 0x00040438, 0x00000000, 0x5100100c, 0x0004043c, 0x00000000, + 0x59958bba, 0x00040440, 0x00000000, 0x0c06f17c, 0x00040444, 0x00000000, + 0x6f104b72, 0x00040448, 0x00000000, 0x25c70d9c, 0x0004044c, 0x00000000, + 0x3881201e, 0x00040450, 0x00000000, 0x09afddaf, 0x00040454, 0x00000000, + 0x367d63cf, 0x00040458, 0x00000000, 0x43707db8, 0x0004045c, 0x00000000, + 0xf6ae26ba, 0x00040460, 0x00000000, 0x1b917652, 0x00040464, 0x00000000, + 0x94b1e4f7, 0x00040468, 0x00000000, 0xe3661b6c, 0x0004046c, 0x00000000, + 0x1eb8d73a, 0x00040470, 0x00000000, 0x15051fa3, 0x00040474, 0x00000000, + 0x9c51a7f1, 0x00040478, 0x00000000, 0xa20efc70, 0x0004047c, 0x00000000, + 0x0ec09f6e, 0x00040480, 0x00000000, 0x34a7d6d3, 0x00040484, 0x00000000, + 0xce831b94, 0x00040488, 0x00000000, 0x5cc72a74, 0x0004048c, 0x00000000, + 0x3b77713c, 0x00040490, 0x00000000, 0x40f64410, 0x00040494, 0x00000000, + 0x69441268, 0x00040498, 0x00000000, 0x2e418b83, 0x0004049c, 0x00000000, + 0xa00d0c8d, 0x000404a0, 0x00000000, 0xb1289fb3, 0x000404a4, 0x00000000, + 0x144832fb, 0x000404a8, 0x00000000, 0x9890d9fa, 0x000404ac, 0x00000000, + 0x9016faf1, 0x000404b0, 0x00000000, 0x4b73ce27, 0x000404b4, 0x00000000, + 0x5f5a3e7b, 0x000404b8, 0x00000000, 0xd4af3a51, 0x000404bc, 0x00000000, + 0x6cf4f543, 0x000404c0, 0x00000000, 0x5fb86ccb, 0x000404c4, 0x00000000, + 0xedefa188, 0x000404c8, 0x00000000, 0xd0de9ee6, 0x000404cc, 0x00000000, + 0xc5831803, 0x000404d0, 0x00000000, 0x8e6de84b, 0x000404d4, 0x00000000, + 0xd0771662, 0x000408d8, 0x00000000, 0x569f7444, 0x004004dc, 0x00000000, + 0x5c6637cd, 0x000408e0, 0x00000000, 0xcae3e690, 0x000408e4, 0x00000000, + 0x43fbd769, 0x000410e8, 0x00000000, 0xef550356, 0x002004ec, 0x00000000, + 0x8f52e279, 0x000408f0, 0x00000000, 0x1f2d8149, 0x001008f4, 0x00000000, + 0x853d0614, 0x000404f8, 0x00000000, 0x4413c6f9, 0x000408fc, 0x00000000, + 0x67096014, 0x00044100, 0x00000000, 0x88f5e915, 0x00040904, 0x00000000, + 0x033f1eff, 0x00040908, 0x00000000, 0x918106bb, 0x0004090c, 0x00000000, + 0x95f0e7ac, 0x00041110, 0x00000000, 0xb48cbab2, 0x00040914, 0x00000000, + 0x05bb3bc6, 0x00040918, 0x00000000, 0x94faba07, 0x0004091c, 0x00000000, + 0x38e92c91, 0x00204120, 0x00000000, 0x0b177b43, 0x00040924, 0x00000000, + 0x48531062, 0x00800528, 0x00000000, 0xaaa964b9, 0x0020112c, 0x00000000, + 0x7690af6f, 0x00402930, 0x00000000, 0x078d7060, 0x00040934, 0x00000000, + 0x5d6e3ec7, 0x00040401, 0x00000000, 0x57e7079a, 0x00040405, 0x00000000, + 0x7467fafd, 0x00040409, 0x00000000, 0x9e826b50, 0x0004040d, 0x00000000, + 0xdc484f18, 0x00040411, 0x00000000, 0x5da8bca3, 0x00040415, 0x00000000, + 0x5ca726d8, 0x00040419, 0x00000000, 0x5dd28360, 0x0004041d, 0x00000000, + 0x57efa0be, 0x00040421, 0x00000000, 0x502363d5, 0x00040425, 0x00000000, + 0x506d14bd, 0x00040429, 0x00000000, 0xb2609506, 0x0010042d, 0x00000000, + 0x9e5cfb8a, 0x00040c31, 0x00000000, 0xc918df3e, 0x00040835, 0x00000000, + 0xc9751c9c, 0x00040839, 0x00000000, 0xc97ef77a, 0x0004083d, 0x00000000, + 0xc927c6b9, 0x00040841, 0x00000000, 0xc92a1ec5, 0x00040845, 0x00000000, + 0xc9e01b31, 0x00040849, 0x00000000, 0x50685f29, 0x0004044d, 0x00000000, + 0xb26219ca, 0x00100451, 0x00000000, 0xb25b6cf9, 0x00100455, 0x00000000, + 0xb2f86101, 0x00100459, 0x00000000, 0x07e3b155, 0x0004085d, 0x00000000, + 0x07e383c6, 0x00040861, 0x00000000, 0xbda98653, 0x00400465, 0x00000000, + 0xbdbd1d0f, 0x00800469, 0x00000000, 0xbd946fd4, 0x0080046d, 0x00000000, + 0xbdf02a4c, 0x00800471, 0x00000000, 0x19e93e85, 0x00040402, 0x00000000, + 0x3b5c2af3, 0x00040406, 0x00000000, 0x47974a35, 0x0004040a, 0x00000000, + 0x105971c2, 0x0004040e, 0x00000000, 0x7d137765, 0x00040412, 0x00000000, + 0xd187febc, 0x00040416, 0x00000000, 0x12f854ac, 0x0004041a, 0x00000000, + 0x40d49ad1, 0x0004041e, 0x00000000, 0x8c0923da, 0x00040422, 0x00000000, + 0xbb8df494, 0x00040426, 0x00000000, 0xf6d8a76e, 0x0004042a, 0x00000000, + 0x7620f1e9, 0x0004042e, 0x00000000, 0x9ebfab10, 0x00040432, 0x00000000, + 0x98a09d0e, 0x00040836, 0x00000000, 0x195d37d2, 0x0010083a, 0x00000000, + 0xf9ec4f85, 0x0010103e, 0x00000000, 0x4626b750, 0x00080842, 0x00000000, + 0x4cc673a0, 0x00200446, 0x00000000, 0xde5b3431, 0x0004044a, 0x00000000, + 0x036ac8a6, 0x000c044e, 0x00000000, +}; + +static CompactRegisterDefaults g_agc_public_reg_defaults_v7 = { + g_agc_public_reg_defaults_v7_tbl0_regs, 489, g_agc_public_reg_defaults_v7_tbl0_ptrs, 78, + g_agc_public_reg_defaults_v7_tbl1_regs, 159, g_agc_public_reg_defaults_v7_tbl1_ptrs, 29, + g_agc_public_reg_defaults_v7_tbl2_regs, 55, g_agc_public_reg_defaults_v7_tbl2_ptrs, 20, + nullptr, 0, nullptr, 0, + g_agc_public_reg_defaults_v7_types, 127, +}; + +// g_agc_public_reg_defaults_v8: regs=(489, 159, 55, 0), ptrs=(78, 29, 20, 0), types=127. +static ShaderRegister g_agc_public_reg_defaults_v8_tbl0_regs[] = { + {0x0202, 0x00cc0010}, {0x0109, 0x00000008}, {0x0104, 0x00000000}, {0x008f, 0x00000000}, + {0x008e, 0x00000000}, {0x02dc, 0x0000aa00}, {0x0001, 0x11000100}, {0x0200, 0x00000000}, + {0x0201, 0x00000000}, {0x0000, 0x00000000}, {0x0006, 0x00000000}, {0x001f, 0x00000000}, + {0x0203, 0x00000000}, {0x02b0, 0x00000000}, {0x02b1, 0x00000000}, {0x010c, 0x00000000}, + {0x010d, 0x00000000}, {0x010b, 0x00000000}, {0x01ff, 0x00000000}, {0x0204, 0x00000000}, + {0x020d, 0x00000000}, {0x0206, 0x0000043f}, {0x02f8, 0x00000000}, {0x0083, 0x0000ffff}, + {0x0313, 0x00006000}, {0x00f0, 0x00000000}, {0x00ea, 0x00000000}, {0x00e9, 0x00000000}, + {0x0292, 0x00000002}, {0x0293, 0x06020000}, {0x00e8, 0x00000000}, {0x0080, 0x00000000}, + {0x0211, 0x00000000}, {0x0210, 0x04000000}, {0x008d, 0x00000000}, {0x0282, 0x00000008}, + {0x0281, 0xffff0000}, {0x0280, 0x00080008}, {0x02df, 0x00000000}, {0x02de, 0x000001e9}, + {0x0205, 0x00000240}, {0x020c, 0x00000021}, {0x02f9, 0x0000002d}, {0x01ba, 0x00000000}, + {0x02a6, 0x00000000}, {0x02ce, 0x00000400}, {0x029b, 0x00000002}, {0x02d6, 0x00000000}, + {0x0103, 0xffffffff}, {0x02a1, 0x00000000}, {0x02ad, 0x00000000}, {0x02d5, 0x00002000}, + {0x02d4, 0x88101000}, {0x02db, 0x00000000}, {0x02f5, 0x00000000}, {0x02f6, 0x00000000}, + {0x02fe, 0x00000000}, {0x02ff, 0x00000000}, {0x0300, 0x00000000}, {0x0301, 0x00000000}, + {0x0302, 0x00000000}, {0x0303, 0x00000000}, {0x0304, 0x00000000}, {0x0305, 0x00000000}, + {0x0306, 0x00000000}, {0x0307, 0x00000000}, {0x0308, 0x00000000}, {0x0309, 0x00000000}, + {0x030a, 0x00000000}, {0x030b, 0x00000000}, {0x030c, 0x00000000}, {0x030d, 0x00000000}, + {0x030e, 0xffffffff}, {0x030f, 0xffffffff}, {0x0311, 0x01fd2002}, {0x0312, 0x03ff0080}, + {0x0105, 0x00000000}, {0x0106, 0x00000000}, {0x0107, 0x00000000}, {0x0108, 0x00000000}, + {0x01e0, 0x20010001}, {0x01e1, 0x20010001}, {0x01e2, 0x20010001}, {0x01e3, 0x20010001}, + {0x01e4, 0x20010001}, {0x01e5, 0x20010001}, {0x01e6, 0x20010001}, {0x01e7, 0x20010001}, + {0x0020, 0x00000000}, {0x0021, 0x00000000}, {0x0084, 0x00000000}, {0x0085, 0x20002000}, + {0x0086, 0x00000000}, {0x0087, 0x20002000}, {0x0088, 0x00000000}, {0x0089, 0x20002000}, + {0x008a, 0x00000000}, {0x008b, 0x20002000}, {0x00db, 0x00000000}, {0x0008, 0x00000000}, + {0x0009, 0x00000000}, {0x0010, 0x80000180}, {0x0011, 0x20000180}, {0x0012, 0x00000000}, + {0x0013, 0x00000000}, {0x0014, 0x00000000}, {0x0015, 0x00000000}, {0x001a, 0x00000000}, + {0x001b, 0x00000000}, {0x001c, 0x00000000}, {0x001d, 0x00000000}, {0x001e, 0x00000000}, + {0x0002, 0x00000000}, {0x0005, 0x00000000}, {0x0007, 0x00000000}, {0x000b, 0x00000000}, + {0x000a, 0x00000000}, {0x00eb, 0xff00ff00}, {0x00ec, 0x00010000}, {0x00f1, 0x00000000}, + {0x00f2, 0x00000000}, {0x0090, 0x80000000}, {0x0091, 0x40004000}, {0x02fa, 0x3f800000}, + {0x02fb, 0x3f800000}, {0x02fc, 0x3f800000}, {0x02fd, 0x3f800000}, {0x02e2, 0x00000000}, + {0x02e3, 0x00000000}, {0x02e0, 0x00000000}, {0x02e1, 0x00000000}, {0x0003, 0x00000000}, + {0x0004, 0x00000000}, {0x0318, 0x00000000}, {0x031b, 0x00000000}, {0x031c, 0x00000000}, + {0x031d, 0x00000000}, {0x031e, 0x00000048}, {0x031f, 0x00000000}, {0x0321, 0x00000000}, + {0x0323, 0x00000000}, {0x0324, 0x00000000}, {0x0325, 0x00000000}, {0x0390, 0x00000000}, + {0x0398, 0x00000000}, {0x03a0, 0x00000000}, {0x03a8, 0x00000000}, {0x03b0, 0x00000000}, + {0x03b8, 0x08c6c000}, {0x0327, 0x00000000}, {0x032a, 0x00000000}, {0x032b, 0x00000000}, + {0x032c, 0x00000000}, {0x032d, 0x00000048}, {0x032e, 0x00000000}, {0x0330, 0x00000000}, + {0x0332, 0x00000000}, {0x0333, 0x00000000}, {0x0334, 0x00000000}, {0x0391, 0x00000000}, + {0x0399, 0x00000000}, {0x03a1, 0x00000000}, {0x03a9, 0x00000000}, {0x03b1, 0x00000000}, + {0x03b9, 0x08c6c000}, {0x0336, 0x00000000}, {0x0339, 0x00000000}, {0x033a, 0x00000000}, + {0x033b, 0x00000000}, {0x033c, 0x00000048}, {0x033d, 0x00000000}, {0x033f, 0x00000000}, + {0x0341, 0x00000000}, {0x0342, 0x00000000}, {0x0343, 0x00000000}, {0x0392, 0x00000000}, + {0x039a, 0x00000000}, {0x03a2, 0x00000000}, {0x03aa, 0x00000000}, {0x03b2, 0x00000000}, + {0x03ba, 0x08c6c000}, {0x0345, 0x00000000}, {0x0348, 0x00000000}, {0x0349, 0x00000000}, + {0x034a, 0x00000000}, {0x034b, 0x00000048}, {0x034c, 0x00000000}, {0x034e, 0x00000000}, + {0x0350, 0x00000000}, {0x0351, 0x00000000}, {0x0352, 0x00000000}, {0x0393, 0x00000000}, + {0x039b, 0x00000000}, {0x03a3, 0x00000000}, {0x03ab, 0x00000000}, {0x03b3, 0x00000000}, + {0x03bb, 0x08c6c000}, {0x0354, 0x00000000}, {0x0357, 0x00000000}, {0x0358, 0x00000000}, + {0x0359, 0x00000000}, {0x035a, 0x00000048}, {0x035b, 0x00000000}, {0x035d, 0x00000000}, + {0x035f, 0x00000000}, {0x0360, 0x00000000}, {0x0361, 0x00000000}, {0x0394, 0x00000000}, + {0x039c, 0x00000000}, {0x03a4, 0x00000000}, {0x03ac, 0x00000000}, {0x03b4, 0x00000000}, + {0x03bc, 0x08c6c000}, {0x0363, 0x00000000}, {0x0366, 0x00000000}, {0x0367, 0x00000000}, + {0x0368, 0x00000000}, {0x0369, 0x00000048}, {0x036a, 0x00000000}, {0x036c, 0x00000000}, + {0x036e, 0x00000000}, {0x036f, 0x00000000}, {0x0370, 0x00000000}, {0x0395, 0x00000000}, + {0x039d, 0x00000000}, {0x03a5, 0x00000000}, {0x03ad, 0x00000000}, {0x03b5, 0x00000000}, + {0x03bd, 0x08c6c000}, {0x0372, 0x00000000}, {0x0375, 0x00000000}, {0x0376, 0x00000000}, + {0x0377, 0x00000000}, {0x0378, 0x00000048}, {0x0379, 0x00000000}, {0x037b, 0x00000000}, + {0x037d, 0x00000000}, {0x037e, 0x00000000}, {0x037f, 0x00000000}, {0x0396, 0x00000000}, + {0x039e, 0x00000000}, {0x03a6, 0x00000000}, {0x03ae, 0x00000000}, {0x03b6, 0x00000000}, + {0x03be, 0x08c6c000}, {0x0381, 0x00000000}, {0x0384, 0x00000000}, {0x0385, 0x00000000}, + {0x0386, 0x00000000}, {0x0387, 0x00000048}, {0x0388, 0x00000000}, {0x038a, 0x00000000}, + {0x038c, 0x00000000}, {0x038d, 0x00000000}, {0x038e, 0x00000000}, {0x0397, 0x00000000}, + {0x039f, 0x00000000}, {0x03a7, 0x00000000}, {0x03af, 0x00000000}, {0x03b7, 0x00000000}, + {0x03bf, 0x08c6c000}, {0x000c, 0x00000000}, {0x000d, 0x40004000}, {0x0191, 0x00000000}, + {0x0192, 0x00000000}, {0x0193, 0x00000000}, {0x0194, 0x00000000}, {0x0195, 0x00000000}, + {0x0196, 0x00000000}, {0x0197, 0x00000000}, {0x0198, 0x00000000}, {0x0199, 0x00000000}, + {0x019a, 0x00000000}, {0x019b, 0x00000000}, {0x019c, 0x00000000}, {0x019d, 0x00000000}, + {0x019e, 0x00000000}, {0x019f, 0x00000000}, {0x01a0, 0x00000000}, {0x01a1, 0x00000000}, + {0x01a2, 0x00000000}, {0x01a3, 0x00000000}, {0x01a4, 0x00000000}, {0x01a5, 0x00000000}, + {0x01a6, 0x00000000}, {0x01a7, 0x00000000}, {0x01a8, 0x00000000}, {0x01a9, 0x00000000}, + {0x01aa, 0x00000000}, {0x01ab, 0x00000000}, {0x01ac, 0x00000000}, {0x01ad, 0x00000000}, + {0x01ae, 0x00000000}, {0x01af, 0x00000000}, {0x01b0, 0x00000000}, {0x016f, 0x00000000}, + {0x0170, 0x00000000}, {0x0171, 0x00000000}, {0x0172, 0x00000000}, {0x0173, 0x00000000}, + {0x0174, 0x00000000}, {0x0175, 0x00000000}, {0x0176, 0x00000000}, {0x0177, 0x00000000}, + {0x0178, 0x00000000}, {0x0179, 0x00000000}, {0x017a, 0x00000000}, {0x017b, 0x00000000}, + {0x017c, 0x00000000}, {0x017d, 0x00000000}, {0x017e, 0x00000000}, {0x017f, 0x00000000}, + {0x0180, 0x00000000}, {0x0181, 0x00000000}, {0x0182, 0x00000000}, {0x0183, 0x00000000}, + {0x0184, 0x00000000}, {0x0185, 0x00000000}, {0x0186, 0x00000000}, {0x0187, 0x00000000}, + {0x0188, 0x00000000}, {0x0189, 0x00000000}, {0x018a, 0x00000000}, {0x018b, 0x00000000}, + {0x018c, 0x00000000}, {0x018d, 0x00000000}, {0x018e, 0x00000000}, {0x010f, 0x3f800000}, + {0x0111, 0x3f800000}, {0x0113, 0x3f800000}, {0x0110, 0x00000000}, {0x0112, 0x00000000}, + {0x0114, 0x00000000}, {0x0094, 0x80000000}, {0x0095, 0x40004000}, {0x00b4, 0x00000000}, + {0x00b5, 0x00000000}, {0x0115, 0x3f800000}, {0x0117, 0x3f800000}, {0x0119, 0x3f800000}, + {0x0116, 0x00000000}, {0x0118, 0x00000000}, {0x011a, 0x00000000}, {0x0096, 0x80000000}, + {0x0097, 0x40004000}, {0x00b6, 0x00000000}, {0x00b7, 0x00000000}, {0x011b, 0x3f800000}, + {0x011d, 0x3f800000}, {0x011f, 0x3f800000}, {0x011c, 0x00000000}, {0x011e, 0x00000000}, + {0x0120, 0x00000000}, {0x0098, 0x80000000}, {0x0099, 0x40004000}, {0x00b8, 0x00000000}, + {0x00b9, 0x00000000}, {0x0121, 0x3f800000}, {0x0123, 0x3f800000}, {0x0125, 0x3f800000}, + {0x0122, 0x00000000}, {0x0124, 0x00000000}, {0x0126, 0x00000000}, {0x009a, 0x80000000}, + {0x009b, 0x40004000}, {0x00ba, 0x00000000}, {0x00bb, 0x00000000}, {0x0127, 0x3f800000}, + {0x0129, 0x3f800000}, {0x012b, 0x3f800000}, {0x0128, 0x00000000}, {0x012a, 0x00000000}, + {0x012c, 0x00000000}, {0x009c, 0x80000000}, {0x009d, 0x40004000}, {0x00bc, 0x00000000}, + {0x00bd, 0x00000000}, {0x012d, 0x3f800000}, {0x012f, 0x3f800000}, {0x0131, 0x3f800000}, + {0x012e, 0x00000000}, {0x0130, 0x00000000}, {0x0132, 0x00000000}, {0x009e, 0x80000000}, + {0x009f, 0x40004000}, {0x00be, 0x00000000}, {0x00bf, 0x00000000}, {0x0133, 0x3f800000}, + {0x0135, 0x3f800000}, {0x0137, 0x3f800000}, {0x0134, 0x00000000}, {0x0136, 0x00000000}, + {0x0138, 0x00000000}, {0x00a0, 0x80000000}, {0x00a1, 0x40004000}, {0x00c0, 0x00000000}, + {0x00c1, 0x00000000}, {0x0139, 0x3f800000}, {0x013b, 0x3f800000}, {0x013d, 0x3f800000}, + {0x013a, 0x00000000}, {0x013c, 0x00000000}, {0x013e, 0x00000000}, {0x00a2, 0x80000000}, + {0x00a3, 0x40004000}, {0x00c2, 0x00000000}, {0x00c3, 0x00000000}, {0x013f, 0x3f800000}, + {0x0141, 0x3f800000}, {0x0143, 0x3f800000}, {0x0140, 0x00000000}, {0x0142, 0x00000000}, + {0x0144, 0x00000000}, {0x00a4, 0x80000000}, {0x00a5, 0x40004000}, {0x00c4, 0x00000000}, + {0x00c5, 0x00000000}, {0x0145, 0x3f800000}, {0x0147, 0x3f800000}, {0x0149, 0x3f800000}, + {0x0146, 0x00000000}, {0x0148, 0x00000000}, {0x014a, 0x00000000}, {0x00a6, 0x80000000}, + {0x00a7, 0x40004000}, {0x00c6, 0x00000000}, {0x00c7, 0x00000000}, {0x014b, 0x3f800000}, + {0x014d, 0x3f800000}, {0x014f, 0x3f800000}, {0x014c, 0x00000000}, {0x014e, 0x00000000}, + {0x0150, 0x00000000}, {0x00a8, 0x80000000}, {0x00a9, 0x40004000}, {0x00c8, 0x00000000}, + {0x00c9, 0x00000000}, {0x0151, 0x3f800000}, {0x0153, 0x3f800000}, {0x0155, 0x3f800000}, + {0x0152, 0x00000000}, {0x0154, 0x00000000}, {0x0156, 0x00000000}, {0x00aa, 0x80000000}, + {0x00ab, 0x40004000}, {0x00ca, 0x00000000}, {0x00cb, 0x00000000}, {0x0157, 0x3f800000}, + {0x0159, 0x3f800000}, {0x015b, 0x3f800000}, {0x0158, 0x00000000}, {0x015a, 0x00000000}, + {0x015c, 0x00000000}, {0x00ac, 0x80000000}, {0x00ad, 0x40004000}, {0x00cc, 0x00000000}, + {0x00cd, 0x00000000}, {0x015d, 0x3f800000}, {0x015f, 0x3f800000}, {0x0161, 0x3f800000}, + {0x015e, 0x00000000}, {0x0160, 0x00000000}, {0x0162, 0x00000000}, {0x00ae, 0x80000000}, + {0x00af, 0x40004000}, {0x00ce, 0x00000000}, {0x00cf, 0x00000000}, {0x0163, 0x3f800000}, + {0x0165, 0x3f800000}, {0x0167, 0x3f800000}, {0x0164, 0x00000000}, {0x0166, 0x00000000}, + {0x0168, 0x00000000}, {0x00b0, 0x80000000}, {0x00b1, 0x40004000}, {0x00d0, 0x00000000}, + {0x00d1, 0x00000000}, {0x0169, 0x3f800000}, {0x016b, 0x3f800000}, {0x016d, 0x3f800000}, + {0x016a, 0x00000000}, {0x016c, 0x00000000}, {0x016e, 0x00000000}, {0x00b2, 0x80000000}, + {0x00b3, 0x40004000}, {0x00d2, 0x00000000}, {0x00d3, 0x00000000}, {0x0081, 0x80000000}, + {0x0082, 0x40004000}, +}; + +static const uint16_t g_agc_public_reg_defaults_v8_tbl0_ptrs[] = { + 0x0000, 0x0001, 0x0002, 0x0003, 0x0004, 0x0005, 0x0006, 0x0007, 0x0008, 0x0009, 0x000a, 0x000b, + 0x000c, 0x000d, 0x000e, 0x000f, 0x0010, 0x0011, 0x0012, 0x0013, 0x0014, 0x0015, 0x0016, 0x0017, + 0x0018, 0x0019, 0x001a, 0x001b, 0x001c, 0x001d, 0x001e, 0x001f, 0x0020, 0x0021, 0x0022, 0x0023, + 0x0024, 0x0025, 0x0026, 0x0027, 0x0028, 0x0029, 0x002a, 0x002b, 0x002c, 0x002d, 0x002e, 0x002f, + 0x0030, 0x0031, 0x0032, 0x0033, 0x0034, 0x0035, 0x0036, 0x0038, 0x0048, 0x004a, 0x004c, 0x0050, + 0x0058, 0x005a, 0x0062, 0x0063, 0x0065, 0x0075, 0x0077, 0x0079, 0x007b, 0x007f, 0x0081, 0x0083, + 0x0085, 0x0105, 0x0107, 0x0127, 0x0147, 0x01e7, +}; + +static ShaderRegister g_agc_public_reg_defaults_v8_tbl1_regs[] = { + {0x0212, 0x00000000}, {0x0213, 0x00000000}, {0x0228, 0x00000000}, {0x0215, 0x00000000}, + {0x0218, 0x00000000}, {0x008a, 0x00000000}, {0x010a, 0x00000000}, {0x000a, 0x00000000}, + {0x008b, 0x00000000}, {0x010b, 0x00000000}, {0x000b, 0x00000000}, {0x0224, 0x00000000}, + {0x0225, 0x00000000}, {0x0226, 0x00000000}, {0x0227, 0x00000000}, {0x0107, 0xffff0000}, + {0x0087, 0x0000ffff}, {0x0007, 0x0000ffff}, {0x020c, 0x00000000}, {0x020d, 0x00000000}, + {0x00c8, 0x00000000}, {0x00c9, 0x00000000}, {0x0088, 0x00000000}, {0x0089, 0x00000000}, + {0x0108, 0x00000000}, {0x0109, 0x00000000}, {0x0148, 0x00000000}, {0x0149, 0x00000000}, + {0x0008, 0x00000000}, {0x0009, 0x00000000}, {0x0280, 0x00000000}, {0x00b2, 0x00000000}, + {0x00b3, 0x00000000}, {0x00b4, 0x00000000}, {0x00b5, 0x00000000}, {0x0132, 0x00000000}, + {0x0133, 0x00000000}, {0x0134, 0x00000000}, {0x0135, 0x00000000}, {0x0032, 0x00000000}, + {0x0033, 0x00000000}, {0x0034, 0x00000000}, {0x0035, 0x00000000}, {0x0082, 0x00000000}, + {0x0083, 0x00000000}, {0x0102, 0x00000000}, {0x0103, 0x00000000}, {0x0240, 0x00000000}, + {0x0241, 0x00000000}, {0x0242, 0x00000000}, {0x0243, 0x00000000}, {0x0244, 0x00000000}, + {0x0245, 0x00000000}, {0x0246, 0x00000000}, {0x0247, 0x00000000}, {0x0248, 0x00000000}, + {0x0249, 0x00000000}, {0x024a, 0x00000000}, {0x024b, 0x00000000}, {0x024c, 0x00000000}, + {0x024d, 0x00000000}, {0x024e, 0x00000000}, {0x024f, 0x00000000}, {0x008c, 0x00000000}, + {0x008d, 0x00000000}, {0x008e, 0x00000000}, {0x008f, 0x00000000}, {0x0090, 0x00000000}, + {0x0091, 0x00000000}, {0x0092, 0x00000000}, {0x0093, 0x00000000}, {0x0094, 0x00000000}, + {0x0095, 0x00000000}, {0x0096, 0x00000000}, {0x0097, 0x00000000}, {0x0098, 0x00000000}, + {0x0099, 0x00000000}, {0x009a, 0x00000000}, {0x009b, 0x00000000}, {0x009c, 0x00000000}, + {0x009d, 0x00000000}, {0x009e, 0x00000000}, {0x009f, 0x00000000}, {0x00a0, 0x00000000}, + {0x00a1, 0x00000000}, {0x00a2, 0x00000000}, {0x00a3, 0x00000000}, {0x00a4, 0x00000000}, + {0x00a5, 0x00000000}, {0x00a6, 0x00000000}, {0x00a7, 0x00000000}, {0x00a8, 0x00000000}, + {0x00a9, 0x00000000}, {0x00aa, 0x00000000}, {0x00ab, 0x00000000}, {0x010c, 0x00000000}, + {0x010d, 0x00000000}, {0x010e, 0x00000000}, {0x010f, 0x00000000}, {0x0110, 0x00000000}, + {0x0111, 0x00000000}, {0x0112, 0x00000000}, {0x0113, 0x00000000}, {0x0114, 0x00000000}, + {0x0115, 0x00000000}, {0x0116, 0x00000000}, {0x0117, 0x00000000}, {0x0118, 0x00000000}, + {0x0119, 0x00000000}, {0x011a, 0x00000000}, {0x011b, 0x00000000}, {0x011c, 0x00000000}, + {0x011d, 0x00000000}, {0x011e, 0x00000000}, {0x011f, 0x00000000}, {0x0120, 0x00000000}, + {0x0121, 0x00000000}, {0x0122, 0x00000000}, {0x0123, 0x00000000}, {0x0124, 0x00000000}, + {0x0125, 0x00000000}, {0x0126, 0x00000000}, {0x0127, 0x00000000}, {0x0128, 0x00000000}, + {0x0129, 0x00000000}, {0x012a, 0x00000000}, {0x012b, 0x00000000}, {0x000c, 0x00000000}, + {0x000d, 0x00000000}, {0x000e, 0x00000000}, {0x000f, 0x00000000}, {0x0010, 0x00000000}, + {0x0011, 0x00000000}, {0x0012, 0x00000000}, {0x0013, 0x00000000}, {0x0014, 0x00000000}, + {0x0015, 0x00000000}, {0x0016, 0x00000000}, {0x0017, 0x00000000}, {0x0018, 0x00000000}, + {0x0019, 0x00000000}, {0x001a, 0x00000000}, {0x001b, 0x00000000}, {0x001c, 0x00000000}, + {0x001d, 0x00000000}, {0x001e, 0x00000000}, {0x001f, 0x00000000}, {0x0020, 0x00000000}, + {0x0021, 0x00000000}, {0x0022, 0x00000000}, {0x0023, 0x00000000}, {0x0024, 0x00000000}, + {0x0025, 0x00000000}, {0x0026, 0x00000000}, {0x0027, 0x00000000}, {0x0028, 0x00000000}, + {0x0029, 0x00000000}, {0x002a, 0x00000000}, {0x002b, 0x00000000}, +}; + +static const uint16_t g_agc_public_reg_defaults_v8_tbl1_ptrs[] = { + 0x0000, 0x0001, 0x0002, 0x0003, 0x0004, 0x0005, 0x0006, 0x0007, 0x0008, 0x0009, 0x000a, 0x000b, + 0x000f, 0x0012, 0x0014, 0x0016, 0x0018, 0x001a, 0x001c, 0x001e, 0x001f, 0x0023, 0x0027, 0x002b, + 0x002d, 0x002f, 0x003f, 0x005f, 0x007f, +}; + +static ShaderRegister g_agc_public_reg_defaults_v8_tbl2_regs[] = { + {0x041f, 0x00000000}, {0x041d, 0x00000000}, {0x041e, 0x00000000}, {0x025b, 0x00000000}, + {0x024a, 0x00000000}, {0x024b, 0x00000000}, {0x025f, 0x00000000}, {0x0262, 0x00000000}, + {0x02bb, 0x00000000}, {0x0384, 0x00000000}, {0x0383, 0x40000040}, {0x0248, 0x00000000}, + {0x0242, 0x00000000}, {0x0380, 0x00000000}, {0x0381, 0x00000000}, {0x02e4, 0x00000000}, + {0x02e5, 0x00000000}, {0x02e6, 0x00000000}, {0x02e7, 0x00000000}, {0x02e8, 0x00000000}, + {0x02e9, 0x00000000}, {0x02ea, 0x00000000}, {0x02eb, 0x00000000}, {0x02c4, 0x00000000}, + {0x02c5, 0x00000000}, {0x02c6, 0x00000000}, {0x02c7, 0x00000000}, {0x02c8, 0x00000000}, + {0x02c9, 0x00000000}, {0x02ca, 0x00000000}, {0x02cb, 0x00000000}, {0x02cc, 0x00000000}, + {0x02cd, 0x00000000}, {0x02ce, 0x00000000}, {0x02cf, 0x00000000}, {0x02d0, 0x00000000}, + {0x02d1, 0x00000000}, {0x02d2, 0x00000000}, {0x02d3, 0x00000000}, {0x02bc, 0x00000000}, + {0x02bd, 0x00000000}, {0x02be, 0x00000000}, {0x02bf, 0x00000000}, {0x0035, 0x00000000}, + {0x0036, 0x00000000}, {0x0037, 0x00000000}, {0x0038, 0x00000000}, {0x0039, 0x00000000}, + {0x003a, 0x00000000}, {0x003b, 0x00000000}, {0x003c, 0x00000000}, {0x00a2, 0x00000000}, + {0x025c, 0x00000000}, {0x025d, 0x00000000}, {0x025e, 0x00000000}, +}; + +static const uint16_t g_agc_public_reg_defaults_v8_tbl2_ptrs[] = { + 0x0000, 0x0001, 0x0002, 0x0003, 0x0004, 0x0005, 0x0006, 0x0007, 0x0008, 0x0009, 0x000a, 0x000b, + 0x000c, 0x000d, 0x000f, 0x0017, 0x0027, 0x002b, 0x0033, 0x0034, +}; + +static uint32_t g_agc_public_reg_defaults_v8_types[] = { + 0xe24f806d, 0x00040400, 0x00000000, 0xf6c28182, 0x00040404, 0x00000000, + 0x6f6e55a5, 0x00040408, 0x00000000, 0x0bc65da4, 0x0004040c, 0x00000000, + 0x9e5ad592, 0x00040410, 0x00000000, 0xbb513b98, 0x00040414, 0x00000000, + 0xab64b23b, 0x00040418, 0x00000000, 0x53c39964, 0x0004041c, 0x00000000, + 0x01396b11, 0x00040420, 0x00000000, 0x7d42019a, 0x00040424, 0x00000000, + 0x3548f523, 0x00040428, 0x00000000, 0xf43ad28a, 0x0004042c, 0x00000000, + 0x6de4c312, 0x00040430, 0x00000000, 0x00a77ae0, 0x00040434, 0x00000000, + 0x00a779b7, 0x00040438, 0x00000000, 0x5100100c, 0x0004043c, 0x00000000, + 0x59958bba, 0x00040440, 0x00000000, 0x0c06f17c, 0x00040444, 0x00000000, + 0x6f104b72, 0x00040448, 0x00000000, 0x25c70d9c, 0x0004044c, 0x00000000, + 0x3881201e, 0x00040450, 0x00000000, 0x09afddaf, 0x00040454, 0x00000000, + 0x367d63cf, 0x00040458, 0x00000000, 0x43707db8, 0x0004045c, 0x00000000, + 0xf6ae26ba, 0x00040460, 0x00000000, 0x1b917652, 0x00040464, 0x00000000, + 0x94b1e4f7, 0x00040468, 0x00000000, 0xe3661b6c, 0x0004046c, 0x00000000, + 0x1eb8d73a, 0x00040470, 0x00000000, 0x15051fa3, 0x00040474, 0x00000000, + 0x9c51a7f1, 0x00040478, 0x00000000, 0xa20efc70, 0x0004047c, 0x00000000, + 0x0ec09f6e, 0x00040480, 0x00000000, 0x34a7d6d3, 0x00040484, 0x00000000, + 0xce831b94, 0x00040488, 0x00000000, 0x5cc72a74, 0x0004048c, 0x00000000, + 0x3b77713c, 0x00040490, 0x00000000, 0x40f64410, 0x00040494, 0x00000000, + 0x69441268, 0x00040498, 0x00000000, 0x2e418b83, 0x0004049c, 0x00000000, + 0xa00d0c8d, 0x000404a0, 0x00000000, 0xb1289fb3, 0x000404a4, 0x00000000, + 0x144832fb, 0x000404a8, 0x00000000, 0x9890d9fa, 0x000404ac, 0x00000000, + 0x9016faf1, 0x000404b0, 0x00000000, 0x4b73ce27, 0x000404b4, 0x00000000, + 0x5f5a3e7b, 0x000404b8, 0x00000000, 0xd4af3a51, 0x000404bc, 0x00000000, + 0x6cf4f543, 0x000404c0, 0x00000000, 0x5fb86ccb, 0x000404c4, 0x00000000, + 0xedefa188, 0x000404c8, 0x00000000, 0xd0de9ee6, 0x000404cc, 0x00000000, + 0xc5831803, 0x000404d0, 0x00000000, 0x8e6de84b, 0x000404d4, 0x00000000, + 0xd0771662, 0x000408d8, 0x00000000, 0x569f7444, 0x004004dc, 0x00000000, + 0x5c6637cd, 0x000408e0, 0x00000000, 0xcae3e690, 0x000408e4, 0x00000000, + 0x43fbd769, 0x000410e8, 0x00000000, 0xef550356, 0x002004ec, 0x00000000, + 0x8f52e279, 0x000408f0, 0x00000000, 0x1f2d8149, 0x001008f4, 0x00000000, + 0x853d0614, 0x000404f8, 0x00000000, 0x4413c6f9, 0x000408fc, 0x00000000, + 0x67096014, 0x00044100, 0x00000000, 0x88f5e915, 0x00040904, 0x00000000, + 0x033f1eff, 0x00040908, 0x00000000, 0x918106bb, 0x0004090c, 0x00000000, + 0x95f0e7ac, 0x00041110, 0x00000000, 0xb48cbab2, 0x00040914, 0x00000000, + 0x05bb3bc6, 0x00040918, 0x00000000, 0x94faba07, 0x0004091c, 0x00000000, + 0x38e92c91, 0x00204120, 0x00000000, 0x0b177b43, 0x00040924, 0x00000000, + 0x48531062, 0x00800528, 0x00000000, 0xaaa964b9, 0x0020112c, 0x00000000, + 0x7690af6f, 0x00402930, 0x00000000, 0x078d7060, 0x00040934, 0x00000000, + 0x5d6e3ec7, 0x00040401, 0x00000000, 0x57e7079a, 0x00040405, 0x00000000, + 0x7467fafd, 0x00040409, 0x00000000, 0x9e826b50, 0x0004040d, 0x00000000, + 0xdc484f18, 0x00040411, 0x00000000, 0x5da8bca3, 0x00040415, 0x00000000, + 0x5ca726d8, 0x00040419, 0x00000000, 0x5dd28360, 0x0004041d, 0x00000000, + 0x57efa0be, 0x00040421, 0x00000000, 0x502363d5, 0x00040425, 0x00000000, + 0x506d14bd, 0x00040429, 0x00000000, 0xb2609506, 0x0010042d, 0x00000000, + 0x9e5cfb8a, 0x00040c31, 0x00000000, 0xc918df3e, 0x00040835, 0x00000000, + 0xc9751c9c, 0x00040839, 0x00000000, 0xc97ef77a, 0x0004083d, 0x00000000, + 0xc927c6b9, 0x00040841, 0x00000000, 0xc92a1ec5, 0x00040845, 0x00000000, + 0xc9e01b31, 0x00040849, 0x00000000, 0x50685f29, 0x0004044d, 0x00000000, + 0xb26219ca, 0x00100451, 0x00000000, 0xb25b6cf9, 0x00100455, 0x00000000, + 0xb2f86101, 0x00100459, 0x00000000, 0x07e3b155, 0x0004085d, 0x00000000, + 0x07e383c6, 0x00040861, 0x00000000, 0xbda98653, 0x00400465, 0x00000000, + 0xbdbd1d0f, 0x00800469, 0x00000000, 0xbd946fd4, 0x0080046d, 0x00000000, + 0xbdf02a4c, 0x00800471, 0x00000000, 0x19e93e85, 0x00040402, 0x00000000, + 0x3b5c2af3, 0x00040406, 0x00000000, 0x47974a35, 0x0004040a, 0x00000000, + 0x105971c2, 0x0004040e, 0x00000000, 0x7d137765, 0x00040412, 0x00000000, + 0xd187febc, 0x00040416, 0x00000000, 0x12f854ac, 0x0004041a, 0x00000000, + 0x40d49ad1, 0x0004041e, 0x00000000, 0x8c0923da, 0x00040422, 0x00000000, + 0xbb8df494, 0x00040426, 0x00000000, 0xf6d8a76e, 0x0004042a, 0x00000000, + 0x7620f1e9, 0x0004042e, 0x00000000, 0x9ebfab10, 0x00040432, 0x00000000, + 0x98a09d0e, 0x00040836, 0x00000000, 0x195d37d2, 0x0010083a, 0x00000000, + 0xf9ec4f85, 0x0010103e, 0x00000000, 0x4626b750, 0x00080842, 0x00000000, + 0x4cc673a0, 0x00200446, 0x00000000, 0xde5b3431, 0x0004044a, 0x00000000, + 0x036ac8a6, 0x000c044e, 0x00000000, +}; + +static CompactRegisterDefaults g_agc_public_reg_defaults_v8 = { + g_agc_public_reg_defaults_v8_tbl0_regs, 489, g_agc_public_reg_defaults_v8_tbl0_ptrs, 78, + g_agc_public_reg_defaults_v8_tbl1_regs, 159, g_agc_public_reg_defaults_v8_tbl1_ptrs, 29, + g_agc_public_reg_defaults_v8_tbl2_regs, 55, g_agc_public_reg_defaults_v8_tbl2_ptrs, 20, + nullptr, 0, nullptr, 0, + g_agc_public_reg_defaults_v8_types, 127, +}; + +// g_agc_public_reg_defaults_v9: regs=(489, 160, 55, 0), ptrs=(78, 29, 20, 0), types=127. +static ShaderRegister g_agc_public_reg_defaults_v9_tbl0_regs[] = { + {0x0202, 0x00cc0010}, {0x0109, 0x00000010}, {0x0104, 0x00000000}, {0x008f, 0x00000000}, + {0x008e, 0x00000000}, {0x02dc, 0x0000aa00}, {0x0001, 0x11000100}, {0x0200, 0x00000000}, + {0x0201, 0x00000000}, {0x0000, 0x00000000}, {0x0006, 0x00000000}, {0x001f, 0x00000000}, + {0x0203, 0x00000000}, {0x02b0, 0x00000000}, {0x02b1, 0x00000000}, {0x010c, 0x00000000}, + {0x010d, 0x00000000}, {0x010b, 0x00000000}, {0x01ff, 0x00000000}, {0x0204, 0x00000000}, + {0x020d, 0x00000000}, {0x0206, 0x0000043f}, {0x02f8, 0x00000000}, {0x0083, 0x0000ffff}, + {0x0313, 0x00006000}, {0x00f0, 0x00000000}, {0x00ea, 0x00000000}, {0x00e9, 0x00000000}, + {0x0292, 0x00000002}, {0x0293, 0x06020000}, {0x00e8, 0x00000000}, {0x0080, 0x00000000}, + {0x0211, 0x00000000}, {0x0210, 0x04000000}, {0x008d, 0x00000000}, {0x0282, 0x00000008}, + {0x0281, 0xffff0000}, {0x0280, 0x00080008}, {0x02df, 0x00000000}, {0x02de, 0x000001e9}, + {0x0205, 0x00000240}, {0x020c, 0x00000021}, {0x02f9, 0x0000002d}, {0x01ba, 0x00000000}, + {0x02a6, 0x00000000}, {0x02ce, 0x00000400}, {0x029b, 0x00000002}, {0x02d6, 0x00000000}, + {0x0103, 0xffffffff}, {0x02a1, 0x00000000}, {0x02ad, 0x00000000}, {0x02d5, 0x00002000}, + {0x02d4, 0x88101000}, {0x02db, 0x00000000}, {0x02f5, 0x00000000}, {0x02f6, 0x00000000}, + {0x02fe, 0x00000000}, {0x02ff, 0x00000000}, {0x0300, 0x00000000}, {0x0301, 0x00000000}, + {0x0302, 0x00000000}, {0x0303, 0x00000000}, {0x0304, 0x00000000}, {0x0305, 0x00000000}, + {0x0306, 0x00000000}, {0x0307, 0x00000000}, {0x0308, 0x00000000}, {0x0309, 0x00000000}, + {0x030a, 0x00000000}, {0x030b, 0x00000000}, {0x030c, 0x00000000}, {0x030d, 0x00000000}, + {0x030e, 0xffffffff}, {0x030f, 0xffffffff}, {0x0311, 0x01fd2002}, {0x0312, 0x03ff0080}, + {0x0105, 0x00000000}, {0x0106, 0x00000000}, {0x0107, 0x00000000}, {0x0108, 0x00000000}, + {0x01e0, 0x20010001}, {0x01e1, 0x20010001}, {0x01e2, 0x20010001}, {0x01e3, 0x20010001}, + {0x01e4, 0x20010001}, {0x01e5, 0x20010001}, {0x01e6, 0x20010001}, {0x01e7, 0x20010001}, + {0x0020, 0x00000000}, {0x0021, 0x00000000}, {0x0084, 0x00000000}, {0x0085, 0x20002000}, + {0x0086, 0x00000000}, {0x0087, 0x20002000}, {0x0088, 0x00000000}, {0x0089, 0x20002000}, + {0x008a, 0x00000000}, {0x008b, 0x20002000}, {0x00db, 0x00000000}, {0x0008, 0x00000000}, + {0x0009, 0x00000000}, {0x0010, 0x80000180}, {0x0011, 0x20000180}, {0x0012, 0x00000000}, + {0x0013, 0x00000000}, {0x0014, 0x00000000}, {0x0015, 0x00000000}, {0x001a, 0x00000000}, + {0x001b, 0x00000000}, {0x001c, 0x00000000}, {0x001d, 0x00000000}, {0x001e, 0x00000000}, + {0x0002, 0x00000000}, {0x0005, 0x00000000}, {0x0007, 0x00000000}, {0x000b, 0x00000000}, + {0x000a, 0x00000000}, {0x00eb, 0xff00ff00}, {0x00ec, 0x00010000}, {0x00f1, 0x00000000}, + {0x00f2, 0x00000000}, {0x0090, 0x80000000}, {0x0091, 0x40004000}, {0x02fa, 0x3f800000}, + {0x02fb, 0x3f800000}, {0x02fc, 0x3f800000}, {0x02fd, 0x3f800000}, {0x02e2, 0x00000000}, + {0x02e3, 0x00000000}, {0x02e0, 0x00000000}, {0x02e1, 0x00000000}, {0x0003, 0x00000000}, + {0x0004, 0x00000000}, {0x0318, 0x00000000}, {0x031b, 0x00000000}, {0x031c, 0x00000000}, + {0x031d, 0x00000000}, {0x031e, 0x00000048}, {0x031f, 0x00000000}, {0x0321, 0x00000000}, + {0x0323, 0x00000000}, {0x0324, 0x00000000}, {0x0325, 0x00000000}, {0x0390, 0x00000000}, + {0x0398, 0x00000000}, {0x03a0, 0x00000000}, {0x03a8, 0x00000000}, {0x03b0, 0x00000000}, + {0x03b8, 0x08c6c000}, {0x0327, 0x00000000}, {0x032a, 0x00000000}, {0x032b, 0x00000000}, + {0x032c, 0x00000000}, {0x032d, 0x00000048}, {0x032e, 0x00000000}, {0x0330, 0x00000000}, + {0x0332, 0x00000000}, {0x0333, 0x00000000}, {0x0334, 0x00000000}, {0x0391, 0x00000000}, + {0x0399, 0x00000000}, {0x03a1, 0x00000000}, {0x03a9, 0x00000000}, {0x03b1, 0x00000000}, + {0x03b9, 0x08c6c000}, {0x0336, 0x00000000}, {0x0339, 0x00000000}, {0x033a, 0x00000000}, + {0x033b, 0x00000000}, {0x033c, 0x00000048}, {0x033d, 0x00000000}, {0x033f, 0x00000000}, + {0x0341, 0x00000000}, {0x0342, 0x00000000}, {0x0343, 0x00000000}, {0x0392, 0x00000000}, + {0x039a, 0x00000000}, {0x03a2, 0x00000000}, {0x03aa, 0x00000000}, {0x03b2, 0x00000000}, + {0x03ba, 0x08c6c000}, {0x0345, 0x00000000}, {0x0348, 0x00000000}, {0x0349, 0x00000000}, + {0x034a, 0x00000000}, {0x034b, 0x00000048}, {0x034c, 0x00000000}, {0x034e, 0x00000000}, + {0x0350, 0x00000000}, {0x0351, 0x00000000}, {0x0352, 0x00000000}, {0x0393, 0x00000000}, + {0x039b, 0x00000000}, {0x03a3, 0x00000000}, {0x03ab, 0x00000000}, {0x03b3, 0x00000000}, + {0x03bb, 0x08c6c000}, {0x0354, 0x00000000}, {0x0357, 0x00000000}, {0x0358, 0x00000000}, + {0x0359, 0x00000000}, {0x035a, 0x00000048}, {0x035b, 0x00000000}, {0x035d, 0x00000000}, + {0x035f, 0x00000000}, {0x0360, 0x00000000}, {0x0361, 0x00000000}, {0x0394, 0x00000000}, + {0x039c, 0x00000000}, {0x03a4, 0x00000000}, {0x03ac, 0x00000000}, {0x03b4, 0x00000000}, + {0x03bc, 0x08c6c000}, {0x0363, 0x00000000}, {0x0366, 0x00000000}, {0x0367, 0x00000000}, + {0x0368, 0x00000000}, {0x0369, 0x00000048}, {0x036a, 0x00000000}, {0x036c, 0x00000000}, + {0x036e, 0x00000000}, {0x036f, 0x00000000}, {0x0370, 0x00000000}, {0x0395, 0x00000000}, + {0x039d, 0x00000000}, {0x03a5, 0x00000000}, {0x03ad, 0x00000000}, {0x03b5, 0x00000000}, + {0x03bd, 0x08c6c000}, {0x0372, 0x00000000}, {0x0375, 0x00000000}, {0x0376, 0x00000000}, + {0x0377, 0x00000000}, {0x0378, 0x00000048}, {0x0379, 0x00000000}, {0x037b, 0x00000000}, + {0x037d, 0x00000000}, {0x037e, 0x00000000}, {0x037f, 0x00000000}, {0x0396, 0x00000000}, + {0x039e, 0x00000000}, {0x03a6, 0x00000000}, {0x03ae, 0x00000000}, {0x03b6, 0x00000000}, + {0x03be, 0x08c6c000}, {0x0381, 0x00000000}, {0x0384, 0x00000000}, {0x0385, 0x00000000}, + {0x0386, 0x00000000}, {0x0387, 0x00000048}, {0x0388, 0x00000000}, {0x038a, 0x00000000}, + {0x038c, 0x00000000}, {0x038d, 0x00000000}, {0x038e, 0x00000000}, {0x0397, 0x00000000}, + {0x039f, 0x00000000}, {0x03a7, 0x00000000}, {0x03af, 0x00000000}, {0x03b7, 0x00000000}, + {0x03bf, 0x08c6c000}, {0x000c, 0x00000000}, {0x000d, 0x40004000}, {0x0191, 0x00000000}, + {0x0192, 0x00000000}, {0x0193, 0x00000000}, {0x0194, 0x00000000}, {0x0195, 0x00000000}, + {0x0196, 0x00000000}, {0x0197, 0x00000000}, {0x0198, 0x00000000}, {0x0199, 0x00000000}, + {0x019a, 0x00000000}, {0x019b, 0x00000000}, {0x019c, 0x00000000}, {0x019d, 0x00000000}, + {0x019e, 0x00000000}, {0x019f, 0x00000000}, {0x01a0, 0x00000000}, {0x01a1, 0x00000000}, + {0x01a2, 0x00000000}, {0x01a3, 0x00000000}, {0x01a4, 0x00000000}, {0x01a5, 0x00000000}, + {0x01a6, 0x00000000}, {0x01a7, 0x00000000}, {0x01a8, 0x00000000}, {0x01a9, 0x00000000}, + {0x01aa, 0x00000000}, {0x01ab, 0x00000000}, {0x01ac, 0x00000000}, {0x01ad, 0x00000000}, + {0x01ae, 0x00000000}, {0x01af, 0x00000000}, {0x01b0, 0x00000000}, {0x016f, 0x00000000}, + {0x0170, 0x00000000}, {0x0171, 0x00000000}, {0x0172, 0x00000000}, {0x0173, 0x00000000}, + {0x0174, 0x00000000}, {0x0175, 0x00000000}, {0x0176, 0x00000000}, {0x0177, 0x00000000}, + {0x0178, 0x00000000}, {0x0179, 0x00000000}, {0x017a, 0x00000000}, {0x017b, 0x00000000}, + {0x017c, 0x00000000}, {0x017d, 0x00000000}, {0x017e, 0x00000000}, {0x017f, 0x00000000}, + {0x0180, 0x00000000}, {0x0181, 0x00000000}, {0x0182, 0x00000000}, {0x0183, 0x00000000}, + {0x0184, 0x00000000}, {0x0185, 0x00000000}, {0x0186, 0x00000000}, {0x0187, 0x00000000}, + {0x0188, 0x00000000}, {0x0189, 0x00000000}, {0x018a, 0x00000000}, {0x018b, 0x00000000}, + {0x018c, 0x00000000}, {0x018d, 0x00000000}, {0x018e, 0x00000000}, {0x010f, 0x3f800000}, + {0x0111, 0x3f800000}, {0x0113, 0x3f800000}, {0x0110, 0x00000000}, {0x0112, 0x00000000}, + {0x0114, 0x00000000}, {0x0094, 0x80000000}, {0x0095, 0x40004000}, {0x00b4, 0x00000000}, + {0x00b5, 0x00000000}, {0x0115, 0x3f800000}, {0x0117, 0x3f800000}, {0x0119, 0x3f800000}, + {0x0116, 0x00000000}, {0x0118, 0x00000000}, {0x011a, 0x00000000}, {0x0096, 0x80000000}, + {0x0097, 0x40004000}, {0x00b6, 0x00000000}, {0x00b7, 0x00000000}, {0x011b, 0x3f800000}, + {0x011d, 0x3f800000}, {0x011f, 0x3f800000}, {0x011c, 0x00000000}, {0x011e, 0x00000000}, + {0x0120, 0x00000000}, {0x0098, 0x80000000}, {0x0099, 0x40004000}, {0x00b8, 0x00000000}, + {0x00b9, 0x00000000}, {0x0121, 0x3f800000}, {0x0123, 0x3f800000}, {0x0125, 0x3f800000}, + {0x0122, 0x00000000}, {0x0124, 0x00000000}, {0x0126, 0x00000000}, {0x009a, 0x80000000}, + {0x009b, 0x40004000}, {0x00ba, 0x00000000}, {0x00bb, 0x00000000}, {0x0127, 0x3f800000}, + {0x0129, 0x3f800000}, {0x012b, 0x3f800000}, {0x0128, 0x00000000}, {0x012a, 0x00000000}, + {0x012c, 0x00000000}, {0x009c, 0x80000000}, {0x009d, 0x40004000}, {0x00bc, 0x00000000}, + {0x00bd, 0x00000000}, {0x012d, 0x3f800000}, {0x012f, 0x3f800000}, {0x0131, 0x3f800000}, + {0x012e, 0x00000000}, {0x0130, 0x00000000}, {0x0132, 0x00000000}, {0x009e, 0x80000000}, + {0x009f, 0x40004000}, {0x00be, 0x00000000}, {0x00bf, 0x00000000}, {0x0133, 0x3f800000}, + {0x0135, 0x3f800000}, {0x0137, 0x3f800000}, {0x0134, 0x00000000}, {0x0136, 0x00000000}, + {0x0138, 0x00000000}, {0x00a0, 0x80000000}, {0x00a1, 0x40004000}, {0x00c0, 0x00000000}, + {0x00c1, 0x00000000}, {0x0139, 0x3f800000}, {0x013b, 0x3f800000}, {0x013d, 0x3f800000}, + {0x013a, 0x00000000}, {0x013c, 0x00000000}, {0x013e, 0x00000000}, {0x00a2, 0x80000000}, + {0x00a3, 0x40004000}, {0x00c2, 0x00000000}, {0x00c3, 0x00000000}, {0x013f, 0x3f800000}, + {0x0141, 0x3f800000}, {0x0143, 0x3f800000}, {0x0140, 0x00000000}, {0x0142, 0x00000000}, + {0x0144, 0x00000000}, {0x00a4, 0x80000000}, {0x00a5, 0x40004000}, {0x00c4, 0x00000000}, + {0x00c5, 0x00000000}, {0x0145, 0x3f800000}, {0x0147, 0x3f800000}, {0x0149, 0x3f800000}, + {0x0146, 0x00000000}, {0x0148, 0x00000000}, {0x014a, 0x00000000}, {0x00a6, 0x80000000}, + {0x00a7, 0x40004000}, {0x00c6, 0x00000000}, {0x00c7, 0x00000000}, {0x014b, 0x3f800000}, + {0x014d, 0x3f800000}, {0x014f, 0x3f800000}, {0x014c, 0x00000000}, {0x014e, 0x00000000}, + {0x0150, 0x00000000}, {0x00a8, 0x80000000}, {0x00a9, 0x40004000}, {0x00c8, 0x00000000}, + {0x00c9, 0x00000000}, {0x0151, 0x3f800000}, {0x0153, 0x3f800000}, {0x0155, 0x3f800000}, + {0x0152, 0x00000000}, {0x0154, 0x00000000}, {0x0156, 0x00000000}, {0x00aa, 0x80000000}, + {0x00ab, 0x40004000}, {0x00ca, 0x00000000}, {0x00cb, 0x00000000}, {0x0157, 0x3f800000}, + {0x0159, 0x3f800000}, {0x015b, 0x3f800000}, {0x0158, 0x00000000}, {0x015a, 0x00000000}, + {0x015c, 0x00000000}, {0x00ac, 0x80000000}, {0x00ad, 0x40004000}, {0x00cc, 0x00000000}, + {0x00cd, 0x00000000}, {0x015d, 0x3f800000}, {0x015f, 0x3f800000}, {0x0161, 0x3f800000}, + {0x015e, 0x00000000}, {0x0160, 0x00000000}, {0x0162, 0x00000000}, {0x00ae, 0x80000000}, + {0x00af, 0x40004000}, {0x00ce, 0x00000000}, {0x00cf, 0x00000000}, {0x0163, 0x3f800000}, + {0x0165, 0x3f800000}, {0x0167, 0x3f800000}, {0x0164, 0x00000000}, {0x0166, 0x00000000}, + {0x0168, 0x00000000}, {0x00b0, 0x80000000}, {0x00b1, 0x40004000}, {0x00d0, 0x00000000}, + {0x00d1, 0x00000000}, {0x0169, 0x3f800000}, {0x016b, 0x3f800000}, {0x016d, 0x3f800000}, + {0x016a, 0x00000000}, {0x016c, 0x00000000}, {0x016e, 0x00000000}, {0x00b2, 0x80000000}, + {0x00b3, 0x40004000}, {0x00d2, 0x00000000}, {0x00d3, 0x00000000}, {0x0081, 0x80000000}, + {0x0082, 0x40004000}, +}; + +static const uint16_t g_agc_public_reg_defaults_v9_tbl0_ptrs[] = { + 0x0000, 0x0001, 0x0002, 0x0003, 0x0004, 0x0005, 0x0006, 0x0007, 0x0008, 0x0009, 0x000a, 0x000b, + 0x000c, 0x000d, 0x000e, 0x000f, 0x0010, 0x0011, 0x0012, 0x0013, 0x0014, 0x0015, 0x0016, 0x0017, + 0x0018, 0x0019, 0x001a, 0x001b, 0x001c, 0x001d, 0x001e, 0x001f, 0x0020, 0x0021, 0x0022, 0x0023, + 0x0024, 0x0025, 0x0026, 0x0027, 0x0028, 0x0029, 0x002a, 0x002b, 0x002c, 0x002d, 0x002e, 0x002f, + 0x0030, 0x0031, 0x0032, 0x0033, 0x0034, 0x0035, 0x0036, 0x0038, 0x0048, 0x004a, 0x004c, 0x0050, + 0x0058, 0x005a, 0x0062, 0x0063, 0x0065, 0x0075, 0x0077, 0x0079, 0x007b, 0x007f, 0x0081, 0x0083, + 0x0085, 0x0105, 0x0107, 0x0127, 0x0147, 0x01e7, +}; + +static ShaderRegister g_agc_public_reg_defaults_v9_tbl1_regs[] = { + {0x0212, 0x00000000}, {0x0213, 0x00000000}, {0x0228, 0x00000000}, {0x0215, 0x00000000}, + {0x0218, 0x00000000}, {0x008a, 0x00000000}, {0x010a, 0x00000000}, {0x000a, 0x00000000}, + {0x008b, 0x00000000}, {0x010b, 0x00000000}, {0x000b, 0x00000000}, {0x0224, 0x00000000}, + {0x0225, 0x00000000}, {0x0226, 0x00000000}, {0x0227, 0x00000000}, {0x0107, 0xffff0000}, + {0x0087, 0x0000ffff}, {0x0081, 0x00000000}, {0x0007, 0x0000ffff}, {0x020c, 0x00000000}, + {0x020d, 0x00000000}, {0x00c8, 0x00000000}, {0x00c9, 0x00000000}, {0x0088, 0x00000000}, + {0x0089, 0x00000000}, {0x0108, 0x00000000}, {0x0109, 0x00000000}, {0x0148, 0x00000000}, + {0x0149, 0x00000000}, {0x0008, 0x00000000}, {0x0009, 0x00000000}, {0x0280, 0x00000000}, + {0x00b2, 0x00000000}, {0x00b3, 0x00000000}, {0x00b4, 0x00000000}, {0x00b5, 0x00000000}, + {0x0132, 0x00000000}, {0x0133, 0x00000000}, {0x0134, 0x00000000}, {0x0135, 0x00000000}, + {0x0032, 0x00000000}, {0x0033, 0x00000000}, {0x0034, 0x00000000}, {0x0035, 0x00000000}, + {0x0082, 0x00000000}, {0x0083, 0x00000000}, {0x0102, 0x00000000}, {0x0103, 0x00000000}, + {0x0240, 0x00000000}, {0x0241, 0x00000000}, {0x0242, 0x00000000}, {0x0243, 0x00000000}, + {0x0244, 0x00000000}, {0x0245, 0x00000000}, {0x0246, 0x00000000}, {0x0247, 0x00000000}, + {0x0248, 0x00000000}, {0x0249, 0x00000000}, {0x024a, 0x00000000}, {0x024b, 0x00000000}, + {0x024c, 0x00000000}, {0x024d, 0x00000000}, {0x024e, 0x00000000}, {0x024f, 0x00000000}, + {0x008c, 0x00000000}, {0x008d, 0x00000000}, {0x008e, 0x00000000}, {0x008f, 0x00000000}, + {0x0090, 0x00000000}, {0x0091, 0x00000000}, {0x0092, 0x00000000}, {0x0093, 0x00000000}, + {0x0094, 0x00000000}, {0x0095, 0x00000000}, {0x0096, 0x00000000}, {0x0097, 0x00000000}, + {0x0098, 0x00000000}, {0x0099, 0x00000000}, {0x009a, 0x00000000}, {0x009b, 0x00000000}, + {0x009c, 0x00000000}, {0x009d, 0x00000000}, {0x009e, 0x00000000}, {0x009f, 0x00000000}, + {0x00a0, 0x00000000}, {0x00a1, 0x00000000}, {0x00a2, 0x00000000}, {0x00a3, 0x00000000}, + {0x00a4, 0x00000000}, {0x00a5, 0x00000000}, {0x00a6, 0x00000000}, {0x00a7, 0x00000000}, + {0x00a8, 0x00000000}, {0x00a9, 0x00000000}, {0x00aa, 0x00000000}, {0x00ab, 0x00000000}, + {0x010c, 0x00000000}, {0x010d, 0x00000000}, {0x010e, 0x00000000}, {0x010f, 0x00000000}, + {0x0110, 0x00000000}, {0x0111, 0x00000000}, {0x0112, 0x00000000}, {0x0113, 0x00000000}, + {0x0114, 0x00000000}, {0x0115, 0x00000000}, {0x0116, 0x00000000}, {0x0117, 0x00000000}, + {0x0118, 0x00000000}, {0x0119, 0x00000000}, {0x011a, 0x00000000}, {0x011b, 0x00000000}, + {0x011c, 0x00000000}, {0x011d, 0x00000000}, {0x011e, 0x00000000}, {0x011f, 0x00000000}, + {0x0120, 0x00000000}, {0x0121, 0x00000000}, {0x0122, 0x00000000}, {0x0123, 0x00000000}, + {0x0124, 0x00000000}, {0x0125, 0x00000000}, {0x0126, 0x00000000}, {0x0127, 0x00000000}, + {0x0128, 0x00000000}, {0x0129, 0x00000000}, {0x012a, 0x00000000}, {0x012b, 0x00000000}, + {0x000c, 0x00000000}, {0x000d, 0x00000000}, {0x000e, 0x00000000}, {0x000f, 0x00000000}, + {0x0010, 0x00000000}, {0x0011, 0x00000000}, {0x0012, 0x00000000}, {0x0013, 0x00000000}, + {0x0014, 0x00000000}, {0x0015, 0x00000000}, {0x0016, 0x00000000}, {0x0017, 0x00000000}, + {0x0018, 0x00000000}, {0x0019, 0x00000000}, {0x001a, 0x00000000}, {0x001b, 0x00000000}, + {0x001c, 0x00000000}, {0x001d, 0x00000000}, {0x001e, 0x00000000}, {0x001f, 0x00000000}, + {0x0020, 0x00000000}, {0x0021, 0x00000000}, {0x0022, 0x00000000}, {0x0023, 0x00000000}, + {0x0024, 0x00000000}, {0x0025, 0x00000000}, {0x0026, 0x00000000}, {0x0027, 0x00000000}, + {0x0028, 0x00000000}, {0x0029, 0x00000000}, {0x002a, 0x00000000}, {0x002b, 0x00000000}, +}; + +static const uint16_t g_agc_public_reg_defaults_v9_tbl1_ptrs[] = { + 0x0000, 0x0001, 0x0002, 0x0003, 0x0004, 0x0005, 0x0006, 0x0007, 0x0008, 0x0009, 0x000a, 0x000b, + 0x000f, 0x0013, 0x0015, 0x0017, 0x0019, 0x001b, 0x001d, 0x001f, 0x0020, 0x0024, 0x0028, 0x002c, + 0x002e, 0x0030, 0x0040, 0x0060, 0x0080, +}; + +static ShaderRegister g_agc_public_reg_defaults_v9_tbl2_regs[] = { + {0x041f, 0x00000000}, {0x041d, 0x00000000}, {0x041e, 0x00000000}, {0x025b, 0x00000000}, + {0x024a, 0x00000000}, {0x024b, 0x00000000}, {0x025f, 0x00000000}, {0x0262, 0x00000000}, + {0x02bb, 0x00000000}, {0x0384, 0x00000000}, {0x0383, 0x40000040}, {0x0248, 0x00000000}, + {0x0242, 0x00000000}, {0x0380, 0x00000000}, {0x0381, 0x00000000}, {0x02e4, 0x00000000}, + {0x02e5, 0x00000000}, {0x02e6, 0x00000000}, {0x02e7, 0x00000000}, {0x02e8, 0x00000000}, + {0x02e9, 0x00000000}, {0x02ea, 0x00000000}, {0x02eb, 0x00000000}, {0x02c4, 0x00000000}, + {0x02c5, 0x00000000}, {0x02c6, 0x00000000}, {0x02c7, 0x00000000}, {0x02c8, 0x00000000}, + {0x02c9, 0x00000000}, {0x02ca, 0x00000000}, {0x02cb, 0x00000000}, {0x02cc, 0x00000000}, + {0x02cd, 0x00000000}, {0x02ce, 0x00000000}, {0x02cf, 0x00000000}, {0x02d0, 0x00000000}, + {0x02d1, 0x00000000}, {0x02d2, 0x00000000}, {0x02d3, 0x00000000}, {0x02bc, 0x00000000}, + {0x02bd, 0x00000000}, {0x02be, 0x00000000}, {0x02bf, 0x00000000}, {0x0035, 0x00000000}, + {0x0036, 0x00000000}, {0x0037, 0x00000000}, {0x0038, 0x00000000}, {0x0039, 0x00000000}, + {0x003a, 0x00000000}, {0x003b, 0x00000000}, {0x003c, 0x00000000}, {0x00a2, 0x00000000}, + {0x025c, 0x00000000}, {0x025d, 0x00000000}, {0x025e, 0x00000000}, +}; + +static const uint16_t g_agc_public_reg_defaults_v9_tbl2_ptrs[] = { + 0x0000, 0x0001, 0x0002, 0x0003, 0x0004, 0x0005, 0x0006, 0x0007, 0x0008, 0x0009, 0x000a, 0x000b, + 0x000c, 0x000d, 0x000f, 0x0017, 0x0027, 0x002b, 0x0033, 0x0034, +}; + +static uint32_t g_agc_public_reg_defaults_v9_types[] = { + 0xe24f806d, 0x00040400, 0x00000000, 0xf6c28182, 0x00040404, 0x00000000, + 0x6f6e55a5, 0x00040408, 0x00000000, 0x0bc65da4, 0x0004040c, 0x00000000, + 0x9e5ad592, 0x00040410, 0x00000000, 0xbb513b98, 0x00040414, 0x00000000, + 0xab64b23b, 0x00040418, 0x00000000, 0x53c39964, 0x0004041c, 0x00000000, + 0x01396b11, 0x00040420, 0x00000000, 0x7d42019a, 0x00040424, 0x00000000, + 0x3548f523, 0x00040428, 0x00000000, 0xf43ad28a, 0x0004042c, 0x00000000, + 0x6de4c312, 0x00040430, 0x00000000, 0x00a77ae0, 0x00040434, 0x00000000, + 0x00a779b7, 0x00040438, 0x00000000, 0x5100100c, 0x0004043c, 0x00000000, + 0x59958bba, 0x00040440, 0x00000000, 0x0c06f17c, 0x00040444, 0x00000000, + 0x6f104b72, 0x00040448, 0x00000000, 0x25c70d9c, 0x0004044c, 0x00000000, + 0x3881201e, 0x00040450, 0x00000000, 0x09afddaf, 0x00040454, 0x00000000, + 0x367d63cf, 0x00040458, 0x00000000, 0x43707db8, 0x0004045c, 0x00000000, + 0xf6ae26ba, 0x00040460, 0x00000000, 0x1b917652, 0x00040464, 0x00000000, + 0x94b1e4f7, 0x00040468, 0x00000000, 0xe3661b6c, 0x0004046c, 0x00000000, + 0x1eb8d73a, 0x00040470, 0x00000000, 0x15051fa3, 0x00040474, 0x00000000, + 0x9c51a7f1, 0x00040478, 0x00000000, 0xa20efc70, 0x0004047c, 0x00000000, + 0x0ec09f6e, 0x00040480, 0x00000000, 0x34a7d6d3, 0x00040484, 0x00000000, + 0xce831b94, 0x00040488, 0x00000000, 0x5cc72a74, 0x0004048c, 0x00000000, + 0x3b77713c, 0x00040490, 0x00000000, 0x40f64410, 0x00040494, 0x00000000, + 0x69441268, 0x00040498, 0x00000000, 0x2e418b83, 0x0004049c, 0x00000000, + 0xa00d0c8d, 0x000404a0, 0x00000000, 0xb1289fb3, 0x000404a4, 0x00000000, + 0x144832fb, 0x000404a8, 0x00000000, 0x9890d9fa, 0x000404ac, 0x00000000, + 0x9016faf1, 0x000404b0, 0x00000000, 0x4b73ce27, 0x000404b4, 0x00000000, + 0x5f5a3e7b, 0x000404b8, 0x00000000, 0xd4af3a51, 0x000404bc, 0x00000000, + 0x6cf4f543, 0x000404c0, 0x00000000, 0x5fb86ccb, 0x000404c4, 0x00000000, + 0xedefa188, 0x000404c8, 0x00000000, 0xd0de9ee6, 0x000404cc, 0x00000000, + 0xc5831803, 0x000404d0, 0x00000000, 0x8e6de84b, 0x000404d4, 0x00000000, + 0xd0771662, 0x000408d8, 0x00000000, 0x569f7444, 0x004004dc, 0x00000000, + 0x5c6637cd, 0x000408e0, 0x00000000, 0xcae3e690, 0x000408e4, 0x00000000, + 0x43fbd769, 0x000410e8, 0x00000000, 0xef550356, 0x002004ec, 0x00000000, + 0x8f52e279, 0x000408f0, 0x00000000, 0x1f2d8149, 0x001008f4, 0x00000000, + 0x853d0614, 0x000404f8, 0x00000000, 0x4413c6f9, 0x000408fc, 0x00000000, + 0x67096014, 0x00044100, 0x00000000, 0x88f5e915, 0x00040904, 0x00000000, + 0x033f1eff, 0x00040908, 0x00000000, 0x918106bb, 0x0004090c, 0x00000000, + 0x95f0e7ac, 0x00041110, 0x00000000, 0xb48cbab2, 0x00040914, 0x00000000, + 0x05bb3bc6, 0x00040918, 0x00000000, 0x94faba07, 0x0004091c, 0x00000000, + 0x38e92c91, 0x00204120, 0x00000000, 0x0b177b43, 0x00040924, 0x00000000, + 0x48531062, 0x00800528, 0x00000000, 0xaaa964b9, 0x0020112c, 0x00000000, + 0x7690af6f, 0x00402930, 0x00000000, 0x078d7060, 0x00040934, 0x00000000, + 0x5d6e3ec7, 0x00040401, 0x00000000, 0x57e7079a, 0x00040405, 0x00000000, + 0x7467fafd, 0x00040409, 0x00000000, 0x9e826b50, 0x0004040d, 0x00000000, + 0xdc484f18, 0x00040411, 0x00000000, 0x5da8bca3, 0x00040415, 0x00000000, + 0x5ca726d8, 0x00040419, 0x00000000, 0x5dd28360, 0x0004041d, 0x00000000, + 0x57efa0be, 0x00040421, 0x00000000, 0x502363d5, 0x00040425, 0x00000000, + 0x506d14bd, 0x00040429, 0x00000000, 0xb2609506, 0x0010042d, 0x00000000, + 0x9e5cfb8a, 0x00041031, 0x00000000, 0xc918df3e, 0x00040835, 0x00000000, + 0xc9751c9c, 0x00040839, 0x00000000, 0xc97ef77a, 0x0004083d, 0x00000000, + 0xc927c6b9, 0x00040841, 0x00000000, 0xc92a1ec5, 0x00040845, 0x00000000, + 0xc9e01b31, 0x00040849, 0x00000000, 0x50685f29, 0x0004044d, 0x00000000, + 0xb26219ca, 0x00100451, 0x00000000, 0xb25b6cf9, 0x00100455, 0x00000000, + 0xb2f86101, 0x00100459, 0x00000000, 0x07e3b155, 0x0004085d, 0x00000000, + 0x07e383c6, 0x00040861, 0x00000000, 0xbda98653, 0x00400465, 0x00000000, + 0xbdbd1d0f, 0x00800469, 0x00000000, 0xbd946fd4, 0x0080046d, 0x00000000, + 0xbdf02a4c, 0x00800471, 0x00000000, 0x19e93e85, 0x00040402, 0x00000000, + 0x3b5c2af3, 0x00040406, 0x00000000, 0x47974a35, 0x0004040a, 0x00000000, + 0x105971c2, 0x0004040e, 0x00000000, 0x7d137765, 0x00040412, 0x00000000, + 0xd187febc, 0x00040416, 0x00000000, 0x12f854ac, 0x0004041a, 0x00000000, + 0x40d49ad1, 0x0004041e, 0x00000000, 0x8c0923da, 0x00040422, 0x00000000, + 0xbb8df494, 0x00040426, 0x00000000, 0xf6d8a76e, 0x0004042a, 0x00000000, + 0x7620f1e9, 0x0004042e, 0x00000000, 0x9ebfab10, 0x00040432, 0x00000000, + 0x98a09d0e, 0x00040836, 0x00000000, 0x195d37d2, 0x0010083a, 0x00000000, + 0xf9ec4f85, 0x0010103e, 0x00000000, 0x4626b750, 0x00080842, 0x00000000, + 0x4cc673a0, 0x00200446, 0x00000000, 0xde5b3431, 0x0004044a, 0x00000000, + 0x036ac8a6, 0x000c044e, 0x00000000, +}; + +static CompactRegisterDefaults g_agc_public_reg_defaults_v9 = { + g_agc_public_reg_defaults_v9_tbl0_regs, 489, g_agc_public_reg_defaults_v9_tbl0_ptrs, 78, + g_agc_public_reg_defaults_v9_tbl1_regs, 160, g_agc_public_reg_defaults_v9_tbl1_ptrs, 29, + g_agc_public_reg_defaults_v9_tbl2_regs, 55, g_agc_public_reg_defaults_v9_tbl2_ptrs, 20, + nullptr, 0, nullptr, 0, + g_agc_public_reg_defaults_v9_types, 127, +}; + +static CompactRegisterDefaults* g_agc_public_reg_defaults_by_version[] = { + &g_agc_public_reg_defaults_v0, // version 0 + &g_agc_public_reg_defaults_v0, // version 1 + &g_agc_public_reg_defaults_v0, // version 2 + &g_agc_public_reg_defaults_v0, // version 3 + &g_agc_public_reg_defaults_v4, // version 4 + &g_agc_public_reg_defaults_v5, // version 5 + &g_agc_public_reg_defaults_v5, // version 6 + &g_agc_public_reg_defaults_v7, // version 7 + &g_agc_public_reg_defaults_v8, // version 8 + &g_agc_public_reg_defaults_v9, // version 9 + &g_agc_public_reg_defaults_v10, // version 10 + &g_agc_public_reg_defaults_v11, // version 11 + &g_agc_public_reg_defaults_v10, // version 12 +}; + +static CompactRegisterDefaults* g_agc_internal_reg_defaults_by_version[] = { + &g_agc_internal_reg_defaults_v0, // version 0 + &g_agc_internal_reg_defaults_v0, // version 1 + &g_agc_internal_reg_defaults_v0, // version 2 + &g_agc_internal_reg_defaults_v0, // version 3 + &g_agc_internal_reg_defaults_v4, // version 4 + &g_agc_internal_reg_defaults_v5, // version 5 + &g_agc_internal_reg_defaults_v5, // version 6 + &g_agc_internal_reg_defaults_v7, // version 7 + &g_agc_internal_reg_defaults_v8, // version 8 + &g_agc_internal_reg_defaults_v9, // version 9 + &g_agc_internal_reg_defaults_v10, // version 10 + &g_agc_internal_reg_defaults_v11, // version 11 + &g_agc_internal_reg_defaults_v10, // version 12 +}; diff --git a/src/libs/ajm.cpp b/src/libs/ajm.cpp new file mode 100644 index 000000000..89473ad62 --- /dev/null +++ b/src/libs/ajm.cpp @@ -0,0 +1,1361 @@ +#include "common/assert.h" +#include "common/common.h" +#include "common/logging/log.h" +#include "libs/audio.h" +#include "libs/errno.h" +#include "libs/libs.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace Libs::Audio::Ajm { + +LIB_NAME("Ajm", "Ajm"); + +struct AjmBatchInfo { + void* buffer; + size_t offset; + size_t size; + void* last_good_job; + const void* last_good_job_ra; +}; + +struct AjmBatchError { + int error_code; + const void* job_addr; + uint32_t cmd_offset; + const void* job_ra; +}; + +struct AjmBuffer { + void* address; + size_t size; +}; + +struct AjmSidebandResult { + int32_t result; + int32_t internal_result; +}; + +struct AjmSidebandStream { + int32_t size_consumed; + int32_t size_produced; + uint64_t total_decoded_samples; +}; + +struct AjmSidebandMFrame { + uint32_t num_frames; + uint32_t reserved; +}; + +struct AjmSidebandFormat { + uint32_t channel_num; + uint32_t channel_mask; + uint32_t sampling_frequency; + uint32_t sample_encoding; + uint32_t bitrate; + uint32_t reserved; +}; + +struct AjmSidebandGaplessDecode { + uint32_t total_samples; + uint16_t skip_samples; + uint16_t skipped_samples; +}; + +struct AjmGaplessState { + AjmSidebandGaplessDecode init {}; + AjmSidebandGaplessDecode current {}; + + [[nodiscard]] bool HasSampleLimit() const { + return init.total_samples != 0 && + init.total_samples != std::numeric_limits::max(); + } + + [[nodiscard]] bool IsEnd() const { return HasSampleLimit() && current.total_samples == 0; } + + void Reset() { + current = init; + current.skipped_samples = 0; + } + + void Set(const AjmSidebandGaplessDecode& params, bool reset) { + init.total_samples = params.total_samples; + init.skip_samples = params.skip_samples; + if (reset || (current.total_samples == 0 && current.skip_samples == 0 && + current.skipped_samples == 0)) { + Reset(); + } + } +}; + +struct AjmSidebandResampleInfo { + float ratio; + int32_t num_samples; + uint32_t reserved[8]; +}; + +struct AjmGetStatisticsResult { + AjmSidebandResult result; + float engine_usage_batch; + float engine_usage_interval[3]; + uint32_t memory[6]; +}; + +struct AjmDecOpusInitializeParameters { + uint32_t channel_num; + uint32_t sample_rate; + uint32_t mapping_family; +}; + +struct AjmSidebandDecOpusCodecInfo { + uint32_t frames_per_packet; +}; + +enum class AjmCodec : uint32_t { + DecMp3 = 0, + DecAt9 = 1, + DecM4aac = 2, + DecUnknownNewABI = 14, + DecOpus = 24, +}; + +enum class AjmSampleEncoding : uint32_t { + S16 = 0, + S32 = 1, + Float = 2, +}; + +struct AjmDecodeResult { + int32_t result = OK; + int32_t internal_result = OK; + size_t input_consumed = 0; + size_t output_written = 0; + uint64_t total_decoded_samples = 0; + uint32_t frames = 0; + uint32_t frames_per_packet = 0; + AjmSidebandFormat format {}; +}; + +constexpr int32_t AJM_ERROR_INVALID_CONTEXT = static_cast(0x80930002u); +constexpr int32_t AJM_ERROR_INVALID_PARAMETER = static_cast(0x80930005u); +constexpr int32_t AJM_ERROR_CODEC_NOT_SUPPORTED = static_cast(0x80930008u); +constexpr int32_t AJM_ERROR_JOB_CREATION = static_cast(0x80930012u); +constexpr int32_t AJM_RESULT_NOT_INITIALIZED = 0x00000001; +constexpr int32_t AJM_RESULT_INVALID_DATA = 0x00000002; +constexpr int32_t AJM_RESULT_INVALID_PARAMETER = 0x00000004; +constexpr int32_t AJM_RESULT_PARTIAL_INPUT = 0x00000008; +constexpr int32_t AJM_RESULT_NOT_ENOUGH_ROOM = 0x00000010; +constexpr int32_t AJM_RESULT_CODEC_ERROR = 0x40000000; +constexpr int32_t AJM_RESULT_FATAL = static_cast(0x80000000u); +constexpr size_t AJM_JOB_CONTROL_SIZE = 48; +constexpr size_t AJM_JOB_RUN_SIZE = 64; +constexpr size_t AJM_JOB_GET_STATISTICS_SIZE = + sizeof(uint32_t) + sizeof(float) * 3 + 8 + AJM_JOB_RUN_SIZE; +constexpr size_t AJM_JOB_SET_RESAMPLE_PARAMETERS_SIZE = + sizeof(float) * 2 + sizeof(uint32_t) + 12 + AJM_JOB_CONTROL_SIZE; +constexpr size_t AJM_SIDEBAND_RESULT_SIZE = sizeof(AjmSidebandResult); +constexpr size_t AJM_SIDEBAND_STREAM_SIZE = sizeof(AjmSidebandStream); +constexpr size_t AJM_SIDEBAND_MFRAME_SIZE = sizeof(AjmSidebandMFrame); +constexpr size_t AJM_SIDEBAND_FORMAT_SIZE = sizeof(AjmSidebandFormat); +constexpr size_t AJM_SIDEBAND_GAPLESS_SIZE = sizeof(AjmSidebandGaplessDecode); +constexpr size_t AJM_SIDEBAND_RESAMPLE_INFO_SIZE = sizeof(AjmSidebandResampleInfo); +constexpr uint64_t AJM_FLAG_RUN_GET_CODEC_INFO = 1ull << 11u; +constexpr uint64_t AJM_FLAG_RUN_MULTIPLE_FRAMES = 1ull << 12u; +constexpr uint64_t AJM_FLAG_CONTROL_RESET = 1ull << 13u; +constexpr uint64_t AJM_FLAG_CONTROL_INITIALIZE = 1ull << 14u; +constexpr uint64_t AJM_FLAG_SIDEBAND_RESAMPLE_INFO = 1ull << 44u; +constexpr uint64_t AJM_FLAG_SIDEBAND_GAPLESS = 1ull << 45u; +constexpr uint64_t AJM_FLAG_SIDEBAND_FORMAT = 1ull << 46u; +constexpr uint64_t AJM_FLAG_SIDEBAND_STREAM = 1ull << 47u; +constexpr uint64_t AJM_INSTANCE_FLAG_FORMAT_OFFSET = 7u; +constexpr uint64_t AJM_INSTANCE_FLAG_FORMAT_MASK = 0x7u; +constexpr uint64_t AJM_INSTANCE_FLAG_MAX_CHANNEL_MASK = 0x7fu; +constexpr uint64_t AJM_INSTANCE_FLAG_CODEC_OFFSET = 32u; +constexpr uint32_t AJM_DEC_OPUS_MAX_CHANNELS_FOR_10CH = 10; +constexpr uint32_t AJM_DEC_OPUS_FRAME_SAMPLES = 960; + +static std::atomic_uint32_t g_ajm_next_instance {1}; +static std::atomic_uint32_t g_ajm_next_batch {1}; + +static uint32_t AjmGetFlagChannelCount(uint64_t flags) { + return static_cast(flags & AJM_INSTANCE_FLAG_MAX_CHANNEL_MASK); +} + +static AjmSampleEncoding AjmGetFlagSampleEncoding(uint64_t flags) { + const auto encoding = static_cast((flags >> AJM_INSTANCE_FLAG_FORMAT_OFFSET) & + AJM_INSTANCE_FLAG_FORMAT_MASK); + switch (encoding) { + case static_cast(AjmSampleEncoding::S16): return AjmSampleEncoding::S16; + case static_cast(AjmSampleEncoding::S32): return AjmSampleEncoding::S32; + case static_cast(AjmSampleEncoding::Float): return AjmSampleEncoding::Float; + default: EXIT("unsupported AJM PCM sample encoding %u\n", encoding); + } + return AjmSampleEncoding::S16; +} + +static size_t AjmBytesPerSample(AjmSampleEncoding encoding) { + switch (encoding) { + case AjmSampleEncoding::S16: return sizeof(int16_t); + case AjmSampleEncoding::S32: return sizeof(int32_t); + case AjmSampleEncoding::Float: return sizeof(float); + default: EXIT("unsupported AJM PCM sample encoding %u\n", static_cast(encoding)); + } + return sizeof(int16_t); +} + +static uint32_t AjmChannelMask(uint32_t channels) { + switch (channels) { + case 1: return 0x4; + case 2: return 0x3; + case 3: return 0x7; + case 4: return 0x33; + case 5: return 0x607; + case 6: return 0x60f; + case 7: return 0x70f; + case 8: return 0x63f; + default: return 0; + } +} + +static AjmSidebandFormat AjmMakeFormat(uint32_t channels, uint32_t sample_rate, + AjmSampleEncoding encoding) { + AjmSidebandFormat format {}; + format.channel_num = channels; + format.channel_mask = AjmChannelMask(channels); + format.sampling_frequency = sample_rate; + format.sample_encoding = static_cast(encoding); + format.bitrate = 0; + return format; +} + +static const char* AjmCodecName(uint32_t codec) { + switch (codec) { + case static_cast(AjmCodec::DecMp3): return "MP3 decoder"; + case static_cast(AjmCodec::DecAt9): return "ATRAC9 decoder"; + case static_cast(AjmCodec::DecM4aac): return "MPEG4-AAC decoder"; + case 3: return "CELP8 decoder"; + case 4: return "CELP8 encoder"; + case 12: return "CELP16 decoder"; + case 13: return "CELP16 encoder"; + case 22: return "HE-VAG decoder"; + case 23: return "LPCM decoder"; + case static_cast(AjmCodec::DecOpus): return "Opus decoder"; + default: return "unknown"; + } +} + +class AjmDecoder { +public: + AjmDecoder(uint32_t channels, uint32_t sample_rate, AjmSampleEncoding encoding) + : m_channels(channels), m_sample_rate(sample_rate), m_sample_encoding(encoding) {} + virtual ~AjmDecoder() = default; + + KYTY_CLASS_NO_COPY(AjmDecoder); + + virtual AjmDecodeResult Initialize(const void* codec_parameters, size_t codec_parameters_size) { + (void)codec_parameters; + (void)codec_parameters_size; + return MakeResult(); + } + + virtual void Reset() { m_total_decoded_samples = 0; } + + virtual AjmDecodeResult Decode(const void* input, size_t input_size, void* output, + size_t output_size, bool multiple_frames, + AjmGaplessState* gapless) = 0; + + virtual void WriteCodecInfo(void* output, size_t output_size, + const AjmDecodeResult& result) const { + (void)output; + (void)output_size; + (void)result; + } + + [[nodiscard]] virtual size_t CodecInfoSize() const { return 0; } + + [[nodiscard]] virtual AjmSidebandFormat GetFormat() const { + return AjmMakeFormat(m_channels, m_sample_rate, m_sample_encoding); + } + + [[nodiscard]] AjmDecodeResult MakeResult() const { + AjmDecodeResult result {}; + result.total_decoded_samples = m_total_decoded_samples; + result.format = GetFormat(); + return result; + } + +protected: + void SetFormat(uint32_t channels, uint32_t sample_rate, AjmSampleEncoding encoding) { + m_channels = channels; + m_sample_rate = sample_rate; + m_sample_encoding = encoding; + } + + uint32_t m_channels = 2; + uint32_t m_sample_rate = 48000; + AjmSampleEncoding m_sample_encoding = AjmSampleEncoding::S16; + uint64_t m_total_decoded_samples = 0; +}; + +} // namespace Libs::Audio::Ajm + +#include "libs/ajm/aac_decoder.h" +#include "libs/ajm/atrac9_decoder.h" +#include "libs/ajm/mp3_decoder.h" + +namespace Libs::Audio::Ajm { + +static_assert(sizeof(AjmDecAt9ConfigDataInfo) == 20); + +int KYTY_SYSV_ABI AjmDecAt9ParseConfigData(const void* config_data, + AjmDecAt9ConfigDataInfo* config_info) { + PRINT_NAME(); + if (config_data == nullptr || config_info == nullptr) { + return AJM_ERROR_INVALID_PARAMETER; + } + + auto* handle = Atrac9GetHandle(); + if (handle == nullptr) { + return AJM_ERROR_INVALID_PARAMETER; + } + + uint8_t config[ATRAC9_CONFIG_DATA_SIZE] {}; + std::memcpy(config, config_data, sizeof(config)); + Atrac9CodecInfo codec_info {}; + const int init_result = Atrac9InitDecoder(handle, config); + const int info_result = + init_result == 0 ? Atrac9GetCodecInfo(handle, &codec_info) : init_result; + Atrac9ReleaseHandle(handle); + + if (info_result != 0 || codec_info.channels <= 0 || codec_info.samplingRate <= 0 || + codec_info.frameSamples <= 0 || codec_info.framesInSuperframe <= 0 || + codec_info.superframeSize <= 0) { + return AJM_ERROR_INVALID_PARAMETER; + } + + config_info->channels = static_cast(codec_info.channels); + config_info->sample_rate = static_cast(codec_info.samplingRate); + config_info->frame_samples_per_channel = static_cast(codec_info.frameSamples); + config_info->superframe_samples_per_channel = + static_cast(codec_info.frameSamples) * + static_cast(codec_info.framesInSuperframe); + config_info->superframe_size = static_cast(codec_info.superframeSize); + return OK; +} + +class AjmOpusDecoder final: public AjmDecoder { +public: + AjmOpusDecoder(uint32_t channels, uint32_t sample_rate, AjmSampleEncoding encoding, + uint64_t flags) + : AjmDecoder(channels, sample_rate, encoding) { + (void)flags; + } + + AjmDecodeResult Initialize(const void* codec_parameters, + size_t codec_parameters_size) override { + auto result = MakeResult(); + + if (codec_parameters == nullptr || + codec_parameters_size < sizeof(AjmDecOpusInitializeParameters)) { + result.result = AJM_RESULT_INVALID_PARAMETER; + return result; + } + + const auto* params = static_cast(codec_parameters); + if (params->channel_num == 0 || params->channel_num > AJM_DEC_OPUS_MAX_CHANNELS_FOR_10CH || + params->sample_rate == 0) { + result.result = AJM_RESULT_INVALID_PARAMETER; + return result; + } + + SetFormat(params->channel_num, params->sample_rate, m_sample_encoding); + m_mapping_family = params->mapping_family; + m_total_decoded_samples = 0; + m_is_initialized = true; + m_frames_per_packet = 1; + + LOGF("AJM Opus initialized: %" PRIu32 " Hz, %" PRIu32 " ch, mapping=%" PRIu32 "\n", + params->sample_rate, params->channel_num, params->mapping_family); + + return MakeResult(); + } + + void Reset() override { m_total_decoded_samples = 0; } + + AjmDecodeResult Decode(const void* input, size_t input_size, void* output, size_t output_size, + bool multiple_frames, AjmGaplessState* gapless) override { + (void)multiple_frames; + auto result = MakeResult(); + + if (input == nullptr || input_size == 0) { + result.result = AJM_RESULT_PARTIAL_INPUT; + return result; + } + if (!m_is_initialized) { + result.result = AJM_RESULT_NOT_INITIALIZED; + return result; + } + + auto samples_to_write = AJM_DEC_OPUS_FRAME_SAMPLES; + auto skip_samples = 0u; + if (gapless != nullptr) { + skip_samples = std::min(samples_to_write, gapless->current.skip_samples); + samples_to_write -= skip_samples; + if (gapless->HasSampleLimit()) { + samples_to_write = + std::min(samples_to_write, gapless->current.total_samples); + } + } + + const auto output_bytes = static_cast(samples_to_write) * + static_cast(m_channels) * + AjmBytesPerSample(m_sample_encoding); + if (output_bytes != 0 && output == nullptr) { + result.result = AJM_RESULT_NOT_ENOUGH_ROOM; + return result; + } + if (output_bytes > output_size) { + result.result = AJM_RESULT_NOT_ENOUGH_ROOM; + return result; + } + + if (output_bytes != 0) { + std::memset(output, 0, output_bytes); + } + + if (gapless != nullptr) { + gapless->current.skip_samples = + static_cast(gapless->current.skip_samples - skip_samples); + if (gapless->HasSampleLimit()) { + gapless->current.total_samples = + (gapless->current.total_samples > samples_to_write + ? gapless->current.total_samples - samples_to_write + : 0); + } + + const auto skipped_samples = AJM_DEC_OPUS_FRAME_SAMPLES - samples_to_write; + const auto skipped_total = std::min( + std::numeric_limits::max(), + static_cast(gapless->current.skipped_samples) + skipped_samples); + gapless->current.skipped_samples = static_cast(skipped_total); + } + + result.input_consumed = input_size; + result.output_written = output_bytes; + result.frames = 1; + result.frames_per_packet = m_frames_per_packet; + m_total_decoded_samples += samples_to_write; + result.total_decoded_samples = m_total_decoded_samples; + result.format = GetFormat(); + return result; + } + + void WriteCodecInfo(void* output, size_t output_size, + const AjmDecodeResult& result) const override { + (void)result; + if (output == nullptr || output_size < sizeof(AjmSidebandDecOpusCodecInfo)) { + return; + } + + auto* info = static_cast(output); + info->frames_per_packet = m_frames_per_packet; + } + + [[nodiscard]] size_t CodecInfoSize() const override { + return sizeof(AjmSidebandDecOpusCodecInfo); + } + +private: + bool m_is_initialized = false; + uint32_t m_mapping_family = 0; + uint32_t m_frames_per_packet = 1; +}; + +struct AjmInstanceState { + uint32_t context = 0; + uint32_t codec = 0; + uint64_t flags = 0; + AjmGaplessState gapless; + std::unique_ptr decoder; +}; + +static std::mutex g_ajm_instances_mutex; +static std::unordered_map g_ajm_instances; + +static bool AjmCodecIsSupported(uint32_t codec) { + return codec == static_cast(AjmCodec::DecMp3) || + codec == static_cast(AjmCodec::DecAt9) || + codec == static_cast(AjmCodec::DecM4aac) || + codec == static_cast(AjmCodec::DecOpus) || + codec == static_cast(AjmCodec::DecUnknownNewABI); +} + +static bool AjmLogCodecSupport(uint32_t codec) { + const auto supported = AjmCodecIsSupported(codec); + if (supported) { + static std::atomic_bool logged {false}; + if (!logged.exchange(true, std::memory_order_relaxed)) { + std::printf("AJM codec=%u, name=%s\n", codec, AjmCodecName(codec)); + } + } else { + std::printf("AJM codec=%u, name=%s\n", codec, AjmCodecName(codec)); + std::printf("AJM unsupported codec=%u (%s)\n", codec, AjmCodecName(codec)); + } + return supported; +} + +static std::unique_ptr AjmCreateDecoder(uint32_t codec, uint64_t flags) { + if (!AjmCodecIsSupported(codec)) { + return {}; + } + + auto channels = AjmGetFlagChannelCount(flags); + if (channels == 0) { + channels = 2; + } + + const auto encoding = AjmGetFlagSampleEncoding(flags); + switch (codec) { + case static_cast(AjmCodec::DecMp3): + return std::make_unique(channels, 48000, encoding, flags); + case static_cast(AjmCodec::DecAt9): + return std::make_unique(channels, 48000, encoding, flags); + case static_cast(AjmCodec::DecM4aac): + return std::make_unique(channels, 48000, encoding, flags); + case static_cast(AjmCodec::DecOpus): + return std::make_unique(channels, 48000, encoding, flags); + default: break; + } + + return {}; +} + +static AjmInstanceState* AjmFindInstanceLocked(uint32_t instance) { + const auto it = g_ajm_instances.find(instance); + if (it == g_ajm_instances.end()) { + return nullptr; + } + return &it->second; +} + +static uint8_t* AjmReserveJob(AjmBatchInfo* info, size_t job_size) { + EXIT_NOT_IMPLEMENTED(info == nullptr); + + if (info->offset + job_size > info->size) { + return nullptr; + } + + auto* job = static_cast(info->buffer) + info->offset; + info->last_good_job = job; + info->last_good_job_ra = nullptr; + info->offset += job_size; + + std::memset(job, 0, job_size); + + return job; +} + +static int AjmAppendJob(AjmBatchInfo* info, size_t job_size, const char* name) { + auto* job = AjmReserveJob(info, job_size); + if (job == nullptr) { + LOGF("\t %s: not enough batch space, offset=0x%016" PRIx64 ", size=0x%016" PRIx64 + ", job=0x%016" PRIx64 "\n", + name, static_cast(info->offset), static_cast(info->size), + static_cast(job_size)); + return AJM_ERROR_JOB_CREATION; + } + + return OK; +} + +static int32_t AjmClampInt32(size_t value) { + return static_cast( + std::min(value, static_cast(std::numeric_limits::max()))); +} + +static void AjmWriteResult(void* result, size_t size, const AjmDecodeResult& decode_result = {}) { + if (result == nullptr || size == 0) { + return; + } + + std::memset(result, 0, size); + + if (size >= AJM_SIDEBAND_RESULT_SIZE) { + auto* sideband = static_cast(result); + sideband->result = decode_result.result; + sideband->internal_result = decode_result.internal_result; + } +} + +static void AjmWriteStreamResult(void* result, size_t result_size, + const AjmDecodeResult& decode_result, bool multiple_frames) { + if (result == nullptr) { + return; + } + + AjmWriteResult(result, result_size, decode_result); + + if (result_size >= AJM_SIDEBAND_RESULT_SIZE + AJM_SIDEBAND_STREAM_SIZE) { + auto* stream = reinterpret_cast(static_cast(result) + + AJM_SIDEBAND_RESULT_SIZE); + stream->size_consumed = AjmClampInt32(decode_result.input_consumed); + stream->size_produced = AjmClampInt32(decode_result.output_written); + stream->total_decoded_samples = decode_result.total_decoded_samples; + } + + if (multiple_frames && result_size >= AJM_SIDEBAND_RESULT_SIZE + AJM_SIDEBAND_STREAM_SIZE + + AJM_SIDEBAND_MFRAME_SIZE) { + auto* mframe = reinterpret_cast( + static_cast(result) + AJM_SIDEBAND_RESULT_SIZE + AJM_SIDEBAND_STREAM_SIZE); + mframe->num_frames = decode_result.frames; + } +} + +static void AjmWriteFormat(void* result, size_t result_size, const AjmSidebandFormat& format, + const AjmDecodeResult& decode_result = {}) { + if (result == nullptr || result_size < AJM_SIDEBAND_RESULT_SIZE + AJM_SIDEBAND_FORMAT_SIZE) { + AjmWriteResult(result, result_size, decode_result); + return; + } + + AjmWriteResult(result, result_size, decode_result); + + auto* format_output = reinterpret_cast(static_cast(result) + + AJM_SIDEBAND_RESULT_SIZE); + *format_output = format; +} + +static void AjmWriteCodecInfo(AjmDecoder* decoder, void* result, size_t result_size, + const AjmDecodeResult& decode_result) { + if (decoder == nullptr || result == nullptr || result_size <= AJM_SIDEBAND_RESULT_SIZE) { + return; + } + decoder->WriteCodecInfo(static_cast(result) + AJM_SIDEBAND_RESULT_SIZE, + result_size - AJM_SIDEBAND_RESULT_SIZE, decode_result); +} + +static void AjmWriteSideband(uint64_t flags, void* sideband_output, size_t sideband_output_size, + AjmDecoder* decoder, const AjmGaplessState* gapless, + const AjmDecodeResult& decode_result) { + if (sideband_output == nullptr || sideband_output_size == 0) { + return; + } + + AjmWriteResult(sideband_output, sideband_output_size, decode_result); + + size_t offset = AJM_SIDEBAND_RESULT_SIZE; + + if ((flags & AJM_FLAG_RUN_GET_CODEC_INFO) != 0 && decoder != nullptr) { + const auto codec_info_size = decoder->CodecInfoSize(); + if (offset + codec_info_size <= sideband_output_size) { + decoder->WriteCodecInfo(static_cast(sideband_output) + offset, + sideband_output_size - offset, decode_result); + offset += codec_info_size; + } + } + + if ((flags & AJM_FLAG_SIDEBAND_RESAMPLE_INFO) != 0 && + offset + AJM_SIDEBAND_RESAMPLE_INFO_SIZE <= sideband_output_size) { + auto* info = reinterpret_cast( + static_cast(sideband_output) + offset); + info->ratio = 1.0f; + offset += AJM_SIDEBAND_RESAMPLE_INFO_SIZE; + } + + if ((flags & AJM_FLAG_SIDEBAND_GAPLESS) != 0 && + offset + AJM_SIDEBAND_GAPLESS_SIZE <= sideband_output_size) { + auto* gapless_output = reinterpret_cast( + static_cast(sideband_output) + offset); + if (gapless != nullptr) { + *gapless_output = gapless->current; + } + offset += AJM_SIDEBAND_GAPLESS_SIZE; + } + + if ((flags & AJM_FLAG_SIDEBAND_FORMAT) != 0 && + offset + AJM_SIDEBAND_FORMAT_SIZE <= sideband_output_size) { + auto* format = + reinterpret_cast(static_cast(sideband_output) + offset); + *format = decode_result.format; + offset += AJM_SIDEBAND_FORMAT_SIZE; + } + + if ((flags & AJM_FLAG_SIDEBAND_STREAM) != 0 && + offset + AJM_SIDEBAND_STREAM_SIZE <= sideband_output_size) { + auto* stream = + reinterpret_cast(static_cast(sideband_output) + offset); + stream->size_consumed = AjmClampInt32(decode_result.input_consumed); + stream->size_produced = AjmClampInt32(decode_result.output_written); + stream->total_decoded_samples = decode_result.total_decoded_samples; + offset += AJM_SIDEBAND_STREAM_SIZE; + } + + if ((flags & AJM_FLAG_RUN_MULTIPLE_FRAMES) != 0 && + offset + AJM_SIDEBAND_MFRAME_SIZE <= sideband_output_size) { + auto* mframe = + reinterpret_cast(static_cast(sideband_output) + offset); + mframe->num_frames = decode_result.frames; + } +} + +static bool AjmCodecIsValid(uint32_t codec) { + switch (codec) { + case static_cast(AjmCodec::DecMp3): + case static_cast(AjmCodec::DecAt9): + case static_cast(AjmCodec::DecM4aac): + case 3: + case 4: + case 12: + case 13: + case static_cast(AjmCodec::DecUnknownNewABI): + case 22: + case 23: + case static_cast(AjmCodec::DecOpus): return true; + default: return false; + } +} + +static AjmDecodeResult AjmDecodeInstance(uint32_t instance, const void* input, size_t input_size, + void* output, size_t output_size, bool multiple_frames) { + std::scoped_lock lock(g_ajm_instances_mutex); + auto* state = AjmFindInstanceLocked(instance); + if (state == nullptr || state->decoder == nullptr) { + AjmDecodeResult result {}; + result.result = AJM_RESULT_INVALID_PARAMETER; + return result; + } + + return state->decoder->Decode(input, input_size, output, output_size, multiple_frames, + &state->gapless); +} + +static AjmDecodeResult AjmDecodeSplitInstance(uint32_t instance, const AjmBuffer* input_buffers, + size_t input_buffers_num, + const AjmBuffer* output_buffers, + size_t output_buffers_num, bool multiple_frames) { + size_t input_size = 0; + size_t output_size = 0; + for (size_t i = 0; i < input_buffers_num && input_buffers != nullptr; i++) { + input_size += input_buffers[i].size; + } + for (size_t i = 0; i < output_buffers_num && output_buffers != nullptr; i++) { + output_size += output_buffers[i].size; + } + + std::vector input(input_size); + size_t input_offset = 0; + for (size_t i = 0; i < input_buffers_num && input_buffers != nullptr; i++) { + if (input_buffers[i].address != nullptr && input_buffers[i].size != 0) { + std::memcpy(input.data() + input_offset, input_buffers[i].address, + input_buffers[i].size); + } + input_offset += input_buffers[i].size; + } + + std::vector output(output_size); + auto result = AjmDecodeInstance(instance, input.data(), input.size(), output.data(), + output.size(), multiple_frames); + + size_t output_offset = 0; + for (size_t i = 0; i < output_buffers_num && output_buffers != nullptr; i++) { + const auto bytes = std::min( + output_buffers[i].size, + (result.output_written > output_offset ? result.output_written - output_offset : 0)); + if (output_buffers[i].address != nullptr && bytes != 0) { + std::memcpy(output_buffers[i].address, output.data() + output_offset, bytes); + } + output_offset += output_buffers[i].size; + } + + return result; +} + +static AjmDecodeResult AjmInitializeInstance(uint32_t instance, const void* codec_parameters, + size_t codec_parameters_size) { + std::scoped_lock lock(g_ajm_instances_mutex); + auto* state = AjmFindInstanceLocked(instance); + if (state == nullptr || state->decoder == nullptr) { + AjmDecodeResult result {}; + result.result = AJM_RESULT_INVALID_PARAMETER; + return result; + } + + state->gapless.Reset(); + return state->decoder->Initialize(codec_parameters, codec_parameters_size); +} + +static AjmDecodeResult AjmControlInstance(uint32_t instance, uint64_t flags, + const void* sideband_input, size_t sideband_input_size, + AjmDecoder** out_decoder, AjmGaplessState** out_gapless) { + std::scoped_lock lock(g_ajm_instances_mutex); + auto* state = AjmFindInstanceLocked(instance); + if (state == nullptr || state->decoder == nullptr) { + AjmDecodeResult result {}; + result.result = AJM_RESULT_INVALID_PARAMETER; + return result; + } + + auto* decoder = state->decoder.get(); + if (out_decoder != nullptr) { + *out_decoder = decoder; + } + if (out_gapless != nullptr) { + *out_gapless = &state->gapless; + } + + AjmDecodeResult result = decoder->MakeResult(); + if ((flags & AJM_FLAG_CONTROL_RESET) != 0) { + state->gapless.Reset(); + decoder->Reset(); + result = decoder->MakeResult(); + } + if ((flags & AJM_FLAG_CONTROL_INITIALIZE) != 0) { + state->gapless.Reset(); + result = decoder->Initialize(sideband_input, sideband_input_size); + } + + return result; +} + +static AjmDecodeResult AjmSetGaplessDecode(uint32_t instance, const void* gapless_decode, + int reset) { + std::scoped_lock lock(g_ajm_instances_mutex); + auto* state = AjmFindInstanceLocked(instance); + if (state == nullptr || state->decoder == nullptr || gapless_decode == nullptr) { + AjmDecodeResult result {}; + result.result = AJM_RESULT_INVALID_PARAMETER; + return result; + } + + state->gapless.Set(*static_cast(gapless_decode), reset != 0); + return state->decoder->MakeResult(); +} + +static AjmDecodeResult AjmGetGaplessDecode(uint32_t instance, + AjmSidebandGaplessDecode* gapless_decode) { + std::scoped_lock lock(g_ajm_instances_mutex); + auto* state = AjmFindInstanceLocked(instance); + if (state == nullptr || state->decoder == nullptr) { + AjmDecodeResult result {}; + result.result = AJM_RESULT_INVALID_PARAMETER; + return result; + } + + if (gapless_decode != nullptr) { + *gapless_decode = state->gapless.current; + } + return state->decoder->MakeResult(); +} + +static AjmSidebandFormat AjmGetInstanceFormat(uint32_t instance, + AjmDecodeResult* out_result = nullptr, + AjmDecoder** out_decoder = nullptr, + AjmGaplessState** out_gapless = nullptr) { + std::scoped_lock lock(g_ajm_instances_mutex); + auto* state = AjmFindInstanceLocked(instance); + if (state == nullptr || state->decoder == nullptr) { + if (out_result != nullptr) { + out_result->result = AJM_RESULT_INVALID_PARAMETER; + } + return AjmMakeFormat(2, 48000, AjmSampleEncoding::S16); + } + + if (out_decoder != nullptr) { + *out_decoder = state->decoder.get(); + } + if (out_gapless != nullptr) { + *out_gapless = &state->gapless; + } + if (out_result != nullptr) { + *out_result = state->decoder->MakeResult(); + out_result->format = state->decoder->GetFormat(); + } + return state->decoder->GetFormat(); +} + +int KYTY_SYSV_ABI AjmInitialize(int64_t reserved, uint32_t* context) { + PRINT_NAME(); + + EXIT_NOT_IMPLEMENTED(context == nullptr); + if (reserved != 0) { + LOGF("\t ignoring reserved = 0x%016" PRIx64 "\n", static_cast(reserved)); + } + + *context = 1; + + return OK; +} + +int KYTY_SYSV_ABI AjmFinalize(uint32_t context) { + PRINT_NAME(); + LOGF("\t context = %" PRIu32 "\n", context); + return OK; +} + +int KYTY_SYSV_ABI AjmModuleRegister(uint32_t context, uint32_t codec, int64_t reserved) { + PRINT_NAME(); + + if (reserved != 0) { + LOGF("\t ignoring reserved = 0x%016" PRIx64 "\n", static_cast(reserved)); + } + + LOGF("\t codec = %u\n", codec); + LOGF("\t %s\n", AjmCodecName(codec)); + + const auto valid = AjmCodecIsValid(codec); + const auto supported = valid && AjmLogCodecSupport(codec); + int result = OK; + if (context != 1) { + result = AJM_ERROR_INVALID_CONTEXT; + } else if (!valid) { + result = AJM_ERROR_INVALID_PARAMETER; + } else if (!supported) { + result = AJM_ERROR_CODEC_NOT_SUPPORTED; + } + + if (!supported) { + LOGF("\t codec is not implemented\n"); + } + + return result; +} + +int KYTY_SYSV_ABI AjmModuleUnregister(uint32_t context, uint32_t codec) { + PRINT_NAME(); + LOGF("\t context = %" PRIu32 ", codec = %" PRIu32 "\n", context, codec); + return OK; +} + +int KYTY_SYSV_ABI AjmInstanceCreate(uint32_t context, uint32_t codec, uint64_t flags, + uint32_t* instance) { + PRINT_NAME(); + + EXIT_NOT_IMPLEMENTED(instance == nullptr); + const auto supported = AjmLogCodecSupport(codec); + + const auto slot = (g_ajm_next_instance.fetch_add(1, std::memory_order_relaxed) & 0x3fffu); + *instance = (codec << 14u) | slot; + + auto state = AjmInstanceState {}; + state.context = context; + state.codec = codec; + state.flags = flags; + state.decoder = AjmCreateDecoder(codec, flags); + + { + std::scoped_lock lock(g_ajm_instances_mutex); + g_ajm_instances[*instance] = std::move(state); + } + + LOGF("\t context = %" PRIu32 "\n" + "\t codec = %" PRIu32 "\n" + "\t flags = 0x%016" PRIx64 "\n" + "\t instance = 0x%08" PRIx32 "\n", + context, codec, flags, *instance); + if (!supported) { + LOGF("\t unsupported codec instance created without decoder\n"); + } + + return OK; +} + +int KYTY_SYSV_ABI AjmInstanceDestroy(uint32_t context, uint32_t instance) { + PRINT_NAME(); + LOGF("\t context = %" PRIu32 ", instance = 0x%08" PRIx32 "\n", context, instance); + + std::scoped_lock lock(g_ajm_instances_mutex); + g_ajm_instances.erase(instance); + + return OK; +} + +int KYTY_SYSV_ABI AjmMemoryRegister(uint32_t context, void* ptr, size_t pages) { + PRINT_NAME(); + LOGF("\t context = %" PRIu32 ", ptr = 0x%016" PRIx64 ", pages = 0x%016" PRIx64 "\n", context, + reinterpret_cast(ptr), static_cast(pages)); + return OK; +} + +int KYTY_SYSV_ABI AjmMemoryUnregister(uint32_t context, void* ptr) { + PRINT_NAME(); + LOGF("\t context = %" PRIu32 ", ptr = 0x%016" PRIx64 "\n", context, + reinterpret_cast(ptr)); + return OK; +} + +int KYTY_SYSV_ABI AjmBatchInitialize(void* buffer, size_t size, AjmBatchInfo* info) { + PRINT_NAME(); + + EXIT_NOT_IMPLEMENTED(buffer == nullptr); + EXIT_NOT_IMPLEMENTED(info == nullptr); + + info->buffer = buffer; + info->offset = 0; + info->size = size; + info->last_good_job = nullptr; + info->last_good_job_ra = nullptr; + + return OK; +} + +int KYTY_SYSV_ABI AjmBatchStart(uint32_t context, const AjmBatchInfo* info, int priority, + AjmBatchError* error, uint32_t* batch) { + PRINT_NAME(); + + EXIT_NOT_IMPLEMENTED(info == nullptr); + EXIT_NOT_IMPLEMENTED(batch == nullptr); + + (void)context; + (void)priority; + + if (error != nullptr) { + std::memset(error, 0, sizeof(AjmBatchError)); + } + + *batch = g_ajm_next_batch.fetch_add(1, std::memory_order_relaxed); + + return OK; +} + +int KYTY_SYSV_ABI AjmBatchWait(uint32_t context, uint32_t batch, uint32_t timeout, + AjmBatchError* error) { + (void)context; + (void)batch; + (void)timeout; + + if (error != nullptr) { + std::memset(error, 0, sizeof(AjmBatchError)); + } + + return OK; +} + +int KYTY_SYSV_ABI AjmBatchCancel(uint32_t context, uint32_t batch) { + PRINT_NAME(); + LOGF("\t context = %" PRIu32 ", batch = %" PRIu32 "\n", context, batch); + return OK; +} + +int KYTY_SYSV_ABI AjmBatchErrorDump(const AjmBatchInfo* info, AjmBatchError* error) { + PRINT_NAME(); + if (error != nullptr) { + std::memset(error, 0, sizeof(AjmBatchError)); + } + LOGF("\t info = 0x%016" PRIx64 "\n", reinterpret_cast(info)); + return OK; +} + +int KYTY_SYSV_ABI AjmBatchJobInitialize(AjmBatchInfo* info, uint32_t instance, + const void* codec_parameters, size_t codec_parameters_size, + void* result) { + PRINT_NAME(); + EXIT_NOT_IMPLEMENTED(info == nullptr); + + const auto decode_result = + AjmInitializeInstance(instance, codec_parameters, codec_parameters_size); + AjmWriteResult(result, AJM_SIDEBAND_RESULT_SIZE, decode_result); + + LOGF("\t instance = 0x%08" PRIx32 ", codec_parameters = 0x%016" PRIx64 ", size = 0x%016" PRIx64 + "\n", + instance, reinterpret_cast(codec_parameters), + static_cast(codec_parameters_size)); + return AjmAppendJob(info, AJM_JOB_CONTROL_SIZE, "initialize"); +} + +int KYTY_SYSV_ABI AjmBatchJobClearContext(AjmBatchInfo* info, uint32_t instance, void* result) { + PRINT_NAME(); + EXIT_NOT_IMPLEMENTED(info == nullptr); + + AjmDecodeResult decode_result {}; + { + std::scoped_lock lock(g_ajm_instances_mutex); + auto* state = AjmFindInstanceLocked(instance); + if (state != nullptr && state->decoder != nullptr) { + state->gapless.Reset(); + state->decoder->Reset(); + decode_result = state->decoder->MakeResult(); + } else { + decode_result.result = AJM_RESULT_INVALID_PARAMETER; + } + } + + AjmWriteResult(result, AJM_SIDEBAND_RESULT_SIZE, decode_result); + LOGF("\t instance = 0x%08" PRIx32 "\n", instance); + return AjmAppendJob(info, AJM_JOB_CONTROL_SIZE, "clear-context"); +} + +int KYTY_SYSV_ABI AjmBatchJobDecode(AjmBatchInfo* info, uint32_t instance, + const void* bitstream_input, size_t bitstream_input_size, + void* pcm_output, size_t pcm_output_size, void* result) { + PRINT_NAME(); + EXIT_NOT_IMPLEMENTED(info == nullptr); + + const auto decode_result = AjmDecodeInstance(instance, bitstream_input, bitstream_input_size, + pcm_output, pcm_output_size, true); + AjmWriteStreamResult( + result, AJM_SIDEBAND_RESULT_SIZE + AJM_SIDEBAND_STREAM_SIZE + AJM_SIDEBAND_MFRAME_SIZE, + decode_result, true); + + LOGF("\t instance = 0x%08" PRIx32 ", input = 0x%016" PRIx64 ", input_size = 0x%016" PRIx64 + ", output = 0x%016" PRIx64 ", output_size = 0x%016" PRIx64 "\n", + instance, reinterpret_cast(bitstream_input), + static_cast(bitstream_input_size), reinterpret_cast(pcm_output), + static_cast(pcm_output_size)); + return AjmAppendJob(info, AJM_JOB_RUN_SIZE, "decode"); +} + +int KYTY_SYSV_ABI AjmBatchJobDecodeSingle(AjmBatchInfo* info, uint32_t instance, + const void* bitstream_input, size_t bitstream_input_size, + void* pcm_output, size_t pcm_output_size, void* result) { + PRINT_NAME(); + EXIT_NOT_IMPLEMENTED(info == nullptr); + + const auto decode_result = AjmDecodeInstance(instance, bitstream_input, bitstream_input_size, + pcm_output, pcm_output_size, false); + AjmWriteStreamResult(result, AJM_SIDEBAND_RESULT_SIZE + AJM_SIDEBAND_STREAM_SIZE, decode_result, + false); + + LOGF("\t instance = 0x%08" PRIx32 "\n", instance); + return AjmAppendJob(info, AJM_JOB_RUN_SIZE, "decode-single"); +} + +int KYTY_SYSV_ABI AjmBatchJobDecodeSplit(AjmBatchInfo* info, uint32_t instance, + const AjmBuffer* input_buffers, size_t input_buffers_num, + const AjmBuffer* output_buffers, size_t output_buffers_num, + void* result) { + PRINT_NAME(); + EXIT_NOT_IMPLEMENTED(info == nullptr); + + const auto decode_result = AjmDecodeSplitInstance(instance, input_buffers, input_buffers_num, + output_buffers, output_buffers_num, true); + AjmWriteStreamResult( + result, AJM_SIDEBAND_RESULT_SIZE + AJM_SIDEBAND_STREAM_SIZE + AJM_SIDEBAND_MFRAME_SIZE, + decode_result, true); + + LOGF("\t instance = 0x%08" PRIx32 ", input_buffers = 0x%016" PRIx64 + ", input_num = 0x%016" PRIx64 ", output_buffers = 0x%016" PRIx64 + ", output_num = 0x%016" PRIx64 "\n", + instance, reinterpret_cast(input_buffers), + static_cast(input_buffers_num), reinterpret_cast(output_buffers), + static_cast(output_buffers_num)); + return AjmAppendJob(info, 16 * (input_buffers_num + output_buffers_num) + 32, "decode-split"); +} + +int KYTY_SYSV_ABI AjmBatchJobEncode(AjmBatchInfo* info, uint32_t instance, const void* pcm_input, + size_t pcm_input_size, void* bitstream_output, + size_t bitstream_output_size, void* result) { + PRINT_NAME(); + EXIT_NOT_IMPLEMENTED(info == nullptr); + + AjmDecodeResult encode_result {}; + encode_result.input_consumed = pcm_input_size; + encode_result.output_written = bitstream_output_size; + encode_result.frames = (pcm_input_size != 0 || bitstream_output_size != 0 ? 1u : 0u); + encode_result.total_decoded_samples = bitstream_output_size; + if (bitstream_output != nullptr && bitstream_output_size != 0) { + std::memset(bitstream_output, 0, bitstream_output_size); + } + + AjmWriteStreamResult( + result, AJM_SIDEBAND_RESULT_SIZE + AJM_SIDEBAND_STREAM_SIZE + AJM_SIDEBAND_MFRAME_SIZE, + encode_result, true); + LOGF("\t instance = 0x%08" PRIx32 ", input = 0x%016" PRIx64 ", output = 0x%016" PRIx64 "\n", + instance, reinterpret_cast(pcm_input), + reinterpret_cast(bitstream_output)); + return AjmAppendJob(info, AJM_JOB_RUN_SIZE, "encode"); +} + +int KYTY_SYSV_ABI AjmBatchJobGetInfo(AjmBatchInfo* info, uint32_t instance, void* result) { + PRINT_NAME(); + EXIT_NOT_IMPLEMENTED(info == nullptr); + + AjmDecodeResult decode_result {}; + const auto format = AjmGetInstanceFormat(instance, &decode_result); + AjmWriteFormat(result, AJM_SIDEBAND_RESULT_SIZE + AJM_SIDEBAND_FORMAT_SIZE, format, + decode_result); + + LOGF("\t instance = 0x%08" PRIx32 "\n", instance); + return AjmAppendJob(info, AJM_JOB_RUN_SIZE, "get-info"); +} + +int KYTY_SYSV_ABI AjmBatchJobGetCodecInfo(AjmBatchInfo* info, uint32_t instance, void* result, + size_t result_size) { + PRINT_NAME(); + EXIT_NOT_IMPLEMENTED(info == nullptr); + + AjmDecodeResult decode_result {}; + AjmDecoder* decoder = nullptr; + (void)AjmGetInstanceFormat(instance, &decode_result, &decoder); + AjmWriteResult(result, result_size, decode_result); + AjmWriteCodecInfo(decoder, result, result_size, decode_result); + + LOGF("\t instance = 0x%08" PRIx32 ", result_size = 0x%016" PRIx64 "\n", instance, + static_cast(result_size)); + return AjmAppendJob(info, AJM_JOB_RUN_SIZE, "get-codec-info"); +} + +int KYTY_SYSV_ABI AjmBatchJobSetGaplessDecode(AjmBatchInfo* info, uint32_t instance, + const void* gapless_decode, int reset, void* result) { + PRINT_NAME(); + EXIT_NOT_IMPLEMENTED(info == nullptr); + const auto decode_result = AjmSetGaplessDecode(instance, gapless_decode, reset); + AjmWriteResult(result, AJM_SIDEBAND_RESULT_SIZE, decode_result); + LOGF("\t instance = 0x%08" PRIx32 ", gapless_decode = 0x%016" PRIx64 ", reset = %d\n", instance, + reinterpret_cast(gapless_decode), reset); + return AjmAppendJob(info, AJM_JOB_CONTROL_SIZE, "set-gapless"); +} + +int KYTY_SYSV_ABI AjmBatchJobGetGaplessDecode(AjmBatchInfo* info, uint32_t instance, void* result) { + PRINT_NAME(); + EXIT_NOT_IMPLEMENTED(info == nullptr); + + AjmDecodeResult decode_result {}; + AjmSidebandGaplessDecode gapless_decode {}; + decode_result = AjmGetGaplessDecode(instance, &gapless_decode); + AjmWriteResult(result, AJM_SIDEBAND_RESULT_SIZE + AJM_SIDEBAND_GAPLESS_SIZE, decode_result); + if (result != nullptr) { + auto* gapless = reinterpret_cast(static_cast(result) + + AJM_SIDEBAND_RESULT_SIZE); + *gapless = gapless_decode; + } + + LOGF("\t instance = 0x%08" PRIx32 "\n", instance); + return AjmAppendJob(info, AJM_JOB_RUN_SIZE, "get-gapless"); +} + +int KYTY_SYSV_ABI AjmBatchJobSetResampleParameters(AjmBatchInfo* info, uint32_t instance, + float ratio, uint32_t flags, void* result) { + PRINT_NAME(); + EXIT_NOT_IMPLEMENTED(info == nullptr); + AjmWriteResult(result, AJM_SIDEBAND_RESULT_SIZE); + LOGF("\t instance = 0x%08" PRIx32 ", ratio = %f, flags = 0x%08" PRIx32 "\n", instance, + static_cast(ratio), flags); + return AjmAppendJob(info, AJM_JOB_SET_RESAMPLE_PARAMETERS_SIZE, "set-resample"); +} + +int KYTY_SYSV_ABI AjmBatchJobSetResampleParametersEx(AjmBatchInfo* info, uint32_t instance, + float ratio_start, + float ratio_change_per_sample, uint32_t flags, + void* result) { + PRINT_NAME(); + EXIT_NOT_IMPLEMENTED(info == nullptr); + AjmWriteResult(result, AJM_SIDEBAND_RESULT_SIZE); + LOGF("\t instance = 0x%08" PRIx32 ", ratio_start = %f, ratio_change = %f, flags = 0x%08" PRIx32 + "\n", + instance, static_cast(ratio_start), static_cast(ratio_change_per_sample), + flags); + return AjmAppendJob(info, AJM_JOB_SET_RESAMPLE_PARAMETERS_SIZE, "set-resample-ex"); +} + +int KYTY_SYSV_ABI AjmBatchJobGetResampleInfo(AjmBatchInfo* info, uint32_t instance, void* result) { + PRINT_NAME(); + EXIT_NOT_IMPLEMENTED(info == nullptr); + + AjmDecodeResult decode_result {}; + (void)AjmGetInstanceFormat(instance, &decode_result); + AjmWriteResult(result, AJM_SIDEBAND_RESULT_SIZE + AJM_SIDEBAND_RESAMPLE_INFO_SIZE, + decode_result); + if (result != nullptr) { + auto* resample = reinterpret_cast(static_cast(result) + + AJM_SIDEBAND_RESULT_SIZE); + resample->ratio = 1.0f; + } + LOGF("\t instance = 0x%08" PRIx32 "\n", instance); + return AjmAppendJob(info, AJM_JOB_RUN_SIZE, "get-resample-info"); +} + +int KYTY_SYSV_ABI AjmBatchJobGetStatistics(AjmBatchInfo* info, float interval, void* result) { + PRINT_NAME(); + EXIT_NOT_IMPLEMENTED(info == nullptr); + AjmWriteResult(result, sizeof(AjmGetStatisticsResult)); + (void)interval; + return AjmAppendJob(info, AJM_JOB_GET_STATISTICS_SIZE, "get-statistics"); +} + +int KYTY_SYSV_ABI AjmBatchJobControl(AjmBatchInfo* info, uint32_t instance, uint64_t flags, + const void* sideband_input, size_t sideband_input_size, + void* sideband_output, size_t sideband_output_size) { + PRINT_NAME(); + EXIT_NOT_IMPLEMENTED(info == nullptr); + + AjmDecoder* decoder = nullptr; + AjmGaplessState* gapless = nullptr; + const auto decode_result = AjmControlInstance(instance, flags, sideband_input, + sideband_input_size, &decoder, &gapless); + AjmWriteSideband(flags, sideband_output, sideband_output_size, decoder, gapless, decode_result); + + LOGF("\t instance = 0x%08" PRIx32 ", flags = 0x%016" PRIx64 ", input = 0x%016" PRIx64 + ", input_size = 0x%016" PRIx64 ", output = 0x%016" PRIx64 ", output_size = 0x%016" PRIx64 + "\n", + instance, flags, reinterpret_cast(sideband_input), + static_cast(sideband_input_size), reinterpret_cast(sideband_output), + static_cast(sideband_output_size)); + return AjmAppendJob(info, AJM_JOB_CONTROL_SIZE, "control"); +} + +int KYTY_SYSV_ABI AjmBatchJobRun(AjmBatchInfo* info, uint32_t instance, uint64_t flags, + const void* data_input, size_t data_input_size, void* data_output, + size_t data_output_size, void* sideband_output, + size_t sideband_output_size) { + PRINT_NAME(); + EXIT_NOT_IMPLEMENTED(info == nullptr); + + auto decode_result = + AjmDecodeInstance(instance, data_input, data_input_size, data_output, data_output_size, + (flags & AJM_FLAG_RUN_MULTIPLE_FRAMES) != 0); + AjmDecoder* decoder = nullptr; + AjmGaplessState* gapless = nullptr; + (void)AjmGetInstanceFormat(instance, nullptr, &decoder, &gapless); + AjmWriteSideband(flags, sideband_output, sideband_output_size, decoder, gapless, decode_result); + + LOGF("\t instance = 0x%08" PRIx32 ", flags = 0x%016" PRIx64 ", input = 0x%016" PRIx64 + ", input_size = 0x%016" PRIx64 ", output = 0x%016" PRIx64 ", output_size = 0x%016" PRIx64 + "\n", + instance, flags, reinterpret_cast(data_input), + static_cast(data_input_size), reinterpret_cast(data_output), + static_cast(data_output_size)); + return AjmAppendJob(info, AJM_JOB_RUN_SIZE, "run"); +} + +int KYTY_SYSV_ABI AjmBatchJobRunSplit(AjmBatchInfo* info, uint32_t instance, uint64_t flags, + const AjmBuffer* input_buffers, size_t input_buffers_num, + const AjmBuffer* output_buffers, size_t output_buffers_num, + void* sideband_output, size_t sideband_output_size) { + PRINT_NAME(); + EXIT_NOT_IMPLEMENTED(info == nullptr); + + auto decode_result = + AjmDecodeSplitInstance(instance, input_buffers, input_buffers_num, output_buffers, + output_buffers_num, (flags & AJM_FLAG_RUN_MULTIPLE_FRAMES) != 0); + AjmDecoder* decoder = nullptr; + AjmGaplessState* gapless = nullptr; + (void)AjmGetInstanceFormat(instance, nullptr, &decoder, &gapless); + AjmWriteSideband(flags, sideband_output, sideband_output_size, decoder, gapless, decode_result); + + LOGF("\t instance = 0x%08" PRIx32 ", flags = 0x%016" PRIx64 ", input_buffers = 0x%016" PRIx64 + ", input_num = 0x%016" PRIx64 ", output_buffers = 0x%016" PRIx64 + ", output_num = 0x%016" PRIx64 "\n", + instance, flags, reinterpret_cast(input_buffers), + static_cast(input_buffers_num), reinterpret_cast(output_buffers), + static_cast(output_buffers_num)); + return AjmAppendJob(info, 16 * (input_buffers_num + output_buffers_num) + 32, "run-split"); +} + +const char* KYTY_SYSV_ABI AjmStrError(int error) { + PRINT_NAME(); + LOGF("\t error = %d\n", error); + return "AJM"; +} + +} // namespace Libs::Audio::Ajm diff --git a/src/libs/ajm/aac_decoder.h b/src/libs/ajm/aac_decoder.h new file mode 100644 index 000000000..56c2ab552 --- /dev/null +++ b/src/libs/ajm/aac_decoder.h @@ -0,0 +1,363 @@ +#pragma once + +#include "common/logging/log.h" +#include "libs/ajm/ffmpeg_decoder_common.h" + +#include +#include +#include +#include +#include +#include + +namespace Libs::Audio::Ajm { + +struct AjmDecM4aacInitializeParameters { + uint32_t config_number; + uint32_t sampling_freq_index; +}; + +struct AjmSidebandDecM4aacCodecInfo { + uint32_t heaac; + uint32_t reserved; +}; + +static_assert(sizeof(AjmDecM4aacInitializeParameters) == 8); +static_assert(sizeof(AjmSidebandDecM4aacCodecInfo) == 8); + +static constexpr uint64_t AJM_INSTANCE_FLAG_DEC_M4AAC_ENABLE_SBR_DECODE = + 1ull << (AJM_INSTANCE_FLAG_CODEC_OFFSET + 0u); +static constexpr uint64_t AJM_INSTANCE_FLAG_DEC_M4AAC_ENABLE_NONDELAY_OUTPUT = + 1ull << (AJM_INSTANCE_FLAG_CODEC_OFFSET + 1u); +static constexpr uint32_t AJM_DEC_M4AAC_CONFIG_NUMBER_ADTS = 1; +static constexpr uint32_t AJM_DEC_M4AAC_CONFIG_NUMBER_RAW = 2; +static constexpr uint32_t AJM_DEC_M4AAC_CONFIG_NUMBER_SAF = 3; +static constexpr uint32_t AJM_DEC_M4AAC_MAX_CHANNELS = 8; +static constexpr uint32_t AJM_DEC_M4AAC_MAX_SAMPLING_FREQ_INDEX = 11; + +class AjmAacDecoder final: public AjmDecoder { +public: + AjmAacDecoder(uint32_t channels, uint32_t sample_rate, AjmSampleEncoding encoding, + uint64_t flags) + : AjmDecoder(channels, sample_rate, encoding), m_flags(flags) { + m_codec = avcodec_find_decoder(AV_CODEC_ID_AAC); + if (m_codec == nullptr) { + LOGF("AJM AAC: avcodec_find_decoder failed\n"); + return; + } + } + + ~AjmAacDecoder() override { Release(); } + + AjmDecodeResult Initialize(const void* codec_parameters, + size_t codec_parameters_size) override { + auto result = MakeResult(); + + if (codec_parameters == nullptr || + codec_parameters_size < sizeof(AjmDecM4aacInitializeParameters)) { + result.result = AJM_RESULT_INVALID_PARAMETER; + return result; + } + + const auto* params = static_cast(codec_parameters); + if (params->config_number != AJM_DEC_M4AAC_CONFIG_NUMBER_ADTS && + params->config_number != AJM_DEC_M4AAC_CONFIG_NUMBER_RAW && + params->config_number != AJM_DEC_M4AAC_CONFIG_NUMBER_SAF) { + result.result = AJM_RESULT_INVALID_PARAMETER; + return result; + } + if (params->config_number == AJM_DEC_M4AAC_CONFIG_NUMBER_RAW && + params->sampling_freq_index > AJM_DEC_M4AAC_MAX_SAMPLING_FREQ_INDEX) { + result.result = AJM_RESULT_INVALID_PARAMETER; + return result; + } + + m_config_number = params->config_number; + m_sampling_freq_index = params->sampling_freq_index; + m_is_initialized = true; + Reset(); + if (m_codec_context == nullptr) { + m_is_initialized = false; + result.result = AJM_RESULT_CODEC_ERROR | AJM_RESULT_FATAL; + return result; + } + + LOGF("AJM AAC initialized: config=%" PRIu32 ", freq_index=%" PRIu32 "\n", m_config_number, + m_sampling_freq_index); + return result; + } + + void Reset() override { + Release(); + m_skip_frames = + ((m_flags & AJM_INSTANCE_FLAG_DEC_M4AAC_ENABLE_NONDELAY_OUTPUT) != 0 ? 0u : 2u); + m_total_decoded_samples = 0; + m_bitrate = 0; + if (m_is_initialized) { + OpenDecoder(); + } + } + + AjmDecodeResult Decode(const void* input, size_t input_size, void* output, size_t output_size, + bool multiple_frames, AjmGaplessState* gapless) override { + (void)multiple_frames; + auto result = MakeResult(); + + if (!m_is_initialized) { + result.result = AJM_RESULT_NOT_INITIALIZED; + return result; + } + if (m_codec_context == nullptr) { + result.result = AJM_RESULT_CODEC_ERROR | AJM_RESULT_FATAL; + return result; + } + if (input == nullptr || input_size == 0) { + result.result = AJM_RESULT_PARTIAL_INPUT; + return result; + } + if (input_size > static_cast(std::numeric_limits::max())) { + result.result = AJM_RESULT_INVALID_PARAMETER; + return result; + } + + size_t output_offset = 0; + if (!DecodePacket(static_cast(input), static_cast(input_size), output, + output_size, &output_offset, gapless, &result)) { + result.input_consumed = input_size; + result.output_written = output_offset; + result.total_decoded_samples = m_total_decoded_samples; + result.format = GetFormat(); + return result; + } + + if (result.frames == 0 && result.result == OK) { + result.result = AJM_RESULT_PARTIAL_INPUT; + } + + result.input_consumed = input_size; + result.output_written = output_offset; + result.total_decoded_samples = m_total_decoded_samples; + result.format = GetFormat(); + return result; + } + + void WriteCodecInfo(void* output, size_t output_size, + const AjmDecodeResult& result) const override { + (void)result; + if (output == nullptr || output_size < sizeof(AjmSidebandDecM4aacCodecInfo)) { + return; + } + + auto* info = static_cast(output); + info->heaac = ((m_flags & AJM_INSTANCE_FLAG_DEC_M4AAC_ENABLE_SBR_DECODE) != 0 ? 1u : 0u); + info->reserved = 0; + } + + [[nodiscard]] size_t CodecInfoSize() const override { + return sizeof(AjmSidebandDecM4aacCodecInfo); + } + + [[nodiscard]] AjmSidebandFormat GetFormat() const override { + auto format = AjmMakeFormat(m_channels, m_sample_rate, m_sample_encoding); + format.bitrate = m_bitrate; + return format; + } + +private: + static uint32_t SamplingRateFromIndex(uint32_t index) { + static constexpr uint32_t SAMPLE_RATES[] = { + 96000, 88200, 64000, 48000, 44100, 32000, 24000, 22050, 16000, 12000, 11025, 8000, + }; + return (index < (sizeof(SAMPLE_RATES) / sizeof(SAMPLE_RATES[0])) ? SAMPLE_RATES[index] + : 48000u); + } + + void Release() { + if (m_codec_context != nullptr) { + avcodec_free_context(&m_codec_context); + } + } + + void OpenDecoder() { + if (m_codec == nullptr) { + return; + } + + m_codec_context = avcodec_alloc_context3(m_codec); + if (m_codec_context == nullptr) { + return; + } + + if (m_config_number == AJM_DEC_M4AAC_CONFIG_NUMBER_RAW) { + const auto sample_index = + std::min(m_sampling_freq_index, AJM_DEC_M4AAC_MAX_SAMPLING_FREQ_INDEX); + const auto channels = std::clamp(m_channels, 1u, AJM_DEC_M4AAC_MAX_CHANNELS); + constexpr uint32_t audio_object_type = 2; + + auto* extradata = static_cast(av_mallocz(2 + AV_INPUT_BUFFER_PADDING_SIZE)); + if (extradata == nullptr) { + Release(); + return; + } + + extradata[0] = static_cast((audio_object_type << 3u) | (sample_index >> 1u)); + extradata[1] = static_cast(((sample_index & 0x1u) << 7u) | + (std::min(channels, 7u) << 3u)); + m_codec_context->extradata = extradata; + m_codec_context->extradata_size = 2; + SetFormat(channels, SamplingRateFromIndex(sample_index), m_sample_encoding); + } + + if (avcodec_open2(m_codec_context, m_codec, nullptr) < 0) { + LOGF("AJM AAC: decoder initialization failed\n"); + Release(); + } + } + + bool WriteFrame(const AVFrame* frame, void* output, size_t output_size, size_t* output_offset, + AjmGaplessState* gapless, AjmDecodeResult* result) { + if (frame == nullptr || output_offset == nullptr || result == nullptr) { + return false; + } + + const auto channels = static_cast(std::max(frame->ch_layout.nb_channels, 1)); + const auto bpf = channels * AjmBytesPerSample(m_sample_encoding); + if (bpf == 0) { + result->result = AJM_RESULT_INVALID_PARAMETER; + return false; + } + + SetFormat(channels, static_cast(std::max(frame->sample_rate, 1)), + m_sample_encoding); + if (m_codec_context != nullptr && m_codec_context->bit_rate > 0) { + m_bitrate = static_cast(m_codec_context->bit_rate); + } + + result->frames++; + if (m_skip_frames > 0) { + m_skip_frames--; + result->format = GetFormat(); + return true; + } + + uint32_t skip_samples = 0; + if (gapless != nullptr && gapless->current.skip_samples > 0) { + skip_samples = std::min(frame->nb_samples, gapless->current.skip_samples); + gapless->current.skip_samples = + static_cast(gapless->current.skip_samples - skip_samples); + } + + uint32_t samples_to_write = static_cast(frame->nb_samples) - skip_samples; + if (gapless != nullptr && gapless->HasSampleLimit()) { + samples_to_write = std::min(samples_to_write, gapless->current.total_samples); + } + + const auto requested_bytes = static_cast(samples_to_write) * bpf; + auto writable_bytes = requested_bytes; + if (output == nullptr || *output_offset >= output_size) { + writable_bytes = 0; + } else { + writable_bytes = std::min(writable_bytes, output_size - *output_offset); + writable_bytes -= writable_bytes % bpf; + } + + if (writable_bytes != 0) { + auto* dst = static_cast(output) + *output_offset; + const auto* src = frame->data[0] + static_cast(skip_samples) * bpf; + std::memcpy(dst, src, writable_bytes); + *output_offset += writable_bytes; + } + + const auto samples_written = static_cast(writable_bytes / bpf); + if (gapless != nullptr && gapless->HasSampleLimit()) { + gapless->current.total_samples -= + std::min(gapless->current.total_samples, samples_written); + } + if (gapless != nullptr) { + const auto skipped_total = + std::min(std::numeric_limits::max(), + static_cast(gapless->current.skipped_samples) + + skip_samples + (samples_to_write - samples_written)); + gapless->current.skipped_samples = static_cast(skipped_total); + } + + m_total_decoded_samples += samples_written; + result->total_decoded_samples = m_total_decoded_samples; + result->format = GetFormat(); + + if (writable_bytes != requested_bytes) { + result->result |= AJM_RESULT_NOT_ENOUGH_ROOM; + return false; + } + return true; + } + + bool DecodePacket(const uint8_t* packet_data, int packet_size, void* output, size_t output_size, + size_t* output_offset, AjmGaplessState* gapless, AjmDecodeResult* result) { + if (packet_data == nullptr || packet_size <= 0 || result == nullptr) { + return true; + } + + AVPacket* packet = av_packet_alloc(); + if (packet == nullptr || av_new_packet(packet, packet_size) < 0) { + av_packet_free(&packet); + result->result = AJM_RESULT_CODEC_ERROR | AJM_RESULT_FATAL; + return false; + } + std::memcpy(packet->data, packet_data, static_cast(packet_size)); + + int rc = avcodec_send_packet(m_codec_context, packet); + av_packet_free(&packet); + if (rc == AVERROR_INVALIDDATA) { + result->result = AJM_RESULT_CODEC_ERROR | AJM_RESULT_INVALID_DATA; + return false; + } + if (rc < 0) { + result->result = AJM_RESULT_PARTIAL_INPUT; + return false; + } + + for (;;) { + AVFrame* frame = av_frame_alloc(); + if (frame == nullptr) { + result->result = AJM_RESULT_CODEC_ERROR | AJM_RESULT_FATAL; + return false; + } + + rc = avcodec_receive_frame(m_codec_context, frame); + if (rc == AVERROR(EAGAIN) || rc == AVERROR_EOF) { + av_frame_free(&frame); + return true; + } + if (rc < 0) { + av_frame_free(&frame); + result->result = AJM_RESULT_CODEC_ERROR | AJM_RESULT_INVALID_DATA; + return false; + } + + AVFrame* converted = AjmConvertFfmpegFrame(frame, m_sample_encoding, result); + av_frame_free(&frame); + if (converted == nullptr) { + return false; + } + + const bool ok = + WriteFrame(converted, output, output_size, output_offset, gapless, result); + av_frame_free(&converted); + if (!ok) { + return false; + } + } + } + + const AVCodec* m_codec = nullptr; + AVCodecContext* m_codec_context = nullptr; + uint64_t m_flags = 0; + uint32_t m_config_number = AJM_DEC_M4AAC_CONFIG_NUMBER_ADTS; + uint32_t m_sampling_freq_index = 3; + uint32_t m_skip_frames = 2; + uint32_t m_bitrate = 0; + bool m_is_initialized = false; +}; + +} // namespace Libs::Audio::Ajm diff --git a/src/libs/ajm/atrac9_decoder.h b/src/libs/ajm/atrac9_decoder.h new file mode 100644 index 000000000..cb7587c56 --- /dev/null +++ b/src/libs/ajm/atrac9_decoder.h @@ -0,0 +1,530 @@ +#pragma once + +#include "common/assert.h" +#include "common/logging/log.h" + +#include +#include +#include +#include +#include +#include +#include + +extern "C" { +#include "libatrac9.h" +} + +namespace Libs::Audio::Ajm { + +struct AjmDecAt9InitializeParameters { + uint8_t config_data[ATRAC9_CONFIG_DATA_SIZE]; + uint32_t reserved; +}; + +struct AjmSidebandDecAt9CodecInfo { + uint32_t super_frame_size; + uint32_t frames_in_super_frame; + uint32_t next_frame_size; + uint32_t frame_samples; +}; + +static constexpr uint64_t AJM_INSTANCE_FLAG_DEC_AT9_PARSE_RIFF_HEADER = + 1ull << (AJM_INSTANCE_FLAG_CODEC_OFFSET + 0u); + +static uint16_t AjmReadLe16(const uint8_t* data) { + return static_cast(data[0]) | (static_cast(data[1]) << 8u); +} + +static uint32_t AjmReadLe32(const uint8_t* data) { + return static_cast(data[0]) | (static_cast(data[1]) << 8u) | + (static_cast(data[2]) << 16u) | (static_cast(data[3]) << 24u); +} + +static bool AjmFourCcEquals(const uint8_t* data, char a, char b, char c, char d) { + return data[0] == static_cast(a) && data[1] == static_cast(b) && + data[2] == static_cast(c) && data[3] == static_cast(d); +} + +class AjmAt9Decoder final: public AjmDecoder { +public: + AjmAt9Decoder(uint32_t channels, uint32_t sample_rate, AjmSampleEncoding encoding, + uint64_t flags) + : AjmDecoder(channels, sample_rate, encoding), + m_parse_riff_header((flags & AJM_INSTANCE_FLAG_DEC_AT9_PARSE_RIFF_HEADER) != 0), + m_handle(Atrac9GetHandle()) {} + + ~AjmAt9Decoder() override { + if (m_handle != nullptr) { + Atrac9ReleaseHandle(m_handle); + } + } + + AjmDecodeResult Initialize(const void* codec_parameters, + size_t codec_parameters_size) override { + auto result = MakeResult(); + + if (codec_parameters == nullptr || codec_parameters_size < ATRAC9_CONFIG_DATA_SIZE) { + result.result = AJM_RESULT_INVALID_PARAMETER; + return result; + } + + const auto* params = static_cast(codec_parameters); + if (!InitializeConfig(params->config_data, &result)) { + return result; + } + + return MakeResult(); + } + + void Reset() override { + if (m_handle != nullptr) { + Atrac9ReleaseHandle(m_handle); + } + m_handle = Atrac9GetHandle(); + m_num_frames = 0; + m_total_decoded_samples = 0; + m_superframe_bytes_remain = + (m_is_initialized ? static_cast(m_codec_info.superframeSize) : 0); + + if (m_has_config && m_handle != nullptr) { + auto result = MakeResult(); + (void)InitializeConfig(m_config_data, &result); + } + } + + AjmDecodeResult Decode(const void* input, size_t input_size, void* output, size_t output_size, + bool multiple_frames, AjmGaplessState* gapless) override { + auto result = MakeResult(); + + if (input == nullptr || input_size == 0) { + result.result = AJM_RESULT_PARTIAL_INPUT; + return result; + } + + const auto* input_bytes = static_cast(input); + auto* output_bytes = static_cast(output); + size_t input_offset = 0; + + if (m_parse_riff_header && input_size >= 12 && + AjmFourCcEquals(input_bytes, 'R', 'I', 'F', 'F')) { + if (!ParseRiffHeader(input_bytes, input_size, &input_offset, gapless, &result)) { + return result; + } + if (input_offset >= input_size) { + result.input_consumed = input_offset; + result.result = AJM_RESULT_PARTIAL_INPUT; + return result; + } + } + + if (!m_is_initialized) { + result.result = AJM_RESULT_NOT_INITIALIZED; + return result; + } + + for (;;) { + if (input_offset >= input_size) { + break; + } + if (gapless != nullptr && gapless->IsEnd()) { + break; + } + + const auto remaining_input = input_size - input_offset; + if (remaining_input < GetMinimumInputSize()) { + if (result.frames == 0) { + result.result = AJM_RESULT_PARTIAL_INPUT; + } + break; + } + + const auto frame_output_bytes = FrameOutputBytes(gapless); + if (frame_output_bytes != 0 && output == nullptr) { + result.result |= AJM_RESULT_NOT_ENOUGH_ROOM; + break; + } + if (frame_output_bytes == 0 && !FrameCanProduceNoOutput(gapless)) { + result.result |= AJM_RESULT_NOT_ENOUGH_ROOM; + break; + } + if (output_size - result.output_written < frame_output_bytes) { + result.result |= AJM_RESULT_NOT_ENOUGH_ROOM; + break; + } + + int bytes_used = 0; + const int ret = DecodeFrame(input_bytes + input_offset, &bytes_used); + if (ret != 0) { + result.result = AJM_RESULT_CODEC_ERROR | AJM_RESULT_FATAL; + result.internal_result = ret; + break; + } + if (bytes_used <= 0 || static_cast(bytes_used) > remaining_input || + static_cast(bytes_used) > m_superframe_bytes_remain) { + result.result = AJM_RESULT_CODEC_ERROR | AJM_RESULT_INVALID_DATA; + break; + } + + const auto samples_written = + CopyDecodedFrame(output_bytes, output_size, &result.output_written, gapless); + + input_offset += static_cast(bytes_used); + result.input_consumed = input_offset; + result.frames++; + m_total_decoded_samples += samples_written; + + AdvanceSuperframe(static_cast(bytes_used), input_size, &input_offset, + &result); + if (gapless != nullptr && gapless->IsEnd()) { + DrainSuperframe(input_bytes, input_size, &input_offset, &result); + } + if (result.result != OK || !multiple_frames) { + break; + } + } + + result.total_decoded_samples = m_total_decoded_samples; + result.format = GetFormat(); + return result; + } + + void WriteCodecInfo(void* output, size_t output_size, + const AjmDecodeResult& result) const override { + (void)result; + if (output == nullptr || output_size < sizeof(AjmSidebandDecAt9CodecInfo)) { + return; + } + + auto* info = static_cast(output); + info->super_frame_size = + m_is_initialized ? static_cast(m_codec_info.superframeSize) : 0; + info->frames_in_super_frame = + m_is_initialized ? static_cast(m_codec_info.framesInSuperframe) : 0; + info->next_frame_size = GetMinimumInputSize(); + info->frame_samples = + m_is_initialized ? static_cast(m_codec_info.frameSamples) : 0; + } + + [[nodiscard]] size_t CodecInfoSize() const override { + return sizeof(AjmSidebandDecAt9CodecInfo); + } + + [[nodiscard]] AjmSidebandFormat GetFormat() const override { + auto format = AjmMakeFormat( + m_is_initialized ? static_cast(m_codec_info.channels) : m_channels, + m_is_initialized ? static_cast(m_codec_info.samplingRate) : m_sample_rate, + m_sample_encoding); + if (m_is_initialized && m_codec_info.framesInSuperframe > 0 && + m_codec_info.frameSamples > 0) { + format.bitrate = + static_cast((static_cast(m_codec_info.samplingRate) * + static_cast(m_codec_info.superframeSize) * 8u) / + (static_cast(m_codec_info.framesInSuperframe) * + static_cast(m_codec_info.frameSamples))); + } + return format; + } + +private: + [[nodiscard]] uint32_t FrameOutputSamples(const AjmGaplessState* gapless) const { + if (!m_is_initialized || m_codec_info.channels <= 0 || m_codec_info.frameSamples <= 0) { + return 0; + } + auto samples = static_cast(m_codec_info.frameSamples); + if (gapless != nullptr) { + const auto skip_samples = std::min(samples, gapless->current.skip_samples); + samples -= skip_samples; + if (gapless->HasSampleLimit()) { + samples = std::min(samples, gapless->current.total_samples); + } + } + return samples; + } + + [[nodiscard]] bool FrameCanProduceNoOutput(const AjmGaplessState* gapless) const { + return m_is_initialized && m_codec_info.channels > 0 && m_codec_info.frameSamples > 0 && + gapless != nullptr && + (gapless->current.skip_samples != 0 || + (gapless->HasSampleLimit() && gapless->current.total_samples == 0)); + } + + [[nodiscard]] size_t FrameOutputBytes(const AjmGaplessState* gapless = nullptr) const { + if (!m_is_initialized || m_codec_info.channels <= 0 || m_codec_info.frameSamples <= 0) { + return 0; + } + return static_cast(m_codec_info.channels) * + static_cast(FrameOutputSamples(gapless)) * + AjmBytesPerSample(m_sample_encoding); + } + + [[nodiscard]] uint32_t GetMinimumInputSize() const { + return (m_is_initialized ? m_superframe_bytes_remain : 0); + } + + bool InitializeConfig(const uint8_t* config_data, AjmDecodeResult* result) { + if (config_data == nullptr || result == nullptr) { + return false; + } + if (m_handle == nullptr) { + m_handle = Atrac9GetHandle(); + } + if (m_handle == nullptr) { + result->result = AJM_RESULT_CODEC_ERROR | AJM_RESULT_FATAL; + return false; + } + + std::memcpy(m_config_data, config_data, ATRAC9_CONFIG_DATA_SIZE); + m_has_config = true; + + const int init_ret = Atrac9InitDecoder(m_handle, m_config_data); + if (init_ret != 0) { + m_is_initialized = false; + result->result = AJM_RESULT_CODEC_ERROR | AJM_RESULT_INVALID_DATA; + result->internal_result = init_ret; + return false; + } + + const int info_ret = Atrac9GetCodecInfo(m_handle, &m_codec_info); + if (info_ret != 0 || m_codec_info.channels <= 0 || m_codec_info.frameSamples <= 0 || + m_codec_info.superframeSize <= 0 || m_codec_info.framesInSuperframe <= 0) { + m_is_initialized = false; + result->result = AJM_RESULT_CODEC_ERROR | AJM_RESULT_INVALID_DATA; + result->internal_result = info_ret; + return false; + } + + m_is_initialized = true; + m_num_frames = 0; + m_total_decoded_samples = 0; + m_superframe_bytes_remain = static_cast(m_codec_info.superframeSize); + SetFormat(static_cast(m_codec_info.channels), + static_cast(m_codec_info.samplingRate), m_sample_encoding); + m_pcm_buffer.assign(FrameOutputBytes(), 0); + + LOGF("AJM ATRAC9 initialized: %d Hz, %d ch, frame_samples=%d, superframe=%d bytes/%d " + "frames\n", + m_codec_info.samplingRate, m_codec_info.channels, m_codec_info.frameSamples, + m_codec_info.superframeSize, m_codec_info.framesInSuperframe); + + return true; + } + + int DecodeFrame(const uint8_t* input, int* bytes_used) { + switch (m_sample_encoding) { + case AjmSampleEncoding::S16: + return Atrac9Decode(m_handle, input, + reinterpret_cast(m_pcm_buffer.data()), bytes_used, 0); + case AjmSampleEncoding::S32: + return Atrac9DecodeS32(m_handle, input, + reinterpret_cast(m_pcm_buffer.data()), bytes_used, + 0); + case AjmSampleEncoding::Float: + return Atrac9DecodeF32( + m_handle, input, reinterpret_cast(m_pcm_buffer.data()), bytes_used, 0); + default: + EXIT("unsupported AJM PCM sample encoding %u\n", + static_cast(m_sample_encoding)); + } + return -1; + } + + uint64_t CopyDecodedFrame(uint8_t* output, size_t output_size, size_t* output_written, + AjmGaplessState* gapless) { + EXIT_IF(output_written == nullptr); + if (!m_is_initialized || m_codec_info.channels <= 0 || m_codec_info.frameSamples <= 0) { + return 0; + } + + const auto channels = static_cast(m_codec_info.channels); + const auto frame_samples = static_cast(m_codec_info.frameSamples); + const auto bytes_per_frame = + static_cast(channels) * AjmBytesPerSample(m_sample_encoding); + auto skip_samples = 0u; + auto samples_to_write = frame_samples; + + if (gapless != nullptr) { + skip_samples = std::min(frame_samples, gapless->current.skip_samples); + gapless->current.skip_samples = + static_cast(gapless->current.skip_samples - skip_samples); + + samples_to_write -= skip_samples; + if (gapless->HasSampleLimit()) { + samples_to_write = + std::min(samples_to_write, gapless->current.total_samples); + } + } + + const auto copy_bytes = static_cast(samples_to_write) * bytes_per_frame; + if (copy_bytes != 0) { + EXIT_IF(output == nullptr); + EXIT_IF(*output_written > output_size || copy_bytes > output_size - *output_written); + std::memcpy(output + *output_written, + m_pcm_buffer.data() + static_cast(skip_samples) * bytes_per_frame, + copy_bytes); + *output_written += copy_bytes; + } + + if (gapless != nullptr) { + if (gapless->HasSampleLimit()) { + gapless->current.total_samples = + (gapless->current.total_samples > samples_to_write + ? gapless->current.total_samples - samples_to_write + : 0); + } + + const auto skipped_samples = frame_samples - samples_to_write; + const auto skipped_total = std::min( + std::numeric_limits::max(), + static_cast(gapless->current.skipped_samples) + skipped_samples); + gapless->current.skipped_samples = static_cast(skipped_total); + } + + return samples_to_write; + } + + void AdvanceSuperframe(uint32_t bytes_used, size_t input_size, size_t* input_offset, + AjmDecodeResult* result) { + EXIT_IF(input_offset == nullptr); + EXIT_IF(result == nullptr); + + m_superframe_bytes_remain -= bytes_used; + m_num_frames++; + + if (m_num_frames < static_cast(m_codec_info.framesInSuperframe)) { + return; + } + + if (m_superframe_bytes_remain != 0) { + const auto remaining_input = + (*input_offset < input_size ? input_size - *input_offset : 0); + if (remaining_input < m_superframe_bytes_remain) { + result->result = AJM_RESULT_PARTIAL_INPUT; + return; + } + *input_offset += m_superframe_bytes_remain; + result->input_consumed = *input_offset; + } + + m_superframe_bytes_remain = static_cast(m_codec_info.superframeSize); + m_num_frames = 0; + } + + void DrainSuperframe(const uint8_t* input, size_t input_size, size_t* input_offset, + AjmDecodeResult* result) { + EXIT_IF(input_offset == nullptr); + EXIT_IF(result == nullptr); + + while (m_num_frames != 0) { + const auto remaining_input = + (*input_offset < input_size ? input_size - *input_offset : 0); + if (remaining_input < GetMinimumInputSize()) { + result->result = AJM_RESULT_PARTIAL_INPUT; + return; + } + + int bytes_used = 0; + const int ret = DecodeFrame(input + *input_offset, &bytes_used); + if (ret != 0) { + result->result = AJM_RESULT_CODEC_ERROR | AJM_RESULT_FATAL; + result->internal_result = ret; + return; + } + if (bytes_used <= 0 || static_cast(bytes_used) > remaining_input || + static_cast(bytes_used) > m_superframe_bytes_remain) { + result->result = AJM_RESULT_CODEC_ERROR | AJM_RESULT_INVALID_DATA; + return; + } + + *input_offset += static_cast(bytes_used); + result->input_consumed = *input_offset; + result->frames++; + AdvanceSuperframe(static_cast(bytes_used), input_size, input_offset, result); + if (result->result != OK) { + return; + } + } + } + + bool ParseRiffHeader(const uint8_t* input, size_t input_size, size_t* data_offset, + AjmGaplessState* gapless, AjmDecodeResult* result) { + EXIT_IF(data_offset == nullptr); + EXIT_IF(result == nullptr); + + *data_offset = 0; + if (input_size < 12 || !AjmFourCcEquals(input, 'R', 'I', 'F', 'F') || + !AjmFourCcEquals(input + 8, 'W', 'A', 'V', 'E')) { + result->result = AJM_RESULT_INVALID_DATA; + return false; + } + + size_t offset = 12; + while (offset + 8 <= input_size) { + const auto* chunk = input + offset; + const auto chunk_size = static_cast(AjmReadLe32(chunk + 4)); + const auto payload = offset + 8; + if (payload > input_size || chunk_size > input_size - payload) { + result->result = AJM_RESULT_PARTIAL_INPUT; + return false; + } + + if (AjmFourCcEquals(chunk, 'f', 'm', 't', ' ')) { + if (chunk_size < 48) { + result->result = AJM_RESULT_INVALID_DATA; + return false; + } + const auto* fmt = input + payload; + static constexpr uint8_t AT9_GUID[16] = {0xd2, 0x42, 0xe1, 0x47, 0xba, 0x36, + 0x8d, 0x4d, 0x88, 0xfc, 0x61, 0x65, + 0x4f, 0x8c, 0x83, 0x6c}; + if (AjmReadLe16(fmt) != 0xfffe || + std::memcmp(fmt + 24, AT9_GUID, sizeof(AT9_GUID)) != 0) { + result->result = AJM_RESULT_INVALID_DATA; + return false; + } + if (!InitializeConfig(fmt + 44, result)) { + return false; + } + } else if (AjmFourCcEquals(chunk, 'f', 'a', 'c', 't')) { + if (chunk_size >= 12 && gapless != nullptr) { + AjmSidebandGaplessDecode gapless_decode {}; + gapless_decode.total_samples = AjmReadLe32(input + payload); + gapless_decode.skip_samples = static_cast(std::min( + AjmReadLe32(input + payload + 4), std::numeric_limits::max())); + gapless->Set(gapless_decode, true); + LOGF("AJM ATRAC9 gapless: total=%" PRIu32 ", skip=%" PRIu16 "\n", + gapless_decode.total_samples, gapless_decode.skip_samples); + } + } else if (AjmFourCcEquals(chunk, 'd', 'a', 't', 'a')) { + *data_offset = payload; + result->input_consumed = payload; + result->format = GetFormat(); + result->total_decoded_samples = m_total_decoded_samples; + return true; + } + + const auto padded_size = chunk_size + (chunk_size & 1u); + if (padded_size > std::numeric_limits::max() - 8 || + offset > std::numeric_limits::max() - 8 - padded_size) { + result->result = AJM_RESULT_INVALID_DATA; + return false; + } + offset += 8 + padded_size; + } + + result->result = AJM_RESULT_PARTIAL_INPUT; + return false; + } + + bool m_parse_riff_header = false; + bool m_is_initialized = false; + bool m_has_config = false; + void* m_handle = nullptr; + uint8_t m_config_data[ATRAC9_CONFIG_DATA_SIZE] {}; + uint32_t m_superframe_bytes_remain = 0; + uint32_t m_num_frames = 0; + Atrac9CodecInfo m_codec_info {}; + std::vector m_pcm_buffer; +}; + +} // namespace Libs::Audio::Ajm diff --git a/src/libs/ajm/ffmpeg_decoder_common.h b/src/libs/ajm/ffmpeg_decoder_common.h new file mode 100644 index 000000000..3b1dffdde --- /dev/null +++ b/src/libs/ajm/ffmpeg_decoder_common.h @@ -0,0 +1,65 @@ +#pragma once + +#include "common/assert.h" + +#include + +extern "C" { +#include +#include +#include +#include +#include +} + +namespace Libs::Audio::Ajm { + +static AVSampleFormat AjmSampleEncodingToAvFormat(AjmSampleEncoding encoding) { + switch (encoding) { + case AjmSampleEncoding::S16: return AV_SAMPLE_FMT_S16; + case AjmSampleEncoding::S32: return AV_SAMPLE_FMT_S32; + case AjmSampleEncoding::Float: return AV_SAMPLE_FMT_FLT; + default: EXIT("unsupported AJM PCM sample encoding %u\n", static_cast(encoding)); + } + return AV_SAMPLE_FMT_S16; +} + +static AVFrame* AjmConvertFfmpegFrame(const AVFrame* frame, AjmSampleEncoding encoding, + AjmDecodeResult* result) { + if (frame == nullptr || result == nullptr) { + return nullptr; + } + + AVFrame* converted = av_frame_alloc(); + if (converted == nullptr) { + result->result = AJM_RESULT_CODEC_ERROR | AJM_RESULT_FATAL; + return nullptr; + } + + converted->format = AjmSampleEncodingToAvFormat(encoding); + converted->sample_rate = frame->sample_rate; + converted->nb_samples = frame->nb_samples; + if (av_channel_layout_copy(&converted->ch_layout, &frame->ch_layout) < 0 || + av_frame_get_buffer(converted, 0) < 0) { + av_frame_free(&converted); + result->result = AJM_RESULT_CODEC_ERROR | AJM_RESULT_FATAL; + return nullptr; + } + + SwrContext* swr = nullptr; + int swr_result = swr_alloc_set_opts2( + &swr, &converted->ch_layout, static_cast(converted->format), + converted->sample_rate, &frame->ch_layout, static_cast(frame->format), + frame->sample_rate, 0, nullptr); + if (swr_result < 0 || swr_init(swr) < 0 || swr_convert_frame(swr, converted, frame) < 0) { + swr_free(&swr); + av_frame_free(&converted); + result->result = AJM_RESULT_CODEC_ERROR | AJM_RESULT_INVALID_DATA; + return nullptr; + } + swr_free(&swr); + + return converted; +} + +} // namespace Libs::Audio::Ajm diff --git a/src/libs/ajm/mp3_decoder.h b/src/libs/ajm/mp3_decoder.h new file mode 100644 index 000000000..b3de890c1 --- /dev/null +++ b/src/libs/ajm/mp3_decoder.h @@ -0,0 +1,393 @@ +#pragma once + +#include "common/logging/log.h" +#include "libs/ajm/ffmpeg_decoder_common.h" + +#include +#include +#include +#include +#include + +namespace Libs::Audio::Ajm { + +struct AjmSidebandDecMp3CodecInfo { + uint32_t header; + uint8_t crc; + uint8_t mode; + uint8_t mode_extension; + uint8_t copyright; + uint8_t original; + uint8_t emphasis; + uint16_t reserved[3]; +}; + +static_assert(sizeof(AjmSidebandDecMp3CodecInfo) == 16); + +struct AjmMp3FrameHeader { + uint32_t header = 0; + uint32_t frame_size = 0; + uint32_t num_channels = 0; + uint32_t samples_per_channel = 0; + uint32_t bitrate = 0; + uint32_t sample_rate = 0; + uint8_t crc = 0; + uint8_t mode = 0; + uint8_t mode_extension = 0; + uint8_t copyright = 0; + uint8_t original = 0; + uint8_t emphasis = 0; +}; + +static uint32_t AjmReadBe32(const uint8_t* data) { + return (static_cast(data[0]) << 24u) | (static_cast(data[1]) << 16u) | + (static_cast(data[2]) << 8u) | static_cast(data[3]); +} + +static bool AjmParseMp3Header(const uint8_t* data, size_t size, AjmMp3FrameHeader* out) { + if (data == nullptr || size < 4 || out == nullptr) { + return false; + } + + const uint32_t header = AjmReadBe32(data); + if ((header & 0xffe00000u) != 0xffe00000u) { + return false; + } + + const auto version_id = (header >> 19u) & 0x3u; + const auto layer_id = (header >> 17u) & 0x3u; + const auto protection_bit = (header >> 16u) & 0x1u; + const auto bitrate_index = (header >> 12u) & 0xfu; + const auto sample_index = (header >> 10u) & 0x3u; + const auto padding = (header >> 9u) & 0x1u; + const auto channel_mode = (header >> 6u) & 0x3u; + const auto mode_extension = (header >> 4u) & 0x3u; + const auto copyright = (header >> 3u) & 0x1u; + const auto original = (header >> 2u) & 0x1u; + const auto emphasis = header & 0x3u; + + if (version_id == 1 || layer_id != 1 || sample_index == 3 || bitrate_index == 0 || + bitrate_index == 15) { + return false; + } + + static constexpr uint32_t SAMPLE_RATES[4][3] = { + {11025, 12000, 8000}, + {0, 0, 0}, + {22050, 24000, 16000}, + {44100, 48000, 32000}, + }; + static constexpr uint32_t BITRATES_MPEG1[16] = { + 0, 32000, 40000, 48000, 56000, 64000, 80000, 96000, + 112000, 128000, 160000, 192000, 224000, 256000, 320000, 0, + }; + static constexpr uint32_t BITRATES_MPEG2[16] = { + 0, 8000, 16000, 24000, 32000, 40000, 48000, 56000, + 64000, 80000, 96000, 112000, 128000, 144000, 160000, 0, + }; + + const auto sample_rate = SAMPLE_RATES[version_id][sample_index]; + const auto bitrate = + (version_id == 3 ? BITRATES_MPEG1[bitrate_index] : BITRATES_MPEG2[bitrate_index]); + if (sample_rate == 0 || bitrate == 0) { + return false; + } + + const bool mpeg1 = (version_id == 3); + + out->header = header; + out->frame_size = + (mpeg1 ? (144u * bitrate) / sample_rate : (72u * bitrate) / sample_rate) + padding; + out->num_channels = (channel_mode == 3 ? 1u : 2u); + out->samples_per_channel = (mpeg1 ? 1152u : 576u); + out->bitrate = bitrate; + out->sample_rate = sample_rate; + out->crc = static_cast(protection_bit); + out->mode = static_cast(channel_mode); + out->mode_extension = static_cast(mode_extension); + out->copyright = static_cast(copyright); + out->original = static_cast(original); + out->emphasis = static_cast(emphasis); + return true; +} + +class AjmMp3Decoder final: public AjmDecoder { +public: + AjmMp3Decoder(uint32_t channels, uint32_t sample_rate, AjmSampleEncoding encoding, + uint64_t flags) + : AjmDecoder(channels, sample_rate, encoding) { + (void)flags; + + m_codec = avcodec_find_decoder(AV_CODEC_ID_MP3); + if (m_codec == nullptr) { + LOGF("AJM MP3: avcodec_find_decoder failed\n"); + return; + } + + m_codec_context = avcodec_alloc_context3(m_codec); + m_parser = av_parser_init(m_codec->id); + if (m_codec_context == nullptr || m_parser == nullptr || + avcodec_open2(m_codec_context, m_codec, nullptr) < 0) { + LOGF("AJM MP3: decoder initialization failed\n"); + Release(); + } + } + + ~AjmMp3Decoder() override { Release(); } + + void Reset() override { + if (m_codec_context != nullptr) { + avcodec_flush_buffers(m_codec_context); + } + if (m_parser != nullptr) { + av_parser_close(m_parser); + m_parser = (m_codec != nullptr ? av_parser_init(m_codec->id) : nullptr); + } + m_total_decoded_samples = 0; + m_have_header = false; + m_bitrate = 0; + } + + AjmDecodeResult Decode(const void* input, size_t input_size, void* output, size_t output_size, + bool multiple_frames, AjmGaplessState* gapless) override { + auto result = MakeResult(); + + if (m_codec_context == nullptr || m_parser == nullptr) { + result.result = AJM_RESULT_CODEC_ERROR | AJM_RESULT_FATAL; + return result; + } + if (input == nullptr || input_size == 0) { + result.result = AJM_RESULT_PARTIAL_INPUT; + return result; + } + if (input_size > static_cast(std::numeric_limits::max())) { + result.result = AJM_RESULT_INVALID_PARAMETER; + return result; + } + + const auto* input_bytes = static_cast(input); + size_t input_offset = 0; + size_t output_offset = 0; + bool decoded = false; + + while (input_offset < input_size) { + AjmMp3FrameHeader header {}; + if (AjmParseMp3Header(input_bytes + input_offset, input_size - input_offset, &header)) { + m_last_header = header; + m_have_header = true; + } + + uint8_t* packet_data = nullptr; + int packet_size = 0; + const int consumed = av_parser_parse2( + m_parser, m_codec_context, &packet_data, &packet_size, input_bytes + input_offset, + static_cast(input_size - input_offset), AV_NOPTS_VALUE, AV_NOPTS_VALUE, 0); + if (consumed < 0) { + result.result = AJM_RESULT_CODEC_ERROR | AJM_RESULT_INVALID_DATA; + break; + } + + input_offset += static_cast(consumed); + if (packet_size > 0) { + if (!DecodePacket(packet_data, packet_size, output, output_size, &output_offset, + gapless, &result)) { + break; + } + decoded = true; + if (!multiple_frames) { + break; + } + } + + if (consumed == 0) { + break; + } + } + + if (!decoded && result.result == OK) { + result.result = AJM_RESULT_PARTIAL_INPUT; + } + + result.input_consumed = input_offset; + result.output_written = output_offset; + result.total_decoded_samples = m_total_decoded_samples; + result.format = GetFormat(); + return result; + } + + void WriteCodecInfo(void* output, size_t output_size, + const AjmDecodeResult& result) const override { + (void)result; + if (output == nullptr || output_size < sizeof(AjmSidebandDecMp3CodecInfo) || + !m_have_header) { + return; + } + + auto* info = static_cast(output); + info->header = m_last_header.header; + info->crc = m_last_header.crc; + info->mode = m_last_header.mode; + info->mode_extension = m_last_header.mode_extension; + info->copyright = m_last_header.copyright; + info->original = m_last_header.original; + info->emphasis = m_last_header.emphasis; + } + + [[nodiscard]] size_t CodecInfoSize() const override { + return sizeof(AjmSidebandDecMp3CodecInfo); + } + + [[nodiscard]] AjmSidebandFormat GetFormat() const override { + auto format = AjmMakeFormat(m_channels, m_sample_rate, m_sample_encoding); + format.bitrate = m_bitrate; + return format; + } + +private: + void Release() { + if (m_parser != nullptr) { + av_parser_close(m_parser); + m_parser = nullptr; + } + if (m_codec_context != nullptr) { + avcodec_free_context(&m_codec_context); + } + } + + bool WriteFrame(const AVFrame* frame, void* output, size_t output_size, size_t* output_offset, + AjmGaplessState* gapless, AjmDecodeResult* result) { + if (frame == nullptr || output_offset == nullptr || result == nullptr) { + return false; + } + + const auto channels = static_cast(std::max(frame->ch_layout.nb_channels, 1)); + const auto bpf = channels * AjmBytesPerSample(m_sample_encoding); + if (bpf == 0) { + result->result = AJM_RESULT_INVALID_PARAMETER; + return false; + } + + SetFormat(channels, static_cast(std::max(frame->sample_rate, 1)), + m_sample_encoding); + if (m_codec_context != nullptr && m_codec_context->bit_rate > 0) { + m_bitrate = static_cast(m_codec_context->bit_rate); + } else if (m_have_header) { + m_bitrate = m_last_header.bitrate; + } + + uint32_t skip_samples = 0; + if (gapless != nullptr && gapless->current.skip_samples > 0) { + skip_samples = std::min(frame->nb_samples, gapless->current.skip_samples); + gapless->current.skip_samples = + static_cast(gapless->current.skip_samples - skip_samples); + } + + uint32_t samples_to_write = static_cast(frame->nb_samples) - skip_samples; + if (gapless != nullptr && gapless->HasSampleLimit()) { + samples_to_write = std::min(samples_to_write, gapless->current.total_samples); + } + + const auto requested_bytes = static_cast(samples_to_write) * bpf; + auto writable_bytes = requested_bytes; + if (output == nullptr || *output_offset >= output_size) { + writable_bytes = 0; + } else { + writable_bytes = std::min(writable_bytes, output_size - *output_offset); + writable_bytes -= writable_bytes % bpf; + } + + if (writable_bytes != 0) { + auto* dst = static_cast(output) + *output_offset; + const auto* src = frame->data[0] + static_cast(skip_samples) * bpf; + std::memcpy(dst, src, writable_bytes); + *output_offset += writable_bytes; + } + + const auto samples_written = static_cast(writable_bytes / bpf); + if (gapless != nullptr && gapless->HasSampleLimit()) { + gapless->current.total_samples -= + std::min(gapless->current.total_samples, samples_written); + } + if (gapless != nullptr) { + const auto skipped_total = + std::min(std::numeric_limits::max(), + static_cast(gapless->current.skipped_samples) + + skip_samples + (samples_to_write - samples_written)); + gapless->current.skipped_samples = static_cast(skipped_total); + } + + m_total_decoded_samples += samples_written; + result->frames++; + result->total_decoded_samples = m_total_decoded_samples; + result->format = GetFormat(); + + if (writable_bytes != requested_bytes) { + result->result |= AJM_RESULT_NOT_ENOUGH_ROOM; + return false; + } + return true; + } + + bool DecodePacket(const uint8_t* packet_data, int packet_size, void* output, size_t output_size, + size_t* output_offset, AjmGaplessState* gapless, AjmDecodeResult* result) { + if (packet_data == nullptr || packet_size <= 0 || result == nullptr) { + return true; + } + + AVPacket* packet = av_packet_alloc(); + if (packet == nullptr || av_new_packet(packet, packet_size) < 0) { + av_packet_free(&packet); + result->result = AJM_RESULT_CODEC_ERROR | AJM_RESULT_FATAL; + return false; + } + std::memcpy(packet->data, packet_data, static_cast(packet_size)); + + int rc = avcodec_send_packet(m_codec_context, packet); + av_packet_free(&packet); + if (rc < 0) { + result->result = AJM_RESULT_CODEC_ERROR | AJM_RESULT_INVALID_DATA; + return false; + } + + for (;;) { + AVFrame* frame = av_frame_alloc(); + if (frame == nullptr) { + result->result = AJM_RESULT_CODEC_ERROR | AJM_RESULT_FATAL; + return false; + } + + rc = avcodec_receive_frame(m_codec_context, frame); + if (rc == AVERROR(EAGAIN) || rc == AVERROR_EOF) { + av_frame_free(&frame); + return true; + } + if (rc < 0) { + av_frame_free(&frame); + result->result = AJM_RESULT_CODEC_ERROR | AJM_RESULT_INVALID_DATA; + return false; + } + + AVFrame* converted = AjmConvertFfmpegFrame(frame, m_sample_encoding, result); + av_frame_free(&frame); + if (converted == nullptr) { + return false; + } + + const bool ok = + WriteFrame(converted, output, output_size, output_offset, gapless, result); + av_frame_free(&converted); + if (!ok) { + return false; + } + } + } + + const AVCodec* m_codec = nullptr; + AVCodecContext* m_codec_context = nullptr; + AVCodecParserContext* m_parser = nullptr; + bool m_have_header = false; + uint32_t m_bitrate = 0; + AjmMp3FrameHeader m_last_header; +}; + +} // namespace Libs::Audio::Ajm diff --git a/src/libs/audio.cpp b/src/libs/audio.cpp new file mode 100644 index 000000000..c7a3eb294 --- /dev/null +++ b/src/libs/audio.cpp @@ -0,0 +1,2639 @@ +#include "libs/audio.h" + +#include "SDL.h" +#include "common/assert.h" +#include "common/common.h" +#include "common/logging/log.h" +#include "common/magicEnum.h" +#include "common/stringUtils.h" +#include "common/threads.h" +#include "kernel/pthread.h" +#include "kernel/semaphore.h" +#include "libs/audio_internal.h" +#include "libs/errno.h" +#include "libs/libs.h" + +#include +#include +#include +#include +#include +#include +#include + +namespace Libs::Audio { + +namespace { + +constexpr int AUDIO_OUT_PORT_TYPE_MAIN = 0; +constexpr int AUDIO_OUT_PORT_TYPE_BGM = 1; +constexpr int AUDIO_OUT_PORT_TYPE_VOICE = 2; +constexpr int AUDIO_OUT_PORT_TYPE_PERSONAL = 3; +constexpr int AUDIO_OUT_PORT_TYPE_PADSPK = 4; +constexpr int AUDIO_OUT_PORT_TYPE_VIBRATION = 10; +constexpr int AUDIO_OUT_PORT_TYPE_AUDIO3D = 126; +constexpr int AUDIO_OUT_PORT_TYPE_AUX = 127; + +constexpr uint32_t AUDIO_OUT_PARAM_FORMAT_MASK = 0x000000ffu; + +static bool audio_out_port_type_is_valid(int type) { + return (type >= AUDIO_OUT_PORT_TYPE_MAIN && type <= AUDIO_OUT_PORT_TYPE_PADSPK) || + type == AUDIO_OUT_PORT_TYPE_VIBRATION || type == AUDIO_OUT_PORT_TYPE_AUDIO3D || + type == AUDIO_OUT_PORT_TYPE_AUX; +} + +} // namespace + +class Audio { +public: + using Format = AudioInternal::Format; + + class Id { + public: + explicit Id(int id): m_id(id - 1) {} + [[nodiscard]] int ToInt() const { return m_id + 1; } + [[nodiscard]] bool IsValid() const { return m_id >= 0; } + + friend class Audio; + + private: + Id() = default; + static Id Invalid() { return {}; } + static Id Create(int audio_id) { + Id r; + r.m_id = audio_id; + return r; + } + [[nodiscard]] int GetId() const { return m_id; } + + int m_id = -1; + }; + + struct OutputParam { + Id handle; + const void* data = nullptr; + }; + + Audio() = default; + virtual ~Audio(); + + KYTY_CLASS_NO_COPY(Audio); + + Id AudioOutOpen(int type, uint32_t samples_num, uint32_t freq, Format format); + bool AudioOutClose(Id handle); + bool AudioOutValid(Id handle); + bool AudioOutSetVolume(Id handle, uint32_t bitflag, const int* volume); + uint32_t AudioOutOutputs(OutputParam* params, uint32_t num); + bool AudioOutGetStatus(Id handle, int* type, int* channels_num); + + Id AudioInOpen(uint32_t type, uint32_t samples_num, uint32_t freq, Format format); + bool AudioInValid(Id handle); + uint32_t AudioInInput(Id handle, void* dest); + + static constexpr int OUT_PORTS_MAX = 32; + static constexpr int IN_PORTS_MAX = 8; + +private: + struct PortOut { + bool used = false; + int type = 0; + uint32_t samples_num = 0; + uint32_t freq = 0; + Format format = Format::Unknown; + uint64_t last_output_time = 0; + int channels_num = 0; + int volume[8] = {}; + + SDL_AudioDeviceID audio_device = 0; + SDL_AudioSpec audio_spec = {}; + }; + + struct PortIn { + bool used = false; + uint32_t type = 0; + uint32_t samples_num = 0; + uint32_t freq = 0; + Format format = Format::Unknown; + uint64_t last_input_time = 0; + }; + + Common::Mutex m_mutex; + PortOut m_out_ports[OUT_PORTS_MAX]; + PortIn m_in_ports[IN_PORTS_MAX]; + + static bool FormatIsFloat(Format format); + static bool FormatIsStd(Format format); + static uint32_t BytesPerSample(Format format); + static uint32_t FrameSize(const PortOut& port); + static SDL_AudioFormat SdlFormat(Format format); + static bool OpenSdlDevice(PortOut* port); + static void CloseSdlDevice(PortOut* port); + static const void* PrepareOutputBuffer(const PortOut& port, const void* data, + std::vector* buffer); + static bool QueueSdlAudio(PortOut* port, const void* data); +}; + +static Audio* g_audio = nullptr; + +namespace AudioInternal { + +int AudioOutOpen(int type, uint32_t samples_num, uint32_t freq, Format format) { + if (g_audio == nullptr) { + return 0; + } + + auto id = g_audio->AudioOutOpen(type, samples_num, freq, format); + return id.IsValid() ? id.ToInt() : 0; +} + +void AudioOutClose(int handle) { + if (g_audio != nullptr && handle > 0) { + (void)g_audio->AudioOutClose(Audio::Id(handle)); + } +} + +uint32_t AudioOutOutputs(const OutputParam* params, uint32_t num) { + if (g_audio == nullptr || params == nullptr || num == 0) { + return 0; + } + + std::vector output_params; + output_params.reserve(num); + for (uint32_t i = 0; i < num; i++) { + if (params[i].handle > 0 && params[i].data != nullptr) { + output_params.push_back( + Audio::OutputParam {Audio::Id(params[i].handle), params[i].data}); + } + } + + if (output_params.empty()) { + return 0; + } + + return g_audio->AudioOutOutputs(output_params.data(), + static_cast(output_params.size())); +} + +} // namespace AudioInternal + +KYTY_SUBSYSTEM_INIT(Audio) { + EXIT_IF(g_audio != nullptr); + + g_audio = new Audio; +} + +KYTY_SUBSYSTEM_UNEXPECTED_SHUTDOWN(Audio) {} + +KYTY_SUBSYSTEM_DESTROY(Audio) { + delete g_audio; + g_audio = nullptr; +} + +Audio::~Audio() { + for (auto& port: m_out_ports) { + CloseSdlDevice(&port); + } +} + +bool Audio::FormatIsFloat(Format format) { + return (format == Format::FloatMono || format == Format::FloatStereo || + format == Format::Float8Ch || format == Format::Float8ChStd); +} + +bool Audio::FormatIsStd(Format format) { + return (format == Format::Signed16bit8ChStd || format == Format::Float8ChStd); +} + +uint32_t Audio::BytesPerSample(Format format) { + return FormatIsFloat(format) ? sizeof(float) : sizeof(int16_t); +} + +uint32_t Audio::FrameSize(const PortOut& port) { + return BytesPerSample(port.format) * port.channels_num; +} + +SDL_AudioFormat Audio::SdlFormat(Format format) { + return FormatIsFloat(format) ? AUDIO_F32SYS : AUDIO_S16SYS; +} + +bool Audio::OpenSdlDevice(PortOut* port) { + EXIT_IF(port == nullptr); + + if (SDL_InitSubSystem(SDL_INIT_AUDIO) < 0) { + LOGF("AudioOut: SDL audio init failed: %s\n", SDL_GetError()); + return false; + } + + SDL_AudioSpec desired {}; + desired.freq = static_cast(port->freq); + desired.format = SdlFormat(port->format); + desired.channels = static_cast(port->channels_num); + desired.samples = static_cast(port->samples_num); + desired.callback = nullptr; + + SDL_AudioSpec obtained {}; + + port->audio_device = + SDL_OpenAudioDevice(nullptr, 0, &desired, &obtained, SDL_AUDIO_ALLOW_ANY_CHANGE); + if (port->audio_device == 0) { + LOGF("AudioOut: SDL_OpenAudioDevice failed: %s\n", SDL_GetError()); + return false; + } + + port->audio_spec = obtained; + SDL_PauseAudioDevice(port->audio_device, 0); + + LOGF("AudioOut: opened SDL device (%d Hz, %u ch, format 0x%04x)\n", obtained.freq, + obtained.channels, obtained.format); + return true; +} + +void Audio::CloseSdlDevice(PortOut* port) { + EXIT_IF(port == nullptr); + + if (port->audio_device != 0 && SDL_WasInit(SDL_INIT_AUDIO) != 0) { + SDL_ClearQueuedAudio(port->audio_device); + SDL_CloseAudioDevice(port->audio_device); + } + + port->audio_device = 0; + port->audio_spec = {}; +} + +const void* Audio::PrepareOutputBuffer(const PortOut& port, const void* data, + std::vector* buffer) { + EXIT_IF(data == nullptr); + EXIT_IF(buffer == nullptr); + + const auto frames = port.samples_num; + const auto channels = static_cast(port.channels_num); + const auto bytes_per_sample = BytesPerSample(port.format); + const auto src_size = frames * channels * bytes_per_sample; + + bool volume_changed = false; + for (uint32_t ch = 0; ch < channels; ch++) { + if (port.volume[ch] != 32768) { + volume_changed = true; + break; + } + } + + if (!volume_changed && !FormatIsStd(port.format)) { + return data; + } + + buffer->resize(src_size); + + static constexpr uint32_t STD_8CH_MAP[8] = {0, 1, 2, 3, 6, 7, 4, 5}; + + if (FormatIsFloat(port.format)) { + auto* dst = reinterpret_cast(buffer->data()); + const auto* src = static_cast(data); + + for (uint32_t frame = 0; frame < frames; frame++) { + for (uint32_t ch = 0; ch < channels; ch++) { + const auto src_ch = + (FormatIsStd(port.format) && channels == 8 ? STD_8CH_MAP[ch] : ch); + dst[frame * channels + ch] = src[frame * channels + src_ch] * + (static_cast(port.volume[ch]) / 32768.0f); + } + } + } else { + auto* dst = reinterpret_cast(buffer->data()); + const auto* src = static_cast(data); + + for (uint32_t frame = 0; frame < frames; frame++) { + for (uint32_t ch = 0; ch < channels; ch++) { + const auto src_ch = + (FormatIsStd(port.format) && channels == 8 ? STD_8CH_MAP[ch] : ch); + int64_t sample = + static_cast(src[frame * channels + src_ch]) * port.volume[ch] / 32768; + if (sample > std::numeric_limits::max()) { + sample = std::numeric_limits::max(); + } else if (sample < std::numeric_limits::min()) { + sample = std::numeric_limits::min(); + } + dst[frame * channels + ch] = static_cast(sample); + } + } + } + + return buffer->data(); +} + +bool Audio::QueueSdlAudio(PortOut* port, const void* data) { + EXIT_IF(port == nullptr); + + if (port->audio_device == 0 || data == nullptr) { + return false; + } + + std::vector prepared_buffer; + const void* prepared_data = PrepareOutputBuffer(*port, data, &prepared_buffer); + const auto prepared_size = FrameSize(*port) * port->samples_num; + + std::vector convert_buffer; + const void* queue_data = prepared_data; + uint32_t queue_size = prepared_size; + + SDL_AudioCVT cvt {}; + const int cvt_result = + SDL_BuildAudioCVT(&cvt, SdlFormat(port->format), static_cast(port->channels_num), + static_cast(port->freq), port->audio_spec.format, + port->audio_spec.channels, port->audio_spec.freq); + + if (cvt_result < 0) { + LOGF("AudioOut: SDL_BuildAudioCVT failed: %s\n", SDL_GetError()); + return false; + } + + if (cvt_result > 0) { + convert_buffer.resize(prepared_size * cvt.len_mult); + std::memcpy(convert_buffer.data(), prepared_data, prepared_size); + + cvt.buf = convert_buffer.data(); + cvt.len = static_cast(prepared_size); + + if (SDL_ConvertAudio(&cvt) < 0) { + LOGF("AudioOut: SDL_ConvertAudio failed: %s\n", SDL_GetError()); + return false; + } + + queue_data = cvt.buf; + queue_size = static_cast(cvt.len_cvt); + } + + const auto min_queued_size = queue_size * 2u; + const auto wait_start = LibKernel::KernelGetProcessTime(); + while (SDL_GetQueuedAudioSize(port->audio_device) > min_queued_size) { + if (LibKernel::KernelGetProcessTime() - wait_start > 200000) { + SDL_ClearQueuedAudio(port->audio_device); + break; + } + Common::Thread::SleepMicro(1000); + } + + if (SDL_QueueAudio(port->audio_device, queue_data, queue_size) < 0) { + LOGF("AudioOut: SDL_QueueAudio failed: %s\n", SDL_GetError()); + return false; + } + + return true; +} + +Audio::Id Audio::AudioOutOpen(int type, uint32_t samples_num, uint32_t freq, Format format) { + Common::LockGuard lock(m_mutex); + + for (int id = 0; id < OUT_PORTS_MAX; id++) { + if (!m_out_ports[id].used) { + auto& port = m_out_ports[id]; + + port.used = true; + port.type = type; + port.samples_num = samples_num; + port.freq = freq; + port.format = format; + port.last_output_time = 0; + + switch (format) { + case Format::Signed16bitMono: + case Format::FloatMono: port.channels_num = 1; break; + case Format::Signed16bitStereo: + case Format::FloatStereo: port.channels_num = 2; break; + case Format::Signed16bit8Ch: + case Format::Float8Ch: + case Format::Signed16bit8ChStd: + case Format::Float8ChStd: port.channels_num = 8; break; + default: EXIT("unknown format"); + } + + for (int i = 0; i < port.channels_num; i++) { + port.volume[i] = 32768; + } + + if (type != AUDIO_OUT_PORT_TYPE_VIBRATION) { + OpenSdlDevice(&port); + } + + return Id::Create(id); + } + } + + return Id::Invalid(); +} + +bool Audio::AudioOutClose(Id handle) { + Common::LockGuard lock(m_mutex); + + if (AudioOutValid(handle)) { + auto& port = m_out_ports[handle.GetId()]; + + CloseSdlDevice(&port); + port = {}; + + return true; + } + + return false; +} + +bool Audio::AudioOutValid(Id handle) { + Common::LockGuard lock(m_mutex); + + return (handle.GetId() >= 0 && handle.GetId() < OUT_PORTS_MAX && + m_out_ports[handle.GetId()].used); +} + +bool Audio::AudioOutGetStatus(Id handle, int* type, int* channels_num) { + Common::LockGuard lock(m_mutex); + + if (AudioOutValid(handle)) { + auto& port = m_out_ports[handle.GetId()]; + + *type = port.type; + *channels_num = port.channels_num; + + return true; + } + + return false; +} + +bool Audio::AudioOutSetVolume(Id handle, uint32_t bitflag, const int* volume) { + Common::LockGuard lock(m_mutex); + + if (AudioOutValid(handle)) { + auto& port = m_out_ports[handle.GetId()]; + + for (int i = 0; i < port.channels_num; i++, bitflag >>= 1u) { + auto bit = bitflag & 0x1u; + + if (bit == 1) { + int src_index = i; + if (port.format == Format::Float8ChStd || + port.format == Format::Signed16bit8ChStd) { + switch (i) { + case 4: src_index = 6; break; + case 5: src_index = 7; break; + case 6: src_index = 4; break; + case 7: src_index = 5; break; + default:; + } + } + port.volume[i] = volume[src_index]; + + LOGF("\t port.volume[%d] = volume[%d] (%d)\n", i, src_index, volume[src_index]); + } + } + + return true; + } + + return false; +} + +uint32_t Audio::AudioOutOutputs(OutputParam* params, uint32_t num) { + EXIT_NOT_IMPLEMENTED(num == 0); + EXIT_NOT_IMPLEMENTED(!AudioOutValid(params[0].handle)); + + const auto& first_port = m_out_ports[params[0].handle.GetId()]; + + uint64_t block_time = (1000000 * first_port.samples_num) / first_port.freq; + uint64_t current_time = LibKernel::KernelGetProcessTime(); + + uint64_t max_wait_time = 0; + + for (uint32_t i = 0; i < num; i++) { + uint64_t next_time = m_out_ports[params[i].handle.GetId()].last_output_time + block_time; + uint64_t wait_time = (next_time > current_time ? next_time - current_time : 0); + max_wait_time = (wait_time > max_wait_time ? wait_time : max_wait_time); + } + + if (max_wait_time != 0) { + Common::Thread::SleepMicro(max_wait_time); + } + + for (uint32_t i = 0; i < num; i++) { + auto& port = m_out_ports[params[i].handle.GetId()]; + + QueueSdlAudio(&port, params[i].data); + } + + for (uint32_t i = 0; i < num; i++) { + m_out_ports[params[i].handle.GetId()].last_output_time = LibKernel::KernelGetProcessTime(); + } + + return first_port.samples_num; +} + +Audio::Id Audio::AudioInOpen(uint32_t type, uint32_t samples_num, uint32_t freq, Format format) { + Common::LockGuard lock(m_mutex); + + for (int id = 0; id < IN_PORTS_MAX; id++) { + if (!m_in_ports[id].used) { + auto& port = m_in_ports[id]; + + port.used = true; + port.type = type; + port.samples_num = samples_num; + port.freq = freq; + port.format = format; + + switch (format) { + case Format::Signed16bitMono: + case Format::Signed16bitStereo: break; + default: EXIT("unknown format"); + } + + return Id::Create(id); + } + } + + return Id::Invalid(); +} + +bool Audio::AudioInValid(Id handle) { + Common::LockGuard lock(m_mutex); + + return (handle.GetId() >= 0 && handle.GetId() < IN_PORTS_MAX && + m_in_ports[handle.GetId()].used); +} + +uint32_t Audio::AudioInInput(Id handle, void* dest) { + EXIT_NOT_IMPLEMENTED(!AudioInValid(handle)); + EXIT_NOT_IMPLEMENTED(dest == nullptr); + + const auto& port = m_in_ports[handle.GetId()]; + + uint64_t block_time = (1000000 * port.samples_num) / port.freq; + uint64_t current_time = LibKernel::KernelGetProcessTime(); + + uint64_t next_time = m_in_ports[handle.GetId()].last_input_time + block_time; + uint64_t wait_time = (next_time > current_time ? next_time - current_time : 0); + + // TODO(): Audio input is not yet implemented, so simulate audio delay + Common::Thread::SleepMicro(wait_time); + + m_in_ports[handle.GetId()].last_input_time = LibKernel::KernelGetProcessTime(); + + return port.samples_num; +} + +namespace AudioOut { + +LIB_NAME("AudioOut", "AudioOut"); + +struct AudioOutOutputParam { + int handle; + const void* ptr; +}; + +struct AudioOutPortState { + uint16_t output; + uint8_t channel; + uint8_t reserved1[1]; + int16_t volume; + uint16_t reroute_counter; + uint64_t flag; + uint64_t reserved2[2]; +}; + +int KYTY_SYSV_ABI AudioOutInit() { + PRINT_NAME(); + + return OK; +} + +int KYTY_SYSV_ABI AudioOutOpen(int user_id, int type, int index, uint32_t len, uint32_t freq, + uint32_t param) { + PRINT_NAME(); + + LOGF("\t user_id = %d\n" + "\t type = %d\n" + "\t index = %d\n" + "\t len = %u\n" + "\t freq = %u\n", + user_id, type, index, len, freq); + + if (!audio_out_port_type_is_valid(type)) { + return AUDIO_OUT_ERROR_INVALID_PORT_TYPE; + } + EXIT_NOT_IMPLEMENTED(index != 0); + + Audio::Format format = Audio::Format::Unknown; + const auto format_param = param & AUDIO_OUT_PARAM_FORMAT_MASK; + + switch (format_param) { + case 0: format = Audio::Format::Signed16bitMono; break; + case 1: format = Audio::Format::Signed16bitStereo; break; + case 2: format = Audio::Format::Signed16bit8Ch; break; + case 3: format = Audio::Format::FloatMono; break; + case 4: format = Audio::Format::FloatStereo; break; + case 5: format = Audio::Format::Float8Ch; break; + case 6: format = Audio::Format::Signed16bit8ChStd; break; + case 7: format = Audio::Format::Float8ChStd; break; + default:; + } + + LOGF("\t param = %u (format=%u, %s)\n", param, format_param, + Common::EnumName(format).c_str()); + + EXIT_NOT_IMPLEMENTED(format == Audio::Format::Unknown); + + EXIT_IF(g_audio == nullptr); + + auto id = g_audio->AudioOutOpen(type, len, freq, format); + + if (!id.IsValid()) { + return AUDIO_OUT_ERROR_PORT_FULL; + } + + return id.ToInt(); +} + +int KYTY_SYSV_ABI AudioOutClose(int handle) { + PRINT_NAME(); + + if (!g_audio->AudioOutClose(Audio::Id(handle))) { + return AUDIO_OUT_ERROR_INVALID_PORT; + } + + return OK; +} + +int KYTY_SYSV_ABI AudioOutGetPortState(int handle, AudioOutPortState* state) { + PRINT_NAME(); + + int type = 0; + int channels_num = 0; + + if (!g_audio->AudioOutGetStatus(Audio::Id(handle), &type, &channels_num)) { + return AUDIO_OUT_ERROR_INVALID_PORT; + } + + EXIT_NOT_IMPLEMENTED(state == nullptr); + + state->reroute_counter = 0; + state->volume = 127; + + switch (type) { + case AUDIO_OUT_PORT_TYPE_MAIN: + case AUDIO_OUT_PORT_TYPE_BGM: + case AUDIO_OUT_PORT_TYPE_AUDIO3D: + state->output = 1; + state->channel = (channels_num > 2 ? 2 : channels_num); + break; + case AUDIO_OUT_PORT_TYPE_VOICE: + case AUDIO_OUT_PORT_TYPE_PERSONAL: + state->output = 0x40; + state->channel = 1; + break; + case AUDIO_OUT_PORT_TYPE_PADSPK: + case AUDIO_OUT_PORT_TYPE_VIBRATION: + state->output = 4; + state->channel = 1; + break; + case AUDIO_OUT_PORT_TYPE_AUX: + state->output = 0x80; + state->channel = 0; + break; + default: EXIT("unknown port type: %d\n", type); + } + + LOGF("\t output = %" PRIu16 "\n" + "\t channel = %" PRIu8 "\n", + state->output, state->channel); + + return OK; +} + +int KYTY_SYSV_ABI AudioOutSetVolume(int handle, uint32_t flag, int* vol) { + PRINT_NAME(); + + LOGF("\t handle = %d\n" + "\t flag = %u\n", + handle, flag); + + EXIT_IF(g_audio == nullptr); + EXIT_NOT_IMPLEMENTED(vol == nullptr); + + if (!g_audio->AudioOutSetVolume(Audio::Id(handle), flag, vol)) { + return AUDIO_OUT_ERROR_INVALID_PORT; + } + + return OK; +} + +int KYTY_SYSV_ABI AudioOutOutputs(AudioOutOutputParam* param, uint32_t num) { + PRINT_NAME(); + + EXIT_NOT_IMPLEMENTED(param == nullptr); + + Audio::OutputParam params[Audio::OUT_PORTS_MAX]; + + EXIT_IF(g_audio == nullptr); + + for (uint32_t i = 0; i < num; i++) { + params[i].handle = Audio::Id(param[i].handle); + params[i].data = param[i].ptr; + + if (!g_audio->AudioOutValid(params[i].handle)) { + return AUDIO_OUT_ERROR_INVALID_PORT; + } + } + + return static_cast(g_audio->AudioOutOutputs(params, num)); +} + +int KYTY_SYSV_ABI AudioOutOutput(int handle, const void* ptr) { + // EXIT_NOT_IMPLEMENTED(ptr == nullptr); + + Audio::OutputParam params[1]; + + EXIT_IF(g_audio == nullptr); + + params[0].handle = Audio::Id(handle); + params[0].data = ptr; + + if (!g_audio->AudioOutValid(params[0].handle)) { + return AUDIO_OUT_ERROR_INVALID_PORT; + } + + return static_cast(g_audio->AudioOutOutputs(params, 1)); +} + +} // namespace AudioOut + +namespace AudioIn { + +LIB_NAME("AudioIn", "AudioIn"); + +int KYTY_SYSV_ABI AudioInOpen(int user_id, uint32_t type, uint32_t index, uint32_t len, + uint32_t freq, uint32_t param) { + PRINT_NAME(); + + LOGF("\t user_id = %d\n" + "\t type = %u\n" + "\t index = %d\n" + "\t len = %u\n" + "\t freq = %u\n", + user_id, type, index, len, freq); + + if (user_id != 255 && user_id != 1) { + LOGF("\t temporary: accepting unsupported audio input user_id %d\n", user_id); + } + EXIT_NOT_IMPLEMENTED(type != 1); + EXIT_NOT_IMPLEMENTED(index != 0); + + Audio::Format format = Audio::Format::Unknown; + + switch (param) { + case 0: format = Audio::Format::Signed16bitMono; break; + case 2: format = Audio::Format::Signed16bitStereo; break; + default: + LOGF("\t temporary: using signed 16-bit stereo for unsupported audio input param %u\n", + param); + format = Audio::Format::Signed16bitStereo; + break; + } + + LOGF("\t param = %u (%s)\n", param, Common::EnumName(format).c_str()); + + EXIT_IF(g_audio == nullptr); + + auto id = g_audio->AudioInOpen(type, len, freq, format); + + if (!id.IsValid()) { + return AUDIO_IN_ERROR_PORT_FULL; + } + + return id.ToInt(); +} + +int KYTY_SYSV_ABI AudioInInput(int handle, void* dest) { + PRINT_NAME(); + + EXIT_NOT_IMPLEMENTED(dest == nullptr); + + EXIT_IF(g_audio == nullptr); + + if (!g_audio->AudioInValid(Audio::Id(handle))) { + return AUDIO_IN_ERROR_INVALID_HANDLE; + } + + return static_cast(g_audio->AudioInInput(Audio::Id(handle), dest)); +} + +} // namespace AudioIn + +namespace VoiceQoS { + +LIB_NAME("VoiceQoS", "VoiceQoS"); + +int KYTY_SYSV_ABI VoiceQoSInit(void* mem_block, uint32_t mem_size, int32_t app_type) { + PRINT_NAME(); + + LOGF("\t mem_block = %016" PRIx64 "\n" + "\t mem_size = %" PRIu32 "\n" + "\t app_type = %" PRId32 "\n", + reinterpret_cast(mem_block), mem_size, app_type); + + return OK; +} + +} // namespace VoiceQoS + +namespace Acm { + +LIB_NAME("Acm", "Acm"); + +struct AcmBatchInfo { + void* buffer; + size_t offset; + size_t buffer_size; +}; + +struct AcmBatchError { + uint32_t reserved[8]; +}; + +static std::atomic_uint32_t g_acm_next_context {1}; +static std::atomic_uint32_t g_acm_next_batch {1}; + +static void acm_advance_batch(AcmBatchInfo* info, size_t bytes) { + if (info == nullptr || info->buffer == nullptr || info->buffer_size == 0) { + return; + } + + info->offset = std::min(info->buffer_size, info->offset + bytes); +} + +int KYTY_SYSV_ABI AcmContextCreate(AcmContextId* context) { + PRINT_NAME(); + + EXIT_NOT_IMPLEMENTED(context == nullptr); + + *context = g_acm_next_context.fetch_add(1, std::memory_order_relaxed); + + LOGF("\t context = %" PRIu32 "\n", *context); + + return OK; +} + +int KYTY_SYSV_ABI AcmContextDestroy(AcmContextId context) { + PRINT_NAME(); + LOGF("\t context = %" PRIu32 "\n", context); + return OK; +} + +int KYTY_SYSV_ABI AcmBatchStartBuffer(AcmContextId context, const void* batch_commands, + size_t batch_size, AcmBatchError* batch_error, + AcmBatchId* batch) { + PRINT_NAME(); + + EXIT_NOT_IMPLEMENTED(batch == nullptr); + + if (batch_error != nullptr) { + std::memset(batch_error, 0, sizeof(AcmBatchError)); + } + + *batch = g_acm_next_batch.fetch_add(1, std::memory_order_relaxed); + + return OK; +} + +int KYTY_SYSV_ABI AcmBatchStartBuffers(AcmContextId context, uint32_t batch_info_count, + const AcmBatchInfo* const batch_info[], + AcmBatchError* batch_error, AcmBatchId* batch) { + PRINT_NAME(); + + EXIT_NOT_IMPLEMENTED(batch_info_count != 0 && batch_info == nullptr); + EXIT_NOT_IMPLEMENTED(batch == nullptr); + + if (batch_error != nullptr) { + std::memset(batch_error, 0, sizeof(AcmBatchError)); + } + + *batch = g_acm_next_batch.fetch_add(1, std::memory_order_relaxed); + + return OK; +} + +int KYTY_SYSV_ABI AcmBatchWait(AcmContextId context, AcmBatchId batch, uint32_t timeout) { + return OK; +} + +int KYTY_SYSV_ABI AcmBatchJobNotification(AcmBatchInfo* batch_info) { + PRINT_NAME(); + acm_advance_batch(batch_info, 2 * 16); + return OK; +} + +int KYTY_SYSV_ABI AcmConvReverbSharedInput(AcmBatchInfo* batch_info, uint32_t block_count, void* in, + uint32_t count, const void* const ir[], + const float* gain, void* const out[]) { + PRINT_NAME(); + (void)block_count; + (void)in; + (void)count; + (void)ir; + (void)gain; + (void)out; + acm_advance_batch(batch_info, 1024); + return OK; +} + +int KYTY_SYSV_ABI AcmConvReverbSharedIr(AcmBatchInfo* batch_info, uint32_t block_count, + const void* ir, uint32_t count, void* const in[], + const float* gain, void* const out[]) { + PRINT_NAME(); + (void)block_count; + (void)ir; + (void)count; + (void)in; + (void)gain; + (void)out; + acm_advance_batch(batch_info, 1024); + return OK; +} + +int KYTY_SYSV_ABI AcmFft(AcmBatchInfo* batch_info, int size, int count, int input_format, + const void* const input[], int output_format, void* const output[], + uint32_t flags) { + PRINT_NAME(); + (void)size; + (void)count; + (void)input_format; + (void)input; + (void)output_format; + (void)output; + (void)flags; + acm_advance_batch(batch_info, 256); + return OK; +} + +int KYTY_SYSV_ABI AcmIfft(AcmBatchInfo* batch_info, int size, int count, int input_format, + const void* const input[], int output_format, void* const output[], + uint32_t flags) { + PRINT_NAME(); + (void)size; + (void)count; + (void)input_format; + (void)input; + (void)output_format; + (void)output; + (void)flags; + acm_advance_batch(batch_info, 256); + return OK; +} + +int KYTY_SYSV_ABI AcmPanner(AcmBatchInfo* batch_info, uint32_t in_count, const float* const in[], + uint32_t biquad_count, uint32_t biquad_update_count, uint32_t out_count, + const void* const parameter[], void* const state[], + const float* const out_init[], float* const out[]) { + PRINT_NAME(); + (void)in_count; + (void)in; + (void)biquad_count; + (void)biquad_update_count; + (void)out_count; + (void)parameter; + (void)state; + (void)out_init; + (void)out; + acm_advance_batch(batch_info, 512); + return OK; +} + +} // namespace Acm + +namespace Audio3d { + +LIB_NAME("Audio3d", "Audio3d"); + +namespace Semaphore = LibKernel::Semaphore; + +struct Audio3dOpenParameters { + size_t size = 0x20; + uint32_t granularity = 256; + uint32_t rate = 0; + uint32_t max_objects = 512; + uint32_t queue_depth = 2; + uint32_t buffer_mode = 2; + uint32_t pad = 0; + // uint32_t num_beds; +}; + +struct Audio3dData { + enum class State { Empty, Ready, Play }; + + std::atomic state = State::Empty; +}; + +struct Audio3dInternal { + Audio3dData* data = nullptr; + Common::Mutex* data_mutex = nullptr; + uint64_t data_delay = 0; + Semaphore::KernelSema playback_sema = nullptr; + Audio3dOpenParameters params = {}; + int user_id = 0; + float late_reverb_level = 0.0f; + float downmix_spread_radius = 2.0f; + int downmix_spread_height_aware = 0; + uint32_t data_index = 0; + bool used = false; + std::atomic_bool playback_finished = false; +}; + +constexpr uint32_t MAX_PORTS = 4; + +static Audio3dInternal g_ports[MAX_PORTS] = {}; + +static void playback_simulate(void* arg) { + auto* port = static_cast(arg); + EXIT_IF(port == nullptr); + EXIT_IF(port->data_mutex == nullptr); + EXIT_IF(port->data == nullptr); + + for (;;) { + int result = Semaphore::KernelWaitSema(port->playback_sema, 1, nullptr); + + if (result != OK) { + break; + } + + Audio3dData* play_data = nullptr; + + port->data_mutex->Lock(); + { + for (uint32_t i = 0; i < port->params.queue_depth; i++) { + uint32_t index = (port->data_index + i) % port->params.queue_depth; + + if (port->data[index].state == Audio3dData::State::Play) { + play_data = &port->data[index]; + break; + } + } + } + port->data_mutex->Unlock(); + + EXIT_IF(play_data == nullptr); + + if (play_data != nullptr) { + // TODO(): Audio output is not yet implemented, so simulate audio delay + Common::Thread::SleepMicro(port->data_delay); + play_data->state = Audio3dData::State::Empty; + } + } + + port->playback_finished = true; +} + +int KYTY_SYSV_ABI Audio3dInitialize(int64_t reserved) { + PRINT_NAME(); + + EXIT_NOT_IMPLEMENTED(reserved != 0); + + return OK; +} + +void KYTY_SYSV_ABI Audio3dGetDefaultOpenParameters(Audio3dOpenParameters* p) { + PRINT_NAME(); + + EXIT_NOT_IMPLEMENTED(sizeof(Audio3dOpenParameters) != 0x20); + + *p = Audio3dOpenParameters(); +} + +int KYTY_SYSV_ABI Audio3dPortOpen(int user_id, const Audio3dOpenParameters* parameters, + uint32_t* id) { + PRINT_NAME(); + + EXIT_NOT_IMPLEMENTED(parameters == nullptr); + EXIT_NOT_IMPLEMENTED(id == nullptr); + EXIT_NOT_IMPLEMENTED(parameters->size != 0x20); + + LOGF("\t user_id = %d\n" + "\t granularity = %u\n" + "\t rate = %u\n" + "\t max_objects = %u\n" + "\t queue_depth = %u\n" + "\t buffer_mode = %u\n", + user_id, parameters->granularity, parameters->rate, parameters->max_objects, + parameters->queue_depth, parameters->buffer_mode); + + EXIT_NOT_IMPLEMENTED(parameters->buffer_mode != 2); + EXIT_NOT_IMPLEMENTED(user_id != 255 && user_id != 1); + + uint32_t port = 0; + for (; port < MAX_PORTS; port++) { + if (!g_ports[port].used) { + break; + } + } + + EXIT_NOT_IMPLEMENTED(port >= MAX_PORTS); + + g_ports[port].user_id = user_id; + g_ports[port].params = *parameters; + g_ports[port].used = true; + + EXIT_IF(g_ports[port].data != nullptr); + EXIT_IF(g_ports[port].data_mutex != nullptr); + EXIT_IF(g_ports[port].playback_sema != nullptr); + + g_ports[port].data = new Audio3dData[parameters->queue_depth]; + g_ports[port].data_index = 0; + g_ports[port].data_mutex = new Common::Mutex; + g_ports[port].data_delay = (1000000 * static_cast(parameters->granularity)) / 48000; + + for (uint32_t d = 0; d < parameters->queue_depth; d++) { + g_ports[port].data[d].state = Audio3dData::State::Empty; + } + + int result = Semaphore::KernelCreateSema(&g_ports[port].playback_sema, "audio3d_play", 0x01, 0, + static_cast(parameters->queue_depth), nullptr); + EXIT_NOT_IMPLEMENTED(result != OK); + + g_ports[port].playback_finished = false; + Common::Thread playback_thread(playback_simulate, &g_ports[port]); + playback_thread.Detach(); + + *id = port; + + return OK; +} + +int KYTY_SYSV_ABI Audio3dPortSetAttribute(uint32_t port_id, uint32_t attribute_id, + const void* attribute, size_t attribute_size) { + PRINT_NAME(); + + EXIT_NOT_IMPLEMENTED(port_id >= MAX_PORTS); + EXIT_NOT_IMPLEMENTED(!g_ports[port_id].used); + EXIT_NOT_IMPLEMENTED(attribute == nullptr); + + LOGF("\t attribute_id = 0x%" PRIx32 "\n", attribute_id); + + switch (attribute_id) { + case 0x10001: + EXIT_NOT_IMPLEMENTED(attribute_size != 4); + g_ports[port_id].late_reverb_level = *static_cast(attribute); + LOGF("\t late_reverb_level = %f\n", g_ports[port_id].late_reverb_level); + break; + case 0x10002: + EXIT_NOT_IMPLEMENTED(attribute_size != 4); + g_ports[port_id].downmix_spread_radius = *static_cast(attribute); + LOGF("\t downmix_spread_radius = %f\n", g_ports[port_id].downmix_spread_radius); + break; + case 0x10003: + EXIT_NOT_IMPLEMENTED(attribute_size != 4); + g_ports[port_id].downmix_spread_height_aware = *static_cast(attribute); + LOGF("\t downmix_spread_height_aware = %d\n", + g_ports[port_id].downmix_spread_height_aware); + break; + default: EXIT("unknown attribute: 0x%" PRIx32 "\n", attribute_id); + } + + return OK; +} + +int KYTY_SYSV_ABI Audio3dPortGetQueueLevel(uint32_t port_id, uint32_t* queue_level, + uint32_t* queue_available) { + PRINT_NAME(); + + EXIT_NOT_IMPLEMENTED(port_id >= MAX_PORTS); + EXIT_NOT_IMPLEMENTED(!g_ports[port_id].used); + EXIT_NOT_IMPLEMENTED(queue_level == nullptr && queue_available == nullptr); + + auto* port = &g_ports[port_id]; + + uint32_t empty_num = 0; + + port->data_mutex->Lock(); + { + for (uint32_t i = 0; i < port->params.queue_depth; i++) { + uint32_t index = (port->data_index + i) % port->params.queue_depth; + + if (port->data[index].state == Audio3dData::State::Empty) { + empty_num++; + } else { + break; + } + } + } + port->data_mutex->Unlock(); + + EXIT_IF(empty_num > port->params.queue_depth); + + LOGF("\t queue_available = %u\n", empty_num); + + if (queue_level != nullptr) { + *queue_level = port->params.queue_depth - empty_num; + } + if (queue_available != nullptr) { + *queue_available = empty_num; + } + + return OK; +} + +int KYTY_SYSV_ABI Audio3dPortAdvance(uint32_t port_id) { + PRINT_NAME(); + + EXIT_NOT_IMPLEMENTED(port_id >= MAX_PORTS); + EXIT_NOT_IMPLEMENTED(!g_ports[port_id].used); + + auto* port = &g_ports[port_id]; + + port->data_mutex->Lock(); + { + uint32_t current_index = port->data_index; + uint32_t next_index = (current_index + 1) % port->params.queue_depth; + + if (port->data[current_index].state == Audio3dData::State::Empty) { + port->data[current_index].state = Audio3dData::State::Ready; + } + + EXIT_NOT_IMPLEMENTED(port->data[current_index].state != Audio3dData::State::Ready); + + port->data_index = next_index; + + LOGF("\t %u -> %u\n", current_index, next_index); + } + port->data_mutex->Unlock(); + + return OK; +} + +int KYTY_SYSV_ABI Audio3dPortPush(uint32_t port_id, uint32_t blocking) { + PRINT_NAME(); + + EXIT_NOT_IMPLEMENTED(port_id >= MAX_PORTS); + EXIT_NOT_IMPLEMENTED(!g_ports[port_id].used); + + auto* port = &g_ports[port_id]; + + EXIT_NOT_IMPLEMENTED(blocking != 1); + + LOGF("\t blocking = %u\n", blocking); + + int data_num = 0; + Audio3dData* first_data = nullptr; + + port->data_mutex->Lock(); + { + first_data = port->data + port->data_index; + + for (uint32_t i = 0; i < port->params.queue_depth; i++) { + uint32_t index = (port->data_index + i) % port->params.queue_depth; + + if (port->data[index].state == Audio3dData::State::Ready) { + port->data[index].state = Audio3dData::State::Play; + data_num++; + } + } + } + port->data_mutex->Unlock(); + + LOGF("\t push num = %d\n", data_num); + + if (data_num > 0) { + Semaphore::KernelSignalSema(port->playback_sema, data_num); + + if (blocking == 1) { + auto wait_time = port->data_delay / 8; + while (first_data->state != Audio3dData::State::Empty) { + Common::Thread::SleepMicro(wait_time); + } + } + } + + return OK; +} + +} // namespace Audio3d + +namespace Ngs2 { + +LIB_NAME("Ngs2", "Ngs2"); + +struct Ngs2SystemOption { + size_t size = 0; + char name[64] = {}; + uintptr_t job_scheduler_options[4] = {}; + uint32_t flags = 0; + uint32_t max_grain_samples = 0; + uint32_t num_grain_samples = 0; + uint32_t sample_rate = 0; + uint32_t max_voice_channels = 0; + uint32_t reserved[5] = {}; +}; + +struct Ngs2RackOption { + size_t size = 0; + char name[64] = {}; + uint32_t flags = 0; + uint32_t max_grain_samples = 0; + uint32_t max_voices = 0; + uint32_t max_input_delay_blocks = 0; + uint32_t max_matrices = 0; + uint32_t max_ports = 0; + uint32_t max_voice_channels = 0; + uint32_t max_output_channels = 0; + uint32_t reserved[18] = {}; +}; + +struct Ngs2MasteringRackOption { + Ngs2RackOption rack_option; + uint32_t max_channels = 0; + uint32_t num_peak_meter_blocks = 0; +}; + +struct Ngs2SubmixerRackOption { + Ngs2RackOption rack_option; + uint32_t max_channels = 0; + uint32_t max_envelope_points = 0; + uint32_t max_filters = 0; + uint32_t max_inputs = 0; + uint32_t num_peak_meter_blocks = 0; +}; + +struct Ngs2SamplerRackOption { + Ngs2RackOption rack_option; + uint32_t max_channel_works = 0; + uint32_t max_codec_caches = 0; + uint32_t max_waveform_blocks = 0; + uint32_t max_envelope_points = 0; + uint32_t max_filters = 0; + uint32_t max_atrac9_decoders = 0; + uint32_t max_atrac9_channel_works = 0; + uint32_t max_ajm_atrac9_decoders = 0; + uint32_t num_peak_meter_blocks = 0; +}; + +struct Ngs2ReverbRackOption { + Ngs2RackOption rack_option; + uint32_t max_channels = 0; + uint32_t reverb_size = 0; +}; + +struct Ngs2CustomModuleOption { + uint32_t size = 0; +}; + +struct Ngs2CustomRackModuleInfo { + const Ngs2CustomModuleOption* option = nullptr; + uint32_t module_id = 0; + uint32_t source_buffer_id = 0; + uint32_t extra_buffer_id = 0; + uint32_t dest_buffer_id = 0; + uint32_t state_offset = 0; + uint32_t state_size = 0; + uint32_t reserved = 0; + uint32_t reserved2 = 0; +}; + +struct Ngs2CustomRackPortInfo { + uint32_t source_buffer_id = 0; + uint32_t reserved = 0; +}; + +struct Ngs2CustomRackOption { + Ngs2RackOption rack_option; + uint32_t state_size = 0; + uint32_t num_buffers = 0; + uint32_t num_modules = 0; + uint32_t reserved = 0; + Ngs2CustomRackModuleInfo module[24]; + Ngs2CustomRackPortInfo port[16]; +}; + +struct Ngs2CustomSubmixerRackOption { + Ngs2CustomRackOption custom_rack_option; + uint32_t max_channels = 0; + uint32_t max_inputs = 0; +}; + +struct Ngs2CustomSamplerRackOption { + Ngs2CustomRackOption custom_rack_option; + uint32_t max_channel_works = 0; + uint32_t max_waveform_blocks = 0; + uint32_t max_atrac9_decoders = 0; + uint32_t max_atrac9_channel_works = 0; + uint32_t max_ajm_atrac9_decoders = 0; + uint32_t max_codec_caches = 0; +}; + +union Ngs2RackOptionUnion { + Ngs2RackOption common; + Ngs2SamplerRackOption sampler; + Ngs2MasteringRackOption mastering; + Ngs2SubmixerRackOption submixer; + Ngs2ReverbRackOption reverb; + Ngs2CustomSubmixerRackOption custom_submixer; + Ngs2CustomSamplerRackOption custom_sampler; +}; + +struct Ngs2ContextBufferInfo { + void* host_buffer = nullptr; + size_t host_buffer_size = 0; + uintptr_t reserved[5] = {}; + uintptr_t user_data = 0; +}; + +struct Ngs2RenderBufferInfo { + void* buffer = nullptr; + size_t buffer_size = 0; + uint32_t waveform_type = 0; + uint32_t num_channels = 0; +}; + +struct Ngs2WaveformFormat { + uint32_t waveform_type = 0; + uint32_t num_channels = 0; + uint32_t sample_rate = 0; + uint32_t config_data = 0; + uint32_t frame_margin = 0; + uint32_t frame_offset = 0; +}; + +struct Ngs2WaveformBlock { + uintptr_t data_offset = 0; + size_t data_size = 0; + uint32_t num_repeats = 0; + uint32_t num_skip_samples = 0; + uint32_t num_samples = 0; + uint32_t reserved = 0; + uintptr_t user_data = 0; +}; + +struct Ngs2WaveformInfo { + Ngs2WaveformFormat format; + uint32_t data_offset = 0; + uint32_t data_size = 0; + uint32_t loop_begin_position = 0; + uint32_t loop_end_position = 0; + uint32_t num_samples = 0; + uint32_t audio_unit_size = 0; + uint32_t num_audio_unit_samples = 0; + uint32_t num_audio_unit_per_frame = 0; + uint32_t audio_frame_size = 0; + uint32_t num_audio_frame_samples = 0; + uint32_t num_delay_samples = 0; + uint32_t num_blocks = 0; + Ngs2WaveformBlock blocks[4]; +}; + +struct Ngs2PanParam { + float angle = 0.0f; + float distance = 0.0f; + float fbw_level = 0.0f; + float lfe_level = 0.0f; +}; + +struct Ngs2PanWork { + float speaker_angles[8] = {}; + float unit_angle = 0.0f; + uint32_t num_speakers = 0; +}; + +struct Ngs2GeomVector { + float x = 0.0f; + float y = 0.0f; + float z = 0.0f; +}; + +struct Ngs2GeomCone { + float inner_level = 0.0f; + float inner_angle = 0.0f; + float outer_level = 0.0f; + float outer_angle = 0.0f; +}; + +struct Ngs2GeomRolloff { + uint32_t model = 0; + float max_distance = 0.0f; + float rolloff_factor = 0.0f; + float reference_distance = 0.0f; +}; + +struct Ngs2GeomListenerParam { + Ngs2GeomVector position; + Ngs2GeomVector orient_front; + Ngs2GeomVector orient_up; + Ngs2GeomVector velocity; + float sound_speed = 0.0f; + uint32_t reserved[2] = {}; +}; + +struct Ngs2GeomListenerWork { + float matrix[4][4] = {}; + Ngs2GeomVector velocity; + float sound_speed = 0.0f; + uint32_t coordinate = 0; + uint32_t reserved[3] = {}; +}; + +struct Ngs2GeomSourceParam { + Ngs2GeomVector position; + Ngs2GeomVector velocity; + Ngs2GeomVector direction; + Ngs2GeomCone cone; + Ngs2GeomRolloff rolloff; + float doppler_factor = 0.0f; + float fbw_level = 0.0f; + float lfe_level = 0.0f; + float max_level = 0.0f; + float min_level = 0.0f; + float radius = 0.0f; + uint32_t num_speakers = 0; + uint32_t matrix_format = 0; + uint32_t reserved[2] = {}; +}; + +struct Ngs2GeomA3dAttribute { + Ngs2GeomVector position; + float volume = 0.0f; + uint32_t reserved[4] = {}; +}; + +struct Ngs2GeomAttribute { + float pitch_ratio = 0.0f; + float level[64] = {}; + Ngs2GeomA3dAttribute a3d_attrib; + uint32_t reserved[4] = {}; +}; + +using Ngs2BufferAllocHandler = int32_t KYTY_SYSV_ABI (*)(Ngs2ContextBufferInfo*); +using Ngs2BufferFreeHandler = int32_t KYTY_SYSV_ABI (*)(Ngs2ContextBufferInfo*); + +struct Ngs2BufferAllocator { + Ngs2BufferAllocHandler alloc_handler = nullptr; + Ngs2BufferFreeHandler free_handler = nullptr; + uintptr_t user_data = 0; +}; + +struct Ngs2Internal { + Ngs2SystemOption option; + Ngs2BufferAllocator allocator; + Ngs2Internal* next = nullptr; + Common::Mutex mutex; +}; + +enum class Ngs2RackType { + Sampler, + Submixer, + Mastering, + Reverb, + CustomSubmixer, + CustomSampler, +}; + +struct Ngs2RackInternal { + Ngs2Internal* ngs = nullptr; + Ngs2RackInternal* next = nullptr; + Ngs2RackType type = Ngs2RackType::Sampler; + Ngs2RackOptionUnion option; + Ngs2BufferAllocator allocator; +}; + +enum class Ngs2VoicePlayState { Empty, Playing, Paused, Stopped }; + +enum class Ngs2VoicePlayEvent { None, Play, Pause, Resume, Stop, StopImm, Kill }; + +struct Ngs2VoiceInternal { + Ngs2VoicePlayEvent event = Ngs2VoicePlayEvent::None; + Ngs2VoicePlayState state = Ngs2VoicePlayState::Empty; + Ngs2RackInternal* rack = nullptr; + uintptr_t callback = 0; + uintptr_t callback_data = 0; + uint32_t callback_flags = 0; +}; + +struct Ngs2VoiceParamHeader { + uint16_t size; + int16_t next; + uint32_t id; +}; + +struct Ngs2VoiceEventParam { + Ngs2VoiceParamHeader header; + uint32_t event_id; +}; + +struct Ngs2VoicePatchParam { + Ngs2VoiceParamHeader header; + uint32_t port; + uint32_t dest_input_id; + uintptr_t dest_handle; +}; + +struct Ngs2VoiceMatrixLevelsParam { + Ngs2VoiceParamHeader header; + uint32_t matrix_id; + uint32_t num_levels; + const float* levels; +}; + +struct Ngs2VoicePortMatrixParam { + Ngs2VoiceParamHeader header; + uint32_t port; + int32_t matrix_id; +}; + +struct Ngs2VoicePortVolumeParam { + Ngs2VoiceParamHeader header; + uint32_t port; + float level; +}; + +struct Ngs2VoicePortDelayParam { + Ngs2VoiceParamHeader header; + uint32_t port; + uint32_t num_samples; +}; + +struct Ngs2VoiceCallbackParam { + Ngs2VoiceParamHeader header; + uintptr_t callback; + uintptr_t callback_data; + uint32_t flags; + uint32_t reserved; +}; + +struct Ngs2VoiceState { + uint32_t state_flags; +}; + +struct Ngs2SamplerVoiceState { + Ngs2VoiceState voice_state; + float envelope_height; + float peak_height; + uint32_t reserved; + uint64_t num_decoded_samples; + uint64_t decoded_data_size; + uint64_t user_data; + const void* waveform_data; +}; + +static Ngs2Internal* g_ngs_list = nullptr; +static Ngs2RackInternal* g_racks_list = nullptr; + +static_assert(sizeof(Ngs2SystemOption) == 144); +static_assert(sizeof(Ngs2RackOption) == 176); + +static uint32_t Ngs2GetStateFlags(const Ngs2VoiceInternal* voice) { + switch (voice->state) { + case Ngs2VoicePlayState::Empty: return 0; + case Ngs2VoicePlayState::Playing: return 0x3; + case Ngs2VoicePlayState::Paused: return 0x5; + case Ngs2VoicePlayState::Stopped: return 0xb; + } + + return 0; +} + +static Ngs2SystemOption Ngs2DefaultSystemOption() { + Ngs2SystemOption option {}; + option.size = sizeof(Ngs2SystemOption); + option.max_grain_samples = 512; + option.num_grain_samples = 256; + option.sample_rate = 48000; + return option; +} + +int KYTY_SYSV_ABI Ngs2SystemResetOption(Ngs2SystemOption* option) { + PRINT_NAME(); + + EXIT_NOT_IMPLEMENTED(option == nullptr); + + *option = Ngs2DefaultSystemOption(); + return OK; +} + +static Ngs2Internal* Ngs2CreateSystemInternal(const Ngs2SystemOption* option, void* host_buffer) { + auto* ngs = new (host_buffer) Ngs2Internal; + + ngs->option = *option; + ngs->next = g_ngs_list; + g_ngs_list = ngs; + + return ngs; +} + +static bool Ngs2RackIsCustom(Ngs2RackType type) { + switch (type) { + case Ngs2RackType::CustomSubmixer: + case Ngs2RackType::CustomSampler: return true; + default: return false; + } +} + +int KYTY_SYSV_ABI Ngs2SystemQueryBufferSize(const Ngs2SystemOption* option, + Ngs2ContextBufferInfo* buffer_info) { + PRINT_NAME(); + + EXIT_NOT_IMPLEMENTED(buffer_info == nullptr); + + auto default_option = Ngs2DefaultSystemOption(); + if (option == nullptr) { + option = &default_option; + LOGF("\t option = nullptr, using reset defaults\n"); + } + + EXIT_NOT_IMPLEMENTED(option->size != sizeof(Ngs2SystemOption)); + + std::memset(buffer_info, 0, sizeof(Ngs2ContextBufferInfo)); + buffer_info->host_buffer_size = sizeof(Ngs2Internal); + + return OK; +} + +int KYTY_SYSV_ABI Ngs2SystemCreate(const Ngs2SystemOption* option, + const Ngs2ContextBufferInfo* buffer_info, uintptr_t* handle) { + PRINT_NAME(); + + EXIT_NOT_IMPLEMENTED(buffer_info == nullptr); + EXIT_NOT_IMPLEMENTED(handle == nullptr); + EXIT_NOT_IMPLEMENTED(buffer_info->host_buffer == nullptr); + EXIT_NOT_IMPLEMENTED(buffer_info->host_buffer_size < sizeof(Ngs2Internal)); + + auto default_option = Ngs2DefaultSystemOption(); + if (option == nullptr) { + option = &default_option; + LOGF("\t option = nullptr, using reset defaults\n"); + } + + EXIT_NOT_IMPLEMENTED(option->size != sizeof(Ngs2SystemOption)); + + auto* ngs = Ngs2CreateSystemInternal(option, buffer_info->host_buffer); + + *handle = reinterpret_cast(ngs); + + return OK; +} + +static void Ngs2FillDefaultRackOption(uint32_t rack_id, Ngs2RackOptionUnion* option) { + EXIT_NOT_IMPLEMENTED(option == nullptr); + + *option = {}; + + switch (rack_id) { + case 0x1000: + option->sampler.rack_option.size = sizeof(Ngs2SamplerRackOption); + option->sampler.rack_option.max_grain_samples = 512; + option->sampler.rack_option.max_voices = 256; + option->sampler.rack_option.max_input_delay_blocks = 0; + option->sampler.rack_option.max_matrices = 1; + option->sampler.rack_option.max_ports = 8; + option->sampler.max_channel_works = 256; + option->sampler.max_codec_caches = 32; + option->sampler.max_waveform_blocks = 4; + option->sampler.max_envelope_points = 4; + option->sampler.max_filters = 8; + option->sampler.max_atrac9_decoders = 256; + option->sampler.max_atrac9_channel_works = 256; + option->sampler.max_ajm_atrac9_decoders = 0; + option->sampler.num_peak_meter_blocks = 8; + break; + case 0x2000: + option->submixer.rack_option.size = sizeof(Ngs2SubmixerRackOption); + option->submixer.rack_option.max_grain_samples = 512; + option->submixer.rack_option.max_voices = 1; + option->submixer.rack_option.max_input_delay_blocks = 1; + option->submixer.rack_option.max_matrices = 1; + option->submixer.rack_option.max_ports = 8; + option->submixer.max_channels = 8; + option->submixer.max_envelope_points = 4; + option->submixer.max_filters = 8; + option->submixer.max_inputs = 1; + option->submixer.num_peak_meter_blocks = 8; + break; + case 0x2001: + option->reverb.rack_option.size = sizeof(Ngs2ReverbRackOption); + option->reverb.rack_option.max_grain_samples = 512; + option->reverb.rack_option.max_voices = 1; + option->reverb.rack_option.max_input_delay_blocks = 1; + option->reverb.rack_option.max_matrices = 1; + option->reverb.rack_option.max_ports = 8; + option->reverb.max_channels = 8; + option->reverb.reverb_size = 1; + break; + case 0x3000: + option->mastering.rack_option.size = sizeof(Ngs2MasteringRackOption); + option->mastering.rack_option.max_grain_samples = 512; + option->mastering.rack_option.max_voices = 1; + option->mastering.rack_option.max_input_delay_blocks = 1; + option->mastering.rack_option.max_matrices = 0; + option->mastering.rack_option.max_ports = 0; + option->mastering.max_channels = 8; + option->mastering.num_peak_meter_blocks = 8; + break; + case 0x4002: + // FIXME: Temporary PS5 progress fallback. This mirrors Prospero reset helper's + // common custom-submixer defaults, but the custom module internals are still stubbed. + option->custom_submixer.custom_rack_option.rack_option.size = + sizeof(Ngs2CustomSubmixerRackOption); + option->custom_submixer.custom_rack_option.rack_option.max_grain_samples = 512; + option->custom_submixer.custom_rack_option.rack_option.max_voices = 1; + option->custom_submixer.custom_rack_option.rack_option.max_input_delay_blocks = 1; + option->custom_submixer.custom_rack_option.rack_option.max_matrices = 1; + option->custom_submixer.custom_rack_option.rack_option.max_ports = 8; + option->custom_submixer.custom_rack_option.num_buffers = 1; + option->custom_submixer.max_channels = 8; + option->custom_submixer.max_inputs = 1; + break; + case 0x4001: + option->custom_sampler.custom_rack_option.rack_option.size = + sizeof(Ngs2CustomSamplerRackOption); + option->custom_sampler.custom_rack_option.rack_option.max_grain_samples = 512; + option->custom_sampler.custom_rack_option.rack_option.max_voices = 256; + option->custom_sampler.custom_rack_option.rack_option.max_input_delay_blocks = 0; + option->custom_sampler.custom_rack_option.rack_option.max_matrices = 1; + option->custom_sampler.custom_rack_option.rack_option.max_ports = 8; + option->custom_sampler.custom_rack_option.num_buffers = 1; + option->custom_sampler.max_channel_works = 256; + option->custom_sampler.max_waveform_blocks = 4; + option->custom_sampler.max_atrac9_decoders = 256; + option->custom_sampler.max_atrac9_channel_works = 256; + option->custom_sampler.max_ajm_atrac9_decoders = 0; + option->custom_sampler.max_codec_caches = 32; + break; + default: EXIT("unknown rack_id for default option: 0x%" PRIx32 "\n", rack_id); + } +} + +int KYTY_SYSV_ABI Ngs2RackQueryBufferSize(uint32_t rack_id, const Ngs2RackOption* option, + Ngs2ContextBufferInfo* buffer_info) { + PRINT_NAME(); + + EXIT_NOT_IMPLEMENTED(buffer_info == nullptr); + + Ngs2RackOptionUnion default_option {}; + if (option == nullptr) { + Ngs2FillDefaultRackOption(rack_id, &default_option); + option = &default_option.common; + LOGF("\t option = nullptr, using reset defaults for rack_id 0x%" PRIx32 "\n", rack_id); + } + + LOGF("\t rack_id = 0x%" PRIx32 "\n" + "\t max_voices = %u\n", + rack_id, option->max_voices); + + buffer_info->host_buffer_size = + sizeof(Ngs2RackInternal) + sizeof(Ngs2VoiceInternal) * option->max_voices; + + return OK; +} + +int KYTY_SYSV_ABI Ngs2SystemCreateWithAllocator(const Ngs2SystemOption* option, + const Ngs2BufferAllocator* allocator, + uintptr_t* handle) { + PRINT_NAME(); + + EXIT_NOT_IMPLEMENTED(allocator == nullptr); + EXIT_NOT_IMPLEMENTED(handle == nullptr); + EXIT_NOT_IMPLEMENTED(allocator->alloc_handler == nullptr); + EXIT_NOT_IMPLEMENTED(allocator->free_handler == nullptr); + + auto default_option = Ngs2DefaultSystemOption(); + if (option == nullptr) { + option = &default_option; + LOGF("\t option = nullptr, using reset defaults\n"); + } + + EXIT_NOT_IMPLEMENTED(option->size != sizeof(Ngs2SystemOption)); + + LOGF("\t name = %.64s\n" + "\t flags = %u\n" + "\t max_grain_samples = %u\n" + "\t num_grain_samples = %u\n" + "\t sample_rate = %u\n" + "\t max_voice_channels = %u\n" + "\t alloc_handler = 0x%016" PRIx64 "\n" + "\t free_handler = 0x%016" PRIx64 "\n" + "\t user_data = 0x%016" PRIx64 "\n", + option->name, option->flags, option->max_grain_samples, option->num_grain_samples, + option->sample_rate, option->max_voice_channels, + reinterpret_cast(allocator->alloc_handler), + reinterpret_cast(allocator->free_handler), + reinterpret_cast(allocator->user_data)); + + Ngs2ContextBufferInfo buf {}; + buf.host_buffer = nullptr; + buf.host_buffer_size = sizeof(Ngs2Internal); + buf.user_data = allocator->user_data; + + int result = allocator->alloc_handler(&buf); + + EXIT_NOT_IMPLEMENTED(result != OK); + EXIT_NOT_IMPLEMENTED(buf.host_buffer == nullptr); + + auto* ngs = Ngs2CreateSystemInternal(option, buf.host_buffer); + ngs->allocator = *allocator; + + *handle = reinterpret_cast(ngs); + + return OK; +} + +int KYTY_SYSV_ABI Ngs2SystemSetGrainSamples(uintptr_t system_handle, uint32_t num_samples) { + PRINT_NAME(); + LOGF("\t system_handle = 0x%016" PRIx64 "\n" + "\t num_samples = %u\n", + static_cast(system_handle), num_samples); + + EXIT_NOT_IMPLEMENTED(system_handle == 0); + + auto* ngs = reinterpret_cast(system_handle); + ngs->option.num_grain_samples = num_samples; + + return OK; +} + +int KYTY_SYSV_ABI Ngs2SystemDestroy(uintptr_t system_handle, Ngs2ContextBufferInfo* buffer_info) { + PRINT_NAME(); + LOGF("\t system_handle = 0x%016" PRIx64 "\n", static_cast(system_handle)); + + if (buffer_info != nullptr) { + std::memset(buffer_info, 0, sizeof(Ngs2ContextBufferInfo)); + } + + return OK; +} + +int KYTY_SYSV_ABI Ngs2RackCreate(uintptr_t system_handle, uint32_t rack_id, + const Ngs2RackOption* option, + const Ngs2ContextBufferInfo* buffer_info, uintptr_t* handle) { + PRINT_NAME(); + + EXIT_NOT_IMPLEMENTED(buffer_info == nullptr); + EXIT_NOT_IMPLEMENTED(handle == nullptr); + EXIT_NOT_IMPLEMENTED(buffer_info->host_buffer == nullptr); + EXIT_NOT_IMPLEMENTED(buffer_info->host_buffer_size == 0); + EXIT_NOT_IMPLEMENTED(system_handle == 0); + + Ngs2RackOptionUnion default_option {}; + if (option == nullptr) { + Ngs2FillDefaultRackOption(rack_id, &default_option); + option = &default_option.common; + LOGF("\t option = nullptr, using reset defaults for rack_id 0x%" PRIx32 + "\n", + rack_id); + } + + EXIT_NOT_IMPLEMENTED(option->size < sizeof(Ngs2RackOption)); + + LOGF("\t rack_id = 0x%" PRIx32 "\n" + "\t name = %.64s\n" + "\t flags = %u\n" + "\t max_grain_samples = %u\n" + "\t max_voices = %u\n" + "\t max_input_delay_blocks = %u\n" + "\t max_matrices = %u\n" + "\t max_ports = %u\n" + "\t max_voice_channels = %u\n" + "\t max_output_channels = %u\n" + "\t host_buffer = 0x%016" PRIx64 "\n" + "\t host_buffer_size = 0x%016" PRIx64 "\n", + rack_id, option->name, option->flags, option->max_grain_samples, option->max_voices, + option->max_input_delay_blocks, option->max_matrices, option->max_ports, + option->max_voice_channels, option->max_output_channels, + reinterpret_cast(buffer_info->host_buffer), + reinterpret_cast(buffer_info->host_buffer_size)); + + auto* ngs = reinterpret_cast(system_handle); + auto* rack = static_cast(buffer_info->host_buffer); + auto* voices = reinterpret_cast(rack + 1); + + Common::LockGuard lock(ngs->mutex); + + switch (rack_id) { + case 0x1000: + EXIT_NOT_IMPLEMENTED(option->size != sizeof(Ngs2SamplerRackOption)); + rack->option.sampler = *reinterpret_cast(option); + rack->type = Ngs2RackType::Sampler; + break; + case 0x2000: + EXIT_NOT_IMPLEMENTED(option->size != sizeof(Ngs2SubmixerRackOption)); + rack->option.submixer = *reinterpret_cast(option); + rack->type = Ngs2RackType::Submixer; + break; + case 0x2001: + EXIT_NOT_IMPLEMENTED(option->size != sizeof(Ngs2ReverbRackOption)); + rack->option.reverb = *reinterpret_cast(option); + rack->type = Ngs2RackType::Reverb; + break; + case 0x3000: + EXIT_NOT_IMPLEMENTED(option->size != sizeof(Ngs2MasteringRackOption)); + rack->option.mastering = *reinterpret_cast(option); + rack->type = Ngs2RackType::Mastering; + break; + case 0x4002: + EXIT_NOT_IMPLEMENTED(option->size != sizeof(Ngs2CustomSubmixerRackOption)); + rack->option.custom_submixer = + *reinterpret_cast(option); + rack->type = Ngs2RackType::CustomSubmixer; + break; + case 0x4001: + EXIT_NOT_IMPLEMENTED(option->size != sizeof(Ngs2CustomSamplerRackOption)); + rack->option.custom_sampler = + *reinterpret_cast(option); + rack->type = Ngs2RackType::CustomSampler; + break; + default: EXIT("unknown rack_id: 0x%" PRIx32 "\n", rack_id); + } + + LOGF("\t type = %s\n", Common::EnumName(rack->type).c_str()); + + rack->allocator = Ngs2BufferAllocator(); + rack->ngs = ngs; + + rack->next = g_racks_list; + g_racks_list = rack; + + for (uint32_t i = 0; i < option->max_voices; i++) { + voices[i].rack = rack; + voices[i].event = Ngs2VoicePlayEvent::None; + voices[i].state = Ngs2VoicePlayState::Empty; + } + + *handle = reinterpret_cast(rack); + + return OK; +} + +int KYTY_SYSV_ABI Ngs2RackCreateWithAllocator(uintptr_t system_handle, uint32_t rack_id, + const Ngs2RackOption* option, + const Ngs2BufferAllocator* allocator, + uintptr_t* handle) { + PRINT_NAME(); + + EXIT_NOT_IMPLEMENTED(allocator == nullptr); + EXIT_NOT_IMPLEMENTED(handle == nullptr); + EXIT_NOT_IMPLEMENTED(allocator->alloc_handler == nullptr); + EXIT_NOT_IMPLEMENTED(allocator->free_handler == nullptr); + EXIT_NOT_IMPLEMENTED(system_handle == 0); + + Ngs2RackOptionUnion default_option {}; + if (option == nullptr) { + Ngs2FillDefaultRackOption(rack_id, &default_option); + option = &default_option.common; + LOGF("\t option = nullptr, using reset defaults for rack_id 0x%" PRIx32 + "\n", + rack_id); + } + + EXIT_NOT_IMPLEMENTED(option->size < sizeof(Ngs2RackOption)); + + LOGF("\t rack_id = 0x%" PRIx32 "\n" + "\t name = %.64s\n" + "\t flags = %u\n" + "\t max_grain_samples = %u\n" + "\t max_voices = %u\n" + "\t max_input_delay_blocks = %u\n" + "\t max_matrices = %u\n" + "\t max_ports = %u\n" + "\t max_voice_channels = %u\n" + "\t max_output_channels = %u\n" + "\t alloc_handler = 0x%016" PRIx64 "\n" + "\t free_handler = 0x%016" PRIx64 "\n" + "\t user_data = 0x%016" PRIx64 "\n", + rack_id, option->name, option->flags, option->max_grain_samples, option->max_voices, + option->max_input_delay_blocks, option->max_matrices, option->max_ports, + option->max_voice_channels, option->max_output_channels, + reinterpret_cast(allocator->alloc_handler), + reinterpret_cast(allocator->free_handler), + reinterpret_cast(allocator->user_data)); + + Ngs2ContextBufferInfo buf {}; + buf.host_buffer = nullptr; + buf.host_buffer_size = 0; + buf.user_data = allocator->user_data; + + Ngs2RackQueryBufferSize(rack_id, option, &buf); + + EXIT_NOT_IMPLEMENTED(buf.host_buffer_size == 0); + + int result = allocator->alloc_handler(&buf); + + EXIT_NOT_IMPLEMENTED(result != OK); + EXIT_NOT_IMPLEMENTED(buf.host_buffer == nullptr); + + result = Ngs2RackCreate(system_handle, rack_id, option, &buf, handle); + + if (result == OK) { + auto* rack = static_cast(buf.host_buffer); + rack->allocator = *allocator; + } + + return result; +} + +int KYTY_SYSV_ABI Ngs2RackDestroy(uintptr_t rack_handle, Ngs2ContextBufferInfo* buffer_info) { + PRINT_NAME(); + LOGF("\t rack_handle = 0x%016" PRIx64 "\n", static_cast(rack_handle)); + + if (buffer_info != nullptr) { + std::memset(buffer_info, 0, sizeof(Ngs2ContextBufferInfo)); + } + + return OK; +} + +int KYTY_SYSV_ABI Ngs2RackLock(uintptr_t rack_handle) { + PRINT_NAME(); + LOGF("\t rack_handle = 0x%016" PRIx64 "\n", static_cast(rack_handle)); + + EXIT_NOT_IMPLEMENTED(rack_handle == 0); + + auto* rack = reinterpret_cast(rack_handle); + + EXIT_NOT_IMPLEMENTED(rack->ngs == nullptr); + + rack->ngs->mutex.Lock(); + + return OK; +} + +int KYTY_SYSV_ABI Ngs2RackUnlock(uintptr_t rack_handle) { + PRINT_NAME(); + LOGF("\t rack_handle = 0x%016" PRIx64 "\n", static_cast(rack_handle)); + + EXIT_NOT_IMPLEMENTED(rack_handle == 0); + + auto* rack = reinterpret_cast(rack_handle); + + EXIT_NOT_IMPLEMENTED(rack->ngs == nullptr); + + rack->ngs->mutex.Unlock(); + + return OK; +} + +int KYTY_SYSV_ABI Ngs2SystemRender(uintptr_t system_handle, const Ngs2RenderBufferInfo* buffer_info, + uint32_t num_buffer_info) { + static std::atomic_uint32_t render_log_count = 0; + const auto log_index = render_log_count.fetch_add(1, std::memory_order_relaxed); + const bool log_this_call = (log_index < 16 || (log_index % 600) == 0); + + if (log_this_call) { + PRINT_NAME(); + LOGF("\t call_count = %" PRIu32 "\n", log_index + 1); + } + + EXIT_NOT_IMPLEMENTED(buffer_info == nullptr); + EXIT_NOT_IMPLEMENTED(system_handle == 0); + EXIT_NOT_IMPLEMENTED(num_buffer_info == 0); + + auto* ngs = reinterpret_cast(system_handle); + + Common::LockGuard lock(ngs->mutex); + + for (uint32_t i = 0; i < num_buffer_info; i++) { + if (buffer_info[i].buffer != nullptr && buffer_info[i].buffer_size != 0) { + std::memset(buffer_info[i].buffer, 0, buffer_info[i].buffer_size); + } + } + + for (auto* rack = g_racks_list; rack != nullptr; rack = rack->next) { + if (rack->ngs == ngs) { + auto* voices = reinterpret_cast(rack + 1); + + for (uint32_t i = 0; i < rack->option.common.max_voices; i++) { + auto& voice = voices[i]; + switch (voice.event) { + case Ngs2VoicePlayEvent::None: + if (voice.state == Ngs2VoicePlayState::Playing || + voice.state == Ngs2VoicePlayState::Stopped) { + voice.state = Ngs2VoicePlayState::Empty; + } + break; + case Ngs2VoicePlayEvent::Play: + if (voice.state == Ngs2VoicePlayState::Empty) { + voice.state = Ngs2VoicePlayState::Playing; + } + break; + case Ngs2VoicePlayEvent::Pause: + if (voice.state == Ngs2VoicePlayState::Playing) { + voice.state = Ngs2VoicePlayState::Paused; + } + break; + case Ngs2VoicePlayEvent::Resume: + if (voice.state == Ngs2VoicePlayState::Paused) { + voice.state = Ngs2VoicePlayState::Playing; + } + break; + case Ngs2VoicePlayEvent::Stop: + if (voice.state == Ngs2VoicePlayState::Playing) { + voice.state = Ngs2VoicePlayState::Stopped; + } + break; + case Ngs2VoicePlayEvent::StopImm: + case Ngs2VoicePlayEvent::Kill: voice.state = Ngs2VoicePlayState::Empty; break; + } + voice.event = Ngs2VoicePlayEvent::None; + } + } + } + + return OK; +} + +int KYTY_SYSV_ABI Ngs2ParseWaveformData(const void* data, size_t data_size, + Ngs2WaveformInfo* info) { + PRINT_NAME(); + LOGF("\t data = 0x%016" PRIx64 ", data_size = 0x%016" PRIx64 "\n", + reinterpret_cast(data), static_cast(data_size)); + + EXIT_NOT_IMPLEMENTED(info == nullptr); + + std::memset(info, 0, sizeof(Ngs2WaveformInfo)); + info->format.waveform_type = 0x80; + info->format.num_channels = 1; + info->format.sample_rate = 48000; + info->data_size = + static_cast(std::min(data_size, std::numeric_limits::max())); + info->num_audio_unit_samples = 1; + info->num_audio_unit_per_frame = 1; + info->num_audio_frame_samples = 1; + return OK; +} + +int KYTY_SYSV_ABI Ngs2CalcWaveformBlock(const Ngs2WaveformFormat* format, uint32_t sample_pos, + uint32_t num_samples, Ngs2WaveformBlock* block) { + PRINT_NAME(); + LOGF("\t format = 0x%016" PRIx64 ", sample_pos = %" PRIu32 ", num_samples = %" PRIu32 "\n", + reinterpret_cast(format), sample_pos, num_samples); + + EXIT_NOT_IMPLEMENTED(block == nullptr); + + std::memset(block, 0, sizeof(Ngs2WaveformBlock)); + block->num_samples = num_samples; + return OK; +} + +int KYTY_SYSV_ABI Ngs2PanInit(Ngs2PanWork* work, const float* speaker_angles, float unit_angle, + uint32_t num_speakers) { + PRINT_NAME(); + LOGF("\t work = 0x%016" PRIx64 ", num_speakers = %" PRIu32 "\n", + reinterpret_cast(work), num_speakers); + + EXIT_NOT_IMPLEMENTED(work == nullptr); + + std::memset(work, 0, sizeof(Ngs2PanWork)); + work->unit_angle = unit_angle; + work->num_speakers = std::min(num_speakers, 8); + if (speaker_angles != nullptr) { + for (uint32_t i = 0; i < work->num_speakers; i++) { + work->speaker_angles[i] = speaker_angles[i]; + } + } + return OK; +} + +int KYTY_SYSV_ABI Ngs2PanGetVolumeMatrix(Ngs2PanWork* work, const Ngs2PanParam* params, + uint32_t num_params, uint32_t matrix_format, + float* out_volume_matrix) { + PRINT_NAME(); + LOGF("\t work = 0x%016" PRIx64 ", params = 0x%016" PRIx64 ", num_params = %" PRIu32 + ", matrix_format = %" PRIu32 "\n", + reinterpret_cast(work), reinterpret_cast(params), num_params, + matrix_format); + + EXIT_NOT_IMPLEMENTED(out_volume_matrix == nullptr && num_params != 0); + + const auto channels = (matrix_format == 0 ? 2u : std::min(matrix_format, 8)); + for (uint32_t p = 0; p < num_params; p++) { + for (uint32_t c = 0; c < channels; c++) { + out_volume_matrix[p * channels + c] = (c == 0 ? 1.0f : 0.0f); + } + } + + return OK; +} + +int KYTY_SYSV_ABI Ngs2GeomResetListenerParam(Ngs2GeomListenerParam* out_listener_param) { + PRINT_NAME(); + + EXIT_NOT_IMPLEMENTED(out_listener_param == nullptr); + + std::memset(out_listener_param, 0, sizeof(Ngs2GeomListenerParam)); + out_listener_param->orient_front.z = 1.0f; + out_listener_param->orient_up.y = 1.0f; + out_listener_param->sound_speed = 343.0f; + + return OK; +} + +int KYTY_SYSV_ABI Ngs2GeomResetSourceParam(Ngs2GeomSourceParam* out_source_param) { + PRINT_NAME(); + + EXIT_NOT_IMPLEMENTED(out_source_param == nullptr); + + std::memset(out_source_param, 0, sizeof(Ngs2GeomSourceParam)); + out_source_param->direction.z = 1.0f; + out_source_param->cone.inner_level = 1.0f; + out_source_param->cone.inner_angle = 360.0f; + out_source_param->cone.outer_level = 1.0f; + out_source_param->cone.outer_angle = 360.0f; + out_source_param->rolloff.model = 0; + out_source_param->rolloff.max_distance = 1000000.0f; + out_source_param->rolloff.rolloff_factor = 1.0f; + out_source_param->rolloff.reference_distance = 1.0f; + out_source_param->doppler_factor = 1.0f; + out_source_param->fbw_level = 1.0f; + out_source_param->lfe_level = 1.0f; + out_source_param->max_level = 1.0f; + out_source_param->min_level = 0.0f; + out_source_param->num_speakers = 2; + out_source_param->matrix_format = 2; + + return OK; +} + +int KYTY_SYSV_ABI Ngs2GeomCalcListener(const Ngs2GeomListenerParam* param, + Ngs2GeomListenerWork* out_work, uint32_t flags) { + PRINT_NAME(); + LOGF("\t flags = 0x%08" PRIx32 "\n", flags); + + EXIT_NOT_IMPLEMENTED(param == nullptr); + EXIT_NOT_IMPLEMENTED(out_work == nullptr); + + std::memset(out_work, 0, sizeof(Ngs2GeomListenerWork)); + for (uint32_t i = 0; i < 4; i++) { + out_work->matrix[i][i] = 1.0f; + } + out_work->velocity = param->velocity; + out_work->sound_speed = (param->sound_speed > 0.0f ? param->sound_speed : 343.0f); + out_work->coordinate = flags & 0x1u; + + return OK; +} + +int KYTY_SYSV_ABI Ngs2GeomApply(const Ngs2GeomListenerWork* listener, + const Ngs2GeomSourceParam* source, Ngs2GeomAttribute* out_attrib, + uint32_t flags) { + PRINT_NAME(); + LOGF("\t flags = 0x%08" PRIx32 "\n", flags); + + EXIT_NOT_IMPLEMENTED(listener == nullptr); + EXIT_NOT_IMPLEMENTED(source == nullptr); + EXIT_NOT_IMPLEMENTED(out_attrib == nullptr); + + std::memset(out_attrib, 0, sizeof(Ngs2GeomAttribute)); + out_attrib->pitch_ratio = 1.0f; + out_attrib->a3d_attrib.position = source->position; + out_attrib->a3d_attrib.volume = std::max(source->min_level, source->max_level); + + const auto channels = + std::min((source->matrix_format == 0 ? 2u : source->matrix_format), 8); + const auto level = (source->max_level > 0.0f ? source->max_level : 1.0f); + for (uint32_t ch = 0; ch < channels; ch++) { + out_attrib->level[ch * 8 + ch] = level; + } + + return OK; +} + +int KYTY_SYSV_ABI Ngs2RackGetVoiceHandle(uintptr_t rack_handle, uint32_t voice_id, + uintptr_t* handle) { + PRINT_NAME(); + + EXIT_NOT_IMPLEMENTED(handle == nullptr); + EXIT_NOT_IMPLEMENTED(rack_handle == 0); + + LOGF("\t voice_id = %u\n", voice_id); + + auto* rack = reinterpret_cast(rack_handle); + auto* voices = reinterpret_cast(rack_handle + sizeof(Ngs2RackInternal)); + + if (voice_id >= rack->option.common.max_voices) { + LOGF("\t warning: voice_id %u >= max_voices %u, using last available stub voice\n", + voice_id, rack->option.common.max_voices); + if (rack->option.common.max_voices == 0) { + return -1; + } + voice_id = rack->option.common.max_voices - 1; + } + + EXIT_IF(voices[voice_id].rack != rack); + + *handle = reinterpret_cast(voices + voice_id); + + return OK; +} + +// NOLINTNEXTLINE(readability-function-cognitive-complexity) +int KYTY_SYSV_ABI Ngs2VoiceControl(uintptr_t voice_handle, const Ngs2VoiceParamHeader* param_list) { + PRINT_NAME(); + + EXIT_NOT_IMPLEMENTED(param_list == nullptr); + EXIT_NOT_IMPLEMENTED(voice_handle == 0); + + auto* voice = reinterpret_cast(voice_handle); + + Common::LockGuard lock(voice->rack->ngs->mutex); + + const auto* param = param_list; + + for (;;) { + LOGF("\t id = 0x%08" PRIx32 "\n" + "\t size = %" PRIu16 "\n" + "\t next = %" PRId16 "\n", + param->id, param->size, param->next); + + auto rack_id = param->id >> 16u; + + EXIT_NOT_IMPLEMENTED(((param->id >> 15u) & 0x1u) != 0); + + switch (rack_id) { + case 0x0000: { + auto cid = param->id & 0x7fffu; + switch (cid) { + case 0x0001: { + EXIT_NOT_IMPLEMENTED(param->size != sizeof(Ngs2VoiceMatrixLevelsParam)); + const auto* ml = reinterpret_cast(param); + LOGF("\t matrix_id = %u\n" + "\t num_levels = %u\n" + "\t levels = 0x%016" PRIx64 "\n", + ml->matrix_id, ml->num_levels, reinterpret_cast(ml->levels)); + break; + } + case 0x0002: { + EXIT_NOT_IMPLEMENTED(param->size != sizeof(Ngs2VoicePortVolumeParam)); + const auto* volume = + reinterpret_cast(param); + LOGF("\t port = %u\n" + "\t level = %f\n", + volume->port, volume->level); + break; + } + case 0x0003: { + EXIT_NOT_IMPLEMENTED(param->size != sizeof(Ngs2VoicePortMatrixParam)); + const auto* pm = reinterpret_cast(param); + LOGF("\t port = %u\n" + "\t matrix_id = %d\n", + pm->port, pm->matrix_id); + break; + } + case 0x0004: { + EXIT_NOT_IMPLEMENTED(param->size != sizeof(Ngs2VoicePortDelayParam)); + const auto* delay = reinterpret_cast(param); + LOGF("\t port = %u\n" + "\t num_samples = %u\n", + delay->port, delay->num_samples); + break; + } + case 0x0005: { + EXIT_NOT_IMPLEMENTED(param->size != sizeof(Ngs2VoicePatchParam)); + const auto* patch = reinterpret_cast(param); + LOGF("\t connect->port = %u\n" + "\t connect->dest_input_id = %u\n" + "\t connect->dest_handle = 0x%016" PRIx64 "\n", + patch->port, patch->dest_input_id, patch->dest_handle); + break; + } + case 0x0006: { + EXIT_NOT_IMPLEMENTED(param->size != sizeof(Ngs2VoiceEventParam)); + const auto* event = reinterpret_cast(param); + switch (event->event_id) { + case 0x0001: voice->event = Ngs2VoicePlayEvent::Play; break; + case 0x0002: voice->event = Ngs2VoicePlayEvent::Stop; break; + case 0x0004: voice->event = Ngs2VoicePlayEvent::StopImm; break; + case 0x0008: voice->event = Ngs2VoicePlayEvent::Kill; break; + case 0x0010: voice->event = Ngs2VoicePlayEvent::Pause; break; + case 0x0020: voice->event = Ngs2VoicePlayEvent::Resume; break; + default: EXIT("unknown event_id: 0x%08" PRIx32 "\n", event->event_id); + } + LOGF("\t event = %u\n", event->event_id); + break; + } + case 0x0007: { + EXIT_NOT_IMPLEMENTED(param->size != sizeof(Ngs2VoiceCallbackParam)); + const auto* callback = + reinterpret_cast(param); + voice->callback = callback->callback; + voice->callback_data = callback->callback_data; + voice->callback_flags = callback->flags; + LOGF("\t callback = 0x%016" PRIx64 "\n" + "\t callback_data = 0x%016" PRIx64 "\n" + "\t flags = 0x%08" PRIx32 "\n", + static_cast(voice->callback), + static_cast(voice->callback_data), voice->callback_flags); + break; + } + default: EXIT("unknown id: 0x%04" PRIx32 "\n", cid); + } + break; + } + case 0x1000: EXIT_NOT_IMPLEMENTED(voice->rack->type != Ngs2RackType::Sampler); break; + case 0x2000: EXIT_NOT_IMPLEMENTED(voice->rack->type != Ngs2RackType::Submixer); break; + case 0x2001: EXIT_NOT_IMPLEMENTED(voice->rack->type != Ngs2RackType::Reverb); break; + case 0x3000: EXIT_NOT_IMPLEMENTED(voice->rack->type != Ngs2RackType::Mastering); break; + case 0x4000: { + EXIT_NOT_IMPLEMENTED(!Ngs2RackIsCustom(voice->rack->type)); + auto cid = param->id & 0xffffu; + auto module_id = (cid >> 8u) & 0xffu; + auto ctl_id = (cid >> 5u) & 0x7u; + auto module_no = cid & 0x1fu; + LOGF("\t custom module_id = 0x%02" PRIx32 ", ctl_id = 0x%" PRIx32 + ", module_no = %" PRIu32 "\n", + module_id, ctl_id, module_no); + break; + } + case 0x4001: + EXIT_NOT_IMPLEMENTED(voice->rack->type != Ngs2RackType::CustomSampler); + break; + case 0x4002: + EXIT_NOT_IMPLEMENTED(voice->rack->type != Ngs2RackType::CustomSubmixer); + break; + default: EXIT("unknown rack_id: 0x%" PRIx32 "\n", rack_id); + } + + if (param->next == 0) { + break; + } + param = reinterpret_cast(reinterpret_cast(param) + + param->next); + } + + return OK; +} + +int KYTY_SYSV_ABI Ngs2VoiceRunCommands(uintptr_t voice_handle, const void* commands, + uint32_t num_commands, uint32_t flags) { + PRINT_NAME(); + + (void)voice_handle; + (void)commands; + (void)num_commands; + (void)flags; + + return OK; +} + +int KYTY_SYSV_ABI Ngs2VoiceGetState(uintptr_t voice_handle, Ngs2VoiceState* state, + size_t state_size) { + PRINT_NAME(); + + EXIT_NOT_IMPLEMENTED(state == nullptr); + EXIT_NOT_IMPLEMENTED(voice_handle == 0); + + auto* voice = reinterpret_cast(voice_handle); + + Common::LockGuard lock(voice->rack->ngs->mutex); + + switch (voice->rack->type) { + case Ngs2RackType::Sampler: + case Ngs2RackType::CustomSampler: { + if (state_size != sizeof(Ngs2SamplerVoiceState)) { + LOGF("\t warning: sampler state_size = 0x%016" PRIx64 ", expected 0x%016" PRIx64 + "\n", + static_cast(state_size), + static_cast(sizeof(Ngs2SamplerVoiceState))); + } + std::memset(state, 0, state_size); + + state->state_flags = Ngs2GetStateFlags(voice); + if (state_size < sizeof(Ngs2SamplerVoiceState)) { + LOGF("\t state_flags = %u\n", state->state_flags); + break; + } + + auto* sampler = reinterpret_cast(state); + sampler->envelope_height = 1.0f; + sampler->peak_height = 0.0f; + sampler->reserved = 0; + sampler->num_decoded_samples = 0; + sampler->user_data = 0; + sampler->waveform_data = nullptr; + LOGF("\t state_flags = %u\n", sampler->voice_state.state_flags); + break; + } + default: EXIT("unknown type: %s\n", Common::EnumName(voice->rack->type).c_str()); + } + + return OK; +} + +int KYTY_SYSV_ABI Ngs2VoiceGetStateFlags(uintptr_t voice_handle, uint32_t* state_flags) { + PRINT_NAME(); + + EXIT_NOT_IMPLEMENTED(state_flags == nullptr); + EXIT_NOT_IMPLEMENTED(voice_handle == 0); + + auto* voice = reinterpret_cast(voice_handle); + + Common::LockGuard lock(voice->rack->ngs->mutex); + + *state_flags = Ngs2GetStateFlags(voice); + + LOGF("\t state_flags = %u\n", *state_flags); + + return OK; +} + +} // namespace Ngs2 + +} // namespace Libs::Audio diff --git a/src/libs/audio.h b/src/libs/audio.h new file mode 100644 index 000000000..7218bdd5d --- /dev/null +++ b/src/libs/audio.h @@ -0,0 +1,353 @@ +#ifndef EMULATOR_INCLUDE_EMULATOR_AUDIO_H_ +#define EMULATOR_INCLUDE_EMULATOR_AUDIO_H_ + +#include "common/abi.h" +#include "common/common.h" +#include "common/subsystems.h" + +namespace Libs::Audio { + +KYTY_SUBSYSTEM_DEFINE(Audio); + +namespace AudioOut { + +struct AudioOutOutputParam; +struct AudioOutPortState; + +int KYTY_SYSV_ABI AudioOutInit(); +int KYTY_SYSV_ABI AudioOutOpen(int user_id, int type, int index, uint32_t len, uint32_t freq, + uint32_t param); +int KYTY_SYSV_ABI AudioOutSetVolume(int handle, uint32_t flag, int* vol); +int KYTY_SYSV_ABI AudioOutOutputs(AudioOutOutputParam* param, uint32_t num); +int KYTY_SYSV_ABI AudioOutOutput(int handle, const void* ptr); +int KYTY_SYSV_ABI AudioOutClose(int handle); +int KYTY_SYSV_ABI AudioOutGetPortState(int handle, AudioOutPortState* state); + +} // namespace AudioOut + +namespace AudioOut2 { + +struct AudioOut2ContextParam; +struct AudioOut2PortParam; +struct AudioOut2Attribute; +struct AudioOut2PortState; +struct AudioOut2SystemState; +struct AudioOut2Position; +struct AudioOut2SpeakerInfo; +struct AudioOut2SystemDebugStateParam; +struct AudioOut2MasteringParamsHeader; +struct AudioOut2MasteringStatesHeader; + +using AudioOut2ContextHandle = uint64_t; +using AudioOut2PortHandle = uint64_t; +using AudioOut2UserHandle = uintptr_t; +using AudioOut2SpeakerArrayHandle = void*; + +int KYTY_SYSV_ABI AudioOut2Initialize(); +int KYTY_SYSV_ABI AudioOut2ContextQueryMemory(const AudioOut2ContextParam* params, + size_t* memory_size); +int KYTY_SYSV_ABI AudioOut2ContextResetParam(AudioOut2ContextParam* params); +int KYTY_SYSV_ABI AudioOut2ContextCreate(const AudioOut2ContextParam* params, void* buffer, + size_t buffer_size, AudioOut2ContextHandle* ctx); +int KYTY_SYSV_ABI AudioOut2ContextDestroy(AudioOut2ContextHandle ctx); +int KYTY_SYSV_ABI AudioOut2ContextSetAttributes(AudioOut2ContextHandle ctx, + const AudioOut2Attribute* attributes, uint32_t num); +int KYTY_SYSV_ABI AudioOut2ContextAdvance(AudioOut2ContextHandle ctx); +int KYTY_SYSV_ABI AudioOut2ContextPush(AudioOut2ContextHandle ctx, uint32_t blocking); +int KYTY_SYSV_ABI AudioOut2ContextGetQueueLevel(AudioOut2ContextHandle ctx, uint32_t* queue_level, + uint32_t* available_queues); +int KYTY_SYSV_ABI AudioOut2PortCreate(AudioOut2ContextHandle ctx, const AudioOut2PortParam* params, + AudioOut2PortHandle* port); +int KYTY_SYSV_ABI AudioOut2PortDestroy(AudioOut2PortHandle port); +int KYTY_SYSV_ABI AudioOut2PortSetAttributes(AudioOut2PortHandle port, + const AudioOut2Attribute* attributes, uint32_t num); +int KYTY_SYSV_ABI AudioOut2PortGetState(AudioOut2PortHandle port, AudioOut2PortState* state); +int KYTY_SYSV_ABI AudioOut2GetSystemState(AudioOut2SystemState* state); +int KYTY_SYSV_ABI AudioOut2UserCreate(uint32_t user_id, AudioOut2UserHandle* handle); +int KYTY_SYSV_ABI AudioOut2UserDestroy(AudioOut2UserHandle handle); +size_t KYTY_SYSV_ABI AudioOut2GetSpeakerArrayMemorySize(uint32_t num_speakers, uint8_t is_3d, + uint8_t is_ambisonics); +int KYTY_SYSV_ABI AudioOut2SpeakerArrayCreate(AudioOut2SpeakerArrayHandle* handle, + const void* vbap_params, const void* ambi_params); +int KYTY_SYSV_ABI AudioOut2SpeakerArrayDestroy(AudioOut2SpeakerArrayHandle handle); +int KYTY_SYSV_ABI AudioOut2GetSpeakerArrayCoefficients( + AudioOut2SpeakerArrayHandle handle, AudioOut2Position pos, float spread, float* coefficients, + uint32_t num_coefficients, uint8_t height_aware, float downmix_spread_radius); +int KYTY_SYSV_ABI AudioOut2GetSpeakerArrayAmbisonicsCoefficients(AudioOut2SpeakerArrayHandle handle, + uint32_t ambisonics_channel, + float* coefficients, + uint32_t num_coefficients); +int KYTY_SYSV_ABI AudioOut2GetSpeakerInfo(AudioOut2SpeakerInfo* info, uint32_t flags); +int KYTY_SYSV_ABI AudioOut2SetSystemDebugState(const AudioOut2SystemDebugStateParam* param); +int KYTY_SYSV_ABI AudioOut2Set3DLatency(uint32_t user_id, uint32_t output, uint32_t latency_us); +int KYTY_SYSV_ABI AudioOut2MasteringInit(uint32_t flags); +int KYTY_SYSV_ABI AudioOut2MasteringSetParam(const AudioOut2MasteringParamsHeader* param, + uint32_t output, uint32_t flags); +int KYTY_SYSV_ABI AudioOut2MasteringGetState(AudioOut2MasteringStatesHeader* state, uint32_t output, + AudioOut2UserHandle user); +int KYTY_SYSV_ABI AudioOut2MasteringTerm(); + +} // namespace AudioOut2 + +namespace AudioIn { + +int KYTY_SYSV_ABI AudioInOpen(int user_id, uint32_t type, uint32_t index, uint32_t len, + uint32_t freq, uint32_t param); +int KYTY_SYSV_ABI AudioInInput(int handle, void* dest); + +} // namespace AudioIn + +namespace VoiceQoS { + +int KYTY_SYSV_ABI VoiceQoSInit(void* mem_block, uint32_t mem_size, int32_t app_type); + +} // namespace VoiceQoS + +namespace Ajm { + +struct AjmBatchInfo; +struct AjmBatchError; +struct AjmBuffer; + +struct AjmDecAt9ConfigDataInfo { + uint32_t channels; + uint32_t sample_rate; + uint32_t frame_samples_per_channel; + uint32_t superframe_samples_per_channel; + uint32_t superframe_size; +}; + +int KYTY_SYSV_ABI AjmInitialize(int64_t reserved, uint32_t* context); +int KYTY_SYSV_ABI AjmModuleRegister(uint32_t context, uint32_t codec, int64_t reserved); +int KYTY_SYSV_ABI AjmModuleUnregister(uint32_t context, uint32_t codec); +int KYTY_SYSV_ABI AjmFinalize(uint32_t context); +int KYTY_SYSV_ABI AjmInstanceCreate(uint32_t context, uint32_t codec, uint64_t flags, + uint32_t* instance); +int KYTY_SYSV_ABI AjmInstanceDestroy(uint32_t context, uint32_t instance); +int KYTY_SYSV_ABI AjmMemoryRegister(uint32_t context, void* ptr, size_t pages); +int KYTY_SYSV_ABI AjmMemoryUnregister(uint32_t context, void* ptr); +int KYTY_SYSV_ABI AjmBatchInitialize(void* buffer, size_t size, AjmBatchInfo* info); +int KYTY_SYSV_ABI AjmBatchStart(uint32_t context, const AjmBatchInfo* info, int priority, + AjmBatchError* error, uint32_t* batch); +int KYTY_SYSV_ABI AjmBatchWait(uint32_t context, uint32_t batch, uint32_t timeout, + AjmBatchError* error); +int KYTY_SYSV_ABI AjmBatchCancel(uint32_t context, uint32_t batch); +int KYTY_SYSV_ABI AjmBatchErrorDump(const AjmBatchInfo* info, AjmBatchError* error); +int KYTY_SYSV_ABI AjmBatchJobInitialize(AjmBatchInfo* info, uint32_t instance, + const void* codec_parameters, size_t codec_parameters_size, + void* result); +int KYTY_SYSV_ABI AjmBatchJobClearContext(AjmBatchInfo* info, uint32_t instance, void* result); +int KYTY_SYSV_ABI AjmBatchJobDecode(AjmBatchInfo* info, uint32_t instance, + const void* bitstream_input, size_t bitstream_input_size, + void* pcm_output, size_t pcm_output_size, void* result); +int KYTY_SYSV_ABI AjmBatchJobDecodeSingle(AjmBatchInfo* info, uint32_t instance, + const void* bitstream_input, size_t bitstream_input_size, + void* pcm_output, size_t pcm_output_size, void* result); +int KYTY_SYSV_ABI AjmBatchJobDecodeSplit(AjmBatchInfo* info, uint32_t instance, + const AjmBuffer* input_buffers, size_t input_buffers_num, + const AjmBuffer* output_buffers, size_t output_buffers_num, + void* result); +int KYTY_SYSV_ABI AjmBatchJobEncode(AjmBatchInfo* info, uint32_t instance, const void* pcm_input, + size_t pcm_input_size, void* bitstream_output, + size_t bitstream_output_size, void* result); +int KYTY_SYSV_ABI AjmBatchJobGetInfo(AjmBatchInfo* info, uint32_t instance, void* result); +int KYTY_SYSV_ABI AjmBatchJobGetCodecInfo(AjmBatchInfo* info, uint32_t instance, void* result, + size_t result_size); +int KYTY_SYSV_ABI AjmBatchJobSetGaplessDecode(AjmBatchInfo* info, uint32_t instance, + const void* gapless_decode, int reset, void* result); +int KYTY_SYSV_ABI AjmBatchJobGetGaplessDecode(AjmBatchInfo* info, uint32_t instance, void* result); +int KYTY_SYSV_ABI AjmBatchJobSetResampleParameters(AjmBatchInfo* info, uint32_t instance, + float ratio, uint32_t flags, void* result); +int KYTY_SYSV_ABI AjmBatchJobSetResampleParametersEx(AjmBatchInfo* info, uint32_t instance, + float ratio_start, + float ratio_change_per_sample, uint32_t flags, + void* result); +int KYTY_SYSV_ABI AjmBatchJobGetResampleInfo(AjmBatchInfo* info, uint32_t instance, void* result); +int KYTY_SYSV_ABI AjmBatchJobGetStatistics(AjmBatchInfo* info, float interval, void* result); +int KYTY_SYSV_ABI AjmBatchJobControl(AjmBatchInfo* info, uint32_t instance, uint64_t flags, + const void* sideband_input, size_t sideband_input_size, + void* sideband_output, size_t sideband_output_size); +int KYTY_SYSV_ABI AjmBatchJobRun(AjmBatchInfo* info, uint32_t instance, uint64_t flags, + const void* data_input, size_t data_input_size, void* data_output, + size_t data_output_size, void* sideband_output, + size_t sideband_output_size); +int KYTY_SYSV_ABI AjmBatchJobRunSplit(AjmBatchInfo* info, uint32_t instance, uint64_t flags, + const AjmBuffer* input_buffers, size_t input_buffers_num, + const AjmBuffer* output_buffers, size_t output_buffers_num, + void* sideband_output, size_t sideband_output_size); +int KYTY_SYSV_ABI AjmDecAt9ParseConfigData(const void* config_data, + AjmDecAt9ConfigDataInfo* config_info); +const char* KYTY_SYSV_ABI AjmStrError(int error); + +} // namespace Ajm + +namespace Acm { + +struct AcmBatchInfo; +struct AcmBatchError; + +using AcmContextId = uint32_t; +using AcmBatchId = uint32_t; + +int KYTY_SYSV_ABI AcmContextCreate(AcmContextId* context); +int KYTY_SYSV_ABI AcmContextDestroy(AcmContextId context); +int KYTY_SYSV_ABI AcmBatchStartBuffer(AcmContextId context, const void* batch_commands, + size_t batch_size, AcmBatchError* batch_error, + AcmBatchId* batch); +int KYTY_SYSV_ABI AcmBatchStartBuffers(AcmContextId context, uint32_t batch_info_count, + const AcmBatchInfo* const batch_info[], + AcmBatchError* batch_error, AcmBatchId* batch); +int KYTY_SYSV_ABI AcmBatchWait(AcmContextId context, AcmBatchId batch, uint32_t timeout); +int KYTY_SYSV_ABI AcmBatchJobNotification(AcmBatchInfo* batch_info); +int KYTY_SYSV_ABI AcmConvReverbSharedInput(AcmBatchInfo* batch_info, uint32_t block_count, void* in, + uint32_t count, const void* const ir[], + const float* gain, void* const out[]); +int KYTY_SYSV_ABI AcmConvReverbSharedIr(AcmBatchInfo* batch_info, uint32_t block_count, + const void* ir, uint32_t count, void* const in[], + const float* gain, void* const out[]); +int KYTY_SYSV_ABI AcmFft(AcmBatchInfo* batch_info, int size, int count, int input_format, + const void* const input[], int output_format, void* const output[], + uint32_t flags); +int KYTY_SYSV_ABI AcmIfft(AcmBatchInfo* batch_info, int size, int count, int input_format, + const void* const input[], int output_format, void* const output[], + uint32_t flags); +int KYTY_SYSV_ABI AcmPanner(AcmBatchInfo* batch_info, uint32_t in_count, const float* const in[], + uint32_t biquad_count, uint32_t biquad_update_count, uint32_t out_count, + const void* const parameter[], void* const state[], + const float* const out_init[], float* const out[]); + +} // namespace Acm + +namespace AvPlayer { + +struct AvPlayerInitData; +struct AvPlayerFrameInfoEx; +struct AvPlayerFrameInfo; +struct AvPlayerInternal; + +using Bool = uint8_t; + +AvPlayerInternal* KYTY_SYSV_ABI AvPlayerInit(AvPlayerInitData* init); +int KYTY_SYSV_ABI AvPlayerInitEx(const void* init_ex, AvPlayerInternal** handle); +int KYTY_SYSV_ABI AvPlayerPostInit(AvPlayerInternal* h, const void* post_init); +int KYTY_SYSV_ABI AvPlayerAddSource(AvPlayerInternal* h, const char* filename); +int KYTY_SYSV_ABI AvPlayerAddSourceEx(AvPlayerInternal* h, uint32_t uri_type, + const void* source_details); +int KYTY_SYSV_ABI AvPlayerStreamCount(AvPlayerInternal* h); +int KYTY_SYSV_ABI AvPlayerGetStreamInfo(AvPlayerInternal* h, uint32_t stream_id, void* info); +int KYTY_SYSV_ABI AvPlayerGetStreamInfoEx(AvPlayerInternal* h, uint32_t stream_id, void* info); +int KYTY_SYSV_ABI AvPlayerEnableStream(AvPlayerInternal* h, uint32_t stream_id); +int KYTY_SYSV_ABI AvPlayerDisableStream(AvPlayerInternal* h, uint32_t stream_id); +int KYTY_SYSV_ABI AvPlayerChangeStream(AvPlayerInternal* h, uint32_t old_stream_id, + uint32_t new_stream_id); +int KYTY_SYSV_ABI AvPlayerStart(AvPlayerInternal* h); +int KYTY_SYSV_ABI AvPlayerStartEx(AvPlayerInternal* h, const void* start_info_ex); +int KYTY_SYSV_ABI AvPlayerStop(AvPlayerInternal* h); +int KYTY_SYSV_ABI AvPlayerPause(AvPlayerInternal* h); +int KYTY_SYSV_ABI AvPlayerResume(AvPlayerInternal* h); +int KYTY_SYSV_ABI AvPlayerSetLooping(AvPlayerInternal* h, Bool loop); +int KYTY_SYSV_ABI AvPlayerSetTrickSpeed(AvPlayerInternal* h, int32_t trick_speed); +int KYTY_SYSV_ABI AvPlayerSetAvSyncMode(AvPlayerInternal* h, uint32_t sync_mode); +int KYTY_SYSV_ABI AvPlayerSetAvailableBandwidth(AvPlayerInternal* h, uint32_t start_bandwidth, + uint32_t minimum_bandwidth, + uint32_t maximum_bandwidth); +Bool KYTY_SYSV_ABI AvPlayerGetVideoData(AvPlayerInternal* h, AvPlayerFrameInfo* video_info); +Bool KYTY_SYSV_ABI AvPlayerGetVideoDataEx(AvPlayerInternal* h, AvPlayerFrameInfoEx* video_info); +Bool KYTY_SYSV_ABI AvPlayerGetAudioData(AvPlayerInternal* h, AvPlayerFrameInfo* audio_info); +Bool KYTY_SYSV_ABI AvPlayerIsActive(AvPlayerInternal* h); +uint64_t KYTY_SYSV_ABI AvPlayerCurrentTime(AvPlayerInternal* h); +int KYTY_SYSV_ABI AvPlayerJumpToTime(AvPlayerInternal* h, uint64_t time_ms); +int KYTY_SYSV_ABI AvPlayerClose(AvPlayerInternal* h); +int KYTY_SYSV_ABI AvPlayerSetLogCallback(void* callback, void* user_data); + +} // namespace AvPlayer + +namespace Audio3d { + +struct Audio3dOpenParameters; + +int KYTY_SYSV_ABI Audio3dInitialize(int64_t reserved); +void KYTY_SYSV_ABI Audio3dGetDefaultOpenParameters(Audio3dOpenParameters* p); +int KYTY_SYSV_ABI Audio3dPortOpen(int user_id, const Audio3dOpenParameters* parameters, + uint32_t* id); +int KYTY_SYSV_ABI Audio3dPortSetAttribute(uint32_t port_id, uint32_t attribute_id, + const void* attribute, size_t attribute_size); +int KYTY_SYSV_ABI Audio3dPortGetQueueLevel(uint32_t port_id, uint32_t* queue_level, + uint32_t* queue_available); +int KYTY_SYSV_ABI Audio3dPortAdvance(uint32_t port_id); +int KYTY_SYSV_ABI Audio3dPortPush(uint32_t port_id, uint32_t blocking); + +} // namespace Audio3d + +namespace Ngs2 { + +struct Ngs2SystemOption; +struct Ngs2RackOption; +struct Ngs2BufferAllocator; +struct Ngs2VoiceParamHeader; +struct Ngs2RenderBufferInfo; +struct Ngs2ContextBufferInfo; +struct Ngs2VoiceState; +struct Ngs2WaveformFormat; +struct Ngs2WaveformBlock; +struct Ngs2WaveformInfo; +struct Ngs2PanWork; +struct Ngs2PanParam; +struct Ngs2GeomListenerParam; +struct Ngs2GeomListenerWork; +struct Ngs2GeomSourceParam; +struct Ngs2GeomAttribute; + +int KYTY_SYSV_ABI Ngs2SystemResetOption(Ngs2SystemOption* option); +int KYTY_SYSV_ABI Ngs2SystemQueryBufferSize(const Ngs2SystemOption* option, + Ngs2ContextBufferInfo* buffer_info); +int KYTY_SYSV_ABI Ngs2SystemCreate(const Ngs2SystemOption* option, + const Ngs2ContextBufferInfo* buffer_info, uintptr_t* handle); +int KYTY_SYSV_ABI Ngs2SystemSetGrainSamples(uintptr_t system_handle, uint32_t num_samples); +int KYTY_SYSV_ABI Ngs2RackQueryBufferSize(uint32_t rack_id, const Ngs2RackOption* option, + Ngs2ContextBufferInfo* buffer_info); +int KYTY_SYSV_ABI Ngs2RackCreate(uintptr_t system_handle, uint32_t rack_id, + const Ngs2RackOption* option, + const Ngs2ContextBufferInfo* buffer_info, uintptr_t* handle); +int KYTY_SYSV_ABI Ngs2SystemCreateWithAllocator(const Ngs2SystemOption* option, + const Ngs2BufferAllocator* allocator, + uintptr_t* handle); +int KYTY_SYSV_ABI Ngs2SystemDestroy(uintptr_t system_handle, Ngs2ContextBufferInfo* buffer_info); +int KYTY_SYSV_ABI Ngs2RackCreateWithAllocator(uintptr_t system_handle, uint32_t rack_id, + const Ngs2RackOption* option, + const Ngs2BufferAllocator* allocator, + uintptr_t* handle); +int KYTY_SYSV_ABI Ngs2RackDestroy(uintptr_t rack_handle, Ngs2ContextBufferInfo* buffer_info); +int KYTY_SYSV_ABI Ngs2RackLock(uintptr_t rack_handle); +int KYTY_SYSV_ABI Ngs2RackUnlock(uintptr_t rack_handle); +int KYTY_SYSV_ABI Ngs2RackGetVoiceHandle(uintptr_t rack_handle, uint32_t voice_id, + uintptr_t* handle); +int KYTY_SYSV_ABI Ngs2VoiceControl(uintptr_t voice_handle, const Ngs2VoiceParamHeader* param_list); +int KYTY_SYSV_ABI Ngs2VoiceRunCommands(uintptr_t voice_handle, const void* commands, + uint32_t num_commands, uint32_t flags); +int KYTY_SYSV_ABI Ngs2VoiceGetState(uintptr_t voice_handle, Ngs2VoiceState* state, + size_t state_size); +int KYTY_SYSV_ABI Ngs2VoiceGetStateFlags(uintptr_t voice_handle, uint32_t* state_flags); +int KYTY_SYSV_ABI Ngs2SystemRender(uintptr_t system_handle, const Ngs2RenderBufferInfo* buffer_info, + uint32_t num_buffer_info); +int KYTY_SYSV_ABI Ngs2ParseWaveformData(const void* data, size_t data_size, Ngs2WaveformInfo* info); +int KYTY_SYSV_ABI Ngs2CalcWaveformBlock(const Ngs2WaveformFormat* format, uint32_t sample_pos, + uint32_t num_samples, Ngs2WaveformBlock* block); +int KYTY_SYSV_ABI Ngs2PanInit(Ngs2PanWork* work, const float* speaker_angles, float unit_angle, + uint32_t num_speakers); +int KYTY_SYSV_ABI Ngs2PanGetVolumeMatrix(Ngs2PanWork* work, const Ngs2PanParam* params, + uint32_t num_params, uint32_t matrix_format, + float* out_volume_matrix); +int KYTY_SYSV_ABI Ngs2GeomResetListenerParam(Ngs2GeomListenerParam* out_listener_param); +int KYTY_SYSV_ABI Ngs2GeomResetSourceParam(Ngs2GeomSourceParam* out_source_param); +int KYTY_SYSV_ABI Ngs2GeomCalcListener(const Ngs2GeomListenerParam* param, + Ngs2GeomListenerWork* out_work, uint32_t flags); +int KYTY_SYSV_ABI Ngs2GeomApply(const Ngs2GeomListenerWork* listener, + const Ngs2GeomSourceParam* source, Ngs2GeomAttribute* out_attrib, + uint32_t flags); + +} // namespace Ngs2 + +} // namespace Libs::Audio + +#endif /* EMULATOR_INCLUDE_EMULATOR_AUDIO_H_ */ diff --git a/src/libs/audio_internal.h b/src/libs/audio_internal.h new file mode 100644 index 000000000..ed7c4a8ef --- /dev/null +++ b/src/libs/audio_internal.h @@ -0,0 +1,33 @@ +#ifndef EMULATOR_INCLUDE_EMULATOR_AUDIO_INTERNAL_H_ +#define EMULATOR_INCLUDE_EMULATOR_AUDIO_INTERNAL_H_ + +#include "common/common.h" + +namespace Libs::Audio::AudioInternal { + +enum class Format { + Unknown, + Signed16bitMono, + Signed16bitStereo, + Signed16bit8Ch, + FloatMono, + FloatStereo, + Float8Ch, + Signed16bit8ChStd, + Float8ChStd, +}; + +struct OutputParam { + int handle = 0; + const void* data = nullptr; +}; + +static constexpr int OUT_PORTS_MAX = 32; + +int AudioOutOpen(int type, uint32_t samples_num, uint32_t freq, Format format); +void AudioOutClose(int handle); +uint32_t AudioOutOutputs(const OutputParam* params, uint32_t num); + +} // namespace Libs::Audio::AudioInternal + +#endif // EMULATOR_INCLUDE_EMULATOR_AUDIO_INTERNAL_H_ diff --git a/src/libs/avPlayer.cpp b/src/libs/avPlayer.cpp new file mode 100644 index 000000000..9f4176766 --- /dev/null +++ b/src/libs/avPlayer.cpp @@ -0,0 +1,1591 @@ +#include "common/common.h" +#include "common/logging/log.h" +#include "common/magicEnum.h" +#include "common/stringUtils.h" +#include "common/threads.h" +#include "kernel/fileSystem.h" +#include "libs/audio.h" +#include "libs/libs.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +extern "C" { +#include +#include +#include +#include +#include +#include +#include +#include +} + +namespace Libs::Audio::AvPlayer { + +LIB_NAME("AvPlayer", "AvPlayer"); + +using AvPlayerAllocate = KYTY_SYSV_ABI void* (*)(void*, uint32_t, uint32_t); +using AvPlayerDeallocate = KYTY_SYSV_ABI void (*)(void*, void*); +using AvPlayerAllocateTexture = KYTY_SYSV_ABI void* (*)(void*, uint32_t, uint32_t); +using AvPlayerDeallocateTexture = KYTY_SYSV_ABI void (*)(void*, void*); +using AvPlayerOpenFile = KYTY_SYSV_ABI int (*)(void*, const char*); +using AvPlayerCloseFile = KYTY_SYSV_ABI int (*)(void*); +using AvPlayerReadOffsetFile = KYTY_SYSV_ABI int (*)(void*, uint8_t*, uint64_t, uint32_t); +using AvPlayerSizeFile = KYTY_SYSV_ABI uint64_t (*)(void*); +using AvPlayerEventCallback = KYTY_SYSV_ABI void (*)(void*, int32_t, int32_t, void*); +using Bool = uint8_t; + +constexpr int32_t AVPLAYER_EVENT_STATE_STOP = 0x01; +constexpr int32_t AVPLAYER_EVENT_STATE_READY = 0x02; +constexpr int32_t AVPLAYER_EVENT_STATE_PLAY = 0x03; +constexpr int32_t AVPLAYER_EVENT_STATE_PAUSE = 0x04; +constexpr int32_t AVPLAYER_EVENT_WARNING_ID = 0x20; +constexpr int32_t AVPLAYER_ERROR_INVALID_PARAMS = -2140536831; +constexpr int32_t AVPLAYER_ERROR_OPERATION_FAILED = -2140536830; +constexpr int32_t AVPLAYER_ERROR_NO_MEMORY = -2140536829; +constexpr int32_t AVPLAYER_ERROR_NOT_SUPPORTED = -2140536828; +constexpr int32_t AVPLAYER_WARNING_LOOPING_BACK = -2140536671; +constexpr int32_t AVPLAYER_WARNING_JUMP_COMPLETE = -2140536669; +constexpr int32_t AVPLAYER_TRICK_SPEED_NORMAL = 100; +constexpr int32_t AVPLAYER_TRICK_SPEED_MIN = 400; +constexpr int32_t AVPLAYER_TRICK_SPEED_MAX = 3200; + +enum AvPlayerUriType : uint32_t { AvPlayerUriTypeSource = 0 }; +enum AvPlayerSourceType : uint32_t { + AvPlayerSourceUnknown = 0, + AvPlayerSourceFileMp4 = 1, + AvPlayerSourceFileWebm = 2, + AvPlayerSourceHls = 8 +}; + +enum AvPlayerDebuglevels : int { + AvplayerDbgNone, + AvplayerDbgInfo, + AvplayerDbgWarnings, + AvplayerDbgAll +}; +enum AvPlayerStreamType : uint32_t { + AvPlayerStreamUnknown = 0, + AvPlayerStreamVideo = 1, + AvPlayerStreamAudio = 2, + AvPlayerStreamTimedText = 3 +}; +enum AvPlayerVideoDecoderType : uint32_t { + AvPlayerVideoDecoderDefault = 0, + AvPlayerVideoDecoderSoftware2 = 1 +}; +enum AvPlayerAudioDecoderType : uint32_t { AvPlayerAudioDecoderDefault = 0 }; +enum AvPlayerAudioChannelOrder : uint32_t { + AvPlayerAudioChannelOrderDefault = 0, + AvPlayerAudioChannelOrderExt = 1, + AvPlayerAudioChannelOrderLsRs = 2 +}; + +struct AvPlayerMemAllocator { + void* object_pointer = nullptr; + AvPlayerAllocate allocate = nullptr; + AvPlayerDeallocate deallocate = nullptr; + AvPlayerAllocateTexture allocate_texture = nullptr; + AvPlayerDeallocateTexture deallocate_texture = nullptr; +}; +struct AvPlayerFileReplacement { + void* object_pointer = nullptr; + AvPlayerOpenFile open = nullptr; + AvPlayerCloseFile close = nullptr; + AvPlayerReadOffsetFile read_offset = nullptr; + AvPlayerSizeFile size = nullptr; +}; +struct AvPlayerEventReplacement { + void* object_pointer = nullptr; + AvPlayerEventCallback event_callback = nullptr; +}; +struct AvPlayerUri { + const char* name = nullptr; + uint32_t length = 0; +}; +struct AvPlayerSourceDetails { + AvPlayerUri uri; + uint8_t reserved1[64] = {}; + AvPlayerSourceType source_type = AvPlayerSourceUnknown; + uint8_t reserved2[44] = {}; +}; +struct AvPlayerInitData { + AvPlayerMemAllocator memory_replacement; + AvPlayerFileReplacement file_replacement; + AvPlayerEventReplacement event_replacement; + AvPlayerDebuglevels debug_level = AvplayerDbgNone; + uint32_t base_priority = 0; + int32_t num_output_video_framebuffers = 0; + Bool auto_start = 0; + uint8_t reserved[3] = {}; + const char* default_language = nullptr; +}; +struct AvPlayerThreadInfo { + uint32_t priority = 0; + uint32_t stack_size = 0; + uint64_t affinity = 0; + uint8_t reserved[32] = {}; +}; +struct AvPlayerInitDataEx { + size_t this_size = 0; + AvPlayerMemAllocator memory_replacement; + AvPlayerFileReplacement file_replacement; + AvPlayerEventReplacement event_replacement; + const char* default_language = nullptr; + AvPlayerDebuglevels debug_level = AvplayerDbgNone; + Bool auto_start = 0; + uint8_t reserved[3] = {}; + AvPlayerThreadInfo audio_decoder; + AvPlayerThreadInfo video_decoder; + AvPlayerThreadInfo demuxer; + AvPlayerThreadInfo event; + AvPlayerThreadInfo call_queue; + AvPlayerThreadInfo http_command_processor; + AvPlayerThreadInfo http_segment_manager; + AvPlayerThreadInfo http_streamlist; + AvPlayerThreadInfo file_streaming; + int32_t num_output_video_framebuffers = 0; + uint8_t reserved2[4] = {}; +}; +struct AvPlayerDecoderInit { + union { + AvPlayerVideoDecoderType video_type; + AvPlayerAudioDecoderType audio_type; + uint8_t reserved[4]; + } decoder_type {}; + uint8_t reserved[4] = {}; + union { + struct { + uint64_t cpu_affinity_mask; + int32_t cpu_thread_priority; + uint8_t decode_input_queue_depth; + uint8_t compute_pipe_id; + uint8_t compute_queue_id; + uint8_t reserved[17]; + } video_sw2; + struct { + AvPlayerAudioChannelOrder audio_channel_order; + uint8_t reserved[28]; + } audio; + uint8_t reserved[32]; + } decoder_params {}; +}; +struct AvPlayerHttpContext { + uint32_t http_context_id = 0; + uint32_t ssl_context_id = 0; +}; +struct AvPlayerPostInitData { + uint32_t demux_video_buffer_size = 0; + uint8_t reserved[4] = {}; + AvPlayerDecoderInit video_decoder_init; + AvPlayerDecoderInit audio_decoder_init; + AvPlayerHttpContext http_context; + uint8_t reserved2[56] = {}; +}; +struct AvPlayerStartInfoEx { + size_t this_size = 0; + uint64_t start_time_milliseconds = 0; +}; +struct AvPlayerAudioEx { + uint16_t channel_count = 0; + uint8_t reserved[2] = {}; + uint32_t sample_rate = 0; + uint32_t size = 0; + uint8_t language_code[4] = {}; + uint8_t reserved1[64] = {}; +}; +struct AvPlayerAudio { + uint16_t channel_count = 0; + uint8_t reserved[2] = {}; + uint32_t sample_rate = 0; + uint32_t size = 0; + uint8_t language_code[4] = {}; +}; +struct AvPlayerVideoEx { + uint32_t width = 0; + uint32_t height = 0; + float aspect_ratio = 0.0f; + uint8_t language_code[4] = {}; + uint8_t reserved[4] = {}; + uint32_t crop_left_offset = 0; + uint32_t crop_right_offset = 0; + uint32_t crop_top_offset = 0; + uint32_t crop_bottom_offset = 0; + uint32_t pitch = 0; + uint8_t luma_bit_depth = 0; + uint8_t chroma_bit_depth = 0; + Bool video_full_range_flag = 0; + uint8_t reserved2[5] = {}; + double framerate = 0.0; + uint32_t colour_primaries = 0; + uint32_t transfer_characteristics = 0; + uint8_t reserved3[16] = {}; +}; +struct AvPlayerVideo { + uint32_t width = 0; + uint32_t height = 0; + float aspect_ratio = 0.0f; + uint8_t language_code[4] = {}; +}; +struct AvPlayerTextPosition { + int16_t top = 0; + int16_t left = 0; + int16_t bottom = 0; + int16_t right = 0; +}; +struct AvPlayerTimedTextEx { + uint8_t language_code[4] = {}; + uint16_t text_size = 0; + uint16_t font_size = 0; + AvPlayerTextPosition position; + uint8_t reserved1[64] = {}; +}; +struct AvPlayerTimedText { + uint8_t language_code[4] = {}; + uint16_t text_size = 0; + uint16_t font_size = 0; + AvPlayerTextPosition position; +}; +union AvPlayerStreamDetailsEx { + AvPlayerAudioEx audio; + AvPlayerVideoEx video; + AvPlayerTimedTextEx subs; + uint8_t reserved1[80]; +}; +union AvPlayerStreamDetails { + AvPlayerAudio audio; + AvPlayerVideo video; + AvPlayerTimedText subs; + uint8_t reserved[16]; +}; +struct AvPlayerFrameInfo { + void* data = nullptr; + uint8_t reserved[4] = {}; + uint64_t time_stamp = 0; + AvPlayerStreamDetails details {}; +}; +struct AvPlayerFrameInfoEx { + void* data = nullptr; + uint8_t reserved[4] = {}; + uint64_t time_stamp = 0; + AvPlayerStreamDetailsEx details {}; +}; +struct AvPlayerStreamInfo { + AvPlayerStreamType type = AvPlayerStreamUnknown; + uint8_t reserved[4] = {}; + AvPlayerStreamDetails details {}; + uint64_t duration = 0; +}; +struct AvPlayerStreamInfoEx { + size_t this_size = 0; + AvPlayerStreamType type = AvPlayerStreamUnknown; + uint8_t reserved[4] = {}; + AvPlayerStreamDetailsEx details {}; + uint64_t duration = 0; +}; + +static_assert(offsetof(AvPlayerPostInitData, video_decoder_init) == 8); +static_assert(offsetof(AvPlayerVideoEx, framerate) == 48); +static_assert(sizeof(AvPlayerStreamDetailsEx) == 80); +static_assert(offsetof(AvPlayerStreamInfoEx, duration) == 96); +static_assert(offsetof(AvPlayerFrameInfoEx, details) == 24); + +static std::string fferr(int e) { + char buf[AV_ERROR_MAX_STRING_SIZE] = {}; + av_make_error_string(buf, sizeof(buf), e); + return buf; +} +static uint64_t to_ms(int64_t v, AVRational tb) { + return v == AV_NOPTS_VALUE ? 0 + : static_cast(std::max( + 0, av_rescale_q(v, tb, AVRational {1, 1000}))); +} +static uint32_t align_up(uint32_t v, uint32_t a) { + return a == 0 ? v : ((v + a - 1) / a) * a; +} +static bool ieq(std::string_view a, std::string_view b) { + return a.size() == b.size() && std::equal(a.begin(), a.end(), b.begin(), [](char l, char r) { + return std::tolower(static_cast(l)) == + std::tolower(static_cast(r)); + }); +} +static void lang(uint8_t out[4], AVDictionary* md) { + std::memset(out, 0, 4); + if (auto* e = av_dict_get(md, "language", nullptr, 0); e != nullptr && e->value != nullptr) { + std::memcpy(out, e->value, std::min(3, std::strlen(e->value))); + } else { + out[0] = 'u'; + out[1] = 'n'; + out[2] = 'd'; + } +} + +static AvPlayerSourceType source_type_from_path(std::string_view path) { + if (auto q = path.find_first_of("?#"); q != std::string_view::npos) { + path = path.substr(0, q); + } + auto dot = path.rfind('.'); + if (dot == std::string_view::npos) { + return AvPlayerSourceUnknown; + } + auto ext = path.substr(dot); + if (auto slash = ext.find('/'); slash != std::string_view::npos) { + ext = ext.substr(0, slash); + } + if (ieq(ext, ".mp4") || ieq(ext, ".m4v") || ieq(ext, ".m4a") || ieq(ext, ".mov") || + ieq(ext, ".m3d")) { + return AvPlayerSourceFileMp4; + } + if (ieq(ext, ".webm")) { + return AvPlayerSourceFileWebm; + } + if (ieq(ext, ".m3u8")) { + return AvPlayerSourceHls; + } + return AvPlayerSourceUnknown; +} + +static bool source_type_is_file(AvPlayerSourceType type) { + return type == AvPlayerSourceUnknown || type == AvPlayerSourceFileMp4 || + type == AvPlayerSourceFileWebm; +} + +static bool codec_is_supported_for_source(AvPlayerSourceType source_type, AVMediaType media_type, + AVCodecID codec_id) { + if (source_type != AvPlayerSourceFileWebm) { + return true; + } + + switch (media_type) { + case AVMEDIA_TYPE_VIDEO: + return codec_id == AV_CODEC_ID_VP8 || codec_id == AV_CODEC_ID_VP9 || + codec_id == AV_CODEC_ID_AV1; + case AVMEDIA_TYPE_AUDIO: + return codec_id == AV_CODEC_ID_OPUS || codec_id == AV_CODEC_ID_VORBIS; + default: return true; + } +} + +struct PacketQueue { + ~PacketQueue() { Clear(); } + void Push(AVPacket* p) { q.push_back(p); } + AVPacket* Pop() { + if (q.empty()) { + return nullptr; + } + auto* p = q.front(); + q.pop_front(); + return p; + } + void Clear() { + for (auto* p: q) { + av_packet_free(&p); + } + q.clear(); + } + std::deque q; +}; + +class GuestBuffer { +public: + GuestBuffer() = default; + GuestBuffer(AvPlayerMemAllocator mem, uint32_t align, uint32_t size, bool texture) + : m_mem(mem), m_texture(texture) { + m_data = + static_cast(texture ? mem.allocate_texture(mem.object_pointer, align, size) + : mem.allocate(mem.object_pointer, align, size)); + } + ~GuestBuffer() { Reset(); } + GuestBuffer(const GuestBuffer&) = delete; + GuestBuffer& operator=(const GuestBuffer&) = delete; + GuestBuffer(GuestBuffer&& r) noexcept { Move(r); } + GuestBuffer& operator=(GuestBuffer&& r) noexcept { + if (this != &r) { + Reset(); + Move(r); + } + return *this; + } + uint8_t* Get() const { return m_data; } + bool Valid() const { return m_data != nullptr; } + +private: + void Move(GuestBuffer& r) { + m_mem = r.m_mem; + m_data = r.m_data; + m_texture = r.m_texture; + r.m_data = nullptr; + } + void Reset() { + if (m_data != nullptr) { + if (m_texture) { + m_mem.deallocate_texture(m_mem.object_pointer, m_data); + } else { + m_mem.deallocate(m_mem.object_pointer, m_data); + } + m_data = nullptr; + } + } + AvPlayerMemAllocator m_mem {}; + uint8_t* m_data = nullptr; + bool m_texture = false; +}; + +class FileStreamer { +public: + explicit FileStreamer(AvPlayerFileReplacement f): file(f) {} + ~FileStreamer() { + if (ctx != nullptr) { + avio_context_free(&ctx); + } + if (opened && file.close != nullptr) { + file.close(file.object_pointer); + } + } + bool Init(const std::string& path) { + if (file.open == nullptr || file.close == nullptr || file.read_offset == nullptr || + file.size == nullptr) { + return false; + } + if (file.open(file.object_pointer, path.c_str()) < 0) { + return false; + } + opened = true; + size = file.size(file.object_pointer); + if (size == 0) { + return false; + } + auto* buf = static_cast(av_malloc(4096)); + if (buf == nullptr) { + return false; + } + ctx = avio_alloc_context(buf, 4096, 0, this, Read, nullptr, Seek); + return ctx != nullptr; + } + AVIOContext* Context() const { return ctx; } + +private: + static int Read(void* opaque, uint8_t* buf, int len) { + auto* s = static_cast(opaque); + if (s->pos >= s->size) { + return AVERROR_EOF; + } + len = static_cast(std::min(len, s->size - s->pos)); + auto r = + s->file.read_offset(s->file.object_pointer, buf, s->pos, static_cast(len)); + if (r <= 0) { + return r == 0 ? AVERROR_EOF : r; + } + s->pos += static_cast(r); + return r; + } + static int64_t Seek(void* opaque, int64_t off, int whence) { + auto* s = static_cast(opaque); + if ((whence & AVSEEK_SIZE) != 0) { + return static_cast(s->size); + } + int64_t p = whence == SEEK_SET + ? off + : (whence == SEEK_CUR + ? static_cast(s->pos) + off + : (whence == SEEK_END ? static_cast(s->size) + off : -1)); + if (p < 0) { + return -1; + } + p = std::min(p, static_cast(s->size)); + s->pos = static_cast(p); + return p; + } + AvPlayerFileReplacement file; + bool opened = false; + uint64_t pos = 0; + uint64_t size = 0; + AVIOContext* ctx = nullptr; +}; + +class Source { +public: + Source(AvPlayerMemAllocator m, AvPlayerFileReplacement f, int32_t video_buffers, bool vdec2) + : mem(m), file(f), + max_video_buffers(std::clamp(video_buffers <= 0 ? 2 : video_buffers, 2, 16)), + use_vdec2(vdec2) {} + ~Source() { Close(); } + int Init(const std::string& p, AvPlayerSourceType requested) { + path = p; + source_type = requested == AvPlayerSourceUnknown ? source_type_from_path(path) : requested; + if (source_type == AvPlayerSourceHls) { + return AVPLAYER_ERROR_NOT_SUPPORTED; + } + if (!source_type_is_file(source_type)) { + return AVPLAYER_ERROR_INVALID_PARAMS; + } + auto* raw = avformat_alloc_context(); + if (raw == nullptr) { + return AVPLAYER_ERROR_NO_MEMORY; + } + if (file.open != nullptr) { + streamer = std::make_unique(file); + if (!streamer->Init(path)) { + avformat_free_context(raw); + return AVPLAYER_ERROR_OPERATION_FAILED; + } + raw->pb = streamer->Context(); + if (auto rc = avformat_open_input(&raw, nullptr, nullptr, nullptr); rc < 0) { + LOGF("\t avformat_open_input callback failed: %s\n", fferr(rc).c_str()); + return AVPLAYER_ERROR_OPERATION_FAILED; + } + } else { + auto real = LibKernel::FileSystem::GetRealFilename(std::string(path.c_str())); + auto real_str = Common::PathToString(real); + if (auto rc = avformat_open_input(&raw, real_str.c_str(), nullptr, nullptr); rc < 0) { + LOGF("\t avformat_open_input failed: %s path=%s\n", fferr(rc).c_str(), + real_str.c_str()); + return AVPLAYER_ERROR_OPERATION_FAILED; + } + } + fmt = raw; + if (auto rc = avformat_find_stream_info(fmt, nullptr); rc < 0) { + LOGF("\t avformat_find_stream_info failed: %s\n", fferr(rc).c_str()); + return AVPLAYER_ERROR_OPERATION_FAILED; + } + return 0; + } + int StreamCount() const { return fmt == nullptr ? 0 : static_cast(fmt->nb_streams); } + int Enable(uint32_t id) { + if (fmt == nullptr || id >= fmt->nb_streams) { + return AVPLAYER_ERROR_OPERATION_FAILED; + } + if (!StreamSupported(static_cast(id))) { + return AVPLAYER_ERROR_NOT_SUPPORTED; + } + switch (fmt->streams[id]->codecpar->codec_type) { + case AVMEDIA_TYPE_VIDEO: video_id = static_cast(id); return 0; + case AVMEDIA_TYPE_AUDIO: audio_id = static_cast(id); return 0; + default: return AVPLAYER_ERROR_NOT_SUPPORTED; + } + } + int Disable(uint32_t id) { + if (video_id == static_cast(id)) { + video_id.reset(); + return 0; + } + if (audio_id == static_cast(id)) { + audio_id.reset(); + return 0; + } + return AVPLAYER_ERROR_OPERATION_FAILED; + } + int Change(uint32_t old_id, uint32_t new_id) { + if (fmt == nullptr || new_id >= fmt->nb_streams) { + return AVPLAYER_ERROR_OPERATION_FAILED; + } + if (!StreamSupported(static_cast(new_id))) { + return AVPLAYER_ERROR_NOT_SUPPORTED; + } + if (audio_id == static_cast(old_id) && + fmt->streams[new_id]->codecpar->codec_type == AVMEDIA_TYPE_AUDIO) { + audio_id = static_cast(new_id); + return Start(CurrentTime()); + } + if (video_id == static_cast(old_id) && + fmt->streams[new_id]->codecpar->codec_type == AVMEDIA_TYPE_VIDEO) { + video_id = static_cast(new_id); + return Start(CurrentTime()); + } + return AVPLAYER_ERROR_OPERATION_FAILED; + } + int Start(uint64_t ms = 0) { + std::lock_guard lock(mutex); + StopNoLock(false); + stopped = false; + paused = false; + eof = false; + if (!video_id && !audio_id) { + AutoEnable(); + } + if (!video_id && !audio_id) { + return AVPLAYER_ERROR_OPERATION_FAILED; + } + if (!OpenCodecs()) { + return AVPLAYER_ERROR_OPERATION_FAILED; + } + if (video_id) { + auto* video_stream = fmt->streams[video_id.value()]; + uint32_t video_size = VideoBufferSize(video_stream); + for (int i = 0; i < max_video_buffers; i++) { + auto buffer = std::make_unique(mem, 0x100, video_size, true); + if (!buffer->Valid()) { + video_buffers.clear(); + return AVPLAYER_ERROR_NO_MEMORY; + } + video_buffers.push_back(std::move(buffer)); + } + } + SeekNoLock(ms); + start_time_ms = ms; + clock_start = std::chrono::steady_clock::now(); + if (video_id) { + ::printf("AvPlayer video started playing\n"); + } + return 0; + } + int Stop() { + std::lock_guard lock(mutex); + bool was_playing_video = !stopped && video_id.has_value(); + StopNoLock(true); + if (was_playing_video) { + ::printf("AvPlayer video stopped\n"); + } + return 0; + } + void Pause() { + paused = true; + pause_time = std::chrono::steady_clock::now(); + } + void Resume() { + if (paused) { + paused_extra += std::chrono::steady_clock::now() - pause_time; + } + paused = false; + seek_video_frame_pending = false; + } + void SetLoop(bool v) { loop = v; } + void SetSync(uint32_t v) { sync_mode = v; } + void SetTrick(int32_t v) { trick_speed = v; } + bool Active() const { return !stopped && !eof; } + uint64_t CurrentTime() const { + if (stopped) { + return 0; + } + using namespace std::chrono; + auto now = paused ? pause_time : steady_clock::now(); + return start_time_ms + + static_cast( + duration_cast(now - clock_start - paused_extra).count()); + } + int Jump(uint64_t ms) { + std::lock_guard lock(mutex); + SeekNoLock(ms); + start_time_ms = ms; + clock_start = std::chrono::steady_clock::now(); + paused_extra = {}; + if (paused) { + pause_time = clock_start; + } + seek_video_frame_pending = paused && video_id.has_value(); + return 0; + } + int Info(uint32_t id, AvPlayerStreamInfo* out) const { + if (out == nullptr) { + return AVPLAYER_ERROR_INVALID_PARAMS; + } + std::memset(out, 0, sizeof(*out)); + if (fmt == nullptr || id >= fmt->nb_streams) { + return AVPLAYER_ERROR_OPERATION_FAILED; + } + FillCommon(id, &out->type, &out->duration, &out->details, nullptr); + return 0; + } + int InfoEx(uint32_t id, AvPlayerStreamInfoEx* out) const { + if (out == nullptr) { + return AVPLAYER_ERROR_INVALID_PARAMS; + } + std::memset(out, 0, sizeof(*out)); + out->this_size = sizeof(*out); + if (fmt == nullptr || id >= fmt->nb_streams) { + return AVPLAYER_ERROR_OPERATION_FAILED; + } + FillCommon(id, &out->type, &out->duration, nullptr, &out->details); + return 0; + } + bool Video(AvPlayerFrameInfoEx* out) { + if (out == nullptr || stopped || !video_id) { + return false; + } + std::lock_guard lock(mutex); + const bool deliver_seek_frame = paused && seek_video_frame_pending; + if (paused && !deliver_seek_frame) { + return false; + } + if (current_video != nullptr) { + video_buffers.push_back(std::move(current_video)); + } + if (!DecodeUntil(video_id.value())) { + if (deliver_seek_frame) { + seek_video_frame_pending = false; + } + return false; + } + *out = current_video_info; + if (deliver_seek_frame) { + seek_video_frame_pending = false; + } + return true; + } + bool Audio(AvPlayerFrameInfo* out) { + if (out == nullptr || paused || stopped || !audio_id || + trick_speed != AVPLAYER_TRICK_SPEED_NORMAL) { + return false; + } + std::lock_guard lock(mutex); + if (!DecodeUntil(audio_id.value())) { + return false; + } + std::memset(out, 0, sizeof(*out)); + out->data = current_audio_info.data; + out->time_stamp = current_audio_info.time_stamp; + out->details.audio.channel_count = current_audio_info.details.audio.channel_count; + out->details.audio.sample_rate = current_audio_info.details.audio.sample_rate; + out->details.audio.size = current_audio_info.details.audio.size; + std::memcpy(out->details.audio.language_code, + current_audio_info.details.audio.language_code, 4); + last_audio_ts = out->time_stamp; + return true; + } + std::optional TakeWarning() { + if (warnings.empty()) { + return std::nullopt; + } + auto w = warnings.front(); + warnings.pop_front(); + return w; + } + +private: + void Close() { + Stop(); + if (fmt != nullptr) { + avformat_close_input(&fmt); + } + streamer.reset(); + } + void StopNoLock(bool mark) { + video_packets.Clear(); + audio_packets.Clear(); + video_buffers.clear(); + current_video.reset(); + current_audio.reset(); + if (sws != nullptr) { + sws_freeContext(sws); + sws = nullptr; + } + sws_src_width = 0; + sws_src_height = 0; + sws_src_format = AV_PIX_FMT_NONE; + if (swr != nullptr) { + swr_free(&swr); + } + if (video_ctx != nullptr) { + avcodec_free_context(&video_ctx); + } + if (audio_ctx != nullptr) { + avcodec_free_context(&audio_ctx); + } + eof = true; + seek_video_frame_pending = false; + if (mark) { + stopped = true; + } + } + void AutoEnable() { + for (uint32_t i = 0; i < fmt->nb_streams; i++) { + if (!StreamSupported(static_cast(i))) { + continue; + } + auto t = fmt->streams[i]->codecpar->codec_type; + if (t == AVMEDIA_TYPE_VIDEO && !video_id) { + video_id = static_cast(i); + } + if (t == AVMEDIA_TYPE_AUDIO && !audio_id) { + audio_id = static_cast(i); + } + } + } + bool StreamSupported(int id) const { + if (fmt == nullptr || id < 0 || id >= static_cast(fmt->nb_streams)) { + return false; + } + auto* s = fmt->streams[id]; + auto t = s->codecpar->codec_type; + if (t != AVMEDIA_TYPE_VIDEO && t != AVMEDIA_TYPE_AUDIO) { + return true; + } + if (!codec_is_supported_for_source(source_type, t, s->codecpar->codec_id)) { + LOGF("\t unsupported codec for source type: stream=%d codec=%d source=%u\n", id, + static_cast(s->codecpar->codec_id), static_cast(source_type)); + return false; + } + if (avcodec_find_decoder(s->codecpar->codec_id) == nullptr) { + LOGF("\t decoder not found: stream=%d codec=%d\n", id, + static_cast(s->codecpar->codec_id)); + return false; + } + return true; + } + bool OpenCodec(int id, AVCodecContext** out) { + auto* s = fmt->streams[id]; + auto* dec = avcodec_find_decoder(s->codecpar->codec_id); + if (dec == nullptr) { + LOGF("\t avcodec_find_decoder failed: stream=%d codec=%d\n", id, + static_cast(s->codecpar->codec_id)); + return false; + } + auto* c = avcodec_alloc_context3(dec); + if (c == nullptr) { + return false; + } + if (avcodec_parameters_to_context(c, s->codecpar) < 0 || + avcodec_open2(c, dec, nullptr) < 0) { + avcodec_free_context(&c); + return false; + } + *out = c; + return true; + } + bool OpenCodecs() { + if (video_id && !OpenCodec(video_id.value(), &video_ctx)) { + return false; + } + if (audio_id && !OpenCodec(audio_id.value(), &audio_ctx)) { + return false; + } + return true; + } + void SeekNoLock(uint64_t ms) { + int sid = video_id.value_or(audio_id.value_or(-1)); + int64_t ts = sid >= 0 ? av_rescale_q(static_cast(ms), AVRational {1, 1000}, + fmt->streams[sid]->time_base) + : static_cast(ms) * 1000; + avformat_seek_file(fmt, sid, INT64_MIN, ts, INT64_MAX, 0); + eof = false; + if (video_ctx != nullptr) { + avcodec_flush_buffers(video_ctx); + } + if (audio_ctx != nullptr) { + avcodec_flush_buffers(audio_ctx); + } + video_packets.Clear(); + audio_packets.Clear(); + } + bool DecodeUntil(int wanted) { + PacketQueue& own = wanted == video_id.value_or(-1) ? video_packets : audio_packets; + while (!eof) { + AVPacket* p = own.Pop(); + if (p == nullptr) { + p = av_packet_alloc(); + auto rc = av_read_frame(fmt, p); + if (rc < 0) { + av_packet_free(&p); + if (rc == AVERROR_EOF && loop) { + warnings.push_back(AVPLAYER_WARNING_LOOPING_BACK); + SeekNoLock(0); + clock_start = std::chrono::steady_clock::now(); + start_time_ms = 0; + continue; + } + eof = true; + return false; + } + if (video_id && p->stream_index == video_id.value() && wanted != p->stream_index) { + video_packets.Push(p); + continue; + } + if (audio_id && p->stream_index == audio_id.value() && wanted != p->stream_index) { + audio_packets.Push(p); + continue; + } + if (p->stream_index != wanted) { + av_packet_free(&p); + continue; + } + } + bool ok = wanted == video_id.value_or(-1) ? DecodeVideo(p) : DecodeAudio(p); + av_packet_free(&p); + if (ok) { + return true; + } + } + return false; + } + bool DecodeVideo(AVPacket* p) { + if (video_ctx == nullptr || avcodec_send_packet(video_ctx, p) < 0) { + return false; + } + AVFrame* f = av_frame_alloc(); + auto rc = avcodec_receive_frame(video_ctx, f); + if (rc < 0) { + av_frame_free(&f); + return false; + } + PrepareVideo(f); + av_frame_free(&f); + return true; + } + bool DecodeAudio(AVPacket* p) { + if (audio_ctx == nullptr || avcodec_send_packet(audio_ctx, p) < 0) { + return false; + } + AVFrame* f = av_frame_alloc(); + auto rc = avcodec_receive_frame(audio_ctx, f); + if (rc < 0) { + av_frame_free(&f); + return false; + } + PrepareAudio(f); + av_frame_free(&f); + return true; + } + uint32_t Width(AVStream* s) const { + return use_vdec2 ? static_cast(s->codecpar->width) + : align_up(static_cast(s->codecpar->width), 16); + } + uint32_t Height(AVStream* s) const { + return use_vdec2 ? static_cast(s->codecpar->height) + : align_up(static_cast(s->codecpar->height), 16); + } + float Aspect(AVStream* s) const { + if (s->codecpar->height == 0) { + return 0.0f; + } + double r = static_cast(s->codecpar->width) / s->codecpar->height; + if (s->sample_aspect_ratio.num > 0 && s->sample_aspect_ratio.den > 0) { + r *= av_q2d(s->sample_aspect_ratio); + } + return static_cast(r); + } + uint64_t Duration(AVStream* s) const { + return s->duration != AV_NOPTS_VALUE + ? to_ms(s->duration, s->time_base) + : (fmt->duration > 0 ? static_cast(fmt->duration / 1000) : 0); + } + uint32_t VideoPitch(AVStream* s) const { return align_up(Width(s), 256u); } + uint32_t VideoBufferSize(AVStream* s) const { return VideoPitch(s) * Height(s) * 3 / 2; } + void FillVideo(AVStream* s, AvPlayerVideo* v) const { + std::memset(v, 0, sizeof(*v)); + v->width = Width(s); + v->height = Height(s); + v->aspect_ratio = Aspect(s); + lang(v->language_code, s->metadata); + } + void FillVideoEx(AVStream* s, AvPlayerVideoEx* v) const { + std::memset(v, 0, sizeof(*v)); + v->width = Width(s); + v->height = Height(s); + v->aspect_ratio = Aspect(s); + v->pitch = VideoPitch(s); + v->luma_bit_depth = 8; + v->chroma_bit_depth = 8; + v->video_full_range_flag = s->codecpar->color_range == AVCOL_RANGE_JPEG ? 1 : 0; + auto fr = s->avg_frame_rate.num != 0 ? s->avg_frame_rate : s->r_frame_rate; + v->framerate = av_q2d(fr); + v->colour_primaries = static_cast(s->codecpar->color_primaries); + v->transfer_characteristics = static_cast(s->codecpar->color_trc); + lang(v->language_code, s->metadata); + } + void FillAudio(AVStream* s, AvPlayerAudio* a, uint32_t size) const { + std::memset(a, 0, sizeof(*a)); + a->channel_count = static_cast(s->codecpar->ch_layout.nb_channels); + a->sample_rate = static_cast(s->codecpar->sample_rate); + a->size = size; + lang(a->language_code, s->metadata); + } + void FillAudioEx(AVStream* s, AvPlayerAudioEx* a, uint32_t size) const { + std::memset(a, 0, sizeof(*a)); + a->channel_count = static_cast(s->codecpar->ch_layout.nb_channels); + a->sample_rate = static_cast(s->codecpar->sample_rate); + a->size = size; + lang(a->language_code, s->metadata); + } + void FillCommon(uint32_t id, AvPlayerStreamType* type, uint64_t* dur, AvPlayerStreamDetails* d, + AvPlayerStreamDetailsEx* ex) const { + auto* s = fmt->streams[id]; + *dur = Duration(s); + switch (s->codecpar->codec_type) { + case AVMEDIA_TYPE_VIDEO: + *type = AvPlayerStreamVideo; + if (d) { + FillVideo(s, &d->video); + } + if (ex) { + FillVideoEx(s, &ex->video); + } + break; + case AVMEDIA_TYPE_AUDIO: + *type = AvPlayerStreamAudio; + if (d) { + FillAudio(s, &d->audio, 0); + } + if (ex) { + FillAudioEx(s, &ex->audio, 0); + } + break; + case AVMEDIA_TYPE_SUBTITLE: *type = AvPlayerStreamTimedText; break; + default: *type = AvPlayerStreamUnknown; break; + } + } + void PrepareVideo(AVFrame* src) { + auto* s = fmt->streams[video_id.value()]; + uint32_t h = Height(s); + uint32_t pitch = VideoPitch(s); + auto* current_video = AllocateVideoBuffer(); + if (current_video == nullptr) { + return; + } + AVFrame* nv12 = src; + AVFrame* tmp = nullptr; + if (src->format != AV_PIX_FMT_NV12) { + tmp = av_frame_alloc(); + if (tmp == nullptr) { + return; + } + tmp->format = AV_PIX_FMT_NV12; + tmp->width = src->width; + tmp->height = src->height; + if (av_frame_get_buffer(tmp, 0) < 0) { + av_frame_free(&tmp); + return; + } + if (sws == nullptr || sws_src_width != src->width || sws_src_height != src->height || + sws_src_format != src->format) { + if (sws != nullptr) { + sws_freeContext(sws); + } + sws = sws_getContext( + src->width, src->height, static_cast(src->format), src->width, + src->height, AV_PIX_FMT_NV12, SWS_FAST_BILINEAR, nullptr, nullptr, nullptr); + sws_src_width = src->width; + sws_src_height = src->height; + sws_src_format = src->format; + } + if (sws == nullptr) { + av_frame_free(&tmp); + return; + } + sws_scale(sws, src->data, src->linesize, 0, src->height, tmp->data, tmp->linesize); + nv12 = tmp; + } + auto* dst = current_video->Get(); + std::memset(dst, 0, static_cast(pitch) * h * 3 / 2); + for (int y = 0; y < src->height; y++) { + std::memcpy(dst + y * pitch, nv12->data[0] + y * nv12->linesize[0], src->width); + } + auto* c = dst + pitch * h; + for (int y = 0; y < src->height / 2; y++) { + std::memcpy(c + y * pitch, nv12->data[1] + y * nv12->linesize[1], src->width); + } + std::memset(¤t_video_info, 0, sizeof(current_video_info)); + current_video_info.data = dst; + current_video_info.time_stamp = to_ms( + src->best_effort_timestamp != AV_NOPTS_VALUE ? src->best_effort_timestamp : src->pts, + s->time_base); + FillVideoEx(s, ¤t_video_info.details.video); + current_video_info.details.video.crop_left_offset = static_cast(src->crop_left); + current_video_info.details.video.crop_right_offset = + static_cast(src->crop_right + (pitch - src->width)); + current_video_info.details.video.crop_top_offset = static_cast(src->crop_top); + current_video_info.details.video.crop_bottom_offset = + static_cast(src->crop_bottom + (h - src->height)); + current_video_info.details.video.pitch = pitch; + if (tmp) { + av_frame_free(&tmp); + } + } + GuestBuffer* AllocateVideoBuffer() { + if (video_buffers.empty()) { + return nullptr; + } + current_video = std::move(video_buffers.front()); + video_buffers.pop_front(); + if (current_video == nullptr || !current_video->Valid()) { + current_video.reset(); + return nullptr; + } + return current_video.get(); + } + void PrepareAudio(AVFrame* src) { + auto timestamp = to_ms( + src->best_effort_timestamp != AV_NOPTS_VALUE ? src->best_effort_timestamp : src->pts, + fmt->streams[audio_id.value()]->time_base); + AVFrame* pcm = src; + AVFrame* tmp = nullptr; + if (src->format != AV_SAMPLE_FMT_S16) { + tmp = av_frame_alloc(); + if (tmp == nullptr) { + return; + } + tmp->format = AV_SAMPLE_FMT_S16; + tmp->sample_rate = src->sample_rate; + tmp->nb_samples = src->nb_samples; + av_channel_layout_copy(&tmp->ch_layout, &src->ch_layout); + if (av_frame_get_buffer(tmp, 0) < 0) { + av_frame_free(&tmp); + return; + } + if (swr == nullptr) { + SwrContext* ctx = nullptr; + AVChannelLayout out; + av_channel_layout_copy(&out, &src->ch_layout); + swr_alloc_set_opts2(&ctx, &out, AV_SAMPLE_FMT_S16, src->sample_rate, + &src->ch_layout, static_cast(src->format), + src->sample_rate, 0, nullptr); + swr = ctx; + swr_init(swr); + av_channel_layout_uninit(&out); + } + if (swr == nullptr || swr_convert_frame(swr, tmp, src) < 0) { + av_frame_free(&tmp); + return; + } + pcm = tmp; + } + auto channels = std::max(1, pcm->ch_layout.nb_channels); + auto size = static_cast(channels * pcm->nb_samples * sizeof(int16_t)); + current_audio = std::make_unique(mem, 0x100, size, false); + if (!current_audio->Valid()) { + if (tmp) { + av_frame_free(&tmp); + } + return; + } + std::memcpy(current_audio->Get(), pcm->data[0], size); + auto* s = fmt->streams[audio_id.value()]; + std::memset(¤t_audio_info, 0, sizeof(current_audio_info)); + current_audio_info.data = current_audio->Get(); + current_audio_info.time_stamp = timestamp; + FillAudioEx(s, ¤t_audio_info.details.audio, size); + current_audio_info.details.audio.channel_count = static_cast(channels); + current_audio_info.details.audio.sample_rate = static_cast(pcm->sample_rate); + if (tmp) { + av_frame_free(&tmp); + } + } + AvPlayerMemAllocator mem; + AvPlayerFileReplacement file; + int max_video_buffers = 2; + bool use_vdec2 = false; + AvPlayerSourceType source_type = AvPlayerSourceUnknown; + std::string path; + std::unique_ptr streamer; + AVFormatContext* fmt = nullptr; + AVCodecContext* video_ctx = nullptr; + AVCodecContext* audio_ctx = nullptr; + SwsContext* sws = nullptr; + SwrContext* swr = nullptr; + int sws_src_width = 0; + int sws_src_height = 0; + int sws_src_format = AV_PIX_FMT_NONE; + std::optional video_id; + std::optional audio_id; + PacketQueue video_packets; + PacketQueue audio_packets; + std::deque> video_buffers; + std::unique_ptr current_video; + std::unique_ptr current_audio; + AvPlayerFrameInfoEx current_video_info {}; + AvPlayerFrameInfoEx current_audio_info {}; + std::deque warnings; + mutable std::mutex mutex; + bool stopped = true; + bool paused = false; + bool eof = true; + bool seek_video_frame_pending = false; + bool loop = false; + int32_t trick_speed = AVPLAYER_TRICK_SPEED_NORMAL; + uint32_t sync_mode = 0; + uint64_t start_time_ms = 0; + uint64_t last_audio_ts = 0; + std::chrono::steady_clock::time_point clock_start {}; + std::chrono::steady_clock::time_point pause_time {}; + std::chrono::steady_clock::duration paused_extra {}; +}; + +struct AvPlayerInternal { + AvPlayerMemAllocator mem; + AvPlayerFileReplacement file; + AvPlayerEventReplacement event; + AvPlayerPostInitData post_init {}; + bool auto_start = false; + int32_t video_buffers = 2; + uint32_t start_bandwidth = 0; + uint32_t minimum_bandwidth = 0; + uint32_t maximum_bandwidth = 0; + std::unique_ptr source; +}; + +static bool valid_allocators(const AvPlayerMemAllocator& m) { + return m.allocate && m.deallocate && m.allocate_texture && m.deallocate_texture; +} +static void emit_event(AvPlayerInternal* h, int32_t id, void* data = nullptr) { + if (h != nullptr && h->event.event_callback != nullptr) { + LOGF("\t event_id = %d\n", id); + h->event.event_callback(h->event.object_pointer, id, 0, data); + } +} +static void pump_warnings(AvPlayerInternal* h) { + if (h == nullptr || h->source == nullptr) { + return; + } + while (auto w = h->source->TakeWarning()) { + int32_t warning = *w; + emit_event(h, AVPLAYER_EVENT_WARNING_ID, &warning); + } +} +static AvPlayerInternal* create_player(const AvPlayerMemAllocator& mem, + const AvPlayerFileReplacement& file, + const AvPlayerEventReplacement& event, bool auto_start, + int32_t video_buffers) { + if (!valid_allocators(mem)) { + return nullptr; + } + auto* h = new AvPlayerInternal; + h->mem = mem; + h->file = file; + h->event = event; + h->auto_start = auto_start; + h->video_buffers = std::clamp(video_buffers <= 0 ? 2 : video_buffers, 2, 16); + h->post_init.demux_video_buffer_size = 4 * 1024 * 1024; + return h; +} +static int add_source(AvPlayerInternal* h, const std::string& filename, AvPlayerSourceType type) { + if (h == nullptr) { + return AVPLAYER_ERROR_INVALID_PARAMS; + } + if (h->source != nullptr) { + return AVPLAYER_ERROR_OPERATION_FAILED; + } + bool vdec2 = + h->post_init.video_decoder_init.decoder_type.video_type == AvPlayerVideoDecoderSoftware2; + auto s = std::make_unique(h->mem, h->file, h->video_buffers, vdec2); + if (auto rc = s->Init(filename, type); rc < 0) { + return rc; + } + h->source = std::move(s); + emit_event(h, AVPLAYER_EVENT_STATE_READY); + if (h->auto_start) { + auto rc = h->source->Start(); + if (rc == 0) { + emit_event(h, AVPLAYER_EVENT_STATE_PLAY); + } + return rc; + } + return 0; +} +static void log_init_common(const AvPlayerMemAllocator& mem, const AvPlayerFileReplacement& file, + const AvPlayerEventReplacement& event, AvPlayerDebuglevels debug_level, + int32_t buffers, uint32_t priority, Bool auto_start, + const char* language) { + LOGF("\t memory_replacement.object_pointer = %016" PRIx64 "\n", + reinterpret_cast(mem.object_pointer)); + LOGF("\t memory_replacement.allocate = %016" PRIx64 + "\n\t memory_replacement.deallocate = %016" PRIx64 "\n", + reinterpret_cast(mem.allocate), reinterpret_cast(mem.deallocate)); + LOGF("\t memory_replacement.allocate_texture = %016" PRIx64 "\n", + reinterpret_cast(mem.allocate_texture)); + LOGF("\t memory_replacement.deallocate_texture = %016" PRIx64 "\n", + reinterpret_cast(mem.deallocate_texture)); + LOGF("\t file_replacement.object_pointer = %016" PRIx64 "\n", + reinterpret_cast(file.object_pointer)); + LOGF("\t file_replacement.open = %016" PRIx64 + "\n\t file_replacement.close = %016" PRIx64 + "\n\t file_replacement.read_offset = %016" PRIx64 + "\n\t file_replacement.size = %016" PRIx64 "\n", + reinterpret_cast(file.open), reinterpret_cast(file.close), + reinterpret_cast(file.read_offset), reinterpret_cast(file.size)); + LOGF("\t event_replacement.object_pointer = %016" PRIx64 "\n", + reinterpret_cast(event.object_pointer)); + LOGF("\t event_replacement.event_callback = %016" PRIx64 "\n", + reinterpret_cast(event.event_callback)); + LOGF("\t debug_level = %s\n\t num_output_video_framebuffers " + "= %d\n\t base_priority " + " = %u\n\t auto_start = %u\n\t default_language " + " = %s\n", + Common::EnumName(debug_level).c_str(), buffers, priority, auto_start, + language == nullptr ? "(null)" : language); +} + +AvPlayerInternal* KYTY_SYSV_ABI AvPlayerInit(AvPlayerInitData* init) { + PRINT_NAME(); + if (init == nullptr) { + return nullptr; + } + log_init_common(init->memory_replacement, init->file_replacement, init->event_replacement, + init->debug_level, init->num_output_video_framebuffers, init->base_priority, + init->auto_start, init->default_language); + return create_player(init->memory_replacement, init->file_replacement, init->event_replacement, + init->auto_start != 0, init->num_output_video_framebuffers); +} +int KYTY_SYSV_ABI AvPlayerInitEx(const void* init_ex, AvPlayerInternal** handle) { + PRINT_NAME(); + if (init_ex == nullptr || handle == nullptr) { + return AVPLAYER_ERROR_INVALID_PARAMS; + } + const auto* init = static_cast(init_ex); + LOGF("\t this_size = %zu\n", init->this_size); + log_init_common(init->memory_replacement, init->file_replacement, init->event_replacement, + init->debug_level, init->num_output_video_framebuffers, 0, init->auto_start, + init->default_language); + *handle = + create_player(init->memory_replacement, init->file_replacement, init->event_replacement, + init->auto_start != 0, init->num_output_video_framebuffers); + return *handle == nullptr ? AVPLAYER_ERROR_INVALID_PARAMS : 0; +} +int KYTY_SYSV_ABI AvPlayerPostInit(AvPlayerInternal* h, const void* post_init) { + PRINT_NAME(); + if (h == nullptr) { + return AVPLAYER_ERROR_INVALID_PARAMS; + } + if (post_init != nullptr) { + h->post_init = *static_cast(post_init); + LOGF("\t demux_video_buffer_size = %u\n", h->post_init.demux_video_buffer_size); + } + return 0; +} +int KYTY_SYSV_ABI AvPlayerAddSource(AvPlayerInternal* h, const char* filename) { + PRINT_NAME(); + if (h == nullptr || filename == nullptr) { + return AVPLAYER_ERROR_INVALID_PARAMS; + } + LOGF("\t filename = %s\n", filename); + return add_source(h, filename, AvPlayerSourceUnknown); +} +int KYTY_SYSV_ABI AvPlayerAddSourceEx(AvPlayerInternal* h, uint32_t uri_type, + const void* source_details) { + PRINT_NAME(); + if (h == nullptr || uri_type != AvPlayerUriTypeSource || source_details == nullptr) { + return AVPLAYER_ERROR_INVALID_PARAMS; + } + const auto* d = static_cast(source_details); + if (d->uri.name == nullptr || d->uri.length == 0) { + return AVPLAYER_ERROR_INVALID_PARAMS; + } + if (d->source_type != AvPlayerSourceUnknown && d->source_type != AvPlayerSourceFileMp4 && + d->source_type != AvPlayerSourceFileWebm && d->source_type != AvPlayerSourceHls) { + return AVPLAYER_ERROR_INVALID_PARAMS; + } + std::string filename(d->uri.name, d->uri.length); + LOGF("\t uri_type = %u\n\t source_type = %u\n\t filename = %s\n", uri_type, + static_cast(d->source_type), filename.c_str()); + return add_source(h, filename, d->source_type); +} +int KYTY_SYSV_ABI AvPlayerStreamCount(AvPlayerInternal* h) { + PRINT_NAME(); + return h == nullptr ? AVPLAYER_ERROR_INVALID_PARAMS + : (h->source == nullptr ? 0 : h->source->StreamCount()); +} +int KYTY_SYSV_ABI AvPlayerGetStreamInfo(AvPlayerInternal* h, uint32_t stream_id, void* info) { + PRINT_NAME(); + if (h == nullptr || info == nullptr) { + return AVPLAYER_ERROR_INVALID_PARAMS; + } + return h->source == nullptr + ? AVPLAYER_ERROR_OPERATION_FAILED + : h->source->Info(stream_id, static_cast(info)); +} +int KYTY_SYSV_ABI AvPlayerGetStreamInfoEx(AvPlayerInternal* h, uint32_t stream_id, void* info) { + PRINT_NAME(); + if (h == nullptr || info == nullptr) { + return AVPLAYER_ERROR_INVALID_PARAMS; + } + return h->source == nullptr + ? AVPLAYER_ERROR_OPERATION_FAILED + : h->source->InfoEx(stream_id, static_cast(info)); +} +int KYTY_SYSV_ABI AvPlayerEnableStream(AvPlayerInternal* h, uint32_t stream_id) { + PRINT_NAME(); + return h == nullptr || h->source == nullptr ? AVPLAYER_ERROR_INVALID_PARAMS + : h->source->Enable(stream_id); +} +int KYTY_SYSV_ABI AvPlayerDisableStream(AvPlayerInternal* h, uint32_t stream_id) { + PRINT_NAME(); + return h == nullptr || h->source == nullptr ? AVPLAYER_ERROR_INVALID_PARAMS + : h->source->Disable(stream_id); +} +int KYTY_SYSV_ABI AvPlayerChangeStream(AvPlayerInternal* h, uint32_t old_stream_id, + uint32_t new_stream_id) { + PRINT_NAME(); + return h == nullptr || h->source == nullptr ? AVPLAYER_ERROR_INVALID_PARAMS + : h->source->Change(old_stream_id, new_stream_id); +} +int KYTY_SYSV_ABI AvPlayerStart(AvPlayerInternal* h) { + PRINT_NAME(); + if (h == nullptr || h->source == nullptr) { + return AVPLAYER_ERROR_INVALID_PARAMS; + } + auto rc = h->source->Start(); + if (rc == 0) { + emit_event(h, AVPLAYER_EVENT_STATE_PLAY); + } + return rc; +} +int KYTY_SYSV_ABI AvPlayerStartEx(AvPlayerInternal* h, const void* start_info_ex) { + PRINT_NAME(); + if (h == nullptr || h->source == nullptr) { + return AVPLAYER_ERROR_INVALID_PARAMS; + } + uint64_t ms = + start_info_ex == nullptr + ? 0 + : static_cast(start_info_ex)->start_time_milliseconds; + auto rc = h->source->Start(ms); + if (rc == 0) { + emit_event(h, AVPLAYER_EVENT_STATE_PLAY); + } + return rc; +} +int KYTY_SYSV_ABI AvPlayerStop(AvPlayerInternal* h) { + PRINT_NAME(); + if (h == nullptr || h->source == nullptr) { + return AVPLAYER_ERROR_INVALID_PARAMS; + } + auto rc = h->source->Stop(); + emit_event(h, AVPLAYER_EVENT_STATE_STOP); + return rc; +} +int KYTY_SYSV_ABI AvPlayerPause(AvPlayerInternal* h) { + PRINT_NAME(); + if (h == nullptr || h->source == nullptr) { + return AVPLAYER_ERROR_INVALID_PARAMS; + } + h->source->Pause(); + emit_event(h, AVPLAYER_EVENT_STATE_PAUSE); + return 0; +} +int KYTY_SYSV_ABI AvPlayerResume(AvPlayerInternal* h) { + PRINT_NAME(); + if (h == nullptr || h->source == nullptr) { + return AVPLAYER_ERROR_INVALID_PARAMS; + } + h->source->Resume(); + emit_event(h, AVPLAYER_EVENT_STATE_PLAY); + return 0; +} +int KYTY_SYSV_ABI AvPlayerSetLooping(AvPlayerInternal* h, Bool loop) { + PRINT_NAME(); + if (h == nullptr || h->source == nullptr) { + return AVPLAYER_ERROR_INVALID_PARAMS; + } + h->source->SetLoop(loop != 0); + return 0; +} +int KYTY_SYSV_ABI AvPlayerSetTrickSpeed(AvPlayerInternal* h, int32_t trick_speed) { + PRINT_NAME(); + if (h == nullptr || h->source == nullptr) { + return AVPLAYER_ERROR_INVALID_PARAMS; + } + trick_speed = std::clamp(trick_speed, -AVPLAYER_TRICK_SPEED_MAX, AVPLAYER_TRICK_SPEED_MAX); + if (trick_speed != AVPLAYER_TRICK_SPEED_NORMAL && trick_speed > -AVPLAYER_TRICK_SPEED_MIN && + trick_speed < AVPLAYER_TRICK_SPEED_MIN) { + return AVPLAYER_ERROR_NOT_SUPPORTED; + } + h->source->SetTrick(trick_speed); + return trick_speed == AVPLAYER_TRICK_SPEED_NORMAL ? 0 : AVPLAYER_ERROR_NOT_SUPPORTED; +} +int KYTY_SYSV_ABI AvPlayerSetAvSyncMode(AvPlayerInternal* h, uint32_t sync_mode) { + PRINT_NAME(); + if (h == nullptr || h->source == nullptr) { + return AVPLAYER_ERROR_INVALID_PARAMS; + } + if (sync_mode > 1) { + return AVPLAYER_ERROR_NOT_SUPPORTED; + } + h->source->SetSync(sync_mode); + return 0; +} +int KYTY_SYSV_ABI AvPlayerSetAvailableBandwidth(AvPlayerInternal* h, uint32_t start_bandwidth, + uint32_t minimum_bandwidth, + uint32_t maximum_bandwidth) { + PRINT_NAME(); + if (h == nullptr || (minimum_bandwidth != 0 && maximum_bandwidth != 0 && + minimum_bandwidth > maximum_bandwidth)) { + return AVPLAYER_ERROR_INVALID_PARAMS; + } + h->start_bandwidth = start_bandwidth; + h->minimum_bandwidth = minimum_bandwidth; + h->maximum_bandwidth = maximum_bandwidth; + return h->source == nullptr ? 0 : AVPLAYER_ERROR_OPERATION_FAILED; +} +Bool KYTY_SYSV_ABI AvPlayerGetVideoData(AvPlayerInternal* h, AvPlayerFrameInfo* video_info) { + PRINT_NAME(); + if (h == nullptr || h->source == nullptr || video_info == nullptr) { + return 0; + } + AvPlayerFrameInfoEx ex {}; + if (!h->source->Video(&ex)) { + pump_warnings(h); + return 0; + } + std::memset(video_info, 0, sizeof(*video_info)); + video_info->data = ex.data; + video_info->time_stamp = ex.time_stamp; + video_info->details.video.width = ex.details.video.width; + video_info->details.video.height = ex.details.video.height; + video_info->details.video.aspect_ratio = ex.details.video.aspect_ratio; + std::memcpy(video_info->details.video.language_code, ex.details.video.language_code, 4); + return 1; +} +Bool KYTY_SYSV_ABI AvPlayerGetVideoDataEx(AvPlayerInternal* h, AvPlayerFrameInfoEx* video_info) { + PRINT_NAME(); + if (h == nullptr || h->source == nullptr || video_info == nullptr) { + return 0; + } + auto ok = h->source->Video(video_info) ? 1 : 0; + pump_warnings(h); + return ok; +} +Bool KYTY_SYSV_ABI AvPlayerGetAudioData(AvPlayerInternal* h, AvPlayerFrameInfo* audio_info) { + PRINT_NAME(); + if (h == nullptr || h->source == nullptr || audio_info == nullptr) { + return 0; + } + auto ok = h->source->Audio(audio_info) ? 1 : 0; + pump_warnings(h); + return ok; +} +Bool KYTY_SYSV_ABI AvPlayerIsActive(AvPlayerInternal* h) { + PRINT_NAME(); + return h != nullptr && h->source != nullptr && h->source->Active() ? 1 : 0; +} +uint64_t KYTY_SYSV_ABI AvPlayerCurrentTime(AvPlayerInternal* h) { + PRINT_NAME(); + return h == nullptr || h->source == nullptr ? 0 : h->source->CurrentTime(); +} +int KYTY_SYSV_ABI AvPlayerJumpToTime(AvPlayerInternal* h, uint64_t time_ms) { + PRINT_NAME(); + if (h == nullptr || h->source == nullptr) { + return AVPLAYER_ERROR_INVALID_PARAMS; + } + auto rc = h->source->Jump(time_ms); + if (rc == 0) { + int32_t w = AVPLAYER_WARNING_JUMP_COMPLETE; + emit_event(h, AVPLAYER_EVENT_WARNING_ID, &w); + } + return rc; +} +int KYTY_SYSV_ABI AvPlayerClose(AvPlayerInternal* h) { + PRINT_NAME(); + if (h == nullptr) { + return AVPLAYER_ERROR_INVALID_PARAMS; + } + h->source.reset(); + delete h; + return 0; +} +int KYTY_SYSV_ABI AvPlayerSetLogCallback(void* callback, void* user_data) { + PRINT_NAME(); + LOGF("\t callback = 0x%016" PRIx64 "\n\t user_data = 0x%016" PRIx64 "\n", + reinterpret_cast(callback), reinterpret_cast(user_data)); + return 0; +} + +} // namespace Libs::Audio::AvPlayer diff --git a/src/libs/controller.cpp b/src/libs/controller.cpp new file mode 100644 index 000000000..2b9cecd6d --- /dev/null +++ b/src/libs/controller.cpp @@ -0,0 +1,546 @@ +#include "libs/controller.h" + +#include "common/assert.h" +#include "common/common.h" +#include "common/logging/log.h" +#include "common/stringUtils.h" +#include "common/threads.h" +#include "kernel/pthread.h" +#include "libs/errno.h" +#include "libs/libs.h" +#include "libs/padData.h" + +#include +#include +#include + +namespace Libs::Controller { + +LIB_NAME("Pad", "Pad"); + +constexpr int PAD_ERROR_INVALID_ARG = -2137915391; /* 0x80920001 */ +constexpr int PAD_ERROR_INVALID_HANDLE = -2137915389; /* 0x80920003 */ + +struct PadControllerInformation { + float touch_pixel_density; + uint16_t touch_resolution_x; + uint16_t touch_resolution_y; + uint8_t stick_dead_zone_left; + uint8_t stick_dead_zone_right; + uint8_t connection_type; + uint8_t connected_count; + bool connected; + int device_class; + uint8_t reserve[8]; +}; + +struct PadVibrationParam { + uint8_t large_motor; + uint8_t small_motor; +}; + +struct ControllerState { + uint64_t time = 0; + uint32_t buttons = 0; + int axes[static_cast(Axis::AxisMax)] = {128, 128, 128, 128, 0, 0}; +}; + +class GameController { +public: + GameController() = default; + virtual ~GameController() = default; + + KYTY_CLASS_NO_COPY(GameController); + + void Connect(int id); + void Disconnect(int id); + void Button(int id, uint32_t button, bool down); + void Axis(int id, Axis axis, int value); + void GetConnectionInfo(bool* flag, int* count); + void ReadState(ControllerState* state, bool* flag, int* count); + int ReadStates(ControllerState* states, int states_num, bool* flag, int* count); + +private: + static constexpr uint32_t STATES_MAX = 64; + + struct StatePrivate { + bool obtained = false; + }; + + void CheckActive(); + [[nodiscard]] ControllerState GetLastState() const; + void AddState(const ControllerState& state); + + Common::Mutex m_mutex; + std::vector m_connected_ids; + int m_active_id = -1; + bool m_connected = false; + int m_connected_count = 0; + ControllerState m_states[STATES_MAX]; + StatePrivate m_private[STATES_MAX]; + ControllerState m_last_state; + uint32_t m_states_num = 0; + uint32_t m_first_state = 0; +}; + +static GameController* g_controller = nullptr; + +static constexpr int KEYBOARD_CONTROLLER_ID = -1000; + +static uint8_t pad_connected_count_to_u8(int connected_count) { + return static_cast(connected_count > 255 ? 255 : connected_count); +} + +static void pad_fill_data(PadData* data, const ControllerState& state, bool connected, + int connected_count) { + EXIT_IF(data == nullptr); + + std::memset(data, 0, sizeof(*data)); + + data->buttons = state.buttons; + data->left_stick_x = state.axes[static_cast(Axis::LeftX)]; + data->left_stick_y = state.axes[static_cast(Axis::LeftY)]; + data->right_stick_x = state.axes[static_cast(Axis::RightX)]; + data->right_stick_y = state.axes[static_cast(Axis::RightY)]; + data->analog_buttons_l2 = state.axes[static_cast(Axis::TriggerLeft)]; + data->analog_buttons_r2 = state.axes[static_cast(Axis::TriggerRight)]; + data->orientation_w = 1.0f; + data->touch_data_touch0_id = 1; + data->touch_data_touch1_id = 2; + data->connected = connected; + data->timestamp = state.time; + data->connected_count = pad_connected_count_to_u8(connected_count); + data->device_unique_data_len = 0; +} + +KYTY_SUBSYSTEM_INIT(Controller) { + EXIT_IF(g_controller != nullptr); + + g_controller = new GameController; + g_controller->Connect(KEYBOARD_CONTROLLER_ID); +} + +KYTY_SUBSYSTEM_UNEXPECTED_SHUTDOWN(Controller) {} + +KYTY_SUBSYSTEM_DESTROY(Controller) {} + +void GameController::Connect(int id) { + Common::LockGuard lock(m_mutex); + + if (std::find(m_connected_ids.begin(), m_connected_ids.end(), id) != m_connected_ids.end()) { + return; + } + + m_connected_ids.push_back(id); + + CheckActive(); +} + +void GameController::Disconnect(int id) { + Common::LockGuard lock(m_mutex); + + const auto it = std::find(m_connected_ids.begin(), m_connected_ids.end(), id); + EXIT_IF(it == m_connected_ids.end()); + + m_connected_ids.erase(it); + + CheckActive(); +} + +void GameController::CheckActive() { + bool reset = false; + int new_active_id = -1; + bool new_connected = false; + + if (!m_connected_ids.empty()) { + new_active_id = m_connected_ids[0]; + for (const auto id: m_connected_ids) { + if (id != KEYBOARD_CONTROLLER_ID) { + new_active_id = id; + break; + } + } + new_connected = true; + } + + const bool was_connected = m_connected; + + if (m_connected != new_connected || m_active_id != new_active_id) { + m_active_id = new_active_id; + m_connected = new_connected; + if (!was_connected && new_connected) { + m_connected_count++; + } + reset = true; + } + + if (reset) { + m_states_num = 0; + m_last_state = ControllerState(); + } +} + +ControllerState GameController::GetLastState() const { + if (m_states_num == 0) { + return m_last_state; + } + + auto last = (m_first_state + m_states_num - 1) % STATES_MAX; + + return m_states[last]; +} + +void GameController::AddState(const ControllerState& state) { + if (m_states_num >= STATES_MAX) { + m_states_num = STATES_MAX - 1; + m_first_state = (m_first_state + 1) % STATES_MAX; + } + + auto index = (m_first_state + m_states_num) % STATES_MAX; + + m_states[index] = state; + m_last_state = state; + + m_private[index].obtained = false; + + m_states_num++; +} + +void GameController::Button(int id, uint32_t button, bool down) { + Common::LockGuard lock(m_mutex); + + if (m_active_id == id) { + auto state = GetLastState(); + + state.time = LibKernel::KernelGetProcessTime(); + + if (down) { + state.buttons |= button; + } else { + state.buttons &= ~button; + } + + AddState(state); + } +} + +void GameController::Axis(int id, Controller::Axis axis, int value) { + Common::LockGuard lock(m_mutex); + + if (m_active_id == id) { + auto state = GetLastState(); + + state.time = LibKernel::KernelGetProcessTime(); + + int axis_id = static_cast(axis); + + EXIT_IF(axis_id < 0 || axis_id >= static_cast(Controller::Axis::AxisMax)); + + state.axes[axis_id] = value; + + if (axis == Controller::Axis::TriggerLeft) { + if (value > 0) { + state.buttons |= PAD_BUTTON_L2; + } else { + state.buttons &= ~PAD_BUTTON_L2; + } + } + + if (axis == Controller::Axis::TriggerRight) { + if (value > 0) { + state.buttons |= PAD_BUTTON_R2; + } else { + state.buttons &= ~PAD_BUTTON_R2; + } + } + + AddState(state); + } +} + +void GameController::GetConnectionInfo(bool* flag, int* count) { + EXIT_IF(flag == nullptr); + EXIT_IF(count == nullptr); + + Common::LockGuard lock(m_mutex); + + *flag = m_connected; + *count = m_connected_count; +} + +void GameController::ReadState(ControllerState* state, bool* flag, int* count) { + EXIT_IF(flag == nullptr); + EXIT_IF(count == nullptr); + EXIT_IF(state == nullptr); + + Common::LockGuard lock(m_mutex); + + *flag = m_connected; + *count = m_connected_count; + *state = GetLastState(); +} + +int GameController::ReadStates(ControllerState* states, int states_num, bool* flag, int* count) { + EXIT_IF(flag == nullptr); + EXIT_IF(count == nullptr); + EXIT_IF(states == nullptr); + EXIT_IF(states_num < 1 || states_num > STATES_MAX); + + Common::LockGuard lock(m_mutex); + + *flag = m_connected; + *count = m_connected_count; + + int ret_num = 0; + + if (m_connected) { + if (m_states_num != 0) { + for (uint32_t i = 0; i < m_states_num; i++) { + if (ret_num >= states_num) { + break; + } + auto index = (m_first_state + i) % STATES_MAX; + if (!m_private[index].obtained) { + m_private[index].obtained = true; + + states[ret_num++] = m_states[index]; + } + } + } + } + + return ret_num; +} + +void ControllerConnect(int id) { + EXIT_IF(g_controller == nullptr); + + g_controller->Connect(id); +} + +void ControllerDisconnect(int id) { + EXIT_IF(g_controller == nullptr); + + g_controller->Disconnect(id); +} + +void ControllerButton(int id, uint32_t button, bool down) { + EXIT_IF(g_controller == nullptr); + + g_controller->Button(id, button, down); +} + +void ControllerAxis(int id, Axis axis, int value) { + EXIT_IF(g_controller == nullptr); + + g_controller->Axis(id, axis, value); +} + +int KYTY_SYSV_ABI PadInit() { + PRINT_NAME(); + + return OK; +} + +int KYTY_SYSV_ABI PadOpen(int user_id, int type, int index, const void* param) { + PRINT_NAME(); + + LOGF("\t user_id = %d\n" + "\t type = %d\n" + "\t index = %d\n" + "\t param = 0x%016" PRIx64 "\n", + user_id, type, index, reinterpret_cast(param)); + + constexpr int pad_error_invalid_arg = -2137915391; /* 0x80920001 */ + + if (user_id != 1000 || (type != 0 && type != 2 && type != 16) || index != 0) { + return pad_error_invalid_arg; + } + + int handle = 1; + + return handle; +} + +int KYTY_SYSV_ABI PadGetHandle(int user_id, int type, int index) { + PRINT_NAME(); + + LOGF("\t user_id = %d\n" + "\t type = %d\n" + "\t index = %d\n", + user_id, type, index); + + constexpr int pad_error_device_no_handle = -2137915384; /* 0x80920008 */ + + if (user_id != 1000 || (type != 0 && type != 2 && type != 16) || index != 0) { + return pad_error_device_no_handle; + } + + return 1; +} + +int KYTY_SYSV_ABI PadSetMotionSensorState(int handle, bool enable) { + PRINT_NAME(); + + if (handle != 1) { + return PAD_ERROR_INVALID_HANDLE; + } + + LOGF("\t enable = %s\n", (enable ? "true" : "false")); + + return OK; +} + +int KYTY_SYSV_ABI PadSetAngularVelocityDeadbandState(int handle, bool enable) { + PRINT_NAME(); + + if (handle != 1) { + return PAD_ERROR_INVALID_HANDLE; + } + + LOGF("\t enable = %s\n", (enable ? "true" : "false")); + + return OK; +} + +int KYTY_SYSV_ABI PadResetOrientation(int handle) { + PRINT_NAME(); + + if (handle != 1) { + return PAD_ERROR_INVALID_HANDLE; + } + + return OK; +} + +int KYTY_SYSV_ABI PadGetControllerInformation(int handle, PadControllerInformation* info) { + PRINT_NAME(); + + EXIT_IF(g_controller == nullptr); + + int connected_count = 0; + bool connected = false; + + g_controller->GetConnectionInfo(&connected, &connected_count); + + if (handle != 1) { + return PAD_ERROR_INVALID_HANDLE; + } + if (info == nullptr) { + return PAD_ERROR_INVALID_ARG; + } + + std::memset(info, 0, sizeof(*info)); + + info->touch_pixel_density = 44.86f; + info->touch_resolution_x = 1920; + info->touch_resolution_y = 943; + info->stick_dead_zone_left = controller_get_axis(-32768, 32767, 8000) - 128; + info->stick_dead_zone_right = controller_get_axis(-32768, 32767, 8000) - 128; + info->connection_type = 0; + info->connected_count = pad_connected_count_to_u8(connected_count); + info->connected = connected; + info->device_class = 0; + + return OK; +} + +int KYTY_SYSV_ABI PadReadState(int handle, PadData* data) { + PRINT_NAME(); + + if (handle != 1) { + return PAD_ERROR_INVALID_HANDLE; + } + if (data == nullptr) { + return PAD_ERROR_INVALID_ARG; + } + + EXIT_IF(g_controller == nullptr); + + int connected_count = 0; + bool connected = false; + ControllerState state; + + g_controller->ReadState(&state, &connected, &connected_count); + + pad_fill_data(data, state, connected, connected_count); + + return OK; +} + +int KYTY_SYSV_ABI PadRead(int handle, PadData* data, int num) { + PRINT_NAME(); + + EXIT_NOT_IMPLEMENTED(num < 1 || num > 64); + if (handle != 1) { + return PAD_ERROR_INVALID_HANDLE; + } + if (data == nullptr) { + return PAD_ERROR_INVALID_ARG; + } + + std::memset(data, 0, sizeof(PadData) * static_cast(num)); + + EXIT_IF(g_controller == nullptr); + + int connected_count = 0; + bool connected = false; + ControllerState states[64] = {}; + + int ret_num = g_controller->ReadStates(states, num, &connected, &connected_count); + + if (!connected || ret_num == 0) { + if (connected) { + g_controller->ReadState(&states[0], &connected, &connected_count); + } + ret_num = 1; + } + + for (int i = 0; i < ret_num; i++) { + pad_fill_data(&data[i], states[i], connected, connected_count); + } + + return ret_num; +} + +int KYTY_SYSV_ABI PadSetVibration(int handle, const PadVibrationParam* param) { + PRINT_NAME(); + + if (handle != 1) { + return PAD_ERROR_INVALID_HANDLE; + } + if (param == nullptr) { + return PAD_ERROR_INVALID_ARG; + } + + LOGF("\t large_motor = %d\n" + "\t small_motor = %d\n", + static_cast(param->large_motor), static_cast(param->small_motor)); + + return OK; +} + +int KYTY_SYSV_ABI PadResetLightBar(int handle) { + PRINT_NAME(); + + if (handle != 1) { + return PAD_ERROR_INVALID_HANDLE; + } + + return OK; +} + +int KYTY_SYSV_ABI PadSetLightBar(int handle, const PadLightBarParam* param) { + PRINT_NAME(); + + if (handle != 1) { + return PAD_ERROR_INVALID_HANDLE; + } + if (param == nullptr) { + return PAD_ERROR_INVALID_ARG; + } + + return OK; +} + +} // namespace Libs::Controller diff --git a/src/libs/controller.h b/src/libs/controller.h new file mode 100644 index 000000000..37c5cdf34 --- /dev/null +++ b/src/libs/controller.h @@ -0,0 +1,70 @@ +#ifndef EMULATOR_INCLUDE_EMULATOR_CONTROLLER_H_ +#define EMULATOR_INCLUDE_EMULATOR_CONTROLLER_H_ + +#include "common/abi.h" +#include "common/common.h" +#include "common/subsystems.h" + +namespace Libs::Controller { + +KYTY_SUBSYSTEM_DEFINE(Controller); + +constexpr uint32_t PAD_BUTTON_L3 = 0x00000002; +constexpr uint32_t PAD_BUTTON_R3 = 0x00000004; +constexpr uint32_t PAD_BUTTON_OPTIONS = 0x00000008; +constexpr uint32_t PAD_BUTTON_UP = 0x00000010; +constexpr uint32_t PAD_BUTTON_RIGHT = 0x00000020; +constexpr uint32_t PAD_BUTTON_DOWN = 0x00000040; +constexpr uint32_t PAD_BUTTON_LEFT = 0x00000080; +constexpr uint32_t PAD_BUTTON_L2 = 0x00000100; +constexpr uint32_t PAD_BUTTON_R2 = 0x00000200; +constexpr uint32_t PAD_BUTTON_L1 = 0x00000400; +constexpr uint32_t PAD_BUTTON_R1 = 0x00000800; +constexpr uint32_t PAD_BUTTON_TRIANGLE = 0x00001000; +constexpr uint32_t PAD_BUTTON_CIRCLE = 0x00002000; +constexpr uint32_t PAD_BUTTON_CROSS = 0x00004000; +constexpr uint32_t PAD_BUTTON_SQUARE = 0x00008000; +constexpr uint32_t PAD_BUTTON_TOUCH_PAD = 0x00100000; + +enum class Axis { + LeftX = 0, + LeftY = 1, + RightX = 2, + RightY = 3, + TriggerLeft = 4, + TriggerRight = 5, + + AxisMax +}; + +struct PadControllerInformation; +struct PadData; +struct PadVibrationParam; +struct PadLightBarParam; + +inline int controller_get_axis(int min, int max, int value) { + int v = (255 * (value - min)) / (max - min); + return (v < 0 ? 0 : (v > 255 ? 255 : v)); +} + +void ControllerConnect(int id); +void ControllerDisconnect(int id); +void ControllerButton(int id, uint32_t button, bool down); +void ControllerAxis(int id, Axis axis, int value); + +int KYTY_SYSV_ABI PadInit(); +int KYTY_SYSV_ABI PadOpen(int user_id, int type, int index, const void* param); +int KYTY_SYSV_ABI PadGetHandle(int user_id, int type, int index); +int KYTY_SYSV_ABI PadSetMotionSensorState(int handle, bool enable); +int KYTY_SYSV_ABI PadSetAngularVelocityDeadbandState(int handle, bool enable); +int KYTY_SYSV_ABI PadResetOrientation(int handle); +int KYTY_SYSV_ABI PadGetControllerInformation(int handle, PadControllerInformation* info); +int KYTY_SYSV_ABI PadReadState(int handle, PadData* data); +int KYTY_SYSV_ABI PadRead(int handle, PadData* data, int num); +int KYTY_SYSV_ABI PadSetVibration(int handle, const PadVibrationParam* param); +int KYTY_SYSV_ABI PadResetLightBar(int handle); +int KYTY_SYSV_ABI PadSetLightBar(int handle, const PadLightBarParam* param); + +} // namespace Libs::Controller + +#endif /* EMULATOR_INCLUDE_EMULATOR_CONTROLLER_H_ */ diff --git a/src/libs/dialog.cpp b/src/libs/dialog.cpp new file mode 100644 index 000000000..2b8fb45bd --- /dev/null +++ b/src/libs/dialog.cpp @@ -0,0 +1,627 @@ +#include "libs/dialog.h" + +#include "common/logging/log.h" +#include "common/stringUtils.h" +#include "libs/errno.h" +#include "libs/libs.h" + +#include +#include + +// NOLINTNEXTLINE(modernize-concat-nested-namespaces) +namespace Libs::Dialog { + +namespace CommonDialog { + +LIB_NAME("CommonDialog", "CommonDialog"); + +int KYTY_SYSV_ABI CommonDialogInitialize() { + PRINT_NAME(); + + return OK; +} + +} // namespace CommonDialog + +namespace ImeDialog { + +LIB_NAME("ImeDialog", "ImeDialog"); + +constexpr int IME_STATUS_NONE = 0; + +int KYTY_SYSV_ABI ImeDialogGetStatus() { + PRINT_NAME(); + + return IME_STATUS_NONE; +} + +} // namespace ImeDialog + +namespace LoginDialog { + +LIB_NAME("LoginDialog", "LoginDialog"); + +constexpr int LOGIN_STATUS_NONE = 0; +constexpr int LOGIN_STATUS_INITIALIZED = 1; +constexpr int LOGIN_STATUS_RUNNING = 2; +constexpr int LOGIN_STATUS_FINISHED = 3; +constexpr int LOGIN_RESULT_OK = 0; +constexpr int LOGIN_MODE_ALL_USERS = 0; +constexpr int LOGIN_MODE_NOT_LOGGED_IN = 1; +constexpr int LOGIN_USER_INVALID = -1; +constexpr int LOGIN_USER_ID = 1000; + +constexpr int LOGIN_ERROR_NOT_INITIALIZED = static_cast(0x81340001u); +constexpr int LOGIN_ERROR_ALREADY_INITIALIZED = static_cast(0x81340002u); +constexpr int LOGIN_ERROR_PARAM_INVALID = static_cast(0x81340003u); +constexpr int LOGIN_ERROR_INVALID_STATE = static_cast(0x81340005u); + +struct LoginDialogResult { + int32_t result; + int32_t selected_user; + int32_t reserved[2]; +}; + +struct LoginDialogParam { + int32_t size; + int32_t mode; + int32_t exclude_users_from_login_list[4]; + int32_t exclude_users_from_logout_list[4]; + int32_t initial_focus; + int32_t reserved[5]; +}; + +static_assert(sizeof(LoginDialogResult) == 16); +static_assert(sizeof(LoginDialogParam) == 64); + +static int g_login_status = LOGIN_STATUS_NONE; +static int g_login_selected_user = LOGIN_USER_ID; + +int KYTY_SYSV_ABI LoginDialogInitialize() { + PRINT_NAME(); + + if (g_login_status != LOGIN_STATUS_NONE) { + return LOGIN_ERROR_ALREADY_INITIALIZED; + } + + g_login_status = LOGIN_STATUS_INITIALIZED; + g_login_selected_user = LOGIN_USER_ID; + + return OK; +} + +int KYTY_SYSV_ABI LoginDialogTerminate() { + PRINT_NAME(); + + if (g_login_status == LOGIN_STATUS_NONE) { + return LOGIN_ERROR_NOT_INITIALIZED; + } + + g_login_status = LOGIN_STATUS_NONE; + g_login_selected_user = LOGIN_USER_ID; + + return OK; +} + +int KYTY_SYSV_ABI LoginDialogOpen(const void* param) { + PRINT_NAME(); + + if (g_login_status != LOGIN_STATUS_INITIALIZED && g_login_status != LOGIN_STATUS_FINISHED) { + return LOGIN_ERROR_INVALID_STATE; + } + if (param == nullptr) { + return LOGIN_ERROR_PARAM_INVALID; + } + + const auto* p = static_cast(param); + if (p->size != sizeof(LoginDialogParam) || + (p->mode != LOGIN_MODE_ALL_USERS && p->mode != LOGIN_MODE_NOT_LOGGED_IN)) { + return LOGIN_ERROR_PARAM_INVALID; + } + + g_login_selected_user = + p->initial_focus != LOGIN_USER_INVALID ? p->initial_focus : LOGIN_USER_ID; + + LOGF("\t size = %d\n" + "\t mode = %d\n" + "\t initial_focus = %d\n" + "\t selected_user = %d\n", + p->size, p->mode, p->initial_focus, g_login_selected_user); + + g_login_status = LOGIN_STATUS_RUNNING; + + return OK; +} + +int KYTY_SYSV_ABI LoginDialogClose() { + PRINT_NAME(); + + if (g_login_status == LOGIN_STATUS_NONE) { + return LOGIN_ERROR_NOT_INITIALIZED; + } + + g_login_status = LOGIN_STATUS_FINISHED; + + return OK; +} + +int KYTY_SYSV_ABI LoginDialogUpdateStatus() { + PRINT_NAME(); + + if (g_login_status == LOGIN_STATUS_RUNNING) { + g_login_status = LOGIN_STATUS_FINISHED; + } + + return g_login_status; +} + +int KYTY_SYSV_ABI LoginDialogGetStatus() { + PRINT_NAME(); + + return g_login_status; +} + +int KYTY_SYSV_ABI LoginDialogGetResult(void* result) { + PRINT_NAME(); + + if (g_login_status == LOGIN_STATUS_NONE) { + return LOGIN_ERROR_NOT_INITIALIZED; + } + if (result == nullptr) { + return LOGIN_ERROR_PARAM_INVALID; + } + + auto* r = static_cast(result); + r->result = LOGIN_RESULT_OK; + r->selected_user = g_login_selected_user; + r->reserved[0] = 0; + r->reserved[1] = 0; + + return OK; +} + +void KYTY_SYSV_ABI LoginDialogParamInitialize(void* param) { + PRINT_NAME(); + + if (param == nullptr) { + return; + } + + auto* p = static_cast(param); + std::memset(p, 0, sizeof(LoginDialogParam)); + p->size = sizeof(LoginDialogParam); + for (int i = 0; i < 4; i++) { + p->exclude_users_from_login_list[i] = LOGIN_USER_INVALID; + p->exclude_users_from_logout_list[i] = LOGIN_USER_INVALID; + } + p->initial_focus = LOGIN_USER_INVALID; +} + +} // namespace LoginDialog + +namespace SaveDataDialog { + +LIB_NAME("SaveDataDialog", "SaveDataDialog"); + +constexpr int SAVE_STATUS_NONE = 0; +constexpr int SAVE_STATUS_INITIALIZED = 1; +constexpr int SAVE_STATUS_FINISHED = 3; +constexpr int SAVE_RESULT_OK = 0; +constexpr int SAVE_BUTTON_ID_OK = 1; + +struct SaveDataDialogParam { + uint8_t base_param[48]; + int32_t size; + int32_t mode; + int32_t disp_type; + uint32_t pad0; + void* anim_param; + void* items; + void* user_msg_param; + void* sys_msg_param; + void* error_code_param; + void* prog_bar_param; + void* user_data; + void* option_param; + void* wizard_param; + uint8_t reserved[16]; +}; + +struct SaveDataDirName { + char data[32]; +}; + +struct SaveDataDialogItems { + int32_t user_id; + int32_t pad0; + const void* title_id; + const SaveDataDirName* dir_names; + uint32_t dir_names_num; +}; + +struct SaveDataDialogResult { + int32_t mode; + int32_t result; + int32_t button_id; + uint32_t pad0; + void* dir_name; + void* param; + void* user_data; + uint8_t reserved[32]; +}; + +static int g_save_status = SAVE_STATUS_NONE; +static int g_save_mode = 0; +static void* g_save_user_data = nullptr; +static char g_save_dir_name[sizeof(SaveDataDirName::data)] {}; + +int KYTY_SYSV_ABI SaveDataDialogInitialize() { + PRINT_NAME(); + + g_save_status = SAVE_STATUS_INITIALIZED; + g_save_mode = 0; + g_save_user_data = nullptr; + g_save_dir_name[0] = '\0'; + + return OK; +} + +int KYTY_SYSV_ABI SaveDataDialogGetStatus() { + PRINT_NAME(); + + return g_save_status; +} + +int KYTY_SYSV_ABI SaveDataDialogUpdateStatus() { + PRINT_NAME(); + + return g_save_status; +} + +int KYTY_SYSV_ABI SaveDataDialogGetResult(void* result) { + PRINT_NAME(); + + if (result != nullptr) { + auto* r = static_cast(result); + r->mode = g_save_mode; + r->result = SAVE_RESULT_OK; + r->button_id = SAVE_BUTTON_ID_OK; + r->user_data = g_save_user_data; + if (r->dir_name != nullptr && g_save_dir_name[0] != '\0') { + std::snprintf(static_cast(r->dir_name)->data, + sizeof(SaveDataDirName::data), "%s", g_save_dir_name); + } + } + + return OK; +} + +int KYTY_SYSV_ABI SaveDataDialogOpen(const void* param) { + PRINT_NAME(); + + const auto* p = static_cast(param); + if (p != nullptr) { + g_save_mode = p->mode; + g_save_user_data = p->user_data; + g_save_dir_name[0] = '\0'; + LOGF("\t size = %d\n" + "\t mode = %d\n" + "\t disp_type = %d\n" + "\t items = 0x%016" PRIx64 "\n" + "\t user_msg_param = 0x%016" PRIx64 "\n" + "\t sys_msg_param = 0x%016" PRIx64 "\n" + "\t prog_bar_param = 0x%016" PRIx64 "\n" + "\t user_data = 0x%016" PRIx64 "\n", + p->size, p->mode, p->disp_type, reinterpret_cast(p->items), + reinterpret_cast(p->user_msg_param), + reinterpret_cast(p->sys_msg_param), + reinterpret_cast(p->prog_bar_param), + reinterpret_cast(p->user_data)); + + const auto* items = static_cast(p->items); + if (items != nullptr && items->dir_names != nullptr) { + for (uint32_t i = 0; i < items->dir_names_num; i++) { + const auto* name = items->dir_names[i].data; + if (name[0] != '\0') { + std::snprintf(g_save_dir_name, sizeof(g_save_dir_name), "%s", name); + break; + } + } + } + } + + g_save_status = SAVE_STATUS_FINISHED; + + return OK; +} + +int KYTY_SYSV_ABI SaveDataDialogClose(const void* close_param) { + PRINT_NAME(); + + LOGF("\t close_param = 0x%016" PRIx64 "\n", reinterpret_cast(close_param)); + + g_save_status = SAVE_STATUS_FINISHED; + + return OK; +} + +int KYTY_SYSV_ABI SaveDataDialogIsReadyToDisplay() { + PRINT_NAME(); + + return 1; +} + +int KYTY_SYSV_ABI SaveDataDialogTerminate() { + PRINT_NAME(); + + g_save_status = SAVE_STATUS_NONE; + g_save_mode = 0; + g_save_user_data = nullptr; + g_save_dir_name[0] = '\0'; + + return 0; +} + +int KYTY_SYSV_ABI SaveDataDialogProgressBarInc(int target, uint32_t delta) { + PRINT_NAME(); + + LOGF("\t target = %d\n" + "\t delta = %u\n", + target, delta); + + return OK; +} + +int KYTY_SYSV_ABI SaveDataDialogProgressBarSetValue(int target, uint32_t rate) { + PRINT_NAME(); + + LOGF("\t target = %d\n" + "\t rate = %u\n", + target, rate); + + return OK; +} + +} // namespace SaveDataDialog + +namespace MsgDialog { + +LIB_NAME("MsgDialog.native", "MsgDialog"); + +constexpr int STATUS_NONE = 0; +constexpr int STATUS_INITIALIZED = 1; +constexpr int STATUS_FINISHED = 3; +constexpr int RESULT_OK = 0; +constexpr int BUTTON_ID_OK = 1; + +struct MsgDialogParam { + uint8_t base_param[48]; + uint64_t size; + int32_t mode; + uint32_t pad0; + void* user_msg_param; + void* prog_bar_param; + void* sys_msg_param; + int32_t user_id; + uint8_t reserved[40]; + uint32_t pad1; +}; + +struct MsgDialogResult { + int32_t mode; + int32_t result; + int32_t button_id; + uint8_t reserved[32]; +}; + +static int g_status = STATUS_NONE; +static int g_mode = 0; + +int KYTY_SYSV_ABI MsgDialogInitialize() { + PRINT_NAME(); + + g_status = STATUS_INITIALIZED; + g_mode = 0; + + return OK; +} + +int KYTY_SYSV_ABI MsgDialogOpen(const void* param) { + PRINT_NAME(); + + const auto* p = static_cast(param); + if (p != nullptr) { + g_mode = p->mode; + LOGF("\t size = 0x%016" PRIx64 "\n" + "\t mode = %d\n" + "\t user_msg_param = 0x%016" PRIx64 "\n" + "\t prog_bar_param = 0x%016" PRIx64 "\n" + "\t sys_msg_param = 0x%016" PRIx64 "\n" + "\t user_id = %d\n", + p->size, p->mode, reinterpret_cast(p->user_msg_param), + reinterpret_cast(p->prog_bar_param), + reinterpret_cast(p->sys_msg_param), p->user_id); + } + + g_status = STATUS_FINISHED; + + return OK; +} + +int KYTY_SYSV_ABI MsgDialogUpdateStatus() { + PRINT_NAME(); + + return g_status; +} + +int KYTY_SYSV_ABI MsgDialogGetStatus() { + PRINT_NAME(); + + return g_status; +} + +int KYTY_SYSV_ABI MsgDialogGetResult(void* result) { + PRINT_NAME(); + + if (result != nullptr) { + auto* r = static_cast(result); + r->mode = g_mode; + r->result = RESULT_OK; + r->button_id = BUTTON_ID_OK; + } + + return OK; +} + +int KYTY_SYSV_ABI MsgDialogTerminate() { + PRINT_NAME(); + + g_status = STATUS_NONE; + g_mode = 0; + + return OK; +} + +int KYTY_SYSV_ABI MsgDialogClose() { + PRINT_NAME(); + + g_status = STATUS_FINISHED; + + return OK; +} + +int KYTY_SYSV_ABI MsgDialogProgressBarInc(int target, uint32_t delta) { + PRINT_NAME(); + + LOGF("\t target = %d\n" + "\t delta = %u\n", + target, delta); + + return OK; +} + +int KYTY_SYSV_ABI MsgDialogProgressBarSetValue(int target, uint32_t rate) { + PRINT_NAME(); + + LOGF("\t target = %d\n" + "\t rate = %u\n", + target, rate); + + return OK; +} + +int KYTY_SYSV_ABI MsgDialogProgressBarSetMsg(int target, const char* msg) { + PRINT_NAME(); + + LOGF("\t target = %d\n" + "\t msg = 0x%016" PRIx64 "\n", + target, reinterpret_cast(msg)); + + return OK; +} + +} // namespace MsgDialog + +namespace ErrorDialog { + +LIB_NAME("ErrorDialog", "ErrorDialog"); + +constexpr int STATUS_NONE = 0; +constexpr int STATUS_INITIALIZED = 1; +constexpr int STATUS_RUNNING = 2; +constexpr int STATUS_FINISHED = 3; + +constexpr int ERROR_NOT_INITIALIZED = static_cast(0x80ed0001); +constexpr int ERROR_ALREADY_INITIALIZED = static_cast(0x80ed0002); +constexpr int ERROR_PARAM_INVALID = static_cast(0x80ed0003); +constexpr int ERROR_INVALID_STATE = static_cast(0x80ed0005); + +struct ErrorDialogParam { + int32_t size; + int32_t error_code; + int32_t user_id; + int32_t reserved; +}; + +static int g_error_status = STATUS_NONE; + +int KYTY_SYSV_ABI ErrorDialogInitialize() { + PRINT_NAME(); + + if (g_error_status != STATUS_NONE) { + return ERROR_ALREADY_INITIALIZED; + } + + g_error_status = STATUS_INITIALIZED; + + return OK; +} + +int KYTY_SYSV_ABI ErrorDialogOpen(const void* param) { + PRINT_NAME(); + + if (g_error_status != STATUS_INITIALIZED && g_error_status != STATUS_FINISHED) { + return ERROR_INVALID_STATE; + } + if (param == nullptr) { + return ERROR_PARAM_INVALID; + } + + const auto* p = static_cast(param); + if (p->size != sizeof(ErrorDialogParam)) { + return ERROR_PARAM_INVALID; + } + + LOGF("\t size = %d\n" + "\t error_code = 0x%08" PRIx32 "\n" + "\t user_id = %d\n", + p->size, static_cast(p->error_code), p->user_id); + + g_error_status = STATUS_RUNNING; + + return OK; +} + +int KYTY_SYSV_ABI ErrorDialogClose() { + PRINT_NAME(); + + if (g_error_status != STATUS_RUNNING) { + return ERROR_INVALID_STATE; + } + + g_error_status = STATUS_FINISHED; + + return OK; +} + +int KYTY_SYSV_ABI ErrorDialogTerminate() { + PRINT_NAME(); + + if (g_error_status == STATUS_NONE) { + return ERROR_NOT_INITIALIZED; + } + + g_error_status = STATUS_NONE; + + return OK; +} + +int KYTY_SYSV_ABI ErrorDialogUpdateStatus() { + PRINT_NAME(); + + if (g_error_status == STATUS_RUNNING) { + g_error_status = STATUS_FINISHED; + } + + return g_error_status; +} + +int KYTY_SYSV_ABI ErrorDialogGetStatus() { + PRINT_NAME(); + + return g_error_status; +} + +} // namespace ErrorDialog + +} // namespace Libs::Dialog diff --git a/src/libs/dialog.h b/src/libs/dialog.h new file mode 100644 index 000000000..c1a8976db --- /dev/null +++ b/src/libs/dialog.h @@ -0,0 +1,77 @@ +#ifndef EMULATOR_INCLUDE_EMULATOR_DIALOG_H_ +#define EMULATOR_INCLUDE_EMULATOR_DIALOG_H_ + +#include "common/abi.h" +#include "common/common.h" + +namespace Libs::Dialog { + +namespace CommonDialog { + +int KYTY_SYSV_ABI CommonDialogInitialize(); + +} // namespace CommonDialog + +namespace ImeDialog { + +int KYTY_SYSV_ABI ImeDialogGetStatus(); + +} // namespace ImeDialog + +namespace LoginDialog { + +int KYTY_SYSV_ABI LoginDialogInitialize(); +int KYTY_SYSV_ABI LoginDialogTerminate(); +int KYTY_SYSV_ABI LoginDialogOpen(const void* param); +int KYTY_SYSV_ABI LoginDialogClose(); +int KYTY_SYSV_ABI LoginDialogUpdateStatus(); +int KYTY_SYSV_ABI LoginDialogGetStatus(); +int KYTY_SYSV_ABI LoginDialogGetResult(void* result); +void KYTY_SYSV_ABI LoginDialogParamInitialize(void* param); + +} // namespace LoginDialog + +namespace SaveDataDialog { + +int KYTY_SYSV_ABI SaveDataDialogInitialize(); +int KYTY_SYSV_ABI SaveDataDialogGetStatus(); +int KYTY_SYSV_ABI SaveDataDialogUpdateStatus(); +int KYTY_SYSV_ABI SaveDataDialogGetResult(void* result); +int KYTY_SYSV_ABI SaveDataDialogOpen(const void* param); +int KYTY_SYSV_ABI SaveDataDialogClose(const void* close_param); +int KYTY_SYSV_ABI SaveDataDialogIsReadyToDisplay(); +int KYTY_SYSV_ABI SaveDataDialogTerminate(); +int KYTY_SYSV_ABI SaveDataDialogProgressBarInc(int target, uint32_t delta); +int KYTY_SYSV_ABI SaveDataDialogProgressBarSetValue(int target, uint32_t rate); + +} // namespace SaveDataDialog + +namespace MsgDialog { + +int KYTY_SYSV_ABI MsgDialogInitialize(); +int KYTY_SYSV_ABI MsgDialogOpen(const void* param); +int KYTY_SYSV_ABI MsgDialogUpdateStatus(); +int KYTY_SYSV_ABI MsgDialogGetStatus(); +int KYTY_SYSV_ABI MsgDialogGetResult(void* result); +int KYTY_SYSV_ABI MsgDialogTerminate(); +int KYTY_SYSV_ABI MsgDialogClose(); +int KYTY_SYSV_ABI MsgDialogProgressBarInc(int target, uint32_t delta); +int KYTY_SYSV_ABI MsgDialogProgressBarSetValue(int target, uint32_t rate); +int KYTY_SYSV_ABI MsgDialogProgressBarSetMsg(int target, const char* msg); + +} // namespace MsgDialog + +namespace ErrorDialog { + +int KYTY_SYSV_ABI ErrorDialogInitialize(); +int KYTY_SYSV_ABI ErrorDialogOpen(const void* param); +int KYTY_SYSV_ABI ErrorDialogClose(); +int KYTY_SYSV_ABI ErrorDialogTerminate(); +int KYTY_SYSV_ABI ErrorDialogUpdateStatus(); +int KYTY_SYSV_ABI ErrorDialogGetStatus(); + +} // namespace ErrorDialog + +} // namespace Libs::Dialog + +#endif /* EMULATOR_INCLUDE_EMULATOR_DIALOG_H_ */ diff --git a/src/libs/errno.cpp b/src/libs/errno.cpp new file mode 100644 index 000000000..10d3baa64 --- /dev/null +++ b/src/libs/errno.cpp @@ -0,0 +1,11 @@ +#include "libs/errno.h" + +namespace Libs::Posix { + +static thread_local int g_errno = 0; + +KYTY_SYSV_ABI int* GetErrorAddr() { + return &g_errno; +} + +} // namespace Libs::Posix diff --git a/src/libs/errno.h b/src/libs/errno.h new file mode 100644 index 000000000..b505caa10 --- /dev/null +++ b/src/libs/errno.h @@ -0,0 +1,666 @@ +#ifndef EMULATOR_INCLUDE_EMULATOR_LIBS_ERRNO_H_ +#define EMULATOR_INCLUDE_EMULATOR_LIBS_ERRNO_H_ + +#include "common/abi.h" + +constexpr int OK = 0; + +// NOLINTNEXTLINE(cppcoreguidelines-macro-usage) +#define POSIX_CALL(func) \ + [&]() { \ + auto result = func; \ + if (result != OK) { \ + *Posix::GetErrorAddr() = LibKernel::KernelToPosix(result); \ + return -1; \ + } \ + return 0; \ + }() + +// NOLINTNEXTLINE(cppcoreguidelines-macro-usage) +#define POSIX_N_CALL(func) \ + [&]() { \ + auto result = func; \ + if (result < 0) { \ + *Posix::GetErrorAddr() = LibKernel::KernelToPosix(static_cast(result)); \ + return static_cast(-1); \ + } \ + return result; \ + }() + +// NOLINTNEXTLINE(cppcoreguidelines-macro-usage) +#define POSIX_PTHREAD_CALL(func) \ + [&]() { \ + auto result = func; \ + if (result != OK) { \ + return LibKernel::KernelToPosix(result); \ + } \ + return 0; \ + }() + +// NOLINTNEXTLINE(cppcoreguidelines-macro-usage) +#define POSIX_NET_CALL(func) \ + [&]() { \ + auto result = func; \ + if (result != OK) { \ + *Posix::GetErrorAddr() = Network::NetToPosix(result); \ + return -1; \ + } \ + return 0; \ + }() + +namespace Libs { + +namespace Posix { + +KYTY_SYSV_ABI int* GetErrorAddr(); + +constexpr int POSIX_EPERM = 1; +constexpr int POSIX_ENOENT = 2; +constexpr int POSIX_ESRCH = 3; +constexpr int POSIX_EINTR = 4; +constexpr int POSIX_EIO = 5; +constexpr int POSIX_ENXIO = 6; +constexpr int POSIX_E2BIG = 7; +constexpr int POSIX_ENOEXEC = 8; +constexpr int POSIX_EBADF = 9; +constexpr int POSIX_ECHILD = 10; +constexpr int POSIX_EDEADLK = 11; +constexpr int POSIX_ENOMEM = 12; +constexpr int POSIX_EACCES = 13; +constexpr int POSIX_EFAULT = 14; +constexpr int POSIX_ENOTBLK = 15; +constexpr int POSIX_EBUSY = 16; +constexpr int POSIX_EEXIST = 17; +constexpr int POSIX_EXDEV = 18; +constexpr int POSIX_ENODEV = 19; +constexpr int POSIX_ENOTDIR = 20; +constexpr int POSIX_EISDIR = 21; +constexpr int POSIX_EINVAL = 22; +constexpr int POSIX_ENFILE = 23; +constexpr int POSIX_EMFILE = 24; +constexpr int POSIX_ENOTTY = 25; +constexpr int POSIX_ETXTBSY = 26; +constexpr int POSIX_EFBIG = 27; +constexpr int POSIX_ENOSPC = 28; +constexpr int POSIX_ESPIPE = 29; +constexpr int POSIX_EROFS = 30; +constexpr int POSIX_EMLINK = 31; +constexpr int POSIX_EPIPE = 32; +constexpr int POSIX_EDOM = 33; +constexpr int POSIX_ERANGE = 34; +constexpr int POSIX_EAGAIN = 35; +constexpr int POSIX_EWOULDBLOCK = 35; +constexpr int POSIX_EINPROGRESS = 36; +constexpr int POSIX_EALREADY = 37; +constexpr int POSIX_ENOTSOCK = 38; +constexpr int POSIX_EDESTADDRREQ = 39; +constexpr int POSIX_EMSGSIZE = 40; +constexpr int POSIX_EPROTOTYPE = 41; +constexpr int POSIX_ENOPROTOOPT = 42; +constexpr int POSIX_EPROTONOSUPPORT = 43; +constexpr int POSIX_ESOCKTNOSUPPORT = 44; +constexpr int POSIX_EOPNOTSUPP = 45; +constexpr int POSIX_ENOTSUP = 45; +constexpr int POSIX_EPFNOSUPPORT = 46; +constexpr int POSIX_EAFNOSUPPORT = 47; +constexpr int POSIX_EADDRINUSE = 48; +constexpr int POSIX_EADDRNOTAVAIL = 49; +constexpr int POSIX_ENETDOWN = 50; +constexpr int POSIX_ENETUNREACH = 51; +constexpr int POSIX_ENETRESET = 52; +constexpr int POSIX_ECONNABORTED = 53; +constexpr int POSIX_ECONNRESET = 54; +constexpr int POSIX_ENOBUFS = 55; +constexpr int POSIX_EISCONN = 56; +constexpr int POSIX_ENOTCONN = 57; +constexpr int POSIX_ESHUTDOWN = 58; +constexpr int POSIX_ETOOMANYREFS = 59; +constexpr int POSIX_ETIMEDOUT = 60; +constexpr int POSIX_ECONNREFUSED = 61; +constexpr int POSIX_ELOOP = 62; +constexpr int POSIX_ENAMETOOLONG = 63; +constexpr int POSIX_EHOSTDOWN = 64; +constexpr int POSIX_EHOSTUNREACH = 65; +constexpr int POSIX_ENOTEMPTY = 66; +constexpr int POSIX_EPROCLIM = 67; +constexpr int POSIX_EUSERS = 68; +constexpr int POSIX_EDQUOT = 69; +constexpr int POSIX_ESTALE = 70; +constexpr int POSIX_EREMOTE = 71; +constexpr int POSIX_EBADRPC = 72; +constexpr int POSIX_ERPCMISMATCH = 73; +constexpr int POSIX_EPROGUNAVAIL = 74; +constexpr int POSIX_EPROGMISMATCH = 75; +constexpr int POSIX_EPROCUNAVAIL = 76; +constexpr int POSIX_ENOLCK = 77; +constexpr int POSIX_ENOSYS = 78; +constexpr int POSIX_EFTYPE = 79; +constexpr int POSIX_EAUTH = 80; +constexpr int POSIX_ENEEDAUTH = 81; +constexpr int POSIX_EIDRM = 82; +constexpr int POSIX_ENOMSG = 83; +constexpr int POSIX_EOVERFLOW = 84; +constexpr int POSIX_ECANCELED = 85; +constexpr int POSIX_EILSEQ = 86; +constexpr int POSIX_ENOATTR = 87; +constexpr int POSIX_EDOOFUS = 88; +constexpr int POSIX_EBADMSG = 89; +constexpr int POSIX_EMULTIHOP = 90; +constexpr int POSIX_ENOLINK = 91; +constexpr int POSIX_EPROTO = 92; +constexpr int POSIX_ENOTCAPABLE = 93; +constexpr int POSIX_ECAPMODE = 94; +constexpr int POSIX_ENOBLK = 95; +constexpr int POSIX_EICV = 96; +constexpr int POSIX_ENOPLAYGOENT = 97; +constexpr int POSIX_EREVOKE = 98; +constexpr int POSIX_ESDKVERSION = 99; +constexpr int POSIX_ESTART = 100; +constexpr int POSIX_ESTOP = 101; +constexpr int POSIX_EINVALID2MB = 102; +constexpr int POSIX_ELAST = 102; +constexpr int POSIX_EADHOC = 160; +constexpr int POSIX_EINACTIVEDISABLED = 163; +constexpr int POSIX_ENETNODATA = 164; +constexpr int POSIX_ENETDESC = 165; +constexpr int POSIX_ENETDESCTIMEDOUT = 166; +constexpr int POSIX_ENETINTR = 167; +constexpr int POSIX_ERETURN = 205; +constexpr int POSIX_EFPOS = 152; +constexpr int POSIX_ENODATA = 1040; +constexpr int POSIX_ENOSR = 1050; +constexpr int POSIX_ENOSTR = 1051; +constexpr int POSIX_ENOTRECOVERABLE = 1056; +constexpr int POSIX_EOTHER = 1062; +constexpr int POSIX_EOWNERDEAD = 1064; +constexpr int POSIX_ETIME = 1074; + +} // namespace Posix + +namespace LibKernel { + +inline int KernelToPosix(int kernel_errno) { + return kernel_errno > -2147352576 && kernel_errno <= -2147352475 ? kernel_errno + 2147352576 + : 1062; +} + +constexpr int KERNEL_ERROR_UNKNOWN = -2147352576; /* 0x80020000 */ +constexpr int KERNEL_ERROR_EPERM = -2147352575; /* 0x80020001 */ +constexpr int KERNEL_ERROR_ENOENT = -2147352574; /* 0x80020002 */ +constexpr int KERNEL_ERROR_ESRCH = -2147352573; /* 0x80020003 */ +constexpr int KERNEL_ERROR_EINTR = -2147352572; /* 0x80020004 */ +constexpr int KERNEL_ERROR_EIO = -2147352571; /* 0x80020005 */ +constexpr int KERNEL_ERROR_ENXIO = -2147352570; /* 0x80020006 */ +constexpr int KERNEL_ERROR_E2BIG = -2147352569; /* 0x80020007 */ +constexpr int KERNEL_ERROR_ENOEXEC = -2147352568; /* 0x80020008 */ +constexpr int KERNEL_ERROR_EBADF = -2147352567; /* 0x80020009 */ +constexpr int KERNEL_ERROR_ECHILD = -2147352566; /* 0x8002000A */ +constexpr int KERNEL_ERROR_EDEADLK = -2147352565; /* 0x8002000B */ +constexpr int KERNEL_ERROR_ENOMEM = -2147352564; /* 0x8002000C */ +constexpr int KERNEL_ERROR_EACCES = -2147352563; /* 0x8002000D */ +constexpr int KERNEL_ERROR_EFAULT = -2147352562; /* 0x8002000E */ +constexpr int KERNEL_ERROR_ENOTBLK = -2147352561; /* 0x8002000F */ +constexpr int KERNEL_ERROR_EBUSY = -2147352560; /* 0x80020010 */ +constexpr int KERNEL_ERROR_EEXIST = -2147352559; /* 0x80020011 */ +constexpr int KERNEL_ERROR_EXDEV = -2147352558; /* 0x80020012 */ +constexpr int KERNEL_ERROR_ENODEV = -2147352557; /* 0x80020013 */ +constexpr int KERNEL_ERROR_ENOTDIR = -2147352556; /* 0x80020014 */ +constexpr int KERNEL_ERROR_EISDIR = -2147352555; /* 0x80020015 */ +constexpr int KERNEL_ERROR_EINVAL = -2147352554; /* 0x80020016 */ +constexpr int KERNEL_ERROR_ENFILE = -2147352553; /* 0x80020017 */ +constexpr int KERNEL_ERROR_EMFILE = -2147352552; /* 0x80020018 */ +constexpr int KERNEL_ERROR_ENOTTY = -2147352551; /* 0x80020019 */ +constexpr int KERNEL_ERROR_ETXTBSY = -2147352550; /* 0x8002001A */ +constexpr int KERNEL_ERROR_EFBIG = -2147352549; /* 0x8002001B */ +constexpr int KERNEL_ERROR_ENOSPC = -2147352548; /* 0x8002001C */ +constexpr int KERNEL_ERROR_ESPIPE = -2147352547; /* 0x8002001D */ +constexpr int KERNEL_ERROR_EROFS = -2147352546; /* 0x8002001E */ +constexpr int KERNEL_ERROR_EMLINK = -2147352545; /* 0x8002001F */ +constexpr int KERNEL_ERROR_EPIPE = -2147352544; /* 0x80020020 */ +constexpr int KERNEL_ERROR_EDOM = -2147352543; /* 0x80020021 */ +constexpr int KERNEL_ERROR_ERANGE = -2147352542; /* 0x80020022 */ +constexpr int KERNEL_ERROR_EAGAIN = -2147352541; /* 0x80020023 */ +constexpr int KERNEL_ERROR_EWOULDBLOCK = -2147352541; /* 0x80020023 */ +constexpr int KERNEL_ERROR_EINPROGRESS = -2147352540; /* 0x80020024 */ +constexpr int KERNEL_ERROR_EALREADY = -2147352539; /* 0x80020025 */ +constexpr int KERNEL_ERROR_ENOTSOCK = -2147352538; /* 0x80020026 */ +constexpr int KERNEL_ERROR_EDESTADDRREQ = -2147352537; /* 0x80020027 */ +constexpr int KERNEL_ERROR_EMSGSIZE = -2147352536; /* 0x80020028 */ +constexpr int KERNEL_ERROR_EPROTOTYPE = -2147352535; /* 0x80020029 */ +constexpr int KERNEL_ERROR_ENOPROTOOPT = -2147352534; /* 0x8002002A */ +constexpr int KERNEL_ERROR_EPROTONOSUPPORT = -2147352533; /* 0x8002002B */ +constexpr int KERNEL_ERROR_ESOCKTNOSUPPORT = -2147352532; /* 0x8002002C */ +constexpr int KERNEL_ERROR_EOPNOTSUPP = -2147352531; /* 0x8002002D */ +constexpr int KERNEL_ERROR_ENOTSUP = -2147352531; /* 0x8002002D */ +constexpr int KERNEL_ERROR_EPFNOSUPPORT = -2147352530; /* 0x8002002E */ +constexpr int KERNEL_ERROR_EAFNOSUPPORT = -2147352529; /* 0x8002002F */ +constexpr int KERNEL_ERROR_EADDRINUSE = -2147352528; /* 0x80020030 */ +constexpr int KERNEL_ERROR_EADDRNOTAVAIL = -2147352527; /* 0x80020031 */ +constexpr int KERNEL_ERROR_ENETDOWN = -2147352526; /* 0x80020032 */ +constexpr int KERNEL_ERROR_ENETUNREACH = -2147352525; /* 0x80020033 */ +constexpr int KERNEL_ERROR_ENETRESET = -2147352524; /* 0x80020034 */ +constexpr int KERNEL_ERROR_ECONNABORTED = -2147352523; /* 0x80020035 */ +constexpr int KERNEL_ERROR_ECONNRESET = -2147352522; /* 0x80020036 */ +constexpr int KERNEL_ERROR_ENOBUFS = -2147352521; /* 0x80020037 */ +constexpr int KERNEL_ERROR_EISCONN = -2147352520; /* 0x80020038 */ +constexpr int KERNEL_ERROR_ENOTCONN = -2147352519; /* 0x80020039 */ +constexpr int KERNEL_ERROR_ESHUTDOWN = -2147352518; /* 0x8002003A */ +constexpr int KERNEL_ERROR_ETOOMANYREFS = -2147352517; /* 0x8002003B */ +constexpr int KERNEL_ERROR_ETIMEDOUT = -2147352516; /* 0x8002003C */ +constexpr int KERNEL_ERROR_ECONNREFUSED = -2147352515; /* 0x8002003D */ +constexpr int KERNEL_ERROR_ELOOP = -2147352514; /* 0x8002003E */ +constexpr int KERNEL_ERROR_ENAMETOOLONG = -2147352513; /* 0x8002003F */ +constexpr int KERNEL_ERROR_EHOSTDOWN = -2147352512; /* 0x80020040 */ +constexpr int KERNEL_ERROR_EHOSTUNREACH = -2147352511; /* 0x80020041 */ +constexpr int KERNEL_ERROR_ENOTEMPTY = -2147352510; /* 0x80020042 */ +constexpr int KERNEL_ERROR_EPROCLIM = -2147352509; /* 0x80020043 */ +constexpr int KERNEL_ERROR_EUSERS = -2147352508; /* 0x80020044 */ +constexpr int KERNEL_ERROR_EDQUOT = -2147352507; /* 0x80020045 */ +constexpr int KERNEL_ERROR_ESTALE = -2147352506; /* 0x80020046 */ +constexpr int KERNEL_ERROR_EREMOTE = -2147352505; /* 0x80020047 */ +constexpr int KERNEL_ERROR_EBADRPC = -2147352504; /* 0x80020048 */ +constexpr int KERNEL_ERROR_ERPCMISMATCH = -2147352503; /* 0x80020049 */ +constexpr int KERNEL_ERROR_EPROGUNAVAIL = -2147352502; /* 0x8002004A */ +constexpr int KERNEL_ERROR_EPROGMISMATCH = -2147352501; /* 0x8002004B */ +constexpr int KERNEL_ERROR_EPROCUNAVAIL = -2147352500; /* 0x8002004C */ +constexpr int KERNEL_ERROR_ENOLCK = -2147352499; /* 0x8002004D */ +constexpr int KERNEL_ERROR_ENOSYS = -2147352498; /* 0x8002004E */ +constexpr int KERNEL_ERROR_EFTYPE = -2147352497; /* 0x8002004F */ +constexpr int KERNEL_ERROR_EAUTH = -2147352496; /* 0x80020050 */ +constexpr int KERNEL_ERROR_ENEEDAUTH = -2147352495; /* 0x80020051 */ +constexpr int KERNEL_ERROR_EIDRM = -2147352494; /* 0x80020052 */ +constexpr int KERNEL_ERROR_ENOMSG = -2147352493; /* 0x80020053 */ +constexpr int KERNEL_ERROR_EOVERFLOW = -2147352492; /* 0x80020054 */ +constexpr int KERNEL_ERROR_ECANCELED = -2147352491; /* 0x80020055 */ +constexpr int KERNEL_ERROR_EILSEQ = -2147352490; /* 0x80020056 */ +constexpr int KERNEL_ERROR_ENOATTR = -2147352489; /* 0x80020057 */ +constexpr int KERNEL_ERROR_EDOOFUS = -2147352488; /* 0x80020058 */ +constexpr int KERNEL_ERROR_EBADMSG = -2147352487; /* 0x80020059 */ +constexpr int KERNEL_ERROR_EMULTIHOP = -2147352486; /* 0x8002005A */ +constexpr int KERNEL_ERROR_ENOLINK = -2147352485; /* 0x8002005B */ +constexpr int KERNEL_ERROR_EPROTO = -2147352484; /* 0x8002005C */ +constexpr int KERNEL_ERROR_ENOTCAPABLE = -2147352483; /* 0x8002005D */ +constexpr int KERNEL_ERROR_ECAPMODE = -2147352482; /* 0x8002005E */ +constexpr int KERNEL_ERROR_ENOBLK = -2147352481; /* 0x8002005F */ +constexpr int KERNEL_ERROR_EICV = -2147352480; /* 0x80020060 */ +constexpr int KERNEL_ERROR_ENOPLAYGOENT = -2147352479; /* 0x80020061 */ +constexpr int KERNEL_ERROR_EREVOKE = -2147352478; /* 0x80020062 */ +constexpr int KERNEL_ERROR_ESDKVERSION = -2147352477; /* 0x80020063 */ +constexpr int KERNEL_ERROR_ESTART = -2147352476; /* 0x80020064 */ +constexpr int KERNEL_ERROR_ESTOP = -2147352475; /* 0x80020065 */ + +} // namespace LibKernel + +namespace VideoOut { + +constexpr int VIDEO_OUT_ERROR_INVALID_VALUE = -2144796671; /* 0x80290001 */ +constexpr int VIDEO_OUT_ERROR_INVALID_ADDRESS = -2144796670; /* 0x80290002 */ +constexpr int VIDEO_OUT_ERROR_INVALID_PIXEL_FORMAT = -2144796669; /* 0x80290003 */ +constexpr int VIDEO_OUT_ERROR_INVALID_PITCH = -2144796668; /* 0x80290004 */ +constexpr int VIDEO_OUT_ERROR_INVALID_RESOLUTION = -2144796667; /* 0x80290005 */ +constexpr int VIDEO_OUT_ERROR_INVALID_FLIP_MODE = -2144796666; /* 0x80290006 */ +constexpr int VIDEO_OUT_ERROR_INVALID_TILING_MODE = -2144796665; /* 0x80290007 */ +constexpr int VIDEO_OUT_ERROR_INVALID_ASPECT_RATIO = -2144796664; /* 0x80290008 */ +constexpr int VIDEO_OUT_ERROR_RESOURCE_BUSY = -2144796663; /* 0x80290009 */ +constexpr int VIDEO_OUT_ERROR_INVALID_INDEX = -2144796662; /* 0x8029000A */ +constexpr int VIDEO_OUT_ERROR_INVALID_HANDLE = -2144796661; /* 0x8029000B */ +constexpr int VIDEO_OUT_ERROR_INVALID_EVENT_QUEUE = -2144796660; /* 0x8029000C */ +constexpr int VIDEO_OUT_ERROR_INVALID_EVENT = -2144796659; /* 0x8029000D */ +constexpr int VIDEO_OUT_ERROR_NO_EMPTY_SLOT = -2144796657; /* 0x8029000F */ +constexpr int VIDEO_OUT_ERROR_SLOT_OCCUPIED = -2144796656; /* 0x80290010 */ +constexpr int VIDEO_OUT_ERROR_FLIP_QUEUE_FULL = -2144796654; /* 0x80290012 */ +constexpr int VIDEO_OUT_ERROR_INVALID_MEMORY = -2144796653; /* 0x80290013 */ +constexpr int VIDEO_OUT_ERROR_MEMORY_NOT_PHYSICALLY_CONTIGUOUS = -2144796652; /* 0x80290014 */ +constexpr int VIDEO_OUT_ERROR_MEMORY_INVALID_ALIGNMENT = -2144796651; /* 0x80290015 */ +constexpr int VIDEO_OUT_ERROR_UNSUPPORTED_OUTPUT_MODE = -2144796650; /* 0x80290016 */ +constexpr int VIDEO_OUT_ERROR_OVERFLOW = -2144796649; /* 0x80290017 */ +constexpr int VIDEO_OUT_ERROR_NO_DEVICE = -2144796648; /* 0x80290018 */ +constexpr int VIDEO_OUT_ERROR_UNAVAILABLE_OUTPUT_MODE = -2144796647; /* 0x80290019 */ +constexpr int VIDEO_OUT_ERROR_INVALID_OPTION = -2144796646; /* 0x8029001A */ +constexpr int VIDEO_OUT_ERROR_PORT_UNSUPPORTED_FUNCTION = -2144796645; /* 0x8029001B */ +constexpr int VIDEO_OUT_ERROR_UNSUPPORTED_OPERATION = -2144796644; /* 0x8029001C */ +constexpr int VIDEO_OUT_ERROR_INVALID_CATEGORY = -2144796643; /* 0x8029001D */ +constexpr int VIDEO_OUT_ERROR_FATAL = -2144796417; /* 0x802900FF */ +constexpr int VIDEO_OUT_ERROR_UNKNOWN = -2144796418; /* 0x802900FE */ +constexpr int VIDEO_OUT_ERROR_ENOMEM = -2144792564; /* 0x8029100C */ + +} // namespace VideoOut + +namespace Audio { + +constexpr int AUDIO_OUT_ERROR_NOT_OPENED = -2144993279; /* 0x80260001 */ +constexpr int AUDIO_OUT_ERROR_BUSY = -2144993278; /* 0x80260002 */ +constexpr int AUDIO_OUT_ERROR_INVALID_PORT = -2144993277; /* 0x80260003 */ +constexpr int AUDIO_OUT_ERROR_INVALID_POINTER = -2144993276; /* 0x80260004 */ +constexpr int AUDIO_OUT_ERROR_PORT_FULL = -2144993275; /* 0x80260005 */ +constexpr int AUDIO_OUT_ERROR_INVALID_SIZE = -2144993274; /* 0x80260006 */ +constexpr int AUDIO_OUT_ERROR_INVALID_FORMAT = -2144993273; /* 0x80260007 */ +constexpr int AUDIO_OUT_ERROR_INVALID_SAMPLE_FREQ = -2144993272; /* 0x80260008 */ +constexpr int AUDIO_OUT_ERROR_INVALID_VOLUME = -2144993271; /* 0x80260009 */ +constexpr int AUDIO_OUT_ERROR_INVALID_PORT_TYPE = -2144993270; /* 0x8026000A */ +constexpr int AUDIO_OUT_ERROR_INVALID_CONF_TYPE = -2144993268; /* 0x8026000C */ +constexpr int AUDIO_OUT_ERROR_OUT_OF_MEMORY = -2144993267; /* 0x8026000D */ +constexpr int AUDIO_OUT_ERROR_ALREADY_INIT = -2144993266; /* 0x8026000E */ +constexpr int AUDIO_OUT_ERROR_NOT_INIT = -2144993265; /* 0x8026000F */ +constexpr int AUDIO_OUT_ERROR_MEMORY = -2144993264; /* 0x80260010 */ +constexpr int AUDIO_OUT_ERROR_SYSTEM_RESOURCE = -2144993263; /* 0x80260011 */ +constexpr int AUDIO_OUT_ERROR_TRANS_EVENT = -2144993262; /* 0x80260012 */ +constexpr int AUDIO_OUT_ERROR_INVALID_FLAG = -2144993261; /* 0x80260013 */ +constexpr int AUDIO_OUT_ERROR_INVALID_MIXLEVEL = -2144993260; /* 0x80260014 */ +constexpr int AUDIO_OUT_ERROR_INVALID_ARG = -2144993259; /* 0x80260015 */ +constexpr int AUDIO_OUT_ERROR_INVALID_PARAM = -2144993258; /* 0x80260016 */ + +constexpr int AUDIO_IN_ERROR_FATAL = -2144993024; /* 0x80260100 */ +constexpr int AUDIO_IN_ERROR_INVALID_HANDLE = -2144993023; /* 0x80260101 */ +constexpr int AUDIO_IN_ERROR_INVALID_SIZE = -2144993022; /* 0x80260102 */ +constexpr int AUDIO_IN_ERROR_INVALID_FREQ = -2144993021; /* 0x80260103 */ +constexpr int AUDIO_IN_ERROR_INVALID_TYPE = -2144993020; /* 0x80260104 */ +constexpr int AUDIO_IN_ERROR_INVALID_POINTER = -2144993019; /* 0x80260105 */ +constexpr int AUDIO_IN_ERROR_INVALID_PARAM = -2144993018; /* 0x80260106 */ +constexpr int AUDIO_IN_ERROR_PORT_FULL = -2144993017; /* 0x80260107 */ +constexpr int AUDIO_IN_ERROR_OUT_OF_MEMORY = -2144993016; /* 0x80260108 */ +constexpr int AUDIO_IN_ERROR_NOT_OPENED = -2144993015; /* 0x80260109 */ +constexpr int AUDIO_IN_ERROR_BUSY = -2144993014; /* 0x8026010A */ +constexpr int AUDIO_IN_ERROR_SYSTEM_MEMORY = -2144993013; /* 0x8026010B */ +constexpr int AUDIO_IN_ERROR_SYSTEM_IPC = -2144993012; /* 0x8026010C */ + +} // namespace Audio + +namespace SystemService { + +constexpr int SYSTEM_SERVICE_ERROR_INTERNAL = -2136932351; /* 0x80A10001 */ +constexpr int SYSTEM_SERVICE_ERROR_UNAVAILABLE = -2136932350; /* 0x80A10002 */ +constexpr int SYSTEM_SERVICE_ERROR_PARAMETER = -2136932349; /* 0x80A10003 */ +constexpr int SYSTEM_SERVICE_ERROR_NO_EVENT = -2136932348; /* 0x80A10004 */ +constexpr int SYSTEM_SERVICE_ERROR_REJECTED = -2136932347; /* 0x80A10005 */ +constexpr int SYSTEM_SERVICE_ERROR_NEED_DISPLAY_SAFE_AREA_SETTINGS = -2136932346; /* 0x80A10006 */ +constexpr int SYSTEM_SERVICE_ERROR_INVALID_URI_LEN = -2136932345; /* 0x80A10007 */ +constexpr int SYSTEM_SERVICE_ERROR_INVALID_URI_SCHEME = -2136932344; /* 0x80A10008 */ +constexpr int SYSTEM_SERVICE_ERROR_NO_APP_INFO = -2136932343; /* 0x80A10009 */ +constexpr int SYSTEM_SERVICE_ERROR_NOT_FLAG_IN_PARAM_SFO = -2136932342; /* 0x80A1000A */ + +} // namespace SystemService + +namespace Network { + +inline int NetToPosix(int kernel_errno) { + return kernel_errno > -2143223551 && kernel_errno <= -2143223316 ? kernel_errno + 2143223551 + 1 + : 204; +} + +constexpr int NET_ERROR_EPERM = -2143223551; /* 0x80410101 */ +constexpr int NET_ERROR_ENOENT = -2143223550; /* 0x80410102 */ +constexpr int NET_ERROR_ESRCH = -2143223549; /* 0x80410103 */ +constexpr int NET_ERROR_EINTR = -2143223548; /* 0x80410104 */ +constexpr int NET_ERROR_EIO = -2143223547; /* 0x80410105 */ +constexpr int NET_ERROR_ENXIO = -2143223546; /* 0x80410106 */ +constexpr int NET_ERROR_E2BIG = -2143223545; /* 0x80410107 */ +constexpr int NET_ERROR_ENOEXEC = -2143223544; /* 0x80410108 */ +constexpr int NET_ERROR_EBADF = -2143223543; /* 0x80410109 */ +constexpr int NET_ERROR_ECHILD = -2143223542; /* 0x8041010A */ +constexpr int NET_ERROR_EDEADLK = -2143223541; /* 0x8041010B */ +constexpr int NET_ERROR_ENOMEM = -2143223540; /* 0x8041010C */ +constexpr int NET_ERROR_EACCES = -2143223539; /* 0x8041010D */ +constexpr int NET_ERROR_EFAULT = -2143223538; /* 0x8041010E */ +constexpr int NET_ERROR_ENOTBLK = -2143223537; /* 0x8041010F */ +constexpr int NET_ERROR_EBUSY = -2143223536; /* 0x80410110 */ +constexpr int NET_ERROR_EEXIST = -2143223535; /* 0x80410111 */ +constexpr int NET_ERROR_EXDEV = -2143223534; /* 0x80410112 */ +constexpr int NET_ERROR_ENODEV = -2143223533; /* 0x80410113 */ +constexpr int NET_ERROR_ENOTDIR = -2143223532; /* 0x80410114 */ +constexpr int NET_ERROR_EISDIR = -2143223531; /* 0x80410115 */ +constexpr int NET_ERROR_EINVAL = -2143223530; /* 0x80410116 */ +constexpr int NET_ERROR_ENFILE = -2143223529; /* 0x80410117 */ +constexpr int NET_ERROR_EMFILE = -2143223528; /* 0x80410118 */ +constexpr int NET_ERROR_ENOTTY = -2143223527; /* 0x80410119 */ +constexpr int NET_ERROR_ETXTBSY = -2143223526; /* 0x8041011A */ +constexpr int NET_ERROR_EFBIG = -2143223525; /* 0x8041011B */ +constexpr int NET_ERROR_ENOSPC = -2143223524; /* 0x8041011C */ +constexpr int NET_ERROR_ESPIPE = -2143223523; /* 0x8041011D */ +constexpr int NET_ERROR_EROFS = -2143223522; /* 0x8041011E */ +constexpr int NET_ERROR_EMLINK = -2143223521; /* 0x8041011F */ +constexpr int NET_ERROR_EPIPE = -2143223520; /* 0x80410120 */ +constexpr int NET_ERROR_EDOM = -2143223519; /* 0x80410121 */ +constexpr int NET_ERROR_ERANGE = -2143223518; /* 0x80410122 */ +constexpr int NET_ERROR_EAGAIN = -2143223517; /* 0x80410123 */ +constexpr int NET_ERROR_EWOULDBLOCK = -2143223517; /* 0x80410123 */ +constexpr int NET_ERROR_EINPROGRESS = -2143223516; /* 0x80410124 */ +constexpr int NET_ERROR_EALREADY = -2143223515; /* 0x80410125 */ +constexpr int NET_ERROR_ENOTSOCK = -2143223514; /* 0x80410126 */ +constexpr int NET_ERROR_EDESTADDRREQ = -2143223513; /* 0x80410127 */ +constexpr int NET_ERROR_EMSGSIZE = -2143223512; /* 0x80410128 */ +constexpr int NET_ERROR_EPROTOTYPE = -2143223511; /* 0x80410129 */ +constexpr int NET_ERROR_ENOPROTOOPT = -2143223510; /* 0x8041012A */ +constexpr int NET_ERROR_EPROTONOSUPPORT = -2143223509; /* 0x8041012B */ +constexpr int NET_ERROR_ESOCKTNOSUPPORT = -2143223508; /* 0x8041012C */ +constexpr int NET_ERROR_EOPNOTSUPP = -2143223507; /* 0x8041012D */ +constexpr int NET_ERROR_ENOTSUP = -2143223507; /* 0x8041012D */ +constexpr int NET_ERROR_EPFNOSUPPORT = -2143223506; /* 0x8041012E */ +constexpr int NET_ERROR_EAFNOSUPPORT = -2143223505; /* 0x8041012F */ +constexpr int NET_ERROR_EADDRINUSE = -2143223504; /* 0x80410130 */ +constexpr int NET_ERROR_EADDRNOTAVAIL = -2143223503; /* 0x80410131 */ +constexpr int NET_ERROR_ENETDOWN = -2143223502; /* 0x80410132 */ +constexpr int NET_ERROR_ENETUNREACH = -2143223501; /* 0x80410133 */ +constexpr int NET_ERROR_ENETRESET = -2143223500; /* 0x80410134 */ +constexpr int NET_ERROR_ECONNABORTED = -2143223499; /* 0x80410135 */ +constexpr int NET_ERROR_ECONNRESET = -2143223498; /* 0x80410136 */ +constexpr int NET_ERROR_ENOBUFS = -2143223497; /* 0x80410137 */ +constexpr int NET_ERROR_EISCONN = -2143223496; /* 0x80410138 */ +constexpr int NET_ERROR_ENOTCONN = -2143223495; /* 0x80410139 */ +constexpr int NET_ERROR_ESHUTDOWN = -2143223494; /* 0x8041013A */ +constexpr int NET_ERROR_ETOOMANYREFS = -2143223493; /* 0x8041013B */ +constexpr int NET_ERROR_ETIMEDOUT = -2143223492; /* 0x8041013C */ +constexpr int NET_ERROR_ECONNREFUSED = -2143223491; /* 0x8041013D */ +constexpr int NET_ERROR_ELOOP = -2143223490; /* 0x8041013E */ +constexpr int NET_ERROR_ENAMETOOLONG = -2143223489; /* 0x8041013F */ +constexpr int NET_ERROR_EHOSTDOWN = -2143223488; /* 0x80410140 */ +constexpr int NET_ERROR_EHOSTUNREACH = -2143223487; /* 0x80410141 */ +constexpr int NET_ERROR_ENOTEMPTY = -2143223486; /* 0x80410142 */ +constexpr int NET_ERROR_EPROCLIM = -2143223485; /* 0x80410143 */ +constexpr int NET_ERROR_EUSERS = -2143223484; /* 0x80410144 */ +constexpr int NET_ERROR_EDQUOT = -2143223483; /* 0x80410145 */ +constexpr int NET_ERROR_ESTALE = -2143223482; /* 0x80410146 */ +constexpr int NET_ERROR_EREMOTE = -2143223481; /* 0x80410147 */ +constexpr int NET_ERROR_EBADRPC = -2143223480; /* 0x80410148 */ +constexpr int NET_ERROR_ERPCMISMATCH = -2143223479; /* 0x80410149 */ +constexpr int NET_ERROR_EPROGUNAVAIL = -2143223478; /* 0x8041014A */ +constexpr int NET_ERROR_EPROGMISMATCH = -2143223477; /* 0x8041014B */ +constexpr int NET_ERROR_EPROCUNAVAIL = -2143223476; /* 0x8041014C */ +constexpr int NET_ERROR_ENOLCK = -2143223475; /* 0x8041014D */ +constexpr int NET_ERROR_ENOSYS = -2143223474; /* 0x8041014E */ +constexpr int NET_ERROR_EFTYPE = -2143223473; /* 0x8041014F */ +constexpr int NET_ERROR_EAUTH = -2143223472; /* 0x80410150 */ +constexpr int NET_ERROR_ENEEDAUTH = -2143223471; /* 0x80410151 */ +constexpr int NET_ERROR_EIDRM = -2143223470; /* 0x80410152 */ +constexpr int NET_ERROR_ENOMS = -2143223469; /* 0x80410153 */ +constexpr int NET_ERROR_EOVERFLOW = -2143223468; /* 0x80410154 */ +constexpr int NET_ERROR_ECANCELED = -2143223467; /* 0x80410155 */ +constexpr int NET_ERROR_EPROTO = -2143223460; /* 0x8041015C */ +constexpr int NET_ERROR_EADHOC = -2143223392; /* 0x804101A0 */ +constexpr int NET_ERROR_ERESERVED161 = -2143223391; /* 0x804101A1 */ +constexpr int NET_ERROR_ERESERVED162 = -2143223390; /* 0x804101A2 */ +constexpr int NET_ERROR_EINACTIVEDISABLED = -2143223389; /* 0x804101A3 */ +constexpr int NET_ERROR_ENODATA = -2143223388; /* 0x804101A4 */ +constexpr int NET_ERROR_EDESC = -2143223387; /* 0x804101A5 */ +constexpr int NET_ERROR_EDESCTIMEDOUT = -2143223386; /* 0x804101A6 */ +constexpr int NET_ERROR_ENETINTR = -2143223385; /* 0x804101A7 */ +constexpr int NET_ERROR_ENOTINIT = -2143223352; /* 0x804101C8 */ +constexpr int NET_ERROR_ENOLIBMEM = -2143223351; /* 0x804101C9 */ +constexpr int NET_ERROR_ERESERVED202 = -2143223350; /* 0x804101CA */ +constexpr int NET_ERROR_ECALLBACK = -2143223349; /* 0x804101CB */ +constexpr int NET_ERROR_EINTERNAL = -2143223348; /* 0x804101CC */ +constexpr int NET_ERROR_ERETURN = -2143223347; /* 0x804101CD */ +constexpr int NET_ERROR_ENOALLOCMEM = -2143223346; /* 0x804101CE */ +constexpr int NET_ERROR_RESOLVER_EINTERNAL = -2143223332; /* 0x804101DC */ +constexpr int NET_ERROR_RESOLVER_EBUSY = -2143223331; /* 0x804101DD */ +constexpr int NET_ERROR_RESOLVER_ENOSPACE = -2143223330; /* 0x804101DE */ +constexpr int NET_ERROR_RESOLVER_EPACKET = -2143223329; /* 0x804101DF */ +constexpr int NET_ERROR_RESOLVER_ERESERVED224 = -2143223328; /* 0x804101E0 */ +constexpr int NET_ERROR_RESOLVER_ENODNS = -2143223327; /* 0x804101E1 */ +constexpr int NET_ERROR_RESOLVER_ETIMEDOUT = -2143223326; /* 0x804101E2 */ +constexpr int NET_ERROR_RESOLVER_ENOSUPPORT = -2143223325; /* 0x804101E3 */ +constexpr int NET_ERROR_RESOLVER_EFORMAT = -2143223324; /* 0x804101E4 */ +constexpr int NET_ERROR_RESOLVER_ESERVERFAILURE = -2143223323; /* 0x804101E5 */ +constexpr int NET_ERROR_RESOLVER_ENOHOST = -2143223322; /* 0x804101E6 */ +constexpr int NET_ERROR_RESOLVER_ENOTIMPLEMENTED = -2143223321; /* 0x804101E7 */ +constexpr int NET_ERROR_RESOLVER_ESERVERREFUSED = -2143223320; /* 0x804101E8 */ +constexpr int NET_ERROR_RESOLVER_ENORECORD = -2143223319; /* 0x804101E9 */ +constexpr int NET_ERROR_RESOLVER_EALIGNMENT = -2143223318; /* 0x804101EA */ +constexpr int NET_ERROR_RESOLVER_ENOTFOUND = -2143223317; /* 0x804101EB */ +constexpr int NET_ERROR_RESOLVER_ENOTINIT = -2143223316; /* 0x804101EC */ + +constexpr int SSL_ERROR_BEFORE_INIT = -2137657343; /* 0x8095F001 */ +constexpr int SSL_ERROR_ALREADY_INITED = -2137657342; /* 0x8095F002 */ +constexpr int SSL_ERROR_BROKEN = -2137657341; /* 0x8095F003 */ +constexpr int SSL_ERROR_NOT_FOUND = -2137657340; /* 0x8095F004 */ +constexpr int SSL_ERROR_INVALID_FORMAT = -2137657339; /* 0x8095F005 */ +constexpr int SSL_ERROR_INVALID_ID = -2137657338; /* 0x8095F006 */ +constexpr int SSL_ERROR_INVALID_VALUE = -2137657337; /* 0x8095F007 */ +constexpr int SSL_ERROR_OUT_OF_SIZE = -2137657336; /* 0x8095F008 */ +constexpr int SSL_ERROR_INTERNAL = -2137657335; /* 0x8095F009 */ +constexpr int SSL_ERROR_INVALID_CERT = -2137657334; /* 0x8095F00A */ +constexpr int SSL_ERROR_CN_CHECK = -2137657333; /* 0x8095F00B */ +constexpr int SSL_ERROR_UNKNOWN_CA = -2137657332; /* 0x8095F00C */ +constexpr int SSL_ERROR_NOT_AFTER_CHECK = -2137657331; /* 0x8095F00D */ +constexpr int SSL_ERROR_NOT_BEFORE_CHECK = -2137657330; /* 0x8095F00E */ +constexpr int SSL_ERROR_EAGAIN = -2137657329; /* 0x8095F00F */ +constexpr int SSL_ERROR_FATAL_ALERT = -2137657328; /* 0x8095F010 */ +constexpr int SSL_ERROR_BUSY = -2137657326; /* 0x8095F012 */ +constexpr int SSL_ERROR_WANT_POLLIN = -2137657325; /* 0x8095F013 */ +constexpr int SSL_ERROR_WANT_POLLOUT = -2137657324; /* 0x8095F014 */ +constexpr int SSL_ERROR_INSUFFICIENT_STACKSIZE = -2137657323; /* 0x8095F015 */ +constexpr int SSL_ERROR_APPLICATION_VERIFICATION_CALLBACK_CHECK = -2137657091; /* 0x8095F0FD */ +constexpr int SSL_ERROR_OUT_OF_MEMORY = -2137712683; /* 0x809517D5 */ +constexpr int SSL_ERROR_READ_TIMEOUT = -2137712880; /* 0x80951710 */ +constexpr int SSL_ERROR_SOCKET_CLOSED = -2137712883; /* 0x8095170D */ +constexpr int SSL_ERROR_BAD_LENGTH = -2137712781; /* 0x80951773 */ +constexpr int SSL_ERROR_INDEX_OOB = -2137712776; /* 0x80951778 */ +constexpr int SSL_ERROR_INVALID_ARG = -2137712774; /* 0x8095177A */ +constexpr int SSL_ERROR_EOF = -2137712772; /* 0x8095177C */ +constexpr int SSL_ERROR_BAD_EXPONENT = -2137712771; /* 0x8095177D */ +constexpr int SSL_ERROR_INCOMPLETE_SEARCH = -2137712770; /* 0x8095177E */ +constexpr int SSL_ERROR_INTERNAL_ERROR = -2137712769; /* 0x8095177F */ +constexpr int SSL_ERROR_BAD_CERT_LENGTH = -2137710382; /* 0x809520D2 */ +constexpr int SSL_ERROR_BAD_SIGN_ALGO = -2137710383; /* 0x809520D1 */ +constexpr int SSL_ERROR_MISMATCH_PUBLIC_KEYS = -2137710381; /* 0x809520D3 */ +constexpr int SSL_ERROR_KEY_BLOB_CORRUPT = -2137710380; /* 0x809520D4 */ + +constexpr int HTTP_ERROR_BEFORE_INIT = -2143088639; /*(0x80431001) */ +constexpr int HTTP_ERROR_ALREADY_INITED = -2143088608; /*(0x80431020) */ +constexpr int HTTP_ERROR_BUSY = -2143088607; /*(0x80431021) */ +constexpr int HTTP_ERROR_OUT_OF_MEMORY = -2143088606; /*(0x80431022) */ +constexpr int HTTP_ERROR_NOT_FOUND = -2143088603; /*(0x80431025) */ +constexpr int HTTP_ERROR_INVALID_VERSION = -2143088534; /*(0x8043106a) */ +constexpr int HTTP_ERROR_INVALID_ID = -2143088384; /*(0x80431100) */ +constexpr int HTTP_ERROR_OUT_OF_SIZE = -2143088380; /*(0x80431104) */ +constexpr int HTTP_ERROR_INVALID_VALUE = -2143088130; /*(0x804311fe) */ +constexpr int HTTP_ERROR_INVALID_URL = -2143080352; /*(0x80433060) */ +constexpr int HTTP_ERROR_UNKNOWN_SCHEME = -2143088543; /*(0x80431061) */ +constexpr int HTTP_ERROR_NETWORK = -2143088541; /*(0x80431063) */ +constexpr int HTTP_ERROR_BAD_RESPONSE = -2143088540; /*(0x80431064) */ +constexpr int HTTP_ERROR_BEFORE_SEND = -2143088539; /*(0x80431065) */ +constexpr int HTTP_ERROR_AFTER_SEND = -2143088538; /*(0x80431066) */ +constexpr int HTTP_ERROR_TIMEOUT = -2143088536; /*(0x80431068) */ +constexpr int HTTP_ERROR_UNKNOWN_AUTH_TYPE = -2143088535; /*(0x80431069) */ +constexpr int HTTP_ERROR_UNKNOWN_METHOD = -2143088533; /*(0x8043106b) */ +constexpr int HTTP_ERROR_READ_BY_HEAD_METHOD = -2143088529; /*(0x8043106f) */ +constexpr int HTTP_ERROR_NOT_IN_COM = -2143088528; /*(0x80431070) */ +constexpr int HTTP_ERROR_NO_CONTENT_LENGTH = -2143088527; /*(0x80431071) */ +constexpr int HTTP_ERROR_CHUNK_ENC = -2143088526; /*(0x80431072) */ +constexpr int HTTP_ERROR_TOO_LARGE_RESPONSE_HEADER = -2143088525; /*(0x80431073) */ +constexpr int HTTP_ERROR_SSL = -2143088523; /*(0x80431075) */ +constexpr int HTTP_ERROR_INSUFFICIENT_STACKSIZE = -2143088522; /*(0x80431076) */ +constexpr int HTTP_ERROR_ABORTED = -2143088512; /*(0x80431080) */ +constexpr int HTTP_ERROR_UNKNOWN = -2143088511; /*(0x80431081) */ +constexpr int HTTP_ERROR_EAGAIN = -2143088510; /*(0x80431082) */ +constexpr int HTTP_ERROR_PROXY = -2143088508; /*(0x80431084) */ +constexpr int HTTP_ERROR_BROKEN = -2143088507; /*(0x80431085) */ +constexpr int HTTP_ERROR_PARSE_HTTP_NOT_FOUND = -2143084507; /*(0x80432025) */ +constexpr int HTTP_ERROR_PARSE_HTTP_INVALID_RESPONSE = -2143084448; /*(0x80432060) */ +constexpr int HTTP_ERROR_PARSE_HTTP_INVALID_VALUE = -2143084034; /*(0x804321fe) */ +constexpr int HTTP_ERROR_RESOLVER_EPACKET = -2143068159; /*(0x80436001) */ +constexpr int HTTP_ERROR_RESOLVER_ENODNS = -2143068158; /*(0x80436002) */ +constexpr int HTTP_ERROR_RESOLVER_ETIMEDOUT = -2143068157; /*(0x80436003) */ +constexpr int HTTP_ERROR_RESOLVER_ENOSUPPORT = -2143068156; /*(0x80436004) */ +constexpr int HTTP_ERROR_RESOLVER_EFORMAT = -2143068155; /*(0x80436005) */ +constexpr int HTTP_ERROR_RESOLVER_ESERVERFAILURE = -2143068154; /*(0x80436006) */ +constexpr int HTTP_ERROR_RESOLVER_ENOHOST = -2143068153; /*(0x80436007) */ +constexpr int HTTP_ERROR_RESOLVER_ENOTIMPLEMENTED = -2143068152; /*(0x80436008) */ +constexpr int HTTP_ERROR_RESOLVER_ESERVERREFUSED = -2143068151; /*(0x80436009) */ +constexpr int HTTP_ERROR_RESOLVER_ENORECORD = -2143068150; /*(0x8043600a) */ +constexpr int HTTPS_ERROR_CERT = -2143072160; /*(0x80435060) */ +constexpr int HTTPS_ERROR_HANDSHAKE = -2143072159; /*(0x80435061) */ +constexpr int HTTPS_ERROR_IO = -2143072158; /*(0x80435062) */ +constexpr int HTTPS_ERROR_INTERNAL = -2143072157; /*(0x80435063) */ +constexpr int HTTPS_ERROR_PROXY = -2143072156; /*(0x80435064) */ + +} // namespace Network + +namespace PlayGo { + +constexpr int PLAYGO_ERROR_UNKNOWN = -2135818239; /* 0x80B20001 */ +constexpr int PLAYGO_ERROR_FATAL = -2135818238; /* 0x80B20002 */ +constexpr int PLAYGO_ERROR_NO_MEMORY = -2135818237; /* 0x80B20003 */ +constexpr int PLAYGO_ERROR_INVALID_ARGUMENT = -2135818236; /* 0x80B20004 */ +constexpr int PLAYGO_ERROR_NOT_INITIALIZED = -2135818235; /* 0x80B20005 */ +constexpr int PLAYGO_ERROR_ALREADY_INITIALIZED = -2135818234; /* 0x80B20006 */ +constexpr int PLAYGO_ERROR_ALREADY_STARTED = -2135818233; /* 0x80B20007 */ +constexpr int PLAYGO_ERROR_NOT_STARTED = -2135818232; /* 0x80B20008 */ +constexpr int PLAYGO_ERROR_BAD_HANDLE = -2135818231; /* 0x80B20009 */ +constexpr int PLAYGO_ERROR_BAD_POINTER = -2135818230; /* 0x80B2000A */ +constexpr int PLAYGO_ERROR_BAD_SIZE = -2135818229; /* 0x80B2000B */ +constexpr int PLAYGO_ERROR_BAD_CHUNK_ID = -2135818228; /* 0x80B2000C */ +constexpr int PLAYGO_ERROR_BAD_SPEED = -2135818227; /* 0x80B2000D */ +constexpr int PLAYGO_ERROR_NOT_SUPPORT_PLAYGO = -2135818226; /* 0x80B2000E */ +constexpr int PLAYGO_ERROR_EPERM = -2135818225; /* 0x80B2000F */ +constexpr int PLAYGO_ERROR_BAD_LOCUS = -2135818224; /* 0x80B20010 */ +constexpr int PLAYGO_ERROR_NEED_DATA_DISC = -2135818223; /* 0x80B20011 */ +constexpr int PLAYGO_ERROR_BAD_OPTIONAL_TYPE = -2135818204; /* 0x80B20024 */ + +} // namespace PlayGo + +namespace UserService { + +constexpr int USER_SERVICE_ERROR_INTERNAL = -2137653247; /* 0x80960001 */ +constexpr int USER_SERVICE_ERROR_NOT_INITIALIZED = -2137653246; /* 0x80960002 */ +constexpr int USER_SERVICE_ERROR_ALREADY_INITIALIZED = -2137653245; /* 0x80960003 */ +constexpr int USER_SERVICE_ERROR_NO_MEMORY = -2137653244; /* 0x80960004 */ +constexpr int USER_SERVICE_ERROR_INVALID_ARGUMENT = -2137653243; /* 0x80960005 */ +constexpr int USER_SERVICE_ERROR_OPERATION_NOT_SUPPORTED = -2137653242; /* 0x80960006 */ +constexpr int USER_SERVICE_ERROR_NO_EVENT = -2137653241; /* 0x80960007 */ +constexpr int USER_SERVICE_ERROR_NOT_LOGGED_IN = -2137653239; /* 0x80960009 */ +constexpr int USER_SERVICE_ERROR_BUFFER_TOO_SHORT = -2137653238; /* 0x8096000A */ + +} // namespace UserService + +namespace SaveData { + +constexpr int SAVE_DATA_ERROR_PARAMETER = -2137063424; /* 0x809F0000 */ +constexpr int SAVE_DATA_ERROR_NOT_INITIALIZED = -2137063423; /* 0x809F0001 */ +constexpr int SAVE_DATA_ERROR_OUT_OF_MEMORY = -2137063422; /* 0x809F0002 */ +constexpr int SAVE_DATA_ERROR_BUSY = -2137063421; /* 0x809F0003 */ +constexpr int SAVE_DATA_ERROR_NOT_MOUNTED = -2137063420; /* 0x809F0004 */ +constexpr int SAVE_DATA_ERROR_NO_PERMISSION = -2137063419; /* 0x809F0005 */ +constexpr int SAVE_DATA_ERROR_FINGERPRINT_MISMATCH = -2137063418; /* 0x809F0006 */ +constexpr int SAVE_DATA_ERROR_EXISTS = -2137063417; /* 0x809F0007 */ +constexpr int SAVE_DATA_ERROR_NOT_FOUND = -2137063416; /* 0x809F0008 */ +constexpr int SAVE_DATA_ERROR_NO_SPACE_FS = -2137063414; /* 0x809F000A */ +constexpr int SAVE_DATA_ERROR_INTERNAL = -2137063413; /* 0x809F000B */ +constexpr int SAVE_DATA_ERROR_MOUNT_FULL = -2137063412; /* 0x809F000C */ +constexpr int SAVE_DATA_ERROR_BAD_MOUNTED = -2137063411; /* 0x809F000D */ +constexpr int SAVE_DATA_ERROR_FILE_NOT_FOUND = -2137063410; /* 0x809F000E */ +constexpr int SAVE_DATA_ERROR_BROKEN = -2137063409; /* 0x809F000F */ +constexpr int SAVE_DATA_ERROR_INVALID_LOGIN_USER = -2137063407; /* 0x809F0011 */ +constexpr int SAVE_DATA_ERROR_MEMORY_NOT_READY = -2137063406; /* 0x809F0012 */ +constexpr int SAVE_DATA_ERROR_BACKUP_BUSY = -2137063405; /* 0x809F0013 */ +constexpr int SAVE_DATA_ERROR_NOT_REGIST_CALLBACK = -2137063403; /* 0x809F0015 */ +constexpr int SAVE_DATA_ERROR_BUSY_FOR_SAVING = -2137063402; /* 0x809F0016 */ +constexpr int SAVE_DATA_ERROR_LIMITATION_OVER = -2137063401; /* 0x809F0017 */ +constexpr int SAVE_DATA_ERROR_EVENT_BUSY = -2137063400; /* 0x809F0018 */ +constexpr int SAVE_DATA_ERROR_PARAMSFO_TRANSFER_TITLE_ID_NOT_FOUND = -2137063399; /* 0x809F0019 */ + +} // namespace SaveData + +} // namespace Libs + +#endif /* EMULATOR_INCLUDE_EMULATOR_LIBS_ERRNO_H_ */ diff --git a/src/libs/guestPrintf.cpp b/src/libs/guestPrintf.cpp new file mode 100644 index 000000000..c004a8ae6 --- /dev/null +++ b/src/libs/guestPrintf.cpp @@ -0,0 +1,808 @@ +// +// Original algorithm is from: +// https://github.com/mpaland/printf +// Marco Paland (info@paland.com) +// 2014-2019, PALANDesign Hannover, Germany +// licensed under The MIT License (MIT) + +#include "libs/guestPrintf.h" + +#include "common/abi.h" +#include "common/assert.h" +#include "common/common.h" +#include "common/logging/log.h" +#include "libs/vaContext.h" + +#include +#include +#include + +namespace Libs { + +constexpr uint32_t FLAGS_ZEROPAD = (1U << 0U); +constexpr uint32_t FLAGS_LEFT = (1U << 1U); +constexpr uint32_t FLAGS_PLUS = (1U << 2U); +constexpr uint32_t FLAGS_SPACE = (1U << 3U); +constexpr uint32_t FLAGS_HASH = (1U << 4U); +constexpr uint32_t FLAGS_UPPERCASE = (1U << 5U); +constexpr uint32_t FLAGS_CHAR = (1U << 6U); +constexpr uint32_t FLAGS_SHORT = (1U << 7U); +constexpr uint32_t FLAGS_LONG = (1U << 8U); +constexpr uint32_t FLAGS_LONG_LONG = (1U << 9U); +constexpr uint32_t FLAGS_PRECISION = (1U << 10U); +constexpr uint32_t FLAGS_ADAPT_EXP = (1U << 11U); + +constexpr size_t PRINTF_NTOA_BUFFER_SIZE = 32U; +constexpr size_t PRINTF_FTOA_BUFFER_SIZE = 32U; +constexpr double PRINTF_MAX_FLOAT = 1e9; +constexpr uint32_t PRINTF_DEFAULT_FLOAT_PRECISION = 6U; + +using out_fct_type = void (*)(char character, std::vector* buffer, size_t idx, + size_t /*maxlen*/); + +// internal null output +static inline void _out_null(char character, std::vector* buffer, size_t /*idx*/, + size_t /*maxlen*/) { + buffer->push_back(character); +} + +static inline bool _is_digit(char ch) { + return (ch >= '0') && (ch <= '9'); +} + +static unsigned int _atoi(const char** str) { + unsigned int i = 0U; + while (_is_digit(**str)) { + i = i * 10U + static_cast(*((*str)++) - '0'); + } + return i; +} + +static size_t _out_rev(out_fct_type out, std::vector* buffer, size_t idx, size_t maxlen, + const char* buf, size_t len, unsigned int width, unsigned int flags) { + const size_t start_idx = idx; + + // pad spaces up to given width + if ((flags & FLAGS_LEFT) == 0 && (flags & FLAGS_ZEROPAD) == 0) { + for (size_t i = len; i < width; i++) { + out(' ', buffer, idx++, maxlen); + } + } + + // reverse string + while (len != 0u) { + out(buf[--len], buffer, idx++, maxlen); + } + + // append pad spaces up to given width + if ((flags & FLAGS_LEFT) != 0u) { + while (idx - start_idx < width) { + out(' ', buffer, idx++, maxlen); + } + } + + return idx; +} + +// internal itoa format +static size_t _ntoa_format(out_fct_type out, std::vector* buffer, size_t idx, size_t maxlen, + char* buf, size_t len, bool negative, unsigned int base, + unsigned int prec, unsigned int width, unsigned int flags) { + // pad leading zeros + if ((flags & FLAGS_LEFT) == 0u) { + if ((width != 0u) && ((flags & FLAGS_ZEROPAD) != 0u) && + (negative || ((flags & (FLAGS_PLUS | FLAGS_SPACE)) != 0u))) { + width--; + } + while ((len < prec) && (len < PRINTF_NTOA_BUFFER_SIZE)) { + buf[len++] = '0'; + } + while (((flags & FLAGS_ZEROPAD) != 0u) && (len < width) && + (len < PRINTF_NTOA_BUFFER_SIZE)) { + buf[len++] = '0'; + } + } + + // handle hash + if ((flags & FLAGS_HASH) != 0u) { + if (((flags & FLAGS_PRECISION) == 0u) && (len != 0u) && ((len == prec) || (len == width))) { + len--; + if ((len != 0u) && (base == 16U)) { + len--; + } + } + if ((base == 16U) && ((flags & FLAGS_UPPERCASE) == 0u) && (len < PRINTF_NTOA_BUFFER_SIZE)) { + buf[len++] = 'x'; + } else if ((base == 16U) && ((flags & FLAGS_UPPERCASE) != 0u) && + (len < PRINTF_NTOA_BUFFER_SIZE)) { + buf[len++] = 'X'; + } else if ((base == 2U) && (len < PRINTF_NTOA_BUFFER_SIZE)) { + buf[len++] = 'b'; + } + if (len < PRINTF_NTOA_BUFFER_SIZE) { + buf[len++] = '0'; + } + } + + if (len < PRINTF_NTOA_BUFFER_SIZE) { + if (negative) { + buf[len++] = '-'; + } else if ((flags & FLAGS_PLUS) != 0u) { + buf[len++] = '+'; // ignore the space if the '+' exists + } else if ((flags & FLAGS_SPACE) != 0u) { + buf[len++] = ' '; + } + } + + return _out_rev(out, buffer, idx, maxlen, buf, len, width, flags); +} + +static size_t _ntoa_long_long(out_fct_type out, std::vector* buffer, size_t idx, + size_t maxlen, uint64_t value, bool negative, uint64_t base, + unsigned int prec, unsigned int width, unsigned int flags) { + char buf[PRINTF_NTOA_BUFFER_SIZE]; + size_t len = 0U; + + // no hash for 0 values + if (value == 0u) { + flags &= ~FLAGS_HASH; + } + + // write if precision != 0 and value is != 0 + if (((flags & FLAGS_PRECISION) == 0u) || (value != 0u)) { + do { + const char digit = static_cast(value % base); + // NOLINTNEXTLINE(bugprone-narrowing-conversions,cppcoreguidelines-narrowing-conversions) + buf[len++] = digit < 10 ? '0' + digit + : ((flags & FLAGS_UPPERCASE) != 0u ? 'A' : 'a') + digit - 10; + value /= base; + } while ((value != 0u) && (len < PRINTF_NTOA_BUFFER_SIZE)); + } + + return _ntoa_format(out, buffer, idx, maxlen, buf, len, negative, + static_cast(base), prec, width, flags); +} + +// internal itoa for 'long' type +static size_t _ntoa_long(out_fct_type out, std::vector* buffer, size_t idx, size_t maxlen, + uint32_t value, bool negative, uint32_t base, unsigned int prec, + unsigned int width, unsigned int flags) { + char buf[PRINTF_NTOA_BUFFER_SIZE]; + size_t len = 0U; + + // no hash for 0 values + if (value == 0u) { + flags &= ~FLAGS_HASH; + } + + // write if precision != 0 and value is != 0 + if (((flags & FLAGS_PRECISION) == 0u) || (value != 0u)) { + do { + char digit = static_cast(value % base); + // NOLINTNEXTLINE(bugprone-narrowing-conversions,cppcoreguidelines-narrowing-conversions) + buf[len++] = digit < 10 ? '0' + digit + : ((flags & FLAGS_UPPERCASE) != 0u ? 'A' : 'a') + digit - 10; + value /= base; + } while ((value != 0u) && (len < PRINTF_NTOA_BUFFER_SIZE)); + } + + return _ntoa_format(out, buffer, idx, maxlen, buf, len, negative, + static_cast(base), prec, width, flags); +} + +static size_t _etoa(out_fct_type out, std::vector* buffer, size_t idx, size_t maxlen, + double value, unsigned int prec, unsigned int width, unsigned int flags); + +// internal ftoa for fixed decimal floating point +// NOLINTNEXTLINE(readability-function-cognitive-complexity) +static size_t _ftoa(out_fct_type out, std::vector* buffer, size_t idx, size_t maxlen, + double value, unsigned int prec, unsigned int width, unsigned int flags) { + char buf[PRINTF_FTOA_BUFFER_SIZE]; + size_t len = 0U; + double diff = 0.0; + + // powers of 10 + static const double pow10[] = {1, 10, 100, 1000, 10000, + 100000, 1000000, 10000000, 100000000, 1000000000}; + + // test for special values + if (value != value) { + return _out_rev(out, buffer, idx, maxlen, "nan", 3, width, flags); + } + if (value < -DBL_MAX) { + return _out_rev(out, buffer, idx, maxlen, "fni-", 4, width, flags); + } + if (value > DBL_MAX) { + return _out_rev(out, buffer, idx, maxlen, (flags & FLAGS_PLUS) != 0u ? "fni+" : "fni", + (flags & FLAGS_PLUS) != 0u ? 4U : 3U, width, flags); + } + + // test for very large values + // standard printf behavior is to print EVERY whole number digit -- which could be 100s of + // characters overflowing your buffers == bad + if ((value > PRINTF_MAX_FLOAT) || (value < -PRINTF_MAX_FLOAT)) { + return _etoa(out, buffer, idx, maxlen, value, prec, width, flags); + } + + // test for negative + bool negative = false; + if (value < 0) { + negative = true; + value = 0 - value; + } + + // set default precision, if not set explicitly + if ((flags & FLAGS_PRECISION) == 0u) { + prec = PRINTF_DEFAULT_FLOAT_PRECISION; + } + // limit precision to 9, cause a prec >= 10 can lead to overflow errors + while ((len < PRINTF_FTOA_BUFFER_SIZE) && (prec > 9U)) { + buf[len++] = '0'; + prec--; + } + + int whole = static_cast(value); + double tmp = (value - whole) * pow10[prec]; + auto frac = static_cast(tmp); + diff = tmp - frac; + + if (diff > 0.5) { + ++frac; + // handle rollover, e.g. case 0.99 with prec 1 is 1.0 + if (frac >= pow10[prec]) { + frac = 0; + ++whole; + } + } else if (diff < 0.5) { + } else if ((frac == 0U) || ((frac & 1U) != 0u)) { + // if halfway, round up if odd OR if last digit is 0 + ++frac; + } + + if (prec == 0U) { + diff = value - static_cast(whole); + if ((!(diff < 0.5) || (diff > 0.5)) && ((static_cast(whole) & 1u) != 0)) { + // exactly 0.5 and ODD, then round up + // 1.5 -> 2, but 2.5 -> 2 + ++whole; + } + } else { + unsigned int count = prec; + // now do fractional part, as an unsigned number + while (len < PRINTF_FTOA_BUFFER_SIZE) { + --count; + buf[len++] = static_cast(48U + (frac % 10U)); + if ((frac /= 10U) == 0u) { + break; + } + } + // add extra 0s + while ((len < PRINTF_FTOA_BUFFER_SIZE) && (count-- > 0U)) { + buf[len++] = '0'; + } + if (len < PRINTF_FTOA_BUFFER_SIZE) { + // add decimal + buf[len++] = '.'; + } + } + + // do whole part, number is reversed + while (len < PRINTF_FTOA_BUFFER_SIZE) { + buf[len++] = static_cast(48 + (whole % 10)); + if ((whole /= 10) == 0) { + break; + } + } + + // pad leading zeros + if (((flags & FLAGS_LEFT) == 0u) && ((flags & FLAGS_ZEROPAD) != 0u)) { + if ((width != 0u) && (negative || ((flags & (FLAGS_PLUS | FLAGS_SPACE)) != 0u))) { + width--; + } + while ((len < width) && (len < PRINTF_FTOA_BUFFER_SIZE)) { + buf[len++] = '0'; + } + } + + if (len < PRINTF_FTOA_BUFFER_SIZE) { + if (negative) { + buf[len++] = '-'; + } else if ((flags & FLAGS_PLUS) != 0u) { + buf[len++] = '+'; // ignore the space if the '+' exists + } else if ((flags & FLAGS_SPACE) != 0u) { + buf[len++] = ' '; + } + } + + return _out_rev(out, buffer, idx, maxlen, buf, len, width, flags); +} + +// internal ftoa variant for exponential floating-point type, contributed by Martijn Jasperse +// +static size_t _etoa(out_fct_type out, std::vector* buffer, size_t idx, size_t maxlen, + double value, unsigned int prec, unsigned int width, unsigned int flags) { + // check for NaN and special values + if ((value != value) || (value > DBL_MAX) || (value < -DBL_MAX)) { + return _ftoa(out, buffer, idx, maxlen, value, prec, width, flags); + } + + // determine the sign + const bool negative = value < 0; + if (negative) { + value = -value; + } + + // default precision + if ((flags & FLAGS_PRECISION) == 0u) { + prec = PRINTF_DEFAULT_FLOAT_PRECISION; + } + + // determine the decimal exponent + // based on the algorithm by David Gay (https://www.ampl.com/netlib/fp/dtoa.c) + union { + uint64_t U; + double F; + } conv {}; + + conv.F = value; + int exp2 = static_cast((conv.U >> 52U) & 0x07FFU) - 1023; // effectively log2 + conv.U = (conv.U & ((1ULL << 52U) - 1U)) | + (1023ULL << 52U); // drop the exponent so conv.F is now in [1,2) + // now approximate log10 from the log2 integer part and an expansion of ln around 1.5 + int expval = static_cast(0.1760912590558 + exp2 * 0.301029995663981 + + (conv.F - 1.5) * 0.289529654602168); + // now we want to compute 10^expval but we want to be sure it won't overflow + // exp2 = static_cast(expval * 3.321928094887362 + 0.5); + exp2 = static_cast(lround(expval * 3.321928094887362)); + const double z = expval * 2.302585092994046 - exp2 * 0.6931471805599453; + const double z2 = z * z; + conv.U = static_cast(exp2 + 1023) << 52U; + // compute exp(z) using continued fractions, see + // https://en.wikipedia.org/wiki/Exponential_function#Continued_fractions_for_ex + conv.F *= 1 + 2 * z / (2 - z + (z2 / (6 + (z2 / (10 + z2 / 14))))); + // correct for rounding errors + if (value < conv.F) { + expval--; + conv.F /= 10; + } + + // the exponent format is "%+03d" and largest value is "307", so set aside 4-5 characters + unsigned int minwidth = ((expval < 100) && (expval > -100)) ? 4U : 5U; + + // in "%g" mode, "prec" is the number of *significant figures* not decimals + if ((flags & FLAGS_ADAPT_EXP) != 0u) { + // do we want to fall-back to "%f" mode? + if ((value >= 1e-4) && (value < 1e6)) { + if (static_cast(prec) > expval) { + prec = static_cast(static_cast(prec) - expval - 1); + } else { + prec = 0; + } + flags |= FLAGS_PRECISION; // make sure _ftoa respects precision + // no characters in exponent + minwidth = 0U; + expval = 0; + } else { + // we use one sigfig for the whole part + if ((prec > 0) && ((flags & FLAGS_PRECISION) != 0u)) { + --prec; + } + } + } + + // will everything fit? + unsigned int fwidth = width; + if (width > minwidth) { + // we didn't fall-back so subtract the characters required for the exponent + fwidth -= minwidth; + } else { + // not enough characters, so go back to default sizing + fwidth = 0U; + } + if (((flags & FLAGS_LEFT) != 0u) && (minwidth != 0u)) { + // if we're padding on the right, DON'T pad the floating part + fwidth = 0U; + } + + // rescale the float value + if (expval != 0) { + value /= conv.F; + } + + // output the floating part + const size_t start_idx = idx; + idx = _ftoa(out, buffer, idx, maxlen, negative ? -value : value, prec, fwidth, + flags & ~FLAGS_ADAPT_EXP); + + // output the exponent part + if (minwidth != 0u) { + // output the exponential symbol + out((flags & FLAGS_UPPERCASE) != 0u ? 'E' : 'e', buffer, idx++, maxlen); + // output the exponent value + idx = _ntoa_long(out, buffer, idx, maxlen, (expval < 0) ? -expval : expval, expval < 0, 10, + 0, minwidth - 1, FLAGS_ZEROPAD | FLAGS_PLUS); + // might need to right-pad spaces + if ((flags & FLAGS_LEFT) != 0u) { + while (idx - start_idx < width) { + out(' ', buffer, idx++, maxlen); + } + } + } + return idx; +} + +static inline unsigned int _strnlen_s(const char* str, size_t maxsize) { + const char* s = nullptr; + for (s = str; (*s != 0) && ((maxsize--) != 0u); ++s) { + ; + } + return static_cast(s - str); +} + +// NOLINTNEXTLINE(readability-function-cognitive-complexity) +static int kyty_printf_internal(bool sn, char* sn_s, size_t sn_n, const char* format, + VaList* va_list) { + std::vector buffer; + + uint32_t flags = 0; + uint32_t width = 0; + uint32_t precision = 0; + uint32_t n = 0; + size_t idx = 0U; + auto maxlen = static_cast(-1); + + // use null output function + auto out = _out_null; + + while (*format != 0) { + // format specifier? %[flags][width][.precision][length] + if (*format != '%') { + // no + out(*format, &buffer, idx++, maxlen); + format++; + continue; + } + + // yes, evaluate it + format++; + + // evaluate flags + flags = 0U; + do { + switch (*format) { + case '0': + flags |= FLAGS_ZEROPAD; + format++; + n = 1U; + break; + case '-': + flags |= FLAGS_LEFT; + format++; + n = 1U; + break; + case '+': + flags |= FLAGS_PLUS; + format++; + n = 1U; + break; + case ' ': + flags |= FLAGS_SPACE; + format++; + n = 1U; + break; + case '#': + flags |= FLAGS_HASH; + format++; + n = 1U; + break; + default: n = 0U; break; + } + } while (n != 0u); + + // evaluate width field + width = 0U; + if (_is_digit(*format)) { + width = _atoi(&format); + } else if (*format == '*') { + // const int w = va_arg(va, int); + const int w = VaArg_int(va_list); + if (w < 0) { + flags |= FLAGS_LEFT; // reverse padding + width = static_cast(-w); + } else { + width = static_cast(w); + } + format++; + } + + // evaluate precision field + precision = 0U; + if (*format == '.') { + flags |= FLAGS_PRECISION; + format++; + if (_is_digit(*format)) { + precision = _atoi(&format); + } else if (*format == '*') { + // const int prec = (int)va_arg(va, int); + const int prec = VaArg_int(va_list); + precision = prec > 0 ? static_cast(prec) : 0U; + format++; + } + } + + // evaluate length field + switch (*format) { + case 'l': + flags |= FLAGS_LONG; + format++; + if (*format == 'l') { + flags |= FLAGS_LONG_LONG; + format++; + } + break; + case 'h': + flags |= FLAGS_SHORT; + format++; + if (*format == 'h') { + flags |= FLAGS_CHAR; + format++; + } + break; + case 't': + flags |= (sizeof(ptrdiff_t) == sizeof(int32_t) ? FLAGS_LONG : FLAGS_LONG_LONG); + format++; + break; + case 'j': + flags |= (sizeof(intmax_t) == sizeof(int32_t) ? FLAGS_LONG : FLAGS_LONG_LONG); + format++; + break; + case 'z': + flags |= (sizeof(size_t) == sizeof(int32_t) ? FLAGS_LONG : FLAGS_LONG_LONG); + format++; + break; + default: break; + } + + // evaluate specifier + switch (*format) { + case 'd': + case 'i': + case 'u': + case 'x': + case 'X': + case 'o': + case 'b': { + // set the base + unsigned int base = 0; + if (*format == 'x' || *format == 'X') { + base = 16U; + } else if (*format == 'o') { + base = 8U; + } else if (*format == 'b') { + base = 2U; + } else { + base = 10U; + flags &= ~FLAGS_HASH; // no hash for dec format + } + // uppercase + if (*format == 'X') { + flags |= FLAGS_UPPERCASE; + } + + // no plus or space flag for u, x, X, o, b + if ((*format != 'i') && (*format != 'd')) { + flags &= ~(FLAGS_PLUS | FLAGS_SPACE); + } + + // ignore '0' flag when precision is given + if ((flags & FLAGS_PRECISION) != 0u) { + flags &= ~FLAGS_ZEROPAD; + } + + // convert the integer + if ((*format == 'i') || (*format == 'd')) { + // signed + if ((flags & FLAGS_LONG_LONG) != 0u || (flags & FLAGS_LONG) != 0u) { + // const long long value = va_arg(va, long long); + auto value = VaArg_long_long(va_list); + idx = _ntoa_long_long(out, &buffer, idx, maxlen, + static_cast(value > 0 ? value : 0 - value), + value < 0, base, precision, width, flags); + } else if ((flags & FLAGS_LONG) != 0u) { + // const long value = va_arg(va, long); + auto value = VaArg_long(va_list); + idx = _ntoa_long(out, &buffer, idx, maxlen, + static_cast(value > 0 ? value : 0 - value), + value < 0, base, precision, width, flags); + } else { + // const int value = (flags & FLAGS_CHAR) ? (char)va_arg(va, int) + // : (flags & FLAGS_SHORT) ? (short int)va_arg(va, int) + // : va_arg(va, int); + int value = + (flags & FLAGS_CHAR) != 0u ? static_cast(VaArg_int(va_list)) + : (flags & FLAGS_SHORT) != 0u ? static_cast(VaArg_int(va_list)) + : VaArg_int(va_list); + idx = _ntoa_long(out, &buffer, idx, maxlen, + static_cast(value > 0 ? value : 0 - value), + value < 0, base, precision, width, flags); + } + } else { + // unsigned + if ((flags & FLAGS_LONG_LONG) != 0u || (flags & FLAGS_LONG) != 0u) { + idx = _ntoa_long_long(out, &buffer, idx, maxlen, + static_cast(VaArg_long_long(va_list)), + false, base, precision, width, flags); + } else if ((flags & FLAGS_LONG) != 0u) { + idx = _ntoa_long(out, &buffer, idx, maxlen, + static_cast(VaArg_long(va_list)), false, base, + precision, width, flags); + } else { + const unsigned int value = + (flags & FLAGS_CHAR) != 0u + ? static_cast(VaArg_int(va_list)) + : (flags & FLAGS_SHORT) != 0u + ? static_cast(VaArg_int(va_list)) + : static_cast(VaArg_int(va_list)); + idx = _ntoa_long(out, &buffer, idx, maxlen, value, false, base, precision, + width, flags); + } + } + format++; + break; + } + case 'f': + case 'F': + if (*format == 'F') { + flags |= FLAGS_UPPERCASE; + } + idx = _ftoa(out, &buffer, idx, maxlen, VaArg_double(va_list), precision, width, + flags); + format++; + break; + case 'e': + case 'E': + case 'g': + case 'G': + if ((*format == 'g') || (*format == 'G')) { + flags |= FLAGS_ADAPT_EXP; + } + if ((*format == 'E') || (*format == 'G')) { + flags |= FLAGS_UPPERCASE; + } + idx = _etoa(out, &buffer, idx, maxlen, VaArg_double(va_list), precision, width, + flags); + format++; + break; + case 'c': { + unsigned int l = 1U; + // pre padding + if ((flags & FLAGS_LEFT) == 0u) { + while (l++ < width) { + out(' ', &buffer, idx++, maxlen); + } + } + // char output + out(static_cast(VaArg_int(va_list)), &buffer, idx++, maxlen); + // post padding + if ((flags & FLAGS_LEFT) != 0u) { + while (l++ < width) { + out(' ', &buffer, idx++, maxlen); + } + } + format++; + break; + } + + case 's': { + // const char* p = va_arg(va, char*); + const char* p = VaArg_ptr(va_list); + unsigned int l = + _strnlen_s(p, precision != 0u ? precision : static_cast(-1)); + // pre padding + if ((flags & FLAGS_PRECISION) != 0u) { + l = (l < precision ? l : precision); + } + if ((flags & FLAGS_LEFT) == 0u) { + while (l++ < width) { + out(' ', &buffer, idx++, maxlen); + } + } + // string output + while ((*p != 0) && (((flags & FLAGS_PRECISION) == 0u) || ((precision--) != 0u))) { + out(*(p++), &buffer, idx++, maxlen); + } + // post padding + if ((flags & FLAGS_LEFT) != 0u) { + while (l++ < width) { + out(' ', &buffer, idx++, maxlen); + } + } + format++; + break; + } + + case 'p': { + width = sizeof(void*) * 2U; + flags |= FLAGS_ZEROPAD | FLAGS_UPPERCASE; + const bool is_ll = sizeof(uintptr_t) == sizeof(int64_t); + if (is_ll) { + idx = _ntoa_long_long(out, &buffer, idx, maxlen, + reinterpret_cast(VaArg_ptr(va_list)), + false, 16U, precision, width, flags); + } else { + idx = _ntoa_long(out, &buffer, idx, maxlen, + static_cast( + reinterpret_cast(VaArg_ptr(va_list))), + false, 16U, precision, width, flags); + } + format++; + break; + } + + case '%': + out('%', &buffer, idx++, maxlen); + format++; + break; + + default: + out(*format, &buffer, idx++, maxlen); + format++; + break; + } + } + + // termination + out(static_cast(0), &buffer, idx < maxlen ? idx : maxlen - 1U, maxlen); + + if (sn) { + int s = snprintf(sn_s, sn_n, "%s", buffer.data()); + EXIT_NOT_IMPLEMENTED(static_cast(s) >= sn_n); + } else { + LOGF_COLOR(Log::Color::BrightMagenta, "%s", buffer.data()); + } + + // return written chars without terminating \0 + return static_cast(idx); +} + +static int kyty_vprintf(const char* format, VaList* va_list) { + return kyty_printf_internal(false, nullptr, 0, format, va_list); +} + +static int kyty_printf_ctx(VaContext* ctx) { + const char* format = VaArg_ptr(&ctx->va_list); + + return kyty_printf_internal(false, nullptr, 0, format, &ctx->va_list); +} + +static int kyty_snprintf_ctx(VaContext* ctx) { + char* s = VaArg_ptr(&ctx->va_list); + size_t n = VaArg_size_t(&ctx->va_list); + const char* format = VaArg_ptr(&ctx->va_list); + + return kyty_printf_internal(true, s, n, format, &ctx->va_list); +} + +static int KYTY_SYSV_ABI kyty_printf_std(VA_ARGS) { + VA_CONTEXT(ctx); // NOLINT(cppcoreguidelines-pro-type-member-init,hicpp-member-init) + + return kyty_printf_ctx(&ctx); +} + +guest_printf_std_func_t GetGuestPrintfStdFunc() { + return reinterpret_cast(kyty_printf_std); +} + +guest_printf_ctx_func_t GetGuestPrintfCtxFunc() { + return kyty_printf_ctx; +} + +guest_snprintf_ctx_func_t GetGuestSnprintfCtxFunc() { + return kyty_snprintf_ctx; +} + +guest_vprintf_func_t GetGuestVprintfFunc() { + return kyty_vprintf; +} + +} // namespace Libs diff --git a/src/libs/guestPrintf.h b/src/libs/guestPrintf.h new file mode 100644 index 000000000..0e71438e6 --- /dev/null +++ b/src/libs/guestPrintf.h @@ -0,0 +1,25 @@ +#ifndef EMULATOR_INCLUDE_EMULATOR_LIBS_GUEST_PRINTF_H_ +#define EMULATOR_INCLUDE_EMULATOR_LIBS_GUEST_PRINTF_H_ + +#include "common/abi.h" +#include "common/common.h" + +namespace Libs { + +struct VaContext; +struct VaList; + +using guest_printf_std_func_t = KYTY_FORMAT_PRINTF(1, 2) KYTY_SYSV_ABI + int (*)(const char* str, ...); +using guest_printf_ctx_func_t = int (*)(VaContext* c); +using guest_snprintf_ctx_func_t = int (*)(VaContext* c); +using guest_vprintf_func_t = int (*)(const char* str, VaList* c); + +guest_printf_std_func_t GetGuestPrintfStdFunc(); +guest_printf_ctx_func_t GetGuestPrintfCtxFunc(); +guest_snprintf_ctx_func_t GetGuestSnprintfCtxFunc(); +guest_vprintf_func_t GetGuestVprintfFunc(); + +} // namespace Libs + +#endif /* EMULATOR_INCLUDE_EMULATOR_LIBS_GUEST_PRINTF_H_ */ diff --git a/src/libs/libAmpr.cpp b/src/libs/libAmpr.cpp new file mode 100644 index 000000000..74696de02 --- /dev/null +++ b/src/libs/libAmpr.cpp @@ -0,0 +1,2754 @@ +#include "common/abi.h" +#include "common/dateTime.h" +#include "common/file.h" +#include "common/logging/log.h" +#include "common/stringUtils.h" +#include "kernel/eventQueue.h" +#include "kernel/fileSystem.h" +#include "kernel/memory.h" +#include "libs/errno.h" +#include "libs/libs.h" +#include "loader/symbolDatabase.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#ifdef min +#undef min +#endif +#ifdef max +#undef max +#endif + +namespace Libs { + +namespace AprShared { + +struct ResultBuffer { + int32_t result = 0; + uint32_t errorOffset = 0; +}; + +struct SubmissionState { + uint64_t command_buffer = 0; + uint64_t result = 0; + int32_t execution_result = 0; + uint32_t error_offset = 0; +}; + +struct ResolvedPathInfo { + int result = OK; + uint32_t file_id = 0xffffffffu; + uint64_t file_size = 0; + bool is_dir = false; + std::string host_path; +}; + +static std::mutex g_mutex; +static uint32_t g_next_submission_id = 1; +static std::unordered_map g_submissions; +static std::unordered_map g_files; +static std::unordered_map g_file_sizes; +static std::unordered_map g_resolved_paths; + +static bool IsValidGuestRange(uint64_t addr, uint64_t size, bool write = false) { + (void)write; + if (size == 0) { + return true; + } + return addr != 0 && addr <= std::numeric_limits::max() - size; +} + +static bool ReadGuestBytes(uint64_t addr, void* out, uint64_t size) { + if (out == nullptr || !IsValidGuestRange(addr, size)) { + return false; + } + std::memcpy(out, reinterpret_cast(addr), static_cast(size)); + return true; +} + +static bool WriteGuestBytes(uint64_t addr, const void* in, uint64_t size) { + if (in == nullptr || !IsValidGuestRange(addr, size, true)) { + return false; + } + std::memcpy(reinterpret_cast(addr), in, static_cast(size)); + return true; +} + +template +static bool ReadGuest(uint64_t addr, T* value) { + return ReadGuestBytes(addr, value, sizeof(T)); +} + +template +static bool WriteGuest(uint64_t addr, const T& value) { + return WriteGuestBytes(addr, &value, sizeof(T)); +} + +static uint32_t ComputeFileId(const char* guest_path) { + uint32_t hash = 2166136261u; + for (auto* p = reinterpret_cast(guest_path); p != nullptr && *p != 0; ++p) { + hash ^= *p; + hash *= 16777619u; + } + return hash & static_cast(std::numeric_limits::max()); +} + +static bool CopyStringToOutput(const std::string& str, char* out, size_t out_size) { + if (str.empty() || str.size() + 1 > out_size) { + return false; + } + + std::memcpy(out, str.c_str(), str.size() + 1); + return true; +} + +static bool ReadGuestCString(uint64_t addr, char* out, size_t out_size) { + if (out == nullptr || out_size == 0 || addr == 0) { + return false; + } + + const auto max_len = static_cast(out_size - 1); + const auto* src = reinterpret_cast(static_cast(addr)); + for (uint64_t pos = 0; pos < max_len; pos++) { + if (addr > std::numeric_limits::max() - pos) { + out[pos] = '\0'; + return false; + } + + const auto ch = src[pos]; + out[pos] = ch; + if (ch == '\0') { + return pos != 0; + } + } + + out[max_len] = '\0'; + return false; +} + +static bool ReadGuestWideCString(uint64_t addr, char* out, size_t out_size) { + if (out == nullptr || out_size == 0 || addr == 0) { + return false; + } + + std::array tmp {}; + const auto max_len = std::min(tmp.size() - 1, out_size - 1); + const auto* src = reinterpret_cast(static_cast(addr)); + for (uint64_t pos = 0; pos < max_len; pos++) { + if (pos > (std::numeric_limits::max() - addr) / sizeof(char16_t)) { + return false; + } + + char16_t ch = 0; + std::memcpy(&ch, src + pos * sizeof(char16_t), sizeof(ch)); + tmp[static_cast(pos)] = ch; + if (ch == u'\0') { + return pos != 0 && CopyStringToOutput(Common::Utf16ToUtf8(tmp.data()), out, out_size); + } + } + + return false; +} + +static bool ReadGuestPathText(uint64_t addr, char* out, size_t out_size) { + std::array narrow {}; + if (ReadGuestCString(addr, narrow.data(), std::min(narrow.size(), out_size))) { + const auto narrow_len = std::strlen(narrow.data()); + if (narrow_len > 1 || !ReadGuestWideCString(addr, out, out_size)) { + std::memcpy(out, narrow.data(), std::strlen(narrow.data()) + 1); + } + return true; + } + + if (ReadGuestWideCString(addr, out, out_size)) { + return true; + } + return false; +} + +static bool ReadPathPointer(uint64_t pointer_addr, char* out, size_t out_size) { + uint64_t path_addr = 0; + if (!ReadGuest(pointer_addr, &path_addr)) { + return false; + } + if (path_addr == 0) { + return false; + } + if (ReadGuestPathText(path_addr, out, out_size)) { + return true; + } + return false; +} + +static bool ResolvePathFromList(uint64_t path_list, uint64_t index, char* out, size_t out_size) { + if (ReadPathPointer(path_list + index * sizeof(uint64_t), out, out_size)) { + return true; + } + if (index != 0) { + return false; + } + if (ReadGuestPathText(path_list, out, out_size)) { + return true; + } + for (uint64_t offset = 0; offset < 0x40; offset += sizeof(uint64_t)) { + if (ReadPathPointer(path_list + offset, out, out_size)) { + return true; + } + } + return false; +} + +static void RegisterHostPathLocked(uint32_t file_id, const std::string& host_path, + uint64_t file_size, bool is_dir) { + g_files[file_id] = host_path; + g_file_sizes[file_id] = is_dir ? 0 : file_size; +} + +static void RegisterHostPath(uint32_t file_id, const std::string& host_path, uint64_t file_size, + bool is_dir) { + std::scoped_lock lock(g_mutex); + RegisterHostPathLocked(file_id, host_path, file_size, is_dir); +} + +static bool TryGetHostPath(uint32_t file_id, std::string* out) { + std::scoped_lock lock(g_mutex); + const auto it = g_files.find(file_id); + if (it == g_files.end()) { + return false; + } + *out = it->second; + return true; +} + +static bool TryGetHostFileSize(uint32_t file_id, uint64_t* out) { + std::scoped_lock lock(g_mutex); + const auto it = g_file_sizes.find(file_id); + if (it == g_file_sizes.end()) { + return false; + } + *out = it->second; + return true; +} + +static int GetHostPathStat(const std::string& host_path, LibKernel::FileSystem::FileStat* st) { + if (st == nullptr) { + return LibKernel::KERNEL_ERROR_EINVAL; + } + + const bool is_dir = Common::File::IsDirectoryExisting(host_path); + const bool is_file = Common::File::IsFileExisting(host_path); + if (!is_dir && !is_file) { + return LibKernel::KERNEL_ERROR_ENOENT; + } + + LibKernel::FileSystem::FileStat stat {}; + stat.st_mode = 0000777u | (is_dir ? 0040000u : 0100000u); + + auto at = Common::DateTime::FromSystemUTC(); + auto wt = at; + + if (is_dir) { + stat.st_size = 0; + stat.st_blksize = 512; + stat.st_blocks = 0; + } else { + stat.st_size = static_cast(Common::File::Size(host_path)); + stat.st_blksize = 512; + stat.st_blocks = (stat.st_size + 511) / 512; + + Common::File::GetLastAccessAndWriteTimeUTC(host_path, &at, &wt); + } + + stat.st_atim.tv_sec = static_cast(at.ToUnix()); + stat.st_atim.tv_nsec = static_cast( + (at.ToUnix() - static_cast(stat.st_atim.tv_sec)) * 1000000000.0); + stat.st_mtim.tv_sec = static_cast(wt.ToUnix()); + stat.st_mtim.tv_nsec = static_cast( + (wt.ToUnix() - static_cast(stat.st_mtim.tv_sec)) * 1000000000.0); + stat.st_ctim = stat.st_atim; + stat.st_birthtim = stat.st_mtim; + *st = stat; + + return OK; +} + +static bool JoinPrefixPath(const char* prefix, const char* path, char* out, size_t out_size) { + if (path == nullptr || out == nullptr || out_size == 0) { + return false; + } + + if (prefix == nullptr || prefix[0] == '\0') { + std::strncpy(out, path, out_size - 1); + out[out_size - 1] = '\0'; + return true; + } + + const auto prefix_len = std::strlen(prefix); + const auto path_len = std::strlen(path); + const bool needs_sep = prefix_len != 0 && path_len != 0 && prefix[prefix_len - 1] != '/' && + prefix[prefix_len - 1] != '\\' && path[0] != '/' && path[0] != '\\'; + const auto total_len = prefix_len + (needs_sep ? 1u : 0u) + path_len; + if (total_len + 1u > out_size) { + return false; + } + + std::memcpy(out, prefix, prefix_len); + auto pos = prefix_len; + if (needs_sep) { + out[pos++] = '/'; + } + std::memcpy(out + pos, path, path_len); + out[total_len] = '\0'; + return true; +} + +static int ResolveOnePath(const char* guest_path, uint32_t* id, uint64_t* size) { + if (guest_path == nullptr || guest_path[0] == '\0') { + return LibKernel::KERNEL_ERROR_EINVAL; + } + + ResolvedPathInfo info {}; + const std::string path(guest_path); + bool found = false; + { + std::scoped_lock lock(g_mutex); + const auto it = g_resolved_paths.find(path); + if (it != g_resolved_paths.end()) { + info = it->second; + found = true; + } + } + + if (!found) { + const auto real_path = LibKernel::FileSystem::GetRealFilename(path); + info.file_id = AprShared::ComputeFileId(guest_path); + info.host_path = Common::PathToString(real_path); + + if (Common::File::IsDirectoryExisting(real_path)) { + info.is_dir = true; + info.file_size = 0x10000; + } else if (Common::File::IsFileExisting(real_path)) { + info.file_size = Common::File::Size(real_path); + } else { + info.result = LibKernel::KERNEL_ERROR_ENOENT; + } + + bool log_missing = false; + { + std::scoped_lock lock(g_mutex); + auto [it, inserted] = g_resolved_paths.emplace(path, info); + if (!inserted) { + info = it->second; + } else if (info.result == OK) { + RegisterHostPathLocked(info.file_id, info.host_path, info.file_size, info.is_dir); + } else if (info.result == LibKernel::KERNEL_ERROR_ENOENT) { + log_missing = true; + } + } + if (log_missing) { + LOGF("\tAPR resolve missing path: %s -> %s\n", guest_path, info.host_path.c_str()); + } + } else if (info.result == OK) { + AprShared::RegisterHostPath(info.file_id, info.host_path, info.file_size, info.is_dir); + } + + if (info.result != OK) { + return info.result; + } + + if (id != nullptr) { + if (!AprShared::IsValidGuestRange(reinterpret_cast(id), sizeof(*id), true)) { + return LibKernel::KERNEL_ERROR_EFAULT; + } + *id = info.file_id; + } + + if (size != nullptr) { + if (!AprShared::IsValidGuestRange(reinterpret_cast(size), sizeof(*size), true)) { + return LibKernel::KERNEL_ERROR_EFAULT; + } + *size = info.file_size; + } + + return OK; +} + +static int ResolvePathsCommon(const void* path_list, uint64_t count, uint32_t* ids, uint64_t* sizes, + uint32_t* error_index, const char* prefix, int* results) { + if (path_list == nullptr || count == 0 || count > 1024 || + (ids == nullptr && sizes == nullptr && results == nullptr)) { + return LibKernel::KERNEL_ERROR_EINVAL; + } + + if (error_index != nullptr && + !AprShared::IsValidGuestRange(reinterpret_cast(error_index), sizeof(*error_index), + true)) { + return LibKernel::KERNEL_ERROR_EFAULT; + } + if (ids != nullptr && !AprShared::IsValidGuestRange(reinterpret_cast(ids), + count * sizeof(*ids), true)) { + return LibKernel::KERNEL_ERROR_EFAULT; + } + if (sizes != nullptr && !AprShared::IsValidGuestRange(reinterpret_cast(sizes), + count * sizeof(*sizes), true)) { + return LibKernel::KERNEL_ERROR_EFAULT; + } + if (results != nullptr && !AprShared::IsValidGuestRange(reinterpret_cast(results), + count * sizeof(*results), true)) { + return LibKernel::KERNEL_ERROR_EFAULT; + } + + const auto path_list_addr = reinterpret_cast(path_list); + bool any_error = false; + int first_error = OK; + uint32_t success_count = 0; + for (uint64_t i = 0; i < count; i++) { + char guest_path[1024] {}; + char resolved_path[2048] {}; + int result = OK; + if (!AprShared::ResolvePathFromList(path_list_addr, i, guest_path, sizeof(guest_path)) || + !JoinPrefixPath(prefix, guest_path, resolved_path, sizeof(resolved_path))) { + result = LibKernel::KERNEL_ERROR_EFAULT; + } else { + result = ResolveOnePath(resolved_path, ids != nullptr ? &ids[i] : nullptr, + sizes != nullptr ? &sizes[i] : nullptr); + } + + if (results != nullptr) { + results[i] = result; + } + if (result == OK) { + success_count++; + } else { + if (ids != nullptr) { + ids[i] = 0xffffffffu; + } + if (sizes != nullptr) { + sizes[i] = 0; + } + if (error_index != nullptr) { + *error_index = static_cast(i); + } + if (!any_error) { + any_error = true; + first_error = result; + } + if (results == nullptr) { + return result; + } + } + } + + return results != nullptr ? static_cast(success_count) : (any_error ? first_error : OK); +} + +static uint32_t AllocateSubmissionId(uint64_t command_buffer, uint64_t result) { + std::scoped_lock lock(g_mutex); + + auto id = g_next_submission_id++; + if (id == 0) { + id = g_next_submission_id++; + } + g_submissions[id] = SubmissionState {command_buffer, result}; + return id; +} + +static void SetSubmissionResult(uint32_t submission_id, int32_t execution_result, + uint32_t error_offset) { + std::scoped_lock lock(g_mutex); + auto it = g_submissions.find(submission_id); + if (it != g_submissions.end()) { + it->second.execution_result = execution_result; + it->second.error_offset = error_offset; + } +} + +static bool CompleteSubmission(uint32_t submission_id, SubmissionState* state) { + std::scoped_lock lock(g_mutex); + auto it = g_submissions.find(submission_id); + if (it == g_submissions.end()) { + return false; + } + if (state != nullptr) { + *state = it->second; + } + g_submissions.erase(it); + return true; +} + +static int WriteResult(void* result, int32_t execution_result = 0, uint32_t error_offset = 0) { + if (result == nullptr) { + return OK; + } + ResultBuffer res {}; + res.result = execution_result; + res.errorOffset = error_offset; + if (!WriteGuest(reinterpret_cast(result), res)) { + return LibKernel::KERNEL_ERROR_EFAULT; + } + return OK; +} + +} // namespace AprShared + +namespace LibAmpr::Ampr { +static int ExecuteAprCommandBuffer(uint64_t command_buffer, int32_t* execution_result, + uint32_t* error_offset); +} + +namespace LibKernelApr { + +LIB_VERSION("libkernel", 1, "libkernel", 1, 1); + +namespace Apr { + +// TODO: change helper name +static int KernelSyscallResult(int result) { + if (result >= OK) { + return result; + } + + // TODO: actually __error() thread local + *Posix::GetErrorAddr() = LibKernel::KernelToPosix(result); + return -1; +} + +static int KYTY_SYSV_ABI ResolveFilepathsToIds(const void* path_list, uint32_t count, uint32_t* ids, + uint32_t* error_index) { + PRINT_NAME(); + + return KernelSyscallResult(AprShared::ResolvePathsCommon(path_list, count, ids, nullptr, + error_index, nullptr, nullptr)); +} + +static int KYTY_SYSV_ABI GetFileStat(uint32_t file_id, LibKernel::FileSystem::FileStat* st) { + PRINT_NAME(); + + if (st == nullptr) { + return KernelSyscallResult(LibKernel::KERNEL_ERROR_EINVAL); + } + + std::string host_path; + if (!AprShared::TryGetHostPath(file_id, &host_path)) { + LOGF("\tAPR stat failed for unknown file id: 0x%08" PRIx32 "\n", file_id); + return KernelSyscallResult(LibKernel::KERNEL_ERROR_ENOENT); + } + if (!AprShared::IsValidGuestRange(reinterpret_cast(st), sizeof(*st), true)) { + return KernelSyscallResult(LibKernel::KERNEL_ERROR_EFAULT); + } + + return KernelSyscallResult(AprShared::GetHostPathStat(host_path, st)); +} + +static int KYTY_SYSV_ABI GetFileSize(uint32_t file_id, uint64_t* size) { + PRINT_NAME(); + + if (size == nullptr) { + return KernelSyscallResult(LibKernel::KERNEL_ERROR_EINVAL); + } + if (!AprShared::IsValidGuestRange(reinterpret_cast(size), sizeof(*size), true)) { + return KernelSyscallResult(LibKernel::KERNEL_ERROR_EFAULT); + } + + std::string host_path; + if (!AprShared::TryGetHostPath(file_id, &host_path)) { + LOGF("\tAPR size failed for unknown file id: 0x%08" PRIx32 "\n", file_id); + return KernelSyscallResult(LibKernel::KERNEL_ERROR_ENOENT); + } + + uint64_t cached_size = 0; + if (AprShared::TryGetHostFileSize(file_id, &cached_size)) { + *size = cached_size; + return OK; + } + + if (Common::File::IsDirectoryExisting(host_path)) { + *size = 0; + return OK; + } + if (!Common::File::IsFileExisting(host_path)) { + return KernelSyscallResult(LibKernel::KERNEL_ERROR_ENOENT); + } + + *size = Common::File::Size(host_path); + return OK; +} + +static int KYTY_SYSV_ABI ResolveFilepathsToIdsAndFileSizes(const void* path_list, uint32_t count, + uint32_t* ids, uint64_t* sizes, + uint32_t* error_index) { + PRINT_NAME(); + + return KernelSyscallResult( + AprShared::ResolvePathsCommon(path_list, count, ids, sizes, error_index, nullptr, nullptr)); +} + +static int KYTY_SYSV_ABI ResolveFilepathsWithPrefixToIds(const char* prefix, const void* path_list, + uint32_t count, uint32_t* ids, + uint32_t* error_index) { + PRINT_NAME(); + + char prefix_buf[1024] {}; + if (prefix != nullptr && !AprShared::ReadGuestPathText(reinterpret_cast(prefix), + prefix_buf, sizeof(prefix_buf))) { + return KernelSyscallResult(LibKernel::KERNEL_ERROR_EFAULT); + } + return KernelSyscallResult(AprShared::ResolvePathsCommon(path_list, count, ids, nullptr, + error_index, prefix_buf, nullptr)); +} + +static int KYTY_SYSV_ABI ResolveFilepathsWithPrefixToIdsAndFileSizes(const char* prefix, + const void* path_list, + uint32_t count, uint32_t* ids, + uint64_t* sizes, + uint32_t* error_index) { + PRINT_NAME(); + + char prefix_buf[1024] {}; + if (prefix != nullptr && !AprShared::ReadGuestPathText(reinterpret_cast(prefix), + prefix_buf, sizeof(prefix_buf))) { + return KernelSyscallResult(LibKernel::KERNEL_ERROR_EFAULT); + } + return KernelSyscallResult(AprShared::ResolvePathsCommon(path_list, count, ids, sizes, + error_index, prefix_buf, nullptr)); +} + +static int KYTY_SYSV_ABI ResolveFilepathsToIdsForEach(const void* path_list, uint32_t count, + uint32_t* ids, int* results) { + PRINT_NAME(); + + return KernelSyscallResult( + AprShared::ResolvePathsCommon(path_list, count, ids, nullptr, nullptr, nullptr, results)); +} + +static int KYTY_SYSV_ABI ResolveFilepathsToIdsAndFileSizesForEach(const void* path_list, + uint32_t count, uint32_t* ids, + uint64_t* sizes, int* results) { + PRINT_NAME(); + + return KernelSyscallResult( + AprShared::ResolvePathsCommon(path_list, count, ids, sizes, nullptr, nullptr, results)); +} + +static int KYTY_SYSV_ABI ResolveFilepathsWithPrefixToIdsForEach(const char* prefix, + const void* path_list, + uint32_t count, uint32_t* ids, + int* results) { + PRINT_NAME(); + + char prefix_buf[1024] {}; + if (prefix != nullptr && !AprShared::ReadGuestPathText(reinterpret_cast(prefix), + prefix_buf, sizeof(prefix_buf))) { + return KernelSyscallResult(LibKernel::KERNEL_ERROR_EFAULT); + } + return KernelSyscallResult(AprShared::ResolvePathsCommon(path_list, count, ids, nullptr, + nullptr, prefix_buf, results)); +} + +static int KYTY_SYSV_ABI ResolveFilepathsWithPrefixToIdsAndFileSizesForEach( + const char* prefix, const void* path_list, uint32_t count, uint32_t* ids, uint64_t* sizes, + int* results) { + PRINT_NAME(); + + char prefix_buf[1024] {}; + if (prefix != nullptr && !AprShared::ReadGuestPathText(reinterpret_cast(prefix), + prefix_buf, sizeof(prefix_buf))) { + return KernelSyscallResult(LibKernel::KERNEL_ERROR_EFAULT); + } + return KernelSyscallResult( + AprShared::ResolvePathsCommon(path_list, count, ids, sizes, nullptr, prefix_buf, results)); +} + +static int KYTY_SYSV_ABI SubmitCommandBufferAndGetResult(void* command_buffer, uint64_t, + void* result, + uint32_t* out_submission_id) { + PRINT_NAME(); + + if (command_buffer == nullptr) { + return KernelSyscallResult(LibKernel::KERNEL_ERROR_EINVAL); + } + + const auto command_buffer_addr = reinterpret_cast(command_buffer); + const auto id = + AprShared::AllocateSubmissionId(command_buffer_addr, reinterpret_cast(result)); + + int32_t execution_result = OK; + uint32_t error_offset = 0; + auto submit_result = LibAmpr::Ampr::ExecuteAprCommandBuffer(command_buffer_addr, + &execution_result, &error_offset); + if (submit_result != OK) { + return KernelSyscallResult(submit_result); + } + AprShared::SetSubmissionResult(id, execution_result, error_offset); + + if (out_submission_id != nullptr) { + if (!AprShared::WriteGuest(reinterpret_cast(out_submission_id), id)) { + return KernelSyscallResult(LibKernel::KERNEL_ERROR_EFAULT); + } + } + + return KernelSyscallResult(AprShared::WriteResult(result, execution_result, error_offset)); +} + +static int KYTY_SYSV_ABI SubmitCommandBuffer(void* command_buffer, uint64_t) { + PRINT_NAME(); + + if (command_buffer == nullptr) { + return KernelSyscallResult(LibKernel::KERNEL_ERROR_EINVAL); + } + + int32_t execution_result = OK; + uint32_t error_offset = 0; + return KernelSyscallResult(LibAmpr::Ampr::ExecuteAprCommandBuffer( + reinterpret_cast(command_buffer), &execution_result, &error_offset)); +} + +static int KYTY_SYSV_ABI SubmitCommandBufferAndGetId(void* command_buffer, uint64_t, + uint32_t* out_submission_id) { + PRINT_NAME(); + + if (command_buffer == nullptr || out_submission_id == nullptr) { + return KernelSyscallResult(LibKernel::KERNEL_ERROR_EINVAL); + } + + const auto command_buffer_addr = reinterpret_cast(command_buffer); + const auto id = AprShared::AllocateSubmissionId(command_buffer_addr, 0); + + int32_t execution_result = OK; + uint32_t error_offset = 0; + auto submit_result = LibAmpr::Ampr::ExecuteAprCommandBuffer(command_buffer_addr, + &execution_result, &error_offset); + if (submit_result != OK) { + return KernelSyscallResult(submit_result); + } + AprShared::SetSubmissionResult(id, execution_result, error_offset); + + if (!AprShared::WriteGuest(reinterpret_cast(out_submission_id), id)) { + return KernelSyscallResult(LibKernel::KERNEL_ERROR_EFAULT); + } + + return OK; +} + +static int KYTY_SYSV_ABI WaitCommandBuffer(uint32_t submission_id) { + PRINT_NAME(); + + AprShared::SubmissionState state {}; + if (!AprShared::CompleteSubmission(submission_id, &state)) { + return KernelSyscallResult(LibKernel::KERNEL_ERROR_ESRCH); + } + if (state.result != 0) { + return KernelSyscallResult(AprShared::WriteResult( + reinterpret_cast(state.result), state.execution_result, state.error_offset)); + } + return OK; +} + +} // namespace Apr + +LIB_DEFINE(InitLibKernel_1_Apr) { + LIB_FUNC("WT-5NKy42fw", Apr::ResolveFilepathsToIds); + LIB_FUNC("ApkYaHb8Sek", Apr::GetFileStat); + LIB_FUNC("WvEu7yl3Ivg", Apr::GetFileSize); + LIB_FUNC("gEpBkcwxUjw", Apr::ResolveFilepathsToIdsAndFileSizes); + LIB_FUNC("i3HWvW35jao", Apr::ResolveFilepathsWithPrefixToIds); + LIB_FUNC("w5fcCG+t31g", Apr::ResolveFilepathsWithPrefixToIdsAndFileSizes); + LIB_FUNC("eYAh2vlCY-U", Apr::ResolveFilepathsToIdsForEach); + LIB_FUNC("QzB4O+bJQyA", Apr::ResolveFilepathsToIdsAndFileSizesForEach); + LIB_FUNC("VB-BtuIW8Xc", Apr::ResolveFilepathsWithPrefixToIdsForEach); + LIB_FUNC("C+Khtbbx2g8", Apr::ResolveFilepathsWithPrefixToIdsAndFileSizesForEach); + LIB_FUNC("ASoW5WE-UPo", Apr::SubmitCommandBufferAndGetResult); + LIB_FUNC("rqwFKI4PAiM", Apr::WaitCommandBuffer); + LIB_FUNC("eE4Szl8sil8", Apr::SubmitCommandBuffer); + LIB_FUNC("qvMUCyyaCSI", Apr::SubmitCommandBufferAndGetId); +} + +} // namespace LibKernelApr + +namespace LibAmpr { + +LIB_VERSION("Ampr", 1, "Ampr", 1, 1); + +namespace Ampr { + +constexpr uint64_t COMMAND_BUFFER_TYPE_OFFSET = 0x00; +constexpr uint64_t COMMAND_BUFFER_OFFSET_OFFSET = 0x04; +constexpr uint64_t COMMAND_BUFFER_NUM_OFFSET = 0x08; +constexpr uint64_t COMMAND_BUFFER_SIZE_OFFSET = 0x0c; +constexpr uint64_t COMMAND_BUFFER_DATA_OFFSET = 0x10; +constexpr uint64_t COMMAND_BUFFER_SIZE = 0x18; +constexpr uint64_t APR_COMMAND_BUFFER_MAP_OFFSET = 0x18; +constexpr uint64_t APR_COMMAND_BUFFER_SG_OFFSET = 0x20; +constexpr uint32_t COMMAND_BUFFER_SIZE_MAX = 64 * 1024 * 1024; +constexpr uint64_t READ_FILE_RECORD_SIZE = 0x14; +constexpr uint64_t READ_FILE_RECORD_SIZE_EXT = 0x18; +constexpr uint64_t READ_GATHER_RECORD_SIZE = 0x08; +constexpr uint64_t READ_GATHER_RECORD_SIZE_EXT = 0x0c; +constexpr uint64_t READ_SCATTER_RECORD_SIZE = 0x0c; +constexpr uint64_t READ_GATHER_SCATTER_SIZE = 0x10; +constexpr uint64_t READ_GATHER_SCATTER_SIZE_EXT = 0x14; +constexpr uint64_t RESET_GATHER_SCATTER_SIZE = 0x04; +constexpr uint64_t KERNEL_EVENT_RECORD_SIZE = 0x20; +constexpr uint64_t APR_MAP_BEGIN_RECORD_SIZE = 0x0c; +constexpr uint64_t APR_MAP_DIRECT_BEGIN_SIZE = 0x10; +constexpr uint64_t APR_MAP_END_RECORD_SIZE = 0x04; +constexpr uint64_t AMM_MAP_RECORD_SIZE = 0x20; +constexpr uint64_t AMM_MAP_DIRECT_RECORD_SIZE = 0x30; +constexpr uint64_t AMM_UNMAP_RECORD_SIZE = 0x20; +constexpr uint64_t AMM_PAGE_SIZE = 0x4000; +constexpr int AMM_MAP_FIXED = 0x10; +constexpr int AMM_USAGE_DIRECT = 0; +constexpr int AMM_USAGE_AUTO = 1; +constexpr int PROT_CPU_READ = 0x01; +constexpr int PROT_CPU_WRITE = 0x02; +constexpr int PROT_CPU_EXEC = 0x04; +constexpr int PROT_GPU_READ = 0x10; +constexpr int PROT_GPU_WRITE = 0x20; +constexpr int PROT_AMPR_READ = 0x40; +constexpr int PROT_AMPR_WRITE = 0x80; +constexpr int PROT_ACP_READ = 0x100; +constexpr int PROT_ACP_WRITE = 0x200; +constexpr uint64_t AMM_VA_START = 0x0000001000000000ull; +constexpr uint64_t AMM_VA_SIZE = 0x0000001000000000ull; +constexpr uint64_t APR_MAX_READ_LENGTH = 0x0000000100000000ull; +constexpr uint64_t APR_MAX_FILE_OFFSET = 0x0000010000000000ull; +constexpr uint64_t APR_MAX_APP_ADDRESS = 0x0000f00000000000ull; +constexpr uint64_t APR_HOST_READ_CHUNK_SIZE = 4 * 1024 * 1024; +constexpr uint32_t APR_TYPE_GATHER_SCATTER_VALID = 0x00010000; +constexpr uint32_t APR_TYPE_MAP_ACTIVE = 0x00020000; + +enum class AmmCommandKind : uint32_t { + MapAuto, + MapDirect, + Unmap, +}; + +struct CommandBufferState { + uint64_t buffer = 0; + uint64_t size = 0; + uint64_t write_offset = 0; + bool header_validated = false; + bool buffer_validated = false; + struct ReadFileCommand { + uint64_t record_offset = 0; + uint32_t file_id = 0; + uint64_t destination = 0; + uint64_t size = 0; + uint64_t file_offset = 0; + }; + struct KernelEventCommand { + uint64_t record_offset = 0; + uint64_t eq = 0; + int32_t id = 0; + uint64_t data = 0; + }; + struct WriteAddressCommand { + uint64_t record_offset = 0; + uint64_t address = 0; + uint64_t value = 0; + }; + struct AmmMapCommand { + uint64_t record_offset = 0; + AmmCommandKind kind = AmmCommandKind::MapAuto; + uint64_t va = 0; + uint64_t dmem_offset = 0; + uint64_t size = 0; + int32_t type = 0; + int32_t prot = 0; + uint8_t gpu_mask_id = 0; + }; + std::vector read_file_commands; + std::vector kernel_event_commands; + std::vector write_address_commands; + std::vector amm_map_commands; + bool gather_scatter_valid = false; + uint32_t gather_scatter_file_id = 0; + uint64_t gather_scatter_destination = 0; + uint64_t gather_scatter_file_offset = 0; +}; + +struct AmmAutoPoolRange { + uint64_t start = 0; + uint64_t size = 0; + uint64_t used = 0; +}; + +struct AmmVirtualAddressRanges { + uint64_t va_start = 0; + uint64_t va_end = 0; + uint64_t multimap_va_start = 0; + uint64_t multimap_va_end = 0; +}; + +struct AmmUsageStatsData { + uint64_t size_in_bytes = 0; + uint16_t num_page_table_pool_entries = 0; + uint16_t snapshot_page_table_pool_allocated_entries = 0; + uint16_t high_watermark_allocated_page_table_pool_entries = 0; + uint16_t reserved1 = 0; + uint32_t ring_idle_flags = 0; +}; +static_assert(sizeof(AmmUsageStatsData) == 0x18); + +static std::mutex g_command_buffer_mutex; +static std::unordered_map g_command_buffers; +static std::unordered_map g_command_buffer_aliases; +static std::mutex g_amm_auto_pool_mutex; +static std::vector g_amm_auto_pool; +using CommandBufferIterator = std::unordered_map::iterator; + +static bool HasQueuedCommands(const CommandBufferState& state) { + return !state.read_file_commands.empty() || !state.kernel_event_commands.empty() || + !state.write_address_commands.empty() || !state.amm_map_commands.empty(); +} + +static void RegisterCommandBufferAliasLocked(uint64_t command_buffer, uint64_t buffer) { + for (auto it = g_command_buffer_aliases.begin(); it != g_command_buffer_aliases.end();) { + if (it->second == command_buffer) { + it = g_command_buffer_aliases.erase(it); + } else { + ++it; + } + } + + if (buffer != 0 && buffer != command_buffer) { + g_command_buffer_aliases[buffer] = command_buffer; + } +} + +static void EraseCommandBufferStateLocked(uint64_t command_buffer) { + g_command_buffers.erase(command_buffer); + for (auto it = g_command_buffer_aliases.begin(); it != g_command_buffer_aliases.end();) { + if (it->first == command_buffer || it->second == command_buffer) { + it = g_command_buffer_aliases.erase(it); + } else { + ++it; + } + } +} + +static bool AddU64(uint64_t a, uint64_t b, uint64_t* out) { + if (out == nullptr || std::numeric_limits::max() - a < b) { + return false; + } + + *out = a + b; + return true; +} + +static bool IsValidAprFileOffset(uint64_t file_offset) { + return file_offset < APR_MAX_FILE_OFFSET; +} + +static bool IsValidAprReadSize(uint64_t size) { + return size != 0 && size <= APR_MAX_READ_LENGTH; +} + +static bool IsValidAprReadRange(uint64_t destination, uint64_t size) { + return IsValidAprReadSize(size) && destination <= APR_MAX_APP_ADDRESS && + APR_MAX_APP_ADDRESS - destination >= size; +} + +static uint64_t KernelErrorU64(int error) { + return static_cast(error); +} + +template +static T ReadCommandBufferUnchecked(uint64_t addr) { + T value {}; + std::memcpy(&value, reinterpret_cast(addr), sizeof(value)); + return value; +} + +template +static void WriteCommandBufferUnchecked(uint64_t addr, const T& value) { + std::memcpy(reinterpret_cast(addr), &value, sizeof(value)); +} + +static bool ValidateCommandBufferHeader(uint64_t command_buffer, + CommandBufferState* state = nullptr) { + if (state != nullptr && state->header_validated) { + return true; + } + if (!AprShared::IsValidGuestRange(command_buffer, COMMAND_BUFFER_SIZE, true)) { + return false; + } + if (state != nullptr) { + state->header_validated = true; + } + return true; +} + +static bool ValidateCommandBufferBacking(CommandBufferState* state) { + if (state == nullptr || state->buffer == 0 || state->size == 0) { + return false; + } + if (state->buffer_validated) { + return true; + } + if (!AprShared::IsValidGuestRange(state->buffer, state->size, true)) { + return false; + } + state->buffer_validated = true; + return true; +} + +static bool LoadCommandBufferStateFromGuest(uint64_t command_buffer, CommandBufferState* state) { + if (state == nullptr || !ValidateCommandBufferHeader(command_buffer)) { + return false; + } + + CommandBufferState loaded {}; + loaded.buffer = + ReadCommandBufferUnchecked(command_buffer + COMMAND_BUFFER_DATA_OFFSET); + loaded.size = ReadCommandBufferUnchecked(command_buffer + COMMAND_BUFFER_SIZE_OFFSET); + loaded.write_offset = + ReadCommandBufferUnchecked(command_buffer + COMMAND_BUFFER_OFFSET_OFFSET); + loaded.header_validated = true; + *state = loaded; + return true; +} + +static bool GetOrCreateCommandBufferStateLocked(uint64_t command_buffer, + CommandBufferIterator* out) { + if (out == nullptr) { + return false; + } + + auto it = g_command_buffers.find(command_buffer); + if (it == g_command_buffers.end()) { + CommandBufferState state {}; + if (!LoadCommandBufferStateFromGuest(command_buffer, &state)) { + return false; + } + it = g_command_buffers.emplace(command_buffer, state).first; + } + + *out = it; + return true; +} + +static bool EnsureCommandBufferRecordSpace(uint64_t command_buffer, CommandBufferState* state, + uint64_t record_size) { + if (state == nullptr || record_size == 0 || + !ValidateCommandBufferHeader(command_buffer, state) || state->buffer == 0 || + record_size > state->size || state->write_offset > state->size - record_size) { + return false; + } + + return ValidateCommandBufferBacking(state); +} + +static bool CommitCommandBufferRecord(uint64_t command_buffer, CommandBufferState* state, + uint64_t record_size) { + if (state == nullptr) { + return false; + } + + state->write_offset += record_size; + if (state->write_offset > std::numeric_limits::max()) { + return false; + } + + WriteCommandBufferUnchecked(command_buffer + COMMAND_BUFFER_OFFSET_OFFSET, + static_cast(state->write_offset)); + const auto num = + ReadCommandBufferUnchecked(command_buffer + COMMAND_BUFFER_NUM_OFFSET); + WriteCommandBufferUnchecked(command_buffer + COMMAND_BUFFER_NUM_OFFSET, num + 1); + return true; +} + +static void UpdateCommandBufferTypeFlagsUnchecked(uint64_t command_buffer, uint32_t set_bits, + uint32_t clear_bits) { + auto type = ReadCommandBufferUnchecked(command_buffer + COMMAND_BUFFER_TYPE_OFFSET); + type |= set_bits; + type &= ~clear_bits; + WriteCommandBufferUnchecked(command_buffer + COMMAND_BUFFER_TYPE_OFFSET, type); +} + +static uint64_t AprReadFileRecordSize(uint64_t file_offset) { + return (file_offset >> 32u) != 0 ? READ_FILE_RECORD_SIZE_EXT : READ_FILE_RECORD_SIZE; +} + +static uint64_t AprReadGatherRecordSize(uint64_t file_offset) { + return file_offset > 0x3ffffu ? READ_GATHER_RECORD_SIZE_EXT : READ_GATHER_RECORD_SIZE; +} + +static uint64_t AprReadGatherScatterRecordSize(uint64_t file_offset) { + return (file_offset >> 32u) != 0 ? READ_GATHER_SCATTER_SIZE_EXT : READ_GATHER_SCATTER_SIZE; +} + +static bool UpdateCommandBufferTypeFlags(uint64_t command_buffer, uint32_t set_bits, + uint32_t clear_bits) { + if (!ValidateCommandBufferHeader(command_buffer)) { + return false; + } + UpdateCommandBufferTypeFlagsUnchecked(command_buffer, set_bits, clear_bits); + return true; +} + +static std::unordered_map::iterator +ResolveCommandBufferStateLocked(uint64_t command_buffer) { + auto it = g_command_buffers.find(command_buffer); + if (it != g_command_buffers.end() && HasQueuedCommands(it->second)) { + return it; + } + + const auto alias = g_command_buffer_aliases.find(command_buffer); + if (alias != g_command_buffer_aliases.end()) { + const auto alias_it = g_command_buffers.find(alias->second); + if (alias_it != g_command_buffers.end()) { + return alias_it; + } + } + + return it; +} + +static bool WriteVisibleCommandBufferPointers(uint64_t command_buffer, uint64_t buffer, + uint64_t size) { + if (size > std::numeric_limits::max() || + !ValidateCommandBufferHeader(command_buffer)) { + return false; + } + + WriteCommandBufferUnchecked(command_buffer + COMMAND_BUFFER_DATA_OFFSET, buffer); + WriteCommandBufferUnchecked(command_buffer + COMMAND_BUFFER_SIZE_OFFSET, + static_cast(size)); + return true; +} + +static bool WriteCommandBufferPointers(uint64_t command_buffer, uint64_t buffer, uint64_t size, + uint64_t write_offset = 0) { + if (write_offset > std::numeric_limits::max() || + size > std::numeric_limits::max() || + !ValidateCommandBufferHeader(command_buffer)) { + return false; + } + + WriteCommandBufferUnchecked(command_buffer + COMMAND_BUFFER_DATA_OFFSET, buffer); + WriteCommandBufferUnchecked(command_buffer + COMMAND_BUFFER_SIZE_OFFSET, + static_cast(size)); + WriteCommandBufferUnchecked(command_buffer + COMMAND_BUFFER_OFFSET_OFFSET, + static_cast(write_offset)); + + std::scoped_lock lock(g_command_buffer_mutex); + auto& state = g_command_buffers[command_buffer]; + state.buffer = buffer; + state.size = size; + state.write_offset = write_offset; + state.header_validated = true; + state.buffer_validated = false; + state.read_file_commands.clear(); + state.kernel_event_commands.clear(); + state.write_address_commands.clear(); + state.amm_map_commands.clear(); + state.gather_scatter_valid = false; + state.gather_scatter_file_id = 0; + state.gather_scatter_destination = 0; + state.gather_scatter_file_offset = 0; + RegisterCommandBufferAliasLocked(command_buffer, buffer); + return true; +} + +static bool TryGetCommandBufferState(uint64_t command_buffer, CommandBufferState* out) { + if (out == nullptr) { + return false; + } + + { + std::scoped_lock lock(g_command_buffer_mutex); + const auto it = ResolveCommandBufferStateLocked(command_buffer); + if (it != g_command_buffers.end()) { + *out = it->second; + return true; + } + } + + CommandBufferState state {}; + if (!LoadCommandBufferStateFromGuest(command_buffer, &state)) { + return false; + } + + std::scoped_lock lock(g_command_buffer_mutex); + g_command_buffers[command_buffer] = state; + *out = state; + return true; +} + +static bool InitializeBaseCommandBuffer(uint64_t command_buffer) { + if (!AprShared::IsValidGuestRange(command_buffer, COMMAND_BUFFER_SIZE, true)) { + return false; + } + + std::memset(reinterpret_cast(command_buffer), 0, COMMAND_BUFFER_SIZE); + + std::scoped_lock lock(g_command_buffer_mutex); + EraseCommandBufferStateLocked(command_buffer); + return true; +} + +static bool WriteGuestZero64Validated(uint64_t addr) { + if (!AprShared::IsValidGuestRange(addr, sizeof(uint64_t), true)) { + return false; + } + WriteCommandBufferUnchecked(addr, uint64_t {0}); + return true; +} + +static bool InitializeAprReservedState(uint64_t command_buffer, uint64_t reserved_state0, + uint64_t reserved_state1) { + if (reserved_state0 == 0) { + reserved_state0 = command_buffer + APR_COMMAND_BUFFER_MAP_OFFSET; + } + if (reserved_state1 == 0) { + reserved_state1 = command_buffer + APR_COMMAND_BUFFER_SG_OFFSET; + } + + if (reserved_state0 == reserved_state1) { + return WriteGuestZero64Validated(reserved_state0); + } + + const auto word_size = static_cast(sizeof(uint64_t)); + const auto span_size = word_size * 2u; + const bool adjacent = (reserved_state0 <= std::numeric_limits::max() - word_size && + reserved_state0 + word_size == reserved_state1) || + (reserved_state1 <= std::numeric_limits::max() - word_size && + reserved_state1 + word_size == reserved_state0); + if (adjacent) { + const auto first = std::min(reserved_state0, reserved_state1); + if (first > std::numeric_limits::max() - span_size || + !AprShared::IsValidGuestRange(first, span_size, true)) { + return false; + } + WriteCommandBufferUnchecked(reserved_state0, uint64_t {0}); + WriteCommandBufferUnchecked(reserved_state1, uint64_t {0}); + return true; + } + + return WriteGuestZero64Validated(reserved_state0) && WriteGuestZero64Validated(reserved_state1); +} + +static bool AppendReadFileRecord(uint64_t command_buffer, uint8_t opcode, uint32_t file_id, + uint64_t destination, uint64_t size, uint64_t file_offset, + uint64_t record_size) { + std::scoped_lock lock(g_command_buffer_mutex); + CommandBufferIterator it; + if (!GetOrCreateCommandBufferStateLocked(command_buffer, &it)) { + return false; + } + + auto& state = it->second; + if (!EnsureCommandBufferRecordSpace(command_buffer, &state, record_size)) { + return false; + } + + const auto record_offset = state.write_offset; + std::memset(reinterpret_cast(state.buffer + record_offset), 0, + static_cast(record_size)); + std::memcpy(reinterpret_cast(state.buffer + record_offset), &opcode, sizeof(opcode)); + state.read_file_commands.push_back({record_offset, file_id, destination, size, file_offset}); + if (!CommitCommandBufferRecord(command_buffer, &state, record_size)) { + return false; + } + state.gather_scatter_valid = true; + state.gather_scatter_file_id = file_id; + if (!AddU64(destination, size, &state.gather_scatter_destination) || + !AddU64(file_offset, size, &state.gather_scatter_file_offset)) { + state.gather_scatter_valid = false; + return false; + } + UpdateCommandBufferTypeFlagsUnchecked(command_buffer, APR_TYPE_GATHER_SCATTER_VALID, 0); + return true; +} + +static bool AppendKernelEventRecord(uint64_t command_buffer, uint64_t eq, int32_t id, + uint64_t data) { + std::array record {}; + const uint32_t type = 2; + std::memcpy(&record[0x00], &type, sizeof(type)); + std::memcpy(&record[0x08], &eq, sizeof(eq)); + std::memcpy(&record[0x10], &id, sizeof(id)); + std::memcpy(&record[0x18], &data, sizeof(data)); + + std::scoped_lock lock(g_command_buffer_mutex); + CommandBufferIterator it; + if (!GetOrCreateCommandBufferStateLocked(command_buffer, &it)) { + return false; + } + + auto& state = it->second; + if (!EnsureCommandBufferRecordSpace(command_buffer, &state, KERNEL_EVENT_RECORD_SIZE)) { + return false; + } + + const auto record_offset = state.write_offset; + std::memcpy(reinterpret_cast(state.buffer + record_offset), record.data(), + record.size()); + state.kernel_event_commands.push_back({record_offset, eq, id, data}); + return CommitCommandBufferRecord(command_buffer, &state, KERNEL_EVENT_RECORD_SIZE); +} + +static bool AppendWriteAddressRecord(uint64_t command_buffer, uint64_t address, uint64_t value, + uint64_t record_size = 0x20) { + std::scoped_lock lock(g_command_buffer_mutex); + CommandBufferIterator it; + if (!GetOrCreateCommandBufferStateLocked(command_buffer, &it)) { + return false; + } + + auto& state = it->second; + if (!EnsureCommandBufferRecordSpace(command_buffer, &state, record_size)) { + return false; + } + + const auto record_offset = state.write_offset; + std::memset(reinterpret_cast(state.buffer + record_offset), 0, + static_cast(record_size)); + state.write_address_commands.push_back({record_offset, address, value}); + return CommitCommandBufferRecord(command_buffer, &state, record_size); +} + +static bool ValidateAmmMapArgs(uint64_t va, uint64_t size) { + return va != 0 && size != 0 && (va & (AMM_PAGE_SIZE - 1u)) == 0 && + (size & (AMM_PAGE_SIZE - 1u)) == 0 && va + size >= va; +} + +static int KYTY_SYSV_ABI AmmCommandBufferConstructor(void* command_buffer) { + PRINT_NAME(); + + (void)command_buffer; + return OK; +} + +static int KYTY_SYSV_ABI AmmCommandBufferDestructor(void* command_buffer) { + PRINT_NAME(); + + (void)command_buffer; + return OK; +} + +static int NormalizeAmmProtection(int prot) { + constexpr int cpu_gpu_bits = + PROT_CPU_READ | PROT_CPU_WRITE | PROT_CPU_EXEC | PROT_GPU_READ | PROT_GPU_WRITE; + + int normalized = prot & cpu_gpu_bits; + + if ((prot & PROT_AMPR_READ) != 0) { + normalized |= PROT_CPU_READ; + } + if ((prot & PROT_AMPR_WRITE) != 0) { + normalized |= PROT_CPU_READ | PROT_CPU_WRITE; + } + if ((prot & PROT_ACP_READ) != 0) { + normalized |= PROT_CPU_READ; + } + if ((prot & PROT_ACP_WRITE) != 0) { + normalized |= PROT_CPU_READ | PROT_CPU_WRITE; + } + + return normalized; +} + +static bool AllocateAmmAutoDirectMemory(uint64_t size, int memory_type, uint64_t* dmem_offset) { + if (dmem_offset == nullptr || size == 0) { + return false; + } + + std::scoped_lock lock(g_amm_auto_pool_mutex); + + for (auto& range: g_amm_auto_pool) { + const auto aligned_used = (range.used + (AMM_PAGE_SIZE - 1u)) & ~(AMM_PAGE_SIZE - 1u); + if (aligned_used <= range.size && size <= range.size - aligned_used) { + *dmem_offset = range.start + aligned_used; + range.used = aligned_used + size; + return true; + } + } + + int64_t allocated = 0; + if (LibKernel::Memory::KernelAllocateDirectMemory( + 0, LibKernel::Memory::KernelGetDirectMemorySize(), size, AMM_PAGE_SIZE, memory_type, + &allocated) != OK) { + return false; + } + + *dmem_offset = static_cast(allocated); + return true; +} + +static int ExecuteAmmMapCommand(const CommandBufferState::AmmMapCommand& command) { + if (!ValidateAmmMapArgs(command.va, command.size)) { + return LibKernel::KERNEL_ERROR_EINVAL; + } + + uint64_t dmem_offset = command.dmem_offset; + if (command.kind == AmmCommandKind::MapAuto && + !AllocateAmmAutoDirectMemory(command.size, command.type, &dmem_offset)) { + return LibKernel::KERNEL_ERROR_EAGAIN; + } + + void* addr = reinterpret_cast(command.va); + return LibKernel::Memory::KernelMapDirectMemory2( + &addr, command.size, command.type, NormalizeAmmProtection(command.prot), AMM_MAP_FIXED, + static_cast(dmem_offset), AMM_PAGE_SIZE); +} + +static bool AppendAmmMapRecord(uint64_t command_buffer, + const CommandBufferState::AmmMapCommand& cmd, uint64_t record_size) { + std::scoped_lock lock(g_command_buffer_mutex); + CommandBufferIterator it; + if (!GetOrCreateCommandBufferStateLocked(command_buffer, &it)) { + return false; + } + + auto& state = it->second; + if (!EnsureCommandBufferRecordSpace(command_buffer, &state, record_size)) { + return false; + } + + auto record = cmd; + record.record_offset = state.write_offset; + auto* record_addr = reinterpret_cast(state.buffer + state.write_offset); + std::memset(record_addr, 0, static_cast(record_size)); + state.amm_map_commands.push_back(record); + return CommitCommandBufferRecord(command_buffer, &state, record_size); +} + +static int ReadHostFileToGuest(const std::string& host_path, uint64_t file_offset, + uint64_t destination, uint64_t size, uint64_t* bytes_read); + +static int ExecuteAprCommandBuffer(uint64_t command_buffer, int32_t* execution_result, + uint32_t* error_offset) { + if (execution_result == nullptr || error_offset == nullptr) { + return LibKernel::KERNEL_ERROR_EINVAL; + } + + *execution_result = OK; + *error_offset = 0; + + CommandBufferState state {}; + if (!TryGetCommandBufferState(command_buffer, &state)) { + return LibKernel::KERNEL_ERROR_EFAULT; + } + + enum class CommandKind { + ReadFile, + KernelEvent, + WriteAddress, + AmmMap, + }; + + struct OrderedCommand { + uint64_t record_offset = 0; + CommandKind kind = CommandKind::ReadFile; + size_t index = 0; + }; + + std::vector ordered; + ordered.reserve(state.read_file_commands.size() + state.kernel_event_commands.size() + + state.write_address_commands.size() + state.amm_map_commands.size()); + for (size_t i = 0; i < state.read_file_commands.size(); i++) { + if (state.read_file_commands[i].record_offset < state.write_offset) { + ordered.push_back( + {state.read_file_commands[i].record_offset, CommandKind::ReadFile, i}); + } + } + for (size_t i = 0; i < state.kernel_event_commands.size(); i++) { + if (state.kernel_event_commands[i].record_offset < state.write_offset) { + ordered.push_back( + {state.kernel_event_commands[i].record_offset, CommandKind::KernelEvent, i}); + } + } + for (size_t i = 0; i < state.write_address_commands.size(); i++) { + if (state.write_address_commands[i].record_offset < state.write_offset) { + ordered.push_back( + {state.write_address_commands[i].record_offset, CommandKind::WriteAddress, i}); + } + } + for (size_t i = 0; i < state.amm_map_commands.size(); i++) { + if (state.amm_map_commands[i].record_offset < state.write_offset) { + ordered.push_back({state.amm_map_commands[i].record_offset, CommandKind::AmmMap, i}); + } + } + + std::sort(ordered.begin(), ordered.end(), [](const OrderedCommand& a, const OrderedCommand& b) { + return a.record_offset < b.record_offset; + }); + + for (const auto& entry: ordered) { + switch (entry.kind) { + case CommandKind::ReadFile: { + const auto& command = state.read_file_commands[entry.index]; + std::string host_path; + if (!AprShared::TryGetHostPath(command.file_id, &host_path)) { + LOGF("\tAPR submit failed for unknown file id: 0x%08" PRIx32 "\n", + command.file_id); + *execution_result = LibKernel::KERNEL_ERROR_ENOENT; + *error_offset = static_cast(command.record_offset); + return OK; + } + + uint64_t bytes_read = 0; + auto result = ReadHostFileToGuest(host_path, command.file_offset, + command.destination, command.size, &bytes_read); + if (result != OK) { + LOGF("\tAPR submit read failed: id=0x%08" PRIx32 ", result=0x%08" PRIx32 + ", path=%s\n", + command.file_id, static_cast(result), host_path.c_str()); + *execution_result = result; + *error_offset = static_cast(command.record_offset); + return OK; + } + } break; + case CommandKind::KernelEvent: { + const auto& command = state.kernel_event_commands[entry.index]; + auto* eq = reinterpret_cast(command.eq); + auto result = LibKernel::EventQueue::KernelTriggerUserEvent( + eq, command.id, reinterpret_cast(command.data)); + if (result != OK) { + LOGF("\tAPR submit event failed: eq=0x%016" PRIx64 ", id=%" PRId32 + ", result=0x%08" PRIx32 "\n", + command.eq, command.id, static_cast(result)); + *execution_result = result; + *error_offset = static_cast(command.record_offset); + return OK; + } + } break; + case CommandKind::WriteAddress: { + const auto& command = state.write_address_commands[entry.index]; + if (!AprShared::WriteGuest(command.address, command.value)) { + LOGF("\tAMPR submit write-address failed: address=0x%016" PRIx64 + " value=0x%016" PRIx64 "\n", + command.address, command.value); + *execution_result = LibKernel::KERNEL_ERROR_EFAULT; + *error_offset = static_cast(command.record_offset); + return OK; + } + } break; + case CommandKind::AmmMap: { + const auto& command = state.amm_map_commands[entry.index]; + int result = OK; + if (command.kind == AmmCommandKind::Unmap) { + result = LibKernel::Memory::KernelMunmap(command.va, command.size); + } else { + result = ExecuteAmmMapCommand(command); + } + + if (result != OK) { + LOGF("\tAMM submit command failed: kind=%u va=0x%016" PRIx64 + " dmem=0x%016" PRIx64 " size=0x%016" PRIx64 " type=%" PRId32 + " prot=0x%08" PRIx32 " result=0x%08" PRIx32 "\n", + static_cast(command.kind), command.va, command.dmem_offset, + command.size, command.type, static_cast(command.prot), + static_cast(result)); + *execution_result = result; + *error_offset = static_cast(command.record_offset); + return OK; + } + } break; + } + } + + return OK; +} + +static bool AdvanceCommandBuffer(uint64_t command_buffer, uint64_t record_size) { + std::scoped_lock lock(g_command_buffer_mutex); + CommandBufferIterator it; + if (!GetOrCreateCommandBufferStateLocked(command_buffer, &it)) { + return false; + } + + auto& state = it->second; + if (!EnsureCommandBufferRecordSpace(command_buffer, &state, record_size)) { + return false; + } + + std::memset(reinterpret_cast(state.buffer + state.write_offset), 0, + static_cast(record_size)); + return CommitCommandBufferRecord(command_buffer, &state, record_size); +} + +static int ReadHostFileToGuest(const std::string& host_path, uint64_t file_offset, + uint64_t destination, uint64_t size, uint64_t* bytes_read) { + if (bytes_read == nullptr) { + return LibKernel::KERNEL_ERROR_EINVAL; + } + + *bytes_read = 0; + if (size == 0) { + return OK; + } + if (!AprShared::IsValidGuestRange(destination, size, true)) { + return LibKernel::KERNEL_ERROR_EFAULT; + } + + Common::File file; + if (!file.Open(host_path, Common::File::Mode::Read)) { + LOGF("\tAPR read missing host file: %s\n", host_path.c_str()); + return LibKernel::KERNEL_ERROR_ENOENT; + } + const auto file_size = file.Size(); + if (file_offset >= file_size) { + file.Close(); + return OK; + } + if (!file.Seek(file_offset)) { + file.Close(); + return LibKernel::KERNEL_ERROR_EINVAL; + } + + const auto readable = std::min(size, file_size - file_offset); + if (readable == 0) { + file.Close(); + return OK; + } + + std::vector buffer( + static_cast(std::min(APR_HOST_READ_CHUNK_SIZE, readable))); + while (*bytes_read < readable) { + const auto request = + static_cast(std::min(buffer.size(), readable - *bytes_read)); + uint32_t read = 0; + file.Read(buffer.data(), request, &read); + if (read == 0) { + break; + } + std::memcpy(reinterpret_cast(destination + *bytes_read), buffer.data(), read); + *bytes_read += read; + } + + file.Close(); + return OK; +} + +static int KYTY_SYSV_ABI CommandBufferConstructor(void* command_buffer) { + PRINT_NAME(); + + if (command_buffer == nullptr) { + return OK; + } + + return InitializeBaseCommandBuffer(reinterpret_cast(command_buffer)) + ? OK + : LibKernel::KERNEL_ERROR_EFAULT; +} + +static int KYTY_SYSV_ABI AprCommandBufferConstructor(void* command_buffer, void* reserved_state0, + void* reserved_state1) { + PRINT_NAME(); + + if (command_buffer == nullptr) { + return OK; + } + + return InitializeAprReservedState(reinterpret_cast(command_buffer), + reinterpret_cast(reserved_state0), + reinterpret_cast(reserved_state1)) + ? OK + : LibKernel::KERNEL_ERROR_EFAULT; +} + +static int KYTY_SYSV_ABI AprCommandBufferDestructor(void* command_buffer) { + PRINT_NAME(); + + (void)command_buffer; + return OK; +} + +static int KYTY_SYSV_ABI CommandBufferDestructor(void* command_buffer) { + PRINT_NAME(); + + if (command_buffer == nullptr) { + return OK; + } + + const auto command_buffer_addr = reinterpret_cast(command_buffer); + std::scoped_lock lock(g_command_buffer_mutex); + EraseCommandBufferStateLocked(command_buffer_addr); + return OK; +} + +static int KYTY_SYSV_ABI CommandBufferSetBuffer(void* command_buffer, void* buffer, uint32_t size) { + PRINT_NAME(); + + if (command_buffer == nullptr) { + return LibKernel::KERNEL_ERROR_EINVAL; + } + + const auto command_buffer_addr = reinterpret_cast(command_buffer); + if (!ValidateCommandBufferHeader(command_buffer_addr)) { + return LibKernel::KERNEL_ERROR_EFAULT; + } + if (buffer == nullptr || (reinterpret_cast(buffer) & 0x3u) != 0 || size == 0 || + size > COMMAND_BUFFER_SIZE_MAX || (size & 0x3u) != 0) { + return LibKernel::KERNEL_ERROR_EINVAL; + } + + const auto old_buffer = + ReadCommandBufferUnchecked(command_buffer_addr + COMMAND_BUFFER_DATA_OFFSET); + if (old_buffer != 0) { + return LibKernel::KERNEL_ERROR_EBUSY; + } + + if (!WriteCommandBufferPointers(command_buffer_addr, reinterpret_cast(buffer), + size)) { + return LibKernel::KERNEL_ERROR_EFAULT; + } + + WriteCommandBufferUnchecked(command_buffer_addr + COMMAND_BUFFER_NUM_OFFSET, int32_t {0}); + return OK; +} + +static int KYTY_SYSV_ABI CommandBufferReset(void* command_buffer) { + PRINT_NAME(); + + if (command_buffer == nullptr) { + return LibKernel::KERNEL_ERROR_EPERM; + } + + const auto command_buffer_addr = reinterpret_cast(command_buffer); + CommandBufferState state {}; + if (!TryGetCommandBufferState(command_buffer_addr, &state)) { + return LibKernel::KERNEL_ERROR_EFAULT; + } + if (state.buffer == 0) { + return LibKernel::KERNEL_ERROR_EPERM; + } + + if (!WriteCommandBufferPointers(command_buffer_addr, state.buffer, state.size)) { + return LibKernel::KERNEL_ERROR_EFAULT; + } + + WriteCommandBufferUnchecked(command_buffer_addr + COMMAND_BUFFER_NUM_OFFSET, int32_t {0}); + return OK; +} + +static void* KYTY_SYSV_ABI CommandBufferClearBuffer(void* command_buffer) { + PRINT_NAME(); + + if (command_buffer == nullptr) { + return nullptr; + } + + const auto command_buffer_addr = reinterpret_cast(command_buffer); + if (!ValidateCommandBufferHeader(command_buffer_addr)) { + return nullptr; + } + const auto buffer = + ReadCommandBufferUnchecked(command_buffer_addr + COMMAND_BUFFER_DATA_OFFSET); + const auto size = + ReadCommandBufferUnchecked(command_buffer_addr + COMMAND_BUFFER_SIZE_OFFSET); + if (buffer == 0 || size == 0) { + return nullptr; + } + + if (!WriteVisibleCommandBufferPointers(command_buffer_addr, 0, 0)) { + return nullptr; + } + + std::scoped_lock lock(g_command_buffer_mutex); + EraseCommandBufferStateLocked(command_buffer_addr); + return reinterpret_cast(buffer); +} + +static int KYTY_SYSV_ABI AprCommandBufferReadFile(void* command_buffer, uint64_t, uint64_t, + uint32_t file_id, void* destination, + uint64_t size, uint64_t file_offset) { + PRINT_NAME(); + + const auto destination_addr = reinterpret_cast(destination); + if (command_buffer == nullptr || !IsValidAprReadRange(destination_addr, size) || + !IsValidAprFileOffset(file_offset)) { + return LibKernel::KERNEL_ERROR_EINVAL; + } + + const auto append_ok = AppendReadFileRecord(reinterpret_cast(command_buffer), 0x17, + file_id, destination_addr, size, file_offset, + AprReadFileRecordSize(file_offset)); + return append_ok ? OK : LibKernel::KERNEL_ERROR_EFAULT; +} + +static uint64_t AlignUp4(uint64_t value) { + return (value + 3u) & ~uint64_t {3}; +} + +static uint64_t MarkerCommandSize(const char* msg, bool with_color) { + const uint64_t msg_size = (msg == nullptr ? 0 : std::strlen(msg) + 1); + return AlignUp4((with_color ? 8u : 4u) + msg_size); +} + +static uint64_t KYTY_SYSV_ABI MeasureCommandSizeReadFile(uint64_t, uint64_t destination, + uint64_t size, uint64_t file_offset) { + PRINT_NAME(); + + return IsValidAprReadRange(destination, size) && IsValidAprFileOffset(file_offset) + ? AprReadFileRecordSize(file_offset) + : KernelErrorU64(LibKernel::KERNEL_ERROR_EINVAL); +} + +static uint64_t KYTY_SYSV_ABI MeasureCommandSizeWriteKernelEventQueue(uint64_t, uint64_t, uint64_t, + uint64_t, uint64_t, + uint64_t) { + PRINT_NAME(); + + return KERNEL_EVENT_RECORD_SIZE; +} + +static uint64_t KYTY_SYSV_ABI MeasureCommandSizeFixed32(uint64_t, uint64_t, uint64_t, uint64_t, + uint64_t, uint64_t) { + PRINT_NAME(); + + return 0x20; +} + +static uint64_t KYTY_SYSV_ABI MeasureCommandSizeWriteAddressOnCompletion(volatile uint64_t*, + uint64_t) { + PRINT_NAME(); + + return 0x20; +} + +static uint64_t KYTY_SYSV_ABI +MeasureCommandSizeWriteAddressFromTimeCounterOnCompletion(volatile uint64_t*) { + PRINT_NAME(); + + return 0x20; +} + +static uint64_t KYTY_SYSV_ABI +MeasureCommandSizeWriteAddressFromCounterOnCompletion(volatile uint64_t*, uint8_t) { + PRINT_NAME(); + + return 0x20; +} + +static uint64_t KYTY_SYSV_ABI +MeasureCommandSizeWriteAddressFromCounterPairOnCompletion(volatile uint64_t*, uint8_t) { + PRINT_NAME(); + + return 0x20; +} + +static uint64_t KYTY_SYSV_ABI MeasureCommandSizeWriteCounterOnCompletion(uint8_t, uint8_t, uint64_t, + uint8_t) { + PRINT_NAME(); + + return 0x20; +} + +static int64_t KYTY_SYSV_ABI MeasureAmmCommandSizeMap(uint64_t, uint64_t, int32_t, int32_t) { + PRINT_NAME(); + + return AMM_MAP_RECORD_SIZE; +} + +static int64_t KYTY_SYSV_ABI MeasureAmmCommandSizeMapWithGpuMaskId(uint64_t, uint64_t, int32_t, + int32_t, uint8_t) { + PRINT_NAME(); + + return AMM_MAP_RECORD_SIZE; +} + +static int64_t KYTY_SYSV_ABI MeasureAmmCommandSizeMapDirect(uint64_t, uint64_t, uint64_t, int32_t, + int32_t) { + PRINT_NAME(); + + return AMM_MAP_DIRECT_RECORD_SIZE; +} + +static int64_t KYTY_SYSV_ABI MeasureAmmCommandSizeMapDirectWithGpuMaskId(uint64_t, uint64_t, + uint64_t, int32_t, int32_t, + uint8_t) { + PRINT_NAME(); + + return AMM_MAP_DIRECT_RECORD_SIZE; +} + +static int64_t KYTY_SYSV_ABI MeasureAmmCommandSizeUnmap(uint64_t, uint64_t) { + PRINT_NAME(); + + return AMM_UNMAP_RECORD_SIZE; +} + +static int64_t KYTY_SYSV_ABI MeasureAprCommandSizeMapBegin(uint64_t va, uint64_t size, int32_t, + int32_t) { + PRINT_NAME(); + + return ValidateAmmMapArgs(va, size) ? APR_MAP_BEGIN_RECORD_SIZE + : LibKernel::KERNEL_ERROR_EINVAL; +} + +static int64_t KYTY_SYSV_ABI MeasureAprCommandSizeMapDirectBegin(uint64_t va, uint64_t dmem_offset, + uint64_t size, int32_t, int32_t) { + PRINT_NAME(); + + return ValidateAmmMapArgs(va, size) && (dmem_offset & (AMM_PAGE_SIZE - 1u)) == 0 + ? APR_MAP_DIRECT_BEGIN_SIZE + : LibKernel::KERNEL_ERROR_EINVAL; +} + +static uint64_t KYTY_SYSV_ABI MeasureCommandSizeNop(uint32_t num_u32) { + PRINT_NAME(); + + return num_u32 == 0 ? sizeof(uint32_t) + : AlignUp4(static_cast(num_u32) * sizeof(uint32_t)); +} + +static uint64_t KYTY_SYSV_ABI MeasureCommandSizeNopWithData(uint32_t num_u32, const uint32_t*) { + PRINT_NAME(); + + return AlignUp4((static_cast(num_u32) + 1u) * sizeof(uint32_t)); +} + +static uint64_t KYTY_SYSV_ABI MeasureCommandSizeReadFileGather(uint64_t size, + uint64_t file_offset) { + PRINT_NAME(); + + return IsValidAprReadSize(size) && IsValidAprFileOffset(file_offset) + ? AprReadGatherRecordSize(file_offset) + : KernelErrorU64(LibKernel::KERNEL_ERROR_EINVAL); +} + +static uint64_t KYTY_SYSV_ABI MeasureCommandSizeReadFileScatter(uint64_t destination, + uint64_t size) { + PRINT_NAME(); + + return IsValidAprReadRange(destination, size) ? READ_SCATTER_RECORD_SIZE + : KernelErrorU64(LibKernel::KERNEL_ERROR_EINVAL); +} + +static uint64_t KYTY_SYSV_ABI MeasureCommandSizeReadFileGatherScatter(uint64_t destination, + uint64_t size, + uint64_t file_offset) { + PRINT_NAME(); + + return IsValidAprReadRange(destination, size) && IsValidAprFileOffset(file_offset) + ? AprReadGatherScatterRecordSize(file_offset) + : KernelErrorU64(LibKernel::KERNEL_ERROR_EINVAL); +} + +static uint64_t KYTY_SYSV_ABI MeasureCommandSizeMarkerWithColor(const char* msg, uint32_t) { + PRINT_NAME(); + + return MarkerCommandSize(msg, true); +} + +static uint64_t KYTY_SYSV_ABI MeasureCommandSizeMarker(const char* msg) { + PRINT_NAME(); + + return MarkerCommandSize(msg, false); +} + +static uint64_t KYTY_SYSV_ABI MeasureCommandSizePopMarker() { + PRINT_NAME(); + + return sizeof(uint32_t); +} + +static uint64_t KYTY_SYSV_ABI CommandBufferGetSize(void* command_buffer) { + PRINT_NAME(); + + if (command_buffer == nullptr) { + return 0; + } + + uint32_t size = 0; + AprShared::ReadGuest(reinterpret_cast(command_buffer) + COMMAND_BUFFER_SIZE_OFFSET, + &size); + return size; +} + +static uint64_t KYTY_SYSV_ABI CommandBufferGetCurrentOffset(void* command_buffer) { + PRINT_NAME(); + + if (command_buffer == nullptr) { + return 0; + } + + uint32_t offset = 0; + AprShared::ReadGuest(reinterpret_cast(command_buffer) + COMMAND_BUFFER_OFFSET_OFFSET, + &offset); + return offset; +} + +static int KYTY_SYSV_ABI CommandBufferGetType(void* command_buffer) { + PRINT_NAME(); + + if (command_buffer == nullptr) { + return 0; + } + + uint32_t type = 0; + AprShared::ReadGuest(reinterpret_cast(command_buffer) + COMMAND_BUFFER_TYPE_OFFSET, + &type); + return static_cast(type); +} + +static void* KYTY_SYSV_ABI CommandBufferGetBufferBaseAddress(void* command_buffer) { + PRINT_NAME(); + + if (command_buffer == nullptr) { + return nullptr; + } + + uint64_t buffer = 0; + AprShared::ReadGuest(reinterpret_cast(command_buffer) + COMMAND_BUFFER_DATA_OFFSET, + &buffer); + return reinterpret_cast(buffer); +} + +static uint32_t KYTY_SYSV_ABI CommandBufferGetNumCommands(void* command_buffer) { + PRINT_NAME(); + + if (command_buffer == nullptr) { + return 0; + } + + uint32_t num = 0; + AprShared::ReadGuest(reinterpret_cast(command_buffer) + COMMAND_BUFFER_NUM_OFFSET, + &num); + return num; +} + +static int AppendNoOpCommand(void* command_buffer, uint64_t size) { + if (command_buffer == nullptr) { + return LibKernel::KERNEL_ERROR_EINVAL; + } + + return AdvanceCommandBuffer(reinterpret_cast(command_buffer), AlignUp4(size)) + ? OK + : LibKernel::KERNEL_ERROR_EFAULT; +} + +static int KYTY_SYSV_ABI CommandBufferWriteKernelEventQueue(void* command_buffer, uint64_t eq, + uint64_t id, uint64_t data, uint64_t, + uint64_t) { + PRINT_NAME(); + + if (command_buffer == nullptr) { + return LibKernel::KERNEL_ERROR_EINVAL; + } + + return AppendKernelEventRecord(reinterpret_cast(command_buffer), eq, + static_cast(id), data) + ? OK + : LibKernel::KERNEL_ERROR_EFAULT; +} + +static int KYTY_SYSV_ABI CommandBufferNop(void* command_buffer, uint32_t num_u32) { + PRINT_NAME(); + + if (num_u32 == 0 || num_u32 > 16) { + return LibKernel::KERNEL_ERROR_EINVAL; + } + return AppendNoOpCommand(command_buffer, static_cast(num_u32) * sizeof(uint32_t)); +} + +static int KYTY_SYSV_ABI CommandBufferNopWithData(void* command_buffer, uint32_t num_u32, + const uint32_t*) { + PRINT_NAME(); + + if (num_u32 > 15) { + return LibKernel::KERNEL_ERROR_EINVAL; + } + return AppendNoOpCommand(command_buffer, + (static_cast(num_u32) + 1u) * sizeof(uint32_t)); +} + +static int KYTY_SYSV_ABI CommandBufferConstructNop(void* command_buffer, uint32_t, const void*, + uint32_t bytes, const uint32_t*) { + PRINT_NAME(); + + return AppendNoOpCommand(command_buffer, sizeof(uint32_t) + bytes); +} + +static int KYTY_SYSV_ABI CommandBufferConstructMarker(void* command_buffer, uint32_t, + const char* msg, const uint32_t* color) { + PRINT_NAME(); + + return AppendNoOpCommand(command_buffer, MarkerCommandSize(msg, color != nullptr)); +} + +static int KYTY_SYSV_ABI CommandBufferSetMarker(void* command_buffer, const char* msg) { + PRINT_NAME(); + + return AppendNoOpCommand(command_buffer, MarkerCommandSize(msg, false)); +} + +static int KYTY_SYSV_ABI CommandBufferSetMarkerWithColor(void* command_buffer, const char* msg, + uint32_t) { + PRINT_NAME(); + + return AppendNoOpCommand(command_buffer, MarkerCommandSize(msg, true)); +} + +static int KYTY_SYSV_ABI CommandBufferPushMarker(void* command_buffer, const char* msg) { + PRINT_NAME(); + + return AppendNoOpCommand(command_buffer, MarkerCommandSize(msg, false)); +} + +static int KYTY_SYSV_ABI CommandBufferPushMarkerWithColor(void* command_buffer, const char* msg, + uint32_t) { + PRINT_NAME(); + + return AppendNoOpCommand(command_buffer, MarkerCommandSize(msg, true)); +} + +static int KYTY_SYSV_ABI CommandBufferPopMarker(void* command_buffer) { + PRINT_NAME(); + + return AppendNoOpCommand(command_buffer, sizeof(uint32_t)); +} + +static int KYTY_SYSV_ABI CommandBufferWaitOnAddress(void* command_buffer, volatile uint64_t*, + uint64_t, uint8_t, uint8_t) { + PRINT_NAME(); + + return AppendNoOpCommand(command_buffer, 0x20); +} + +static int KYTY_SYSV_ABI CommandBufferWaitOnCounter(void* command_buffer, uint8_t, uint8_t, + uint64_t, uint8_t, uint8_t, uint64_t, uint8_t) { + PRINT_NAME(); + + return AppendNoOpCommand(command_buffer, 0x20); +} + +static int AppendWriteAddressCommand(void* command_buffer, volatile uint64_t* address, + uint64_t value) { + if (command_buffer == nullptr || address == nullptr) { + return LibKernel::KERNEL_ERROR_EINVAL; + } + return AppendWriteAddressRecord(reinterpret_cast(command_buffer), + reinterpret_cast(address), value) + ? OK + : LibKernel::KERNEL_ERROR_EBUSY; +} + +static int KYTY_SYSV_ABI CommandBufferWriteAddress(void* command_buffer, volatile uint64_t* address, + uint64_t value, uint32_t) { + PRINT_NAME(); + + return AppendWriteAddressCommand(command_buffer, address, value); +} + +static int KYTY_SYSV_ABI CommandBufferWriteAddressOnCompletion(void* command_buffer, + volatile uint64_t* address, + uint64_t value) { + PRINT_NAME(); + + return AppendWriteAddressCommand(command_buffer, address, value); +} + +static int KYTY_SYSV_ABI CommandBufferWriteCounter(void* command_buffer, uint8_t, uint8_t, uint64_t, + uint8_t, uint32_t) { + PRINT_NAME(); + + return AppendNoOpCommand(command_buffer, 0x20); +} + +static int KYTY_SYSV_ABI CommandBufferWriteCounterOnCompletion(void* command_buffer, uint8_t, + uint8_t, uint64_t, uint8_t) { + PRINT_NAME(); + + return AppendNoOpCommand(command_buffer, 0x20); +} + +static int KYTY_SYSV_ABI CommandBufferWriteAddressFromTimeCounter(void* command_buffer, + volatile uint64_t* address, + uint32_t) { + PRINT_NAME(); + + return AppendWriteAddressCommand(command_buffer, address, uint64_t {0}); +} + +static int KYTY_SYSV_ABI CommandBufferWriteAddressFromTimeCounterOnCompletion( + void* command_buffer, volatile uint64_t* address) { + PRINT_NAME(); + + return AppendWriteAddressCommand(command_buffer, address, uint64_t {0}); +} + +static int KYTY_SYSV_ABI CommandBufferWriteAddressFromCounter(void* command_buffer, + volatile uint64_t* address, uint8_t, + uint32_t) { + PRINT_NAME(); + + return AppendWriteAddressCommand(command_buffer, address, uint64_t {0}); +} + +static int KYTY_SYSV_ABI CommandBufferWriteAddressFromCounterOnCompletion( + void* command_buffer, volatile uint64_t* address, uint8_t) { + PRINT_NAME(); + + return AppendWriteAddressCommand(command_buffer, address, uint64_t {0}); +} + +static int KYTY_SYSV_ABI CommandBufferWriteAddressFromCounterPair(void* command_buffer, + volatile uint64_t* address, + uint8_t, uint32_t) { + PRINT_NAME(); + + return AppendWriteAddressCommand(command_buffer, address, uint64_t {0}); +} + +static int KYTY_SYSV_ABI CommandBufferWriteAddressFromCounterPairOnCompletion( + void* command_buffer, volatile uint64_t* address, uint8_t) { + PRINT_NAME(); + + return AppendWriteAddressCommand(command_buffer, address, uint64_t {0}); +} + +static int KYTY_SYSV_ABI AprCommandBufferReadFileGather(void* command_buffer, uint64_t, uint64_t, + uint64_t size, uint64_t file_offset) { + PRINT_NAME(); + + if (command_buffer == nullptr || !IsValidAprReadSize(size) || + !IsValidAprFileOffset(file_offset)) { + return LibKernel::KERNEL_ERROR_EINVAL; + } + + CommandBufferState state {}; + const auto command_buffer_addr = reinterpret_cast(command_buffer); + if (!TryGetCommandBufferState(command_buffer_addr, &state)) { + return LibKernel::KERNEL_ERROR_EFAULT; + } + if (!state.gather_scatter_valid || + !IsValidAprReadRange(state.gather_scatter_destination, size)) { + return LibKernel::KERNEL_ERROR_EINVAL; + } + + return AppendReadFileRecord(command_buffer_addr, 0x18, state.gather_scatter_file_id, + state.gather_scatter_destination, size, file_offset, + AprReadGatherRecordSize(file_offset)) + ? OK + : LibKernel::KERNEL_ERROR_EFAULT; +} + +static int KYTY_SYSV_ABI AprCommandBufferReadFileScatter(void* command_buffer, uint64_t, uint64_t, + void* destination, uint64_t size) { + PRINT_NAME(); + + const auto destination_addr = reinterpret_cast(destination); + if (command_buffer == nullptr || !IsValidAprReadRange(destination_addr, size)) { + return LibKernel::KERNEL_ERROR_EINVAL; + } + + CommandBufferState state {}; + const auto command_buffer_addr = reinterpret_cast(command_buffer); + if (!TryGetCommandBufferState(command_buffer_addr, &state)) { + return LibKernel::KERNEL_ERROR_EFAULT; + } + if (!state.gather_scatter_valid || !IsValidAprFileOffset(state.gather_scatter_file_offset)) { + return LibKernel::KERNEL_ERROR_EINVAL; + } + + return AppendReadFileRecord(command_buffer_addr, 0x19, state.gather_scatter_file_id, + destination_addr, size, state.gather_scatter_file_offset, + READ_SCATTER_RECORD_SIZE) + ? OK + : LibKernel::KERNEL_ERROR_EFAULT; +} + +static int KYTY_SYSV_ABI AprCommandBufferReadFileGatherScatter(void* command_buffer, uint64_t, + uint64_t, void* destination, + uint64_t size, + uint64_t file_offset) { + PRINT_NAME(); + + const auto destination_addr = reinterpret_cast(destination); + if (command_buffer == nullptr || !IsValidAprReadRange(destination_addr, size) || + !IsValidAprFileOffset(file_offset)) { + return LibKernel::KERNEL_ERROR_EINVAL; + } + + CommandBufferState state {}; + const auto command_buffer_addr = reinterpret_cast(command_buffer); + if (!TryGetCommandBufferState(command_buffer_addr, &state)) { + return LibKernel::KERNEL_ERROR_EFAULT; + } + if (!state.gather_scatter_valid) { + return LibKernel::KERNEL_ERROR_EINVAL; + } + + return AppendReadFileRecord(command_buffer_addr, 0x1a, state.gather_scatter_file_id, + destination_addr, size, file_offset, + AprReadGatherScatterRecordSize(file_offset)) + ? OK + : LibKernel::KERNEL_ERROR_EFAULT; +} + +static int KYTY_SYSV_ABI AprCommandBufferResetGatherScatterState(void* command_buffer, uint64_t, + uint64_t) { + PRINT_NAME(); + + if (command_buffer == nullptr) { + return LibKernel::KERNEL_ERROR_EINVAL; + } + + const auto command_buffer_addr = reinterpret_cast(command_buffer); + if (AppendNoOpCommand(command_buffer, RESET_GATHER_SCATTER_SIZE) != OK) { + return LibKernel::KERNEL_ERROR_EFAULT; + } + + std::scoped_lock lock(g_command_buffer_mutex); + auto it = g_command_buffers.find(command_buffer_addr); + if (it != g_command_buffers.end()) { + it->second.gather_scatter_valid = false; + it->second.gather_scatter_file_id = 0; + it->second.gather_scatter_destination = 0; + it->second.gather_scatter_file_offset = 0; + } + UpdateCommandBufferTypeFlags(command_buffer_addr, 0, APR_TYPE_GATHER_SCATTER_VALID); + return OK; +} + +static int KYTY_SYSV_ABI AprCommandBufferMapBegin(void* command_buffer, uint64_t va, uint64_t size, + int32_t, int32_t) { + PRINT_NAME(); + + if (command_buffer == nullptr || !ValidateAmmMapArgs(va, size)) { + return LibKernel::KERNEL_ERROR_EINVAL; + } + + if (AppendNoOpCommand(command_buffer, APR_MAP_BEGIN_RECORD_SIZE) != OK) { + return LibKernel::KERNEL_ERROR_EFAULT; + } + + UpdateCommandBufferTypeFlags(reinterpret_cast(command_buffer), APR_TYPE_MAP_ACTIVE, + 0); + return OK; +} + +static int KYTY_SYSV_ABI AprCommandBufferMapDirectBegin(void* command_buffer, uint64_t va, + uint64_t dmem_offset, uint64_t size, + int32_t, int32_t) { + PRINT_NAME(); + + if (command_buffer == nullptr || !ValidateAmmMapArgs(va, size) || + (dmem_offset & (AMM_PAGE_SIZE - 1u)) != 0) { + return LibKernel::KERNEL_ERROR_EINVAL; + } + + if (AppendNoOpCommand(command_buffer, APR_MAP_DIRECT_BEGIN_SIZE) != OK) { + return LibKernel::KERNEL_ERROR_EFAULT; + } + + UpdateCommandBufferTypeFlags(reinterpret_cast(command_buffer), APR_TYPE_MAP_ACTIVE, + 0); + return OK; +} + +static int KYTY_SYSV_ABI AprCommandBufferMapEnd(void* command_buffer) { + PRINT_NAME(); + + if (command_buffer == nullptr) { + return LibKernel::KERNEL_ERROR_EPERM; + } + + uint32_t type = 0; + const auto command_buffer_addr = reinterpret_cast(command_buffer); + if (!AprShared::ReadGuest(command_buffer_addr + COMMAND_BUFFER_TYPE_OFFSET, &type)) { + return LibKernel::KERNEL_ERROR_EFAULT; + } + if ((type & APR_TYPE_MAP_ACTIVE) == 0) { + return LibKernel::KERNEL_ERROR_EPERM; + } + + if (AppendNoOpCommand(command_buffer, APR_MAP_END_RECORD_SIZE) != OK) { + return LibKernel::KERNEL_ERROR_EFAULT; + } + + UpdateCommandBufferTypeFlags(command_buffer_addr, 0, APR_TYPE_MAP_ACTIVE); + return OK; +} + +static int KYTY_SYSV_ABI AmmCommandBufferMap(void* command_buffer, uint64_t va, uint64_t size, + int32_t type, int32_t prot) { + PRINT_NAME(); + + if (command_buffer == nullptr || !ValidateAmmMapArgs(va, size)) { + return LibKernel::KERNEL_ERROR_EINVAL; + } + + CommandBufferState::AmmMapCommand command {}; + command.kind = AmmCommandKind::MapAuto; + command.va = va; + command.size = size; + command.type = type; + command.prot = prot; + + return AppendAmmMapRecord(reinterpret_cast(command_buffer), command, + AMM_MAP_RECORD_SIZE) + ? OK + : LibKernel::KERNEL_ERROR_EBUSY; +} + +static int KYTY_SYSV_ABI AmmCommandBufferMapWithGpuMaskId(void* command_buffer, uint64_t va, + uint64_t size, int32_t type, int32_t prot, + uint8_t gpu_mask_id) { + PRINT_NAME(); + + if (command_buffer == nullptr || !ValidateAmmMapArgs(va, size)) { + return LibKernel::KERNEL_ERROR_EINVAL; + } + + CommandBufferState::AmmMapCommand command {}; + command.kind = AmmCommandKind::MapAuto; + command.va = va; + command.size = size; + command.type = type; + command.prot = prot; + command.gpu_mask_id = gpu_mask_id; + + return AppendAmmMapRecord(reinterpret_cast(command_buffer), command, + AMM_MAP_RECORD_SIZE) + ? OK + : LibKernel::KERNEL_ERROR_EBUSY; +} + +static int KYTY_SYSV_ABI AmmCommandBufferMapDirect(void* command_buffer, uint64_t va, + uint64_t dmem_offset, uint64_t size, + int32_t type, int32_t prot) { + PRINT_NAME(); + + if (command_buffer == nullptr || !ValidateAmmMapArgs(va, size) || + (dmem_offset & (AMM_PAGE_SIZE - 1u)) != 0) { + return LibKernel::KERNEL_ERROR_EINVAL; + } + + CommandBufferState::AmmMapCommand command {}; + command.kind = AmmCommandKind::MapDirect; + command.va = va; + command.dmem_offset = dmem_offset; + command.size = size; + command.type = type; + command.prot = prot; + + return AppendAmmMapRecord(reinterpret_cast(command_buffer), command, + AMM_MAP_DIRECT_RECORD_SIZE) + ? OK + : LibKernel::KERNEL_ERROR_EBUSY; +} + +static int KYTY_SYSV_ABI AmmCommandBufferMapDirectWithGpuMaskId(void* command_buffer, uint64_t va, + uint64_t dmem_offset, uint64_t size, + int32_t type, int32_t prot, + uint8_t gpu_mask_id) { + PRINT_NAME(); + + if (command_buffer == nullptr || !ValidateAmmMapArgs(va, size) || + (dmem_offset & (AMM_PAGE_SIZE - 1u)) != 0) { + return LibKernel::KERNEL_ERROR_EINVAL; + } + + CommandBufferState::AmmMapCommand command {}; + command.kind = AmmCommandKind::MapDirect; + command.va = va; + command.dmem_offset = dmem_offset; + command.size = size; + command.type = type; + command.prot = prot; + command.gpu_mask_id = gpu_mask_id; + + return AppendAmmMapRecord(reinterpret_cast(command_buffer), command, + AMM_MAP_DIRECT_RECORD_SIZE) + ? OK + : LibKernel::KERNEL_ERROR_EBUSY; +} + +static int KYTY_SYSV_ABI AmmCommandBufferUnmap(void* command_buffer, uint64_t va, uint64_t size) { + PRINT_NAME(); + + if (command_buffer == nullptr || !ValidateAmmMapArgs(va, size)) { + return LibKernel::KERNEL_ERROR_EINVAL; + } + + CommandBufferState::AmmMapCommand command {}; + command.kind = AmmCommandKind::Unmap; + command.va = va; + command.size = size; + + return AppendAmmMapRecord(reinterpret_cast(command_buffer), command, + AMM_UNMAP_RECORD_SIZE) + ? OK + : LibKernel::KERNEL_ERROR_EBUSY; +} + +static int KYTY_SYSV_ABI AmmGiveDirectMemory(int64_t search_start, int64_t search_end, + uint64_t size, uint64_t align, int32_t usage, + int64_t* dmem_offset) { + PRINT_NAME(); + + if (dmem_offset == nullptr || size == 0 || + (usage != AMM_USAGE_DIRECT && usage != AMM_USAGE_AUTO)) { + return LibKernel::KERNEL_ERROR_EINVAL; + } + + int64_t allocated = 0; + int result = LibKernel::Memory::KernelAllocateDirectMemory(search_start, search_end, size, + align, 0, &allocated); + if (result != OK) { + return result; + } + + *dmem_offset = allocated; + if (usage == AMM_USAGE_AUTO) { + std::scoped_lock lock(g_amm_auto_pool_mutex); + g_amm_auto_pool.push_back(AmmAutoPoolRange {static_cast(allocated), size, 0}); + } + + return OK; +} + +static void KYTY_SYSV_ABI AmmGetVirtualAddressRanges(uint64_t* va_start, uint64_t* va_end, + uint64_t* multimap_va_start, + uint64_t* multimap_va_end) { + PRINT_NAME(); + + void* reserve_addr = reinterpret_cast(AMM_VA_START); + if (LibKernel::Memory::KernelReserveVirtualRange(&reserve_addr, AMM_VA_SIZE, AMM_MAP_FIXED, + 0x10000) == OK) { + (void)LibKernel::Memory::KernelSetVirtualRangeName(reserve_addr, AMM_VA_SIZE, "AMM"); + } + + if (va_start != nullptr) { + AprShared::WriteGuest(reinterpret_cast(va_start), AMM_VA_START); + } + if (va_end != nullptr) { + AprShared::WriteGuest(reinterpret_cast(va_end), AMM_VA_START + AMM_VA_SIZE); + } + if (multimap_va_start != nullptr) { + AprShared::WriteGuest(reinterpret_cast(multimap_va_start), + AMM_VA_START + AMM_VA_SIZE / 2u); + } + if (multimap_va_end != nullptr) { + AprShared::WriteGuest(reinterpret_cast(multimap_va_end), + AMM_VA_START + AMM_VA_SIZE); + } +} + +static int KYTY_SYSV_ABI AmmGetUsageStatsData(AmmUsageStatsData* stats) { + PRINT_NAME(); + + if (stats == nullptr || stats->size_in_bytes > sizeof(AmmUsageStatsData)) { + return LibKernel::KERNEL_ERROR_EINVAL; + } + + AmmUsageStatsData out {}; + out.size_in_bytes = stats->size_in_bytes; + out.num_page_table_pool_entries = 512; + out.ring_idle_flags = 0x7; + const auto write_size = std::min(stats->size_in_bytes, sizeof(out)); + return AprShared::WriteGuestBytes(reinterpret_cast(stats), &out, write_size) + ? OK + : LibKernel::KERNEL_ERROR_EFAULT; +} + +static int KYTY_SYSV_ABI AmmSetPageTablePoolOccupancyNotificationThreshold(uint32_t) { + PRINT_NAME(); + + return OK; +} + +static int KYTY_SYSV_ABI AmmSubmitCommandBuffer(void* command_buffer_base, uint32_t, uint32_t) { + PRINT_NAME(); + + if (command_buffer_base == nullptr) { + return LibKernel::KERNEL_ERROR_EINVAL; + } + + int32_t execution_result = OK; + uint32_t error_offset = 0; + return ExecuteAprCommandBuffer(reinterpret_cast(command_buffer_base), + &execution_result, &error_offset); +} + +static int KYTY_SYSV_ABI AmmSubmitCommandBufferAndGetId(void* command_buffer_base, uint32_t, + uint32_t, uint32_t* out_submission_id) { + PRINT_NAME(); + + if (command_buffer_base == nullptr || out_submission_id == nullptr) { + return LibKernel::KERNEL_ERROR_EINVAL; + } + + const auto command_buffer_addr = reinterpret_cast(command_buffer_base); + const auto id = AprShared::AllocateSubmissionId(command_buffer_addr, 0); + + int32_t execution_result = OK; + uint32_t error_offset = 0; + auto submit_result = + ExecuteAprCommandBuffer(command_buffer_addr, &execution_result, &error_offset); + if (submit_result != OK) { + return submit_result; + } + AprShared::SetSubmissionResult(id, execution_result, error_offset); + + if (!AprShared::WriteGuest(reinterpret_cast(out_submission_id), id)) { + return LibKernel::KERNEL_ERROR_EFAULT; + } + + return OK; +} + +static int KYTY_SYSV_ABI AmmSubmitCommandBufferAndGetResult(void* command_buffer_base, uint32_t, + uint32_t, void* result, + uint32_t* out_submission_id) { + PRINT_NAME(); + + if (command_buffer_base == nullptr) { + return LibKernel::KERNEL_ERROR_EINVAL; + } + + const auto command_buffer_addr = reinterpret_cast(command_buffer_base); + const auto id = + AprShared::AllocateSubmissionId(command_buffer_addr, reinterpret_cast(result)); + + int32_t execution_result = OK; + uint32_t error_offset = 0; + auto submit_result = + ExecuteAprCommandBuffer(command_buffer_addr, &execution_result, &error_offset); + if (submit_result != OK) { + return submit_result; + } + AprShared::SetSubmissionResult(id, execution_result, error_offset); + + if (out_submission_id != nullptr && + !AprShared::WriteGuest(reinterpret_cast(out_submission_id), id)) { + return LibKernel::KERNEL_ERROR_EFAULT; + } + + return AprShared::WriteResult(result, execution_result, error_offset); +} + +static int KYTY_SYSV_ABI AmmWaitCommandBufferCompletion(uint32_t submission_id) { + PRINT_NAME(); + + AprShared::SubmissionState state {}; + if (!AprShared::CompleteSubmission(submission_id, &state)) { + return LibKernel::KERNEL_ERROR_ESRCH; + } + + if (state.result != 0) { + return AprShared::WriteResult(reinterpret_cast(state.result), state.execution_result, + state.error_offset); + } + + return OK; +} + +} // namespace Ampr + +LIB_DEFINE(InitAmpr_1_Ampr) { + LIB_FUNC("8aI7R7WaOlc", Ampr::CommandBufferConstructor); + LIB_FUNC("a8uLzYY--tM", Ampr::AprCommandBufferConstructor); + LIB_FUNC("Qs1xtplKo0U", Ampr::AprCommandBufferDestructor); + LIB_FUNC("GuchCTefuZw", Ampr::CommandBufferDestructor); + LIB_FUNC("N-FSPA4S3nI", Ampr::CommandBufferSetBuffer); + LIB_FUNC("baQO9ez2gL4", Ampr::CommandBufferReset); + LIB_FUNC("ULvXMDz56po", Ampr::CommandBufferClearBuffer); + LIB_FUNC("VEDMaQmJZng", Ampr::CommandBufferGetType); + LIB_FUNC("RPCAhx-aabE", Ampr::CommandBufferGetBufferBaseAddress); + LIB_FUNC("gzndltBEzWc", Ampr::CommandBufferGetNumCommands); + LIB_FUNC("GmOguNIsuKk", Ampr::CommandBufferConstructNop); + LIB_FUNC("tNn5WBkta60", Ampr::CommandBufferNop); + LIB_FUNC("pFQ9UHpO52s", Ampr::CommandBufferNopWithData); + LIB_FUNC("4UkZbYKVF7c", Ampr::CommandBufferConstructMarker); + LIB_FUNC("sWbST0oQKsc", Ampr::CommandBufferSetMarkerWithColor); + LIB_FUNC("4quckD2y7Pg", Ampr::CommandBufferSetMarker); + LIB_FUNC("f12ObAMEi9A", Ampr::CommandBufferPushMarkerWithColor); + LIB_FUNC("dXPaz65HNmk", Ampr::CommandBufferPushMarker); + LIB_FUNC("mv0O8Zg0woU", Ampr::CommandBufferPopMarker); + LIB_FUNC("DLfoNxTFNVk", Ampr::CommandBufferWaitOnAddress); + LIB_FUNC("cQb8Zr8Q0Y0", Ampr::CommandBufferWaitOnCounter); + LIB_FUNC("j0+3uJMxYJY", Ampr::CommandBufferWriteAddress); + LIB_FUNC("sJXyWHjP-F8", Ampr::CommandBufferWriteAddressOnCompletion); + LIB_FUNC("jK+yuYCI7MA", Ampr::CommandBufferWriteCounter); + LIB_FUNC("3wn42MWTzTs", Ampr::CommandBufferWriteCounterOnCompletion); + LIB_FUNC("bt3LHR9xjK4", Ampr::CommandBufferWriteAddressFromTimeCounter); + LIB_FUNC("FI2JD0jAHCs", Ampr::CommandBufferWriteAddressFromTimeCounterOnCompletion); + LIB_FUNC("t4ExS+SwLjs", Ampr::CommandBufferWriteAddressFromCounter); + LIB_FUNC("gSF5OsXdfIg", Ampr::CommandBufferWriteAddressFromCounterOnCompletion); + LIB_FUNC("enZm-6GjWqw", Ampr::CommandBufferWriteAddressFromCounterPair); + LIB_FUNC("ZLWtNUP6R5E", Ampr::CommandBufferWriteAddressFromCounterPairOnCompletion); + LIB_FUNC("vWU-odnS+fU", Ampr::MeasureCommandSizeReadFile); + LIB_FUNC("sSAUCCU1dv4", Ampr::MeasureCommandSizeWriteKernelEventQueue); + LIB_FUNC("Zi3dBUjgyXI", Ampr::MeasureCommandSizeWriteKernelEventQueue); + LIB_FUNC("C+IEj+BsAFM", Ampr::MeasureCommandSizeWriteAddressOnCompletion); + LIB_FUNC("x7SQEXfeovg", Ampr::MeasureCommandSizeWriteAddressFromTimeCounterOnCompletion); + LIB_FUNC("32AcaTaBPSY", Ampr::MeasureCommandSizeWriteAddressFromCounterOnCompletion); + LIB_FUNC("vxC58+DRk-U", Ampr::MeasureCommandSizeWriteAddressFromCounterPairOnCompletion); + LIB_FUNC("4muPEJ-x5N8", Ampr::MeasureCommandSizeWriteCounterOnCompletion); + LIB_FUNC("0BMj1hgG+kE", Ampr::MeasureCommandSizeFixed32); + LIB_FUNC("ClnsFLLLcss", Ampr::MeasureCommandSizeFixed32); + LIB_FUNC("4fgtGfXDrFc", Ampr::MeasureCommandSizeFixed32); + LIB_FUNC("gAtc79UTt5E", Ampr::MeasureCommandSizeFixed32); + LIB_FUNC("JYd9g9L+TmE", Ampr::MeasureCommandSizeFixed32); + LIB_FUNC("2Hw8gjMdwSY", Ampr::MeasureCommandSizeFixed32); + LIB_FUNC("I-Qm+MEso5c", Ampr::MeasureCommandSizeFixed32); + LIB_FUNC("6hbai6KIXkk", Ampr::MeasureAmmCommandSizeMap); + LIB_FUNC("m+fYyX8oFqw", Ampr::MeasureAmmCommandSizeMapWithGpuMaskId); + LIB_FUNC("ZFDZoN9IbVU", Ampr::MeasureAmmCommandSizeMapDirect); + LIB_FUNC("KUjtdPZJo5I", Ampr::MeasureAmmCommandSizeMapDirectWithGpuMaskId); + LIB_FUNC("Ayg6PIon2wA", Ampr::MeasureAmmCommandSizeUnmap); + LIB_FUNC("NNIZ-FMyz3M", Ampr::MeasureCommandSizeNop); + LIB_FUNC("Xp85BP3+BBI", Ampr::MeasureCommandSizeNopWithData); + LIB_FUNC("qesF88X4DRg", Ampr::MeasureCommandSizeReadFileGather); + LIB_FUNC("7nXGDGMXSqo", Ampr::MeasureCommandSizeReadFileScatter); + LIB_FUNC("DXmgc5op8Yw", Ampr::MeasureCommandSizeReadFileGatherScatter); + LIB_FUNC("rddQYXM0CjM", Ampr::MeasureCommandSizePopMarker); + LIB_FUNC("kdFImtTD0hc", Ampr::MeasureAprCommandSizeMapBegin); + LIB_FUNC("qvbdJc7bG+s", Ampr::MeasureAprCommandSizeMapDirectBegin); + LIB_FUNC("iwTNhyaemnw", Ampr::MeasureCommandSizePopMarker); + LIB_FUNC("tmfr97+ED5I", Ampr::MeasureCommandSizeMarkerWithColor); + LIB_FUNC("VGkEj4d6-Kg", Ampr::MeasureCommandSizeMarker); + LIB_FUNC("3OfeY4pzDV0", Ampr::MeasureCommandSizeMarkerWithColor); + LIB_FUNC("0RdLmAh7WVo", Ampr::MeasureCommandSizeMarker); + LIB_FUNC("pbnNnahE8vk", Ampr::MeasureCommandSizePopMarker); + LIB_FUNC("tZDDEo2tE5k", Ampr::CommandBufferGetSize); + LIB_FUNC("GnxKOHEawhk", Ampr::CommandBufferGetCurrentOffset); + LIB_FUNC("H896Pt-yB4I", Ampr::CommandBufferWriteKernelEventQueue); + LIB_FUNC("o67gODLFpls", Ampr::CommandBufferWriteKernelEventQueue); + LIB_FUNC("mQ16-QdKv7k", Ampr::AprCommandBufferReadFile); + LIB_FUNC("mZSbNJVJpV8", Ampr::AprCommandBufferReadFileGather); + LIB_FUNC("Jg-AgkdJHkk", Ampr::AprCommandBufferReadFileScatter); + LIB_FUNC("BVmR1H8l+XI", Ampr::AprCommandBufferReadFileGatherScatter); + LIB_FUNC("YPxkUDhgoNI", Ampr::AprCommandBufferResetGatherScatterState); + LIB_FUNC("Eul7AGEpjLo", Ampr::AprCommandBufferMapBegin); + LIB_FUNC("bFEs0Gs6D2A", Ampr::AprCommandBufferMapDirectBegin); + LIB_FUNC("X169CE6G3Y4", Ampr::AprCommandBufferMapEnd); + LIB_FUNC("Q07J7XpvhrU", Ampr::AmmGiveDirectMemory); + LIB_FUNC("wkQR9+xTFKY", Ampr::AmmGetVirtualAddressRanges); + LIB_FUNC("KqiWXLgCVe0", Ampr::AmmGetUsageStatsData); + LIB_FUNC("touqMEt6qXQ", Ampr::AmmSetPageTablePoolOccupancyNotificationThreshold); + LIB_FUNC("EDq5bqCqYpA", Ampr::AmmCommandBufferConstructor); + LIB_FUNC("pvUFDOHilnE", Ampr::AmmCommandBufferDestructor); + LIB_FUNC("JEVYGhDc97M", Ampr::AmmCommandBufferMap); + LIB_FUNC("ojBkmG7+CgE", Ampr::AmmCommandBufferMapWithGpuMaskId); + LIB_FUNC("8TBE+9XCZbI", Ampr::AmmCommandBufferMapDirect); + LIB_FUNC("kOfZlhbVAkc", Ampr::AmmCommandBufferMapDirectWithGpuMaskId); + LIB_FUNC("M-VFI2DJWQA", Ampr::AmmCommandBufferUnmap); + LIB_FUNC("lwS-7y3jcBI", Ampr::AmmSubmitCommandBuffer); + LIB_FUNC("NnKhlMJtIsI", Ampr::AmmSubmitCommandBufferAndGetId); + LIB_FUNC("OJf3vCckPAM", Ampr::AmmSubmitCommandBufferAndGetResult); + LIB_FUNC("HXymib4T8gc", Ampr::AmmWaitCommandBufferCompletion); +} + +} // namespace LibAmpr + +LIB_DEFINE(InitAmpr_1) { + LibAmpr::InitAmpr_1_Ampr(s); +} + +} // namespace Libs diff --git a/src/libs/libAppContent.cpp b/src/libs/libAppContent.cpp new file mode 100644 index 000000000..fc2d625db --- /dev/null +++ b/src/libs/libAppContent.cpp @@ -0,0 +1,254 @@ +#include "common/abi.h" +#include "common/assert.h" +#include "common/common.h" +#include "common/emulatorConfig.h" +#include "common/logging/log.h" +#include "common/stringUtils.h" +#include "libs/errno.h" +#include "libs/libs.h" +#include "loader/symbolDatabase.h" +#include "loader/systemContent.h" + +#include +#include +#include + +namespace Libs { + +LIB_VERSION("AppContent", 1, "AppContentUtil", 1, 1); + +namespace AppContent { + +constexpr uint32_t APP_CONTENT_APPPARAM_SKU_FLAG_FULL = 3; +constexpr int APP_CONTENT_ERROR_PARAMETER = -2133262334; /* 0x80D90002 */ +constexpr int APP_CONTENT_ERROR_NOT_MOUNTED = -2133262332; /* 0x80D90004 */ +constexpr int APP_CONTENT_ERROR_DRM_NO_ENTITLEMENT = -2133262329; /* 0x80D90007 */ + +static std::string g_title_id; + +struct AppContentInitParam { + char reserved[32]; +}; + +struct AppContentBootParam { + char reserved1[4]; + uint32_t attr; + char reserved2[32]; +}; + +struct AppContentMountPoint { + char data[16]; +}; + +struct NpUnifiedEntitlementLabel { + char data[17]; + char padding[3]; +}; + +static bool TryGetTitleIdFromContentId(std::string* title_id) { + std::string content_id; + if (!Loader::SystemContentParamSfoGetString("CONTENT_ID", &content_id) || content_id.empty()) { + return false; + } + + const uint32_t dash = Common::FindIndex(content_id, '-'); + if (!Common::IndexValid(content_id, dash)) { + return false; + } + + const uint32_t underscore = Common::FindIndex(content_id, '_', dash + 1); + if (!Common::IndexValid(content_id, underscore) || underscore <= dash + 1) { + return false; + } + + *title_id = Common::Mid(content_id, dash + 1, underscore - dash - 1); + return !title_id->empty(); +} + +static bool ResolveTitleId(std::string* title_id) { + if (Loader::SystemContentParamSfoGetString("TITLE_ID", title_id) && !title_id->empty()) { + return true; + } + + if (TryGetTitleIdFromContentId(title_id)) { + LOGF("\t PS5 mode: TITLE_ID missing, using CONTENT_ID title segment\n"); + return true; + } + + return false; +} + +int KYTY_SYSV_ABI AppContentInitialize(const AppContentInitParam* init_param, + AppContentBootParam* boot_param) { + PRINT_NAME(); + + LOGF("\t init_param = 0x%016" PRIx64 "\n" + "\t boot_param = 0x%016" PRIx64 "\n", + reinterpret_cast(init_param), reinterpret_cast(boot_param)); + + if (boot_param != nullptr) { + std::memset(boot_param, 0, sizeof(*boot_param)); + } + + if (ResolveTitleId(&g_title_id)) { + LOGF("\t title_id = %s\n", g_title_id.c_str()); + } else { + LOGF("\t TITLE_ID missing\n"); + } + + return OK; +} + +int KYTY_SYSV_ABI AppContentAppParamGetInt(uint32_t param_id, int32_t* value) { + PRINT_NAME(); + + EXIT_NOT_IMPLEMENTED(value == nullptr); + + *value = 0; + bool found = false; + + LOGF("\t param_id = %u\n", param_id); + + switch (param_id) { + case 0: + *value = APP_CONTENT_APPPARAM_SKU_FLAG_FULL; + found = true; + break; + case 1: found = Loader::SystemContentParamSfoGetInt("USER_DEFINED_PARAM_1", value); break; + case 2: found = Loader::SystemContentParamSfoGetInt("USER_DEFINED_PARAM_2", value); break; + case 3: found = Loader::SystemContentParamSfoGetInt("USER_DEFINED_PARAM_3", value); break; + case 4: found = Loader::SystemContentParamSfoGetInt("USER_DEFINED_PARAM_4", value); break; + default: EXIT("unknown param_id: %u\n", param_id); + } + + LOGF("\t value = %d [%s]\n", *value, found ? "found" : "not found"); + + return OK; +} + +int KYTY_SYSV_ABI AppContentDownloadDataGetAvailableSpaceKb(const AppContentMountPoint* mount_point, + size_t* available_space_kb) { + PRINT_NAME(); + + LOGF("\t mount_point = 0x%016" PRIx64 "\n" + "\t available_space_kb = 0x%016" PRIx64 "\n", + reinterpret_cast(mount_point), reinterpret_cast(available_space_kb)); + + if (available_space_kb == nullptr) { + return APP_CONTENT_ERROR_PARAMETER; + } + + std::error_code ec; + const auto space = std::filesystem::space(std::filesystem::current_path(), ec); + if (ec) { + LOGF("\t filesystem::space failed: %s\n", ec.message().c_str()); + *available_space_kb = 1024u * 1024u; + } else { + *available_space_kb = static_cast(space.available / 1024u); + } + + if (mount_point != nullptr) { + char mount_name[sizeof(mount_point->data) + 1] {}; + std::memcpy(mount_name, mount_point->data, sizeof(mount_point->data)); + LOGF("\t mount = %s\n", mount_name); + } + LOGF("\t available_space_kb = %" PRIu64 "\n", static_cast(*available_space_kb)); + + return OK; +} + +int KYTY_SYSV_ABI AppContentAddcontMount(uint32_t service_label, + const NpUnifiedEntitlementLabel* entitlement_label, + AppContentMountPoint* mount_point) { + PRINT_NAME(); + + LOGF("\t service_label = %" PRIu32 "\n" + "\t entitlement_label = 0x%016" PRIx64 "\n" + "\t mount_point = 0x%016" PRIx64 "\n", + service_label, reinterpret_cast(entitlement_label), + reinterpret_cast(mount_point)); + + if (entitlement_label == nullptr || mount_point == nullptr) { + return APP_CONTENT_ERROR_PARAMETER; + } + + std::memset(mount_point->data, 0, sizeof(mount_point->data)); + + return APP_CONTENT_ERROR_DRM_NO_ENTITLEMENT; +} + +int KYTY_SYSV_ABI AppContentAddcontUnmount(const AppContentMountPoint* mount_point) { + PRINT_NAME(); + + LOGF("\t mount_point = 0x%016" PRIx64 "\n", reinterpret_cast(mount_point)); + + if (mount_point == nullptr) { + return APP_CONTENT_ERROR_PARAMETER; + } + + if (mount_point->data[0] == '\0') { + return APP_CONTENT_ERROR_NOT_MOUNTED; + } + + return OK; +} + +} // namespace AppContent + +namespace LibAppContentTemporary { + +namespace AppContentTemporary { + +using AppContentMountPoint = AppContent::AppContentMountPoint; + +static int KYTY_SYSV_ABI AppContentTemporaryDataMount2(uint32_t option, + AppContentMountPoint* mount_point) { + PRINT_NAME(); + + LOGF("\t option = 0x%08" PRIx32 "\n" + "\t mount_point = 0x%016" PRIx64 "\n", + option, reinterpret_cast(mount_point)); + + if (mount_point == nullptr) { + return AppContent::APP_CONTENT_ERROR_PARAMETER; + } + + std::memset(mount_point->data, 0, sizeof(mount_point->data)); + std::memcpy(mount_point->data, "/temp0", sizeof("/temp0") - 1); + + LOGF("\t mount = %s\n", mount_point->data); + + return OK; +} + +static int KYTY_SYSV_ABI AppContentTemporaryDataFormat(const AppContentMountPoint* mount_point) { + PRINT_NAME(); + + LOGF("\t mount_point = 0x%016" PRIx64 "\n", reinterpret_cast(mount_point)); + + if (mount_point == nullptr) { + return AppContent::APP_CONTENT_ERROR_PARAMETER; + } + + return OK; +} + +} // namespace AppContentTemporary + +LIB_DEFINE(InitAppContent_1_Temporary) { + LIB_FUNC("buYbeLOGWmA", AppContentTemporary::AppContentTemporaryDataMount2); + LIB_FUNC("a5N7lAG0y2Q", AppContentTemporary::AppContentTemporaryDataFormat); +} + +} // namespace LibAppContentTemporary + +LIB_DEFINE(InitAppContent_1) { + LIB_FUNC("R9lA82OraNs", AppContent::AppContentInitialize); + LIB_FUNC("99b82IKXpH4", AppContent::AppContentAppParamGetInt); + LIB_FUNC("VANhIWcqYak", AppContent::AppContentAddcontMount); + LIB_FUNC("3rHWaV-1KC4", AppContent::AppContentAddcontUnmount); + LIB_FUNC("Gl6w5i0JokY", AppContent::AppContentDownloadDataGetAvailableSpaceKb); + LibAppContentTemporary::InitAppContent_1_Temporary(s); +} + +} // namespace Libs diff --git a/src/libs/libAudio.cpp b/src/libs/libAudio.cpp new file mode 100644 index 000000000..abc1b975c --- /dev/null +++ b/src/libs/libAudio.cpp @@ -0,0 +1,1134 @@ +#include "common/abi.h" +#include "libs/audio.h" +#include "libs/libs.h" +#include "loader/symbolDatabase.h" + +#include +#include +#include + +namespace Libs { + +namespace LibAudioPropagation { + +LIB_VERSION("AudioPropagation", 1, "AudioPropagation", 1, 0); + +using AudioPropagationHandle = uint64_t; + +struct AudioPropagationStructDescriptor { + uint32_t id; + size_t size; +}; + +struct AudioPropagationSystemMemory { + AudioPropagationStructDescriptor desc; + void* p_cpu_mem; + size_t size_cpu_mem; + void* p_gpu_mem; + size_t size_gpu_mem; +}; + +static AudioPropagationHandle AudioPropagationNextHandle() { + static std::atomic_uint64_t next_handle {1}; + return next_handle.fetch_add(1, std::memory_order_relaxed); +} + +static int32_t KYTY_SYSV_ABI AudioPropagationSystemQueryMemory( + const void* /*options*/, AudioPropagationSystemMemory* out_memory) { + PRINT_NAME(); + + if (out_memory != nullptr) { + out_memory->p_cpu_mem = nullptr; + out_memory->size_cpu_mem = 0; + out_memory->p_gpu_mem = nullptr; + out_memory->size_gpu_mem = 0; + } + + return 0; +} + +static int32_t KYTY_SYSV_ABI +AudioPropagationSystemCreate(const void* /*options*/, AudioPropagationSystemMemory* /*memory*/, + AudioPropagationHandle* out_system_handle) { + PRINT_NAME(); + + if (out_system_handle != nullptr) { + *out_system_handle = AudioPropagationNextHandle(); + } + + return 0; +} + +static int32_t KYTY_SYSV_ABI AudioPropagationRoomCreate(AudioPropagationHandle /*system_handle*/, + AudioPropagationHandle* out_room_handle) { + PRINT_NAME(); + + if (out_room_handle != nullptr) { + *out_room_handle = AudioPropagationNextHandle(); + } + + return 0; +} + +static int32_t KYTY_SYSV_ABI AudioPropagationSystemRegisterMaterial( + AudioPropagationHandle /*system_handle*/, const void* /*material*/, + AudioPropagationHandle* out_material_handle) { + PRINT_NAME(); + + if (out_material_handle != nullptr) { + *out_material_handle = AudioPropagationNextHandle(); + } + + return 0; +} + +static int32_t KYTY_SYSV_ABI +AudioPropagationSystemSetAttributes(AudioPropagationHandle /*system_handle*/, + const void* /*attributes*/, uint32_t /*num_attributes*/) { + PRINT_NAME(); + + return 0; +} + +static int32_t KYTY_SYSV_ABI AudioPropagationSystemGetRays(AudioPropagationHandle /*system_handle*/, + void* /*rays*/, uint32_t* num_rays) { + PRINT_NAME(); + + if (num_rays != nullptr) { + *num_rays = 0; + } + + return 0; +} + +LIB_DEFINE(InitAudio_1_AudioPropagation) { + LIB_FUNC("7xyAxrusLko", LibAudioPropagation::AudioPropagationSystemQueryMemory); + LIB_FUNC("aNEqtSHdUSo", LibAudioPropagation::AudioPropagationSystemCreate); + LIB_FUNC("8bI5h8req30", LibAudioPropagation::AudioPropagationRoomCreate); + LIB_FUNC("CPLV6G-eXmk", LibAudioPropagation::AudioPropagationSystemRegisterMaterial); + LIB_FUNC("kIdb+iQUzCs", LibAudioPropagation::AudioPropagationSystemSetAttributes); + LIB_FUNC("ht-QXT3zGxo", LibAudioPropagation::AudioPropagationSystemGetRays); +} + +} // namespace LibAudioPropagation + +namespace LibAudioOut { + +LIB_VERSION("AudioOut", 1, "AudioOut", 1, 1); + +namespace AudioOut = Audio::AudioOut; + +LIB_DEFINE(InitAudio_1_AudioOut) { + LIB_FUNC("JfEPXVxhFqA", AudioOut::AudioOutInit); + LIB_FUNC("ekNvsT22rsY", AudioOut::AudioOutOpen); + LIB_FUNC("b+uAV89IlxE", AudioOut::AudioOutSetVolume); + LIB_FUNC("w3PdaSTSwGE", AudioOut::AudioOutOutputs); + LIB_FUNC("QOQtbeDqsT4", AudioOut::AudioOutOutput); + LIB_FUNC("s1--uE9mBFw", AudioOut::AudioOutClose); + LIB_FUNC("GrQ9s4IrNaQ", AudioOut::AudioOutGetPortState); +} + +} // namespace LibAudioOut + +namespace LibAudioOut2 { + +LIB_VERSION("AudioOut2", 1, "AudioOut", 1, 1); + +namespace AudioOut2 = Audio::AudioOut2; + +LIB_DEFINE(InitAudio_1_AudioOut2) { + LIB_FUNC("g2tViFIohHE", AudioOut2::AudioOut2Initialize); + LIB_FUNC("pDmme7Bgm6E", AudioOut2::AudioOut2ContextQueryMemory); + LIB_FUNC("t5YrizufpQc", AudioOut2::AudioOut2ContextResetParam); + LIB_FUNC("0x6o1VVAYSY", AudioOut2::AudioOut2ContextCreate); + LIB_FUNC("on6ZH7Abo10", AudioOut2::AudioOut2ContextDestroy); + LIB_FUNC("4dq2rblWlg0", AudioOut2::AudioOut2ContextSetAttributes); + LIB_FUNC("PE2zHMqLSHs", AudioOut2::AudioOut2ContextAdvance); + LIB_FUNC("aII9h5nli9U", AudioOut2::AudioOut2ContextPush); + LIB_FUNC("R7d0F1g2qsU", AudioOut2::AudioOut2ContextGetQueueLevel); + LIB_FUNC("JK2wamZPzwM", AudioOut2::AudioOut2PortCreate); + LIB_FUNC("cd+Rtw+D1x8", AudioOut2::AudioOut2PortDestroy); + LIB_FUNC("8XTArSPyWHk", AudioOut2::AudioOut2PortSetAttributes); + LIB_FUNC("gatEUKG+Ea4", AudioOut2::AudioOut2PortGetState); + LIB_FUNC("bkBN+CMLwRc", AudioOut2::AudioOut2GetSystemState); + LIB_FUNC("xywYcRB7nbQ", AudioOut2::AudioOut2UserCreate); + LIB_FUNC("IaZXJ9M79uo", AudioOut2::AudioOut2UserDestroy); + LIB_FUNC("G1YOKDJYX2Y", AudioOut2::AudioOut2GetSpeakerArrayMemorySize); + LIB_FUNC("+k91hoTuoA8", AudioOut2::AudioOut2SpeakerArrayCreate); + LIB_FUNC("erCWQR5eKiQ", AudioOut2::AudioOut2SpeakerArrayDestroy); + LIB_FUNC("4BlZurolOAo", AudioOut2::AudioOut2GetSpeakerArrayCoefficients); + LIB_FUNC("28QqMnuuJ9Y", AudioOut2::AudioOut2GetSpeakerArrayAmbisonicsCoefficients); + LIB_FUNC("DImz2Ft9E2g", AudioOut2::AudioOut2GetSpeakerInfo); + LIB_FUNC("b1HWDUC8zaE", AudioOut2::AudioOut2SetSystemDebugState); + LIB_FUNC("TViD1EZXkNI", AudioOut2::AudioOut2Set3DLatency); + LIB_FUNC("XHl38ZNknbs", AudioOut2::AudioOut2MasteringInit); + LIB_FUNC("v8iOE+j8a5o", AudioOut2::AudioOut2MasteringSetParam); + LIB_FUNC("VZidxi2cYh0", AudioOut2::AudioOut2MasteringGetState); + LIB_FUNC("2bbBBOkH4CY", AudioOut2::AudioOut2MasteringTerm); +} + +} // namespace LibAudioOut2 + +namespace LibAudioIn { + +LIB_VERSION("AudioIn", 1, "AudioIn", 1, 1); + +namespace AudioIn = Audio::AudioIn; + +LIB_DEFINE(InitAudio_1_AudioIn) { + LIB_FUNC("5NE8Sjc7VC8", AudioIn::AudioInOpen); + LIB_FUNC("LozEOU8+anM", AudioIn::AudioInInput); +} + +} // namespace LibAudioIn + +namespace LibVoiceQoS { + +LIB_VERSION("VoiceQoS", 1, "VoiceQoS", 0, 0); + +namespace VoiceQoS = Audio::VoiceQoS; + +LIB_DEFINE(InitAudio_1_VoiceQoS) { + LIB_FUNC("U8IfNl6-Css", VoiceQoS::VoiceQoSInit); +} + +} // namespace LibVoiceQoS + +namespace LibVoice { + +LIB_VERSION("Voice", 1, "Voice", 1, 1); + +struct VoiceInitParam { + int32_t app_type; + uint64_t on_event; + void* user_data; + uint8_t reserved[32 - sizeof(int32_t) - sizeof(uint64_t) - sizeof(void*)]; +}; + +struct VoicePortParam { + int32_t port_type; + uint16_t threshold; + uint16_t mute; + float volume; + union { + struct { + int32_t bitrate; + } voice; + struct { + uint32_t buffer_size; + int32_t data_type; + int32_t sample_rate; + } pcmaudio; + struct { + int32_t user_id; + int32_t type; + int32_t index; + } device; + }; +}; + +struct VoicePortInfo { + int32_t port_type; + int32_t state; + uint32_t* edge; + uint32_t byte_count; + uint32_t frame_size; + uint16_t edge_count; + uint16_t reserved; +}; + +struct VoiceStartParam { + void* container; + uint32_t mem_size; + uint8_t reserved[32 - sizeof(void*) - sizeof(uint32_t)]; +}; + +struct VoicePort { + bool used = false; + int32_t port_type = -1; + uint32_t bitrate = 48000; + uint32_t frame_size = 1; + uint32_t byte_count = 0; + uint16_t edge_count = 0; + float volume = 1.0f; +}; + +static std::array g_voice_ports; +static uint32_t g_next_voice_port_id = 1; +static bool g_voice_started = false; + +static uint32_t alloc_voice_port(const VoicePortParam* param) { + for (uint32_t i = 0; i < static_cast(g_voice_ports.size()) - 1; i++) { + const auto id = g_next_voice_port_id; + g_next_voice_port_id++; + if (g_next_voice_port_id == 0xff) { + g_next_voice_port_id = 1; + } + + if (!g_voice_ports[id].used) { + auto& port = g_voice_ports[id]; + port = {}; + port.used = true; + port.port_type = (param != nullptr ? param->port_type : -1); + port.bitrate = 48000; + port.frame_size = 1; + port.byte_count = 0; + port.edge_count = 0; + port.volume = (param != nullptr && param->volume > 0.0f ? param->volume : 1.0f); + + if (param != nullptr && param->port_type == 2 && param->voice.bitrate > 0) { + port.bitrate = static_cast(param->voice.bitrate); + } + + return id; + } + } + + return 0xff; +} + +static VoicePort* get_voice_port(uint32_t port_id) { + if (port_id >= g_voice_ports.size() || port_id == 0xff || !g_voice_ports[port_id].used) { + return nullptr; + } + + return &g_voice_ports[port_id]; +} + +int KYTY_SYSV_ABI VoiceInit(VoiceInitParam* /*param*/, int32_t /*version*/) { + PRINT_NAME(); + g_voice_started = false; + for (auto& port: g_voice_ports) { + port = {}; + } + g_next_voice_port_id = 1; + return 0; +} + +int KYTY_SYSV_ABI VoiceSetThreadsParams(void* /*params*/) { + PRINT_NAME(); + return 0; +} + +int KYTY_SYSV_ABI VoiceCreatePort(uint32_t* port_id, const VoicePortParam* param) { + PRINT_NAME(); + + if (port_id == nullptr) { + return -2142369787; + } + + const auto id = alloc_voice_port(param); + if (id == 0xff) { + return -2142369784; + } + + *port_id = id; + return 0; +} + +int KYTY_SYSV_ABI VoiceConnectIPortToOPort(uint32_t input_port_id, uint32_t output_port_id) { + PRINT_NAME(); + + auto* input_port = get_voice_port(input_port_id); + auto* output_port = get_voice_port(output_port_id); + + if (input_port != nullptr) { + input_port->edge_count = 1; + } + if (output_port != nullptr) { + output_port->edge_count = 1; + } + + return 0; +} + +int KYTY_SYSV_ABI VoiceDisconnectIPortFromOPort(uint32_t input_port_id, uint32_t output_port_id) { + PRINT_NAME(); + + auto* input_port = get_voice_port(input_port_id); + auto* output_port = get_voice_port(output_port_id); + + if (input_port != nullptr) { + input_port->edge_count = 0; + } + if (output_port != nullptr) { + output_port->edge_count = 0; + } + + return 0; +} + +int KYTY_SYSV_ABI VoiceDeletePort(uint32_t port_id) { + PRINT_NAME(); + + if (auto* port = get_voice_port(port_id); port != nullptr) { + *port = {}; + } + + return 0; +} + +int KYTY_SYSV_ABI VoiceEnd() { + PRINT_NAME(); + + g_voice_started = false; + for (auto& port: g_voice_ports) { + port = {}; + } + g_next_voice_port_id = 1; + + return 0; +} + +int KYTY_SYSV_ABI VoiceStart(const VoiceStartParam* /*param*/) { + PRINT_NAME(); + g_voice_started = true; + return 0; +} + +int KYTY_SYSV_ABI VoiceStop() { + PRINT_NAME(); + g_voice_started = false; + return 0; +} + +int KYTY_SYSV_ABI VoiceReadFromOPort(uint32_t /*output_port_id*/, void* data, uint32_t* size) { + PRINT_NAME(); + + if (size == nullptr) { + return -2142369787; + } + + if (data != nullptr && *size != 0) { + std::memset(data, 0, *size); + } + + *size = 0; + return 0; +} + +int KYTY_SYSV_ABI VoiceWriteToIPort(uint32_t /*input_port_id*/, const void* /*data*/, + uint32_t* size, int16_t /*frame_gaps*/) { + PRINT_NAME(); + + if (size == nullptr) { + return -2142369787; + } + + return 0; +} + +int KYTY_SYSV_ABI VoiceGetPortInfo(uint32_t port_id, VoicePortInfo* info) { + PRINT_NAME(); + + if (info == nullptr) { + return -2142369787; + } + + const auto* port = get_voice_port(port_id); + + const auto edge = info->edge; + std::memset(info, 0, sizeof(*info)); + info->edge = edge; + info->port_type = (port != nullptr ? port->port_type : -1); + info->state = (g_voice_started ? 1 : 0); + info->byte_count = (port != nullptr ? port->byte_count : 0); + info->frame_size = (port != nullptr && port->frame_size != 0 ? port->frame_size : 1); + info->edge_count = (port != nullptr ? port->edge_count : 0); + + return 0; +} + +int KYTY_SYSV_ABI VoiceGetPortAttr(uint32_t /*port_id*/, int32_t attr, void* value, int32_t size) { + PRINT_NAME(); + + if (value == nullptr || size <= 0) { + return -2142369787; + } + + std::memset(value, 0, static_cast(size)); + if (attr == 1001 && size >= static_cast(sizeof(bool))) { + *static_cast(value) = true; + } else if (attr == 1003 && size >= static_cast(sizeof(uint16_t))) { + *static_cast(value) = 0; + } + + return 0; +} + +int KYTY_SYSV_ABI VoiceGetBitRate(uint32_t port_id, uint32_t* bitrate) { + PRINT_NAME(); + + if (bitrate == nullptr) { + return -2142369787; + } + + const auto* port = get_voice_port(port_id); + *bitrate = (port != nullptr && port->bitrate != 0 ? port->bitrate : 48000); + + return 0; +} + +int KYTY_SYSV_ABI VoiceSetVolume(uint32_t port_id, float volume) { + PRINT_NAME(); + + if (auto* port = get_voice_port(port_id); port != nullptr) { + port->volume = volume; + } + + return 0; +} + +int KYTY_SYSV_ABI VoiceGetVolume(uint32_t port_id, float* volume) { + PRINT_NAME(); + + if (volume == nullptr) { + return -2142369787; + } + + const auto* port = get_voice_port(port_id); + *volume = (port != nullptr ? port->volume : 1.0f); + + return 0; +} + +LIB_DEFINE(InitAudio_1_Voice) { + LIB_FUNC("9TrhuGzberQ", VoiceInit); + LIB_FUNC("clyKUyi3RYU", VoiceSetThreadsParams); + LIB_FUNC("nXpje5yNpaE", VoiceCreatePort); + LIB_FUNC("oV9GAdJ23Gw", VoiceConnectIPortToOPort); + LIB_FUNC("ajVj3QG2um4", VoiceDisconnectIPortFromOPort); + LIB_FUNC("b7kJI+nx2hg", VoiceDeletePort); + LIB_FUNC("Oo0S5PH7FIQ", VoiceEnd); + LIB_FUNC("54phPH2LZls", VoiceStart); + LIB_FUNC("Ao2YNSA7-Qo", VoiceStop); + LIB_FUNC("cQ6DGsQEjV4", VoiceReadFromOPort); + LIB_FUNC("YeJl6yDlhW0", VoiceWriteToIPort); + LIB_FUNC("CrLqDwWLoXM", VoiceGetPortInfo); + LIB_FUNC("elcxZTEfHZM", VoiceGetPortAttr); + LIB_FUNC("cJLufzou6bc", VoiceGetBitRate); + LIB_FUNC("QBFoAIjJoXQ", VoiceSetVolume); + LIB_FUNC("jjkCjneOYSs", VoiceGetVolume); +} + +} // namespace LibVoice + +namespace LibAjm { + +LIB_VERSION("Ajm", 1, "Ajm", 1, 1); + +namespace Ajm = Audio::Ajm; + +LIB_DEFINE(InitAudio_1_Ajm) { + LIB_FUNC("dl+4eHSzUu4", Ajm::AjmInitialize); + LIB_FUNC("Q3dyFuwGn64", Ajm::AjmModuleRegister); + LIB_FUNC("Wi7DtlLV+KI", Ajm::AjmModuleUnregister); + LIB_FUNC("MHur6qCsUus", Ajm::AjmFinalize); + LIB_FUNC("AxoDrINp4J8", Ajm::AjmInstanceCreate); + LIB_FUNC("RbLbuKv8zho", Ajm::AjmInstanceDestroy); + LIB_FUNC("bkRHEYG6lEM", Ajm::AjmMemoryRegister); + LIB_FUNC("pIpGiaYkHkM", Ajm::AjmMemoryUnregister); + LIB_FUNC("MmpF1XsQiHw", Ajm::AjmBatchInitialize); + LIB_FUNC("5tOfnaClcqM", Ajm::AjmBatchStart); + LIB_FUNC("-qLsfDAywIY", Ajm::AjmBatchWait); + LIB_FUNC("NVDXiUesSbA", Ajm::AjmBatchCancel); + LIB_FUNC("WfAiBW8Wcek", Ajm::AjmBatchErrorDump); + LIB_FUNC("AxhcqVv5AYU", Ajm::AjmStrError); + LIB_FUNC("ezM2OhNxzck", Ajm::AjmBatchJobInitialize); + LIB_FUNC("uJ3m8INuikg", Ajm::AjmBatchJobClearContext); + LIB_FUNC("39WxhR-ePew", Ajm::AjmBatchJobDecode); + LIB_FUNC("5LLWbpP5xi8", Ajm::AjmBatchJobDecodeSingle); + LIB_FUNC("SJ3i0DXP8vg", Ajm::AjmBatchJobDecodeSplit); + LIB_FUNC("SlVIGK1Kl38", Ajm::AjmBatchJobEncode); + LIB_FUNC("TBWW4aPfWcA", Ajm::AjmBatchJobGetInfo); + LIB_FUNC("uSrXaxT+oPQ", Ajm::AjmBatchJobGetCodecInfo); + LIB_FUNC("SkEwpiu3tZg", Ajm::AjmBatchJobSetGaplessDecode); + LIB_FUNC("Esr9db8S1S0", Ajm::AjmBatchJobGetGaplessDecode); + LIB_FUNC("81HsnXFbWS4", Ajm::AjmBatchJobSetResampleParameters); + LIB_FUNC("5ldnD16rYZw", Ajm::AjmBatchJobSetResampleParametersEx); + LIB_FUNC("JkdNCocpu1M", Ajm::AjmBatchJobGetResampleInfo); + LIB_FUNC("3cAg7xN995U", Ajm::AjmBatchJobGetStatistics); + LIB_FUNC("7FZsbyVRM4U", Ajm::AjmBatchJobControl); + LIB_FUNC("jVCWcthifr8", Ajm::AjmBatchJobRun); + LIB_FUNC("Z9NVCesiP0Q", Ajm::AjmBatchJobRunSplit); + LIB_FUNC("1t3ixYNXyuc", Ajm::AjmDecAt9ParseConfigData); +} + +} // namespace LibAjm + +namespace LibAcm { + +LIB_VERSION("Acm", 1, "Acm", 1, 1); + +namespace Acm = Audio::Acm; + +LIB_DEFINE(InitAudio_1_Acm) { + LIB_FUNC("ZIXln2K3XMk", Acm::AcmContextCreate); + LIB_FUNC("jBgBjAj02R8", Acm::AcmContextDestroy); + LIB_FUNC("tW9W+CAG4FE", Acm::AcmBatchStartBuffer); + LIB_FUNC("8fe55ktlNVo", Acm::AcmBatchStartBuffers); + LIB_FUNC("RLN3gRlXJLE", Acm::AcmBatchWait); + LIB_FUNC("r7z5YQFZo+U", Acm::AcmBatchJobNotification); + LIB_FUNC("u70oWo92SYQ", Acm::AcmConvReverbSharedInput); + LIB_FUNC("9nLbWmRDpa8", Acm::AcmConvReverbSharedIr); + LIB_FUNC("KovqaFbmtsM", Acm::AcmFft); + LIB_FUNC("DR-ZCmvVR9Q", Acm::AcmIfft); + LIB_FUNC("LA4RCNKnFjg", Acm::AcmPanner); +} + +} // namespace LibAcm + +namespace LibAvPlayer { + +LIB_VERSION("AvPlayer", 1, "AvPlayer", 1, 0); + +namespace AvPlayer = Audio::AvPlayer; + +LIB_DEFINE(InitAudio_1_AvPlayer) { + LIB_FUNC("aS66RI0gGgo", AvPlayer::AvPlayerInit); + LIB_FUNC("o9eWRkSL+M4", AvPlayer::AvPlayerInitEx); + LIB_FUNC("HD1YKVU26-M", AvPlayer::AvPlayerPostInit); + LIB_FUNC("KMcEa+rHsIo", AvPlayer::AvPlayerAddSource); + LIB_FUNC("x8uvuFOPZhU", AvPlayer::AvPlayerAddSourceEx); + LIB_FUNC("hdTyRzCXQeQ", AvPlayer::AvPlayerStreamCount); + LIB_FUNC("d8FcbzfAdQw", AvPlayer::AvPlayerGetStreamInfo); + LIB_FUNC("ctTAcF5DiKQ", AvPlayer::AvPlayerGetStreamInfoEx); + LIB_FUNC("BOVKAzRmuTQ", AvPlayer::AvPlayerDisableStream); + LIB_FUNC("ODJK2sn9w4A", AvPlayer::AvPlayerEnableStream); + LIB_FUNC("buMCiJftcfw", AvPlayer::AvPlayerChangeStream); + LIB_FUNC("ET4Gr-Uu07s", AvPlayer::AvPlayerStart); + LIB_FUNC("NxSdL9t-KXk", AvPlayer::AvPlayerStartEx); + LIB_FUNC("ZC17w3vB5Lo", AvPlayer::AvPlayerStop); + LIB_FUNC("9y5v+fGN4Wk", AvPlayer::AvPlayerPause); + LIB_FUNC("w5moABNwnRY", AvPlayer::AvPlayerResume); + LIB_FUNC("k-q+xOxdc3E", AvPlayer::AvPlayerSetAvSyncMode); + LIB_FUNC("N6Oy-EjduiY", AvPlayer::AvPlayerSetAvailableBandwidth); + LIB_FUNC("OVths0xGfho", AvPlayer::AvPlayerSetLooping); + LIB_FUNC("av8Z++94rs0", AvPlayer::AvPlayerSetTrickSpeed); + LIB_FUNC("o3+RWnHViSg", AvPlayer::AvPlayerGetVideoData); + LIB_FUNC("JdksQu8pNdQ", AvPlayer::AvPlayerGetVideoDataEx); + LIB_FUNC("Wnp1OVcrZgk", AvPlayer::AvPlayerGetAudioData); + LIB_FUNC("UbQoYawOsfY", AvPlayer::AvPlayerIsActive); + LIB_FUNC("wwM99gjFf1Y", AvPlayer::AvPlayerCurrentTime); + LIB_FUNC("XC9wM+xULz8", AvPlayer::AvPlayerJumpToTime); + LIB_FUNC("NkJwDzKmIlw", AvPlayer::AvPlayerClose); + LIB_FUNC("eBTreZ84JFY", AvPlayer::AvPlayerSetLogCallback); +} + +} // namespace LibAvPlayer + +namespace LibAudiodec { + +LIB_VERSION("Audiodec", 1, "Audiodec", 1, 1); + +static constexpr int32_t AUDIODEC_ERROR_INVALID_TYPE = -2139160575; // 0x807F0001 +static constexpr int32_t AUDIODEC_ERROR_ARG = -2139160574; // 0x807F0002 +static constexpr int32_t AUDIODEC_ERROR_INVALID_PARAM_SIZE = -2139160572; // 0x807F0004 +static constexpr int32_t AUDIODEC_ERROR_INVALID_BSI_INFO_SIZE = -2139160571; // 0x807F0005 +static constexpr int32_t AUDIODEC_ERROR_INVALID_AU_INFO_SIZE = -2139160570; // 0x807F0006 +static constexpr int32_t AUDIODEC_ERROR_INVALID_PCM_ITEM_SIZE = -2139160569; // 0x807F0007 +static constexpr int32_t AUDIODEC_ERROR_INVALID_CTRL_POINTER = -2139160568; // 0x807F0008 +static constexpr int32_t AUDIODEC_ERROR_INVALID_PARAM_POINTER = -2139160567; // 0x807F0009 +static constexpr int32_t AUDIODEC_ERROR_INVALID_BSI_INFO_POINTER = -2139160566; // 0x807F000A +static constexpr int32_t AUDIODEC_ERROR_INVALID_AU_INFO_POINTER = -2139160565; // 0x807F000B +static constexpr int32_t AUDIODEC_ERROR_INVALID_PCM_ITEM_POINTER = -2139160564; // 0x807F000C +static constexpr int32_t AUDIODEC_ERROR_INVALID_AU_POINTER = -2139160563; // 0x807F000D +static constexpr int32_t AUDIODEC_ERROR_INVALID_PCM_POINTER = -2139160562; // 0x807F000E +static constexpr int32_t AUDIODEC_ERROR_INVALID_HANDLE = -2139160561; // 0x807F000F +static constexpr int32_t AUDIODEC_ERROR_INVALID_WORD_LENGTH = -2139160560; // 0x807F0010 +static constexpr int32_t AUDIODEC_ERROR_INVALID_AU_SIZE = -2139160559; // 0x807F0011 +static constexpr int32_t AUDIODEC_ERROR_INVALID_PCM_SIZE = -2139160558; // 0x807F0012 + +static constexpr uint32_t AUDIODEC_TYPE_AT9 = 0x0001; +static constexpr uint32_t AUDIODEC_TYPE_MP3 = 0x0002; +static constexpr uint32_t AUDIODEC_TYPE_M4AAC = 0x0003; + +static constexpr int32_t AUDIODEC_WORD_SZ_24BIT = 0; +static constexpr int32_t AUDIODEC_WORD_SZ_16BIT = 1; +static constexpr int32_t AUDIODEC_WORD_SZ_FLOAT = 2; + +static constexpr uint32_t AUDIODEC_AT9_MAX_FRAME_SIZE = 2048; +static constexpr uint32_t AUDIODEC_AT9_MAX_FRAME_SAMPLES = 256; +static constexpr uint32_t AUDIODEC_MP3_MAX_FRAME_SIZE = 1441; +static constexpr uint32_t AUDIODEC_MP3_MAX_FRAME_SAMPLES = 1152; +static constexpr uint32_t AUDIODEC_M4AAC_MAX_FRAME_SIZE = 4608; +static constexpr uint32_t AUDIODEC_M4AAC_MAX_FRAME_SAMPLES = 2048; + +struct AudiodecAuInfo { + uint32_t uiSize; + void* pAuAddr; + uint32_t uiAuSize; +}; + +struct AudiodecPcmItem { + uint32_t uiSize; + void* pPcmAddr; + uint32_t uiPcmSize; +}; + +struct AudiodecCtrl { + void* pParam; + void* pBsiInfo; + AudiodecAuInfo* pAuInfo; + AudiodecPcmItem* pPcmItem; +}; + +struct AudiodecParamAt9 { + uint32_t uiSize; + int32_t iBwPcm; + uint8_t uiConfigData[4]; +}; + +struct AudiodecAt9Info { + uint32_t uiSize; + uint32_t uiChannel; + uint32_t uiBitrate; + uint32_t uiSamplingRate; + uint32_t uiSuperFrameSize; + uint32_t uiFramesInSuperFrame; + uint32_t uiNextFrameSize; + uint32_t uiFrameSamples; + int32_t iResult; +}; + +struct AudiodecParamMp3 { + uint32_t uiSize; + int32_t iBwPcm; +}; + +struct AudiodecMp3Info { + uint32_t uiSize; + uint32_t uiHeader; + uint8_t ucCrc; + uint8_t ucMode; + uint8_t ucModeExtension; + uint8_t ucCopyright; + uint8_t ucOriginal; + uint8_t ucEmphasis; + uint8_t ucReserved[2]; + int32_t iResult; +}; + +struct AudiodecParamM4aac { + uint32_t uiSize; + int32_t iBwPcm; + uint32_t uiConfigNumber; + uint32_t uiSamplingFreqIndex; + uint32_t uiMaxChannels; + uint32_t uiEnableHeaac; +}; + +struct AudiodecM4aacInfo { + uint32_t uiSize; + uint32_t uiSamplingFreq; + uint32_t uiNumberOfChannels; + uint32_t uiHeaac; + int32_t iResult; +}; + +struct AudiodecDecoder { + bool used = false; + uint32_t codec_type = 0; + int32_t word_size = AUDIODEC_WORD_SZ_16BIT; + uint32_t channels = 2; + uint32_t sample_rate = 48000; + uint32_t frame_bytes = AUDIODEC_MP3_MAX_FRAME_SIZE; + uint32_t frame_samples = AUDIODEC_MP3_MAX_FRAME_SAMPLES; +}; + +static Common::Mutex g_audiodec_mutex; +static std::array g_audiodec_init_count {}; +static std::array g_audiodec_decoders {}; + +static bool audiodec_is_valid_type(uint32_t codec_type) { + switch (codec_type) { + case AUDIODEC_TYPE_AT9: + case AUDIODEC_TYPE_MP3: + case AUDIODEC_TYPE_M4AAC: return true; + default: return false; + } +} + +static uint32_t audiodec_sample_rate_from_index(uint32_t index) { + static constexpr std::array rates = {96000, 88200, 64000, 48000, 44100, 32000, + 24000, 22050, 16000, 12000, 11025, 8000}; + return index < rates.size() ? rates[index] : 48000; +} + +static int32_t audiodec_word_size_bytes(int32_t word_size) { + switch (word_size) { + case AUDIODEC_WORD_SZ_24BIT: return 4; + case AUDIODEC_WORD_SZ_16BIT: return 2; + case AUDIODEC_WORD_SZ_FLOAT: return 4; + default: return 0; + } +} + +static int32_t audiodec_validate_ctrl(const AudiodecCtrl* ctrl, uint32_t codec_type, bool decode) { + if (ctrl == nullptr) { + return AUDIODEC_ERROR_INVALID_CTRL_POINTER; + } + if (ctrl->pParam == nullptr) { + return AUDIODEC_ERROR_INVALID_PARAM_POINTER; + } + if (ctrl->pBsiInfo == nullptr) { + return AUDIODEC_ERROR_INVALID_BSI_INFO_POINTER; + } + if (ctrl->pAuInfo == nullptr) { + return AUDIODEC_ERROR_INVALID_AU_INFO_POINTER; + } + if (ctrl->pPcmItem == nullptr) { + return AUDIODEC_ERROR_INVALID_PCM_ITEM_POINTER; + } + if (ctrl->pAuInfo->uiSize != sizeof(AudiodecAuInfo)) { + return AUDIODEC_ERROR_INVALID_AU_INFO_SIZE; + } + if (ctrl->pPcmItem->uiSize != sizeof(AudiodecPcmItem)) { + return AUDIODEC_ERROR_INVALID_PCM_ITEM_SIZE; + } + if (decode && ctrl->pAuInfo->pAuAddr == nullptr) { + return AUDIODEC_ERROR_INVALID_AU_POINTER; + } + if (decode && ctrl->pPcmItem->pPcmAddr == nullptr) { + return AUDIODEC_ERROR_INVALID_PCM_POINTER; + } + + switch (codec_type) { + case AUDIODEC_TYPE_AT9: { + auto* param = static_cast(ctrl->pParam); + auto* info = static_cast(ctrl->pBsiInfo); + if (param->uiSize != sizeof(AudiodecParamAt9)) { + return AUDIODEC_ERROR_INVALID_PARAM_SIZE; + } + if (info->uiSize != sizeof(AudiodecAt9Info)) { + return AUDIODEC_ERROR_INVALID_BSI_INFO_SIZE; + } + if (audiodec_word_size_bytes(param->iBwPcm) == 0) { + return AUDIODEC_ERROR_INVALID_WORD_LENGTH; + } + break; + } + case AUDIODEC_TYPE_MP3: { + auto* param = static_cast(ctrl->pParam); + auto* info = static_cast(ctrl->pBsiInfo); + if (param->uiSize != sizeof(AudiodecParamMp3)) { + return AUDIODEC_ERROR_INVALID_PARAM_SIZE; + } + if (info->uiSize != sizeof(AudiodecMp3Info)) { + return AUDIODEC_ERROR_INVALID_BSI_INFO_SIZE; + } + if (audiodec_word_size_bytes(param->iBwPcm) == 0) { + return AUDIODEC_ERROR_INVALID_WORD_LENGTH; + } + break; + } + case AUDIODEC_TYPE_M4AAC: { + auto* param = static_cast(ctrl->pParam); + auto* info = static_cast(ctrl->pBsiInfo); + if (param->uiSize < sizeof(AudiodecParamM4aac)) { + return AUDIODEC_ERROR_INVALID_PARAM_SIZE; + } + if (info->uiSize != sizeof(AudiodecM4aacInfo)) { + return AUDIODEC_ERROR_INVALID_BSI_INFO_SIZE; + } + if (audiodec_word_size_bytes(param->iBwPcm) == 0) { + return AUDIODEC_ERROR_INVALID_WORD_LENGTH; + } + break; + } + default: return AUDIODEC_ERROR_INVALID_TYPE; + } + + if (decode && ctrl->pAuInfo->uiAuSize == 0) { + return AUDIODEC_ERROR_INVALID_AU_SIZE; + } + if (decode && ctrl->pPcmItem->uiPcmSize == 0) { + return AUDIODEC_ERROR_INVALID_PCM_SIZE; + } + + return 0; +} + +static void audiodec_fill_info(AudiodecCtrl* ctrl, const AudiodecDecoder& decoder) { + switch (decoder.codec_type) { + case AUDIODEC_TYPE_AT9: { + auto* info = static_cast(ctrl->pBsiInfo); + info->uiChannel = decoder.channels; + info->uiBitrate = 0; + info->uiSamplingRate = decoder.sample_rate; + info->uiSuperFrameSize = decoder.frame_bytes; + info->uiFramesInSuperFrame = 1; + info->uiNextFrameSize = decoder.frame_bytes; + info->uiFrameSamples = decoder.frame_samples; + info->iResult = 0; + break; + } + case AUDIODEC_TYPE_MP3: { + auto* info = static_cast(ctrl->pBsiInfo); + info->uiHeader = 0; + info->ucCrc = 0; + info->ucMode = decoder.channels == 1 ? 3 : 0; + info->ucModeExtension = 0; + info->ucCopyright = 0; + info->ucOriginal = 0; + info->ucEmphasis = 0; + info->ucReserved[0] = 0; + info->ucReserved[1] = 0; + info->iResult = 0; + break; + } + case AUDIODEC_TYPE_M4AAC: { + auto* info = static_cast(ctrl->pBsiInfo); + info->uiSamplingFreq = decoder.sample_rate; + info->uiNumberOfChannels = decoder.channels; + info->uiHeaac = 0; + info->iResult = 0; + break; + } + default: break; + } +} + +static int32_t KYTY_SYSV_ABI AudiodecInitLibrary(uint32_t codec_type) { + PRINT_NAME(); + + if (!audiodec_is_valid_type(codec_type)) { + return AUDIODEC_ERROR_INVALID_TYPE; + } + + Common::LockGuard lock(g_audiodec_mutex); + g_audiodec_init_count[codec_type]++; + + return 0; +} + +static int32_t KYTY_SYSV_ABI AudiodecTermLibrary(uint32_t codec_type) { + PRINT_NAME(); + + if (!audiodec_is_valid_type(codec_type)) { + return AUDIODEC_ERROR_INVALID_TYPE; + } + + Common::LockGuard lock(g_audiodec_mutex); + if (g_audiodec_init_count[codec_type] > 0) { + g_audiodec_init_count[codec_type]--; + } + + return 0; +} + +static int32_t KYTY_SYSV_ABI AudiodecCreateDecoder(AudiodecCtrl* ctrl, uint32_t codec_type) { + PRINT_NAME(); + + if (!audiodec_is_valid_type(codec_type)) { + return AUDIODEC_ERROR_INVALID_TYPE; + } + + int32_t ret = audiodec_validate_ctrl(ctrl, codec_type, false); + if (ret < 0) { + return ret; + } + + Common::LockGuard lock(g_audiodec_mutex); + if (g_audiodec_init_count[codec_type] == 0) { + return AUDIODEC_ERROR_ARG; + } + + for (uint32_t i = 0; i < g_audiodec_decoders.size(); i++) { + if (!g_audiodec_decoders[i].used) { + auto& decoder = g_audiodec_decoders[i]; + decoder = {}; + decoder.used = true; + decoder.codec_type = codec_type; + decoder.sample_rate = 48000; + + switch (codec_type) { + case AUDIODEC_TYPE_AT9: { + auto* param = static_cast(ctrl->pParam); + decoder.word_size = param->iBwPcm; + decoder.channels = 2; + decoder.frame_bytes = AUDIODEC_AT9_MAX_FRAME_SIZE; + decoder.frame_samples = AUDIODEC_AT9_MAX_FRAME_SAMPLES; + break; + } + case AUDIODEC_TYPE_MP3: { + auto* param = static_cast(ctrl->pParam); + decoder.word_size = param->iBwPcm; + decoder.channels = 2; + decoder.frame_bytes = AUDIODEC_MP3_MAX_FRAME_SIZE; + decoder.frame_samples = AUDIODEC_MP3_MAX_FRAME_SAMPLES; + break; + } + case AUDIODEC_TYPE_M4AAC: { + auto* param = static_cast(ctrl->pParam); + decoder.word_size = param->iBwPcm; + decoder.channels = param->uiMaxChannels == 0 ? 2 : param->uiMaxChannels; + if (decoder.channels > 8) { + decoder.channels = 8; + } + decoder.sample_rate = + audiodec_sample_rate_from_index(param->uiSamplingFreqIndex); + decoder.frame_bytes = AUDIODEC_M4AAC_MAX_FRAME_SIZE; + decoder.frame_samples = AUDIODEC_M4AAC_MAX_FRAME_SAMPLES; + break; + } + } + + audiodec_fill_info(ctrl, decoder); + + return static_cast(i + 1); + } + } + + return AUDIODEC_ERROR_ARG; +} + +static int32_t KYTY_SYSV_ABI AudiodecDeleteDecoder(int32_t handle) { + PRINT_NAME(); + + if (handle <= 0 || static_cast(handle) > g_audiodec_decoders.size()) { + return AUDIODEC_ERROR_INVALID_HANDLE; + } + + Common::LockGuard lock(g_audiodec_mutex); + auto& decoder = g_audiodec_decoders[static_cast(handle - 1)]; + if (!decoder.used) { + return AUDIODEC_ERROR_INVALID_HANDLE; + } + + decoder = {}; + + return 0; +} + +static int32_t KYTY_SYSV_ABI AudiodecDecode(int32_t handle, AudiodecCtrl* ctrl) { + PRINT_NAME(); + + if (handle <= 0 || static_cast(handle) > g_audiodec_decoders.size()) { + return AUDIODEC_ERROR_INVALID_HANDLE; + } + + Common::LockGuard lock(g_audiodec_mutex); + auto& decoder = g_audiodec_decoders[static_cast(handle - 1)]; + if (!decoder.used) { + return AUDIODEC_ERROR_INVALID_HANDLE; + } + + int32_t ret = audiodec_validate_ctrl(ctrl, decoder.codec_type, true); + if (ret < 0) { + return ret; + } + + const uint32_t word_bytes = static_cast(audiodec_word_size_bytes(decoder.word_size)); + const uint32_t wanted_pcm = decoder.frame_samples * decoder.channels * word_bytes; + const uint32_t produced_pcm = + ctrl->pPcmItem->uiPcmSize < wanted_pcm ? ctrl->pPcmItem->uiPcmSize : wanted_pcm; + const uint32_t consumed_data = ctrl->pAuInfo->uiAuSize < decoder.frame_bytes + ? ctrl->pAuInfo->uiAuSize + : decoder.frame_bytes; + + std::memset(ctrl->pPcmItem->pPcmAddr, 0, produced_pcm); + ctrl->pPcmItem->uiPcmSize = produced_pcm; + ctrl->pAuInfo->uiAuSize = consumed_data; + audiodec_fill_info(ctrl, decoder); + + return 0; +} + +static int32_t KYTY_SYSV_ABI AudiodecClearContext(int32_t handle) { + PRINT_NAME(); + + if (handle <= 0 || static_cast(handle) > g_audiodec_decoders.size()) { + return AUDIODEC_ERROR_INVALID_HANDLE; + } + + Common::LockGuard lock(g_audiodec_mutex); + if (!g_audiodec_decoders[static_cast(handle - 1)].used) { + return AUDIODEC_ERROR_INVALID_HANDLE; + } + + return 0; +} + +LIB_DEFINE(InitAudio_1_Audiodec) { + LIB_FUNC("VjhsmxpcezI", AudiodecInitLibrary); + LIB_FUNC("h5jSB2QIDV0", AudiodecTermLibrary); + LIB_FUNC("O3f1sLMWRvs", AudiodecCreateDecoder); + LIB_FUNC("Tp+ZEy69mLk", AudiodecDeleteDecoder); + LIB_FUNC("KHXHMDLkILw", AudiodecDecode); + LIB_FUNC("6Vf9WTLDoss", AudiodecClearContext); +} + +} // namespace LibAudiodec + +namespace LibAudio3d { + +LIB_VERSION("Audio3d", 1, "Audio3d", 1, 1); + +namespace Audio3d = Audio::Audio3d; + +LIB_DEFINE(InitAudio_1_Audio3d) { + LIB_FUNC("UmCvjSmuZIw", Audio3d::Audio3dInitialize); + LIB_FUNC("Im+jOoa5WAI", Audio3d::Audio3dGetDefaultOpenParameters); + LIB_FUNC("XeDDK0xJWQA", Audio3d::Audio3dPortOpen); + LIB_FUNC("Yq9bfUQ0uJg", Audio3d::Audio3dPortSetAttribute); + LIB_FUNC("YaaDbDwKpFM", Audio3d::Audio3dPortGetQueueLevel); + LIB_FUNC("lw0qrdSjZt8", Audio3d::Audio3dPortAdvance); + LIB_FUNC("VEVhZ9qd4ZY", Audio3d::Audio3dPortPush); +} + +} // namespace LibAudio3d + +namespace LibNgs2 { + +LIB_VERSION("Ngs2", 1, "Ngs2", 1, 1); + +namespace Ngs2 = Audio::Ngs2; + +LIB_DEFINE(InitAudio_1_Ngs2) { + LIB_FUNC("AQkj7C0f3PY", Ngs2::Ngs2SystemResetOption); + LIB_FUNC("pgFAiLR5qT4", Ngs2::Ngs2SystemQueryBufferSize); + LIB_FUNC("koBbCMvOKWw", Ngs2::Ngs2SystemCreate); + LIB_FUNC("mPYgU4oYpuY", Ngs2::Ngs2SystemCreateWithAllocator); + LIB_FUNC("u-WrYDaJA3k", Ngs2::Ngs2SystemDestroy); + LIB_FUNC("U546k6orxQo", Ngs2::Ngs2RackCreateWithAllocator); + LIB_FUNC("cLV4aiT9JpA", Ngs2::Ngs2RackCreate); + LIB_FUNC("0eFLVCfWVds", Ngs2::Ngs2RackQueryBufferSize); + LIB_FUNC("lCqD7oycmIM", Ngs2::Ngs2RackDestroy); + LIB_FUNC("MwmHz8pAdAo", Ngs2::Ngs2RackGetVoiceHandle); + LIB_FUNC("MzTa7VLjogY", Ngs2::Ngs2RackLock); + LIB_FUNC("++YZ7P9e87U", Ngs2::Ngs2RackUnlock); + LIB_FUNC("uu94irFOGpA", Ngs2::Ngs2VoiceControl); + LIB_FUNC("AbYvTOZ8Pts", Ngs2::Ngs2VoiceRunCommands); + LIB_FUNC("-TOuuAQ-buE", Ngs2::Ngs2VoiceGetState); + LIB_FUNC("rEh728kXk3w", Ngs2::Ngs2VoiceGetStateFlags); + LIB_FUNC("i0VnXM-C9fc", Ngs2::Ngs2SystemRender); + LIB_FUNC("l4Q2dWEH6UM", Ngs2::Ngs2SystemSetGrainSamples); + LIB_FUNC("hyVLT2VlOYk", Ngs2::Ngs2ParseWaveformData); + LIB_FUNC("3pCNbVM11UA", Ngs2::Ngs2CalcWaveformBlock); + LIB_FUNC("xa8oL9dmXkM", Ngs2::Ngs2PanInit); + LIB_FUNC("gbMKV+8Enuo", Ngs2::Ngs2PanGetVolumeMatrix); + LIB_FUNC("7Lcfo8SmpsU", Ngs2::Ngs2GeomResetListenerParam); + LIB_FUNC("0lbbayqDNoE", Ngs2::Ngs2GeomResetSourceParam); + LIB_FUNC("1WsleK-MTkE", Ngs2::Ngs2GeomCalcListener); + LIB_FUNC("eF8yRCC6W64", Ngs2::Ngs2GeomApply); +} + +} // namespace LibNgs2 + +LIB_DEFINE(InitAudio_1) { + LibAudioOut::InitAudio_1_AudioOut(s); + LibAudioOut2::InitAudio_1_AudioOut2(s); + LibAudioIn::InitAudio_1_AudioIn(s); + LibVoiceQoS::InitAudio_1_VoiceQoS(s); + LibVoice::InitAudio_1_Voice(s); + LibAjm::InitAudio_1_Ajm(s); + LibAcm::InitAudio_1_Acm(s); + LibAvPlayer::InitAudio_1_AvPlayer(s); + LibAudiodec::InitAudio_1_Audiodec(s); + LibAudio3d::InitAudio_1_Audio3d(s); + LibNgs2::InitAudio_1_Ngs2(s); + LibAudioPropagation::InitAudio_1_AudioPropagation(s); +} + +} // namespace Libs diff --git a/src/libs/libAudio2.cpp b/src/libs/libAudio2.cpp new file mode 100644 index 000000000..04fac4d85 --- /dev/null +++ b/src/libs/libAudio2.cpp @@ -0,0 +1,929 @@ +#include "common/assert.h" +#include "common/common.h" +#include "common/logging/log.h" +#include "common/threads.h" +#include "kernel/pthread.h" +#include "libs/audio.h" +#include "libs/audio_internal.h" +#include "libs/errno.h" +#include "libs/libs.h" + +#include +#include +#include +#include +#include + +namespace Libs::Audio { + +namespace { + +constexpr int AUDIO_OUT_PORT_TYPE_MAIN = 0; +constexpr int AUDIO_OUT_PORT_TYPE_BGM = 1; +constexpr int AUDIO_OUT_PORT_TYPE_VOICE = 2; +constexpr int AUDIO_OUT_PORT_TYPE_PERSONAL = 3; +constexpr int AUDIO_OUT_PORT_TYPE_PADSPK = 4; +constexpr int AUDIO_OUT_PORT_TYPE_VIBRATION = 10; +constexpr int AUDIO_OUT_PORT_TYPE_AUX = 127; + +} // namespace + +namespace AudioOut2 { + +LIB_NAME("AudioOut2", "AudioOut"); + +struct AudioOut2ContextParam { + uint32_t max_ports; + uint32_t max_object_ports; + uint32_t guarantee_object_ports; + uint32_t queue_depth; + uint32_t num_grains; + uint32_t flags; + uint32_t reserved[10]; +}; + +struct AudioOut2PortParam { + uint16_t port_type; + uint16_t pad; + uint32_t data_format; + uint32_t sampling_freq; + uint32_t flags; + AudioOut2UserHandle user_handle; + uint32_t reserved[10]; +}; + +struct AudioOut2Attribute { + uint32_t attribute_id; + int32_t reserved; + const void* value; + size_t value_size; +}; + +struct AudioOut2Pcm { + const void* data; +}; + +struct AudioOut2Position { + float x; + float y; + float z; +}; + +struct AudioOut2PortState { + uint16_t output; + uint8_t num_channels; + uint8_t pad1; + int16_t volume; + uint16_t reroute_counter; + uint32_t flags; + uint32_t pad2; + uint64_t reserved[6]; +}; + +struct AudioOut2SystemState { + float loudness; + uint32_t pad; + uint64_t reserved[7]; +}; + +struct AudioOut2SpeakerAngle { + int16_t azimuth; + int16_t elevation; +}; + +struct AudioOut2SpeakerInfo { + uint8_t type; + uint8_t pad1; + int16_t pad2; + uint32_t available_bits; + uint32_t flags; + uint32_t pad3; + AudioOut2SpeakerAngle speaker_angle[16]; +}; + +struct AudioOut2SystemDebugStateParam { + uint32_t debug_state_id; + int32_t reserved; + void* param; + size_t param_size; +}; + +struct AudioOut2MasteringParamsHeader { + uint32_t params_id; +}; + +struct AudioOut2MasteringStatesHeader { + uint32_t states_id; +}; + +struct AudioOut2MasteringStatesDescriptor { + uint32_t id; + uint32_t size; +}; + +static constexpr uint32_t AUDIO_OUT2_MASTERING_MAX_SYSTEM_CHANNELS = 8; +static constexpr uint32_t AUDIO_OUT2_MASTERING_COMPRESSOR_BANDS = 3; +static constexpr uint32_t AUDIO_OUT2_MASTERING_COMPRESSOR_BANDS_V2 = 4; + +struct AudioOut2MasteringCompressorStates { + AudioOut2MasteringStatesDescriptor descriptor; + uint32_t reserved[2]; + float input_rms[AUDIO_OUT2_MASTERING_COMPRESSOR_BANDS] + [AUDIO_OUT2_MASTERING_MAX_SYSTEM_CHANNELS]; + float compression_coeff[AUDIO_OUT2_MASTERING_COMPRESSOR_BANDS] + [AUDIO_OUT2_MASTERING_MAX_SYSTEM_CHANNELS]; +}; + +struct AudioOut2MasteringCompressorStatesV2 { + AudioOut2MasteringStatesDescriptor descriptor; + uint32_t reserved[2]; + float input_rms[AUDIO_OUT2_MASTERING_COMPRESSOR_BANDS_V2] + [AUDIO_OUT2_MASTERING_MAX_SYSTEM_CHANNELS]; + float compression_coeff[AUDIO_OUT2_MASTERING_COMPRESSOR_BANDS_V2] + [AUDIO_OUT2_MASTERING_MAX_SYSTEM_CHANNELS]; +}; + +struct AudioOut2MasteringLimiterStates { + AudioOut2MasteringStatesDescriptor descriptor; + uint32_t reserved[2]; + float input_peak[AUDIO_OUT2_MASTERING_MAX_SYSTEM_CHANNELS]; + float output_peak[AUDIO_OUT2_MASTERING_MAX_SYSTEM_CHANNELS]; + float gain_peak[AUDIO_OUT2_MASTERING_MAX_SYSTEM_CHANNELS]; +}; + +struct AudioOut2MasteringStates { + AudioOut2MasteringStatesHeader states_header; + uint32_t reserved[3]; + AudioOut2MasteringCompressorStates compressor_states; + AudioOut2MasteringLimiterStates limiter_states; +}; + +struct AudioOut2MasteringStatesV2 { + AudioOut2MasteringStatesHeader states_header; + uint32_t reserved[3]; + AudioOut2MasteringCompressorStatesV2 compressor_states; + AudioOut2MasteringLimiterStates limiter_states; +}; + +static std::atomic_uint64_t g_audioout2_next_context {1}; +static std::atomic_uint64_t g_audioout2_next_port {1}; +static std::atomic_uint64_t g_audioout2_next_user {1}; + +struct AudioOut2ContextState { + bool used = false; + AudioOut2ContextHandle handle = 0; + uint32_t queue_depth = 4; + uint32_t queued = 0; + uint32_t num_grains = 512; + uint64_t last_update = 0; +}; + +struct AudioOut2PortStateEntry { + bool used = false; + AudioOut2PortHandle handle = 0; + AudioOut2ContextHandle context = 0; + uint16_t port_type = 0; + uint32_t data_format = 0; + uint32_t sampling_freq = 48000; + uint32_t samples_num = 512; + AudioInternal::Format audio_format = AudioInternal::Format::Unknown; + int audio_handle = 0; + const void* pcm_data = nullptr; +}; + +struct AudioOut2SpeakerArrayState { + bool used = false; + uint32_t num_speakers = 0; + uint8_t is_3d = 0; + uint8_t is_ambisonics = 0; + AudioOut2SpeakerArrayHandle handle = nullptr; +}; + +struct AudioOut2LatencyState { + bool used = false; + uint32_t user_id = 0; + uint32_t output = 0; + uint32_t latency_us = 0; +}; + +static Common::Mutex g_audioout2_context_mutex; +static std::array g_audioout2_contexts; +static Common::Mutex g_audioout2_port_mutex; +static std::array g_audioout2_ports; +static Common::Mutex g_audioout2_speaker_array_mutex; +static std::array g_audioout2_speaker_arrays; +static Common::Mutex g_audioout2_latency_mutex; +static std::array g_audioout2_latencies; + +static constexpr int AUDIO_OUT2_ERROR_NOT_READY = -2144960504; /* 0x80268008 */ +static constexpr int AUDIO_OUT2_ERROR_PORT_FULL = -2144960494; /* 0x80268012 */ +static constexpr int AUDIO_OUT2_ERROR_INVALID_PARAM = -2144960511; /* 0x80268001 */ +static constexpr int AUDIO_OUT2_ERROR_MASTERING_INVALID_API_PARAM = -2144959999; /* 0x80268201 */ +static constexpr int AUDIO_OUT2_ERROR_MASTERING_INVALID_STATES_ID = -2144959996; /* 0x80268204 */ +static constexpr uint32_t AUDIO_OUT2_PORT_ATTRIBUTE_ID_PCM = 0; +static constexpr uint32_t AUDIO_OUT2_MASTERING_OUTPUT_RECORDING = 2; +static constexpr uint32_t AUDIO_OUT2_MASTERING_STATES_ID_DEFAULT = 1; +static constexpr uint32_t AUDIO_OUT2_MASTERING_STATES_ID_V2 = 2; +static constexpr uint32_t AUDIO_OUT2_MASTERING_STATES_STRUCT_ID_COMPRESSOR_DEFAULT = 0x01010001; +static constexpr uint32_t AUDIO_OUT2_MASTERING_STATES_STRUCT_ID_COMPRESSOR_V2 = 0x01020001; +static constexpr uint32_t AUDIO_OUT2_MASTERING_STATES_STRUCT_ID_LIMITER_DEFAULT = 0x01010003; + +static AudioOut2ContextState* audioout2_find_context_locked(AudioOut2ContextHandle ctx) { + for (auto& state: g_audioout2_contexts) { + if (state.used && state.handle == ctx) { + return &state; + } + } + + return nullptr; +} + +static uint32_t audioout2_grain_micros(uint32_t grains) { + const auto sample_count = (grains == 0 ? 512u : grains); + return std::max((sample_count * 1000000u) / 48000u, 1000u); +} + +static uint8_t audioout2_data_format_channels(uint32_t data_format) { + const auto channels = (data_format >> 8u) & 0xffu; + return static_cast(channels == 0 ? 2u : std::min(channels, 16u)); +} + +static AudioInternal::Format audioout2_data_format_to_audio_format(uint32_t data_format) { + const auto channels = audioout2_data_format_channels(data_format); + const auto data_type = data_format & 0x7fu; + const auto is_std = (data_format & 0x80u) != 0; + + switch (data_type) { + case 0: + switch (channels) { + case 1: return AudioInternal::Format::FloatMono; + case 2: return AudioInternal::Format::FloatStereo; + case 8: + return is_std ? AudioInternal::Format::Float8ChStd + : AudioInternal::Format::Float8Ch; + default: break; + } + break; + case 1: + switch (channels) { + case 1: return AudioInternal::Format::Signed16bitMono; + case 2: return AudioInternal::Format::Signed16bitStereo; + case 8: + return is_std ? AudioInternal::Format::Signed16bit8ChStd + : AudioInternal::Format::Signed16bit8Ch; + default: break; + } + break; + default: break; + } + + return AudioInternal::Format::Unknown; +} + +static int audioout2_port_type_to_audio_out_type(uint16_t port_type) { + switch (port_type & 0xffu) { + case 0: return AUDIO_OUT_PORT_TYPE_MAIN; + case 1: return AUDIO_OUT_PORT_TYPE_BGM; + case 2: return AUDIO_OUT_PORT_TYPE_VOICE; + case 3: return AUDIO_OUT_PORT_TYPE_PADSPK; + case 4: return AUDIO_OUT_PORT_TYPE_PERSONAL; + case 5: return AUDIO_OUT_PORT_TYPE_AUX; + case 6: return AUDIO_OUT_PORT_TYPE_VIBRATION; + default: return AUDIO_OUT_PORT_TYPE_MAIN; + } +} + +static bool audioout2_port_type_is_object(uint16_t port_type) { + return (port_type & 0xff00u) == 0x0100u; +} + +static void audioout2_update_context_locked(AudioOut2ContextState* state) { + if (state == nullptr) { + return; + } + + const auto now = LibKernel::KernelGetProcessTime(); + if (state->last_update == 0 || state->queued == 0) { + state->last_update = now; + return; + } + + const auto grain_micros = static_cast(audioout2_grain_micros(state->num_grains)); + if (now <= state->last_update || grain_micros == 0) { + return; + } + + const auto elapsed = now - state->last_update; + const auto drained = std::min(state->queued, elapsed / grain_micros); + if (drained == 0) { + return; + } + + state->queued -= static_cast(drained); + state->last_update += drained * grain_micros; + if (state->queued == 0) { + state->last_update = now; + } +} + +static AudioOut2PortStateEntry* audioout2_find_port_locked(AudioOut2PortHandle port) { + for (auto& state: g_audioout2_ports) { + if (state.used && state.handle == port) { + return &state; + } + } + + return nullptr; +} + +static uint32_t audioout2_context_grains(AudioOut2ContextHandle ctx) { + uint32_t samples_num = 512; + + g_audioout2_context_mutex.Lock(); + if (auto* state = audioout2_find_context_locked(ctx); state != nullptr) { + samples_num = (state->num_grains == 0 ? 512u : state->num_grains); + } + g_audioout2_context_mutex.Unlock(); + + return samples_num; +} + +static void audioout2_queue_context_audio(AudioOut2ContextHandle ctx) { + std::vector params; + params.reserve(AudioInternal::OUT_PORTS_MAX); + + g_audioout2_port_mutex.Lock(); + for (const auto& state: g_audioout2_ports) { + if (state.used && state.context == ctx && state.audio_handle > 0 && + state.pcm_data != nullptr && params.size() < AudioInternal::OUT_PORTS_MAX) { + params.push_back(AudioInternal::OutputParam {state.audio_handle, state.pcm_data}); + } + } + g_audioout2_port_mutex.Unlock(); + + if (params.empty()) { + return; + } + + (void)AudioInternal::AudioOutOutputs(params.data(), static_cast(params.size())); +} + +static void audioout2_close_audio_handle(int audio_handle) { + if (audio_handle > 0) { + AudioInternal::AudioOutClose(audio_handle); + } +} + +int KYTY_SYSV_ABI AudioOut2Initialize() { + PRINT_NAME(); + + return OK; +} + +int KYTY_SYSV_ABI AudioOut2ContextResetParam(AudioOut2ContextParam* params) { + PRINT_NAME(); + + EXIT_NOT_IMPLEMENTED(params == nullptr); + + std::memset(params, 0, sizeof(AudioOut2ContextParam)); + params->max_ports = 256; + params->max_object_ports = 256; + params->guarantee_object_ports = 0; + params->queue_depth = 4; + params->num_grains = 512; + params->flags = 1; + + return OK; +} + +int KYTY_SYSV_ABI AudioOut2ContextQueryMemory(const AudioOut2ContextParam* params, + size_t* memory_size) { + PRINT_NAME(); + + EXIT_NOT_IMPLEMENTED(params == nullptr); + EXIT_NOT_IMPLEMENTED(memory_size == nullptr); + + const auto queue_depth = (params->queue_depth == 0 ? 4u : params->queue_depth); + *memory_size = 0x10000u + static_cast(queue_depth) * 0x590u; + + LOGF("\t memory_size = 0x%016" PRIx64 "\n", static_cast(*memory_size)); + + return OK; +} + +int KYTY_SYSV_ABI AudioOut2ContextCreate(const AudioOut2ContextParam* params, void* buffer, + size_t buffer_size, AudioOut2ContextHandle* ctx) { + PRINT_NAME(); + + EXIT_NOT_IMPLEMENTED(params == nullptr); + EXIT_NOT_IMPLEMENTED(ctx == nullptr); + + *ctx = g_audioout2_next_context.fetch_add(1, std::memory_order_relaxed); + + g_audioout2_context_mutex.Lock(); + auto* state = audioout2_find_context_locked(0); + if (state == nullptr) { + for (auto& candidate: g_audioout2_contexts) { + if (!candidate.used) { + state = &candidate; + break; + } + } + } + EXIT_NOT_IMPLEMENTED(state == nullptr); + *state = AudioOut2ContextState {}; + state->used = true; + state->handle = *ctx; + state->queue_depth = (params->queue_depth == 0 ? 4u : params->queue_depth); + state->queued = 0; + state->num_grains = (params->num_grains == 0 ? 512u : params->num_grains); + state->last_update = LibKernel::KernelGetProcessTime(); + g_audioout2_context_mutex.Unlock(); + + LOGF("\t buffer = 0x%016" PRIx64 "\n" + "\t buffer_size = 0x%016" PRIx64 "\n" + "\t ctx = 0x%016" PRIx64 "\n" + "\t queue_depth = %" PRIu32 ", num_grains = %" PRIu32 "\n", + reinterpret_cast(buffer), static_cast(buffer_size), *ctx, + state->queue_depth, state->num_grains); + + return OK; +} + +int KYTY_SYSV_ABI AudioOut2ContextDestroy(AudioOut2ContextHandle ctx) { + PRINT_NAME(); + LOGF("\t ctx = 0x%016" PRIx64 "\n", ctx); + + std::array audio_handles {}; + size_t audio_handles_num = 0; + + g_audioout2_port_mutex.Lock(); + for (auto& port_state: g_audioout2_ports) { + if (port_state.used && port_state.context == ctx) { + if (port_state.audio_handle > 0 && audio_handles_num < audio_handles.size()) { + audio_handles[audio_handles_num++] = port_state.audio_handle; + } + port_state = AudioOut2PortStateEntry {}; + } + } + g_audioout2_port_mutex.Unlock(); + + for (size_t i = 0; i < audio_handles_num; i++) { + audioout2_close_audio_handle(audio_handles[i]); + } + + g_audioout2_context_mutex.Lock(); + if (auto* state = audioout2_find_context_locked(ctx); state != nullptr) { + *state = AudioOut2ContextState {}; + } + g_audioout2_context_mutex.Unlock(); + + return OK; +} + +int KYTY_SYSV_ABI AudioOut2ContextSetAttributes(AudioOut2ContextHandle ctx, + const AudioOut2Attribute* attributes, + uint32_t num) { + PRINT_NAME(); + LOGF("\t ctx = 0x%016" PRIx64 ", num = %" PRIu32 "\n", ctx, num); + EXIT_NOT_IMPLEMENTED(num != 0 && attributes == nullptr); + return OK; +} + +int KYTY_SYSV_ABI AudioOut2ContextAdvance(AudioOut2ContextHandle ctx) { + g_audioout2_context_mutex.Lock(); + if (auto* state = audioout2_find_context_locked(ctx); state != nullptr) { + audioout2_update_context_locked(state); + } + g_audioout2_context_mutex.Unlock(); + + return OK; +} + +int KYTY_SYSV_ABI AudioOut2ContextPush(AudioOut2ContextHandle ctx, uint32_t blocking) { + uint32_t sleep_micros = audioout2_grain_micros(512); + + for (;;) { + g_audioout2_context_mutex.Lock(); + if (auto* state = audioout2_find_context_locked(ctx); state != nullptr) { + audioout2_update_context_locked(state); + sleep_micros = audioout2_grain_micros(state->num_grains); + if (state->queued < state->queue_depth) { + if (state->queued == 0) { + state->last_update = LibKernel::KernelGetProcessTime(); + } + state->queued++; + g_audioout2_context_mutex.Unlock(); + audioout2_queue_context_audio(ctx); + return OK; + } + } + g_audioout2_context_mutex.Unlock(); + + if (blocking == 0) { + return AUDIO_OUT2_ERROR_NOT_READY; + } + + Common::Thread::SleepMicro(sleep_micros); + } +} + +int KYTY_SYSV_ABI AudioOut2ContextGetQueueLevel(AudioOut2ContextHandle ctx, uint32_t* queue_level, + uint32_t* available_queues) { + if (queue_level != nullptr) { + *queue_level = 0; + } + if (available_queues != nullptr) { + *available_queues = 4; + } + + g_audioout2_context_mutex.Lock(); + if (auto* state = audioout2_find_context_locked(ctx); state != nullptr) { + audioout2_update_context_locked(state); + if (queue_level != nullptr) { + *queue_level = state->queued; + } + if (available_queues != nullptr) { + *available_queues = + (state->queued < state->queue_depth ? state->queue_depth - state->queued : 0); + } + } + g_audioout2_context_mutex.Unlock(); + + return OK; +} + +int KYTY_SYSV_ABI AudioOut2PortCreate(AudioOut2ContextHandle ctx, const AudioOut2PortParam* params, + AudioOut2PortHandle* port) { + EXIT_NOT_IMPLEMENTED(params == nullptr); + EXIT_NOT_IMPLEMENTED(port == nullptr); + + const auto next_port = g_audioout2_next_port.fetch_add(1, std::memory_order_relaxed); + + g_audioout2_port_mutex.Lock(); + auto* port_state = audioout2_find_port_locked(0); + if (port_state == nullptr) { + for (auto& candidate: g_audioout2_ports) { + if (!candidate.used) { + port_state = &candidate; + break; + } + } + } + g_audioout2_port_mutex.Unlock(); + + if (next_port > g_audioout2_ports.size() || port_state == nullptr) { + return AUDIO_OUT2_ERROR_PORT_FULL; + } + + *port = next_port; + + const auto samples_num = audioout2_context_grains(ctx); + const auto audio_format = audioout2_data_format_to_audio_format(params->data_format); + const auto audio_type = audioout2_port_type_to_audio_out_type(params->port_type); + int audio_handle = 0; + + if (audio_format != AudioInternal::Format::Unknown && + !audioout2_port_type_is_object(params->port_type)) { + audio_handle = AudioInternal::AudioOutOpen(audio_type, samples_num, params->sampling_freq, + audio_format); + } + + g_audioout2_port_mutex.Lock(); + *port_state = AudioOut2PortStateEntry {}; + port_state->used = true; + port_state->handle = *port; + port_state->context = ctx; + port_state->port_type = params->port_type; + port_state->data_format = params->data_format; + port_state->sampling_freq = params->sampling_freq; + port_state->samples_num = samples_num; + port_state->audio_format = audio_format; + port_state->audio_handle = audio_handle; + g_audioout2_port_mutex.Unlock(); + + if (next_port <= 16 || (next_port % 600) == 0) { + PRINT_NAME(); + LOGF("\t ctx = 0x%016" PRIx64 "\n" + "\t port = 0x%016" PRIx64 "\n" + "\t port_type = %" PRIu16 "\n" + "\t data_format = 0x%08" PRIx32 "\n" + "\t sampling_freq = %" PRIu32 "\n", + ctx, *port, params->port_type, params->data_format, params->sampling_freq); + } + + return OK; +} + +int KYTY_SYSV_ABI AudioOut2PortDestroy(AudioOut2PortHandle port) { + PRINT_NAME(); + LOGF("\t port = 0x%016" PRIx64 "\n", port); + + int audio_handle = 0; + g_audioout2_port_mutex.Lock(); + if (auto* state = audioout2_find_port_locked(port); state != nullptr) { + audio_handle = state->audio_handle; + *state = AudioOut2PortStateEntry {}; + } + g_audioout2_port_mutex.Unlock(); + + audioout2_close_audio_handle(audio_handle); + + return OK; +} + +int KYTY_SYSV_ABI AudioOut2PortSetAttributes(AudioOut2PortHandle port, + const AudioOut2Attribute* attributes, uint32_t num) { + EXIT_NOT_IMPLEMENTED(num != 0 && attributes == nullptr); + + const void* pcm_data = nullptr; + bool has_pcm = false; + for (uint32_t i = 0; i < num; i++) { + if (attributes[i].attribute_id == AUDIO_OUT2_PORT_ATTRIBUTE_ID_PCM && + attributes[i].value != nullptr && attributes[i].value_size >= sizeof(AudioOut2Pcm)) { + AudioOut2Pcm pcm {}; + std::memcpy(&pcm, attributes[i].value, sizeof(AudioOut2Pcm)); + pcm_data = pcm.data; + has_pcm = true; + } + } + + if (has_pcm) { + g_audioout2_port_mutex.Lock(); + if (auto* state = audioout2_find_port_locked(port); state != nullptr) { + state->pcm_data = pcm_data; + } + g_audioout2_port_mutex.Unlock(); + } + + return OK; +} + +int KYTY_SYSV_ABI AudioOut2PortGetState(AudioOut2PortHandle port, AudioOut2PortState* state) { + PRINT_NAME(); + + EXIT_NOT_IMPLEMENTED(state == nullptr); + + std::memset(state, 0, sizeof(AudioOut2PortState)); + state->output = 1; + state->num_channels = 2; + state->volume = 127; + state->reroute_counter = 0; + state->flags = 0; + + g_audioout2_port_mutex.Lock(); + if (auto* port_state = audioout2_find_port_locked(port); port_state != nullptr) { + state->num_channels = audioout2_data_format_channels(port_state->data_format); + } + g_audioout2_port_mutex.Unlock(); + + // LOGF("\t port = 0x%016" PRIx64 "\n", port); + // LOGF("\t num_channels = %" PRIu8 "\n", state->num_channels); + + return OK; +} + +int KYTY_SYSV_ABI AudioOut2GetSystemState(AudioOut2SystemState* state) { + PRINT_NAME(); + EXIT_NOT_IMPLEMENTED(state == nullptr); + std::memset(state, 0, sizeof(AudioOut2SystemState)); + return OK; +} + +int KYTY_SYSV_ABI AudioOut2UserCreate(uint32_t user_id, AudioOut2UserHandle* handle) { + PRINT_NAME(); + EXIT_NOT_IMPLEMENTED(handle == nullptr); + *handle = static_cast( + g_audioout2_next_user.fetch_add(1, std::memory_order_relaxed)); + LOGF("\t user_id = %" PRIu32 ", handle = 0x%016" PRIx64 "\n", user_id, + static_cast(*handle)); + return OK; +} + +int KYTY_SYSV_ABI AudioOut2UserDestroy(AudioOut2UserHandle handle) { + PRINT_NAME(); + LOGF("\t handle = 0x%016" PRIx64 "\n", static_cast(handle)); + return OK; +} + +size_t KYTY_SYSV_ABI AudioOut2GetSpeakerArrayMemorySize(uint32_t num_speakers, uint8_t is_3d, + uint8_t is_ambisonics) { + PRINT_NAME(); + + const auto speakers = std::clamp(num_speakers, 1, 32); + const auto size = static_cast(0x400 + speakers * (is_ambisonics != 0 ? 0x100 : 0x40) + + (is_3d != 0 ? 0x200 : 0)); + + LOGF("\t num_speakers = %" PRIu32 "\n" + "\t is_3d = %" PRIu8 "\n" + "\t is_ambisonics = %" PRIu8 "\n" + "\t memory_size = 0x%016" PRIx64 "\n", + num_speakers, is_3d, is_ambisonics, static_cast(size)); + + return size; +} + +int KYTY_SYSV_ABI AudioOut2SpeakerArrayCreate(AudioOut2SpeakerArrayHandle* handle, + const void* vbap_params, const void* ambi_params) { + PRINT_NAME(); + EXIT_NOT_IMPLEMENTED(handle == nullptr); + + *handle = nullptr; + + g_audioout2_speaker_array_mutex.Lock(); + for (auto& state: g_audioout2_speaker_arrays) { + if (!state.used) { + state = AudioOut2SpeakerArrayState {}; + state.used = true; + state.handle = &state; + state.num_speakers = 2; + *handle = state.handle; + break; + } + } + g_audioout2_speaker_array_mutex.Unlock(); + + LOGF("\t handle = 0x%016" PRIx64 "\n" + "\t vbap_params = 0x%016" PRIx64 "\n" + "\t ambi_params = 0x%016" PRIx64 "\n", + reinterpret_cast(*handle), reinterpret_cast(vbap_params), + reinterpret_cast(ambi_params)); + + return (*handle != nullptr ? OK : AUDIO_OUT2_ERROR_PORT_FULL); +} + +int KYTY_SYSV_ABI AudioOut2SpeakerArrayDestroy(AudioOut2SpeakerArrayHandle handle) { + PRINT_NAME(); + LOGF("\t handle = 0x%016" PRIx64 "\n", reinterpret_cast(handle)); + + g_audioout2_speaker_array_mutex.Lock(); + for (auto& state: g_audioout2_speaker_arrays) { + if (state.used && state.handle == handle) { + state = AudioOut2SpeakerArrayState {}; + break; + } + } + g_audioout2_speaker_array_mutex.Unlock(); + + return OK; +} + +int KYTY_SYSV_ABI AudioOut2GetSpeakerArrayCoefficients( + AudioOut2SpeakerArrayHandle handle, AudioOut2Position pos, float spread, float* coefficients, + uint32_t num_coefficients, uint8_t height_aware, float downmix_spread_radius) { + PRINT_NAME(); + EXIT_NOT_IMPLEMENTED(coefficients == nullptr && num_coefficients != 0); + + if (coefficients != nullptr) { + std::fill(coefficients, coefficients + num_coefficients, 0.0f); + if (num_coefficients > 0) { + coefficients[0] = 1.0f; + } + if (num_coefficients > 1) { + coefficients[1] = 1.0f; + } + } + + LOGF("\t handle = 0x%016" PRIx64 ", coeffs = %" PRIu32 + ", pos = (%f, %f, %f), spread = %f, height = %" PRIu8 ", downmix = %f\n", + reinterpret_cast(handle), num_coefficients, static_cast(pos.x), + static_cast(pos.y), static_cast(pos.z), static_cast(spread), + height_aware, static_cast(downmix_spread_radius)); + + return OK; +} + +int KYTY_SYSV_ABI AudioOut2GetSpeakerArrayAmbisonicsCoefficients(AudioOut2SpeakerArrayHandle handle, + uint32_t ambisonics_channel, + float* coefficients, + uint32_t num_coefficients) { + PRINT_NAME(); + EXIT_NOT_IMPLEMENTED(coefficients == nullptr && num_coefficients != 0); + + if (coefficients != nullptr) { + std::fill(coefficients, coefficients + num_coefficients, 0.0f); + if (num_coefficients > 0) { + coefficients[0] = + (ambisonics_channel == 0 || ambisonics_channel == 64 ? 0.70710677f : 1.0f); + } + } + + LOGF("\t handle = 0x%016" PRIx64 ", channel = %" PRIu32 ", coeffs = %" PRIu32 "\n", + reinterpret_cast(handle), ambisonics_channel, num_coefficients); + + return OK; +} + +int KYTY_SYSV_ABI AudioOut2GetSpeakerInfo(AudioOut2SpeakerInfo* info, uint32_t flags) { + EXIT_NOT_IMPLEMENTED(info == nullptr); + std::memset(info, 0, sizeof(AudioOut2SpeakerInfo)); + info->type = 0; + info->available_bits = 0x03; + info->flags = 0; + info->speaker_angle[0] = {-30, 0}; + info->speaker_angle[1] = {30, 0}; + + return OK; +} + +int KYTY_SYSV_ABI AudioOut2SetSystemDebugState(const AudioOut2SystemDebugStateParam* param) { + PRINT_NAME(); + LOGF("\t param = 0x%016" PRIx64 "\n", reinterpret_cast(param)); + return OK; +} + +int KYTY_SYSV_ABI AudioOut2Set3DLatency(uint32_t user_id, uint32_t output, uint32_t latency_us) { + // Not sure + + PRINT_NAME(); + LOGF("\t user_id = %" PRIu32 ", output = %" PRIu32 ", latency_us = %" PRIu32 "\n", user_id, + output, latency_us); + + if (output > AUDIO_OUT2_MASTERING_OUTPUT_RECORDING) { + return AUDIO_OUT2_ERROR_INVALID_PARAM; + } + + g_audioout2_latency_mutex.Lock(); + AudioOut2LatencyState* free_state = nullptr; + for (auto& state: g_audioout2_latencies) { + if (state.used && state.user_id == user_id && state.output == output) { + state.latency_us = latency_us; + g_audioout2_latency_mutex.Unlock(); + return OK; + } + if (!state.used && free_state == nullptr) { + free_state = &state; + } + } + + if (free_state != nullptr) { + *free_state = AudioOut2LatencyState {true, user_id, output, latency_us}; + } + g_audioout2_latency_mutex.Unlock(); + + return (free_state != nullptr ? OK : AUDIO_OUT2_ERROR_PORT_FULL); +} + +int KYTY_SYSV_ABI AudioOut2MasteringInit(uint32_t flags) { + PRINT_NAME(); + LOGF("\t flags = 0x%08" PRIx32 "\n", flags); + return OK; +} + +int KYTY_SYSV_ABI AudioOut2MasteringSetParam(const AudioOut2MasteringParamsHeader* param, + uint32_t output, uint32_t flags) { + PRINT_NAME(); + LOGF("\t param = 0x%016" PRIx64 ", output = %" PRIu32 ", flags = 0x%08" PRIx32 "\n", + reinterpret_cast(param), output, flags); + return OK; +} + +int KYTY_SYSV_ABI AudioOut2MasteringGetState(AudioOut2MasteringStatesHeader* state, uint32_t output, + AudioOut2UserHandle user) { + PRINT_NAME(); + LOGF("\t state = 0x%016" PRIx64 ", output = %" PRIu32 ", user = 0x%016" PRIx64 "\n", + reinterpret_cast(state), output, static_cast(user)); + + if (state == nullptr) { + return AUDIO_OUT2_ERROR_MASTERING_INVALID_API_PARAM; + } + + const auto states_id = state->states_id; + switch (states_id) { + case AUDIO_OUT2_MASTERING_STATES_ID_DEFAULT: { + auto* full_state = reinterpret_cast(state); + std::memset(full_state, 0, sizeof(AudioOut2MasteringStates)); + full_state->states_header.states_id = AUDIO_OUT2_MASTERING_STATES_ID_DEFAULT; + full_state->compressor_states.descriptor.id = + AUDIO_OUT2_MASTERING_STATES_STRUCT_ID_COMPRESSOR_DEFAULT; + full_state->compressor_states.descriptor.size = + sizeof(AudioOut2MasteringCompressorStates); + full_state->limiter_states.descriptor.id = + AUDIO_OUT2_MASTERING_STATES_STRUCT_ID_LIMITER_DEFAULT; + full_state->limiter_states.descriptor.size = sizeof(AudioOut2MasteringLimiterStates); + return OK; + } + case AUDIO_OUT2_MASTERING_STATES_ID_V2: { + auto* full_state = reinterpret_cast(state); + std::memset(full_state, 0, sizeof(AudioOut2MasteringStatesV2)); + full_state->states_header.states_id = AUDIO_OUT2_MASTERING_STATES_ID_V2; + full_state->compressor_states.descriptor.id = + AUDIO_OUT2_MASTERING_STATES_STRUCT_ID_COMPRESSOR_V2; + full_state->compressor_states.descriptor.size = + sizeof(AudioOut2MasteringCompressorStatesV2); + full_state->limiter_states.descriptor.id = + AUDIO_OUT2_MASTERING_STATES_STRUCT_ID_LIMITER_DEFAULT; + full_state->limiter_states.descriptor.size = sizeof(AudioOut2MasteringLimiterStates); + return OK; + } + default: return AUDIO_OUT2_ERROR_MASTERING_INVALID_STATES_ID; + } +} + +int KYTY_SYSV_ABI AudioOut2MasteringTerm() { + PRINT_NAME(); + return OK; +} + +} // namespace AudioOut2 +} // namespace Libs::Audio diff --git a/src/libs/libC.cpp b/src/libs/libC.cpp new file mode 100644 index 000000000..b33a428cb --- /dev/null +++ b/src/libs/libC.cpp @@ -0,0 +1,898 @@ +#include "common/abi.h" +#include "common/assert.h" +#include "common/common.h" +#include "common/logging/log.h" +#include "common/singleton.h" +#include "common/stringUtils.h" +#include "graphics/host_gpu/hostMemory.h" +#include "kernel/pthread.h" +#include "libs/errno.h" +#include "libs/guestPrintf.h" +#include "libs/libs.h" +#include "libs/vaContext.h" +#include "loader/runtimeLinker.h" +#include "loader/symbolDatabase.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace Libs { + +namespace LibKernel { +void KernelDispatchPendingSignalForCurrentThread(); +} // namespace LibKernel + +namespace LibC { + +LIB_VERSION("libc", 1, "libc", 1, 1); + +static uint32_t g_need_flag = 1; + +using cxa_destructor_func_t = KYTY_SYSV_ABI void (*)(void*); +using atexit_func_t = KYTY_SYSV_ABI void (*)(); + +struct CxaDestructor { + cxa_destructor_func_t destructor_func; + void* destructor_object; + void* module_id; +}; + +struct CContext { + std::list cxa; + std::list atexit; +}; + +struct InitEnvParams { + int argc; + uint32_t pad; + const char* argv[3]; +}; + +static int g_argc = 0; +static const char* const* g_argv = nullptr; +static const char* const* g_envp = nullptr; + +int GetArgc() { + return g_argc; +} + +const char** GetArgv() { + return const_cast(g_argv); +} + +static KYTY_SYSV_ABI void exit(int code) { + PRINT_NAME(); + + ::exit(code); +} + +static void PrintAbortStringCandidate(const char* name, uint64_t addr) { + if (!Graphics::HostMemoryIsReadable(addr)) { + return; + } + + const auto* ptr = reinterpret_cast(static_cast(addr)); + char buf[257] {}; + size_t len = 0; + + for (; len < sizeof(buf) - 1 && Graphics::HostMemoryIsReadable(addr + len); len++) { + const auto c = static_cast(ptr[len]); + if (c == 0) { + break; + } + if (!std::isprint(c) && c != '\n' && c != '\r' && c != '\t') { + return; + } + buf[len] = static_cast(c); + } + + if (len >= 4) { + LOGF("\t %s string = \"%s\"\n", name, buf); + } +} + +static void PrintAbortWideStringCandidate(const char* name, uint64_t addr) { + if (!Graphics::HostMemoryIsReadable(addr)) { + return; + } + + const auto* ptr = reinterpret_cast(static_cast(addr)); + char buf[257] {}; + size_t len = 0; + + for (; + len < sizeof(buf) - 1 && Graphics::HostMemoryIsReadable(addr + len * sizeof(uint16_t) + 1); + len++) { + const auto c = ptr[len]; + if (c == 0) { + break; + } + if (c > 0x7f || + (!std::isprint(static_cast(c)) && c != '\n' && c != '\r' && c != '\t')) { + return; + } + buf[len] = static_cast(c); + } + + if (len >= 4) { + LOGF("\t %s u16 = \"%s\"\n", name, buf); + } +} + +static void PrintAbortBytesCandidate(const char* name, uint64_t addr) { + if (!Graphics::HostMemoryIsReadable(addr)) { + return; + } + + uint8_t bytes[64] {}; + size_t len = 0; + + for (; len < sizeof(bytes) && Graphics::HostMemoryIsReadable(addr + len); len++) { + bytes[len] = *reinterpret_cast(static_cast(addr + len)); + } + + if (len == 0) { + return; + } + + LOGF("\t %s bytes =", name); + for (size_t i = 0; i < len; i++) { + LOGF(" %02" PRIx8, bytes[i]); + } + LOGF("\n"); +} + +static void PrintAbortPointerCandidate(const char* name, uint64_t addr) { + PrintAbortStringCandidate(name, addr); + PrintAbortWideStringCandidate(name, addr); + PrintAbortBytesCandidate(name, addr); +} + +static bool LooksLikeAbortPointer(uint64_t addr) { + return Graphics::HostMemoryIsReadable(addr) && addr >= 0x10000; +} + +static void PrintAbortPointerArrayCandidate(const char* name, uint64_t addr) { + if (!LooksLikeAbortPointer(addr)) { + return; + } + + for (int i = 0; i < 8; i++) { + const auto slot_addr = addr + static_cast(i) * sizeof(uint64_t); + if (!Graphics::HostMemoryIsReadable(slot_addr)) { + break; + } + + const auto value = *reinterpret_cast(static_cast(slot_addr)); + if (!LooksLikeAbortPointer(value)) { + continue; + } + + LOGF("\t %s[%d] = 0x%016" PRIx64 "\n", name, i, value); + const auto child_name = fmt::format("{}[{}]", name, i); + PrintAbortPointerCandidate(child_name.c_str(), value); + } +} + +[[noreturn]] static KYTY_SYSV_ABI void abort(uint64_t arg0, uint64_t arg1, uint64_t arg2, + uint64_t arg3, uint64_t arg4, uint64_t arg5) { + PRINT_NAME(); + + uint64_t rbp = 0; + uint64_t rsp = 0; +#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS + asm volatile("movq %%rbp, %0" : "=r"(rbp)); + asm volatile("movq %%rsp, %0" : "=r"(rsp)); +#endif + + const auto ret = reinterpret_cast(__builtin_return_address(0)); + + LOGF("Guest abort diagnostics:\n" + "\t return = 0x%016" PRIx64 "\n" + "\t rbp = 0x%016" PRIx64 "\n" + "\t rsp = 0x%016" PRIx64 "\n" + "\t arg0 = 0x%016" PRIx64 "\n" + "\t arg1 = 0x%016" PRIx64 "\n" + "\t arg2 = 0x%016" PRIx64 "\n" + "\t arg3 = 0x%016" PRIx64 "\n" + "\t arg4 = 0x%016" PRIx64 "\n" + "\t arg5 = 0x%016" PRIx64 "\n", + ret, rbp, rsp, arg0, arg1, arg2, arg3, arg4, arg5); + + PrintAbortPointerCandidate("arg0", arg0); + PrintAbortPointerCandidate("arg1", arg1); + PrintAbortPointerCandidate("arg2", arg2); + PrintAbortPointerCandidate("arg3", arg3); + PrintAbortPointerCandidate("arg4", arg4); + PrintAbortPointerCandidate("arg5", arg5); + PrintAbortPointerArrayCandidate("arg0", arg0); + PrintAbortPointerArrayCandidate("arg1", arg1); + PrintAbortPointerArrayCandidate("arg2", arg2); + PrintAbortPointerArrayCandidate("arg3", arg3); + PrintAbortPointerArrayCandidate("arg4", arg4); + PrintAbortPointerArrayCandidate("arg5", arg5); + + if (rbp != 0) { + Common::Singleton::Instance()->StackTrace(rbp); + } + + if (rsp != 0) { + LOGF("Guest abort stack words:\n"); + auto* linker = Common::Singleton::Instance(); + for (int i = 0; i < 16; i++) { + const auto addr = rsp + static_cast(i) * sizeof(uint64_t); + if (!Graphics::HostMemoryIsReadable(addr)) { + break; + } + const auto value = *reinterpret_cast(static_cast(addr)); + auto* program = linker->FindProgramByAddr(value); + if (program != nullptr) { + auto module_name = Common::PathToString(program->file_name.filename()); + LOGF("\t [%02d] 0x%016" PRIx64 " %s+0x%016" PRIx64 "\n", i, value, + module_name.c_str(), value - program->base_vaddr); + } else { + LOGF("\t [%02d] 0x%016" PRIx64 "\n", i, value); + } + PrintAbortPointerCandidate(fmt::format("stack[{:02d}]", i).c_str(), value); + PrintAbortPointerArrayCandidate(fmt::format("stack[{:02d}]", i).c_str(), value); + } + } + + EXIT("Guest abort()\n"); + std::abort(); +} + +static KYTY_SYSV_ABI int* libc_error() { + PRINT_NAME(); + + return Posix::GetErrorAddr(); +} + +static KYTY_SYSV_ABI void init_env(const InitEnvParams* params) { + PRINT_NAME(); + + if (params == nullptr) { + g_argc = 0; + g_argv = nullptr; + g_envp = nullptr; + return; + } + + constexpr int argv_capacity = static_cast(sizeof(params->argv) / sizeof(params->argv[0])); + + EXIT_NOT_IMPLEMENTED(params->argc < 0 || params->argc >= argv_capacity); + + g_argc = params->argc; + g_argv = params->argv; + g_envp = params->argv + params->argc + 1; + + LOGF("\t argc = %d\n" + "\t argv = 0x%016" PRIx64 "\n" + "\t envp = 0x%016" PRIx64 "\n", + g_argc, reinterpret_cast(g_argv), reinterpret_cast(g_envp)); + + for (int i = 0; i < g_argc; i++) { + LOGF("\t argv[%d] = %s\n", i, g_argv[i] != nullptr ? g_argv[i] : ""); + } +} + +static KYTY_SYSV_ABI int atexit(atexit_func_t func) { + PRINT_NAME(); + + if (func != nullptr) { + Common::Singleton::Instance()->atexit.push_front(func); + } + + return 0; +} + +static KYTY_SYSV_ABI int libc_printf(VA_ARGS) { + VA_CONTEXT(ctx); // NOLINT(cppcoreguidelines-pro-type-member-init,hicpp-member-init) + + PRINT_NAME(); + + return GetGuestPrintfCtxFunc()(&ctx); +} + +static KYTY_SYSV_ABI int puts(const char* s) { + PRINT_NAME(); + + return GetGuestPrintfStdFunc()("%s\n", s); +} + +static KYTY_SYSV_ABI int setenv(const char* name, const char* value, int overwrite) { + PRINT_NAME(); + + LOGF("\t name = %s\n" + "\t value = %s\n" + "\t overwrite = %d\n", + (name != nullptr ? name : ""), (value != nullptr ? value : ""), overwrite); + + if (name == nullptr || value == nullptr || name[0] == '\0' || + std::strchr(name, '=') != nullptr) { + *Posix::GetErrorAddr() = Posix::POSIX_EINVAL; + return -1; + } + + if (overwrite == 0 && std::getenv(name) != nullptr) { + return 0; + } + +#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS + const auto ret = ::_putenv_s(name, value); + if (ret != 0) { + *Posix::GetErrorAddr() = ret; + return -1; + } + return 0; +#else + const auto ret = ::setenv(name, value, overwrite); + if (ret != 0) { + *Posix::GetErrorAddr() = errno; + return -1; + } + return 0; +#endif +} + +static KYTY_SYSV_ABI int64_t libc_time(int64_t* timer) { + const auto now = static_cast(std::time(nullptr)); + if (timer != nullptr) { + *timer = now; + } + return now; +} + +static KYTY_SYSV_ABI double libc_difftime(int64_t time1, int64_t time0) { + return std::difftime(static_cast(time1), static_cast(time0)); +} + +static KYTY_SYSV_ABI std::tm* libc_gmtime(const int64_t* timer) { + if (timer == nullptr) { + return nullptr; + } + + thread_local std::tm result {}; + const auto t = static_cast(*timer); + +#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS + if (_gmtime64_s(&result, &t) != 0) { + return nullptr; + } +#else + if (gmtime_r(&t, &result) == nullptr) { + return nullptr; + } +#endif + + return &result; +} + +static KYTY_SYSV_ABI std::tm* libc_localtime(const int64_t* timer) { + if (timer == nullptr) { + return nullptr; + } + + thread_local std::tm result {}; + const auto t = static_cast(*timer); + +#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS + if (_localtime64_s(&result, &t) != 0) { + return nullptr; + } +#else + if (localtime_r(&t, &result) == nullptr) { + return nullptr; + } +#endif + + return &result; +} + +static KYTY_SYSV_ABI int64_t libc_mktime(std::tm* timeptr) { + if (timeptr == nullptr) { + return -1; + } + +#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS + return static_cast(_mktime64(timeptr)); +#else + return static_cast(std::mktime(timeptr)); +#endif +} + +static KYTY_SYSV_ABI size_t libc_strftime(char* str, size_t count, const char* format, + const std::tm* timeptr) { + if (str == nullptr || format == nullptr || timeptr == nullptr) { + return 0; + } + + return std::strftime(str, count, format, timeptr); +} + +static KYTY_SYSV_ABI void catchReturnFromMain(int status) { + PRINT_NAME(); + + uint64_t ret_addr = reinterpret_cast(__builtin_return_address(0)); + uint64_t rsp = 0; + uint64_t rbp = 0; +#if defined(__x86_64__) || defined(_M_X64) + asm volatile("movq %%rsp, %0\n\t" + "movq %%rbp, %1\n\t" + : "=r"(rsp), "=r"(rbp)); +#endif + + LOGF("\t status = %d\n" + "\t ret_addr = 0x%016" PRIx64 "\n" + "\t rsp = 0x%016" PRIx64 "\n" + "\t rbp = 0x%016" PRIx64 "\n", + status, ret_addr, rsp, rbp); + if (rsp != 0) { + auto* stack = reinterpret_cast(rsp); + for (uint32_t i = 0; i < 12; i++) { + LOGF("\t stack[%02" PRIu32 "] = 0x%016" PRIx64 "\n", i, stack[i]); + } + } + + ::printf("return from main = %d\n", status); +} + +enum class ThrdResult : int { + Success = 0, + NoMem = 1, + TimedOut = 2, + Busy = 3, + Error = 4, + Perm = 5, +}; + +static int mtx_result(int result) { + return (result == OK ? static_cast(ThrdResult::Success) + : static_cast(ThrdResult::Error)); +} + +static KYTY_SYSV_ABI int MtxInit(LibKernel::PthreadMutex* mtx, int type) { + PRINT_NAME(); + + if (mtx == nullptr) { + return static_cast(ThrdResult::Error); + } + + constexpr int mtx_recursive = 0x100; + if ((type & mtx_recursive) == 0) { + return mtx_result(LibKernel::PthreadMutexInit(mtx, nullptr)); + } + + LibKernel::PthreadMutexattr attr = nullptr; + int result = LibKernel::PthreadMutexattrInit(&attr); + result = (result == OK ? LibKernel::PthreadMutexattrSettype(&attr, 2) : result); + result = (result == OK ? LibKernel::PthreadMutexInit(mtx, &attr) : result); + (void)LibKernel::PthreadMutexattrDestroy(&attr); + + return mtx_result(result); +} + +static KYTY_SYSV_ABI int MtxInitWithName(LibKernel::PthreadMutex* mtx, int type, + const char* /*name*/) { + PRINT_NAME(); + + return MtxInit(mtx, type); +} + +static KYTY_SYSV_ABI int MtxInitWithDefaultNameOverride(LibKernel::PthreadMutex* mtx, int type, + const char* /*name*/) { + PRINT_NAME(); + + return MtxInit(mtx, type); +} + +static KYTY_SYSV_ABI void MtxDestroy(LibKernel::PthreadMutex* mtx) { + PRINT_NAME(); + + if (mtx != nullptr) { + (void)LibKernel::PthreadMutexDestroy(mtx); + } +} + +static KYTY_SYSV_ABI int MtxLock(LibKernel::PthreadMutex* mtx) { + PRINT_NAME(); + + return mtx_result(LibKernel::PthreadMutexLock(mtx)); +} + +static KYTY_SYSV_ABI int MtxTrylock(LibKernel::PthreadMutex* mtx) { + PRINT_NAME(); + + const auto result = LibKernel::PthreadMutexTrylock(mtx); + if (result == OK) { + return static_cast(ThrdResult::Success); + } + return static_cast(ThrdResult::Busy); +} + +static KYTY_SYSV_ABI int MtxTimedlock(LibKernel::PthreadMutex* mtx, const void* /*xtime*/) { + PRINT_NAME(); + + return mtx_result(LibKernel::PthreadMutexLock(mtx)); +} + +static KYTY_SYSV_ABI int MtxUnlock(LibKernel::PthreadMutex* mtx) { + PRINT_NAME(); + + return mtx_result(LibKernel::PthreadMutexUnlock(mtx)); +} + +static KYTY_SYSV_ABI int MtxCurrentOwns(LibKernel::PthreadMutex* /*mtx*/) { + PRINT_NAME(); + + return 0; +} + +using execute_once_func_t = KYTY_SYSV_ABI int (*)(void*, void*, void**); + +static std::mutex g_execute_once_mutex; +static std::condition_variable g_execute_once_cv; + +static KYTY_SYSV_ABI int std_execute_once(int* flag, execute_once_func_t func, void* arg) { + PRINT_NAME(); + + if (flag == nullptr || func == nullptr) { + return 0; + } + + constexpr int once_init = 0; + constexpr int once_done = 1; + constexpr int once_running = 2; + + { + std::unique_lock lock(g_execute_once_mutex); + while (*flag == once_running) { + g_execute_once_cv.wait_for(lock, std::chrono::microseconds(10000)); + if (*flag == once_running) { + lock.unlock(); + LibKernel::KernelDispatchPendingSignalForCurrentThread(); + lock.lock(); + } + } + if (*flag == once_done) { + return 1; + } + *flag = once_running; + } + + void* callback_context = nullptr; + int result = func(nullptr, arg, &callback_context); + + { + std::lock_guard lock(g_execute_once_mutex); + *flag = (result != 0 ? once_done : once_init); + } + g_execute_once_cv.notify_all(); + + return result; +} + +static KYTY_SYSV_ABI int cxa_atexit(cxa_destructor_func_t func, void* arg, void* d) { + PRINT_NAME(); + + auto* cc = Common::Singleton::Instance(); + + CxaDestructor c {}; + c.destructor_func = func; + c.destructor_object = arg; + c.module_id = d; + + cc->cxa.push_back(c); + + return 0; +} + +void KYTY_SYSV_ABI cxa_finalize(void* d) { + PRINT_NAME(); + + auto* cc = Common::Singleton::Instance(); + + for (auto i = cc->cxa.rbegin(); i != cc->cxa.rend(); ++i) { + auto& c = *i; + if ((d == nullptr || c.module_id == d) && c.destructor_func != nullptr) { + auto* func = c.destructor_func; + auto* object = c.destructor_object; + c.destructor_func = nullptr; + func(object); + } + } +} + +} // namespace LibC + +namespace LibcInternalExt { + +LIB_VERSION("LibcInternalExt", 1, "LibcInternal", 1, 1); + +static uint64_t g_mspace_atomic_id_mask = 0; +static uint64_t g_mstate_table[64] = {0}; + +struct Info { + uint64_t size; + uint32_t unknown1; + uint32_t unknown2; + uint64_t* mspace_atomic_id_mask; + uint64_t* mstate_table; +}; + +void KYTY_SYSV_ABI LibcHeapGetTraceInfo(Info* info) { + PRINT_NAME(); + + EXIT_NOT_IMPLEMENTED(info->size != 32); + + info->mspace_atomic_id_mask = &g_mspace_atomic_id_mask; + info->mstate_table = g_mstate_table; +} + +uint64_t KYTY_SYSV_ABI LibcInternalExtUnknownQBS714Jr3g(uint64_t arg0, uint64_t arg1, uint64_t arg2, + uint64_t arg3, uint64_t arg4, + uint64_t arg5) { + PRINT_NAME(); + + LOGF("\t arg0 = 0x%016" PRIx64 "\n" + "\t arg1 = 0x%016" PRIx64 "\n" + "\t arg2 = 0x%016" PRIx64 "\n" + "\t arg3 = 0x%016" PRIx64 "\n" + "\t arg4 = 0x%016" PRIx64 "\n" + "\t arg5 = 0x%016" PRIx64 "\n", + arg0, arg1, arg2, arg3, arg4, arg5); + + return 0; +} + +LIB_DEFINE(InitLibcInternalExt_1) { + LIB_FUNC("NWtTN10cJzE", LibcInternalExt::LibcHeapGetTraceInfo); + LIB_FUNC("qBS714-Jr3g", LibcInternalExt::LibcInternalExtUnknownQBS714Jr3g); +} + +} // namespace LibcInternalExt + +namespace LibcInternal { + +LIB_VERSION("LibcInternal", 1, "LibcInternal", 1, 1); + +static uint32_t g_need_flag = 1; + +int KYTY_SYSV_ABI vprintf(const char* str, VaList* c) { + PRINT_NAME(); + + return GetGuestVprintfFunc()(str, c); +} + +static KYTY_SYSV_ABI int snprintf(VA_ARGS) { + VA_CONTEXT(ctx); // NOLINT(cppcoreguidelines-pro-type-member-init,hicpp-member-init) + + PRINT_NAME(); + + return GetGuestSnprintfCtxFunc()(&ctx); +} + +int KYTY_SYSV_ABI fflush(FILE* stream) { + PRINT_NAME(); + + EXIT_NOT_IMPLEMENTED(stream != stdout); + + return ::fflush(stream); +} + +void* KYTY_SYSV_ABI memset(void* s, int c, size_t n) { + PRINT_NAME(); + + return ::memset(s, c, n); +} + +void* KYTY_SYSV_ABI memcpy(void* dest, const void* src, size_t n) { + return ::memcpy(dest, src, n); +} + +void* KYTY_SYSV_ABI memmove(void* dest, const void* src, size_t n) { + return ::memmove(dest, src, n); +} + +int KYTY_SYSV_ABI memcmp(const void* s1, const void* s2, size_t n) { + return ::memcmp(s1, s2, n); +} + +int KYTY_SYSV_ABI strcmp(const char* s1, const char* s2) { + return ::strcmp(s1, s2); +} + +int KYTY_SYSV_ABI strncmp(const char* s1, const char* s2, size_t n) { + return ::strncmp(s1, s2, n); +} + +size_t KYTY_SYSV_ABI strlen(const char* s) { + return ::strlen(s); +} + +char* KYTY_SYSV_ABI strcpy(char* dest, const char* src) { + return ::strcpy(dest, src); +} + +char* KYTY_SYSV_ABI strncpy(char* dest, const char* src, size_t count) { + return ::strncpy(dest, src, count); +} + +char* KYTY_SYSV_ABI strcat(char* dest, const char* src) { + return ::strcat(dest, src); +} + +const char* KYTY_SYSV_ABI strchr(const char* s, int c) { + return ::strchr(s, c); +} + +char* KYTY_SYSV_ABI strrchr(const char* s, int c) { + return const_cast(::strrchr(s, c)); +} + +char* KYTY_SYSV_ABI strstr(const char* haystack, const char* needle) { + static int log_count = 0; + if (log_count++ < 8) { + LOGF("LibcInternal::strstr(\"%s\", \"%s\")\n", (haystack != nullptr ? haystack : ""), + (needle != nullptr ? needle : "")); + } + return const_cast(::strstr(haystack, needle)); +} + +long KYTY_SYSV_ABI strtol(const char* str, char** endptr, int base) { + static int log_count = 0; + if (log_count++ < 16) { + LOGF("LibcInternal::strtol(\"%s\", base=%d)\n", (str != nullptr ? str : ""), base); + } + return ::strtol(str, endptr, base); +} + +unsigned long KYTY_SYSV_ABI strtoul(const char* str, char** endptr, int base) { + static int log_count = 0; + if (log_count++ < 16) { + LOGF("LibcInternal::strtoul(\"%s\", base=%d)\n", (str != nullptr ? str : ""), base); + } + return ::strtoul(str, endptr, base); +} + +int KYTY_SYSV_ABI atoi(const char* str) { + return ::atoi(str); +} + +float KYTY_SYSV_ABI sinf(float x) { + return std::sinf(x); +} + +float KYTY_SYSV_ABI cosf(float x) { + return std::cosf(x); +} + +void KYTY_SYSV_ABI sincosf(float x, float* sinp, float* cosp) { + if (sinp != nullptr) { + *sinp = std::sinf(x); + } + if (cosp != nullptr) { + *cosp = std::cosf(x); + } +} + +double KYTY_SYSV_ABI sin(double x) { + return std::sin(x); +} + +double KYTY_SYSV_ABI cos(double x) { + return std::cos(x); +} + +void KYTY_SYSV_ABI sincos(double x, double* sinp, double* cosp) { + if (sinp != nullptr) { + *sinp = std::sin(x); + } + if (cosp != nullptr) { + *cosp = std::cos(x); + } +} + +int KYTY_SYSV_ABI LibcHeapErrorReportForGame(uint64_t msp, uint64_t ptr, uint64_t error, + uint64_t arg3, uint64_t arg4, uint64_t arg5) { + PRINT_NAME(); + + LOGF("\t temporary: heap error report ignored, msp=0x%016" PRIx64 ", ptr=0x%016" PRIx64 + ", error=0x%016" PRIx64 ", args=(0x%016" PRIx64 ",0x%016" PRIx64 ",0x%016" PRIx64 ")\n", + msp, ptr, error, arg3, arg4, arg5); + + return 0; +} + +LIB_DEFINE(InitLibcInternal_1) { + LibcInternalExt::InitLibcInternalExt_1(s); + + LIB_OBJECT("ZT4ODD2Ts9o", &LibcInternal::g_need_flag); + LIB_OBJECT("2sWzhYqFH4E", stdout); + + LIB_FUNC("GMpvxPFW924", LibcInternal::vprintf); + LIB_FUNC("MUjC4lbHrK4", LibcInternal::fflush); + LIB_FUNC("8zTFvBIAIN8", LibcInternal::memset); + LIB_FUNC("Q3VBxCXhUHs", LibcInternal::memcpy); + LIB_FUNC("+P6FRGH4LfA", LibcInternal::memmove); + LIB_FUNC("DfivPArhucg", LibcInternal::memcmp); + LIB_FUNC("aesyjrHVWy4", LibcInternal::strcmp); + LIB_FUNC("Ovb2dSJOAuE", LibcInternal::strncmp); + LIB_FUNC("j4ViWNHEgww", LibcInternal::strlen); + LIB_FUNC("5Xa2ACNECdo", LibcInternal::strcpy); + LIB_FUNC("6sJWiWSRuqk", LibcInternal::strncpy); + LIB_FUNC("Ls4tzzhimqQ", LibcInternal::strcat); + LIB_FUNC("ob5xAW4ln-0", LibcInternal::strchr); + LIB_FUNC("9yDWMxEFdJU", LibcInternal::strrchr); + LIB_FUNC("viiwFMaNamA", LibcInternal::strstr); + LIB_FUNC("mXlxhmLNMPg", LibcInternal::strtol); + LIB_FUNC("QxmSHBCuKTk", LibcInternal::strtoul); + LIB_FUNC("zlfEH8FmyUA", LibcInternal::strtoul); + LIB_FUNC("fPxypibz2MY", LibcInternal::atoi); + LIB_FUNC("Q4rRL34CEeE", LibcInternal::sinf); + LIB_FUNC("-P6FNMzk2Kc", LibcInternal::cosf); + LIB_FUNC("pztV4AF18iI", LibcInternal::sincosf); + LIB_FUNC("H8ya2H00jbI", LibcInternal::sin); + LIB_FUNC("2WE3BTYVwKM", LibcInternal::cos); + LIB_FUNC("jMB7EFyu30Y", LibcInternal::sincos); + LIB_FUNC("eLdDw6l0-bU", LibcInternal::snprintf); + + LIB_FUNC("L1SBTkC+Cvw", LibC::abort); + LIB_FUNC("tsvEmnenz48", LibC::cxa_atexit); + LIB_FUNC("H2e8t5ScQGc", LibC::cxa_finalize); + LIB_FUNC("DiGVep5yB5w", LibC::std_execute_once); + + LIB_FUNC("al3JzFI9MQ0", LibcInternal::LibcHeapErrorReportForGame); +} + +} // namespace LibcInternal + +LIB_USING(LibC); + +LIB_DEFINE(InitLibC_1) { + LibcInternal::InitLibcInternal_1(s); + + LIB_OBJECT("P330P3dFF68", &LibC::g_need_flag); + + LIB_FUNC("uMei1W9uyNo", LibC::exit); + LIB_FUNC("L1SBTkC+Cvw", LibC::abort); + LIB_FUNC("9BcDykPmo1I", LibC::libc_error); + LIB_FUNC("bzQExy189ZI", LibC::init_env); + LIB_FUNC("8G2LB+A3rzg", LibC::atexit); + LIB_FUNC("hcuQgD53UxM", LibC::libc_printf); + LIB_FUNC("YQ0navp+YIc", LibC::puts); + LIB_FUNC("M4YYbSFfJ8g", LibC::setenv); + LIB_FUNC("wLlFkwG9UcQ", LibC::libc_time); + LIB_FUNC("-VVn74ZyhEs", LibC::libc_difftime); + LIB_FUNC("1mecP7RgI2A", LibC::libc_gmtime); + LIB_FUNC("efhK-YSUYYQ", LibC::libc_localtime); + LIB_FUNC("n7AepwR0s34", LibC::libc_mktime); + LIB_FUNC("Av3zjWi64Kw", LibC::libc_strftime); + LIB_FUNC("XKRegsFpEpk", LibC::catchReturnFromMain); + LIB_FUNC("tsvEmnenz48", LibC::cxa_atexit); + LIB_FUNC("H2e8t5ScQGc", LibC::cxa_finalize); + LIB_FUNC("DiGVep5yB5w", LibC::std_execute_once); + LIB_FUNC("YaHc3GS7y7g", LibC::MtxInit); + LIB_FUNC("tgioGpKtmbE", LibC::MtxInitWithName); + LIB_FUNC("JHp7ogc1+HY", LibC::MtxInitWithDefaultNameOverride); + LIB_FUNC("5Lf51jvohTQ", LibC::MtxDestroy); + LIB_FUNC("iS4aWbUonl0", LibC::MtxLock); + LIB_FUNC("k6pGNMwJB08", LibC::MtxTrylock); + LIB_FUNC("hPzYSd5Nasc", LibC::MtxTimedlock); + LIB_FUNC("gTuXQwP9rrs", LibC::MtxUnlock); + LIB_FUNC("VYQwFs4CC4Y", LibC::MtxCurrentOwns); +} + +} // namespace Libs diff --git a/src/libs/libDbgAsan.cpp b/src/libs/libDbgAsan.cpp new file mode 100644 index 000000000..11000cb03 --- /dev/null +++ b/src/libs/libDbgAsan.cpp @@ -0,0 +1,235 @@ +#include "common/abi.h" +#include "common/assert.h" +#include "common/common.h" +#include "common/logging/log.h" +#include "common/platform/sysDbg.h" +#include "common/stringUtils.h" +#include "common/threads.h" +#include "common/virtualMemory.h" +#include "kernel/memory.h" +#include "libs/libs.h" +#include "loader/symbolDatabase.h" + +#include +#include + +namespace Libs { + +LIB_VERSION("DbgAddressSanitizer", 1, "DbgAddressSanitizer", 1, 1); + +namespace DbgAddressSanitizer { + +struct SourceLocation { + const char* filename; + int line_no; + int column_no; +}; + +struct Global { + uintptr_t beg; + uintptr_t size; + uintptr_t size_with_redzone; + const char* name; + const char* module_name; + uintptr_t has_dynamic_init; + SourceLocation* location; + uintptr_t odr_indicator; +}; + +struct Shadow { + uintptr_t start = 0; + int count = 0; +}; + +struct AsanContext { + Common::Mutex mutex; + std::vector shadows; +}; + +static uint32_t g_fake_stack = 0; +static AsanContext* g_ctx = nullptr; + +static void AllocShadow(uintptr_t min_addr, uintptr_t max_addr) { + EXIT_IF(g_ctx == nullptr); + + Common::LockGuard lock(g_ctx->mutex); + + static const uintptr_t page_size = 0x10000ull; + + auto min_shadow = ((min_addr >> 3u) + 0x10000000000ull) & ~(page_size - 1); + auto max_shadow = ((max_addr >> 3u) + 0x10000000000ull) & ~(page_size - 1); + + auto size = max_shadow - min_shadow + page_size; + + LOGF("\t shadow = %016" PRIx64 "\n" + "\t size = %016" PRIx64 "\n", + static_cast(min_shadow), static_cast(size)); + + for (uintptr_t page = min_shadow; page <= max_shadow; page += page_size) { + const auto it = std::find_if(g_ctx->shadows.begin(), g_ctx->shadows.end(), + [page](const Shadow& s) { return s.start == page; }); + if (it != g_ctx->shadows.end()) { + it->count++; + } else { + if (!Common::VirtualMemory::AllocFixed(page, page_size, + Common::VirtualMemory::Mode::ReadWrite)) { + EXIT("can't allocate shadow memory\n"); + } + + Shadow s; + s.start = page; + s.count = 1; + g_ctx->shadows.push_back(s); + } + } +} + +static void FreeShadow(uintptr_t min_addr, uintptr_t max_addr) { + EXIT_IF(g_ctx == nullptr); + + Common::LockGuard lock(g_ctx->mutex); + + static const uintptr_t page_size = 0x10000ull; + + auto min_shadow = ((min_addr >> 3u) + 0x10000000000ull) & ~(page_size - 1); + auto max_shadow = ((max_addr >> 3u) + 0x10000000000ull) & ~(page_size - 1); + + auto size = max_shadow - min_shadow + page_size; + + LOGF("\t shadow = %016" PRIx64 "\n" + "\t size = %016" PRIx64 "\n", + static_cast(min_shadow), static_cast(size)); + + for (uintptr_t page = min_shadow; page <= max_shadow; page += page_size) { + const auto it = std::find_if(g_ctx->shadows.begin(), g_ctx->shadows.end(), + [page](const Shadow& s) { return s.start == page; }); + if (it != g_ctx->shadows.end()) { + it->count--; + + if (it->count <= 0) { + if (!Common::VirtualMemory::Free(page)) { + EXIT("can't free shadow memory\n"); + } + + g_ctx->shadows.erase(it); + } + } + } +} + +static void KernelAlloc(uintptr_t addr, size_t size) { + AllocShadow(addr, addr + size - 1); +} + +static void KernelFree(uintptr_t addr, size_t size) { + FreeShadow(addr, addr + size - 1); +} + +static void KYTY_SYSV_ABI asan_init() { + PRINT_NAME(); + + EXIT_NOT_IMPLEMENTED(g_ctx != nullptr); + + g_ctx = new AsanContext; + + sys_dbg_stack_info_t s {}; + SysStackUsage(s); + +#if KYTY_PLATFORM == KYTY_PLATFORM_LINUX + AllocShadow(s.addr, reinterpret_cast(&s)); +#else + AllocShadow(s.reserved_addr, reinterpret_cast(&s)); +#endif + + LibKernel::Memory::RegisterCallbacks(KernelAlloc, KernelFree); +} + +static void KYTY_SYSV_ABI asan_version_mismatch_check_v8() { + PRINT_NAME(); +} + +static void KYTY_SYSV_ABI asan_register_elf_globals(uintptr_t* flag, Global* start, Global* stop) { + PRINT_NAME(); + + LOGF("\t flag = %016" PRIx64 "\n" + "\t start = %016" PRIx64 "\n" + "\t stop = %016" PRIx64 "\n", + reinterpret_cast(flag), reinterpret_cast(start), + reinterpret_cast(stop)); + + if (flag == nullptr || start == nullptr || stop == nullptr) { + return; + } + + if (*flag != 0) { + return; + } + + auto num = stop - start; + + if (num != 0) { + uintptr_t min_addr = UINTPTR_MAX; + uintptr_t max_addr = 0; + + for (ptrdiff_t i = 0; i < num; i++) { + auto* g = start + i; + + LOGF("asan global:\n" + "\t beg = %016" PRIx64 "\n" + "\t size = %016" PRIx64 "\n" + "\t size_with_redzone = %016" PRIx64 "\n" + "\t name = %s\n" + "\t module_name = %s\n" + "\t has_dynamic_init = %" PRIu64 "\n" + "\t odr_indicator = %016" PRIx64 "\n", + static_cast(g->beg), static_cast(g->size), + static_cast(g->size_with_redzone), g->name, g->module_name, + static_cast(g->has_dynamic_init), + static_cast(g->odr_indicator)); + if (g->location != nullptr) { + LOGF("\t location = %s:%d\n", g->location->filename, g->location->line_no); + } + + min_addr = std::min(g->beg, min_addr); + max_addr = std::max(g->beg + g->size_with_redzone - 1, max_addr); + } + + AllocShadow(min_addr, max_addr); + } + + *flag = 0; +} + +static void KYTY_SYSV_ABI asan_before_dynamic_init(const char* module_name) { + PRINT_NAME(); + + LOGF("\t name = %s\n", module_name); +} + +static void KYTY_SYSV_ABI asan_after_dynamic_init() { + PRINT_NAME(); +} + +static void* KYTY_SYSV_ABI asan_memcpy(void* dst, const void* src, size_t size) { + return std::memcpy(dst, src, size); +} + +static void* KYTY_SYSV_ABI asan_memset(void* dst, int ch, size_t size) { + return std::memset(dst, ch, size); +} + +} // namespace DbgAddressSanitizer + +LIB_DEFINE(InitDbgAddressSanitizer_1) { + LIB_OBJECT("OmecxEgrhCs", &DbgAddressSanitizer::g_fake_stack); + + LIB_FUNC("xLoVJKHvRMU", DbgAddressSanitizer::asan_init); + LIB_FUNC("GY2I-Okc7q0", DbgAddressSanitizer::asan_version_mismatch_check_v8); + LIB_FUNC("W+2HQAGdPuU", DbgAddressSanitizer::asan_register_elf_globals); + LIB_FUNC("G9xSYlaNv+Y", DbgAddressSanitizer::asan_before_dynamic_init); + LIB_FUNC("z5Gy5T3tptE", DbgAddressSanitizer::asan_after_dynamic_init); + LIB_FUNC("g+Neom2EHO8", DbgAddressSanitizer::asan_memcpy); + LIB_FUNC("q-hZf47nHhQ", DbgAddressSanitizer::asan_memset); +} + +} // namespace Libs diff --git a/src/libs/libDebug.cpp b/src/libs/libDebug.cpp new file mode 100644 index 000000000..7d35a5769 --- /dev/null +++ b/src/libs/libDebug.cpp @@ -0,0 +1,29 @@ +#include "common/abi.h" +#include "common/common.h" +#include "common/stringUtils.h" +#include "libs/libs.h" +#include "loader/symbolDatabase.h" + +namespace Libs { + +namespace LibRazorCpu { + +LIB_VERSION("RazorCpu", 1, "RazorCpu", 1, 1); + +static KYTY_SYSV_ABI uint32_t RazorCpuIsCapturing() { + PRINT_NAME(); + + return 0; +} + +LIB_DEFINE(InitLibRazorCpu_1) { + LIB_FUNC("EboejOQvLL4", LibRazorCpu::RazorCpuIsCapturing); +} + +} // namespace LibRazorCpu + +LIB_DEFINE(InitDebug_1) { + LibRazorCpu::InitLibRazorCpu_1(s); +} + +} // namespace Libs diff --git a/src/libs/libDialog.cpp b/src/libs/libDialog.cpp new file mode 100644 index 000000000..a27bd5a60 --- /dev/null +++ b/src/libs/libDialog.cpp @@ -0,0 +1,136 @@ +#include "common/abi.h" +#include "libs/dialog.h" +#include "libs/libs.h" +#include "loader/symbolDatabase.h" + +namespace Libs { + +namespace LibCommonDialog { + +LIB_VERSION("CommonDialog", 1, "CommonDialog", 1, 1); + +namespace CommonDialog = Dialog::CommonDialog; + +LIB_DEFINE(InitDialog_1_CommonDialog) { + LIB_FUNC("uoUpLGNkygk", CommonDialog::CommonDialogInitialize); +} + +} // namespace LibCommonDialog + +namespace LibImeDialog { + +LIB_VERSION("ImeDialog", 1, "ImeDialog", 1, 1); + +namespace ImeDialog = Dialog::ImeDialog; + +LIB_DEFINE(InitDialog_1_ImeDialog) { + LIB_FUNC("IADmD4tScBY", ImeDialog::ImeDialogGetStatus); +} + +} // namespace LibImeDialog + +namespace LibLoginDialog { + +LIB_VERSION("LoginDialog", 1, "LoginDialog", 1, 1); + +namespace LoginDialog = Dialog::LoginDialog; + +LIB_DEFINE(InitDialog_1_LoginDialog) { + LIB_FUNC("qP-EvQRl2Hc", LoginDialog::LoginDialogInitialize); + LIB_FUNC("vMQJRUKsf3U", LoginDialog::LoginDialogTerminate); + LIB_FUNC("S56ra1+Tymg", LoginDialog::LoginDialogOpen); + LIB_FUNC("F0XIzrG5yvw", LoginDialog::LoginDialogClose); + LIB_FUNC("2rc+egSfb5A", LoginDialog::LoginDialogUpdateStatus); + LIB_FUNC("HAiWUEwEfGo", LoginDialog::LoginDialogGetStatus); + LIB_FUNC("Btkx21f1M8k", LoginDialog::LoginDialogGetResult); + LIB_FUNC("3NPobi5lNmk", LoginDialog::LoginDialogParamInitialize); +} + +} // namespace LibLoginDialog + +namespace LibSaveDataDialog { + +LIB_VERSION("SaveDataDialog", 1, "SaveDataDialog", 1, 1); + +namespace SaveDataDialog = Dialog::SaveDataDialog; + +LIB_DEFINE(InitDialog_1_SaveDataDialog) { + LIB_FUNC("ERKzksauAJA", SaveDataDialog::SaveDataDialogGetStatus); + LIB_FUNC("KK3Bdg1RWK0", SaveDataDialog::SaveDataDialogUpdateStatus); + LIB_FUNC("YuH2FA7azqQ", SaveDataDialog::SaveDataDialogTerminate); + LIB_FUNC("V-uEeFKARJU", SaveDataDialog::SaveDataDialogProgressBarInc); + LIB_FUNC("hay1CfTmLyA", SaveDataDialog::SaveDataDialogProgressBarSetValue); +} + +} // namespace LibSaveDataDialog + +namespace LibSaveDataDialogNative { + +LIB_VERSION("SaveDataDialog.native", 1, "SaveDataDialog", 1, 1); + +namespace SaveDataDialog = Dialog::SaveDataDialog; + +LIB_DEFINE(InitDialog_1_SaveDataDialogNative) { + LIB_FUNC("fH46Lag88XY", SaveDataDialog::SaveDataDialogClose); + LIB_FUNC("yEiJ-qqr6Cg", SaveDataDialog::SaveDataDialogGetResult); + LIB_FUNC("ERKzksauAJA", SaveDataDialog::SaveDataDialogGetStatus); + LIB_FUNC("s9e3+YpRnzw", SaveDataDialog::SaveDataDialogInitialize); + LIB_FUNC("en7gNVnh878", SaveDataDialog::SaveDataDialogIsReadyToDisplay); + LIB_FUNC("4tPhsP6FpDI", SaveDataDialog::SaveDataDialogOpen); + LIB_FUNC("V-uEeFKARJU", SaveDataDialog::SaveDataDialogProgressBarInc); + LIB_FUNC("hay1CfTmLyA", SaveDataDialog::SaveDataDialogProgressBarSetValue); + LIB_FUNC("YuH2FA7azqQ", SaveDataDialog::SaveDataDialogTerminate); + LIB_FUNC("KK3Bdg1RWK0", SaveDataDialog::SaveDataDialogUpdateStatus); +} + +} // namespace LibSaveDataDialogNative + +namespace LibMsgDialog { + +LIB_VERSION("MsgDialog.native", 1, "MsgDialog", 1, 1); + +namespace MsgDialog = Dialog::MsgDialog; + +LIB_DEFINE(InitDialog_1_MsgDialog) { + LIB_FUNC("HTrcDKlFKuM", MsgDialog::MsgDialogClose); + LIB_FUNC("Lr8ovHH9l6A", MsgDialog::MsgDialogGetResult); + LIB_FUNC("CWVW78Qc3fI", MsgDialog::MsgDialogGetStatus); + LIB_FUNC("lDqxaY1UbEo", MsgDialog::MsgDialogInitialize); + LIB_FUNC("b06Hh0DPEaE", MsgDialog::MsgDialogOpen); + LIB_FUNC("Gc5k1qcK4fs", MsgDialog::MsgDialogProgressBarInc); + LIB_FUNC("6H-71OdrpXM", MsgDialog::MsgDialogProgressBarSetMsg); + LIB_FUNC("wTpfglkmv34", MsgDialog::MsgDialogProgressBarSetValue); + LIB_FUNC("ePw-kqZmelo", MsgDialog::MsgDialogTerminate); + LIB_FUNC("6fIC3XKt2k0", MsgDialog::MsgDialogUpdateStatus); +} + +} // namespace LibMsgDialog + +namespace LibErrorDialog { + +LIB_VERSION("ErrorDialog", 1, "ErrorDialog", 1, 1); + +namespace ErrorDialog = Dialog::ErrorDialog; + +LIB_DEFINE(InitDialog_1_ErrorDialog) { + LIB_FUNC("ekXHb1kDBl0", ErrorDialog::ErrorDialogClose); + LIB_FUNC("t2FvHRXzgqk", ErrorDialog::ErrorDialogGetStatus); + LIB_FUNC("I88KChlynSs", ErrorDialog::ErrorDialogInitialize); + LIB_FUNC("M2ZF-ClLhgY", ErrorDialog::ErrorDialogOpen); + LIB_FUNC("9XAxK2PMwk8", ErrorDialog::ErrorDialogTerminate); + LIB_FUNC("WWiGuh9XfgQ", ErrorDialog::ErrorDialogUpdateStatus); +} + +} // namespace LibErrorDialog + +LIB_DEFINE(InitDialog_1) { + LibCommonDialog::InitDialog_1_CommonDialog(s); + LibImeDialog::InitDialog_1_ImeDialog(s); + LibLoginDialog::InitDialog_1_LoginDialog(s); + LibSaveDataDialog::InitDialog_1_SaveDataDialog(s); + LibSaveDataDialogNative::InitDialog_1_SaveDataDialogNative(s); + LibMsgDialog::InitDialog_1_MsgDialog(s); + LibErrorDialog::InitDialog_1_ErrorDialog(s); +} + +} // namespace Libs diff --git a/src/libs/libFont.cpp b/src/libs/libFont.cpp new file mode 100644 index 000000000..6375c2309 --- /dev/null +++ b/src/libs/libFont.cpp @@ -0,0 +1,2291 @@ +#include "common/abi.h" +#include "common/assert.h" +#include "common/common.h" +#include "common/logging/log.h" +#include "libs/errno.h" +#include "libs/libs.h" +#include "loader/symbolDatabase.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#if defined(_MSC_VER) +#pragma warning(push) +#pragma warning(disable : 4244 4267 4701) +#endif +#if defined(__clang__) +#pragma clang diagnostic push +#pragma clang diagnostic ignored "-Wimplicit-fallthrough" +#pragma clang diagnostic ignored "-Wsign-compare" +#pragma clang diagnostic ignored "-Wunused-function" +#endif +#define STB_TRUETYPE_IMPLEMENTATION +#include "stb_truetype.h" +#if defined(__clang__) +#pragma clang diagnostic pop +#endif +#if defined(_MSC_VER) +#pragma warning(pop) +#endif + +extern "C" { +extern const uint8_t avpriv_vga16_font[4096]; +} + +namespace Libs { + +LIB_VERSION("Font", 1, "Font", 1, 1); +namespace Font { + +using FontLibrarySelection = void*; +using FontLibrary = void*; +using FontHandle = void*; +using FontRendererSelection = void*; +using FontRenderer = void*; +using FontMemoryDestroyCallback = void (*)(void* font_memory, void* object, void* destroy_arg); +using FontMemoryInterface = void; +using FontOpenDetail = void; +using FontGenerateGlyphDetail = void; +using FontString = void*; +using FontTextCharacter = const void*; +using FontRenderCharacter = const void*; +using FontWritingLine = void*; + +struct FontTextSource; +union FontTextParseResult; +using FontTextParseFunction = KYTY_SYSV_ABI int (*)(FontTextSource* font_text_source, void** order, + FontTextParseResult* result); +using FontCreateStringOrdersFunction = KYTY_SYSV_ABI void (*)(void* order_callback_object, + uint32_t order_sheet_form, + const void* order_sheet); + +constexpr int SCE_FONT_TEXT_PARSER_RESULT_FONT_CODE = 1; +constexpr int SCE_FONT_TEXT_PARSER_RESULT_TERMINATE = 0; +constexpr int SCE_FONT_TEXT_PARSER_RESULT_ERROR = -1; +constexpr int SCE_FONT_WRITING_FORM_HORIZONTAL = 0x10; +constexpr int FONT_BITMAP_MAX_DIM = 128; + +struct FontMemory { + uint16_t type; + uint16_t attr; + uint32_t size; + void* address; + void* mspace_object; + const FontMemoryInterface* mem_interface; + FontMemoryDestroyCallback destroy_callback; + void* destroy_object; + void* user_object; + void* parent_object; +}; + +struct LibraryState { + const FontMemory* memory; + uint64_t edition; +}; + +struct RendererState { + const FontMemory* memory; + FontRendererSelection selection; + uint64_t edition; +}; + +struct FontGlyphMetrics { + float width; + float height; + struct { + float bearing_x; + float bearing_y; + float advance; + } horizontal; + struct { + float bearing_x; + float bearing_y; + float advance; + } vertical; +}; + +struct FontRenderSurface { + void* buffer; + int32_t width_byte; + int8_t pixel_size_byte; + uint8_t system_ext0; + uint8_t system_ext1; + uint8_t system_ext2; + int32_t width; + int32_t height; + struct { + uint32_t x0; + uint32_t y0; + uint32_t x1; + uint32_t y1; + } scissor; + uint32_t system_use[22]; +}; + +struct FontTransImage { + uint8_t* address; + uint32_t width_byte; + uint32_t image_width; + uint32_t image_height; +}; + +struct FontSurfaceImage { + uint8_t* address; + uint32_t width_byte; + uint8_t pixel_size_byte; + uint8_t pixel_format; +}; + +struct FontGlyphImageMetrics { + float bearing_x; + float bearing_y; + float advance; + float stride; + uint32_t width; + uint32_t height; +}; + +struct FontRenderResult { + const FontTransImage* trans_image; + FontSurfaceImage surface_image; + struct { + uint32_t x; + uint32_t y; + uint32_t w; + uint32_t h; + } update_rect; + FontGlyphImageMetrics image_metrics; +}; + +struct FontState { + FontLibrary library; + const void* data; + uint32_t size; + uint32_t font_set_type; + uint32_t open_mode; + int attribute; + FontRenderer renderer; + float scale_w; + float scale_h; + float effect_weight_x; + float effect_weight_y; + uint32_t effect_weight_mode; + float effect_slant; + float render_scale_w; + float render_scale_h; + float render_effect_weight_x; + float render_effect_weight_y; + uint32_t render_effect_weight_mode; + float render_effect_slant; + stbtt_fontinfo font_info; + const uint8_t* ttf_data; + uint32_t ttf_size; + bool has_ttf; + std::array fallback_image; + FontTransImage trans_image; +}; + +struct FontHorizontalLayout { + float base_line_y; + float line_height; + float effect_height; +}; + +struct FontVerticalLayout { + float base_line_x; + float line_width; + float effect_width; +}; + +struct GlyphState { + FontState* font; + uint32_t code; + int attribute; + FontGlyphMetrics metrics; + std::array image; + FontTransImage trans_image; +}; + +union FontTextParseResult { + void* reserved[4]; + + struct { + FontHandle font; + uint32_t code; + } font_code; + + struct { + void* reserved; + uint32_t terminate_code; + } terminate; + + struct { + void* reserved; + int error_code; + } error; +}; + +struct FontTextSource { + uint64_t system_use0; + const void* start; + const void* end; + const void* current; + FontTextParseFunction text_parser; + void* text_object; + FontHandle default_font; + void* system_use[5]; +}; + +struct FontCreateStringDetail { + uint16_t detail_id; + uint8_t detail_type; + uint8_t detections; + uint32_t orders_option; + FontHandle default_font; + struct { + FontCreateStringOrdersFunction function; + void* object; + } orders; +}; + +struct FontStringState; + +struct FontCharacterState { + FontStringState* string; + uint32_t index; + FontHandle font; + uint32_t code; + void* order; +}; + +struct FontStringState { + const FontMemory* memory; + int writing_form; + uint32_t terminate_code; + void* terminate_order; + std::vector characters; +}; + +union FontTextCodes { + struct { + void* order; + uint32_t code; + uint32_t reserved; + } text; + void* system_reserved[8]; +}; + +struct FontWriting { + void* system_use[32]; +}; + +struct FontWritingExtent { + float top; + float bottom; + float left; + float right; +}; + +struct FontWritingMetrics { + float advance_x; + float advance_y; + FontWritingExtent extent; +}; + +struct FontWritingStep { + float x; + float y; + float advance_x; + float advance_y; + FontHandle font; + struct { + uint32_t character_count : 8; + uint32_t invisible_glyph : 1; + } profile; + uint32_t glyph_code; + struct { + float x; + float y; + } positioning; + FontGlyphMetrics glyph_metrics; +}; + +struct FontWritingLetterStep { + float x; + float y; + float advance_x; + float advance_y; + struct { + uint32_t texts_count; + uint32_t texts_index; + uint32_t glyphs_count; + uint32_t glyphs_index; + uint8_t base_components_count; + uint8_t base_components_index; + uint8_t base_focus_count; + uint8_t base_focus_index; + uint8_t opposite_direction; + uint8_t character_text_count; + uint8_t base_text_count; + uint8_t mark_text_count; + uint32_t marks_text_count; + uint32_t marks_text_number; + uint32_t reserved[3]; + int32_t text_letter_offset; + } components; +}; + +struct FontWritingLineStep { + float x; + float y; + float advance_x; + float advance_y; + float spacing_progress; + void* writing_orderer; + struct { + float x; + float y; + } adjusting; + FontWritingMetrics metrics; +}; + +struct FontWritingState { + FontStringState* string; + FontCharacterState* character; + FontWritingStep step; + FontWritingLetterStep letter_step; + FontWritingMetrics metrics; + int invisible_mask; +}; + +struct FontWritingLineState { + const FontMemory* memory; + int writing_form; + std::vector steps; + size_t cursor; + FontWritingMetrics metrics; +}; + +static std::mutex g_glyph_mutex; +static std::unordered_set g_glyphs; +static std::mutex g_writing_mutex; +static std::unordered_set g_writing_states; + +static void register_glyph(GlyphState* glyph) { + std::scoped_lock lock(g_glyph_mutex); + g_glyphs.insert(glyph); +} + +static bool unregister_glyph(void* glyph) { + std::scoped_lock lock(g_glyph_mutex); + return g_glyphs.erase(glyph) != 0; +} + +static bool is_known_glyph(void* glyph) { + std::scoped_lock lock(g_glyph_mutex); + return g_glyphs.find(glyph) != g_glyphs.end(); +} + +static void register_writing_state(FontWritingState* state) { + std::scoped_lock lock(g_writing_mutex); + g_writing_states.insert(state); +} + +static bool unregister_writing_state(void* state) { + std::scoped_lock lock(g_writing_mutex); + return g_writing_states.erase(state) != 0; +} + +static bool is_known_writing_state(void* state) { + std::scoped_lock lock(g_writing_mutex); + return g_writing_states.find(state) != g_writing_states.end(); +} + +static int get_text_source_writing_form(const FontTextSource* source) { + if (source == nullptr) { + return SCE_FONT_WRITING_FORM_HORIZONTAL; + } + + const auto value = reinterpret_cast(source->system_use[0]); + return value != 0 ? static_cast(value) : SCE_FONT_WRITING_FORM_HORIZONTAL; +} + +static void set_text_source_writing_form(FontTextSource* source, int writing_form) { + if (source != nullptr) { + source->system_use[0] = reinterpret_cast(static_cast(writing_form)); + } +} + +static void append_character(FontStringState* string, FontHandle font, uint32_t code, void* order) { + if (string == nullptr) { + return; + } + + const auto index = static_cast(string->characters.size()); + string->characters.push_back(FontCharacterState {string, index, font, code, order}); +} + +static FontHandle get_string_default_font(const FontTextSource* source, + const FontCreateStringDetail* detail) { + if (detail != nullptr && detail->default_font != nullptr) { + return detail->default_font; + } + return source != nullptr ? source->default_font : nullptr; +} + +static void parse_text_bytes(FontTextSource* source, FontHandle default_font, + FontStringState* string) { + if (source == nullptr || source->start == nullptr || source->end == nullptr) { + return; + } + + auto* ptr = static_cast(source->start); + auto* end = static_cast(source->end); + + while (ptr < end) { + const auto* code_start = ptr; + uint32_t code = *ptr++; + + if ((code & 0x80u) != 0u) { + const uint32_t first = code; + if ((first & 0xe0u) == 0xc0u && ptr < end) { + code = ((first & 0x1fu) << 6u) | (*ptr++ & 0x3fu); + } else if ((first & 0xf0u) == 0xe0u && ptr + 1 < end) { + code = ((first & 0x0fu) << 12u) | ((ptr[0] & 0x3fu) << 6u) | (ptr[1] & 0x3fu); + ptr += 2; + } else if ((first & 0xf8u) == 0xf0u && ptr + 2 < end) { + code = ((first & 0x07u) << 18u) | ((ptr[0] & 0x3fu) << 12u) | + ((ptr[1] & 0x3fu) << 6u) | (ptr[2] & 0x3fu); + ptr += 3; + } + } + + append_character(string, default_font, code, const_cast(code_start)); + } + + source->current = end; +} + +static int parse_text_with_callback(FontTextSource* source, FontHandle default_font, + FontStringState* string) { + if (source == nullptr || source->text_parser == nullptr) { + parse_text_bytes(source, default_font, string); + return OK; + } + + constexpr uint32_t max_font_string_characters = 65536; + + for (uint32_t i = 0; i < max_font_string_characters; i++) { + FontTextParseResult result {}; + void* order = nullptr; + const int ret = source->text_parser(source, &order, &result); + + if (ret == SCE_FONT_TEXT_PARSER_RESULT_FONT_CODE) { + auto font = result.font_code.font != nullptr ? result.font_code.font : default_font; + append_character(string, font, result.font_code.code, order); + continue; + } + + if (ret == SCE_FONT_TEXT_PARSER_RESULT_TERMINATE) { + string->terminate_code = result.terminate.terminate_code; + string->terminate_order = order != nullptr ? order : result.terminate.reserved; + if (string->characters.empty() && source->start != source->end) { + parse_text_bytes(source, default_font, string); + } + return OK; + } + + if (ret == SCE_FONT_TEXT_PARSER_RESULT_ERROR) { + return result.error.error_code != 0 ? result.error.error_code : -1; + } + + return -1; + } + + return OK; +} + +static FontCharacterState* get_character(FontTextCharacter character) { + return const_cast(static_cast(character)); +} + +static FontCharacterState* get_next_character(FontCharacterState* character) { + if (character == nullptr || character->string == nullptr) { + return nullptr; + } + + const auto next = character->index + 1; + return next < character->string->characters.size() ? &character->string->characters[next] + : nullptr; +} + +static FontCharacterState* get_prev_character(FontCharacterState* character) { + if (character == nullptr || character->string == nullptr || character->index == 0) { + return nullptr; + } + + return &character->string->characters[character->index - 1]; +} + +static uint32_t count_characters_until(FontCharacterState* first, FontCharacterState* last) { + if (first == nullptr || first->string == nullptr) { + return 0; + } + + uint32_t count = 0; + for (auto* c = first; c != nullptr; c = get_next_character(c)) { + if (last != nullptr && c == last) { + break; + } + count++; + } + return count; +} + +static void fill_text_codes(FontTextCodes* text_codes, FontCharacterState* character, + FontCharacterState* term_character) { + if (text_codes == nullptr || character == nullptr) { + return; + } + + std::memset(text_codes, 0, sizeof(FontTextCodes)); + text_codes->text.order = character->order; + text_codes->text.code = character->code; + text_codes->system_reserved[2] = character; + text_codes->system_reserved[3] = term_character; +} + +static void fill_metrics(FontGlyphMetrics* metrics, const FontState* font = nullptr); +static void fill_metrics_for_code(FontGlyphMetrics* metrics, const FontState* font, uint32_t code); + +static void fill_writing_metrics(FontWritingMetrics* metrics, uint32_t character_count) { + if (metrics == nullptr) { + return; + } + + const float advance = static_cast(character_count) * 8.0f; + metrics->advance_x = advance; + metrics->advance_y = 0.0f; + metrics->extent.top = -12.0f; + metrics->extent.bottom = 4.0f; + metrics->extent.left = 0.0f; + metrics->extent.right = advance; +} + +static void fill_writing_step(FontWritingStep* step, FontCharacterState* character) { + if (step == nullptr) { + return; + } + + std::memset(step, 0, sizeof(FontWritingStep)); + step->advance_x = 8.0f; + step->advance_y = 0.0f; + step->font = character != nullptr ? character->font : nullptr; + step->profile.character_count = character != nullptr ? 1u : 0u; + step->profile.invisible_glyph = 0u; + step->glyph_code = character != nullptr ? character->code : 0u; + fill_metrics_for_code(&step->glyph_metrics, + static_cast(character != nullptr ? character->font : nullptr), + character != nullptr ? character->code : static_cast('?')); +} + +static void fill_writing_letter_step(FontWritingLetterStep* letter_step, + FontCharacterState* character) { + if (letter_step == nullptr) { + return; + } + + std::memset(letter_step, 0, sizeof(FontWritingLetterStep)); + letter_step->advance_x = 8.0f; + letter_step->advance_y = 0.0f; + + if (character == nullptr) { + return; + } + + letter_step->components.texts_count = 1; + letter_step->components.texts_index = character->index; + letter_step->components.glyphs_count = 1; + letter_step->components.glyphs_index = 0; + letter_step->components.base_components_count = 1; + letter_step->components.base_focus_count = 1; + letter_step->components.character_text_count = 1; + letter_step->components.base_text_count = 1; +} + +static FontWritingState* get_writing_state(FontWriting* font_writing) { + if (font_writing == nullptr) { + return nullptr; + } + + auto* state = static_cast(font_writing->system_use[0]); + return is_known_writing_state(state) ? state : nullptr; +} + +static void clear_writing_line(FontWritingLineState* line) { + if (line == nullptr) { + return; + } + + line->steps.clear(); + line->cursor = 0; + fill_writing_metrics(&line->metrics, 0); +} + +static bool font_is_bold(const FontState* font) { + if (font == nullptr) { + return false; + } + return (font->font_set_type & 0x0fu) >= 0x07u || font->effect_weight_x > 0.0f || + font->render_effect_weight_x > 0.0f; +} + +static bool font_is_italic(const FontState* font) { + if (font == nullptr) { + return false; + } + return (font->font_set_type & 0x00100000u) != 0 || font->effect_slant != 0.0f || + font->render_effect_slant != 0.0f; +} + +static const char* fallback_font_file_name(bool bold, bool italic) { + if (bold && italic) { + return "Roboto-BoldItalic.ttf"; + } + if (bold) { + return "Roboto-Bold.ttf"; + } + if (italic) { + return "Roboto-Italic.ttf"; + } + return "Roboto-Regular.ttf"; +} + +static std::vector read_file_bytes(const std::filesystem::path& path) { + std::ifstream file(path, std::ios::binary | std::ios::ate); + if (!file) { + return {}; + } + + const auto size = file.tellg(); + if (size <= 0) { + return {}; + } + + std::vector data(static_cast(size)); + file.seekg(0); + file.read(reinterpret_cast(data.data()), size); + if (!file) { + return {}; + } + + return data; +} + +static std::vector fallback_font_candidates(const char* file_name) { + const auto rel = + std::filesystem::path("3rdparty") / "tracy" / "profiler" / "src" / "font" / file_name; + std::vector paths; + + auto source_root = std::filesystem::path(__FILE__).parent_path().parent_path().parent_path(); + paths.push_back(source_root / rel); + + std::error_code ec; + auto cwd = std::filesystem::current_path(ec); + if (!ec) { + paths.push_back(cwd / rel); + paths.push_back(cwd / ".." / rel); + paths.push_back(cwd / ".." / ".." / rel); + } + + return paths; +} + +static std::vector load_fallback_font_data(bool bold, bool italic) { + for (const auto& path: fallback_font_candidates(fallback_font_file_name(bold, italic))) { + auto data = read_file_bytes(path); + if (!data.empty()) { + return data; + } + } + + return {}; +} + +static const std::vector& fallback_font_data(bool bold, bool italic) { + static const auto regular = load_fallback_font_data(false, false); + static const auto bold_data = load_fallback_font_data(true, false); + static const auto italic_data = load_fallback_font_data(false, true); + static const auto bold_italic = load_fallback_font_data(true, true); + + if (bold && italic && !bold_italic.empty()) { + return bold_italic; + } + if (bold && !bold_data.empty()) { + return bold_data; + } + if (italic && !italic_data.empty()) { + return italic_data; + } + return regular; +} + +static bool ttf_data_has_supported_magic(const uint8_t* data, uint32_t size) { + if (data == nullptr || size < 12) { + return false; + } + + return (data[0] == 0x00 && data[1] == 0x01 && data[2] == 0x00 && data[3] == 0x00) || + (data[0] == 't' && data[1] == 'r' && data[2] == 'u' && data[3] == 'e'); +} + +static bool init_stb_font(FontState* font, const uint8_t* data, uint32_t size) { + if (font == nullptr || !ttf_data_has_supported_magic(data, size)) { + return false; + } + + stbtt_fontinfo info {}; + if (stbtt_InitFont(&info, data, 0) == 0) { + return false; + } + + font->font_info = info; + font->ttf_data = data; + font->ttf_size = size; + font->has_ttf = true; + + return true; +} + +static void init_font_face(FontState* font) { + if (font == nullptr) { + return; + } + + font->ttf_data = nullptr; + font->ttf_size = 0; + font->has_ttf = false; + + if (init_stb_font(font, static_cast(font->data), font->size)) { + return; + } + + const auto& data = fallback_font_data(font_is_bold(font), font_is_italic(font)); + if (!data.empty()) { + (void)init_stb_font(font, data.data(), static_cast(data.size())); + } +} + +static bool ensure_stb_font(FontState* font) { + if (font == nullptr) { + return false; + } + if (!font->has_ttf) { + init_font_face(font); + } + return font->has_ttf; +} + +static float stb_font_pixel_height(const FontState* font) { + const float scale = (font != nullptr && font->scale_h > 1.0f ? font->scale_h : 16.0f); + return static_cast(std::clamp(static_cast(scale + 0.5f), 8, FONT_BITMAP_MAX_DIM)); +} + +static float stb_font_scale(FontState* font) { + return stbtt_ScaleForPixelHeight(&font->font_info, stb_font_pixel_height(font)); +} + +static uint32_t stb_supported_codepoint(FontState* font, uint32_t code) { + if (font == nullptr || !ensure_stb_font(font)) { + return static_cast('?'); + } + if (code > 0x10ffffu || stbtt_FindGlyphIndex(&font->font_info, static_cast(code)) == 0) { + return static_cast('?'); + } + return code; +} + +static bool get_stb_metrics(FontState* font, uint32_t code, FontGlyphMetrics* metrics) { + if (metrics == nullptr || !ensure_stb_font(font)) { + return false; + } + + code = stb_supported_codepoint(font, code); + const float scale = stb_font_scale(font); + + int advance = 0; + int lsb = 0; + int x0 = 0; + int y0 = 0; + int x1 = 0; + int y1 = 0; + stbtt_GetCodepointHMetrics(&font->font_info, static_cast(code), &advance, &lsb); + stbtt_GetCodepointBitmapBox(&font->font_info, static_cast(code), scale, scale, &x0, &y0, + &x1, &y1); + + const float width = static_cast(std::max(x1 - x0, 0)); + const float height = static_cast(std::max(y1 - y0, 0)); + const float advance_f = std::max(static_cast(advance) * scale, width); + + metrics->width = width; + metrics->height = height; + metrics->horizontal.bearing_x = static_cast(x0); + metrics->horizontal.bearing_y = static_cast(-y0); + metrics->horizontal.advance = advance_f; + metrics->vertical.bearing_x = 0.0f; + metrics->vertical.bearing_y = 0.0f; + metrics->vertical.advance = stb_font_pixel_height(font); + + return true; +} + +static bool init_stb_image(std::array* image, + FontTransImage* trans_image, uint32_t code, FontState* font) { + if (image == nullptr || trans_image == nullptr || !ensure_stb_font(font)) { + return false; + } + + code = stb_supported_codepoint(font, code); + const float scale = stb_font_scale(font); + + int x0 = 0; + int y0 = 0; + int x1 = 0; + int y1 = 0; + stbtt_GetCodepointBitmapBox(&font->font_info, static_cast(code), scale, scale, &x0, &y0, + &x1, &y1); + + image->fill(0); + + const auto width = static_cast(std::clamp(x1 - x0, 0, FONT_BITMAP_MAX_DIM)); + const auto height = static_cast(std::clamp(y1 - y0, 0, FONT_BITMAP_MAX_DIM)); + trans_image->address = image->data(); + trans_image->width_byte = FONT_BITMAP_MAX_DIM; + trans_image->image_width = width; + trans_image->image_height = height; + + if (width == 0 || height == 0) { + return true; + } + + stbtt_MakeCodepointBitmap(&font->font_info, image->data(), static_cast(width), + static_cast(height), FONT_BITMAP_MAX_DIM, scale, scale, + static_cast(code)); + + return true; +} + +static int scaled_font_height(const FontState* font) { + const float scale = (font != nullptr && font->scale_h > 1.0f ? font->scale_h : 16.0f); + return std::clamp(static_cast(scale + 0.5f), 8, FONT_BITMAP_MAX_DIM); +} + +static int scaled_font_width(const FontState* font) { + return std::clamp((scaled_font_height(font) + 1) / 2, 4, FONT_BITMAP_MAX_DIM); +} + +static void fill_bitmap_metrics(FontGlyphMetrics* metrics, const FontState* font) { + if (metrics == nullptr) { + return; + } + + const float width = static_cast(scaled_font_width(font)); + const float height = static_cast(scaled_font_height(font)); + const float ascent = height * 0.75f; + + metrics->width = width; + metrics->height = height; + metrics->horizontal.bearing_x = 0.0f; + metrics->horizontal.bearing_y = ascent; + metrics->horizontal.advance = width; + metrics->vertical.bearing_x = 0.0f; + metrics->vertical.bearing_y = 0.0f; + metrics->vertical.advance = height; +} + +static void fill_metrics_for_code(FontGlyphMetrics* metrics, const FontState* font, uint32_t code) { + if (metrics == nullptr) { + return; + } + + if (get_stb_metrics(const_cast(font), code, metrics)) { + return; + } + + fill_bitmap_metrics(metrics, font); +} + +static void fill_metrics(FontGlyphMetrics* metrics, const FontState* font) { + fill_metrics_for_code(metrics, font, static_cast('?')); +} + +static uint32_t normalize_bitmap_font_code(uint32_t code) { + if (code < 256) { + return code; + } + return static_cast('?'); +} + +static void init_image(std::array* image, + FontTransImage* trans_image, uint32_t code, const FontState* font) { + EXIT_NOT_IMPLEMENTED(image == nullptr); + EXIT_NOT_IMPLEMENTED(trans_image == nullptr); + + if (init_stb_image(image, trans_image, code, const_cast(font))) { + return; + } + + image->fill(0); + code = normalize_bitmap_font_code(code); + + const auto width = static_cast(scaled_font_width(font)); + const auto height = static_cast(scaled_font_height(font)); + + for (uint32_t y = 0; y < height; y++) { + const uint32_t src_y = std::min((y * 16u) / height, 15u); + const uint8_t row = avpriv_vga16_font[code * 16 + src_y]; + for (uint32_t x = 0; x < width; x++) { + const uint32_t src_x = std::min((x * 8u) / width, 7u); + if ((row & (0x80u >> src_x)) != 0) { + (*image)[y * FONT_BITMAP_MAX_DIM + x] = 0xff; + } + } + } + + trans_image->address = image->data(); + trans_image->width_byte = FONT_BITMAP_MAX_DIM; + trans_image->image_width = width; + trans_image->image_height = height; +} + +static void fill_render_result(FontState* font, FontRenderSurface* surf, float x, float y, + FontGlyphMetrics* metrics, FontRenderResult* result) { + FontGlyphMetrics local_metrics {}; + auto* out_metrics = (metrics != nullptr ? metrics : &local_metrics); + if (metrics == nullptr) { + fill_metrics(out_metrics, font); + } + + if (font == nullptr || result == nullptr) { + return; + } + + const auto top_x = std::max(static_cast(x + out_metrics->horizontal.bearing_x), 0); + const auto top_y = std::max(static_cast(y - out_metrics->horizontal.bearing_y), 0); + + result->trans_image = &font->trans_image; + result->surface_image.address = + (surf != nullptr ? static_cast(surf->buffer) : nullptr); + result->surface_image.width_byte = + (surf != nullptr ? static_cast(std::max(surf->width_byte, 0)) : 0); + result->surface_image.pixel_size_byte = + (surf != nullptr ? static_cast(std::max(surf->pixel_size_byte, 0)) : 0); + result->surface_image.pixel_format = 0; + result->update_rect.x = static_cast(top_x); + result->update_rect.y = static_cast(top_y); + result->update_rect.w = font->trans_image.image_width; + result->update_rect.h = font->trans_image.image_height; + result->image_metrics.bearing_x = 0.0f; + result->image_metrics.bearing_y = out_metrics->horizontal.bearing_y; + result->image_metrics.advance = out_metrics->horizontal.advance; + result->image_metrics.stride = out_metrics->horizontal.advance; + result->image_metrics.width = font->trans_image.image_width; + result->image_metrics.height = font->trans_image.image_height; +} + +static void draw_to_surface(const FontTransImage& image, FontRenderSurface* surf, float x, + float y) { + if (surf == nullptr || surf->buffer == nullptr || surf->width <= 0 || surf->height <= 0 || + surf->width_byte <= 0 || surf->pixel_size_byte <= 0) { + return; + } + + auto* dst = static_cast(surf->buffer); + const int pixel_size = surf->pixel_size_byte; + const int start_x = std::max(static_cast(x), 0); + const int start_y = std::max(static_cast(y), 0); + const int end_x = std::min(start_x + static_cast(image.image_width), surf->width); + const int end_y = std::min(start_y + static_cast(image.image_height), surf->height); + const uint32_t sx0 = + std::min(surf->scissor.x0, static_cast(std::max(surf->width, 0))); + const uint32_t sy0 = + std::min(surf->scissor.y0, static_cast(std::max(surf->height, 0))); + const uint32_t sx1 = + std::min(surf->scissor.x1, static_cast(std::max(surf->width, 0))); + const uint32_t sy1 = + std::min(surf->scissor.y1, static_cast(std::max(surf->height, 0))); + + for (int yy = start_y; yy < end_y; yy++) { + if (static_cast(yy) < sy0 || static_cast(yy) >= sy1) { + continue; + } + for (int xx = start_x; xx < end_x; xx++) { + if (static_cast(xx) < sx0 || static_cast(xx) >= sx1) { + continue; + } + const auto src = image.address[(yy - start_y) * image.width_byte + (xx - start_x)]; + if (src == 0) { + continue; + } + uint8_t* pixel = dst + yy * surf->width_byte + xx * pixel_size; + for (int i = 0; i < pixel_size; i++) { + pixel[i] = src; + } + } + } +} + +int KYTY_SYSV_ABI FontMemoryInit(FontMemory* font_memory, void* address, uint32_t size_byte, + const FontMemoryInterface* mem_interface, void* mspace_object, + FontMemoryDestroyCallback destroy_callback, void* destroy_object) { + PRINT_NAME(); + + if (font_memory == nullptr) { + return -1; + } + + font_memory->type = 1; + font_memory->attr = 0; + font_memory->size = size_byte; + font_memory->address = address; + font_memory->mspace_object = mspace_object; + font_memory->mem_interface = mem_interface; + font_memory->destroy_callback = destroy_callback; + font_memory->destroy_object = destroy_object; + font_memory->user_object = nullptr; + font_memory->parent_object = nullptr; + + LOGF("\t address = 0x%016" PRIx64 ", size = %" PRIu32 "\n", reinterpret_cast(address), + size_byte); + + return OK; +} + +int KYTY_SYSV_ABI FontMemoryTerm(FontMemory* font_memory) { + PRINT_NAME(); + + if (font_memory != nullptr) { + font_memory->type = 0; + } + + return OK; +} + +int KYTY_SYSV_ABI FontCreateLibraryWithEdition(const FontMemory* memory, + FontLibrarySelection selection, uint64_t edition, + FontLibrary* library) { + PRINT_NAME(); + + if (library == nullptr) { + return -1; + } + + auto* state = new LibraryState {}; + state->memory = memory; + state->edition = edition; + + *library = state; + + LOGF("\t memory = 0x%016" PRIx64 ", selection = 0x%016" PRIx64 ", edition = 0x%016" PRIx64 + ", library = 0x%016" PRIx64 "\n", + reinterpret_cast(memory), reinterpret_cast(selection), edition, + reinterpret_cast(*library)); + + return OK; +} + +int KYTY_SYSV_ABI FontCreateLibrary(const FontMemory* memory, FontLibrarySelection selection, + FontLibrary* library) { + return FontCreateLibraryWithEdition(memory, selection, 0, library); +} + +int KYTY_SYSV_ABI FontDestroyLibrary(FontLibrary* library) { + PRINT_NAME(); + + if (library != nullptr) { + delete static_cast(*library); + *library = nullptr; + } + + return OK; +} + +int KYTY_SYSV_ABI FontCreateRendererWithEdition(const FontMemory* memory, + FontRendererSelection selection, uint64_t edition, + FontRenderer* renderer) { + PRINT_NAME(); + + if (renderer == nullptr) { + return -1; + } + + auto* state = new RendererState {}; + state->memory = memory; + state->selection = selection; + state->edition = edition; + + *renderer = state; + + LOGF("\t memory = 0x%016" PRIx64 ", selection = 0x%016" PRIx64 ", edition = 0x%016" PRIx64 + ", renderer = 0x%016" PRIx64 "\n", + reinterpret_cast(memory), reinterpret_cast(selection), edition, + reinterpret_cast(*renderer)); + + return OK; +} + +int KYTY_SYSV_ABI FontDestroyRenderer(FontRenderer* renderer) { + PRINT_NAME(); + + if (renderer != nullptr) { + delete static_cast(*renderer); + *renderer = nullptr; + } + + return OK; +} + +static int open_font(FontLibrary library, const void* data, uint32_t size, uint32_t font_set_type, + uint32_t open_mode, FontHandle* handle) { + if (handle == nullptr) { + return -1; + } + + auto* state = new FontState {}; + state->library = library; + state->data = data; + state->size = size; + state->font_set_type = font_set_type; + state->open_mode = open_mode; + state->attribute = 0; + state->renderer = nullptr; + state->scale_w = 8.0f; + state->scale_h = 16.0f; + state->effect_weight_x = 0.0f; + state->effect_weight_y = 0.0f; + state->effect_weight_mode = 0; + state->effect_slant = 0.0f; + state->render_scale_w = 8.0f; + state->render_scale_h = 16.0f; + state->render_effect_weight_x = 0.0f; + state->render_effect_weight_y = 0.0f; + state->render_effect_weight_mode = 0; + state->render_effect_slant = 0.0f; + state->ttf_data = nullptr; + state->ttf_size = 0; + state->has_ttf = false; + init_font_face(state); + init_image(&state->fallback_image, &state->trans_image, static_cast('?'), state); + + *handle = state; + + LOGF("\t library = 0x%016" PRIx64 ", data = 0x%016" PRIx64 ", size = %" PRIu32 + ", font_set = %" PRIu32 ", open_mode = 0x%08" PRIx32 ", handle = 0x%016" PRIx64 "\n", + reinterpret_cast(library), reinterpret_cast(data), size, font_set_type, + open_mode, reinterpret_cast(*handle)); + + return OK; +} + +int KYTY_SYSV_ABI FontOpenFontSet(FontLibrary library, uint32_t font_set_type, uint32_t open_mode, + const FontOpenDetail* detail, FontHandle* handle) { + PRINT_NAME(); + + return open_font(library, detail, 0, font_set_type, open_mode, handle); +} + +int KYTY_SYSV_ABI FontOpenFontMemory(FontLibrary library, const void* font_address, + uint32_t font_size, const FontOpenDetail* /*detail*/, + FontHandle* handle) { + PRINT_NAME(); + + return open_font(library, font_address, font_size, 0, 0, handle); +} + +int KYTY_SYSV_ABI FontOpenFontInstance(FontHandle font_handle, void* setup_font, + FontHandle* out_font_handle) { + PRINT_NAME(); + + if (out_font_handle == nullptr) { + return -1; + } + + if (font_handle == nullptr) { + return open_font(nullptr, setup_font, 0, 0, 0, out_font_handle); + } + + const auto* source = static_cast(font_handle); + return open_font(source->library, source->data, source->size, source->font_set_type, + source->open_mode, out_font_handle); +} + +int KYTY_SYSV_ABI FontCloseFont(FontHandle font_handle) { + PRINT_NAME(); + + auto* font = static_cast(font_handle); + delete font; + + return OK; +} + +int KYTY_SYSV_ABI FontDefineAttribute(FontHandle font_handle, int attribute, int* old_attribute) { + PRINT_NAME(); + + auto* font = static_cast(font_handle); + if (old_attribute != nullptr) { + *old_attribute = (font != nullptr ? font->attribute : 0); + } + if (font != nullptr) { + font->attribute = attribute; + } + + LOGF("\t handle = 0x%016" PRIx64 ", attribute = %d\n", reinterpret_cast(font_handle), + attribute); + + return OK; +} + +int KYTY_SYSV_ABI FontBindRenderer(FontHandle font_handle, FontRenderer renderer) { + PRINT_NAME(); + + auto* font = static_cast(font_handle); + if (font == nullptr) { + return -1; + } + + font->renderer = renderer; + + LOGF("\t handle = 0x%016" PRIx64 ", renderer = 0x%016" PRIx64 "\n", + reinterpret_cast(font_handle), reinterpret_cast(renderer)); + + return OK; +} + +int KYTY_SYSV_ABI FontUnbindRenderer(FontHandle font_handle) { + PRINT_NAME(); + + auto* font = static_cast(font_handle); + if (font == nullptr) { + return -1; + } + + font->renderer = nullptr; + + LOGF("\t handle = 0x%016" PRIx64 "\n", reinterpret_cast(font_handle)); + + return OK; +} + +int KYTY_SYSV_ABI FontRebindRenderer(FontHandle font_handle) { + PRINT_NAME(); + + auto* font = static_cast(font_handle); + if (font == nullptr) { + return -1; + } + + LOGF("\t handle = 0x%016" PRIx64 ", renderer = 0x%016" PRIx64 "\n", + reinterpret_cast(font_handle), reinterpret_cast(font->renderer)); + + return OK; +} + +int KYTY_SYSV_ABI FontSetScalePixel(FontHandle font_handle, float w, float h) { + PRINT_NAME(); + + auto* font = static_cast(font_handle); + if (font == nullptr) { + return -1; + } + + font->scale_w = w; + font->scale_h = h; + + LOGF("\t handle = 0x%016" PRIx64 ", w = %f, h = %f\n", reinterpret_cast(font_handle), + static_cast(w), static_cast(h)); + + return OK; +} + +int KYTY_SYSV_ABI FontSetEffectSlant(FontHandle font_handle, float slant_ratio) { + PRINT_NAME(); + + auto* font = static_cast(font_handle); + if (font == nullptr) { + return -1; + } + + font->effect_slant = slant_ratio; + + LOGF("\t handle = 0x%016" PRIx64 ", slant = %f\n", reinterpret_cast(font_handle), + static_cast(slant_ratio)); + + return OK; +} + +int KYTY_SYSV_ABI FontSetEffectWeight(FontHandle font_handle, float weight_x_scale, + float weight_y_scale, uint32_t mode) { + PRINT_NAME(); + + auto* font = static_cast(font_handle); + if (font == nullptr) { + return -1; + } + + font->effect_weight_x = weight_x_scale; + font->effect_weight_y = weight_y_scale; + font->effect_weight_mode = mode; + + LOGF("\t handle = 0x%016" PRIx64 ", weight_x = %f, weight_y = %f, mode = 0x%08" PRIx32 "\n", + reinterpret_cast(font_handle), static_cast(weight_x_scale), + static_cast(weight_y_scale), mode); + + return OK; +} + +int KYTY_SYSV_ABI FontSetupRenderScalePixel(FontHandle font_handle, float w, float h) { + PRINT_NAME(); + + auto* font = static_cast(font_handle); + if (font == nullptr) { + return -1; + } + + font->render_scale_w = w; + font->render_scale_h = h; + + LOGF("\t handle = 0x%016" PRIx64 ", w = %f, h = %f\n", reinterpret_cast(font_handle), + static_cast(w), static_cast(h)); + + return OK; +} + +int KYTY_SYSV_ABI FontSetupRenderEffectSlant(FontHandle font_handle, float slant_ratio) { + PRINT_NAME(); + + auto* font = static_cast(font_handle); + if (font == nullptr) { + return -1; + } + + font->render_effect_slant = slant_ratio; + + LOGF("\t handle = 0x%016" PRIx64 ", slant = %f\n", reinterpret_cast(font_handle), + static_cast(slant_ratio)); + + return OK; +} + +int KYTY_SYSV_ABI FontSetupRenderEffectWeight(FontHandle font_handle, float weight_x_scale, + float weight_y_scale, uint32_t mode) { + PRINT_NAME(); + + auto* font = static_cast(font_handle); + if (font == nullptr) { + return -1; + } + + font->render_effect_weight_x = weight_x_scale; + font->render_effect_weight_y = weight_y_scale; + font->render_effect_weight_mode = mode; + + LOGF("\t handle = 0x%016" PRIx64 ", weight_x = %f, weight_y = %f, mode = 0x%08" PRIx32 "\n", + reinterpret_cast(font_handle), static_cast(weight_x_scale), + static_cast(weight_y_scale), mode); + + return OK; +} + +int KYTY_SYSV_ABI FontGetHorizontalLayout(FontHandle font_handle, FontHorizontalLayout* layout) { + PRINT_NAME(); + + if (font_handle == nullptr || layout == nullptr) { + return -1; + } + + auto* font = static_cast(font_handle); + if (ensure_stb_font(font)) { + const float scale = stb_font_scale(font); + int ascent = 0; + int descent = 0; + int line_gap = 0; + stbtt_GetFontVMetrics(&font->font_info, &ascent, &descent, &line_gap); + layout->base_line_y = static_cast(ascent) * scale; + layout->line_height = static_cast(ascent - descent + line_gap) * scale; + layout->effect_height = layout->line_height; + } else { + layout->base_line_y = static_cast(scaled_font_height(font)) * 0.75f; + layout->line_height = static_cast(scaled_font_height(font)); + layout->effect_height = layout->line_height; + } + + LOGF("\t handle = 0x%016" PRIx64 ", layout = 0x%016" PRIx64 "\n", + reinterpret_cast(font_handle), reinterpret_cast(layout)); + + return OK; +} + +int KYTY_SYSV_ABI FontGetVerticalLayout(FontHandle font_handle, FontVerticalLayout* layout) { + PRINT_NAME(); + + if (font_handle == nullptr || layout == nullptr) { + return -1; + } + + layout->base_line_x = 0.0f; + layout->line_width = 16.0f; + layout->effect_width = 16.0f; + + LOGF("\t handle = 0x%016" PRIx64 ", layout = 0x%016" PRIx64 "\n", + reinterpret_cast(font_handle), reinterpret_cast(layout)); + + return OK; +} + +int KYTY_SYSV_ABI FontSupportSystemFonts(FontLibrary library) { + PRINT_NAME(); + + LOGF("\t library = 0x%016" PRIx64 "\n", reinterpret_cast(library)); + + return OK; +} + +int KYTY_SYSV_ABI FontSupportExternalFonts(FontLibrary library, uint32_t font_max, + uint32_t formats) { + PRINT_NAME(); + + LOGF("\t library = 0x%016" PRIx64 ", font_max = %" PRIu32 ", formats = 0x%08" PRIx32 "\n", + reinterpret_cast(library), font_max, formats); + + return OK; +} + +int KYTY_SYSV_ABI FontAttachDeviceCacheBuffer(FontLibrary library, void* buffer, uint32_t size) { + PRINT_NAME(); + + LOGF("\t library = 0x%016" PRIx64 ", buffer = 0x%016" PRIx64 ", size = %" PRIu32 "\n", + reinterpret_cast(library), reinterpret_cast(buffer), size); + + return OK; +} + +int KYTY_SYSV_ABI FontTextSourceInit(FontTextSource* font_text_source, const void* text_address, + uint32_t text_size_byte, FontTextParseFunction text_parser, + void* text_object) { + PRINT_NAME(); + + if (font_text_source == nullptr) { + return -1; + } + + std::memset(font_text_source, 0, sizeof(FontTextSource)); + + uint32_t resolved_text_size = text_size_byte; + if (text_address != nullptr && resolved_text_size == 0) { + constexpr uint32_t max_terminated_text_size = 4096; + const auto* text = static_cast(text_address); + while (resolved_text_size < max_terminated_text_size && text[resolved_text_size] != 0) { + resolved_text_size++; + } + } + + font_text_source->start = text_address; + font_text_source->end = text_address != nullptr + ? static_cast(text_address) + resolved_text_size + : nullptr; + font_text_source->current = text_address; + font_text_source->text_parser = text_parser; + font_text_source->text_object = text_object; + font_text_source->system_use[1] = + reinterpret_cast(static_cast(resolved_text_size)); + set_text_source_writing_form(font_text_source, SCE_FONT_WRITING_FORM_HORIZONTAL); + + LOGF("\t source = 0x%016" PRIx64 ", text = 0x%016" PRIx64 ", size = %" PRIu32 + ", resolved_size = %" PRIu32 ", parser = 0x%016" PRIx64 ", object = 0x%016" PRIx64 "\n", + reinterpret_cast(font_text_source), reinterpret_cast(text_address), + text_size_byte, resolved_text_size, reinterpret_cast(text_parser), + reinterpret_cast(text_object)); + + return OK; +} + +int KYTY_SYSV_ABI FontTextSourceRewind(FontTextSource* font_text_source) { + PRINT_NAME(); + + if (font_text_source == nullptr) { + return -1; + } + + font_text_source->current = font_text_source->start; + + LOGF("\t source = 0x%016" PRIx64 "\n", reinterpret_cast(font_text_source)); + + return OK; +} + +int KYTY_SYSV_ABI FontTextSourceSetWritingForm(FontTextSource* font_text_source, int writing_form) { + PRINT_NAME(); + + if (font_text_source == nullptr) { + return -1; + } + + set_text_source_writing_form(font_text_source, writing_form); + + LOGF("\t source = 0x%016" PRIx64 ", writing_form = 0x%08x\n", + reinterpret_cast(font_text_source), writing_form); + + return OK; +} + +int KYTY_SYSV_ABI FontTextSourceSetDefaultFont(FontTextSource* font_text_source, + FontHandle default_font) { + PRINT_NAME(); + + if (font_text_source == nullptr) { + return -1; + } + + font_text_source->default_font = default_font; + + LOGF("\t source = 0x%016" PRIx64 ", default_font = 0x%016" PRIx64 "\n", + reinterpret_cast(font_text_source), reinterpret_cast(default_font)); + + return OK; +} + +int KYTY_SYSV_ABI FontCreateString(const FontMemory* memory, FontTextSource* font_text_source, + const FontCreateStringDetail* string_detail, + FontString* font_string) { + PRINT_NAME(); + + if (font_string == nullptr) { + return -1; + } + + auto* state = new FontStringState {}; + state->memory = memory; + state->writing_form = get_text_source_writing_form(font_text_source); + state->terminate_code = 0; + state->terminate_order = nullptr; + + auto default_font = get_string_default_font(font_text_source, string_detail); + const int ret = parse_text_with_callback(font_text_source, default_font, state); + if (ret != OK) { + delete state; + *font_string = nullptr; + return ret; + } + + *font_string = state; + + LOGF("\t memory = 0x%016" PRIx64 ", source = 0x%016" PRIx64 ", detail = 0x%016" PRIx64 + ", string = 0x%016" PRIx64 ", chars = %zu\n", + reinterpret_cast(memory), reinterpret_cast(font_text_source), + reinterpret_cast(string_detail), reinterpret_cast(*font_string), + state->characters.size()); + + return OK; +} + +int KYTY_SYSV_ABI FontDestroyString(FontString* font_string) { + PRINT_NAME(); + + LOGF("\t string_ptr = 0x%016" PRIx64 "\n", reinterpret_cast(font_string)); + + if (font_string != nullptr) { + delete static_cast(*font_string); + *font_string = nullptr; + } + + return OK; +} + +int KYTY_SYSV_ABI FontStringGetWritingForm(FontString font_string) { + PRINT_NAME(); + + const auto* state = static_cast(font_string); + return state != nullptr ? state->writing_form : SCE_FONT_WRITING_FORM_HORIZONTAL; +} + +uint32_t KYTY_SYSV_ABI FontStringGetTerminateCode(FontString font_string) { + PRINT_NAME(); + + const auto* state = static_cast(font_string); + return state != nullptr ? state->terminate_code : 0; +} + +void* KYTY_SYSV_ABI FontStringGetTerminateOrder(FontString font_string) { + PRINT_NAME(); + + const auto* state = static_cast(font_string); + return state != nullptr ? state->terminate_order : nullptr; +} + +FontTextCharacter KYTY_SYSV_ABI FontStringRefersTextCharacters(FontString font_string, + uint32_t* character_count) { + PRINT_NAME(); + + auto* state = static_cast(font_string); + if (character_count != nullptr) { + *character_count = state != nullptr ? static_cast(state->characters.size()) : 0; + } + + LOGF("\t string = 0x%016" PRIx64 ", count = %" PRIu32 "\n", + reinterpret_cast(font_string), + state != nullptr ? static_cast(state->characters.size()) : 0); + + return state != nullptr && !state->characters.empty() ? &state->characters.front() : nullptr; +} + +FontRenderCharacter KYTY_SYSV_ABI +FontStringRefersRenderCharacters(FontString font_string, FontTextCharacter start_character, + FontTextCharacter last_character, uint32_t* character_count) { + PRINT_NAME(); + + auto* state = static_cast(font_string); + auto* first = get_character(start_character); + auto* last = get_character(last_character); + if (first == nullptr && state != nullptr && !state->characters.empty()) { + first = &state->characters.front(); + } + + if (character_count != nullptr) { + *character_count = count_characters_until(first, last); + } + + LOGF("\t string = 0x%016" PRIx64 ", start = 0x%016" PRIx64 ", last = 0x%016" PRIx64 + ", count = %" PRIu32 "\n", + reinterpret_cast(font_string), reinterpret_cast(start_character), + reinterpret_cast(last_character), + character_count != nullptr ? *character_count : count_characters_until(first, last)); + + return first; +} + +FontTextCharacter KYTY_SYSV_ABI FontCharacterRefersTextNext(FontTextCharacter text_character) { + PRINT_NAME(); + + return get_next_character(get_character(text_character)); +} + +FontTextCharacter KYTY_SYSV_ABI FontCharacterRefersTextBack(FontTextCharacter text_character) { + PRINT_NAME(); + + return get_prev_character(get_character(text_character)); +} + +FontTextCodes* KYTY_SYSV_ABI FontCharactersRefersTextCodes(FontTextCharacter text_character, + FontTextCharacter term_character, + FontTextCodes* text_codes) { + PRINT_NAME(); + + auto* character = get_character(text_character); + fill_text_codes(text_codes, character, get_character(term_character)); + + return text_codes; +} + +FontTextCodes* KYTY_SYSV_ABI FontTextCodesStepNext(FontTextCodes* text_codes_step) { + PRINT_NAME(); + + if (text_codes_step == nullptr) { + return nullptr; + } + + auto* current = static_cast(text_codes_step->system_reserved[2]); + auto* term = static_cast(text_codes_step->system_reserved[3]); + auto* next = get_next_character(current); + if (next == nullptr || next == term) { + return nullptr; + } + + fill_text_codes(text_codes_step, next, term); + return text_codes_step; +} + +FontTextCodes* KYTY_SYSV_ABI FontTextCodesStepBack(FontTextCodes* text_codes_step) { + PRINT_NAME(); + + if (text_codes_step == nullptr) { + return nullptr; + } + + auto* current = static_cast(text_codes_step->system_reserved[2]); + auto* term = static_cast(text_codes_step->system_reserved[3]); + auto* prev = get_prev_character(current); + if (prev == nullptr) { + return nullptr; + } + + fill_text_codes(text_codes_step, prev, term); + return text_codes_step; +} + +int KYTY_SYSV_ABI FontCharacterGetBidiLevel(FontTextCharacter text_character, int* bidi_level) { + PRINT_NAME(); + + if (text_character == nullptr || bidi_level == nullptr) { + return -1; + } + + *bidi_level = 0; + return OK; +} + +uint32_t KYTY_SYSV_ABI FontCharacterLooksWhiteSpace(FontTextCharacter text_character) { + PRINT_NAME(); + + auto* character = get_character(text_character); + if (character == nullptr) { + return 0; + } + + const auto code = character->code; + return (code == ' ' || code == '\t' || code == '\n' || code == '\r') ? 1u : 0u; +} + +uint32_t KYTY_SYSV_ABI FontCharacterLooksFormatCharacters(FontTextCharacter text_character) { + PRINT_NAME(); + + return text_character != nullptr ? 0u : 0u; +} + +int KYTY_SYSV_ABI FontCharacterGetTextFontCode(FontTextCharacter text_character, + FontHandle* font_handle, uint32_t* text_code) { + PRINT_NAME(); + + auto* character = get_character(text_character); + if (character == nullptr) { + return -1; + } + + if (font_handle != nullptr) { + *font_handle = character->font; + } + if (text_code != nullptr) { + *text_code = character->code; + } + + return OK; +} + +int KYTY_SYSV_ABI FontCharacterGetTextOrder(FontTextCharacter text_character, void** text_order) { + PRINT_NAME(); + + auto* character = get_character(text_character); + if (character == nullptr || text_order == nullptr) { + return -1; + } + + *text_order = character->order; + return OK; +} + +int KYTY_SYSV_ABI FontCharacterGetSyllableStringState(FontTextCharacter text_character, + int* syllable_string_state) { + PRINT_NAME(); + + if (text_character == nullptr || syllable_string_state == nullptr) { + return -1; + } + + *syllable_string_state = 0; + return OK; +} + +int KYTY_SYSV_ABI FontWritingInit(FontWriting* font_writing, FontString font_string, + FontRenderCharacter font_character) { + PRINT_NAME(); + + if (font_writing == nullptr) { + return -1; + } + + if (auto* old_state = get_writing_state(font_writing); old_state != nullptr) { + unregister_writing_state(old_state); + delete old_state; + } + std::memset(font_writing, 0, sizeof(FontWriting)); + + auto* state = new FontWritingState {}; + state->string = static_cast(font_string); + state->character = get_character(font_character); + state->invisible_mask = 0; + if (state->character == nullptr && state->string != nullptr && + !state->string->characters.empty()) { + state->character = &state->string->characters.front(); + } + + fill_writing_step(&state->step, state->character); + fill_writing_letter_step(&state->letter_step, state->character); + fill_writing_metrics(&state->metrics, state->step.profile.character_count); + register_writing_state(state); + + font_writing->system_use[0] = state; + font_writing->system_use[1] = &state->step; + font_writing->system_use[2] = state->character; + font_writing->system_use[3] = &state->letter_step; + font_writing->system_use[4] = &state->metrics; + + LOGF("\t writing = 0x%016" PRIx64 ", string = 0x%016" PRIx64 ", character = 0x%016" PRIx64 + ", step = 0x%016" PRIx64 ", letter_step = 0x%016" PRIx64 "\n", + reinterpret_cast(font_writing), reinterpret_cast(font_string), + reinterpret_cast(font_character), reinterpret_cast(&state->step), + reinterpret_cast(&state->letter_step)); + + return OK; +} + +int KYTY_SYSV_ABI FontWritingSetMaskInvisible(FontWriting* font_writing, int mask) { + PRINT_NAME(); + + auto* state = get_writing_state(font_writing); + if (state == nullptr) { + return -1; + } + + state->invisible_mask = mask; + return OK; +} + +const FontWritingStep* KYTY_SYSV_ABI FontWritingRefersRenderStep(FontWriting* font_writing) { + PRINT_NAME(); + + auto* state = get_writing_state(font_writing); + auto* step = state != nullptr ? &state->step : nullptr; + LOGF("\t writing = 0x%016" PRIx64 ", step = 0x%016" PRIx64 "\n", + reinterpret_cast(font_writing), reinterpret_cast(step)); + return step; +} + +FontTextCharacter KYTY_SYSV_ABI FontWritingRefersRenderStepCharacter(FontWriting* font_writing, + const void** letter_step) { + PRINT_NAME(); + + auto* state = get_writing_state(font_writing); + if (letter_step != nullptr) { + *letter_step = state != nullptr ? &state->letter_step : nullptr; + } + LOGF("\t writing = 0x%016" PRIx64 ", character = 0x%016" PRIx64 ", letter_step = 0x%016" PRIx64 + "\n", + reinterpret_cast(font_writing), + reinterpret_cast(state != nullptr ? state->character : nullptr), + reinterpret_cast(letter_step != nullptr ? *letter_step : nullptr)); + return state != nullptr ? state->character : nullptr; +} + +int KYTY_SYSV_ABI FontWritingGetRenderMetrics(FontWriting* font_writing, + FontWritingMetrics* writing_metrics) { + PRINT_NAME(); + + auto* state = get_writing_state(font_writing); + if (state == nullptr || writing_metrics == nullptr) { + return -1; + } + + *writing_metrics = state->metrics; + return OK; +} + +int KYTY_SYSV_ABI FontCreateWritingLine(const FontMemory* memory, int writing_form, + const void* writing_line_detail, + FontWritingLine* writing_line) { + PRINT_NAME(); + + if (writing_line == nullptr) { + return -1; + } + + auto* state = new FontWritingLineState {}; + state->memory = memory; + state->writing_form = writing_form; + clear_writing_line(state); + *writing_line = state; + + LOGF("\t memory = 0x%016" PRIx64 ", writing_form = 0x%08x" + ", detail = 0x%016" PRIx64 ", line = 0x%016" PRIx64 "\n", + reinterpret_cast(memory), writing_form, + reinterpret_cast(writing_line_detail), + reinterpret_cast(*writing_line)); + + return OK; +} + +int KYTY_SYSV_ABI FontDestroyWritingLine(FontWritingLine* writing_line) { + PRINT_NAME(); + + if (writing_line != nullptr) { + delete static_cast(*writing_line); + *writing_line = nullptr; + } + + return OK; +} + +int KYTY_SYSV_ABI FontWritingLineClear(FontWritingLine writing_line) { + PRINT_NAME(); + + auto* state = static_cast(writing_line); + if (state == nullptr) { + return -1; + } + + clear_writing_line(state); + return OK; +} + +int KYTY_SYSV_ABI FontWritingLineWritesOrder(FontWritingLine writing_line, + uint64_t writing_attribute, + const FontWritingMetrics* writing_metrics, + void* writing_orderer) { + PRINT_NAME(); + + auto* state = static_cast(writing_line); + if (state == nullptr) { + return -1; + } + + FontWritingMetrics order_metrics {}; + if (writing_metrics != nullptr) { + order_metrics = *writing_metrics; + } else { + fill_writing_metrics(&order_metrics, 1); + } + + FontWritingLineStep step {}; + step.x = state->metrics.advance_x; + step.y = 0.0f; + step.advance_x = order_metrics.advance_x; + step.advance_y = order_metrics.advance_y; + step.spacing_progress = 0.0f; + step.writing_orderer = writing_orderer; + step.metrics = order_metrics; + state->steps.push_back(step); + state->cursor = 0; + + const float old_advance = state->metrics.advance_x; + state->metrics.advance_x += order_metrics.advance_x; + state->metrics.advance_y += order_metrics.advance_y; + if (state->steps.size() == 1) { + state->metrics.extent.top = order_metrics.extent.top; + state->metrics.extent.bottom = order_metrics.extent.bottom; + state->metrics.extent.left = old_advance + order_metrics.extent.left; + state->metrics.extent.right = old_advance + order_metrics.extent.right; + } else { + state->metrics.extent.top = std::min(state->metrics.extent.top, order_metrics.extent.top); + state->metrics.extent.bottom = + std::max(state->metrics.extent.bottom, order_metrics.extent.bottom); + state->metrics.extent.left = + std::min(state->metrics.extent.left, old_advance + order_metrics.extent.left); + state->metrics.extent.right = + std::max(state->metrics.extent.right, old_advance + order_metrics.extent.right); + } + + LOGF("\t line = 0x%016" PRIx64 ", attr = 0x%016" PRIx64 ", metrics = 0x%016" PRIx64 + ", orderer = 0x%016" PRIx64 ", steps = %zu, advance = %f\n", + reinterpret_cast(writing_line), writing_attribute, + reinterpret_cast(writing_metrics), reinterpret_cast(writing_orderer), + state->steps.size(), static_cast(state->metrics.advance_x)); + + return OK; +} + +int KYTY_SYSV_ABI FontWritingLineGetOrderingSpace(FontWritingLine writing_line, float* head_space, + float* inline_space, float* tail_space, + float* advance_space) { + PRINT_NAME(); + + auto* state = static_cast(writing_line); + if (state == nullptr) { + return -1; + } + + const float advance = state->metrics.advance_x; + if (head_space != nullptr) { + *head_space = advance; + } + if (inline_space != nullptr) { + *inline_space = 0.0f; + } + if (tail_space != nullptr) { + *tail_space = advance; + } + if (advance_space != nullptr) { + *advance_space = 0.0f; + } + + return OK; +} + +FontWritingLineStep* KYTY_SYSV_ABI FontWritingLineRefersRenderStep(FontWritingLine writing_line) { + PRINT_NAME(); + + auto* state = static_cast(writing_line); + if (state == nullptr || state->cursor >= state->steps.size()) { + LOGF("\t line = 0x%016" PRIx64 ", step = 0x%016" PRIx64 "\n", + reinterpret_cast(writing_line), uint64_t {0}); + return nullptr; + } + + auto* step = &state->steps[state->cursor++]; + LOGF("\t line = 0x%016" PRIx64 ", step = 0x%016" PRIx64 + ", index = %zu/%zu, orderer = 0x%016" PRIx64 "\n", + reinterpret_cast(writing_line), reinterpret_cast(step), state->cursor, + state->steps.size(), reinterpret_cast(step->writing_orderer)); + return step; +} + +int KYTY_SYSV_ABI FontWritingLineGetRenderMetrics(FontWritingLine writing_line, + FontWritingMetrics* writing_metrics) { + PRINT_NAME(); + + auto* state = static_cast(writing_line); + if (state == nullptr || writing_metrics == nullptr) { + return -1; + } + + *writing_metrics = state->metrics; + return OK; +} + +int KYTY_SYSV_ABI FontGetRenderCharGlyphMetrics(FontHandle font_handle, uint32_t code, + FontGlyphMetrics* metrics) { + PRINT_NAME(); + + LOGF("\t handle = 0x%016" PRIx64 ", code = 0x%08" PRIx32 ", metrics = 0x%016" PRIx64 "\n", + reinterpret_cast(font_handle), code, reinterpret_cast(metrics)); + + fill_metrics_for_code(metrics, static_cast(font_handle), code); + + return OK; +} + +int KYTY_SYSV_ABI FontGetCharGlyphMetrics(FontHandle font_handle, uint32_t code, + FontGlyphMetrics* metrics) { + PRINT_NAME(); + + LOGF("\t handle = 0x%016" PRIx64 ", code = 0x%08" PRIx32 ", metrics = 0x%016" PRIx64 "\n", + reinterpret_cast(font_handle), code, reinterpret_cast(metrics)); + + fill_metrics_for_code(metrics, static_cast(font_handle), code); + + return OK; +} + +void KYTY_SYSV_ABI FontRenderSurfaceInit(FontRenderSurface* surf, void* buffer, int buf_width_byte, + int pixel_size_byte, int width, int height) { + PRINT_NAME(); + + if (surf == nullptr) { + return; + } + + std::memset(surf, 0, sizeof(FontRenderSurface)); + + surf->buffer = buffer; + surf->width_byte = buf_width_byte; + surf->pixel_size_byte = static_cast(pixel_size_byte); + surf->width = std::max(width, 0); + surf->height = std::max(height, 0); + surf->scissor.x0 = 0; + surf->scissor.y0 = 0; + surf->scissor.x1 = static_cast(surf->width); + surf->scissor.y1 = static_cast(surf->height); + + LOGF("\t surf = 0x%016" PRIx64 ", buffer = 0x%016" PRIx64 + ", width_byte = %d, pixel_size = %d, width = %d, height = %d\n", + reinterpret_cast(surf), reinterpret_cast(buffer), buf_width_byte, + pixel_size_byte, width, height); +} + +void KYTY_SYSV_ABI FontRenderSurfaceSetScissor(FontRenderSurface* surf, uint32_t x0, uint32_t y0, + uint32_t x1, uint32_t y1) { + PRINT_NAME(); + + if (surf == nullptr) { + return; + } + + surf->scissor.x0 = x0; + surf->scissor.y0 = y0; + surf->scissor.x1 = x1; + surf->scissor.y1 = y1; + + LOGF("\t surf = 0x%016" PRIx64 ", x0 = %" PRIu32 ", y0 = %" PRIu32 ", x1 = %" PRIu32 + ", y1 = %" PRIu32 "\n", + reinterpret_cast(surf), x0, y0, x1, y1); +} + +int KYTY_SYSV_ABI FontGenerateCharGlyph(FontHandle font_handle, uint32_t code, + const FontGenerateGlyphDetail* detail, void** font_glyph) { + PRINT_NAME(); + + if (font_handle == nullptr || font_glyph == nullptr) { + return -1; + } + + auto* font = static_cast(font_handle); + auto* glyph = new GlyphState {}; + glyph->font = font; + glyph->code = code; + fill_metrics_for_code(&glyph->metrics, font, code); + init_image(&glyph->image, &glyph->trans_image, code, font); + register_glyph(glyph); + + *font_glyph = glyph; + + LOGF("\t handle = 0x%016" PRIx64 ", code = 0x%08" PRIx32 ", detail = 0x%016" PRIx64 + ", glyph = 0x%016" PRIx64 "\n", + reinterpret_cast(font_handle), code, reinterpret_cast(detail), + reinterpret_cast(*font_glyph)); + + return OK; +} + +int KYTY_SYSV_ABI FontDeleteGlyph(const FontMemory* memory, void** font_glyph) { + PRINT_NAME(); + + LOGF("\t memory = 0x%016" PRIx64 ", glyph_ptr = 0x%016" PRIx64 "\n", + reinterpret_cast(memory), reinterpret_cast(font_glyph)); + + if (font_glyph != nullptr) { + if (unregister_glyph(*font_glyph)) { + delete static_cast(*font_glyph); + } + *font_glyph = nullptr; + } + + return OK; +} + +int KYTY_SYSV_ABI FontGlyphDefineAttribute(void* font_glyph, int attribute, int* old_attribute) { + PRINT_NAME(); + + auto* glyph = (is_known_glyph(font_glyph) ? static_cast(font_glyph) : nullptr); + if (old_attribute != nullptr) { + *old_attribute = (glyph != nullptr ? glyph->attribute : 0); + } + if (glyph != nullptr) { + glyph->attribute = attribute; + } + + LOGF("\t glyph = 0x%016" PRIx64 ", attribute = %d\n", reinterpret_cast(font_glyph), + attribute); + + return OK; +} + +int KYTY_SYSV_ABI FontRenderCharGlyphImageHorizontal(FontHandle font_handle, uint32_t code, + FontRenderSurface* surf, float x, float y, + FontGlyphMetrics* metrics, + FontRenderResult* result) { + PRINT_NAME(); + + auto* font = static_cast(font_handle); + if (font_handle == nullptr) { + return -1; + } + + init_image(&font->fallback_image, &font->trans_image, code, font); + FontGlyphMetrics local_metrics {}; + auto* draw_metrics = (metrics != nullptr ? metrics : &local_metrics); + fill_metrics_for_code(draw_metrics, font, code); + fill_render_result(font, surf, x, y, draw_metrics, result); + const auto top_x = x + draw_metrics->horizontal.bearing_x; + const auto top_y = y - draw_metrics->horizontal.bearing_y; + draw_to_surface(font->trans_image, surf, top_x, top_y); + + LOGF("\t handle = 0x%016" PRIx64 ", code = 0x%08" PRIx32 ", surf = 0x%016" PRIx64 + ", x = %f, y = %f, result = 0x%016" PRIx64 "\n", + reinterpret_cast(font_handle), code, reinterpret_cast(surf), + static_cast(x), static_cast(y), reinterpret_cast(result)); + + return OK; +} + +} // namespace Font + +LIB_DEFINE(InitFont_1) { + LIB_FUNC("CUKn5pX-NVY", Font::FontAttachDeviceCacheBuffer); + LIB_FUNC("vzHs3C8lWJk", Font::FontCloseFont); + LIB_FUNC("WaSFJoRWXaI", Font::FontCreateRendererWithEdition); + LIB_FUNC("exAxkyVLt0s", Font::FontDestroyRenderer); + LIB_FUNC("3OdRkSjOcog", Font::FontBindRenderer); + LIB_FUNC("Z2cdsqJH+5k", Font::FontRebindRenderer); + LIB_FUNC("1QjhKxrsOB8", Font::FontUnbindRenderer); + LIB_FUNC("N1EBMeGhf7E", Font::FontSetScalePixel); + LIB_FUNC("TMtqoFQjjbA", Font::FontSetEffectSlant); + LIB_FUNC("v0phZwa4R5o", Font::FontSetEffectWeight); + LIB_FUNC("6vGCkkQJOcI", Font::FontSetupRenderScalePixel); + LIB_FUNC("lz9y9UFO2UU", Font::FontSetupRenderEffectSlant); + LIB_FUNC("XIGorvLusDQ", Font::FontSetupRenderEffectWeight); + LIB_FUNC("imxVx8lm+KM", Font::FontGetHorizontalLayout); + LIB_FUNC("3BrWWFU+4ts", Font::FontGetVerticalLayout); + LIB_FUNC("8h-SOB-asgk", Font::FontDefineAttribute); + LIB_FUNC("nWrfPI4Okmg", Font::FontCreateLibrary); + LIB_FUNC("n590hj5Oe-k", Font::FontCreateLibraryWithEdition); + LIB_FUNC("MO24vDhmS4E", Font::FontCreateString); + LIB_FUNC("7rogx92EEyc", Font::FontCreateWritingLine); + LIB_FUNC("SSCaczu2aMQ", Font::FontDestroyString); + LIB_FUNC("PEjv7CVDRYs", Font::FontDestroyWritingLine); + LIB_FUNC("LHDoRWVFGqk", Font::FontDeleteGlyph); + LIB_FUNC("FXP359ygujs", Font::FontDestroyLibrary); + LIB_FUNC("C-4Qw5Srlyw", Font::FontGenerateCharGlyph); + LIB_FUNC("L97d+3OgMlE", Font::FontGetCharGlyphMetrics); + LIB_FUNC("IQtleGLL5pQ", Font::FontGetRenderCharGlyphMetrics); + LIB_FUNC("8-zmgsxkBek", Font::FontGlyphDefineAttribute); + LIB_FUNC("whrS4oksXc4", Font::FontMemoryInit); + LIB_FUNC("h6hIgxXEiEc", Font::FontMemoryTerm); + LIB_FUNC("JzCH3SCFnAU", Font::FontOpenFontInstance); + LIB_FUNC("KXUpebrFk1U", Font::FontOpenFontMemory); + LIB_FUNC("cKYtVmeSTcw", Font::FontOpenFontSet); + LIB_FUNC("3G4zhgKuxE8", Font::FontRenderCharGlyphImageHorizontal); + LIB_FUNC("kAenWy1Zw5o", Font::FontRenderCharGlyphImageHorizontal); + LIB_FUNC("i6UNdSig1uE", Font::FontRenderCharGlyphImageHorizontal); + LIB_FUNC("gdUCnU0gHdI", Font::FontRenderSurfaceInit); + LIB_FUNC("vRxf4d0ulPs", Font::FontRenderSurfaceSetScissor); + LIB_FUNC("ObkDGDBsVtw", Font::FontStringGetTerminateCode); + LIB_FUNC("+B-xlbiWDJ4", Font::FontStringGetTerminateOrder); + LIB_FUNC("o1vIEHeb6tw", Font::FontStringGetWritingForm); + LIB_FUNC("hq5LffQjz-s", Font::FontStringRefersRenderCharacters); + LIB_FUNC("Avv7OApgCJk", Font::FontStringRefersTextCharacters); + LIB_FUNC("oaJ1BpN2FQk", Font::FontTextSourceInit); + LIB_FUNC("VRFd3diReec", Font::FontTextSourceRewind); + LIB_FUNC("eCRMCSk96NU", Font::FontTextSourceSetDefaultFont); + LIB_FUNC("OqQKX0h5COw", Font::FontTextSourceSetWritingForm); + LIB_FUNC("6DFUkCwQLa8", Font::FontCharacterGetBidiLevel); + LIB_FUNC("coCrV6IWplE", Font::FontCharacterGetSyllableStringState); + LIB_FUNC("zN3+nuA0SFQ", Font::FontCharacterGetTextFontCode); + LIB_FUNC("mxgmMj-Mq-o", Font::FontCharacterGetTextOrder); + LIB_FUNC("-P6X35Rq2-E", Font::FontCharacterLooksFormatCharacters); + LIB_FUNC("SaRlqtqaCew", Font::FontCharacterLooksWhiteSpace); + LIB_FUNC("6Gqlv5KdTbU", Font::FontCharacterRefersTextBack); + LIB_FUNC("BkjBP+YC19w", Font::FontCharacterRefersTextNext); + LIB_FUNC("lVSR5ftvNag", Font::FontCharactersRefersTextCodes); + LIB_FUNC("IPoYwwlMx-g", Font::FontTextCodesStepBack); + LIB_FUNC("olSmXY+XP1E", Font::FontTextCodesStepNext); + LIB_FUNC("fljdejMcG1c", Font::FontWritingGetRenderMetrics); + LIB_FUNC("fD5rqhEXKYQ", Font::FontWritingInit); + LIB_FUNC("1+DgKL0haWQ", Font::FontWritingLineClear); + LIB_FUNC("JQKWIsS9joE", Font::FontWritingLineGetOrderingSpace); + LIB_FUNC("nlU2VnfpqTM", Font::FontWritingLineGetRenderMetrics); + LIB_FUNC("+FYcYefsVX0", Font::FontWritingLineRefersRenderStep); + LIB_FUNC("wyKFUOWdu3Q", Font::FontWritingLineWritesOrder); + LIB_FUNC("W-2WOXEHGck", Font::FontWritingRefersRenderStep); + LIB_FUNC("f4Onl7efPEY", Font::FontWritingRefersRenderStepCharacter); + LIB_FUNC("BbCZjJizU4A", Font::FontWritingSetMaskInvisible); + LIB_FUNC("mz2iTY0MK4A", Font::FontSupportExternalFonts); + LIB_FUNC("SsRbbCiWoGw", Font::FontSupportSystemFonts); +} + +} // namespace Libs diff --git a/src/libs/libFontFt.cpp b/src/libs/libFontFt.cpp new file mode 100644 index 000000000..3cfee91fb --- /dev/null +++ b/src/libs/libFontFt.cpp @@ -0,0 +1,45 @@ +#include "common/abi.h" +#include "common/common.h" +#include "libs/errno.h" +#include "libs/libs.h" +#include "loader/symbolDatabase.h" + +namespace Libs { + +LIB_VERSION("FontFt", 1, "FontFt", 1, 1); + +namespace FontFt { + +struct SelectionState { + int value; + int kind; +}; + +void* KYTY_SYSV_ABI FontSelectLibraryFt(int value) { + PRINT_NAME(); + + static SelectionState s_selection {}; + s_selection.value = value; + s_selection.kind = 0; + + return &s_selection; +} + +void* KYTY_SYSV_ABI FontSelectRendererFt(int value) { + PRINT_NAME(); + + static SelectionState s_selection {}; + s_selection.value = value; + s_selection.kind = 1; + + return &s_selection; +} + +} // namespace FontFt + +LIB_DEFINE(InitFontFt_1) { + LIB_FUNC("oM+XCzVG3oM", FontFt::FontSelectLibraryFt); + LIB_FUNC("Xx974EW-QFY", FontFt::FontSelectRendererFt); +} + +} // namespace Libs diff --git a/src/libs/libGraphicsDriver.cpp b/src/libs/libGraphicsDriver.cpp new file mode 100644 index 000000000..5bdeca50b --- /dev/null +++ b/src/libs/libGraphicsDriver.cpp @@ -0,0 +1,324 @@ +#include "common/abi.h" +#include "libs/agc.h" +#include "libs/libs.h" +#include "loader/symbolDatabase.h" + +#include + +namespace Libs { + +namespace LibGen5 { + +LIB_VERSION("Graphics5", 1, "Graphics5", 1, 1); + +namespace Gen5 = Graphics::Gen5; + +template +static T ReadUnaligned(const uint8_t* ptr, uint32_t offset) { + T ret {}; + std::memcpy(&ret, ptr + offset, sizeof(T)); + return ret; +} + +static uint32_t BitExtract(uint32_t value, uint32_t start, uint32_t len) { + return (value >> start) & ((1u << len) - 1u); +} + +static void Graphics5UnknownDbWriteDefault(uint64_t* out, uint32_t first) { + constexpr uint64_t default_entry = 0x10000000ull; + + for (uint32_t i = first; i < 32u; i++) { + const auto low = static_cast(default_entry) + i; + const auto high = static_cast(default_entry >> 37u) * 32u + i; + out[i] = (static_cast(high) << 32u) | low; + } +} + +static uint32_t Graphics5UnknownDbApplyTwoBitField(uint32_t value, uint32_t field, uint32_t shift) { + const auto mask = 0x3u << shift; + + switch (field & 0x3u) { + case 0: return value & ~mask; + case 1: return (value & ~mask) | (0x1u << shift); + case 2: return (value & ~mask) | (0x2u << shift); + default: return value | mask; + } +} + +static uint32_t Graphics5UnknownDbApplyFinalMask(uint32_t flags, uint32_t src, + uint32_t mask_value) { + flags &= 0xffffffe0u; + flags |= BitExtract(mask_value, 8u, 5u); + flags &= 0xfffffbffu; + flags |= ((src & 0x400000u) != 0 ? 0x400u : ((src >> 14u) & 0x400u)); + return flags; +} + +static int KYTY_SYSV_ABI Graphics5UnknownDb(uint64_t arg0, uint64_t arg1, uint64_t arg2, + uint64_t arg3, uint64_t arg4, uint64_t arg5) { + PRINT_NAME(); + + (void)arg3; + (void)arg4; + (void)arg5; + + auto* out = reinterpret_cast(arg0); + + if (arg2 == 0) { + Graphics5UnknownDbWriteDefault(out, 0); + return 0; + } + + const auto* desc = reinterpret_cast(arg1); + const auto* src = reinterpret_cast(arg2); + + const auto count = ReadUnaligned(src, 0x50u); + + if (count == 0) { + Graphics5UnknownDbWriteDefault(out, 0); + return 0; + } + + const auto mask_count = ReadUnaligned(desc, 0x56u); + const auto* masks = reinterpret_cast(ReadUnaligned(desc, 0x38u)); + const auto* src_entries = + reinterpret_cast(ReadUnaligned(src, 0x30u)); + + for (uint32_t i = 0; i < count; i++) { + auto src_value = src_entries[i]; + auto mask_index = static_cast(mask_count); + + for (uint32_t j = 0; j < mask_count; j++) { + if ((static_cast(masks[j]) ^ static_cast(src_value)) == 0) { + mask_index = j; + break; + } + } + + const bool has_mask = (mask_index < mask_count); + const auto mode = (src_value >> 20u) & 0x3u; + auto flags = 0u; + + if (mode == 0) { + flags = (((src_value >> 24u) & 0x1u) | (has_mask ? 0u : 1u)) << 5u; + flags = Graphics5UnknownDbApplyTwoBitField(flags, (src_value >> 28u) & 0x3u, 8u); + } else { + const auto shifted_mode = (src_value << 4u) & 0x03000000u; + flags = shifted_mode + 0x80000u; + + if (mode == 2) { + flags &= 0xffefffdfu; + if (has_mask) { + flags |= ((~(masks[mask_index] & src_value) >> 16u) & 0x20u); + } else { + flags |= 0x20u; + } + flags = Graphics5UnknownDbApplyTwoBitField(flags, (src_value >> 30u) & 0x3u, 8u); + flags = Graphics5UnknownDbApplyTwoBitField(flags, (src_value >> 30u) & 0x3u, 21u); + } else { + if (has_mask) { + const auto masked = masks[mask_index] & src_value; + flags &= 0xffffffdfu; + flags |= (masked >> 15u) & 0x20u; + flags ^= 0x20u; + flags &= 0xffefffffu; + flags |= ((~masked >> 1u) & 0x100000u); + flags = + Graphics5UnknownDbApplyTwoBitField(flags, (src_value >> 30u) & 0x3u, 8u); + } else { + flags |= 0x100020u; + flags = + Graphics5UnknownDbApplyTwoBitField(flags, (src_value >> 28u) & 0x3u, 8u); + } + flags = Graphics5UnknownDbApplyTwoBitField(flags, (src_value >> 30u) & 0x3u, 21u); + } + } + + if (has_mask) { + flags = Graphics5UnknownDbApplyFinalMask(flags, src_value, masks[mask_index]); + } else { + flags &= 0xfffffbe0u; + } + + out[i] = (static_cast(flags) << 32u) | (0x10000000u + i); + } + + if (count < 32u) { + Graphics5UnknownDbWriteDefault(out, count); + } + + return 0; +} + +LIB_DEFINE(InitGraphicsDriver_1) { + PRINT_NAME_ENABLE(true); + + LIB_FUNC("23LRUSvYu1M", Gen5::GraphicsInit); + LIB_FUNC("2JtWUUiYBXs", Gen5::GraphicsGetRegisterDefaults2); + LIB_FUNC("wRbq6ZjNop4", Gen5::GraphicsGetRegisterDefaults2Internal); + LIB_FUNC("f3dg2CSgRKY", Gen5::GraphicsCreateShader); + LIB_FUNC("dolOmWH+huQ", Gen5::GraphicsUnknownGetFusedShaderSize); + LIB_FUNC("fd5Bp5tGTgo", Gen5::GraphicsUnknownFuseShaderHalves); + LIB_FUNC("vcmNN+AAXnY", Gen5::GraphicsSetCxRegIndirectPatchSetAddress); + LIB_FUNC("Qrj4c+61z4A", Gen5::GraphicsSetShRegIndirectPatchSetAddress); + LIB_FUNC("6lNcCp+fxi4", Gen5::GraphicsSetUcRegIndirectPatchSetAddress); + LIB_FUNC("whb1RL7K4Ss", Gen5::GraphicsSetCxRegIndirectPatchSetNumRegisters); + LIB_FUNC("nCUgItdN2ms", Gen5::GraphicsSetShRegIndirectPatchSetNumRegisters); + LIB_FUNC("fRG-JOH5+sI", Gen5::GraphicsSetUcRegIndirectPatchSetNumRegisters); + LIB_FUNC("d-6uF9sZDIU", Gen5::GraphicsSetCxRegIndirectPatchAddRegisters); + LIB_FUNC("z2duB-hHQSM", Gen5::GraphicsSetShRegIndirectPatchAddRegisters); + LIB_FUNC("vRoArM9zaIk", Gen5::GraphicsSetUcRegIndirectPatchAddRegisters); + LIB_FUNC("D9sr1xGUriE", Gen5::GraphicsCreatePrimState); + LIB_FUNC("Y3ymLfZ1384", Gen5::GraphicsUpdatePrimState); + LIB_FUNC("HV4j+E0MBHE", Gen5::GraphicsCreateInterpolantMapping); + LIB_FUNC("V++UgBtQhn0", Gen5::GraphicsGetDataPacketPayloadAddress); + LIB_FUNC("s+VGAMDQ0AQ", Gen5::GraphicsGetDataPacketPayloadRange); + LIB_FUNC("fPSCdQxgpSw", Gen5::GraphicsWriteDataPatchSetAddressOrOffset); + LIB_FUNC("bxca0BK4FNg", Gen5::GraphicsUnknownJumpPatchSetTarget); + LIB_FUNC("2BS4EtAaF28", Gen5::GraphicsJumpPatchSetTarget); + LIB_FUNC("h9z6+0hEydk", Gen5::GraphicsSuspendPoint); + LIB_FUNC("qj7QZpgr9Uw", Gen5::GraphicsUnknownQj7QZpgr9Uw); + LIB_FUNC("BfBDZGbti7A", Gen5::GraphicsGetIsTrinityMode); + LIB_FUNC("dbOlWdppb4o", LibGen5::Graphics5UnknownDb); + + LIB_FUNC("F0ZXt5q0ZTA", Gen5::GraphicsDriverGetDefaultOwner); + LIB_FUNC("F0Y42t-3e18", Gen5::GraphicsDriverInitResourceRegistration); + LIB_FUNC("AOLcoIkQDgM", Gen5::GraphicsDriverQueryResourceRegistrationUserMemoryRequirements); + LIB_FUNC("uJziRsODk1c", Gen5::GraphicsDriverGetResourceRegistrationMaxNameLength); + LIB_FUNC("X-Nm5KLREeg", Gen5::GraphicsDriverRegisterOwner); + LIB_FUNC("W5z4eZrjEas", Gen5::GraphicsDriverRegisterResource); + LIB_FUNC("pWLG7WOpVcw", Gen5::GraphicsDriverUnregisterResource); + LIB_FUNC("3AyTaWcF-H8", Gen5::GraphicsDriverRegisterWorkloadStream); + + LIB_FUNC("LtTouSCZjHM", Gen5::GraphicsCbNop); + LIB_FUNC("t7PlZ9nt5Lc", Gen5::GraphicsCbNopGetSize); + LIB_FUNC("k3GhuSNmBLU", Gen5::GraphicsCbDispatch); + LIB_FUNC("Abendgtz+3o", Gen5::GraphicsCbDispatchGetSize); + LIB_FUNC("w1KFAHVqpaU", Gen5::GraphicsCbBranch); + LIB_FUNC("n2fD4A+pb+g", Gen5::GraphicsCbSetShRegisterRangeDirect); + LIB_FUNC("bxGoVxpdSPQ", Gen5::GraphicsCbSetShRegisterRangeDirectGetSize); + LIB_FUNC("UZbQjYAwwXM", Gen5::GraphicsCbSetShRegistersDirect); + LIB_FUNC("wr23dPKyWc0", Gen5::GraphicsCbReleaseMem); + LIB_FUNC("hL7C0IRpWZI", Gen5::GraphicsCbQueueEndOfPipeActionGetSize); + LIB_FUNC("T6xuVw0KUJo", Gen5::GraphicsDebugRaiseException); + LIB_FUNC("JrtiDtKeS38", Gen5::GraphicsAcbResetQueue); + LIB_FUNC("cFazmnXpJOE", Gen5::GraphicsAcbEventWrite); + LIB_FUNC("KT-hTp-Ch14", Gen5::GraphicsAcbAcquireMem); + LIB_FUNC("ewobAQeMo5k", Gen5::GraphicsAcbAcquireMemGetSize); + LIB_FUNC("qyM2bxYFPAk", Gen5::GraphicsAcbCondExec); + LIB_FUNC("ozKzBP4aki4", Gen5::GraphicsAcbCondExecGetSize); + LIB_FUNC("htn36gPnBk4", Gen5::GraphicsAcbWaitRegMem); + LIB_FUNC("-RnpfpxIhec", Gen5::GraphicsAcbDmaData); + LIB_FUNC("qzMN2XKGA4k", Gen5::GraphicsAcbCopyData); + LIB_FUNC("j3EtxFkSIhQ", Gen5::GraphicsAcbDispatchIndirect); + LIB_FUNC("eZ4+17OQz4Q", Gen5::GraphicsAcbWriteData); + LIB_FUNC("xAeBOa0A3kk", Gen5::GraphicsAcbSetMarker); + LIB_FUNC("cpCILPya5Zk", Gen5::GraphicsAcbPushMarker); + LIB_FUNC("6mFxkVqdmbQ", Gen5::GraphicsAcbPopMarker); + LIB_FUNC("TRO721eVt4g", Gen5::GraphicsDcbResetQueue); + LIB_FUNC("MWiElSNE8j8", Gen5::GraphicsDcbWaitUntilSafeForRendering); + LIB_FUNC("LFSPFmGc9Hg", Gen5::GraphicsDcbSetWorkloadsActive); + LIB_FUNC("hEK26Wdny6s", Gen5::GraphicsDcbSetWorkloadComplete); + LIB_FUNC("QhCbS4X9Rl8", Gen5::GraphicsDcbSetMarker); + LIB_FUNC("pFLArOT53+w", Gen5::GraphicsDcbSetShRegisterDirect); + LIB_FUNC("LHFXRrlTPD8", Gen5::GraphicsDcbSetCxRegisterDirect); + LIB_FUNC("1DeUNpRIDDA", Gen5::GraphicsDcbSetCxRegisterDirectGetSize); + LIB_FUNC("w4-d0n60hdo", Gen5::GraphicsDcbSetUcRegisterDirect); + LIB_FUNC("ZvwO9euwYzc", Gen5::GraphicsDcbSetCxRegistersIndirect); + LIB_FUNC("-HOOCn0JY48", Gen5::GraphicsDcbSetShRegistersIndirect); + LIB_FUNC("hvUfkUIQcOE", Gen5::GraphicsDcbSetUcRegistersIndirect); + LIB_FUNC("GIIW2J37e70", Gen5::GraphicsDcbSetIndexSize); + LIB_FUNC("l4fM9K-Lyks", Gen5::GraphicsDcbSetIndexBuffer); + LIB_FUNC("8N2tmT3jmC8", Gen5::GraphicsDcbSetIndexCount); + LIB_FUNC("tSBxhAPyytQ", Gen5::GraphicsDcbSetNumInstances); + LIB_FUNC("6DFuRKT4C9w", Gen5::GraphicsDcbSetNumInstancesGetSize); + LIB_FUNC("q88lQ+GP5Yk", Gen5::GraphicsDcbDrawIndex); + LIB_FUNC("6ee9Hd3EWXQ", Gen5::GraphicsDcbDrawIndexGetSize); + LIB_FUNC("Ikfdt-rIqCE", Gen5::GraphicsUnknownIkfdtRIqCE); + LIB_FUNC("Rlx+bykm0r0", Gen5::GraphicsDcbDrawIndexMultiInstanced); + LIB_FUNC("mR9j7+SfM34", Gen5::GraphicsDcbDrawIndexMultiInstancedGetSize); + LIB_FUNC("Yw0jKSqop+E", Gen5::GraphicsDcbDrawIndexAuto); + LIB_FUNC("WrdP9Zxx3lQ", Gen5::GraphicsDcbDrawIndexAutoGetSize); + LIB_FUNC("B+aG9DUnTKA", Gen5::GraphicsDcbDrawIndexOffset); + LIB_FUNC("qMlfB1ZhMDc", Gen5::GraphicsDcbDrawIndexOffsetGetSize); + LIB_FUNC("RmaJwLtc8rY", Gen5::GraphicsDcbSetBaseIndirectArgs); + LIB_FUNC("1q1titRBL6o", Gen5::GraphicsDcbDrawIndirect); + LIB_FUNC("cxPZ4Wgvdj8", Gen5::GraphicsDcbDrawIndirectGetSize); + LIB_FUNC("t1vNu082-jM", Gen5::GraphicsDcbDrawIndexIndirect); + LIB_FUNC("ypVBz4uPKcQ", Gen5::GraphicsDcbDrawIndexIndirectMulti); + LIB_FUNC("CtB+A9-VxO0", Gen5::GraphicsDcbDispatchIndirect); + LIB_FUNC("w8HVkEeXPv8", Gen5::GraphicsDcbDispatchIndirectGetSize); + LIB_FUNC("aJf+j5yntiU", Gen5::GraphicsDcbEventWrite); + LIB_FUNC("57labkp+rSQ", Gen5::GraphicsDcbAcquireMem); + LIB_FUNC("-vnlTPPXPrw", Gen5::GraphicsDcbAcquireMemGetSize); + LIB_FUNC("1rZSWUv1IRc", Gen5::GraphicsDcbCopyData); + LIB_FUNC("WmAc2MEj6Io", Gen5::GraphicsDcbDmaData); + LIB_FUNC("xSAR0LTcRKM", Gen5::GraphicsDcbJump); + LIB_FUNC("VEGu4dixjUg", Gen5::GraphicsDcbJumpGetSize); + LIB_FUNC("QIXCsbipds0", Gen5::GraphicsDcbRewindGetSize); + LIB_FUNC("zfcxg-ewMK8", Gen5::GraphicsDcbRewind); + LIB_FUNC("BIPexNBSGog", Gen5::GraphicsDcbCondExec); + LIB_FUNC("ou16V5hh5sg", Gen5::GraphicsDcbCondExecGetSize); + LIB_FUNC("bbFueFP+J4k", Gen5::GraphicsDcbSetPredication); + LIB_FUNC("-KRzWekV120", Gen5::GraphicsUnknownKRzWekV120); + LIB_FUNC("IxYiarKlXxM", Gen5::GraphicsDmaDataPatchSetDstAddressOrOffset); + LIB_FUNC("cdDRpqcFGbU", Gen5::GraphicsDmaDataPatchSetSrcAddressOrOffsetOrImmediate); + LIB_FUNC("Lkf86B98qPc", Gen5::GraphicsGetPacketSize); + LIB_FUNC("w6Dj1VJt5qY", Gen5::GraphicsSetPacketPredication); + LIB_FUNC("n8vgpaQg6dA", Gen5::GraphicsSetRangePredication); + LIB_FUNC("i1jyy49AjXU", Gen5::GraphicsDcbWriteData); + LIB_FUNC("p9tI+yTvx68", Gen5::GraphicsDcbWriteDataGetSize); + LIB_FUNC("vuSXe69VILM", Gen5::GraphicsDcbGetLodStats); + LIB_FUNC("VmW0Tdpy420", Gen5::GraphicsDcbWaitRegMem); + LIB_FUNC("43WJ08sSugE", Gen5::GraphicsDcbWaitOnAddressGetSize); + LIB_FUNC("+kSrjIVxKFE", Gen5::GraphicsDcbPushMarker); + LIB_FUNC("H7uZqCoNuWk", Gen5::GraphicsDcbPopMarker); + LIB_FUNC("u2T2DiA5hRI", Gen5::GraphicsDcbStallCommandBufferParser); + LIB_FUNC("3KDcnM3lrcU", Gen5::GraphicsWaitRegMemPatchAddress); + LIB_FUNC("7nOoijNPvEU", Gen5::GraphicsWaitRegMemPatchReference); + LIB_FUNC("0fWWK5uG9rQ", Gen5::GraphicsQueueEndOfPipeActionPatchAddress); + LIB_FUNC("MlEw1feXcjg", Gen5::GraphicsQueueEndOfPipeActionPatchData); + LIB_FUNC("ORWsxIbk4TE", Gen5::GraphicsCondExecPatchSetEnd); + LIB_FUNC("YWTKOju587o", Gen5::GraphicsCondExecPatchSetCommandAddress); + LIB_FUNC("k-JpyR2dYAM", Gen5::GraphicsCondExecPatchSetEnd); + LIB_FUNC("3ZWa3AoyWZQ", Gen5::GraphicsCondExecPatchSetCommandAddress); + LIB_FUNC("YUeqkyT7mEQ", Gen5::GraphicsDcbSetFlip); +} + +} // namespace LibGen5 + +namespace LibGen5Driver { + +LIB_VERSION("Graphics5Driver", 1, "Graphics5Driver", 1, 1); + +namespace Gen5Driver = Graphics::Gen5Driver; + +LIB_DEFINE(InitGraphicsDriver_1) { + PRINT_NAME_ENABLE(true); + + LIB_FUNC("UglJIZjGssM", Gen5Driver::GraphicsDriverSubmitDcb); + LIB_FUNC("AhGvpITrf4M", Gen5Driver::GraphicsDriverSubmitDcb); + LIB_FUNC("6UzEidRZwkg", Gen5Driver::GraphicsDriverSubmitMultiDcbs); + LIB_FUNC("+T8Xo6LtFJI", Gen5Driver::GraphicsDriverSubmitMultiDcbs); + LIB_FUNC("b4fpgH5ZXxQ", Gen5Driver::GraphicsDriverSubmitCommandBuffer); + LIB_FUNC("Fj7r9EHzF38", Gen5Driver::GraphicsDriverSubmitMultiCommandBuffers); + LIB_FUNC("gSRnr79F8tQ", Gen5Driver::GraphicsDriverSubmitAcb); + LIB_FUNC("HF3YllT3mXU", Gen5Driver::GraphicsDriverSubmitMultiAcbs); + LIB_FUNC("w2rJhmD+dsE", Gen5Driver::GraphicsDriverAddEqEvent); + LIB_FUNC("DL2RXaXOy88", Gen5Driver::GraphicsDriverDeleteEqEvent); + LIB_FUNC("5CdQTZIQPxM", Gen5Driver::GraphicsDriverGetEqEventType); + LIB_FUNC("Zw7uUVPulbw", Gen5Driver::GraphicsDriverGetEqContextId); + LIB_FUNC("XlNp7jzGiPo", Gen5Driver::GraphicsDriverSetTFRing); + LIB_FUNC("MM4IZSEYytQ", Gen5Driver::GraphicsDriverSetHsOffchipParam); + LIB_FUNC("Ddwk4gLT5j0", Gen5Driver::GraphicsDriverIsCaptureInProgress); + LIB_FUNC("U9ueyEhSkF4", Gen5Driver::GraphicsDriverUnknownU9ueyEhSkF4); +} + +} // namespace LibGen5Driver + +LIB_DEFINE(InitGraphicsDriver_1) { + LibGen5::InitGraphicsDriver_1(s); + LibGen5Driver::InitGraphicsDriver_1(s); +} + +} // namespace Libs diff --git a/src/libs/libJson2.cpp b/src/libs/libJson2.cpp new file mode 100644 index 000000000..7fd27d69c --- /dev/null +++ b/src/libs/libJson2.cpp @@ -0,0 +1,992 @@ +#include "common/abi.h" +#include "common/common.h" +#include "common/logging/log.h" +#include "libs/libs.h" +#include "loader/symbolDatabase.h" + +#include +#include +#include +#include +#include + +namespace Libs { + +namespace LibJson2 { + +LIB_VERSION("Json2", 1, "Json2", 1, 1); + +enum JsonValueType : uint32_t { + JsonValueTypeNull = 0, + JsonValueTypeBoolean, + JsonValueTypeInteger, + JsonValueTypeUInteger, + JsonValueTypeReal, + JsonValueTypeString, + JsonValueTypeArray, + JsonValueTypeObject, +}; + +constexpr int32_t JSON_ERROR_PARSE_INVALID_CHAR = -2138799871; // 0x80848101 +constexpr int32_t JSON_ERROR_INVALID_ARGUMENT = -2138799840; // 0x80848120 + +struct JsonValue; + +struct JsonString { + std::string* impl; +}; + +struct JsonArray { + std::vector* impl; +}; + +struct JsonObject { + std::map* impl; +}; + +struct JsonValue { + JsonValue* parent; + void* rootparam; + union { + bool boolean; + int64_t integer; + uint64_t uinteger; + double real; + JsonString* string; + JsonArray* array; + JsonObject* object; + }; + char padding[4]; + uint32_t type; +}; + +struct JsonInitParameter2 { + void* allocator; + void* user_data; + size_t file_buffer_size; + uint32_t special_float_format_type; + uint32_t reserved[3]; +}; + +static std::string* JsonStringImpl(JsonString* self) { + if (self == nullptr) { + return nullptr; + } + if (self->impl == nullptr) { + self->impl = new std::string; + } + return self->impl; +} + +static const std::string* JsonStringImpl(const JsonString* self) { + static const std::string empty; + return (self != nullptr && self->impl != nullptr ? self->impl : &empty); +} + +static std::vector* JsonArrayImpl(JsonArray* self) { + if (self == nullptr) { + return nullptr; + } + if (self->impl == nullptr) { + self->impl = new std::vector; + } + return self->impl; +} + +static const std::vector* JsonArrayImpl(const JsonArray* self) { + static const std::vector empty; + return (self != nullptr && self->impl != nullptr ? self->impl : &empty); +} + +static std::map* JsonObjectImpl(JsonObject* self) { + if (self == nullptr) { + return nullptr; + } + if (self->impl == nullptr) { + self->impl = new std::map; + } + return self->impl; +} + +static const std::map* JsonObjectImpl(const JsonObject* self) { + static const std::map empty; + return (self != nullptr && self->impl != nullptr ? self->impl : &empty); +} + +static JsonValue* JsonStaticNullValue() { + static JsonValue value {}; + value.type = JsonValueTypeNull; + return &value; +} + +static JsonString* JsonStaticString() { + static JsonString str {new std::string}; + return &str; +} + +static JsonArray* JsonStaticArray() { + static JsonArray array {new std::vector}; + return &array; +} + +static JsonObject* JsonStaticObject() { + static JsonObject object {new std::map}; + return &object; +} + +static void JsonValueInit(JsonValue* self) { + if (self != nullptr) { + std::memset(self, 0, sizeof(JsonValue)); + self->type = JsonValueTypeNull; + } +} + +static void JsonValueClear(JsonValue* self); + +static JsonValue* JsonValueNew() { + auto* value = new JsonValue; + JsonValueInit(value); + return value; +} + +static JsonString* JsonStringNew(const std::string& value) { + auto* str = new JsonString {}; + str->impl = new std::string(value); + return str; +} + +static JsonArray* JsonArrayNew() { + auto* array = new JsonArray {}; + array->impl = new std::vector; + return array; +} + +static JsonObject* JsonObjectNew() { + auto* object = new JsonObject {}; + object->impl = new std::map; + return object; +} + +static void JsonValueDelete(JsonValue* self) { + if (self != nullptr) { + JsonValueClear(self); + delete self; + } +} + +static void JsonStringDelete(JsonString* self) { + if (self != nullptr) { + delete self->impl; + delete self; + } +} + +static void JsonArrayDelete(JsonArray* self) { + if (self != nullptr) { + if (self->impl != nullptr) { + for (auto* value: *self->impl) { + JsonValueDelete(value); + } + } + delete self->impl; + delete self; + } +} + +static void JsonObjectDelete(JsonObject* self) { + if (self != nullptr) { + if (self->impl != nullptr) { + for (auto& item: *self->impl) { + JsonValueDelete(item.second); + } + } + delete self->impl; + delete self; + } +} + +static void JsonValueClear(JsonValue* self) { + if (self == nullptr) { + return; + } + + switch (self->type) { + case JsonValueTypeString: JsonStringDelete(self->string); break; + case JsonValueTypeArray: JsonArrayDelete(self->array); break; + case JsonValueTypeObject: JsonObjectDelete(self->object); break; + default: break; + } + + self->parent = nullptr; + self->rootparam = nullptr; + self->uinteger = 0; + self->type = JsonValueTypeNull; +} + +static void JsonValueCopy(JsonValue* dst, const JsonValue* src) { + if (dst == nullptr || src == nullptr || dst == src) { + return; + } + + JsonValueClear(dst); + dst->type = src->type; + switch (src->type) { + case JsonValueTypeBoolean: dst->boolean = src->boolean; break; + case JsonValueTypeInteger: dst->integer = src->integer; break; + case JsonValueTypeUInteger: dst->uinteger = src->uinteger; break; + case JsonValueTypeReal: dst->real = src->real; break; + case JsonValueTypeString: dst->string = JsonStringNew(*JsonStringImpl(src->string)); break; + case JsonValueTypeArray: { + dst->array = JsonArrayNew(); + for (auto* value: *JsonArrayImpl(src->array)) { + auto* copy = JsonValueNew(); + JsonValueCopy(copy, value); + copy->parent = dst; + dst->array->impl->push_back(copy); + } + break; + } + case JsonValueTypeObject: { + dst->object = JsonObjectNew(); + for (const auto& item: *JsonObjectImpl(src->object)) { + auto* copy = JsonValueNew(); + JsonValueCopy(copy, item.second); + copy->parent = dst; + (*dst->object->impl)[item.first] = copy; + } + break; + } + default: dst->uinteger = 0; break; + } +} + +static JsonValue* JsonObjectLookup(JsonObject* object, const std::string& key, bool create) { + auto* impl = JsonObjectImpl(object); + if (impl == nullptr) { + return JsonStaticNullValue(); + } + auto it = impl->find(key); + if (it != impl->end()) { + return it->second; + } + if (!create) { + return JsonStaticNullValue(); + } + auto* value = JsonValueNew(); + (*impl)[key] = value; + return value; +} + +static bool JsonValueFromNlohmann(JsonValue* out, const nlohmann::json& value) { + if (out == nullptr) { + return false; + } + + JsonValueClear(out); + if (value.is_null()) { + return true; + } + if (value.is_boolean()) { + out->type = JsonValueTypeBoolean; + out->boolean = value.get(); + return true; + } + if (value.is_number_integer() && !value.is_number_unsigned()) { + out->type = JsonValueTypeInteger; + out->integer = value.get(); + return true; + } + if (value.is_number_unsigned()) { + out->type = JsonValueTypeUInteger; + out->uinteger = value.get(); + return true; + } + if (value.is_number_float()) { + out->type = JsonValueTypeReal; + out->real = value.get(); + return true; + } + if (value.is_string()) { + out->type = JsonValueTypeString; + out->string = JsonStringNew(value.get_ref()); + return true; + } + if (value.is_array()) { + out->type = JsonValueTypeArray; + out->array = JsonArrayNew(); + for (const auto& item: value) { + auto* child = JsonValueNew(); + if (!JsonValueFromNlohmann(child, item)) { + JsonValueDelete(child); + return false; + } + child->parent = out; + out->array->impl->push_back(child); + } + return true; + } + if (value.is_object()) { + out->type = JsonValueTypeObject; + out->object = JsonObjectNew(); + for (auto it = value.begin(); it != value.end(); ++it) { + auto* child = JsonValueNew(); + if (!JsonValueFromNlohmann(child, *it)) { + JsonValueDelete(child); + return false; + } + child->parent = out; + (*out->object->impl)[it.key()] = child; + } + return true; + } + return false; +} + +static void JsonSerializeValue(const JsonValue* value, std::string* out) { + if (value == nullptr) { + out->append("null"); + return; + } + switch (value->type) { + case JsonValueTypeBoolean: out->append(value->boolean ? "true" : "false"); break; + case JsonValueTypeInteger: out->append(std::to_string(value->integer)); break; + case JsonValueTypeUInteger: out->append(std::to_string(value->uinteger)); break; + case JsonValueTypeReal: out->append(std::to_string(value->real)); break; + case JsonValueTypeString: + out->push_back('"'); + out->append(*JsonStringImpl(value->string)); + out->push_back('"'); + break; + case JsonValueTypeArray: { + out->push_back('['); + bool first = true; + for (auto* item: *JsonArrayImpl(value->array)) { + if (!first) { + out->push_back(','); + } + first = false; + JsonSerializeValue(item, out); + } + out->push_back(']'); + break; + } + case JsonValueTypeObject: { + out->push_back('{'); + bool first = true; + for (const auto& item: *JsonObjectImpl(value->object)) { + if (!first) { + out->push_back(','); + } + first = false; + out->push_back('"'); + out->append(item.first); + out->append("\":"); + JsonSerializeValue(item.second, out); + } + out->push_back('}'); + break; + } + default: out->append("null"); break; + } +} + +static void* KYTY_SYSV_ABI JsonMemAllocatorCtor(void* self) { + PRINT_NAME(); + + return self; +} + +static JsonInitParameter2* KYTY_SYSV_ABI JsonInitParameter2Ctor(JsonInitParameter2* self) { + PRINT_NAME(); + + if (self != nullptr) { + self->allocator = nullptr; + self->user_data = nullptr; + self->file_buffer_size = 0; + self->special_float_format_type = 0; + self->reserved[0] = 0; + self->reserved[1] = 0; + self->reserved[2] = 0; + } + + return self; +} + +static void KYTY_SYSV_ABI JsonInitParameter2SetAllocator(JsonInitParameter2* self, void* allocator, + void* user_data) { + PRINT_NAME(); + + if (self != nullptr) { + self->allocator = allocator; + self->user_data = user_data; + } +} + +static void KYTY_SYSV_ABI JsonInitParameter2SetFileBufferSize(JsonInitParameter2* self, + size_t size) { + PRINT_NAME(); + + if (self != nullptr) { + self->file_buffer_size = size; + } +} + +static void KYTY_SYSV_ABI JsonInitParameter2SetSpecialFloatFormatType(JsonInitParameter2* self, + uint32_t type) { + PRINT_NAME(); + + if (self != nullptr) { + self->special_float_format_type = type; + } +} + +static void* KYTY_SYSV_ABI JsonInitializerCtor(void* self) { + PRINT_NAME(); + + return self; +} + +static int KYTY_SYSV_ABI JsonInitializerInitialize(void* self, + const JsonInitParameter2* init_param) { + PRINT_NAME(); + + LOGF("\t self = 0x%016" PRIx64 "\n" + "\t init_param = 0x%016" PRIx64 "\n", + reinterpret_cast(self), reinterpret_cast(init_param)); + + return 0; +} + +static int KYTY_SYSV_ABI JsonInitializerInitializeV1(void* self, const void* init_param) { + PRINT_NAME(); + + LOGF("\t self = 0x%016" PRIx64 "\n" + "\t init_param = 0x%016" PRIx64 "\n", + reinterpret_cast(self), reinterpret_cast(init_param)); + + return 0; +} + +static void KYTY_SYSV_ABI JsonInitializerTerminate(void* self) { + PRINT_NAME(); + + LOGF("\t self = 0x%016" PRIx64 "\n", reinterpret_cast(self)); +} + +static void KYTY_SYSV_ABI JsonInitializerDtor(void* self) { + PRINT_NAME(); + + LOGF("\t self = 0x%016" PRIx64 "\n", reinterpret_cast(self)); +} + +static void KYTY_SYSV_ABI JsonMemAllocatorDtor(void* self) { + PRINT_NAME(); + + LOGF("\t self = 0x%016" PRIx64 "\n", reinterpret_cast(self)); +} + +static void* KYTY_SYSV_ABI JsonValueCtor(void* self) { + PRINT_NAME(); + + LOGF("\t self = 0x%016" PRIx64 "\n", reinterpret_cast(self)); + + if (self != nullptr) { + JsonValueInit(reinterpret_cast(self)); + } + + return self; +} + +static JsonValue* KYTY_SYSV_ABI JsonValueBoolCtor(JsonValue* self, bool value) { + PRINT_NAME(); + + JsonValueInit(self); + if (self != nullptr) { + self->type = JsonValueTypeBoolean; + self->boolean = value; + } + return self; +} + +static JsonValue* KYTY_SYSV_ABI JsonValueIntCtor(JsonValue* self, int64_t value) { + PRINT_NAME(); + + JsonValueInit(self); + if (self != nullptr) { + self->type = JsonValueTypeInteger; + self->integer = value; + } + return self; +} + +static JsonValue* KYTY_SYSV_ABI JsonValueDoubleCtor(JsonValue* self, double value) { + PRINT_NAME(); + + JsonValueInit(self); + if (self != nullptr) { + self->type = JsonValueTypeReal; + self->real = value; + } + return self; +} + +static JsonValue* KYTY_SYSV_ABI JsonValueCStringCtor(JsonValue* self, const char* value) { + PRINT_NAME(); + + JsonValueInit(self); + if (self != nullptr) { + self->type = JsonValueTypeString; + self->string = JsonStringNew(value != nullptr ? value : ""); + } + return self; +} + +static JsonValue* KYTY_SYSV_ABI JsonValueStringCtor(JsonValue* self, const JsonString* value) { + PRINT_NAME(); + + JsonValueInit(self); + if (self != nullptr) { + self->type = JsonValueTypeString; + self->string = JsonStringNew(*JsonStringImpl(value)); + } + return self; +} + +static JsonValue* KYTY_SYSV_ABI JsonValueObjectCtor(JsonValue* self, const JsonObject* value) { + PRINT_NAME(); + + JsonValueInit(self); + if (self != nullptr) { + self->type = JsonValueTypeObject; + self->object = JsonObjectNew(); + for (const auto& item: *JsonObjectImpl(value)) { + auto* copy = JsonValueNew(); + JsonValueCopy(copy, item.second); + (*self->object->impl)[item.first] = copy; + } + } + return self; +} + +static void KYTY_SYSV_ABI JsonValueDtor(void* self) { + PRINT_NAME(); + + LOGF("\t self = 0x%016" PRIx64 "\n", reinterpret_cast(self)); + + JsonValueClear(reinterpret_cast(self)); +} + +static JsonValue* KYTY_SYSV_ABI JsonValueAssign(JsonValue* self, const JsonValue* src) { + PRINT_NAME(); + + JsonValueCopy(self, src); + return self; +} + +static void KYTY_SYSV_ABI JsonValueSetBool(JsonValue* self, bool value) { + PRINT_NAME(); + + JsonValueClear(self); + if (self != nullptr) { + self->type = JsonValueTypeBoolean; + self->boolean = value; + } +} + +static void KYTY_SYSV_ABI JsonValueSetInt(JsonValue* self, int64_t value) { + PRINT_NAME(); + + JsonValueClear(self); + if (self != nullptr) { + self->type = JsonValueTypeInteger; + self->integer = value; + } +} + +static void KYTY_SYSV_ABI JsonValueSetUInt(JsonValue* self, uint64_t value) { + PRINT_NAME(); + + JsonValueClear(self); + if (self != nullptr) { + self->type = JsonValueTypeUInteger; + self->uinteger = value; + } +} + +static void KYTY_SYSV_ABI JsonValueSetDouble(JsonValue* self, double value) { + PRINT_NAME(); + + JsonValueClear(self); + if (self != nullptr) { + self->type = JsonValueTypeReal; + self->real = value; + } +} + +static void KYTY_SYSV_ABI JsonValueSetString(JsonValue* self, const JsonString* value) { + PRINT_NAME(); + + JsonValueClear(self); + if (self != nullptr) { + self->type = JsonValueTypeString; + self->string = JsonStringNew(*JsonStringImpl(value)); + } +} + +static void KYTY_SYSV_ABI JsonValueSetType(JsonValue* self, uint32_t type) { + PRINT_NAME(); + + JsonValueClear(self); + if (self == nullptr) { + return; + } + self->type = type; + switch (type) { + case JsonValueTypeString: self->string = JsonStringNew(""); break; + case JsonValueTypeArray: self->array = JsonArrayNew(); break; + case JsonValueTypeObject: self->object = JsonObjectNew(); break; + default: self->uinteger = 0; break; + } +} + +static uint32_t KYTY_SYSV_ABI JsonValueGetType(const JsonValue* self) { + PRINT_NAME(); + + return (self != nullptr ? self->type : JsonValueTypeNull); +} + +static JsonArray* KYTY_SYSV_ABI JsonValueReferArray(JsonValue* self) { + PRINT_NAME(); + + return (self != nullptr && self->type == JsonValueTypeArray ? self->array : nullptr); +} + +static JsonObject* KYTY_SYSV_ABI JsonValueReferObject(JsonValue* self) { + PRINT_NAME(); + + return (self != nullptr && self->type == JsonValueTypeObject ? self->object : nullptr); +} + +static const JsonString* KYTY_SYSV_ABI JsonValueGetString(const JsonValue* self) { + PRINT_NAME(); + + return (self != nullptr && self->type == JsonValueTypeString ? self->string + : JsonStaticString()); +} + +static const JsonArray* KYTY_SYSV_ABI JsonValueGetArray(const JsonValue* self) { + PRINT_NAME(); + + return (self != nullptr && self->type == JsonValueTypeArray ? self->array : JsonStaticArray()); +} + +static const JsonObject* KYTY_SYSV_ABI JsonValueGetObject(const JsonValue* self) { + PRINT_NAME(); + + return (self != nullptr && self->type == JsonValueTypeObject ? self->object + : JsonStaticObject()); +} + +static const bool* KYTY_SYSV_ABI JsonValueGetBoolean(const JsonValue* self) { + PRINT_NAME(); + + static const bool false_value = false; + return (self != nullptr && self->type == JsonValueTypeBoolean ? &self->boolean : &false_value); +} + +static const int64_t* KYTY_SYSV_ABI JsonValueGetInteger(const JsonValue* self) { + PRINT_NAME(); + + static const int64_t zero = 0; + return (self != nullptr && self->type == JsonValueTypeInteger ? &self->integer : &zero); +} + +static const double* KYTY_SYSV_ABI JsonValueGetReal(const JsonValue* self) { + PRINT_NAME(); + + static const double zero = 0.0; + return (self != nullptr && self->type == JsonValueTypeReal ? &self->real : &zero); +} + +static const JsonValue* KYTY_SYSV_ABI JsonValueIndexString(const JsonValue* self, const char* key) { + PRINT_NAME(); + + if (self == nullptr || self->type != JsonValueTypeObject) { + return JsonStaticNullValue(); + } + return JsonObjectLookup(self->object, key != nullptr ? key : "", false); +} + +static const JsonValue* KYTY_SYSV_ABI JsonValueIndexUInt(const JsonValue* self, uint64_t index) { + PRINT_NAME(); + + if (self == nullptr || self->type != JsonValueTypeArray) { + return JsonStaticNullValue(); + } + const auto* impl = JsonArrayImpl(self->array); + return (index < impl->size() ? (*impl)[static_cast(index)] : JsonStaticNullValue()); +} + +static int32_t KYTY_SYSV_ABI JsonValueSerialize(JsonValue* self, JsonString* dst) { + PRINT_NAME(); + + auto* impl = JsonStringImpl(dst); + if (impl != nullptr) { + impl->clear(); + JsonSerializeValue(self, impl); + } + return 0; +} + +static JsonString* KYTY_SYSV_ABI JsonStringCtor(JsonString* self) { + PRINT_NAME(); + + if (self != nullptr) { + self->impl = new std::string; + } + return self; +} + +static JsonString* KYTY_SYSV_ABI JsonStringCStringCtor(JsonString* self, const char* str) { + PRINT_NAME(); + + if (self != nullptr) { + self->impl = new std::string(str != nullptr ? str : ""); + } + return self; +} + +static void KYTY_SYSV_ABI JsonStringDtor(JsonString* self) { + PRINT_NAME(); + + if (self != nullptr) { + delete self->impl; + self->impl = nullptr; + } +} + +static JsonString* KYTY_SYSV_ABI JsonStringAssign(JsonString* self, const JsonString* src) { + PRINT_NAME(); + + auto* impl = JsonStringImpl(self); + if (impl != nullptr) { + *impl = *JsonStringImpl(src); + } + return self; +} + +static const char* KYTY_SYSV_ABI JsonStringCStr(const JsonString* self) { + PRINT_NAME(); + + return JsonStringImpl(self)->c_str(); +} + +static size_t KYTY_SYSV_ABI JsonStringLength(const JsonString* self) { + PRINT_NAME(); + + return JsonStringImpl(self)->length(); +} + +static JsonArray* KYTY_SYSV_ABI JsonArrayPushBack(JsonArray* self, const JsonValue* value) { + PRINT_NAME(); + + auto* impl = JsonArrayImpl(self); + if (impl != nullptr) { + auto* copy = JsonValueNew(); + JsonValueCopy(copy, value); + impl->push_back(copy); + } + return self; +} + +static JsonValue* KYTY_SYSV_ABI JsonArrayBack(const JsonArray* self) { + PRINT_NAME(); + + const auto* impl = JsonArrayImpl(self); + return (!impl->empty() ? impl->back() : JsonStaticNullValue()); +} + +static size_t KYTY_SYSV_ABI JsonArraySize(const JsonArray* self) { + PRINT_NAME(); + + return JsonArrayImpl(self)->size(); +} + +static JsonObject* KYTY_SYSV_ABI JsonObjectCtor(JsonObject* self) { + PRINT_NAME(); + + if (self != nullptr) { + self->impl = new std::map; + } + return self; +} + +static JsonObject* KYTY_SYSV_ABI JsonObjectCopyCtor(JsonObject* self, const JsonObject* src) { + PRINT_NAME(); + + if (self != nullptr) { + self->impl = new std::map; + for (const auto& item: *JsonObjectImpl(src)) { + auto* copy = JsonValueNew(); + JsonValueCopy(copy, item.second); + (*self->impl)[item.first] = copy; + } + } + return self; +} + +static void KYTY_SYSV_ABI JsonObjectDtor(JsonObject* self) { + PRINT_NAME(); + + if (self != nullptr) { + if (self->impl != nullptr) { + for (auto& item: *self->impl) { + JsonValueDelete(item.second); + } + self->impl->clear(); + } + delete self->impl; + self->impl = nullptr; + } +} + +static JsonObject* KYTY_SYSV_ABI JsonObjectAssign(JsonObject* self, const JsonObject* src) { + PRINT_NAME(); + + JsonObjectDtor(self); + JsonObjectCopyCtor(self, src); + return self; +} + +static JsonValue* KYTY_SYSV_ABI JsonObjectIndex(JsonObject* self, const JsonString* key) { + PRINT_NAME(); + + return JsonObjectLookup(self, *JsonStringImpl(key), true); +} + +static void KYTY_SYSV_ABI JsonObjectClear(JsonObject* self) { + PRINT_NAME(); + + auto* impl = JsonObjectImpl(self); + if (impl != nullptr) { + for (auto& item: *impl) { + JsonValueDelete(item.second); + } + impl->clear(); + } +} + +static int32_t KYTY_SYSV_ABI JsonParserParse(JsonValue* dst, const char* src, size_t size) { + PRINT_NAME(); + + if (dst == nullptr || src == nullptr) { + return JSON_ERROR_INVALID_ARGUMENT; + } + + JsonValue parsed {}; + JsonValueInit(&parsed); + auto json = nlohmann::json::parse(src, src + size, nullptr, false); + if (json.is_discarded() || !JsonValueFromNlohmann(&parsed, json)) { + JsonValueClear(&parsed); + return JSON_ERROR_PARSE_INVALID_CHAR; + } + + JsonValueCopy(dst, &parsed); + JsonValueClear(&parsed); + return 0; +} + +static int32_t KYTY_SYSV_ABI JsonInitializerSetGlobalNullAccessCallback(void* self, void* callback, + void* context) { + PRINT_NAME(); + + LOGF("\t self = 0x%016" PRIx64 "\n" + "\t callback = 0x%016" PRIx64 "\n" + "\t context = 0x%016" PRIx64 "\n", + reinterpret_cast(self), reinterpret_cast(callback), + reinterpret_cast(context)); + + return 0; +} + +static int32_t KYTY_SYSV_ABI JsonValueCount(const JsonValue* self) { + PRINT_NAME(); + + if (self == nullptr) { + return 0; + } + + switch (self->type) { + case JsonValueTypeArray: return static_cast(JsonArrayImpl(self->array)->size()); + case JsonValueTypeObject: return static_cast(JsonObjectImpl(self->object)->size()); + default: return 0; + } +} + +LIB_DEFINE(InitNet_1_Json2) { + LIB_FUNC("-hJRce8wn1U", LibJson2::JsonMemAllocatorCtor); + LIB_FUNC("WSOuge5IsCg", LibJson2::JsonInitParameter2Ctor); + LIB_FUNC("GvGvswb0v34", LibJson2::JsonInitParameter2Ctor); + LIB_FUNC("I2QC8PYhJWY", LibJson2::JsonInitParameter2SetAllocator); + LIB_FUNC("W72B9ylU2JA", LibJson2::JsonInitParameter2SetAllocator); + LIB_FUNC("Eu95jmqn5Rw", LibJson2::JsonInitParameter2SetFileBufferSize); + LIB_FUNC("WVZBP4IyM+E", LibJson2::JsonInitParameter2SetSpecialFloatFormatType); + LIB_FUNC("cK6bYHf-Q5E", LibJson2::JsonInitializerCtor); + LIB_FUNC("IXW-z8pggfg", LibJson2::JsonInitializerInitialize); + LIB_FUNC("Cxwy7wHq4J0", LibJson2::JsonInitializerInitializeV1); + LIB_FUNC("PR5k1penBLM", LibJson2::JsonInitializerTerminate); + LIB_FUNC("RujUxbr3haM", LibJson2::JsonInitializerDtor); + LIB_FUNC("OcAgPxcq5Vk", LibJson2::JsonMemAllocatorDtor); + LIB_FUNC("qBMjqyBn3OM", LibJson2::JsonValueCtor); + LIB_FUNC("-wa17B7TGnw", LibJson2::JsonValueCtor); + LIB_FUNC("WTtYf+cNnXI", LibJson2::JsonValueDtor); + LIB_FUNC("0eUrW9JAxM0", LibJson2::JsonValueDtor); + LIB_FUNC("S5JxQnoGF3E", LibJson2::JsonParserParse); + LIB_FUNC("HwDt5lD9Bfo", LibJson2::JsonValueIndexString); + LIB_FUNC("epJ6x2LV0kU", LibJson2::JsonValueGetString); + LIB_FUNC("L1KAkYWml-M", LibJson2::JsonStringCStr); + LIB_FUNC("OJPTonqdg0I", LibJson2::JsonObjectCtor); + LIB_FUNC("sZIoMRGO+jk", LibJson2::JsonValueStringCtor); + LIB_FUNC("ERuf9y0DY84", LibJson2::JsonObjectIndex); + LIB_FUNC("4zrm6VrgIAw", LibJson2::JsonValueAssign); + LIB_FUNC("a+W7HHlwpBs", LibJson2::JsonObjectCopyCtor); + LIB_FUNC("5JmzZt8twAo", LibJson2::JsonObjectDtor); + LIB_FUNC("nM5XqdeXFPw", LibJson2::JsonValueReferArray); + LIB_FUNC("-NxEk7XLkDY", LibJson2::JsonValueReferObject); + LIB_FUNC("zQtLRTqceMY", LibJson2::JsonArrayPushBack); + LIB_FUNC("0lLK8+kDqmE", LibJson2::JsonValueIntCtor); + LIB_FUNC("urOpESTBZmo", LibJson2::JsonObjectAssign); + LIB_FUNC("zTwZdI8AZ5Y", LibJson2::JsonValueGetBoolean); + LIB_FUNC("R7FDWtcN6f8", LibJson2::JsonValueSerialize); + LIB_FUNC("oH8aBmLU+fc", LibJson2::JsonObjectClear); + LIB_FUNC("bAM9Qwofus0", LibJson2::JsonArrayBack); + LIB_FUNC("UeuWT+yNdCQ", LibJson2::JsonValueBoolCtor); + LIB_FUNC("3xUXnmUkXfo", LibJson2::JsonValueObjectCtor); + LIB_FUNC("cn9svYGWKDQ", LibJson2::JsonStringAssign); + LIB_FUNC("b9V6fmppLXY", LibJson2::JsonValueCStringCtor); + LIB_FUNC("EUH+EmT-v9E", LibJson2::JsonStringLength); + LIB_FUNC("XlWbvieLj2M", LibJson2::JsonValueIndexUInt); + LIB_FUNC("IlsmvBtMkak", LibJson2::JsonValueGetObject); + LIB_FUNC("SHtAad20YYM", LibJson2::JsonValueGetType); + LIB_FUNC("DIxvoy7Ngvk", LibJson2::JsonValueGetInteger); + LIB_FUNC("qSmqLXXCPas", LibJson2::JsonStringCtor); + LIB_FUNC("ONT8As5R1ug", LibJson2::JsonValueGetArray); + LIB_FUNC("3qrge7L-AU4", LibJson2::JsonValueGetReal); + LIB_FUNC("sOmU4vnx3s0", LibJson2::JsonValueDoubleCtor); + LIB_FUNC("rQGJeNjOuUk", LibJson2::JsonArraySize); + LIB_FUNC("5yHuiWXo2gg", LibJson2::JsonValueSetBool); + LIB_FUNC("QxVVYhP-mvg", LibJson2::JsonValueSetInt); + LIB_FUNC("SIe1ZmW7e7s", LibJson2::JsonValueSetUInt); + LIB_FUNC("BSmWDIkV4w4", LibJson2::JsonValueSetDouble); + LIB_FUNC("6l3Bv2gysNc", LibJson2::JsonValueSetString); + LIB_FUNC("9KUZFjI1IxA", LibJson2::JsonStringCStringCtor); + LIB_FUNC("cG1VE2HMl6c", LibJson2::JsonStringDtor); + LIB_FUNC("IKQimvG9Wqs", LibJson2::JsonValueSetType); + LIB_FUNC("RBw+4NukeGQ", LibJson2::JsonValueCount); + LIB_FUNC("+drDFyAS6u4", LibJson2::JsonInitializerSetGlobalNullAccessCallback); + LIB_FUNC("00oCq0RwSAY", LibJson2::JsonInitializerSetGlobalNullAccessCallback); +} + +} // namespace LibJson2 + +} // namespace Libs diff --git a/src/libs/libKernel.cpp b/src/libs/libKernel.cpp new file mode 100644 index 000000000..f08fa6a17 --- /dev/null +++ b/src/libs/libKernel.cpp @@ -0,0 +1,3092 @@ +#include "common/abi.h" +#include "common/assert.h" +#include "common/common.h" +#include "common/emulatorConfig.h" +#include "common/file.h" +#include "common/logging/log.h" +#include "common/singleton.h" +#include "common/stringUtils.h" +#include "common/threads.h" +#include "kernel/eventFlag.h" +#include "kernel/eventQueue.h" +#include "kernel/fileSystem.h" +#include "kernel/memory.h" +#include "kernel/pthread.h" +#include "kernel/semaphore.h" +#include "libs/errno.h" +#include "libs/libs.h" +#include "libs/network.h" +#include "loader/elf.h" +#include "loader/runtimeLinker.h" +#include "loader/symbolDatabase.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS +#ifndef NOMINMAX +#define NOMINMAX +#endif +#include +#endif + +namespace Libs { + +namespace LibC { +int GetArgc(); +const char** GetArgv(); +} // namespace LibC + +LIB_VERSION("libkernel", 1, "libkernel", 1, 1); + +namespace LibKernelApr { +LIB_DEFINE(InitLibKernel_1_Apr); +} // namespace LibKernelApr + +namespace LibKernel { + +using KernelModule = int32_t; +using get_thread_atexit_count_func_t = KYTY_SYSV_ABI int (*)(KernelModule); +using thread_atexit_report_func_t = KYTY_SYSV_ABI void (*)(KernelModule); + +static uint32_t sha1_rol(uint32_t value, uint32_t bits) { + return (value << bits) | (value >> (32u - bits)); +} + +static std::array sha1_digest(const uint8_t* data, size_t size) { + uint32_t h0 = 0x67452301u; + uint32_t h1 = 0xefcdab89u; + uint32_t h2 = 0x98badcfeu; + uint32_t h3 = 0x10325476u; + uint32_t h4 = 0xc3d2e1f0u; + + std::vector msg(data, data + size); + const uint64_t bit_len = static_cast(size) * 8u; + + msg.push_back(0x80u); + while ((msg.size() % 64u) != 56u) { + msg.push_back(0u); + } + for (int i = 7; i >= 0; i--) { + msg.push_back(static_cast((bit_len >> (i * 8)) & 0xffu)); + } + + for (size_t offset = 0; offset < msg.size(); offset += 64u) { + uint32_t w[80] = {}; + for (int i = 0; i < 16; i++) { + const auto j = offset + static_cast(i) * 4u; + w[i] = (static_cast(msg[j + 0]) << 24u) | + (static_cast(msg[j + 1]) << 16u) | + (static_cast(msg[j + 2]) << 8u) | static_cast(msg[j + 3]); + } + for (int i = 16; i < 80; i++) { + w[i] = sha1_rol(w[i - 3] ^ w[i - 8] ^ w[i - 14] ^ w[i - 16], 1); + } + + uint32_t a = h0; + uint32_t b = h1; + uint32_t c = h2; + uint32_t d = h3; + uint32_t e = h4; + + for (int i = 0; i < 80; i++) { + uint32_t f = 0; + uint32_t k = 0; + if (i < 20) { + f = (b & c) | ((~b) & d); + k = 0x5a827999u; + } else if (i < 40) { + f = b ^ c ^ d; + k = 0x6ed9eba1u; + } else if (i < 60) { + f = (b & c) | (b & d) | (c & d); + k = 0x8f1bbcdcu; + } else { + f = b ^ c ^ d; + k = 0xca62c1d6u; + } + + const auto temp = sha1_rol(a, 5) + f + e + k + w[i]; + e = d; + d = c; + c = sha1_rol(b, 30); + b = a; + a = temp; + } + + h0 += a; + h1 += b; + h2 += c; + h3 += d; + h4 += e; + } + + std::array digest = {}; + const uint32_t h[5] = {h0, h1, h2, h3, h4}; + for (int i = 0; i < 5; i++) { + digest[static_cast(i) * 4u + 0u] = static_cast((h[i] >> 24u) & 0xffu); + digest[static_cast(i) * 4u + 1u] = static_cast((h[i] >> 16u) & 0xffu); + digest[static_cast(i) * 4u + 2u] = static_cast((h[i] >> 8u) & 0xffu); + digest[static_cast(i) * 4u + 3u] = static_cast(h[i] & 0xffu); + } + return digest; +} + +static std::string kernel_symbol_to_nid(const char* symbol) { + static constexpr std::array salt = {0x51, 0x8d, 0x64, 0xa6, 0x35, 0xde, + 0xd8, 0xc1, 0xe6, 0xb0, 0x39, 0xb1, + 0xc3, 0xe5, 0x52, 0x30}; + static constexpr char codes[] = + "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+-"; + + const auto symbol_size = std::strlen(symbol); + std::vector input(symbol_size + salt.size()); + std::memcpy(input.data(), symbol, symbol_size); + std::memcpy(input.data() + symbol_size, salt.data(), salt.size()); + + const auto hash = sha1_digest(input.data(), input.size()); + + uint64_t digest = 0; + std::memcpy(&digest, hash.data(), sizeof(digest)); + + char nid[12] = {}; + for (int i = 0; i < 10; i++) { + nid[i] = codes[(digest >> (58 - i * 6)) & 0x3fu]; + } + nid[10] = codes[(digest & 0xfu) * 4u]; + return std::string(nid); +} + +static const Loader::SymbolRecord* kernel_find_export_symbol(const Loader::SymbolDatabase* symbols, + const char* symbol) { + EXIT_IF(symbols == nullptr); + EXIT_IF(symbol == nullptr); + + const auto nid = kernel_symbol_to_nid(symbol); + + if (const auto* record = symbols->FindByNid(nid, Loader::SymbolType::Func); record != nullptr) { + return record; + } + if (const auto* record = symbols->FindByNid(nid, Loader::SymbolType::Object); + record != nullptr) { + return record; + } + if (const auto* record = symbols->FindByNid(nid, Loader::SymbolType::NoType); + record != nullptr) { + return record; + } + + const auto symbol_name = std::string(symbol); + if (const auto* record = symbols->FindByName(symbol_name, Loader::SymbolType::Func); + record != nullptr) { + return record; + } + if (const auto* record = symbols->FindByName(symbol_name, Loader::SymbolType::Object); + record != nullptr) { + return record; + } + return symbols->FindByName(symbol_name, Loader::SymbolType::NoType); +} + +static void* KYTY_SYSV_ABI KernelApplicationHeapGetMem(uint64_t alignment, uint64_t size) { + static std::atomic_uint32_t log_count = 0; + + if (alignment < 0x10u) { + alignment = 0x10u; + } + if ((alignment & (alignment - 1u)) != 0u) { + return nullptr; + } + + auto* rt = Common::Singleton::Instance(); + void* ptr = rt->ApplicationHeapMemalign(alignment, size); + + const auto index = log_count.fetch_add(1, std::memory_order_relaxed); + if (index < 8 || (index % 600) == 0) { + LOGF("\t KernelApplicationHeapGetMem alignment = 0x%016" PRIx64 ", size = 0x%016" PRIx64 + ", ptr = 0x%016" PRIx64 "\n", + alignment, size, reinterpret_cast(ptr)); + } + + return ptr; +} + +int KYTY_SYSV_ABI KernelDlsym(KernelModule handle, const char* symbol, void** addr) { + static std::atomic_uint32_t log_count = 0; + const auto index = log_count.fetch_add(1, std::memory_order_relaxed); + if (index < 32 || (index % 600) == 0) { + PRINT_NAME(); + LOGF("\t call_count = %" PRIu32 ", handle = %" PRId32 ", symbol = %s\n", index + 1, handle, + symbol != nullptr ? symbol : "(null)"); + } + + if (addr == nullptr) { + return KERNEL_ERROR_EFAULT; + } + + if (symbol == nullptr) { + *addr = nullptr; + return KERNEL_ERROR_EFAULT; + } + + auto* rt = Common::Singleton::Instance(); + auto* program = rt->FindProgramById(handle); + if (program == nullptr || program->export_symbols == nullptr) { + *addr = nullptr; + LOGF("\t unresolved dlsym: handle = %" PRId32 ", symbol = %s, reason = module not found\n", + handle, symbol); + return KERNEL_ERROR_ESRCH; + } + + const auto* record = kernel_find_export_symbol(program->export_symbols.get(), symbol); + if (record != nullptr) { + *addr = reinterpret_cast(record->vaddr); + if (index < 32 || (index % 600) == 0) { + LOGF("\t resolved = 0x%016" PRIx64 " (%s)\n", record->vaddr, record->name.c_str()); + } + return OK; + } + + if (handle == 0 && std::strcmp(symbol, "scriptingGetMem") == 0) { + *addr = reinterpret_cast(&KernelApplicationHeapGetMem); + if (index < 32 || (index % 600) == 0) { + LOGF("\t resolved = 0x%016" PRIx64 " (application heap scriptingGetMem)\n", + reinterpret_cast(*addr)); + } + return OK; + } + + *addr = nullptr; + LOGF("\t unresolved dlsym: handle = %" PRId32 ", module = %s, symbol = %s\n", handle, + Common::PathToString(program->file_name).c_str(), symbol); + return KERNEL_ERROR_ESRCH; +} + +#pragma pack(1) + +struct KernelLoadModuleOpt { + size_t size; +}; + +struct KernelUnloadModuleOpt { + size_t size; +}; + +struct TlsInfo { + Loader::Program* program; + uint64_t offset; +}; + +struct MallocReplace { + uint64_t size = sizeof(MallocReplace); + void* malloc_initialize = nullptr; + void* malloc_finalize = nullptr; + void* malloc = nullptr; + void* free = nullptr; + void* calloc = nullptr; + void* realloc = nullptr; + void* memalign = nullptr; + void* reallocalign = nullptr; + void* posix_memalign = nullptr; + void* malloc_stats = nullptr; + void* malloc_stats_fast = nullptr; + void* malloc_usable_size = nullptr; + void* aligned_alloc = nullptr; +}; + +struct NewReplace { + uint64_t size = sizeof(NewReplace); + void* new_p = nullptr; + void* new_nothrow = nullptr; + void* new_array = nullptr; + void* new_array_nothrow = nullptr; + void* delete_p = nullptr; + void* delete_nothrow = nullptr; + void* delete_array = nullptr; + void* delete_array_nothrow = nullptr; + void* delete_with_size = nullptr; + void* delete_with_size_nothrow = nullptr; + void* delete_array_with_size = nullptr; + void* delete_array_with_size_nothrow = nullptr; +}; + +struct ModuleInfo { + uint64_t size; + uint64_t info[32]; + KernelModule handle; + uint8_t pad[156]; +}; + +struct ModuleInfoForUnwind { + uint64_t st_size; + char name[256]; + uint64_t eh_frame_hdr_addr; + uint64_t eh_frame_addr; + uint64_t eh_frame_size; + uint64_t seg0_addr; + uint64_t seg0_size; +}; + +static_assert(sizeof(ModuleInfoForUnwind) == 304); + +#pragma pack() + +constexpr size_t PROGNAME_MAX_SIZE = 511; + +static uint64_t g_stack_chk_guard = 0xDeadBeef00000007; +static char g_progname_buf[PROGNAME_MAX_SIZE + 1] = {0}; +static const char* g_progname = g_progname_buf; +static std::atomic_uint64_t g_gpo_state_bits {0}; + +static get_thread_atexit_count_func_t g_get_thread_atexit_count_func = nullptr; +static thread_atexit_report_func_t g_thread_atexit_report_func = nullptr; + +void SetProgName(const std::string& name) { + strncpy(g_progname_buf, name.c_str(), PROGNAME_MAX_SIZE); +} + +static KYTY_SYSV_ABI int* get_error_addr() { + return Posix::GetErrorAddr(); +} + +static KYTY_SYSV_ABI int getargc() { + PRINT_NAME(); + + return LibC::GetArgc(); +} + +static KYTY_SYSV_ABI const char** getargv() { + PRINT_NAME(); + + return LibC::GetArgv(); +} + +static void dump_stack_chk_fail_context(uint64_t caller, uint64_t rsp) { + LOGF("stack_chk_fail caller = 0x%016" PRIx64 "\n", caller); + LOGF("stack_chk_fail guard = 0x%016" PRIx64 " @ 0x%016" PRIx64 "\n", g_stack_chk_guard, + reinterpret_cast(&g_stack_chk_guard)); + + auto* linker = Common::Singleton::Instance(); + auto* program = linker->FindProgramByAddr(caller); + + LOGF("stack_chk_fail rsp = 0x%016" PRIx64 "\n" + "stack_chk_fail caller/saved-guard scan:\n", + rsp); + for (int i = -32; i < 512; i++) { + auto addr = rsp + static_cast(i) * 8; + auto value = *reinterpret_cast(addr); + + if (value == caller) { + auto saved_guard_addr = addr + 0x4a8; + auto saved_guard = *reinterpret_cast(saved_guard_addr); + LOGF(" return addr at 0x%016" PRIx64 + "; saved guard candidate [ret+0x4a8] = 0x%016" PRIx64 " -> 0x%016" PRIx64 "\n", + addr, saved_guard_addr, saved_guard); + } + + if (value == g_stack_chk_guard) { + LOGF(" guard value found at stack 0x%016" PRIx64 "\n", addr); + } + } + + LOGF("stack_chk_fail stack qwords:\n"); + for (int i = -4; i < 20; i++) { + auto addr = rsp + static_cast(i) * 8; + auto value = *reinterpret_cast(addr); + auto* p = linker->FindProgramByAddr(value); + auto module_name = + (p == nullptr ? std::string() : Common::PathToString(p->file_name.filename())); + LOGF(" [%+03d] 0x%016" PRIx64 " : 0x%016" PRIx64 " %s\n", i, addr, value, + module_name.c_str()); + } + + if (program == nullptr) { + LOGF("stack_chk_fail caller module = \n"); + return; + } + + auto caller_module_name = Common::PathToString(program->file_name.filename()); + LOGF("stack_chk_fail caller module = %s, base = 0x%016" PRIx64 ", offset = 0x%016" PRIx64 "\n", + caller_module_name.c_str(), program->base_vaddr, caller - program->base_vaddr); + + auto read_byte = [program](uint64_t vaddr) { + auto aligned = vaddr & ~0x07ull; + uint64_t value = Loader::RuntimeLinker::ReadFromElf(program, aligned); + return static_cast((value >> ((vaddr - aligned) * 8u)) & 0xffu); + }; + + if (caller >= 5 && read_byte(caller - 5) == 0xe8) { + uint32_t rel = 0; + for (uint32_t i = 0; i < 4; i++) { + rel |= static_cast(read_byte(caller - 4 + i)) << (i * 8u); + } + auto target = caller + static_cast(rel); + LOGF("stack_chk_fail preceding call = rel32 at 0x%016" PRIx64 ", target = 0x%016" PRIx64 + "\n", + caller - 5, target); + } else if (caller >= 6 && read_byte(caller - 6) == 0xff && + (read_byte(caller - 5) & 0x38u) == 0x10) { + LOGF("stack_chk_fail preceding call = indirect at 0x%016" PRIx64 "\n", caller - 6); + } + + auto dump_start = (caller >= 64 ? caller - 64 : 0); + auto dump_end = caller + 32; + + LOGF("stack_chk_fail caller bytes from ELF:\n"); + for (auto addr = dump_start; addr < dump_end; addr += 16) { + LOGF(" 0x%016" PRIx64 ":", addr); + for (uint32_t i = 0; i < 16; i++) { + LOGF("%c%02" PRIx8, (addr + i == caller ? '>' : ' '), read_byte(addr + i)); + } + LOGF("\n"); + } +} + +static KYTY_SYSV_ABI void stack_chk_fail() { + PRINT_NAME(); + + uint64_t rsp = 0; + asm volatile("movq %%rsp, %0" : "=r"(rsp)); + dump_stack_chk_fail_context(reinterpret_cast(__builtin_return_address(0)), rsp); + + EXIT("stack fail!!!"); +} + +static KYTY_SYSV_ABI int sigprocmask(int /*how*/, const void* /*set*/, void* /*oset*/) { + // PRINT_NAME(); + + // LOGF("\t how = %d\n", how); + // LOGF("\t set = %016" PRIx64 "\n", reinterpret_cast(set)); + // LOGF("\t oset = %016" PRIx64 "\n", reinterpret_cast(oset)); + + return 0; +} + +static bool IsAllowedExceptionSignal(int signum) { + constexpr int POSIX_SIGHUP = 1; + constexpr int POSIX_SIGILL = 4; + constexpr int POSIX_SIGFPE = 8; + constexpr int POSIX_SIGBUS = 10; + constexpr int POSIX_SIGSEGV = 11; + constexpr int POSIX_SIGUSR1 = 30; + + return signum == POSIX_SIGHUP || signum == POSIX_SIGILL || signum == POSIX_SIGFPE || + signum == POSIX_SIGBUS || signum == POSIX_SIGSEGV || signum == POSIX_SIGUSR1; +} + +static void* g_exception_handlers[130] = {}; + +using exception_handler_func_t = KYTY_SYSV_ABI void (*)(int, void*); + +static thread_local bool g_dispatching_signal_handler = false; + +struct SignalDispatchScope { + SignalDispatchScope(): previous(g_dispatching_signal_handler) { + g_dispatching_signal_handler = true; + } + ~SignalDispatchScope() { g_dispatching_signal_handler = previous; } + + bool previous; +}; + +bool KernelIsDispatchingSignalOnCurrentThread() { + return g_dispatching_signal_handler; +} + +static void QueuePendingSignal(Pthread thread, int signum) { + PthreadQueuePendingSignal(thread, signum); +} + +static void WaitForSignalDispatch(Pthread thread, int signum) { + constexpr auto DISPATCH_WAIT_STEP = std::chrono::microseconds(1000); + constexpr auto DISPATCH_WAIT_MAX = std::chrono::milliseconds(2); + + auto waited = std::chrono::microseconds(0); + while (PthreadHasPendingSignal(thread, signum) && waited < DISPATCH_WAIT_MAX) { + Common::Thread::SleepMicro(DISPATCH_WAIT_STEP.count()); + waited += DISPATCH_WAIT_STEP; + } +} + +static bool TakePendingSignal(Pthread thread, int signum) { + return PthreadTakePendingSignal(thread, signum); +} + +struct SignalMcontext { + uint64_t mc_onstack; + uint64_t mc_rdi; + uint64_t mc_rsi; + uint64_t mc_rdx; + uint64_t mc_rcx; + uint64_t mc_r8; + uint64_t mc_r9; + uint64_t mc_rax; + uint64_t mc_rbx; + uint64_t mc_rbp; + uint64_t mc_r10; + uint64_t mc_r11; + uint64_t mc_r12; + uint64_t mc_r13; + uint64_t mc_r14; + uint64_t mc_r15; + int mc_trapno; + uint16_t mc_fs; + uint16_t mc_gs; + uint64_t mc_addr; + int mc_flags; + uint16_t mc_es; + uint16_t mc_ds; + uint64_t mc_err; + uint64_t mc_rip; + uint64_t mc_cs; + uint64_t mc_rflags; + uint64_t mc_reserved_0xb8[8]; + uint64_t mc_rsp; + uint64_t mc_ss; + uint64_t mc_len; + uint64_t mc_fpformat; + uint64_t mc_ownedfp; + uint64_t mc_lbrfrom; + uint64_t mc_lbrto; + uint64_t mc_aux1; + uint64_t mc_aux2; + uint64_t mc_fpstate[104]; + uint64_t mc_fsbase; + uint64_t mc_gsbase; + uint64_t mc_spare[6]; +}; + +struct SignalStack { + void* ss_sp; + uint64_t ss_size; + int ss_flags; + int _align; +}; + +struct SignalUcontext { + SignalMcontext uc_mcontext; +}; + +static_assert(offsetof(SignalUcontext, uc_mcontext) == 0x00); +static_assert(offsetof(SignalMcontext, mc_rip) == 0xa0); +static_assert(offsetof(SignalUcontext, uc_mcontext) + offsetof(SignalMcontext, mc_rsp) == 0xf8); + +static SignalUcontext CreateSignalUcontext( +#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS + const CONTEXT* source_ctx = nullptr +#endif +) { + SignalUcontext ctx = {}; + +#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS + CONTEXT host_ctx = {}; + if (source_ctx != nullptr) { + host_ctx = *source_ctx; + } else { + host_ctx.ContextFlags = CONTEXT_CONTROL | CONTEXT_INTEGER | CONTEXT_SEGMENTS; + RtlCaptureContext(&host_ctx); + } + + ctx.uc_mcontext.mc_r8 = host_ctx.R8; + ctx.uc_mcontext.mc_r9 = host_ctx.R9; + ctx.uc_mcontext.mc_r10 = host_ctx.R10; + ctx.uc_mcontext.mc_r11 = host_ctx.R11; + ctx.uc_mcontext.mc_r12 = host_ctx.R12; + ctx.uc_mcontext.mc_r13 = host_ctx.R13; + ctx.uc_mcontext.mc_r14 = host_ctx.R14; + ctx.uc_mcontext.mc_r15 = host_ctx.R15; + ctx.uc_mcontext.mc_rdi = host_ctx.Rdi; + ctx.uc_mcontext.mc_rsi = host_ctx.Rsi; + ctx.uc_mcontext.mc_rbp = host_ctx.Rbp; + ctx.uc_mcontext.mc_rbx = host_ctx.Rbx; + ctx.uc_mcontext.mc_rdx = host_ctx.Rdx; + ctx.uc_mcontext.mc_rax = host_ctx.Rax; + ctx.uc_mcontext.mc_rcx = host_ctx.Rcx; + ctx.uc_mcontext.mc_rsp = host_ctx.Rsp; + ctx.uc_mcontext.mc_rip = host_ctx.Rip; + ctx.uc_mcontext.mc_cs = host_ctx.SegCs; + ctx.uc_mcontext.mc_fs = host_ctx.SegFs; + ctx.uc_mcontext.mc_gs = host_ctx.SegGs; + ctx.uc_mcontext.mc_ss = host_ctx.SegSs; + ctx.uc_mcontext.mc_ds = host_ctx.SegDs; + ctx.uc_mcontext.mc_es = host_ctx.SegEs; + ctx.uc_mcontext.mc_rflags = host_ctx.EFlags; + ctx.uc_mcontext.mc_len = sizeof(SignalMcontext); +#else + ctx.uc_mcontext.mc_len = sizeof(SignalMcontext); + uint64_t stack_marker = 0; + ctx.uc_mcontext.mc_rsp = reinterpret_cast(&stack_marker); + ctx.uc_mcontext.mc_rbp = reinterpret_cast(__builtin_frame_address(0)); + ctx.uc_mcontext.mc_rip = reinterpret_cast(__builtin_return_address(0)); +#endif + + return ctx; +} + +static SignalUcontext CreateCurrentGuestCallSignalUcontext(uint64_t rip) { + auto ctx = CreateSignalUcontext(); + + uint64_t rsp = 0; + uint64_t rbp = 0; + uint64_t rbx = 0; + uint64_t r12 = 0; + uint64_t r13 = 0; + uint64_t r14 = 0; + uint64_t r15 = 0; + asm volatile("movq %%rsp, %0\n\t" + "movq %%rbp, %1\n\t" + "movq %%rbx, %2\n\t" + "movq %%r12, %3\n\t" + "movq %%r13, %4\n\t" + "movq %%r14, %5\n\t" + "movq %%r15, %6\n\t" + : "=r"(rsp), "=r"(rbp), "=r"(rbx), "=r"(r12), "=r"(r13), "=r"(r14), "=r"(r15) + : + : "memory"); + + ctx.uc_mcontext.mc_rax = 0; + ctx.uc_mcontext.mc_rcx = 0; + ctx.uc_mcontext.mc_rdx = 0; + ctx.uc_mcontext.mc_rsi = 0; + ctx.uc_mcontext.mc_rdi = 0; + ctx.uc_mcontext.mc_r8 = 0; + ctx.uc_mcontext.mc_r9 = 0; + ctx.uc_mcontext.mc_r10 = 0; + ctx.uc_mcontext.mc_r11 = 0; + ctx.uc_mcontext.mc_rbx = rbx; + ctx.uc_mcontext.mc_rbp = rbp; + ctx.uc_mcontext.mc_rsp = rsp; + ctx.uc_mcontext.mc_r12 = r12; + ctx.uc_mcontext.mc_r13 = r13; + ctx.uc_mcontext.mc_r14 = r14; + ctx.uc_mcontext.mc_r15 = r15; + ctx.uc_mcontext.mc_rip = rip; + + return ctx; +} + +static bool IsGuestCodeAddress(uint64_t addr) { + return addr >= 0x0000000900000000ull && addr < 0x0000000980000000ull; +} + +static bool IsAddressInRange(uint64_t addr, uint64_t base, uint64_t size) { + return size != 0 && addr >= base && addr < base + size; +} + +static uint64_t GetFallbackStackPointer(Pthread thread) { + uint64_t stack_addr = 0; + uint64_t stack_size = 0; + if (!PthreadGetGuestStack(thread, &stack_addr, &stack_size)) { + return 0; + } + + const auto stack_top = (stack_addr + stack_size) & ~0xfull; + if (stack_top <= stack_addr + 0x100) { + return stack_top; + } + return stack_top - 0x100; +} + +static void SanitizeNonGuestSignalUcontext(SignalUcontext* ctx, Pthread thread) { + if (ctx == nullptr || IsGuestCodeAddress(ctx->uc_mcontext.mc_rip)) { + return; + } + + uint64_t stack_addr = 0; + uint64_t stack_size = 0; + const bool has_stack = PthreadGetGuestStack(thread, &stack_addr, &stack_size); + const bool rsp_valid = + has_stack && IsAddressInRange(ctx->uc_mcontext.mc_rsp, stack_addr, stack_size); + const bool rbp_valid = + has_stack && IsAddressInRange(ctx->uc_mcontext.mc_rbp, stack_addr, stack_size); + const auto fallback_sp = GetFallbackStackPointer(thread); + + ctx->uc_mcontext.mc_rax = 0; + ctx->uc_mcontext.mc_rbx = 0; + ctx->uc_mcontext.mc_rcx = 0; + ctx->uc_mcontext.mc_rdx = 0; + ctx->uc_mcontext.mc_rsi = 0; + ctx->uc_mcontext.mc_rdi = 0; + ctx->uc_mcontext.mc_rbp = (rbp_valid ? ctx->uc_mcontext.mc_rbp : fallback_sp); + ctx->uc_mcontext.mc_rsp = (rsp_valid ? ctx->uc_mcontext.mc_rsp : fallback_sp); + ctx->uc_mcontext.mc_r8 = 0; + ctx->uc_mcontext.mc_r9 = 0; + ctx->uc_mcontext.mc_r10 = 0; + ctx->uc_mcontext.mc_r11 = 0; + ctx->uc_mcontext.mc_r12 = 0; + ctx->uc_mcontext.mc_r13 = 0; + ctx->uc_mcontext.mc_r14 = 0; + ctx->uc_mcontext.mc_r15 = 0; + ctx->uc_mcontext.mc_rip = 0; +} + +#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS +static void ApplySignalUcontext(CONTEXT* dst_ctx, const SignalUcontext& src_ctx) { + if (dst_ctx == nullptr) { + return; + } + + dst_ctx->R8 = src_ctx.uc_mcontext.mc_r8; + dst_ctx->R9 = src_ctx.uc_mcontext.mc_r9; + dst_ctx->R10 = src_ctx.uc_mcontext.mc_r10; + dst_ctx->R11 = src_ctx.uc_mcontext.mc_r11; + dst_ctx->R12 = src_ctx.uc_mcontext.mc_r12; + dst_ctx->R13 = src_ctx.uc_mcontext.mc_r13; + dst_ctx->R14 = src_ctx.uc_mcontext.mc_r14; + dst_ctx->R15 = src_ctx.uc_mcontext.mc_r15; + dst_ctx->Rdi = src_ctx.uc_mcontext.mc_rdi; + dst_ctx->Rsi = src_ctx.uc_mcontext.mc_rsi; + dst_ctx->Rbp = src_ctx.uc_mcontext.mc_rbp; + dst_ctx->Rbx = src_ctx.uc_mcontext.mc_rbx; + dst_ctx->Rdx = src_ctx.uc_mcontext.mc_rdx; + dst_ctx->Rax = src_ctx.uc_mcontext.mc_rax; + dst_ctx->Rcx = src_ctx.uc_mcontext.mc_rcx; + dst_ctx->Rsp = src_ctx.uc_mcontext.mc_rsp; + dst_ctx->Rip = src_ctx.uc_mcontext.mc_rip; + dst_ctx->SegCs = static_cast(src_ctx.uc_mcontext.mc_cs); + dst_ctx->SegFs = src_ctx.uc_mcontext.mc_fs; + dst_ctx->SegGs = src_ctx.uc_mcontext.mc_gs; + dst_ctx->SegSs = static_cast(src_ctx.uc_mcontext.mc_ss); + dst_ctx->SegDs = src_ctx.uc_mcontext.mc_ds; + dst_ctx->SegEs = src_ctx.uc_mcontext.mc_es; + dst_ctx->EFlags = static_cast(src_ctx.uc_mcontext.mc_rflags); +} +#endif + +#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS +enum KytyQueueUserApcFlags : ULONG_PTR { + KytyQueueUserApcFlagsNone = 0, + KytyQueueUserApcFlagsSpecialUserApc = 1, + KytyQueueUserApcFlagsCallbackContext = 0x00010000, + KytyQueueUserApcFlagsMaxValue = 2, +}; + +union KytyUserApcOption { + ULONG_PTR UserApcFlags; + HANDLE MemoryReserveHandle; +}; + +using KytyPsApcRoutine = void (*)(void* apc_arg1, void* apc_arg2, void* apc_arg3, PCONTEXT context); +using NtQueueApcThreadExFunc = uint64_t(WINAPI*)(HANDLE thread, KytyUserApcOption option, + KytyPsApcRoutine routine, void* arg1, void* arg2, + void* arg3); + +static NtQueueApcThreadExFunc GetNtQueueApcThreadEx() { + static auto* nt_queue_apc_thread_ex = reinterpret_cast( + GetProcAddress(GetModuleHandleA("ntdll.dll"), "NtQueueApcThreadEx")); + return nt_queue_apc_thread_ex; +} + +static void SignalApcHandler(void* arg1, void* arg2, void* /*arg3*/, PCONTEXT context) { + auto* thread = static_cast(arg1); + const auto signum = static_cast(reinterpret_cast(arg2)); + if (!TakePendingSignal(thread, signum)) { + return; + } + + auto* handler = reinterpret_cast(g_exception_handlers[signum]); + if (handler != nullptr) { + SignalDispatchScope scope; + auto ctx = CreateSignalUcontext(context); + auto guest_context = IsGuestCodeAddress(ctx.uc_mcontext.mc_rip); + if (!guest_context) { + SanitizeNonGuestSignalUcontext(&ctx, thread); + std::thread([thread, signum, handler, ctx]() mutable { + SignalDispatchScope helper_scope; + auto* previous_self = PthreadSwapSelfForSignal(thread); + handler(signum, &ctx); + PthreadSwapSelfForSignal(previous_self); + }).detach(); + return; + } + handler(signum, &ctx); + if (guest_context) { + ApplySignalUcontext(context, ctx); + } + } +} + +static bool DispatchSignalWithSuspendedThreadContext(HANDLE target_thread, Pthread thread, + int signum, exception_handler_func_t handler) { + if (target_thread == nullptr || thread == nullptr || handler == nullptr) { + return false; + } + + if (!PthreadHasPendingSignal(thread, signum)) { + return true; + } + + if (SuspendThread(target_thread) == static_cast(-1)) { + return false; + } + + bool dispatched = false; + CONTEXT host_ctx {}; + host_ctx.ContextFlags = CONTEXT_FULL | CONTEXT_SEGMENTS; + if (GetThreadContext(target_thread, &host_ctx) != FALSE) { + if (TakePendingSignal(thread, signum)) { + auto ctx = CreateSignalUcontext(&host_ctx); + auto guest_context = IsGuestCodeAddress(ctx.uc_mcontext.mc_rip); + HANDLE helper_thread_handle = nullptr; + if (guest_context) { + if (DuplicateHandle(GetCurrentProcess(), target_thread, GetCurrentProcess(), + &helper_thread_handle, THREAD_SUSPEND_RESUME, FALSE, + 0) == FALSE) { + ResumeThread(target_thread); + return false; + } + } else { + SanitizeNonGuestSignalUcontext(&ctx, thread); + ResumeThread(target_thread); + } + + std::thread([helper_thread_handle, thread, signum, handler, ctx]() mutable { + SignalDispatchScope scope; + auto* previous_self = PthreadSwapSelfForSignal(thread); + handler(signum, &ctx); + PthreadSwapSelfForSignal(previous_self); + if (helper_thread_handle != nullptr) { + ResumeThread(helper_thread_handle); + CloseHandle(helper_thread_handle); + } + }).detach(); + dispatched = true; + return true; + } + dispatched = true; + } + + ResumeThread(target_thread); + + return dispatched; +} +#endif + +void KernelDispatchPendingSignalForCurrentThread() { + Pthread current = PthreadSelfOrNull(); + if (current == nullptr) { + return; + } + + for (int signum = 0; signum < static_cast(std::size(g_exception_handlers)); signum++) { + if (!TakePendingSignal(current, signum)) { + continue; + } + + auto* handler = reinterpret_cast(g_exception_handlers[signum]); + if (handler != nullptr) { + SignalDispatchScope scope; + auto ctx = CreateSignalUcontext(); + SanitizeNonGuestSignalUcontext(&ctx, current); + handler(signum, &ctx); + } + return; + } +} + +static int KYTY_SYSV_ABI KernelInstallExceptionHandler(int signum, void* handler) { + PRINT_NAME(); + + LOGF("\t signum = %d\n" + "\t handler = 0x%016" PRIx64 "\n", + signum, reinterpret_cast(handler)); + + if (!IsAllowedExceptionSignal(signum)) { + return KERNEL_ERROR_EINVAL; + } + if (g_exception_handlers[signum] != nullptr) { + return KERNEL_ERROR_EAGAIN; + } + + g_exception_handlers[signum] = handler; + return OK; +} + +static int KYTY_SYSV_ABI KernelRemoveExceptionHandler(int signum) { + PRINT_NAME(); + + LOGF("\t signum = %d\n", signum); + + if (!IsAllowedExceptionSignal(signum)) { + return KERNEL_ERROR_EINVAL; + } + + g_exception_handlers[signum] = nullptr; + return OK; +} + +static int KYTY_SYSV_ABI KernelRaiseException(Pthread thread, int signum) { + constexpr int POSIX_SIGUSR1 = 30; + + if (signum != POSIX_SIGUSR1 || thread == nullptr) { + return KERNEL_ERROR_EINVAL; + } + + auto* handler = reinterpret_cast(g_exception_handlers[signum]); + if (handler != nullptr) { +#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS + const auto target_thread_id = PthreadGetHostThreadId(thread); + auto* nt_queue_apc_thread_ex = GetNtQueueApcThreadEx(); + if (target_thread_id == 0) { + return KERNEL_ERROR_EINVAL; + } + if (target_thread_id == static_cast(GetCurrentThreadId())) { + SignalDispatchScope scope; + auto ctx = CreateCurrentGuestCallSignalUcontext( + reinterpret_cast(__builtin_return_address(0))); + handler(signum, &ctx); + return OK; + } + if (nt_queue_apc_thread_ex == nullptr) { + return KERNEL_ERROR_EINVAL; + } + + HANDLE target_thread = + OpenThread(THREAD_SET_CONTEXT | THREAD_SUSPEND_RESUME | THREAD_GET_CONTEXT, FALSE, + static_cast(target_thread_id)); + if (target_thread == nullptr) { + return KERNEL_ERROR_EINVAL; + } + + QueuePendingSignal(thread, signum); + KytyUserApcOption option {}; + option.UserApcFlags = KytyQueueUserApcFlagsSpecialUserApc; + + const auto status = + nt_queue_apc_thread_ex(target_thread, option, SignalApcHandler, thread, + reinterpret_cast(static_cast(signum)), nullptr); + + if (status != 0) { + TakePendingSignal(thread, signum); + CloseHandle(target_thread); + LOGF("\t NtQueueApcThreadEx failed: target_os_thread=%" PRIu64 ", status=0x%016" PRIx64 + "\n", + target_thread_id, status); + return KERNEL_ERROR_EINVAL; + } + + Common::CondVar::SignalThread(PthreadGetUniqueId(thread)); + PthreadWakeForSignal(thread); + WaitForSignalDispatch(thread, signum); + if (PthreadHasPendingSignal(thread, signum)) { + DispatchSignalWithSuspendedThreadContext(target_thread, thread, signum, handler); + } + CloseHandle(target_thread); + return OK; +#else + auto ctx = CreateSignalUcontext(); + handler(signum, &ctx); +#endif + } + + return OK; +} + +static KYTY_SYSV_ABI KernelModule KernelLoadStartModule(const char* module_file_name, size_t args, + const void* argp, uint32_t flags, + const KernelLoadModuleOpt* opt, int* res) { + PRINT_NAME(); + + // EXIT_NOT_IMPLEMENTED(!Common::Thread::IsMainThread()); + + LOGF("\tmodule_file_name = %s\n", module_file_name); + + EXIT_NOT_IMPLEMENTED(flags != 0); + EXIT_NOT_IMPLEMENTED(opt != nullptr); + + auto* rt = Common::Singleton::Instance(); + + auto module_path = FileSystem::GetRealFilename(std::string(module_file_name)); + if (!Common::File::IsFileExisting(module_path)) { + LOGF("\tmodule real path missing = %s\n", Common::PathToString(module_path).c_str()); + if (res != nullptr) { + *res = KERNEL_ERROR_ENOENT; + } + return KERNEL_ERROR_ENOENT; + } + + auto* program = rt->FindProgramByFileName(module_path); + if (program != nullptr) { + if (res != nullptr) { + *res = OK; + } + return static_cast(program->unique_id); + } + + program = rt->LoadProgram(module_path); + + auto handle = program->unique_id; + + program->dbg_print_reloc = true; + + rt->RelocateProgram(program); + + int result = rt->StartModule(program, args, argp, nullptr); + + LOGF("\tmodule_start() result = %d\n", result); + + EXIT_NOT_IMPLEMENTED(result < 0); + + if (res != nullptr) { + *res = result; + } + + return static_cast(handle); +} + +static int KYTY_SYSV_ABI KernelStopUnloadModule(KernelModule handle, size_t args, const void* argp, + uint32_t flags, const KernelUnloadModuleOpt* opt, + int* res) { + PRINT_NAME(); + + // EXIT_NOT_IMPLEMENTED(!Common::Thread::IsMainThread()); + + auto* rt = Common::Singleton::Instance(); + + EXIT_NOT_IMPLEMENTED(flags != 0); + EXIT_NOT_IMPLEMENTED(opt != nullptr); + + auto* program = rt->FindProgramById(handle); + + if (program == nullptr) { + LOGF("\tinvalid module handle = %" PRId32 "\n", handle); + return OK; + } + + if (g_get_thread_atexit_count_func != nullptr && + g_get_thread_atexit_count_func(program->unique_id) > 0) { + LOGF("KernelStopUnloadModule: cannot unload %s\n", + Common::PathToString(program->file_name).c_str()); + if (g_thread_atexit_report_func != nullptr) { + g_thread_atexit_report_func(program->unique_id); + } + return KERNEL_ERROR_EBUSY; + } + + int result = rt->StopModule(program, args, argp, nullptr); + + LOGF("\tmodule_stop() result = %d\n", result); + + EXIT_NOT_IMPLEMENTED(result < 0); + + if (res != nullptr) { + *res = result; + } + + rt->UnloadProgram(program); + + return OK; +} + +static void* KYTY_SYSV_ABI tls_get_addr(TlsInfo* info) { + PRINT_NAME(); + + // EXIT_NOT_IMPLEMENTED(!Common::Thread::IsMainThread()); + + return Loader::RuntimeLinker::TlsGetAddr(info->program) + info->offset; +} + +static void* KYTY_SYSV_ABI KernelGetProcParam() { + PRINT_NAME(); + + auto* rt = Common::Singleton::Instance(); // NOLINT + + return reinterpret_cast(rt->GetProcParam()); +} + +static int KYTY_SYSV_ABI KernelIsTrinityMode() { + static std::atomic_uint32_t call_count {0}; + + auto count = call_count.fetch_add(1, std::memory_order_relaxed); + if (count < 8 || (count % 1024) == 0) { + PRINT_NAME(); + LOGF("\tcall = %u, return = 0 (base PS5 mode)\n", count + 1); + } + + return 0; +} + +static int KYTY_SYSV_ABI KernelFsync(int fd) { + PRINT_NAME(); + + LOGF("\t fd = %d\n", fd); + + return OK; +} + +static void KYTY_SYSV_ABI KernelSync() { + PRINT_NAME(); +} + +static int KYTY_SYSV_ABI getpid() { + PRINT_NAME(); + + return 100; +} + +static void KYTY_SYSV_ABI KernelRtldSetApplicationHeapAPI(void* api[]) { + PRINT_NAME(); + + auto* rt = Common::Singleton::Instance(); + rt->SetApplicationHeapApi(api); + + if (api == nullptr) { + return; + } + + for (int i = 0; i < 10; i++) { + LOGF("\tapi[%d] = 0x%016" PRIx64 "\n", i, reinterpret_cast(api[i])); + } + + [[maybe_unused]] auto* heap_malloc = api[0]; + [[maybe_unused]] auto* heap_free = api[1]; + [[maybe_unused]] auto* heap_posix_memalign = api[6]; +} + +static int64_t KYTY_SYSV_ABI write(int d, const char* str, int64_t size) { + // PRINT_NAME(); + + EXIT_NOT_IMPLEMENTED(d < 0); + + if (Network::Net::IsSocket(d)) { + return Network::Net::Send(d, str, static_cast(size), 0); + } + + if (d > 2) { + return POSIX_N_CALL(FileSystem::KernelWrite(d, str, size)); + } + + int size_int = static_cast(size); + + return size_int; +} + +static int KYTY_SYSV_ABI open(const char* path, int flags, int mode) { + return POSIX_N_CALL(FileSystem::KernelOpen(path, flags, mode)); +} + +static int KYTY_SYSV_ABI close(int d) { + if (d < 0) { + *Posix::GetErrorAddr() = Posix::POSIX_EBADF; + return -1; + } + + if (d <= 2) { + return 0; + } + + if (Network::Net::IsSocket(d)) { + return POSIX_NET_CALL(Network::Net::SocketClose(d)); + } + + return POSIX_CALL(FileSystem::KernelClose(d)); +} + +static int64_t KYTY_SYSV_ABI read(int d, void* buf, uint64_t nbytes) { + // PRINT_NAME(); + + if (buf == nullptr) { + *Posix::GetErrorAddr() = Posix::POSIX_EFAULT; + return -1; + } + + if (d < 0 || d == 1 || d == 2) { + *Posix::GetErrorAddr() = Posix::POSIX_EBADF; + return -1; + } + + if (Network::Net::IsSocket(d)) { + return Network::Net::Recv(d, buf, nbytes, 0); + } + + if (d > 2) { + return POSIX_N_CALL(FileSystem::KernelRead(d, buf, nbytes)); + } + + if (nbytes == 0) { + return 0; + } + + auto* str = std::fgets(static_cast(buf), static_cast(nbytes), stdin); + + return (str != nullptr ? static_cast(strlen(str)) : 0); +} + +static bool DecodeEhFramePointer(const uint8_t* data, const uint8_t* end, uint8_t encoding, + uint64_t field_addr, uint64_t* value) { + EXIT_IF(value == nullptr); + + if (encoding == 0xff) { + return false; + } + + uint64_t out = 0; + auto format = encoding & 0x0f; + auto app = encoding & 0x70; + bool is_signed = false; + + switch (format) { + case 0x00: + if (data + sizeof(uint64_t) > end) { + return false; + } + std::memcpy(&out, data, sizeof(uint64_t)); + break; + case 0x03: + if (data + sizeof(uint32_t) > end) { + return false; + } + { + uint32_t tmp = 0; + std::memcpy(&tmp, data, sizeof(tmp)); + out = tmp; + } + break; + case 0x04: + if (data + sizeof(uint64_t) > end) { + return false; + } + std::memcpy(&out, data, sizeof(uint64_t)); + break; + case 0x0b: + if (data + sizeof(int32_t) > end) { + return false; + } + { + int32_t tmp = 0; + std::memcpy(&tmp, data, sizeof(tmp)); + out = static_cast(static_cast(tmp)); + is_signed = true; + } + break; + case 0x0c: + if (data + sizeof(int64_t) > end) { + return false; + } + std::memcpy(&out, data, sizeof(int64_t)); + is_signed = true; + break; + default: return false; + } + + if (app == 0x10) { + out = (is_signed ? static_cast(static_cast(field_addr) + + static_cast(out)) + : field_addr + out); + } else if (app != 0) { + return false; + } + + *value = out; + return true; +} + +int KYTY_SYSV_ABI KernelGetModuleInfoForUnwind(uint64_t addr, int flags, + ModuleInfoForUnwind* info) { + if (flags >= 3) { + if (info != nullptr) { + std::memset(info, 0, sizeof(ModuleInfoForUnwind)); + } + return KERNEL_ERROR_EINVAL; + } + + if (info == nullptr) { + return KERNEL_ERROR_EFAULT; + } + + if (info->st_size < sizeof(ModuleInfoForUnwind)) { + return KERNEL_ERROR_EINVAL; + } + + auto* rt = Common::Singleton::Instance(); + auto* program = rt->FindProgramByAddr(addr); + if (program == nullptr || program->elf == nullptr) { + if (addr < 0x800000000ull) { + // TODO(unwind): guest unwinding can reach a Kyty host return address below the guest VA + // range. Report a synthetic boundary with no unwind tables so libc stops cleanly + // instead of raising. + std::memset(info, 0, sizeof(ModuleInfoForUnwind)); + info->st_size = sizeof(ModuleInfoForUnwind); + std::snprintf(info->name, sizeof(info->name), "%s", "KytyHostBoundary"); + info->seg0_addr = addr & ~0xfffffull; + info->seg0_size = 0x100000; + return OK; + } + + return KERNEL_ERROR_ESRCH; + } + + const auto* ehdr = program->elf->GetEhdr(); + const auto* phdr = program->elf->GetPhdr(); + if (ehdr == nullptr || phdr == nullptr) { + return KERNEL_ERROR_ESRCH; + } + + constexpr Loader::Elf64_Word PT_GNU_EH_FRAME = 0x6474e550; + + uint64_t eh_frame_hdr_size = 0; + + std::memset(info, 0, sizeof(ModuleInfoForUnwind)); + info->st_size = sizeof(ModuleInfoForUnwind); + + if (program->dynamic_info != nullptr && program->dynamic_info->so_name != nullptr && + program->dynamic_info->so_name[0] != '\0') { + std::snprintf(info->name, sizeof(info->name), "%s", program->dynamic_info->so_name); + } else { + auto fallback_name = Common::PathToString(program->file_name.filename()); + std::snprintf(info->name, sizeof(info->name), "%s", fallback_name.c_str()); + } + + uint64_t image_start = UINT64_MAX; + uint64_t image_end = 0; + + for (int i = 0; i < ehdr->e_phnum; i++) { + if (phdr[i].p_memsz == 0) { + continue; + } + + if (phdr[i].p_type == Loader::PT_LOAD || phdr[i].p_type == Loader::PT_OS_RELRO) { + const auto start = program->base_vaddr + phdr[i].p_vaddr; + const auto end = start + phdr[i].p_memsz; + image_start = std::min(image_start, start); + image_end = std::max(image_end, end); + } + + if (phdr[i].p_type == PT_GNU_EH_FRAME) { + info->eh_frame_hdr_addr = program->base_vaddr + phdr[i].p_vaddr; + eh_frame_hdr_size = phdr[i].p_memsz; + } + } + + const auto* shdr = program->elf->GetShdr(); + if (shdr != nullptr) { + for (int i = 0; i < ehdr->e_shnum; i++) { + const char* section_name = program->elf->GetSectionName(i); + if (section_name != nullptr && std::strcmp(section_name, ".eh_frame") == 0) { + info->eh_frame_addr = program->base_vaddr + shdr[i].sh_addr; + info->eh_frame_size = shdr[i].sh_size; + break; + } + } + } + + if (info->eh_frame_hdr_addr != 0 && eh_frame_hdr_size >= 4) { + const auto* hdr = reinterpret_cast(info->eh_frame_hdr_addr); + const auto* end = hdr + eh_frame_hdr_size; + if (hdr[0] == 1) { + uint64_t eh_frame_addr = 0; + if (DecodeEhFramePointer(hdr + 4, end, hdr[1], info->eh_frame_hdr_addr + 4, + &eh_frame_addr)) { + info->eh_frame_addr = eh_frame_addr; + if (info->eh_frame_size == 0 && info->eh_frame_hdr_addr > eh_frame_addr) { + info->eh_frame_size = info->eh_frame_hdr_addr - eh_frame_addr; + } else if (info->eh_frame_size == 0 && + program->base_size_aligned > + info->eh_frame_hdr_addr - program->base_vaddr) { + info->eh_frame_size = program->base_size_aligned - + (info->eh_frame_hdr_addr - program->base_vaddr); + } + } + } + } + + if (image_start != UINT64_MAX && image_end > image_start) { + info->seg0_addr = image_start; + info->seg0_size = image_end - image_start; + } + + if (info->eh_frame_addr != 0 && info->eh_frame_size == 0) { + for (int i = 0; i < ehdr->e_phnum; i++) { + if (phdr[i].p_memsz == 0 || + (phdr[i].p_type != Loader::PT_LOAD && phdr[i].p_type != Loader::PT_OS_RELRO)) { + continue; + } + + const auto start = program->base_vaddr + phdr[i].p_vaddr; + const auto end = start + phdr[i].p_memsz; + if (info->eh_frame_addr >= start && info->eh_frame_addr < end) { + info->eh_frame_size = end - info->eh_frame_addr; + break; + } + } + } + + return OK; +} + +static int KYTY_SYSV_ABI KernelGetModuleInfoFromAddr(uint64_t addr, int n, ModuleInfo* r) { + PRINT_NAME(); + + LOGF("\taddr = %016" PRIx64 "\n" + "\tn = %d\n", + addr, n); + + EXIT_NOT_IMPLEMENTED(n != 2); + EXIT_NOT_IMPLEMENTED(r == nullptr); + + auto* rt = Common::Singleton::Instance(); + + auto* p = rt->FindProgramByAddr(addr); + + if (p == nullptr) { + LOGF("\thandle: not found\n"); + r->handle = 0; + return -1; + } + + r->handle = p->unique_id; + + LOGF("\thandle: %d\n", r->handle); + + return 0; +} + +static void KYTY_SYSV_ABI KernelDebugRaiseExceptionOnReleaseMode(int /*c1*/, int /*c2*/) { + PRINT_NAME(); +} + +static void KYTY_SYSV_ABI KernelDebugRaiseException(int /*c1*/, int /*c2*/) { + PRINT_NAME(); +} + +static void KYTY_SYSV_ABI exit(int code) { + PRINT_NAME(); + + ::exit(code); +} + +static KYTY_SYSV_ABI MallocReplace* KernelGetSanitizerMallocReplaceExternal() { + PRINT_NAME(); + + static MallocReplace ret; + + return &ret; +} + +static KYTY_SYSV_ABI NewReplace* KernelGetSanitizerNewReplaceExternal() { + PRINT_NAME(); + + static NewReplace ret; + + return &ret; +} + +static KYTY_SYSV_ABI int elf_phdr_match_addr(ModuleInfo* m, uint64_t dtor_vaddr) { + PRINT_NAME(); + + EXIT_NOT_IMPLEMENTED(m == nullptr); + + auto* rt = Common::Singleton::Instance(); + auto* p = rt->FindProgramByAddr(dtor_vaddr); + int result = (p != nullptr && p->unique_id == m->handle) ? 1 : 0; + + LOGF("\thandle = %" PRId32 "\n" + "\tdtor_vaddr = %016" PRIx64 "\n" + "\tmatch = %s\n", + m->handle, dtor_vaddr, result == 1 ? "true" : "false"); + + return result; +} + +int KYTY_SYSV_ABI KernelUuidCreate(uint32_t* uuid) { + PRINT_NAME(); + + if (uuid == nullptr) { + return KERNEL_ERROR_EINVAL; + } + + static thread_local std::mt19937 rng(std::random_device {}()); + uuid[0] = rng(); + uuid[1] = rng(); + uuid[2] = rng(); + uuid[3] = rng(); + + return OK; +} + +static KYTY_SYSV_ABI int KernelGetOpenPsId(void* open_ps_id) { + PRINT_NAME(); + + if (open_ps_id == nullptr) { + return KERNEL_ERROR_EINVAL; + } + + static constexpr uint8_t id[16] = { + 0x4b, 0x79, 0x74, 0x79, 0x4f, 0x70, 0x65, 0x6e, + 0x50, 0x73, 0x49, 0x64, 0x00, 0x00, 0x00, 0x01, + }; + + std::memcpy(open_ps_id, id, sizeof(id)); + + return OK; +} + +static KYTY_SYSV_ABI void pthread_cxa_finalize(void* /*p*/) { + PRINT_NAME(); +} + +void KYTY_SYSV_ABI KernelSetThreadAtexitCount(get_thread_atexit_count_func_t func) { + PRINT_NAME(); + + EXIT_NOT_IMPLEMENTED(g_get_thread_atexit_count_func != nullptr); + + g_get_thread_atexit_count_func = func; +} + +void KYTY_SYSV_ABI KernelSetThreadAtexitReport(thread_atexit_report_func_t func) { + PRINT_NAME(); + + EXIT_NOT_IMPLEMENTED(g_thread_atexit_report_func != nullptr); + + g_thread_atexit_report_func = func; +} + +int KYTY_SYSV_ABI KernelRtldThreadAtexitIncrement(uint64_t* /*c*/) { + PRINT_NAME(); + + //__sync_fetch_and_add(c, 1); + + return 0; +} + +int KYTY_SYSV_ABI KernelRtldThreadAtexitDecrement(uint64_t* /*c*/) { + PRINT_NAME(); + + //__sync_fetch_and_sub(c, 1); + + return 0; +} + +static KYTY_SYSV_ABI int KernelGetCurrentCpu() { + PRINT_NAME(); + + return 0; +} + +int KYTY_SYSV_ABI clock_gettime(int clock_id, LibKernel::KernelTimespec* time) { + PRINT_NAME(); + + return POSIX_CALL(LibKernel::KernelClockGettime(clock_id, time)); +} + +int KYTY_SYSV_ABI clock_getres(int clock_id, LibKernel::KernelTimespec* res) { + PRINT_NAME(); + + return POSIX_CALL(LibKernel::KernelClockGetres(clock_id, res)); +} + +void KYTY_SYSV_ABI KernelSetGPO(uint32_t bits) { + g_gpo_state_bits.store(bits, std::memory_order_relaxed); + + static std::atomic_uint32_t call_count {0}; + auto count = call_count.fetch_add(1, std::memory_order_relaxed); + if (count < 16 || (count % 1024) == 0) { + PRINT_NAME(); + LOGF("\tcall = %u, bits = 0x%08" PRIx32 "\n", count + 1, bits); + } +} + +uint64_t KYTY_SYSV_ABI KernelGetGPI() { + auto bits = g_gpo_state_bits.load(std::memory_order_relaxed); + + static std::atomic_uint32_t call_count {0}; + auto count = call_count.fetch_add(1, std::memory_order_relaxed); + if (count < 16 || (count % 1024) == 0) { + PRINT_NAME(); + LOGF("\tcall = %u, return = 0x%016" PRIx64 "\n", count + 1, bits); + } + + return bits; +} + +} // namespace LibKernel + +namespace Posix { + +LIB_VERSION("Posix", 1, "libkernel", 1, 1); + +int KYTY_SYSV_ABI getpagesize() { + PRINT_NAME(); + + return 0x4000; +} + +int KYTY_SYSV_ABI clock_gettime(int clock_id, LibKernel::KernelTimespec* time) { + PRINT_NAME(); + + return POSIX_CALL(LibKernel::KernelClockGettime(clock_id, time)); +} + +int KYTY_SYSV_ABI clock_getres(int clock_id, LibKernel::KernelTimespec* res) { + PRINT_NAME(); + + return POSIX_CALL(LibKernel::KernelClockGetres(clock_id, res)); +} + +struct KernelTimezone { + int32_t tz_minuteswest; + int32_t tz_dsttime; +}; + +int KYTY_SYSV_ABI gettimeofday(LibKernel::KernelTimeval* time, KernelTimezone* timezone) { + PRINT_NAME(); + + if (time != nullptr) { + int result = LibKernel::KernelGettimeofday(time); + if (result != OK) { + *GetErrorAddr() = LibKernel::KernelToPosix(result); + return -1; + } + } + + if (timezone != nullptr) { + timezone->tz_minuteswest = 0; + timezone->tz_dsttime = 0; + } + + return 0; +} + +int KYTY_SYSV_ABI nanosleep(const LibKernel::KernelTimespec* rqtp, + LibKernel::KernelTimespec* rmtp) { + PRINT_NAME(); + + return POSIX_CALL(LibKernel::KernelNanosleep(rqtp, rmtp)); +} + +int KYTY_SYSV_ABI stat(const char* path, LibKernel::FileSystem::FileStat* sb) { + PRINT_NAME(); + + return POSIX_CALL(LibKernel::FileSystem::KernelStat(path, sb)); +} + +int KYTY_SYSV_ABI mkdir(const char* path, uint16_t mode) { + PRINT_NAME(); + + return POSIX_CALL(LibKernel::FileSystem::KernelMkdir(path, mode)); +} + +int64_t KYTY_SYSV_ABI lseek(int d, int64_t offset, int whence) { + PRINT_NAME(); + + return POSIX_N_CALL(LibKernel::FileSystem::KernelLseek(d, offset, whence)); +} + +int64_t KYTY_SYSV_ABI pread(int d, void* buf, size_t nbytes, int64_t offset) { + PRINT_NAME(); + + return POSIX_N_CALL(LibKernel::FileSystem::KernelPread(d, buf, nbytes, offset)); +} + +int64_t KYTY_SYSV_ABI pwrite(int d, const void* buf, size_t nbytes, int64_t offset) { + PRINT_NAME(); + + return POSIX_N_CALL(LibKernel::FileSystem::KernelPwrite(d, buf, nbytes, offset)); +} + +int KYTY_SYSV_ABI flock(int d, int operation) { + PRINT_NAME(); + + constexpr int lock_sh = 0x01; + constexpr int lock_ex = 0x02; + constexpr int lock_nb = 0x04; + constexpr int lock_un = 0x08; + + auto lock_mode = operation & ~(lock_nb); + if (lock_mode != lock_sh && lock_mode != lock_ex && lock_mode != lock_un) { + *GetErrorAddr() = POSIX_EINVAL; + return -1; + } + + LibKernel::FileSystem::FileStat sb {}; + auto result = LibKernel::FileSystem::KernelFstat(d, &sb); + if (result != OK) { + *GetErrorAddr() = LibKernel::KernelToPosix(result); + return -1; + } + + return 0; +} + +int64_t KYTY_SYSV_ABI fstat(int d, LibKernel::FileSystem::FileStat* sb) { + PRINT_NAME(); + + return POSIX_N_CALL(LibKernel::FileSystem::KernelFstat(d, sb)); +} + +int KYTY_SYSV_ABI socket(int family, int type, int protocol) { + PRINT_NAME(); + return Network::Net::Socket(family, type, protocol); +} + +int KYTY_SYSV_ABI bind(int s, const void* addr, uint32_t addrlen) { + PRINT_NAME(); + return Network::Net::Bind(s, addr, addrlen); +} + +int KYTY_SYSV_ABI connect(int s, const void* addr, uint32_t addrlen) { + PRINT_NAME(); + return Network::Net::Connect(s, addr, addrlen); +} + +int KYTY_SYSV_ABI listen(int s, int backlog) { + PRINT_NAME(); + return Network::Net::Listen(s, backlog); +} + +int KYTY_SYSV_ABI accept(int s, void* addr, uint32_t* addrlen) { + PRINT_NAME(); + return Network::Net::Accept(s, addr, addrlen); +} + +int KYTY_SYSV_ABI getsockname(int s, void* addr, uint32_t* addrlen) { + PRINT_NAME(); + return Network::Net::Getsockname(s, addr, addrlen); +} + +int KYTY_SYSV_ABI getsockopt(int s, int level, int optname, void* optval, uint32_t* optlen) { + PRINT_NAME(); + return Network::Net::Getsockopt(s, level, optname, optval, optlen); +} + +int KYTY_SYSV_ABI setsockopt(int s, int level, int optname, const void* optval, uint32_t optlen) { + PRINT_NAME(); + return Network::Net::Setsockopt(s, level, optname, optval, optlen); +} + +int KYTY_SYSV_ABI select(int nfds, void* readfds, void* writefds, void* exceptfds, + const void* timeout) { + PRINT_NAME(); + return Network::Net::Select(nfds, readfds, writefds, exceptfds, timeout); +} + +int64_t KYTY_SYSV_ABI send(int s, const void* buf, uint64_t len, int flags) { + PRINT_NAME(); + return Network::Net::Send(s, buf, len, flags); +} + +int64_t KYTY_SYSV_ABI recv(int s, void* buf, uint64_t len, int flags) { + PRINT_NAME(); + return Network::Net::Recv(s, buf, len, flags); +} + +const char* KYTY_SYSV_ABI inet_ntop(int af, const void* src, char* dst, uint32_t size) { + PRINT_NAME(); + + const char* result = Network::Net::NetInetNtop(af, src, dst, size); + if (result == nullptr) { + *GetErrorAddr() = (af == 2 ? POSIX_ENOSPC : POSIX_EAFNOSUPPORT); + } + + return result; +} + +uint64_t KYTY_SYSV_ABI cfwBSQyr5Ys(uint64_t a1, uint64_t a2, uint64_t a3, uint64_t a4, uint64_t a5, + uint64_t a6) { + PRINT_NAME(); + + LOGF("\t return = 0x%016" PRIx64 "\n" + "\t a1 = 0x%016" PRIx64 "\n" + "\t a2 = 0x%016" PRIx64 "\n" + "\t a3 = 0x%016" PRIx64 "\n" + "\t a4 = 0x%016" PRIx64 "\n" + "\t a5 = 0x%016" PRIx64 "\n" + "\t a6 = 0x%016" PRIx64 "\n" + "\t result = 0\n", + reinterpret_cast(__builtin_return_address(0)), a1, a2, a3, a4, a5, a6); + + return 0; +} + +uint64_t KYTY_SYSV_ABI KernelSyncOnAddressV1(uint64_t op, uint64_t address, uint64_t value, + uint64_t size, uint64_t timeout, uint64_t flags) { + static std::atomic_uint32_t log_count = 0; + const auto index = log_count.fetch_add(1, std::memory_order_relaxed); + + if (index < 16) { + LOGF("\t libkernel_sync_on_address_v1: op=0x%016" PRIx64 ", address=0x%016" PRIx64 + ", value=0x%016" PRIx64 ", size=0x%016" PRIx64 ", timeout=0x%016" PRIx64 + ", flags=0x%016" PRIx64 "\n", + op, address, value, size, timeout, flags); + } + + if (op != 0 && address == 0 && value == 0 && size == 0) { + // This unsupported form is used as a yield/wait by some Unity jobs. + // SleepMicro() uses a sub-millisecond busy wait on Windows, which can + // pin every worker thread when the guest polls this path. + Common::Thread::Sleep(timeout == 0 ? 1 : 2); + } + + return 0; +} + +LIB_DEFINE(InitLibKernel_1_Posix) { + LIB_FUNC("k+AXqu2-eBc", getpagesize); + LIB_FUNC("lLMT9vJAck0", clock_gettime); + LIB_FUNC("smIj7eqzZE8", clock_getres); + LIB_FUNC("n88vx3C5nW8", gettimeofday); + LIB_FUNC("NhpspxdjEKU", nanosleep); + LIB_FUNC("yS8U2TGCe1A", nanosleep); + LIB_FUNC("E6ao34wPw+U", stat); + LIB_FUNC("JGMio+21L4c", mkdir); + LIB_FUNC("pDuPEf3m4fI", Posix::sem_init); + LIB_FUNC("cDW233RAwWo", Posix::sem_destroy); + LIB_FUNC("YCV5dGGBcCo", Posix::sem_wait); + LIB_FUNC("WBWzsRifCEA", Posix::sem_trywait); + LIB_FUNC("4SbrhCozqQU", Posix::sem_reltimedwait_np); + LIB_FUNC("w5IHyvahg-o", Posix::sem_timedwait); + LIB_FUNC("IKP8typ0QUk", Posix::sem_post); + LIB_FUNC("Bq+LRV-N6Hk", Posix::sem_getvalue); + + LIB_FUNC("OxhIB8LB-PQ", Posix::pthread_create); + LIB_FUNC("Jmi+9w9u0E4", Posix::pthread_create_name_np); + LIB_FUNC("+U1R4WtXvoc", Posix::pthread_detach); + LIB_FUNC("FJrT5LuUBAU", Posix::pthread_exit); + LIB_FUNC("EotR8a3ASf4", Posix::pthread_self); + LIB_FUNC("9vyP6Z7bqzc", Posix::pthread_rename_np); + LIB_FUNC("lZzFeSxPl08", Posix::pthread_setcancelstate); + LIB_FUNC("a2P9wYGeZvc", Posix::pthread_setprio); + LIB_FUNC("FIs3-UQT9sg", Posix::pthread_getschedparam); + LIB_FUNC("P41kTWUS3EI", Posix::pthread_getschedparam); + LIB_FUNC("Xs9hdiD7sAA", Posix::pthread_setschedparam); + LIB_FUNC("oIRFTjoILbg", Posix::pthread_setschedparam); + LIB_FUNC("B5GmVDKwpn0", Posix::pthread_yield); + LIB_FUNC("h9CcP3J0oVM", Posix::pthread_join); + LIB_FUNC("wtkt-teR1so", Posix::pthread_attr_init); + LIB_FUNC("zHchY8ft5pk", Posix::pthread_attr_destroy); + LIB_FUNC("vQm4fDEsWi8", Posix::pthread_attr_getstack); + LIB_FUNC("2Q0z6rnBrTE", Posix::pthread_attr_setstacksize); + LIB_FUNC("Ucsu-OK+els", Posix::pthread_attr_get_np); + LIB_FUNC("RtLRV-pBTTY", Posix::pthread_attr_getschedpolicy); + LIB_FUNC("JarMIy8kKEY", Posix::pthread_attr_setschedpolicy); + LIB_FUNC("E+tyo3lp5Lw", Posix::pthread_attr_setdetachstate); + LIB_FUNC("euKRgm0Vn2M", Posix::pthread_attr_setschedparam); + LIB_FUNC("7ZlAakEf0Qg", Posix::pthread_attr_setinheritsched); + LIB_FUNC("0qOtCR-ZHck", Posix::pthread_attr_getstacksize); + LIB_FUNC("VUT1ZSrHT0I", Posix::pthread_attr_getdetachstate); + LIB_FUNC("JKyG3SWyA10", Posix::pthread_attr_setguardsize); + LIB_FUNC("JNkVVsVDmOk", Posix::pthread_attr_getguardsize); + LIB_FUNC("qlk9pSLsUmM", Posix::pthread_attr_getschedparam); + LIB_FUNC("FXPWHNk8Of0", Posix::pthread_attr_getschedparam); + LIB_FUNC("7Xl257M4VNI", LibKernel::PthreadEqual); + LIB_FUNC("7H0iTOciTLo", Posix::pthread_mutex_lock); + LIB_FUNC("K-jXhbt2gn4", Posix::pthread_mutex_trylock); + LIB_FUNC("2Z+PpY6CaJg", Posix::pthread_mutex_unlock); + LIB_FUNC("ttHNfU+qDBU", Posix::pthread_mutex_init); + LIB_FUNC("dQHWEsJtoE4", Posix::pthread_mutexattr_init); + LIB_FUNC("mDmgMOGVUqg", Posix::pthread_mutexattr_settype); + LIB_FUNC("5txKfcMUAok", Posix::pthread_mutexattr_setprotocol); + LIB_FUNC("HF7lK46xzjY", Posix::pthread_mutexattr_destroy); + LIB_FUNC("ltCfaGr2JGE", Posix::pthread_mutex_destroy); + LIB_FUNC("mkx2fVhNMsg", Posix::pthread_cond_broadcast); + LIB_FUNC("2MOy+rUfuhQ", Posix::pthread_cond_signal); + LIB_FUNC("Op8TBGY5KHg", Posix::pthread_cond_wait); + LIB_FUNC("Z4QosVuAsA0", Posix::pthread_once); + LIB_FUNC("1471ajPzxh0", Posix::pthread_rwlock_destroy); + LIB_FUNC("sIlRvQqsN2Y", Posix::pthread_rwlock_wrlock); + LIB_FUNC("SFxTMOfuCkE", LibKernel::PthreadRwlockTryrdlock); + LIB_FUNC("XhWHn6P5R7U", LibKernel::PthreadRwlockTrywrlock); + LIB_FUNC("mqULNdimTn0", Posix::pthread_key_create); + LIB_FUNC("6BpEZuDT7YI", Posix::pthread_key_delete); + LIB_FUNC("WrOLvHU0yQM", Posix::pthread_setspecific); + LIB_FUNC("0-KXaS70xy4", Posix::pthread_getspecific); + LIB_FUNC("-quPa4SEJUw", Posix::pthread_getstack); + LIB_FUNC("YQOfxL4QfeU", LibKernel::Memory::KernelMprotect); + LIB_FUNC("UqDGjXA5yUM", LibKernel::Memory::KernelMunmap); + LIB_FUNC("AqBioC2vF3I", LibKernel::read); + LIB_FUNC("TU-d9PfIHPM", Posix::socket); + LIB_FUNC("KuOmgKoqCdY", Posix::bind); + LIB_FUNC("XVL8So3QJUk", Posix::connect); + LIB_FUNC("pxnCmagrtao", Posix::listen); + LIB_FUNC("3e+4Iv7IJ8U", Posix::accept); + LIB_FUNC("RenI1lL1WFk", Posix::getsockname); + LIB_FUNC("6O8EwYOgH9Y", Posix::getsockopt); + LIB_FUNC("fFxGkxF2bVo", Posix::setsockopt); + LIB_FUNC("T8fER+tIGgk", Posix::select); + LIB_FUNC("fZOeZIOEmLw", Posix::send); + LIB_FUNC("Ez8xjo9UF4E", Posix::recv); + LIB_FUNC("5jRCs2axtr4", Posix::inet_ntop); + LIB_FUNC("cfwBSQyr5Ys", cfwBSQyr5Ys); +} + +} // namespace Posix + +namespace Coredump { + +LIB_VERSION("Coredump", 1, "Coredump", 1, 1); + +static uint64_t g_coredump_handler = 0; +static uint64_t g_coredump_handler_context = 0; + +int KYTY_SYSV_ABI sceCoredumpRegisterCoredumpHandler(uint64_t handler, size_t stack_size, + uint64_t context) { + PRINT_NAME(); + + g_coredump_handler = handler; + g_coredump_handler_context = context; + + LOGF("\t handler = 0x%016" PRIx64 "\n" + "\t stack = 0x%016" PRIx64 "\n" + "\t context = 0x%016" PRIx64 "\n", + handler, static_cast(stack_size), context); + + return OK; +} + +int KYTY_SYSV_ABI sceCoredumpUnregisterCoredumpHandler() { + PRINT_NAME(); + + g_coredump_handler = 0; + g_coredump_handler_context = 0; + + return OK; +} + +} // namespace Coredump + +namespace Fiber { + +LIB_VERSION("Fiber", 1, "Fiber", 1, 1); + +constexpr int32_t FIBER_ERROR_NULL = -2141650943; /* 0x80590001 */ +constexpr int32_t FIBER_ERROR_ALIGNMENT = -2141650942; /* 0x80590002 */ +constexpr int32_t FIBER_ERROR_RANGE = -2141650941; /* 0x80590003 */ +constexpr int32_t FIBER_ERROR_INVALID = -2141650940; /* 0x80590004 */ +constexpr int32_t FIBER_ERROR_PERMISSION = -2141650939; /* 0x80590005 */ +constexpr int32_t FIBER_ERROR_STATE = -2141650938; /* 0x80590006 */ + +constexpr uint32_t FIBER_MAGIC_START = 0xdef1649c; +constexpr uint32_t FIBER_MAGIC_END = 0xb37592a0; +constexpr uint32_t FIBER_OPT_MAGIC = 0xbb40e64d; +constexpr uint64_t FIBER_STACK_MAGIC = 0x7149f2ca7149f2ca; +constexpr uint64_t FIBER_CONTEXT_MIN_SIZE = 512; +constexpr size_t FIBER_MAX_NAME_LENGTH = 31; +constexpr uint32_t FIBER_STATE_RUNNING = 1; +constexpr uint32_t FIBER_STATE_IDLE = 2; +constexpr uint32_t FIBER_STATE_TERMINATED = 3; +constexpr uint32_t FIBER_FLAG_SET_FPU_REGS = 0x100; + +using FiberEntry = KYTY_SYSV_ABI void (*)(uint64_t arg_on_initialize, uint64_t arg_on_run); + +struct FiberOptParam { + uint32_t magic; +}; + +struct FiberCpuContext { + uint64_t rbx; + uint64_t rbp; + uint64_t rdi; + uint64_t rsi; + uint64_t r12; + uint64_t r13; + uint64_t r14; + uint64_t r15; + uint64_t rsp; + uint64_t rip; +}; + +static_assert(sizeof(FiberCpuContext) == 80); + +struct FiberObject { + uint32_t magic_start; + uint32_t state; + FiberEntry entry; + uint64_t arg_on_initialize; + void* addr_context; + uint64_t size_context; + char name[FIBER_MAX_NAME_LENGTH + 1]; + void* context; + uint32_t flags; + uint32_t padding; + void* context_start; + void* context_end; + FiberCpuContext saved_context; + uint64_t arg_on_run; + uint64_t arg_on_return; + bool context_valid; + uint32_t magic_end; +}; + +static_assert(sizeof(FiberObject) <= 256); + +struct FiberInfo { + uint64_t size; + FiberEntry entry; + uint64_t arg_on_initialize; + void* addr_context; + uint64_t size_context; + char name[FIBER_MAX_NAME_LENGTH + 1]; + uint64_t size_context_margin; + uint8_t padding[48]; +}; + +static_assert(sizeof(FiberInfo) == 128); + +static bool FiberIsValid(const FiberObject* fiber) { + return fiber != nullptr && fiber->magic_start == FIBER_MAGIC_START && + fiber->magic_end == FIBER_MAGIC_END; +} + +static thread_local FiberObject* g_current_fiber = nullptr; +static thread_local FiberObject* g_thread_return_fiber = nullptr; +static thread_local FiberObject* g_starting_fiber = nullptr; +static thread_local FiberObject* g_pending_idle_fiber = nullptr; +static thread_local FiberCpuContext g_thread_fiber_context {}; +static std::mutex g_fiber_owner_mutex; +static std::unordered_map g_fiber_owner_thread; +static std::unordered_map g_fiber_current_by_thread; + +static uint32_t FiberLoadState(const FiberObject* fiber) { + auto& state = const_cast(fiber->state); + return std::atomic_ref(state).load(std::memory_order_acquire); +} + +static uint64_t FiberCurrentHostThreadId() { +#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS + return static_cast(::GetCurrentThreadId()); +#else + return std::hash {}(std::this_thread::get_id()); +#endif +} + +static void FiberSetOwner(FiberObject* fiber) { + std::lock_guard lock(g_fiber_owner_mutex); + g_fiber_owner_thread[fiber] = FiberCurrentHostThreadId(); +} + +static void FiberClearOwner(FiberObject* fiber) { + std::lock_guard lock(g_fiber_owner_mutex); + g_fiber_owner_thread.erase(fiber); +} + +static uint64_t FiberGetOwner(FiberObject* fiber) { + std::lock_guard lock(g_fiber_owner_mutex); + auto it = g_fiber_owner_thread.find(fiber); + return it != g_fiber_owner_thread.end() ? it->second : 0; +} + +static void FiberSetCurrentFiber(FiberObject* fiber) { + g_current_fiber = fiber; + + std::lock_guard lock(g_fiber_owner_mutex); + const auto thread_id = FiberCurrentHostThreadId(); + if (fiber != nullptr) { + g_fiber_current_by_thread[thread_id] = fiber; + } else { + g_fiber_current_by_thread.erase(thread_id); + } +} + +static void FiberStoreState(FiberObject* fiber, uint32_t state) { + std::atomic_ref(fiber->state).store(state, std::memory_order_release); + if (state == FIBER_STATE_RUNNING) { + FiberSetOwner(fiber); + } else { + FiberClearOwner(fiber); + } +} + +static void FiberDeferIdle(FiberObject* fiber) { + g_pending_idle_fiber = fiber; +} + +static void FiberCommitDeferredIdle() { + auto* fiber = g_pending_idle_fiber; + g_pending_idle_fiber = nullptr; + if (fiber != nullptr) { + FiberStoreState(fiber, FIBER_STATE_IDLE); + } +} + +static bool FiberCompareExchangeState(FiberObject* fiber, uint32_t expected, uint32_t desired) { + const bool ok = std::atomic_ref(fiber->state) + .compare_exchange_strong(expected, desired, std::memory_order_acq_rel, + std::memory_order_acquire); + if (ok) { + if (desired == FIBER_STATE_RUNNING) { + FiberSetOwner(fiber); + } else { + FiberClearOwner(fiber); + } + } + return ok; +} + +static bool FiberWaitAndEnterRunning(FiberObject* fiber, uint32_t* observed_state) { + const auto start = std::chrono::steady_clock::now(); + uint32_t spin = 0; + auto& state = fiber->state; + auto ref = std::atomic_ref(state); + + for (;;) { + uint32_t expected = FIBER_STATE_IDLE; + if (ref.compare_exchange_strong(expected, FIBER_STATE_RUNNING, std::memory_order_acq_rel, + std::memory_order_acquire)) { + FiberSetOwner(fiber); + return true; + } + if (observed_state != nullptr) { + *observed_state = expected; + } + if (expected != FIBER_STATE_RUNNING) { + return false; + } + if (std::chrono::steady_clock::now() - start >= std::chrono::milliseconds(5)) { + return false; + } + if (spin++ < 64) { + std::this_thread::yield(); + } else { + std::this_thread::sleep_for(std::chrono::microseconds(100)); + } + } +} + +static bool FiberRepairStaleRunningOnThisThread(FiberObject* fiber, uint32_t observed_state) { + if (observed_state != FIBER_STATE_RUNNING || fiber == g_current_fiber) { + return false; + } + + const auto owner = FiberGetOwner(fiber); + if (owner != 0) { + return false; + } + + return FiberCompareExchangeState(fiber, FIBER_STATE_RUNNING, FIBER_STATE_IDLE); +} + +static void FiberSetContextValid(FiberObject* fiber, bool valid) { + fiber->context_valid = valid; + fiber->context = valid ? &fiber->saved_context : nullptr; +} + +#if defined(__x86_64__) || defined(_M_X64) +__attribute__((noinline, returns_twice)) static int FiberSaveContext(FiberCpuContext* ctx) { + int ret = 0; + asm volatile("movq %[ctx], %%r10\n\t" + "movq %%rbx, 0(%%r10)\n\t" + "movq %%rbp, 8(%%r10)\n\t" + "movq %%rdi, 16(%%r10)\n\t" + "movq %%rsi, 24(%%r10)\n\t" + "movq %%r12, 32(%%r10)\n\t" + "movq %%r13, 40(%%r10)\n\t" + "movq %%r14, 48(%%r10)\n\t" + "movq %%r15, 56(%%r10)\n\t" + "leaq 8(%%rsp), %%r11\n\t" + "movq %%r11, 64(%%r10)\n\t" + "movq (%%rsp), %%r11\n\t" + "movq %%r11, 72(%%r10)\n\t" + "xorl %%eax, %%eax\n\t" + : "=a"(ret) + : [ctx] "r"(ctx) + : "memory", "r10", "r11"); + return ret; +} + +__attribute__((noreturn, noinline)) static void FiberRestoreContext(FiberCpuContext* ctx, + uint64_t ret) { + asm volatile("movq %[ctx], %%r10\n\t" + "movq 72(%%r10), %%r11\n\t" + "movq 0(%%r10), %%rbx\n\t" + "movq 8(%%r10), %%rbp\n\t" + "movq 16(%%r10), %%rdi\n\t" + "movq 24(%%r10), %%rsi\n\t" + "movq 32(%%r10), %%r12\n\t" + "movq 40(%%r10), %%r13\n\t" + "movq 48(%%r10), %%r14\n\t" + "movq 56(%%r10), %%r15\n\t" + "movq 64(%%r10), %%rsp\n\t" + "movq %[ret], %%rax\n\t" + "jmp *%%r11\n\t" + : + : [ctx] "r"(ctx), [ret] "r"(ret) + : "memory", "rax", "r10", "r11"); + __builtin_unreachable(); +} +#else +static int FiberSaveContext(FiberCpuContext* ctx) { + (void)ctx; + return 0; +} + +static void FiberRestoreContext(FiberCpuContext* ctx, uint64_t ret) { + (void)ctx; + (void)ret; + EXIT("Fiber context switching is only implemented on x86_64\n"); +} +#endif + +[[noreturn]] static void FiberStartTrampoline(); + +[[noreturn]] static void FiberStartOnGuestStack(FiberObject* fiber) { + FiberCpuContext ctx {}; + const auto stack_top = reinterpret_cast(fiber->addr_context) + + static_cast(fiber->size_context); + auto rsp = (stack_top & ~static_cast(0x0f)); +#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS + rsp -= 4u * sizeof(uint64_t); +#endif + rsp -= sizeof(uint64_t); + *reinterpret_cast(rsp) = 0; + + ctx.rsp = rsp; + ctx.rip = reinterpret_cast(&FiberStartTrampoline); + + g_starting_fiber = fiber; + FiberRestoreContext(&ctx, 1); +} + +[[noreturn]] static void FiberStartTrampoline() { + auto* fiber = g_starting_fiber; + g_starting_fiber = nullptr; + if (fiber == nullptr) { + EXIT("Fiber start without current fiber\n"); + } + + FiberCommitDeferredIdle(); + FiberSetCurrentFiber(fiber); + + fiber->entry(fiber->arg_on_initialize, fiber->arg_on_run); + + FiberStoreState(fiber, FIBER_STATE_TERMINATED); + FiberSetContextValid(fiber, false); + fiber->arg_on_return = 0; + g_thread_return_fiber = fiber; + FiberSetCurrentFiber(nullptr); + + FiberRestoreContext(&g_thread_fiber_context, 1); +} + +int32_t KYTY_SYSV_ABI FiberInitialize(FiberObject* fiber, const char* name, FiberEntry entry, + uint64_t arg_on_initialize, void* addr_context, + uint64_t size_context, const FiberOptParam* opt_param, + uint32_t build_version) { + PRINT_NAME(); + + if (fiber == nullptr || name == nullptr || entry == nullptr) { + return FIBER_ERROR_NULL; + } + + if ((reinterpret_cast(fiber) & 7u) != 0 || + (reinterpret_cast(addr_context) & 15u) != 0 || + (opt_param != nullptr && (reinterpret_cast(opt_param) & 7u) != 0)) { + return FIBER_ERROR_ALIGNMENT; + } + + if (size_context != 0 && size_context < FIBER_CONTEXT_MIN_SIZE) { + return FIBER_ERROR_RANGE; + } + + if ((size_context & 15u) != 0 || (addr_context == nullptr && size_context != 0) || + (addr_context != nullptr && size_context == 0) || + (opt_param != nullptr && opt_param->magic != FIBER_OPT_MAGIC)) { + return FIBER_ERROR_INVALID; + } + + std::memset(fiber, 0, sizeof(*fiber)); + std::strncpy(fiber->name, name, FIBER_MAX_NAME_LENGTH); + fiber->name[FIBER_MAX_NAME_LENGTH] = '\0'; + + fiber->magic_start = FIBER_MAGIC_START; + fiber->state = FIBER_STATE_IDLE; + fiber->entry = entry; + fiber->arg_on_initialize = arg_on_initialize; + fiber->addr_context = addr_context; + fiber->size_context = size_context; + fiber->flags = (build_version >= 0x03500000 ? FIBER_FLAG_SET_FPU_REGS : 0); + fiber->context_start = addr_context; + fiber->context_end = + (addr_context != nullptr ? static_cast(addr_context) + size_context : nullptr); + std::memset(&fiber->saved_context, 0, sizeof(fiber->saved_context)); + FiberSetContextValid(fiber, false); + fiber->magic_end = FIBER_MAGIC_END; + + if (addr_context != nullptr) { + *static_cast(addr_context) = FIBER_STACK_MAGIC; + } + + LOGF("\t fiber init: %s, entry = 0x%016" PRIx64 ", context = 0x%016" PRIx64 ", size = %" PRIu64 + "\n", + fiber->name, reinterpret_cast(entry), reinterpret_cast(addr_context), + size_context); + + return OK; +} + +int32_t KYTY_SYSV_ABI FiberInitializeInternal(FiberObject* fiber, const char* name, + FiberEntry entry, uint64_t arg_on_initialize, + void* addr_context, uint64_t size_context, + const FiberOptParam* opt_param, uint32_t flags, + uint32_t build_version) { + auto ret = FiberInitialize(fiber, name, entry, arg_on_initialize, addr_context, size_context, + opt_param, build_version); + if (ret == OK && fiber != nullptr) { + fiber->flags |= flags; + } + return ret; +} + +int32_t KYTY_SYSV_ABI FiberOptParamInitialize(FiberOptParam* opt_param) { + PRINT_NAME(); + + if (opt_param == nullptr) { + return FIBER_ERROR_NULL; + } + if ((reinterpret_cast(opt_param) & 7u) != 0) { + return FIBER_ERROR_ALIGNMENT; + } + + std::memset(opt_param, 0, 128); + opt_param->magic = FIBER_OPT_MAGIC; + + return OK; +} + +int32_t KYTY_SYSV_ABI FiberFinalize(FiberObject* fiber) { + PRINT_NAME(); + + if (fiber == nullptr) { + return FIBER_ERROR_NULL; + } + if (!FiberIsValid(fiber)) { + return FIBER_ERROR_INVALID; + } + if (!FiberCompareExchangeState(fiber, FIBER_STATE_IDLE, FIBER_STATE_TERMINATED)) { + return FIBER_ERROR_STATE; + } + + return OK; +} + +int32_t KYTY_SYSV_ABI FiberRun(FiberObject* fiber, uint64_t arg_on_run, uint64_t* arg_on_return) { + PRINT_NAME(); + + if (!FiberIsValid(fiber)) { + return FIBER_ERROR_INVALID; + } + if (g_current_fiber != nullptr) { + return FIBER_ERROR_PERMISSION; + } + if (!FiberCompareExchangeState(fiber, FIBER_STATE_IDLE, FIBER_STATE_RUNNING)) { + return FIBER_ERROR_STATE; + } + + fiber->arg_on_run = arg_on_run; + fiber->arg_on_return = 0; + g_thread_return_fiber = nullptr; + + if (FiberSaveContext(&g_thread_fiber_context) == 0) { + if (fiber->context_valid) { + FiberRestoreContext(&fiber->saved_context, 1); + } + FiberStartOnGuestStack(fiber); + } + + FiberCommitDeferredIdle(); + FiberSetCurrentFiber(nullptr); + auto* returned_fiber = (g_thread_return_fiber != nullptr ? g_thread_return_fiber : fiber); + g_thread_return_fiber = nullptr; + + if (arg_on_return != nullptr) { + *arg_on_return = returned_fiber->arg_on_return; + } + + return (FiberLoadState(returned_fiber) == FIBER_STATE_TERMINATED ? FIBER_ERROR_STATE : OK); +} + +int32_t KYTY_SYSV_ABI FiberSwitch(FiberObject* fiber, uint64_t arg_on_run, + uint64_t* arg_on_return) { + PRINT_NAME(); + + if (!FiberIsValid(fiber)) { + return FIBER_ERROR_INVALID; + } + if (g_current_fiber == nullptr) { + return FIBER_ERROR_PERMISSION; + } + + for (;;) { + uint32_t observed_state = 0; + if (FiberWaitAndEnterRunning(fiber, &observed_state)) { + break; + } + if (FiberRepairStaleRunningOnThisThread(fiber, observed_state)) { + continue; + } + return FIBER_ERROR_STATE; + } + + auto* caller = g_current_fiber; + + fiber->arg_on_run = arg_on_run; + fiber->arg_on_return = 0; + + if (FiberSaveContext(&caller->saved_context) == 0) { + FiberSetContextValid(caller, true); + FiberDeferIdle(caller); + if (fiber->context_valid) { + FiberRestoreContext(&fiber->saved_context, 1); + } + FiberStartOnGuestStack(fiber); + } + + FiberCommitDeferredIdle(); + FiberSetCurrentFiber(caller); + FiberStoreState(caller, FIBER_STATE_RUNNING); + + if (arg_on_return != nullptr) { + *arg_on_return = caller->arg_on_run; + } + + return OK; +} + +int32_t KYTY_SYSV_ABI FiberGetSelf(FiberObject** fiber) { + PRINT_NAME(); + + if (fiber == nullptr) { + return FIBER_ERROR_NULL; + } + + *fiber = g_current_fiber; + + return OK; +} + +int32_t KYTY_SYSV_ABI FiberReturnToThread(uint64_t arg_on_return, uint64_t* arg_on_run) { + PRINT_NAME(); + + if (g_current_fiber == nullptr) { + return FIBER_ERROR_PERMISSION; + } + + auto* fiber = g_current_fiber; + fiber->arg_on_return = arg_on_return; + + if (FiberSaveContext(&fiber->saved_context) == 0) { + FiberSetContextValid(fiber, true); + g_thread_return_fiber = fiber; + FiberDeferIdle(fiber); + FiberRestoreContext(&g_thread_fiber_context, 1); + } + + FiberCommitDeferredIdle(); + FiberSetCurrentFiber(fiber); + FiberStoreState(fiber, FIBER_STATE_RUNNING); + if (arg_on_run != nullptr) { + *arg_on_run = fiber->arg_on_run; + } + + return OK; +} + +int32_t KYTY_SYSV_ABI FiberGetInfo(FiberObject* fiber, FiberInfo* fiber_info) { + PRINT_NAME(); + + if (fiber == nullptr || fiber_info == nullptr) { + return FIBER_ERROR_NULL; + } + if (!FiberIsValid(fiber)) { + return FIBER_ERROR_INVALID; + } + + std::memset(fiber_info, 0, sizeof(*fiber_info)); + fiber_info->size = sizeof(*fiber_info); + fiber_info->entry = fiber->entry; + fiber_info->arg_on_initialize = fiber->arg_on_initialize; + fiber_info->addr_context = fiber->addr_context; + fiber_info->size_context = fiber->size_context; + std::strncpy(fiber_info->name, fiber->name, FIBER_MAX_NAME_LENGTH); + + return OK; +} + +int32_t KYTY_SYSV_ABI FiberStartContextSizeCheck(uint32_t flags) { + PRINT_NAME(); + LOGF("\t flags = 0x%08" PRIx32 "\n", flags); + return OK; +} + +int32_t KYTY_SYSV_ABI FiberStopContextSizeCheck() { + PRINT_NAME(); + return OK; +} + +int32_t KYTY_SYSV_ABI FiberRename(FiberObject* fiber, const char* name) { + PRINT_NAME(); + + if (fiber == nullptr || name == nullptr) { + return FIBER_ERROR_NULL; + } + if (!FiberIsValid(fiber)) { + return FIBER_ERROR_INVALID; + } + + std::strncpy(fiber->name, name, FIBER_MAX_NAME_LENGTH); + fiber->name[FIBER_MAX_NAME_LENGTH] = '\0'; + + return OK; +} + +int32_t KYTY_SYSV_ABI FiberGetThreadFramePointerAddress(uint64_t* addr_frame_pointer) { + PRINT_NAME(); + + if (addr_frame_pointer == nullptr) { + return FIBER_ERROR_NULL; + } + + *addr_frame_pointer = 0; + + return OK; +} + +} // namespace Fiber + +int chmod(const char* path, int mode) { + PRINT_NAME(); + + return OK; +} + +int KYTY_SYSV_ABI KernelAioInitializeImpl(void* param, int32_t size) { + PRINT_NAME(); + + LOGF("\t param = 0x%016" PRIx64 "\n" + "\t size = 0x%08" PRIx32 "\n", + reinterpret_cast(param), static_cast(size)); + + return OK; +} + +void KYTY_SYSV_ABI KernelAioInitializeParam(void* param) { + PRINT_NAME(); + + LOGF("\t param = 0x%016" PRIx64 "\n", reinterpret_cast(param)); +} + +constexpr int32_t KERNEL_AIO_STATE_SUBMITTED = 1; +constexpr int32_t KERNEL_AIO_STATE_PROCESSING = 2; +constexpr int32_t KERNEL_AIO_STATE_COMPLETED = 3; +constexpr int32_t KERNEL_AIO_STATE_ABORTED = 4; +constexpr int32_t KERNEL_AIO_MAX_QUEUE = 512; +constexpr int32_t KERNEL_AIO_MAX_REQUESTS = 128; + +struct KernelAioResult { + int64_t return_value; + uint32_t state; +}; + +struct KernelAioRwRequest { + int64_t offset; + size_t nbyte; + void* buf; + KernelAioResult* result; + int32_t fd; +}; + +static std::array g_kernel_aio_state = {}; +static std::atomic_int32_t g_kernel_aio_next_id {1}; + +static int32_t kernel_aio_next_id() { + const auto sequence = g_kernel_aio_next_id.fetch_add(1, std::memory_order_relaxed); + return ((sequence - 1) % (KERNEL_AIO_MAX_QUEUE - 1)) + 1; +} + +static bool kernel_aio_is_valid_id(int32_t id) { + return id > 0 && id < KERNEL_AIO_MAX_QUEUE; +} + +int KYTY_SYSV_ABI KernelAioSubmitReadCommands(KernelAioRwRequest* req, int32_t size, int32_t prio, + int32_t* id) { + PRINT_NAME(); + + if (req == nullptr || id == nullptr) { + return LibKernel::KERNEL_ERROR_EFAULT; + } + + if (size <= 0 || size > KERNEL_AIO_MAX_REQUESTS) { + return LibKernel::KERNEL_ERROR_EINVAL; + } + + const auto submit_id = kernel_aio_next_id(); + g_kernel_aio_state[submit_id].store(KERNEL_AIO_STATE_PROCESSING, std::memory_order_release); + + for (int32_t i = 0; i < size; i++) { + if (req[i].result == nullptr) { + g_kernel_aio_state[submit_id].store(KERNEL_AIO_STATE_ABORTED, + std::memory_order_release); + return LibKernel::KERNEL_ERROR_EFAULT; + } + + req[i].result->state = KERNEL_AIO_STATE_SUBMITTED; + + const auto result = + LibKernel::FileSystem::KernelPread(req[i].fd, req[i].buf, req[i].nbyte, req[i].offset); + + req[i].result->return_value = result; + req[i].result->state = (result < 0 ? KERNEL_AIO_STATE_ABORTED : KERNEL_AIO_STATE_COMPLETED); + } + + g_kernel_aio_state[submit_id].store(KERNEL_AIO_STATE_COMPLETED, std::memory_order_release); + *id = submit_id; + + (void)prio; + return OK; +} + +int KYTY_SYSV_ABI KernelAioSubmitWriteCommands(KernelAioRwRequest* req, int32_t size, int32_t prio, + int32_t* id) { + PRINT_NAME(); + + if (req == nullptr || id == nullptr) { + return LibKernel::KERNEL_ERROR_EFAULT; + } + + if (size <= 0 || size > KERNEL_AIO_MAX_REQUESTS) { + return LibKernel::KERNEL_ERROR_EINVAL; + } + + const auto submit_id = kernel_aio_next_id(); + g_kernel_aio_state[submit_id].store(KERNEL_AIO_STATE_PROCESSING, std::memory_order_release); + + for (int32_t i = 0; i < size; i++) { + if (req[i].result == nullptr) { + g_kernel_aio_state[submit_id].store(KERNEL_AIO_STATE_ABORTED, + std::memory_order_release); + return LibKernel::KERNEL_ERROR_EFAULT; + } + + req[i].result->state = KERNEL_AIO_STATE_SUBMITTED; + + const auto result = + LibKernel::FileSystem::KernelPwrite(req[i].fd, req[i].buf, req[i].nbyte, req[i].offset); + + req[i].result->return_value = result; + req[i].result->state = (result < 0 ? KERNEL_AIO_STATE_ABORTED : KERNEL_AIO_STATE_COMPLETED); + } + + g_kernel_aio_state[submit_id].store(KERNEL_AIO_STATE_COMPLETED, std::memory_order_release); + *id = submit_id; + + (void)prio; + return OK; +} + +int KYTY_SYSV_ABI KernelAioWaitRequest(int32_t id, int32_t* state, uint32_t* usec) { + PRINT_NAME(); + + if (state == nullptr) { + return LibKernel::KERNEL_ERROR_EFAULT; + } + + if (!kernel_aio_is_valid_id(id)) { + return LibKernel::KERNEL_ERROR_EINVAL; + } + + uint32_t waited = 0; + auto current = g_kernel_aio_state[id].load(std::memory_order_acquire); + while (current == KERNEL_AIO_STATE_PROCESSING) { + if (usec != nullptr && *usec != 0 && waited >= *usec) { + *state = current; + return LibKernel::KERNEL_ERROR_ETIMEDOUT; + } + + Common::Thread::SleepMicro(10); + waited += 10; + current = g_kernel_aio_state[id].load(std::memory_order_acquire); + } + + *state = current; + return OK; +} + +int KYTY_SYSV_ABI KernelAioDeleteRequest(int32_t id, int32_t* ret) { + PRINT_NAME(); + + if (ret == nullptr) { + return LibKernel::KERNEL_ERROR_EFAULT; + } + + if (!kernel_aio_is_valid_id(id)) { + return LibKernel::KERNEL_ERROR_EINVAL; + } + + g_kernel_aio_state[id].store(KERNEL_AIO_STATE_ABORTED, std::memory_order_release); + *ret = OK; + + return OK; +} + +namespace FileSystem = LibKernel::FileSystem; +namespace Memory = LibKernel::Memory; +namespace EventQueue = LibKernel::EventQueue; +namespace EventFlag = LibKernel::EventFlag; +namespace Semaphore = LibKernel::Semaphore; + +LIB_DEFINE(InitFiber_1) { + LIB_USING(Fiber); + + LIB_FUNC("hVYD7Ou2pCQ", Fiber::FiberInitialize); + LIB_FUNC("7+OJIpko9RY", Fiber::FiberInitializeInternal); + LIB_FUNC("asjUJJ+aa8s", Fiber::FiberOptParamInitialize); + LIB_FUNC("JeNX5F-NzQU", Fiber::FiberFinalize); + LIB_FUNC("a0LLrZWac0M", Fiber::FiberRun); + LIB_FUNC("PFT2S-tJ7Uk", Fiber::FiberSwitch); + LIB_FUNC("p+zLIOg27zU", Fiber::FiberGetSelf); + LIB_FUNC("B0ZX2hx9DMw", Fiber::FiberReturnToThread); + LIB_FUNC("avfGJ94g36Q", Fiber::FiberRun); + LIB_FUNC("ZqhZFuzKT6U", Fiber::FiberSwitch); + LIB_FUNC("uq2Y5BFz0PE", Fiber::FiberGetInfo); + LIB_FUNC("Lcqty+QNWFc", Fiber::FiberStartContextSizeCheck); + LIB_FUNC("Kj4nXMpnM8Y", Fiber::FiberStopContextSizeCheck); + LIB_FUNC("JzyT91ucGDc", Fiber::FiberRename); + LIB_FUNC("0dy4JtMUcMQ", Fiber::FiberGetThreadFramePointerAddress); +} + +LIB_DEFINE(InitCoredump_1) { + LIB_USING(Coredump); + + LIB_FUNC("8zLSfEfW5AU", Coredump::sceCoredumpRegisterCoredumpHandler); + LIB_FUNC("fFkhOgztiCA", Coredump::sceCoredumpUnregisterCoredumpHandler); +} + +LIB_DEFINE(InitLibKernel_1_FS) { + LIB_FUNC("1G3lF1Gg1k8", FileSystem::KernelOpen); + LIB_FUNC("UK2Tl2DWUns", FileSystem::KernelClose); + LIB_FUNC("Cg4srZ6TKbU", FileSystem::KernelRead); + LIB_FUNC("4wSze92BhLI", FileSystem::KernelWrite); + LIB_FUNC("+r3rMFwItV4", FileSystem::KernelPread); + LIB_FUNC("nKWi-N2HBV4", FileSystem::KernelPwrite); + LIB_FUNC("eV9wAD2riIA", FileSystem::KernelStat); + LIB_FUNC("kBwCPsYX-m4", FileSystem::KernelFstat); + LIB_FUNC("AUXVxWeJU-A", FileSystem::KernelUnlink); + LIB_FUNC("52NcYU9+lEo", FileSystem::KernelRename); + LIB_FUNC("taRWhTJFTgE", FileSystem::KernelGetdirentries); + LIB_FUNC("oib76F-12fk", FileSystem::KernelLseek); + LIB_FUNC("j2AIqSqJP0w", FileSystem::KernelGetdents); + LIB_FUNC("1-LFLmRFxxM", FileSystem::KernelMkdir); + LIB_FUNC("naInUjYt3so", FileSystem::KernelRmdir); + LIB_FUNC("uWyW3v98sU4", FileSystem::KernelCheckReachability); +} + +LIB_DEFINE(InitLibKernel_1_Mem) { + LIB_FUNC("mL8NDH86iQI", Memory::KernelMapNamedFlexibleMemory); + LIB_FUNC("IWIBBdTHit4", Memory::KernelMapFlexibleMemory); + LIB_FUNC("DGMG3JshrZU", Memory::KernelSetVirtualRangeName); + // 6xx + LIB_FUNC("4h6F1LLbTiw", Memory::KernelMapFlexibleMemory); + LIB_FUNC("cQke9UuBQOk", Memory::KernelMunmap); + LIB_FUNC("pO96TwzOm5E", Memory::KernelGetDirectMemorySize); + LIB_FUNC("C0f7TJcbfac", Memory::KernelAvailableDirectMemorySize); + LIB_FUNC("tZ2yplY8MBY", Memory::KernelGetPageTableStats); + LIB_FUNC("rTXw65xmLIA", Memory::KernelAllocateDirectMemory); + LIB_FUNC("B+vc2AO2Zrc", Memory::KernelAllocateMainDirectMemory); + LIB_FUNC("rVjRvHJ0X6c", Memory::KernelVirtualQuery); + LIB_FUNC("yDBwVAolDgg", Memory::KernelIsStack); + LIB_FUNC("7oxv3PPCumo", Memory::KernelReserveVirtualRange); + LIB_FUNC("L-Q3LEjIbgA", Memory::KernelMapDirectMemory); + LIB_FUNC("BQQniolj9tQ", Memory::KernelMapDirectMemory2); + LIB_FUNC("NcaWUxfMNIQ", Memory::KernelMapNamedDirectMemory); + LIB_FUNC("BohYr-F7-is", Memory::KernelSetPrtAperture); + LIB_FUNC("L0v2Go5jOuM", Memory::KernelGetPrtAperture); + LIB_FUNC("hwVSPCmp5tM", Memory::KernelCheckedReleaseDirectMemory); + LIB_FUNC("MBuItvba6z8", Memory::KernelReleaseDirectMemory); + LIB_FUNC("jh+8XiK4LeE", Memory::KernelIsAddressSanitizerEnabled); + LIB_FUNC("WFcfL2lzido", Memory::KernelQueryMemoryProtection); + LIB_FUNC("BHouLQzh0X0", Memory::KernelDirectMemoryQuery); + LIB_FUNC("aNz11fnnzi4", Memory::KernelAvailableFlexibleMemorySize); + LIB_FUNC("n1-v6FgU7MQ", Memory::KernelConfiguredFlexibleMemorySize); + LIB_FUNC("vSMAm3cxYTY", Memory::KernelMprotect); + LIB_FUNC("9bfdLIyuwCY", Memory::KernelMtypeprotect); + LIB_FUNC("2SKEx6bSq-4", Memory::KernelBatchMap); + LIB_FUNC("kBJzF8x4SyE", Memory::KernelBatchMap2); + LIB_FUNC("pU-QydtGcGY", Memory::KernelMemoryPoolReserve); + LIB_FUNC("qCSfqDILlns", Memory::KernelMemoryPoolExpand); + LIB_FUNC("Vzl66WmfLvk", Memory::KernelMemoryPoolCommit); + LIB_FUNC("LXo1tpFqJGs", Memory::KernelMemoryPoolDecommit); + LIB_FUNC("YN878uKRBbE", Memory::KernelMemoryPoolBatch); + LIB_FUNC("bvD+95Q6asU", Memory::KernelMemoryPoolGetBlockStats); +} + +LIB_DEFINE(InitLibKernel_1_Equeue) { + LIB_FUNC("D0OdFMjp46I", EventQueue::KernelCreateEqueue); + LIB_FUNC("jpFjmgAC5AE", EventQueue::KernelDeleteEqueue); + LIB_FUNC("fzyMKs9kim0", EventQueue::KernelWaitEqueue); + LIB_FUNC("vz+pg2zdopI", EventQueue::KernelGetEventUserData); + LIB_FUNC("mJ7aghmgvfc", EventQueue::KernelGetEventId); + LIB_FUNC("23CPPI1tyBY", EventQueue::KernelGetEventFilter); + LIB_FUNC("kwGyyjohI50", EventQueue::KernelGetEventData); + LIB_FUNC("Q0qr9AyqJSk", EventQueue::KernelGetEventFflags); + LIB_FUNC("Uu-iDFC9aUc", EventQueue::KernelGetEventError); + LIB_FUNC("4R6-OvI2cEA", EventQueue::KernelAddUserEvent); + LIB_FUNC("WDszmSbWuDk", EventQueue::KernelAddUserEventEdge); + LIB_FUNC("F6e0kwo4cnk", EventQueue::KernelTriggerUserEvent); + LIB_FUNC("LJDwdSNTnDg", EventQueue::KernelDeleteUserEvent); + LIB_FUNC("R74tt43xP6k", EventQueue::KernelAddHRTimerEvent); + LIB_FUNC("J+LF6LwObXU", EventQueue::KernelDeleteHRTimerEvent); + LIB_FUNC("bBfz7kMF2Ho", EventQueue::KernelAddAmprEvent); + LIB_FUNC("vuae5JPNt9A", EventQueue::KernelAddAmprSystemEvent); + LIB_FUNC("bMmid3pfyjo", EventQueue::KernelDeleteAmprEvent); + LIB_FUNC("Ij+ryuEClXQ", EventQueue::KernelDeleteAmprSystemEvent); +} + +LIB_DEFINE(InitLibKernel_1_EventFlag) { + LIB_FUNC("PZku4ZrXJqg", EventFlag::KernelCancelEventFlag); + LIB_FUNC("7uhBFWRAS60", EventFlag::KernelClearEventFlag); + LIB_FUNC("BpFoboUJoZU", EventFlag::KernelCreateEventFlag); + LIB_FUNC("8mql9OcQnd4", EventFlag::KernelDeleteEventFlag); + LIB_FUNC("9lvj5DjHZiA", EventFlag::KernelPollEventFlag); + LIB_FUNC("JTvBflhYazQ", EventFlag::KernelWaitEventFlag); + LIB_FUNC("IOnSvHzqu6A", EventFlag::KernelSetEventFlag); +} + +LIB_DEFINE(InitLibKernel_1_Semaphore) { + LIB_FUNC("188x57JYp0g", Semaphore::KernelCreateSema); + LIB_FUNC("R1Jvn8bSCW8", Semaphore::KernelDeleteSema); + LIB_FUNC("Zxa0VhQVTsk", Semaphore::KernelWaitSema); + LIB_FUNC("12wOHk8ywb0", Semaphore::KernelPollSema); + LIB_FUNC("4czppHBiriw", Semaphore::KernelSignalSema); + LIB_FUNC("4DM06U2BNEY", Semaphore::KernelCancelSema); + LIB_FUNC("GEnUkDZoUwY", Semaphore::PthreadSemInit); + LIB_FUNC("Vwc+L05e6oE", Semaphore::PthreadSemDestroy); + LIB_FUNC("C36iRE0F5sE", Semaphore::PthreadSemWait); + LIB_FUNC("H2a+IN9TP0E", Semaphore::PthreadSemTrywait); + LIB_FUNC("fjN6NQHhK8k", Semaphore::PthreadSemTimedwait); + LIB_FUNC("aishVAiFaYM", Semaphore::PthreadSemPost); + LIB_FUNC("DjpBvGlaWbQ", Semaphore::PthreadSemGetvalue); +} + +LIB_DEFINE(InitLibKernel_1_Pthread) { + LIB_FUNC("9UK1vLZQft4", LibKernel::PthreadMutexLock); + LIB_FUNC("tn3VlD0hG60", LibKernel::PthreadMutexUnlock); + LIB_FUNC("2Of0f+3mhhE", LibKernel::PthreadMutexDestroy); + LIB_FUNC("cmo1RIYva9o", LibKernel::PthreadMutexInit); + LIB_FUNC("upoVrzMHFeE", LibKernel::PthreadMutexTrylock); + LIB_FUNC("smWEktiyyG0", LibKernel::PthreadMutexattrDestroy); + LIB_FUNC("F8bUHwAG284", LibKernel::PthreadMutexattrInit); + LIB_FUNC("iMp8QpE+XO4", LibKernel::PthreadMutexattrSettype); + LIB_FUNC("1FGvU0i9saQ", LibKernel::PthreadMutexattrSetprotocol); + + LIB_FUNC("aI+OeCz8xrQ", LibKernel::PthreadSelf); + LIB_FUNC("EotR8a3ASf4", LibKernel::PthreadSelf); + LIB_FUNC("6UgtwV+0zb4", LibKernel::PthreadCreate); + LIB_FUNC("3PtV6p3QNX4", LibKernel::PthreadEqual); + LIB_FUNC("onNY9Byn-W8", LibKernel::PthreadJoin); + LIB_FUNC("4qGrR6eoP9Y", LibKernel::PthreadDetach); + LIB_FUNC("3kg7rT0NQIs", LibKernel::PthreadExit); + LIB_FUNC("OAmWq+OHSjw", LibKernel::PthreadSetcancelstate); + LIB_FUNC("sCJd99Phct0", LibKernel::PthreadSetcanceltype); + LIB_FUNC("How7B8Oet6k", LibKernel::PthreadGetname); + LIB_FUNC("GBUY7ywdULE", LibKernel::PthreadRename); + LIB_FUNC("bt3CTBKmGyI", LibKernel::PthreadSetaffinity); + LIB_FUNC("rcrVFJsQWRY", LibKernel::PthreadGetaffinity); + LIB_FUNC("1tKyG7RlMJo", LibKernel::PthreadGetprio); + LIB_FUNC("W0Hpm2X0uPE", LibKernel::PthreadSetprio); + LIB_FUNC("T72hz6ffq08", LibKernel::PthreadYield); + LIB_FUNC("6XG4B33N09g", LibKernel::PthreadYield); + LIB_FUNC("CBNtXOoef-E", Posix::sched_get_priority_max); + LIB_FUNC("m0iS6jNsXds", Posix::sched_get_priority_min); + LIB_FUNC("EI-5-jlq2dE", LibKernel::PthreadGetthreadid); + LIB_FUNC("geDaqgH9lTg", LibKernel::PthreadKeyCreate); + LIB_FUNC("PrdHuuDekhY", LibKernel::PthreadKeyDelete); + LIB_FUNC("+BzXYkqYeLE", LibKernel::PthreadSetspecific); + LIB_FUNC("eoht7mQOCmo", LibKernel::PthreadGetspecific); + + LIB_FUNC("62KCwEMmzcM", LibKernel::PthreadAttrDestroy); + LIB_FUNC("x1X76arYMxU", LibKernel::PthreadAttrGet); + LIB_FUNC("8+s5BzZjxSg", LibKernel::PthreadAttrGetaffinity); + LIB_FUNC("nsYoNRywwNg", LibKernel::PthreadAttrInit); + LIB_FUNC("JaRMy+QcpeU", LibKernel::PthreadAttrGetdetachstate); + LIB_FUNC("-quPa4SEJUw", LibKernel::PthreadAttrGetstack); + LIB_FUNC("Ru36fiTtJzA", LibKernel::PthreadAttrGetstackaddr); + LIB_FUNC("-fA+7ZlGDQs", LibKernel::PthreadAttrGetstacksize); + LIB_FUNC("txHtngJ+eyc", LibKernel::PthreadAttrGetguardsize); + LIB_FUNC("9RnL-m0+diQ", LibKernel::PthreadAttrGetsolosched); + LIB_FUNC("Bvn74vj6oLo", LibKernel::PthreadAttrSetstack); + LIB_FUNC("F+yfmduIBB8", LibKernel::PthreadAttrSetstackaddr); + LIB_FUNC("UTXzJbWhhTE", LibKernel::PthreadAttrSetstacksize); + LIB_FUNC("2Q0z6rnBrTE", LibKernel::PthreadAttrSetstacksize); + LIB_FUNC("-Wreprtu0Qs", LibKernel::PthreadAttrSetdetachstate); + LIB_FUNC("El+cQ20DynU", LibKernel::PthreadAttrSetguardsize); + LIB_FUNC("eXbUSpEaTsA", LibKernel::PthreadAttrSetinheritsched); + LIB_FUNC("DzES9hQF4f4", LibKernel::PthreadAttrSetschedparam); + LIB_FUNC("4+h9EzwKF4I", LibKernel::PthreadAttrSetschedpolicy); + LIB_FUNC("Dk6FC-TI+7Q", LibKernel::PthreadAttrSetsolosched); + LIB_FUNC("3qxgM4ezETA", LibKernel::PthreadAttrSetaffinity); + LIB_FUNC("FXPWHNk8Of0", LibKernel::PthreadAttrGetschedparam); + + LIB_FUNC("6ULAa0fq4jA", LibKernel::PthreadRwlockInit); + LIB_FUNC("ytQULN-nhL4", LibKernel::PthreadRwlockInit); + LIB_FUNC("BB+kb08Tl9A", LibKernel::PthreadRwlockDestroy); + LIB_FUNC("Ox9i0c7L5w0", LibKernel::PthreadRwlockRdlock); + LIB_FUNC("iGjsr1WAtI0", LibKernel::PthreadRwlockRdlock); + LIB_FUNC("+L98PIbGttk", LibKernel::PthreadRwlockUnlock); + LIB_FUNC("EgmLo6EWgso", LibKernel::PthreadRwlockUnlock); + LIB_FUNC("mqdNorrB+gI", LibKernel::PthreadRwlockWrlock); + LIB_FUNC("sIlRvQqsN2Y", LibKernel::PthreadRwlockWrlock); + LIB_FUNC("bIHoZCTomsI", LibKernel::PthreadRwlockTrywrlock); + LIB_FUNC("i2ifZ3fS2fo", LibKernel::PthreadRwlockattrDestroy); + LIB_FUNC("yOfGg-I1ZII", LibKernel::PthreadRwlockattrInit); + LIB_FUNC("qsdmgXjqSgk", LibKernel::PthreadRwlockattrDestroy); + LIB_FUNC("xFebsA4YsFI", LibKernel::PthreadRwlockattrInit); + LIB_FUNC("h-OifiouBd8", LibKernel::PthreadRwlockattrSettype); + + LIB_FUNC("2Tb92quprl0", LibKernel::PthreadCondInit); + LIB_FUNC("0TyVk4MSLt0", LibKernel::PthreadCondInit); + LIB_FUNC("g+PZd2hiacg", LibKernel::PthreadCondDestroy); + LIB_FUNC("RXXqi4CtF8w", LibKernel::PthreadCondDestroy); + LIB_FUNC("WKAXJ4XBPQ4", LibKernel::PthreadCondWait); + LIB_FUNC("JGgj7Uvrl+A", LibKernel::PthreadCondBroadcast); + LIB_FUNC("kDh-NfxgMtE", LibKernel::PthreadCondSignal); + LIB_FUNC("2MOy+rUfuhQ", LibKernel::PthreadCondSignal); + LIB_FUNC("o69RpYO-Mu0", LibKernel::PthreadCondSignalto); + LIB_FUNC("BmMjYxmew1w", LibKernel::PthreadCondTimedwait); + LIB_FUNC("27bAgiJmOh0", Posix::pthread_cond_timedwait); + LIB_FUNC("m5-2bsNfv7s", LibKernel::PthreadCondattrInit); + LIB_FUNC("mKoTx03HRWA", Posix::pthread_condattr_init); + LIB_FUNC("waPcxYiR3WA", LibKernel::PthreadCondattrDestroy); + LIB_FUNC("dJcuQVn6-Iw", Posix::pthread_condattr_destroy); + LIB_FUNC("EjllaAqAPZo", Posix::pthread_condattr_setclock); + + LIB_FUNC("wRYVA5Zolso", LibKernel::KernelClockGetres); + LIB_FUNC("QBi7HCK03hw", LibKernel::KernelClockGettime); + LIB_FUNC("ejekcaNQNq0", LibKernel::KernelGettimeofday); + LIB_FUNC("kOcnerypnQA", LibKernel::KernelGettimezone); + LIB_FUNC("0NTHN1NKONI", LibKernel::KernelConvertLocaltimeToUtc); + LIB_FUNC("-o5uEDpN+oY", LibKernel::KernelConvertUtcToLocaltime); + LIB_FUNC("n88vx3C5nW8", Posix::gettimeofday); + LIB_FUNC("-2IRUCO--PM", LibKernel::KernelReadTsc); + LIB_FUNC("1j3S3n-tTW4", LibKernel::KernelGetTscFrequency); + LIB_FUNC("4J2sUJmuHZQ", LibKernel::KernelGetProcessTime); + LIB_FUNC("BNowx2l588E", LibKernel::KernelGetProcessTimeCounterFrequency); + LIB_FUNC("fgxnMeTNUtY", LibKernel::KernelGetProcessTimeCounter); + + LIB_FUNC("7H0iTOciTLo", Posix::pthread_mutex_lock); + LIB_FUNC("2Z+PpY6CaJg", Posix::pthread_mutex_unlock); + LIB_FUNC("IafI2PxcPnQ", LibKernel::PthreadMutexTimedlock); + LIB_FUNC("Io9+nTKXZtA", Posix::pthread_mutex_timedlock); + LIB_FUNC("mkx2fVhNMsg", Posix::pthread_cond_broadcast); + LIB_FUNC("Op8TBGY5KHg", Posix::pthread_cond_wait); + LIB_FUNC("14bOACANTBo", Posix::pthread_once); + LIB_FUNC("Z4QosVuAsA0", Posix::pthread_once); + LIB_FUNC("Oy6IpwgtYOk", Posix::lseek); + LIB_FUNC("ezv-RSBNKqI", Posix::pread); + LIB_FUNC("C2kJ-byS5rM", Posix::pwrite); + LIB_FUNC("9eMlfusH4sU", Posix::flock); + LIB_FUNC("mqQMh1zPPT8", Posix::fstat); + + LIB_FUNC("z0dtnPxYgtg", chmod); + LIB_FUNC("VAzswvTOCzI", FileSystem::KernelUnlink); + LIB_FUNC("JGMio+21L4c", Posix::mkdir); + LIB_FUNC("wuCroIGjt2g", FileSystem::KernelOpen); + LIB_FUNC("bY-PO6JhzhQ", FileSystem::KernelClose); + LIB_FUNC("FN4gaPmuFV8", FileSystem::KernelWrite); +} + +static void AddLibkernelUnityFunc(Loader::SymbolDatabase* s, const char* nid, uint64_t func, + const std::string& dbg_name) { + Loader::SymbolResolve sr {}; + sr.name = nid; + sr.library = "libkernel_unity"; + sr.library_version = 1; + sr.module = "libkernel"; + sr.module_version_major = 1; + sr.module_version_minor = 1; + sr.type = Loader::SymbolType::Func; + s->Add(sr, func, dbg_name); +} + +LIB_DEFINE(InitLibKernel_1) { + InitLibKernel_1_FS(s); + InitLibKernel_1_Mem(s); + InitLibKernel_1_Equeue(s); + InitLibKernel_1_EventFlag(s); + InitLibKernel_1_Semaphore(s); + InitLibKernel_1_Pthread(s); + LibKernelApr::InitLibKernel_1_Apr(s); + Posix::InitLibKernel_1_Posix(s); + + LIB_OBJECT("f7uOxY9mM1U", &LibKernel::g_stack_chk_guard); + LIB_OBJECT("djxxOmW6-aw", &LibKernel::g_progname); + + LIB_FUNC("1jfXLRVzisc", LibKernel::KernelUsleep); + LIB_FUNC("QcteRwbsnV0", LibKernel::KernelUsleep); + LIB_FUNC("QvsZxomvUHs", LibKernel::KernelNanosleep); + LIB_FUNC("LwG8g3niqwA", LibKernel::KernelDlsym); + LIB_FUNC("-ZR+hG7aDHw", LibKernel::KernelSleep); + LIB_FUNC("6c3rCVE-fTU", LibKernel::open); + LIB_FUNC("6xVpy0Fdq+I", LibKernel::sigprocmask); + LIB_FUNC("6Z83sYWFlA8", LibKernel::exit); + LIB_FUNC("8OnWXlgQlvo", LibKernel::KernelRtldThreadAtexitDecrement); + LIB_FUNC("959qrazPIrg", LibKernel::KernelGetProcParam); + LIB_FUNC("tU5e3f9gSiU", LibKernel::KernelIsTrinityMode); + LIB_FUNC("fTx66l5iWIA", LibKernel::KernelFsync); + LIB_FUNC("uvT2iYBBnkY", LibKernel::KernelSync); + LIB_FUNC("HoLVWNanBBc", LibKernel::getpid); + LIB_FUNC("9BcDykPmo1I", LibKernel::get_error_addr); + LIB_FUNC("k+AXqu2-eBc", Posix::getpagesize); + LIB_FUNC("bnZxYgAFeA0", LibKernel::KernelGetSanitizerNewReplaceExternal); + LIB_FUNC("ca7v6Cxulzs", LibKernel::KernelSetGPO); + LIB_FUNC("4oXYe9Xmk0Q", LibKernel::KernelGetGPI); + LIB_FUNC("DRuBt2pvICk", LibKernel::read); + LIB_FUNC("AqBioC2vF3I", LibKernel::read); + LIB_FUNC("f7KBOafysXo", LibKernel::KernelGetModuleInfoFromAddr); + LIB_FUNC("RpQJJVKTiFM", LibKernel::KernelGetModuleInfoForUnwind); + LIB_FUNC("Fjc4-n1+y2g", LibKernel::elf_phdr_match_addr); + LIB_FUNC("FxVZqBAA7ks", LibKernel::write); + LIB_FUNC("FJmglmTMdr4", LibKernel::getargv); + LIB_FUNC("kbw4UHHSYy0", LibKernel::pthread_cxa_finalize); + LIB_FUNC("lLMT9vJAck0", LibKernel::clock_gettime); + LIB_FUNC("5TgME6AYty4", KernelAioDeleteRequest); + LIB_FUNC("HgX7+AORI58", KernelAioSubmitReadCommands); + LIB_FUNC("KOF-oJbQVvc", KernelAioWaitRequest); + LIB_FUNC("XQ8C8y+de+E", KernelAioSubmitWriteCommands); + LIB_FUNC("nu4a0-arQis", KernelAioInitializeParam); + LIB_FUNC("il03nluKfMk", LibKernel::KernelRaiseException); + LIB_FUNC("iKJMWrAumPE", LibKernel::getargc); + LIB_FUNC("NNtFaKJbPt0", LibKernel::close); + LIB_FUNC("OMDRKKAZ8I4", LibKernel::KernelDebugRaiseException); + LIB_FUNC("Ou3iL1abvng", LibKernel::stack_chk_fail); + LIB_FUNC("p5EcQeEeJAE", LibKernel::KernelRtldSetApplicationHeapAPI); + LIB_FUNC("pB-yGZ2nQ9o", LibKernel::KernelSetThreadAtexitCount); + LIB_FUNC("py6L8jiVAN8", LibKernel::KernelGetSanitizerMallocReplaceExternal); + LIB_FUNC("Qhv5ARAoOEc", LibKernel::KernelRemoveExceptionHandler); + LIB_FUNC("QKd0qM58Qes", LibKernel::KernelStopUnloadModule); + LIB_FUNC("rNhWz+lvOMU", LibKernel::KernelSetThreadDtors); + LIB_FUNC("Tz4RNUCBbGI", LibKernel::KernelRtldThreadAtexitIncrement); + LIB_FUNC("vNe1w4diLCs", LibKernel::tls_get_addr); + LIB_FUNC("vYU8P9Td2Zo", KernelAioInitializeImpl); + LIB_FUNC("WhCc1w3EhSI", LibKernel::KernelSetThreadAtexitReport); + LIB_FUNC("WkwEd3N7w0Y", LibKernel::KernelInstallExceptionHandler); + LIB_FUNC("g0VTBxfJyu0", LibKernel::KernelGetCurrentCpu); + LIB_FUNC("wzvqT4UqKX8", LibKernel::KernelLoadStartModule); + LIB_FUNC("Xjoosiw+XPI", LibKernel::KernelUuidCreate); + LIB_FUNC("DLORcroUqbc", LibKernel::KernelGetOpenPsId); + LIB_FUNC("zE-wXIZjLoM", LibKernel::KernelDebugRaiseExceptionOnReleaseMode); + LIB_FUNC("Hc4CaR6JBL0", Posix::KernelSyncOnAddressV1); + LIB_FUNC("q2y-wDIVWZA", Posix::KernelSyncOnAddressV1); + + AddLibkernelUnityFunc(s, "Qhv5ARAoOEc", + reinterpret_cast(LibKernel::KernelRemoveExceptionHandler), + "LibKernel::KernelRemoveExceptionHandler"); + AddLibkernelUnityFunc(s, "il03nluKfMk", + reinterpret_cast(LibKernel::KernelRaiseException), + "LibKernel::KernelRaiseException"); + AddLibkernelUnityFunc(s, "WkwEd3N7w0Y", + reinterpret_cast(LibKernel::KernelInstallExceptionHandler), + "LibKernel::KernelInstallExceptionHandler"); +} + +} // namespace Libs diff --git a/src/libs/libNet.cpp b/src/libs/libNet.cpp new file mode 100644 index 000000000..4861e123b --- /dev/null +++ b/src/libs/libNet.cpp @@ -0,0 +1,3982 @@ +#include "common/abi.h" +#include "common/assert.h" +#include "common/common.h" +#include "common/logging/log.h" +#include "libs/errno.h" +#include "libs/libs.h" +#include "libs/network.h" +#include "loader/symbolDatabase.h" + +#include +#include +#include +#include +#include +#include + +// NOLINTNEXTLINE(cppcoreguidelines-macro-usage) +#define NET_CALL(func) \ + [&]() { \ + auto result = func; \ + if (result < 0) { \ + *GetNetErrorAddr() = result; \ + } \ + return result; \ + }() + +namespace Libs { + +namespace Network::Net { +struct NetEtherAddr; +} // namespace Network::Net + +namespace LibNet { + +LIB_VERSION("Net", 1, "Net", 1, 1); + +static thread_local int g_net_errno = 0; + +namespace Net = Network::Net; + +KYTY_SYSV_ABI int* GetNetErrorAddr() { + return &g_net_errno; +} + +static int KYTY_SYSV_ABI NetInit() { + return NET_CALL(Net::NetInit()); +} + +static int PosixToNetError(int error) { + if (error >= Posix::POSIX_EPERM && error <= Posix::POSIX_ECANCELED) { + return Network::NET_ERROR_EPERM + error - Posix::POSIX_EPERM; + } + + switch (error) { + case Posix::POSIX_EPROTO: return Network::NET_ERROR_EPROTO; + case Posix::POSIX_EADHOC: return Network::NET_ERROR_EADHOC; + case Posix::POSIX_EINACTIVEDISABLED: return Network::NET_ERROR_EINACTIVEDISABLED; + case Posix::POSIX_ENETNODATA: return Network::NET_ERROR_ENODATA; + case Posix::POSIX_ENETDESC: return Network::NET_ERROR_EDESC; + case Posix::POSIX_ENETDESCTIMEDOUT: return Network::NET_ERROR_EDESCTIMEDOUT; + case Posix::POSIX_ENETINTR: return Network::NET_ERROR_ENETINTR; + case Posix::POSIX_ERETURN: return Network::NET_ERROR_ERETURN; + default: return Network::NET_ERROR_EIO; + } +} + +static int FinishSocketCall(int result) { + if (result >= 0) { + return result; + } + + const int net_error = PosixToNetError(*Posix::GetErrorAddr()); + *GetNetErrorAddr() = net_error; + return net_error; +} + +int KYTY_SYSV_ABI NetSocket(const char* name, int family, int type, int protocol) { + LOGF("\t name = %s\n", name != nullptr ? name : ""); + + return FinishSocketCall(Net::Socket(family, type, protocol)); +} + +int KYTY_SYSV_ABI NetAccept(int s, void* addr, uint32_t* addrlen) { + return FinishSocketCall(Net::Accept(s, addr, addrlen)); +} + +int KYTY_SYSV_ABI NetBind(int s, const void* addr, uint32_t addrlen) { + return FinishSocketCall(Net::Bind(s, addr, addrlen)); +} + +int KYTY_SYSV_ABI NetListen(int s, int backlog) { + return FinishSocketCall(Net::Listen(s, backlog)); +} + +int KYTY_SYSV_ABI NetShutdown(int s, int how) { + return FinishSocketCall(Net::Shutdown(s, how)); +} + +int KYTY_SYSV_ABI NetPoolCreate(const char* name, int size, int flags) { + return NET_CALL(Net::NetPoolCreate(name, size, flags)); +} + +int KYTY_SYSV_ABI NetPoolDestroy(int memid) { + return NET_CALL(Net::NetPoolDestroy(memid)); +} + +int KYTY_SYSV_ABI NetResolverCreate(const char* name, int memid, int flags) { + return NET_CALL(Net::NetResolverCreate(name, memid, flags)); +} + +int KYTY_SYSV_ABI NetResolverStartNtoa(int rid, const char* hostname, void* addr, int timeout, + int retry, int flags) { + return NET_CALL(Net::NetResolverStartNtoa(rid, hostname, addr, timeout, retry, flags)); +} + +int KYTY_SYSV_ABI NetInetPton(int af, const char* src, void* dst) { + return NET_CALL(Net::NetInetPton(af, src, dst)); +} + +const char* KYTY_SYSV_ABI NetInetNtop(int af, const void* src, char* dst, uint32_t size) { + const char* result = Net::NetInetNtop(af, src, dst, size); + if (result == nullptr) { + *GetNetErrorAddr() = PosixToNetError(*Posix::GetErrorAddr()); + } + return result; +} + +int KYTY_SYSV_ABI NetEtherNtostr(const Net::NetEtherAddr* n, char* str, size_t len) { + return NET_CALL(Net::NetEtherNtostr(n, str, len)); +} + +int KYTY_SYSV_ABI NetGetMacAddress(Net::NetEtherAddr* addr, int flags) { + return NET_CALL(Net::NetGetMacAddress(addr, flags)); +} + +int KYTY_SYSV_ABI NetGetSockInfo(int s, void* info, int n, int flags) { + return NET_CALL(Net::NetGetSockInfo(s, info, n, flags)); +} + +int KYTY_SYSV_ABI NetEpollCreate(const char* name, int flags) { + return FinishSocketCall(Net::EpollCreate(name, flags)); +} + +int KYTY_SYSV_ABI NetEpollControl(int eid, int op, int id, const Net::NetEpollEvent* event) { + return FinishSocketCall(Net::EpollControl(eid, op, id, event)); +} + +int KYTY_SYSV_ABI NetEpollWait(int eid, Net::NetEpollEvent* events, int maxevents, int timeout) { + return FinishSocketCall(Net::EpollWait(eid, events, maxevents, timeout)); +} + +int KYTY_SYSV_ABI NetEpollDestroy(int eid) { + return FinishSocketCall(Net::EpollDestroy(eid)); +} + +int KYTY_SYSV_ABI NetSocketClose(int s) { + return NET_CALL(Net::SocketClose(s)); +} + +int KYTY_SYSV_ABI NetSetsockopt(int s, int level, int optname, const void* optval, + uint32_t optlen) { + return NET_CALL(Net::Setsockopt(s, level, optname, optval, optlen)); +} + +uint32_t KYTY_SYSV_ABI NetHtonl(uint32_t host32) { + return ((host32 & 0x000000ffu) << 24u) | ((host32 & 0x0000ff00u) << 8u) | + ((host32 & 0x00ff0000u) >> 8u) | ((host32 & 0xff000000u) >> 24u); +} + +uint16_t KYTY_SYSV_ABI NetHtons(uint16_t host16) { + return static_cast(((host16 & 0x00ffu) << 8u) | ((host16 & 0xff00u) >> 8u)); +} + +uint32_t KYTY_SYSV_ABI NetNtohl(uint32_t net32) { + return NetHtonl(net32); +} + +uint16_t KYTY_SYSV_ABI NetNtohs(uint16_t net16) { + return NetHtons(net16); +} + +LIB_DEFINE(InitNet_1_Net) { + LIB_FUNC("Nlev7Lg8k3A", LibNet::NetInit); + LIB_FUNC("HQOwnfMGipQ", LibNet::GetNetErrorAddr); + LIB_FUNC("PIWqhn9oSxc", LibNet::NetAccept); + LIB_FUNC("bErx49PgxyY", LibNet::NetBind); + LIB_FUNC("dgJBaeJnGpo", LibNet::NetPoolCreate); + LIB_FUNC("K7RlrTkI-mw", LibNet::NetPoolDestroy); + LIB_FUNC("C4UgDHHPvdw", LibNet::NetResolverCreate); + LIB_FUNC("Nd91WaWmG2w", LibNet::NetResolverStartNtoa); + LIB_FUNC("8Kcp5d-q1Uo", LibNet::NetInetPton); + LIB_FUNC("9vA2aW+CHuA", LibNet::NetInetNtop); + LIB_FUNC("v6M4txecCuo", LibNet::NetEtherNtostr); + LIB_FUNC("6Oc0bLsIYe0", LibNet::NetGetMacAddress); + LIB_FUNC("hLuXdjHnhiI", LibNet::NetGetSockInfo); + LIB_FUNC("SF47kB2MNTo", LibNet::NetEpollCreate); + LIB_FUNC("ZVw46bsasAk", LibNet::NetEpollControl); + LIB_FUNC("drjIbDbA7UQ", LibNet::NetEpollWait); + LIB_FUNC("Inp1lfL+Jdw", LibNet::NetEpollDestroy); + LIB_FUNC("kOj1HiAGE54", LibNet::NetListen); + LIB_FUNC("TSM6whtekok", LibNet::NetShutdown); + LIB_FUNC("Q4qBuN-c0ZM", LibNet::NetSocket); + LIB_FUNC("45ggEzakPJQ", LibNet::NetSocketClose); + LIB_FUNC("2mKX2Spso7I", LibNet::NetSetsockopt); + LIB_FUNC("9T2pDF2Ryqg", LibNet::NetHtonl); + LIB_FUNC("iWQWrwiSt8A", LibNet::NetHtons); + LIB_FUNC("pQGpHYopAIY", LibNet::NetNtohl); + LIB_FUNC("Rbvt+5Y2iEw", LibNet::NetNtohs); +} + +} // namespace LibNet + +namespace LibSsl { + +LIB_VERSION("Ssl", 1, "Ssl", 1, 1); + +namespace Ssl = Network::Ssl; + +LIB_DEFINE(InitNet_1_Ssl) { + LIB_FUNC("hdpVEUDFW3s", Ssl::SslInit); + LIB_FUNC("0K1yQ6Lv-Yc", Ssl::SslTerm); + LIB_FUNC("TDfQqO-gMbY", Ssl::SslGetCaCerts); + LIB_FUNC("qIvLs0gYxi0", Ssl::SslFreeCaCerts); +} + +} // namespace LibSsl + +namespace LibHttp { + +LIB_VERSION("Http", 1, "Http", 1, 1); + +namespace Http = Network::Http; + +constexpr int HTTP_ERROR_INVALID_VALUE = -2143088130; /* 0x804311fe */ +constexpr int HTTP_ERROR_OUT_OF_MEMORY = -2143088606; /* 0x80431022 */ +constexpr int HTTP_ERROR_INVALID_URL = -2143080352; /* 0x80433060 */ + +struct SceHttpUriElement { + int opaque = 0; + char* scheme = nullptr; + char* username = nullptr; + char* password = nullptr; + char* hostname = nullptr; + char* path = nullptr; + char* query = nullptr; + char* fragment = nullptr; + uint16_t port = 0; + uint8_t reserved[10] {}; +}; + +struct UriPart { + const char* begin = nullptr; + size_t len = 0; +}; + +static bool IsUriSchemeChar(char c, bool first) { + const auto ch = static_cast(c); + return first ? std::isalpha(ch) != 0 + : (std::isalnum(ch) != 0 || c == '+' || c == '-' || c == '.'); +} + +static char* CopyUriPart(char*& dst, const UriPart& part) { + if (part.begin == nullptr) { + return nullptr; + } + + auto* out = dst; + std::memcpy(out, part.begin, part.len); + out[part.len] = '\0'; + dst += part.len + 1; + return out; +} + +static int ParseEmptyUri(SceHttpUriElement* out, void* pool, size_t* require, size_t prepare) { + constexpr size_t needed = 3; + + if (require != nullptr) { + *require = needed; + } + + if (out != nullptr) { + std::memset(out, 0, sizeof(*out)); + out->opaque = 1; + } + + if (out != nullptr && pool != nullptr) { + if (prepare != 0 && prepare < needed) { + return HTTP_ERROR_OUT_OF_MEMORY; + } + + auto* dst = static_cast(pool); + out->scheme = dst++; + out->hostname = dst++; + out->path = dst; + out->scheme[0] = '\0'; + out->hostname[0] = '\0'; + out->path[0] = '\0'; + } + + return 0; +} + +static int KYTY_SYSV_ABI HttpUriParse(SceHttpUriElement* out, const char* src_url, void* pool, + size_t* require, size_t prepare) { + PRINT_NAME(); + + LOGF("\t out = 0x%016" PRIx64 "\n", reinterpret_cast(out)); + LOGF("\t src_url = %s\n", src_url != nullptr ? src_url : "(null)"); + LOGF("\t pool = 0x%016" PRIx64 "\n", reinterpret_cast(pool)); + LOGF("\t require = 0x%016" PRIx64 "\n", reinterpret_cast(require)); + LOGF("\t prepare = %" PRIu64 "\n", static_cast(prepare)); + + if (src_url == nullptr) { + return HTTP_ERROR_INVALID_URL; + } + + if (out == nullptr && pool == nullptr && require == nullptr) { + return HTTP_ERROR_INVALID_VALUE; + } + + if (src_url[0] == '\0') { + return ParseEmptyUri(out, pool, require, prepare); + } + + UriPart scheme {}; + UriPart username {}; + UriPart password {}; + UriPart hostname {}; + UriPart path {}; + UriPart query {}; + UriPart fragment {}; + auto opaque = false; + uint16_t port = 0; + + const char* cursor = src_url; + if (!IsUriSchemeChar(*cursor, true)) { + return HTTP_ERROR_INVALID_URL; + } + + while (*cursor != '\0' && *cursor != ':') { + if (!IsUriSchemeChar(*cursor, false)) { + return HTTP_ERROR_INVALID_URL; + } + cursor++; + } + + if (*cursor != ':') { + return HTTP_ERROR_INVALID_URL; + } + + scheme = {src_url, static_cast(cursor - src_url)}; + cursor++; + + if (cursor[0] == '/' && cursor[1] == '/') { + cursor += 2; + const char* authority = cursor; + const char* authority_end = authority; + while (*authority_end != '\0' && *authority_end != '/' && *authority_end != '?' && + *authority_end != '#') { + authority_end++; + } + + const char* host_begin = authority; + for (const char* p = authority; p < authority_end; p++) { + if (*p == '@') { + const char* colon = static_cast( + std::memchr(authority, ':', static_cast(p - authority))); + if (colon != nullptr) { + username = {authority, static_cast(colon - authority)}; + password = {colon + 1, static_cast(p - colon - 1)}; + } else { + username = {authority, static_cast(p - authority)}; + } + host_begin = p + 1; + } + } + + const char* host_end = authority_end; + const char* port_begin = nullptr; + if (host_begin < authority_end && *host_begin == '[') { + const char* close = static_cast( + std::memchr(host_begin, ']', static_cast(authority_end - host_begin))); + if (close == nullptr) { + return HTTP_ERROR_INVALID_URL; + } + host_end = close + 1; + if (host_end < authority_end && *host_end == ':') { + port_begin = host_end + 1; + } + } else { + for (const char* p = host_begin; p < authority_end; p++) { + if (*p == ':') { + host_end = p; + port_begin = p + 1; + break; + } + } + } + + if (host_begin < host_end) { + hostname = {host_begin, static_cast(host_end - host_begin)}; + } + + if (port_begin != nullptr) { + if (port_begin == authority_end) { + return HTTP_ERROR_INVALID_URL; + } + + uint32_t port_value = 0; + for (const char* p = port_begin; p < authority_end; p++) { + if (std::isdigit(static_cast(*p)) == 0) { + return HTTP_ERROR_INVALID_URL; + } + port_value = port_value * 10u + static_cast(*p - '0'); + if (port_value > 65535u) { + return HTTP_ERROR_INVALID_URL; + } + } + port = static_cast(port_value); + } + + cursor = authority_end; + } else { + opaque = true; + } + + const char* path_begin = cursor; + while (*cursor != '\0' && *cursor != '?' && *cursor != '#') { + cursor++; + } + if (cursor > path_begin) { + path = {path_begin, static_cast(cursor - path_begin)}; + } + + if (*cursor == '?') { + const char* query_begin = cursor; + cursor++; + while (*cursor != '\0' && *cursor != '#') { + cursor++; + } + query = {query_begin, static_cast(cursor - query_begin)}; + } + + if (*cursor == '#') { + const char* fragment_begin = cursor; + while (*cursor != '\0') { + cursor++; + } + fragment = {fragment_begin, static_cast(cursor - fragment_begin)}; + } + + size_t needed = 0; + for (const auto& part: {scheme, username, password, hostname, path, query, fragment}) { + if (part.begin != nullptr) { + needed += part.len + 1; + } + } + + if (require != nullptr) { + *require = needed; + } + + if (out != nullptr) { + std::memset(out, 0, sizeof(*out)); + out->opaque = opaque ? 1 : 0; + out->port = port; + } + + if (out != nullptr && pool != nullptr) { + if (prepare != 0 && prepare < needed) { + return HTTP_ERROR_OUT_OF_MEMORY; + } + + auto* dst = static_cast(pool); + out->scheme = CopyUriPart(dst, scheme); + out->username = CopyUriPart(dst, username); + out->password = CopyUriPart(dst, password); + out->hostname = CopyUriPart(dst, hostname); + out->path = CopyUriPart(dst, path); + out->query = CopyUriPart(dst, query); + out->fragment = CopyUriPart(dst, fragment); + } + + return 0; +} + +static int KYTY_SYSV_ABI HttpUriEscape(char* out, size_t* require, size_t prepare, const char* in) { + PRINT_NAME(); + + LOGF("\t out = 0x%016" PRIx64 "\n", reinterpret_cast(out)); + LOGF("\t require = 0x%016" PRIx64 "\n", reinterpret_cast(require)); + LOGF("\t prepare = %" PRIu64 "\n", static_cast(prepare)); + LOGF("\t in = %s\n", in != nullptr ? in : "(null)"); + + if (in == nullptr) { + return HTTP_ERROR_INVALID_VALUE; + } + + auto is_unreserved = [](unsigned char c) { + return std::isalnum(c) != 0 || c == '-' || c == '_' || c == '.' || c == '~'; + }; + + size_t needed = 1; + for (const auto* p = reinterpret_cast(in); *p != 0; p++) { + needed += is_unreserved(*p) ? 1 : 3; + } + + if (require != nullptr) { + *require = needed; + } + + if (out == nullptr) { + return 0; + } + + if (prepare < needed) { + return HTTP_ERROR_OUT_OF_MEMORY; + } + + static constexpr char HEX[] = "0123456789ABCDEF"; + auto* dst = out; + for (const auto* p = reinterpret_cast(in); *p != 0; p++) { + if (is_unreserved(*p)) { + *dst++ = static_cast(*p); + } else { + *dst++ = '%'; + *dst++ = HEX[(*p >> 4u) & 0x0fu]; + *dst++ = HEX[*p & 0x0fu]; + } + } + *dst = '\0'; + + return 0; +} + +static void AppendUriPart(std::string* dst, const char* part) { + EXIT_IF(dst == nullptr); + + if (part != nullptr) { + dst->append(part); + } +} + +static int KYTY_SYSV_ABI HttpUriBuild(char* out, size_t* require, size_t prepare, + const SceHttpUriElement* src_element, uint32_t option) { + PRINT_NAME(); + + LOGF("\t out = 0x%016" PRIx64 "\n", reinterpret_cast(out)); + LOGF("\t require = 0x%016" PRIx64 "\n", reinterpret_cast(require)); + LOGF("\t prepare = %" PRIu64 "\n", static_cast(prepare)); + LOGF("\t src_element = 0x%016" PRIx64 "\n", reinterpret_cast(src_element)); + LOGF("\t option = 0x%08" PRIx32 "\n", option); + + if (src_element == nullptr) { + return HTTP_ERROR_INVALID_VALUE; + } + + if (out == nullptr && require == nullptr) { + return HTTP_ERROR_INVALID_VALUE; + } + + std::string uri; + if (src_element->scheme != nullptr) { + uri.append(src_element->scheme); + uri.push_back(':'); + } + + if (src_element->opaque == 0) { + if (src_element->hostname != nullptr) { + uri.append("//"); + if (src_element->username != nullptr) { + uri.append(src_element->username); + if (src_element->password != nullptr) { + uri.push_back(':'); + uri.append(src_element->password); + } + uri.push_back('@'); + } + uri.append(src_element->hostname); + if (src_element->port != 0) { + uri.push_back(':'); + uri.append(std::to_string(src_element->port)); + } + } + } + + AppendUriPart(&uri, src_element->path); + AppendUriPart(&uri, src_element->query); + AppendUriPart(&uri, src_element->fragment); + + const auto needed = uri.size() + 1; + if (require != nullptr) { + *require = needed; + } + + if (out != nullptr) { + if (prepare != 0 && prepare < needed) { + return HTTP_ERROR_OUT_OF_MEMORY; + } + + std::memcpy(out, uri.c_str(), needed); + } + + return 0; +} + +LIB_DEFINE(InitNet_1_Http) { + LIB_FUNC("A9cVMUtEp4Y", Http::HttpInit); + LIB_FUNC("Ik-KpLTlf7Q", Http::HttpTerm); + LIB_FUNC("0gYjPTR-6cY", Http::HttpCreateTemplate); + LIB_FUNC("4I8vEpuEhZ8", Http::HttpDeleteTemplate); + LIB_FUNC("s2-NPIvz+iA", Http::HttpSetNonblock); + LIB_FUNC("htyBOoWeS58", Http::HttpsSetSslCallback); + LIB_FUNC("jUjp+yqMNdQ", Http::HttpsSetMinSslVersion); + LIB_FUNC("mSQCxzWTwVI", Http::HttpsDisableOption); + LIB_FUNC("6381dWF+xsQ", Http::HttpCreateEpoll); + LIB_FUNC("wYhXVfS2Et4", Http::HttpDestroyEpoll); + LIB_FUNC("-xm7kZQNpHI", Http::HttpSetEpoll); + LIB_FUNC("59tL1AQBb8U", Http::HttpUnsetEpoll); + LIB_FUNC("Kiwv9r4IZCc", Http::HttpCreateConnection); + LIB_FUNC("qgxDBjorUxs", Http::HttpCreateConnectionWithURL); + LIB_FUNC("P6A3ytpsiYc", Http::HttpDeleteConnection); + LIB_FUNC("tsGVru3hCe8", Http::HttpCreateRequest); + LIB_FUNC("Aeu5wVKkF9w", Http::HttpCreateRequest); + LIB_FUNC("Cnp77podkCU", Http::HttpCreateRequestWithURL2); + LIB_FUNC("qe7oZ+v4PWA", Http::HttpDeleteRequest); + LIB_FUNC("PTiFIUxCpJc", Http::HttpSetRequestContentLength); + LIB_FUNC("EY28T2bkN7k", Http::HttpAddRequestHeader); + LIB_FUNC("1e2BNwI-XzE", Http::HttpSendRequest); + LIB_FUNC("hvG6GfBMXg8", Http::HttpAbortRequest); + LIB_FUNC("qISjDHrxONc", Http::HttpWaitRequest); + LIB_FUNC("0a2TBNfE3BU", Http::HttpGetStatusCode); + LIB_FUNC("aCYPMSUIaP8", Http::HttpGetAllResponseHeaders); + LIB_FUNC("yuO2H2Uvnos", Http::HttpGetResponseContentLength); + LIB_FUNC("Tc-hAYDKtQc", Http::HttpSetResolveTimeOut); + LIB_FUNC("K1d1LqZRQHQ", Http::HttpSetResolveRetry); + LIB_FUNC("0S9tTH0uqTU", Http::HttpSetConnectTimeOut); + LIB_FUNC("xegFfZKBVlw", Http::HttpSetSendTimeOut); + LIB_FUNC("yigr4V0-HTM", Http::HttpSetRecvTimeOut); + LIB_FUNC("T-mGo9f3Pu4", Http::HttpSetAutoRedirect); + LIB_FUNC("qFg2SuyTJJY", Http::HttpSetAuthEnabled); + LIB_FUNC("IWalAn-guFs", LibHttp::HttpUriParse); + LIB_FUNC("YuOW3dDAKYc", LibHttp::HttpUriEscape); + LIB_FUNC("5LZA+KPISVA", LibHttp::HttpUriBuild); +} + +} // namespace LibHttp + +namespace LibHttp2 { + +LIB_VERSION("Http2", 1, "Http2", 1, 1); + +constexpr int HTTP2_ERROR_INVALID_ID = -2122641152; /* 0x817B1100 */ +constexpr int HTTP2_ERROR_NULL_POINTER = -2122640859; /* 0x817B1225 */ + +struct Http2Options { + bool auth_enabled = false; + bool auto_redirect = false; + bool inflate_gzip = false; + uint32_t ssl_options = 0; + uint32_t resolve_timeout_us = 0; + uint32_t connect_timeout_us = 0; + uint32_t connection_wait_timeout_us = 0; + uint32_t send_timeout_us = 0; + uint32_t recv_timeout_us = 0; + uint32_t timeout_us = 0; + void* ssl_callback = nullptr; + void* ssl_callback_arg = nullptr; + void* redirect_callback = nullptr; + void* redirect_user_arg = nullptr; +}; + +struct Http2Context { + int libnet_mem_id = 0; + int libssl_ctx_id = 0; + size_t pool_size = 0; + int max_concurrent_requests = 0; +}; + +struct Http2Template { + int context_id = 0; + std::string user_agent; + int http_ver = 0; + int is_auto_proxy_conf = 0; + Http2Options options; +}; + +struct Http2Request { + int tmpl_id = 0; + std::string method; + std::string url; + uint64_t content_length = 0; + std::vector> headers; + bool sent = false; + int status_code = 204; + std::string response_headers = "HTTP/2 204 No Content\r\n\r\n"; + std::string response_body; + size_t read_offset = 0; + int async_result = 0; + int async_event = 0; + Http2Options options; +}; + +struct Http2AsyncResult { + int event_type; + int req_id; + int result; + uint8_t padding[4]; + void* reserved; +}; + +static int g_http2_next_context_id = 1; +static int g_http2_next_template_id = 1; +static int g_http2_next_request_id = 1; + +static std::map g_http2_contexts; +static std::map g_http2_templates; +static std::map g_http2_requests; + +static Http2Options* GetHttp2Options(int id) { + auto request = g_http2_requests.find(id); + if (request != g_http2_requests.end()) { + return &request->second.options; + } + + auto tmpl = g_http2_templates.find(id); + if (tmpl != g_http2_templates.end()) { + return &tmpl->second.options; + } + + return nullptr; +} + +static int KYTY_SYSV_ABI Http2Init(int libnet_mem_id, int libssl_ctx_id, size_t pool_size, + int max_concurrent_request) { + PRINT_NAME(); + + LOGF("\t libnet_mem_id = %d\n" + "\t libssl_ctx_id = %d\n" + "\t pool_size = %" PRIu64 "\n" + "\t max_concurrent_request = %d\n", + libnet_mem_id, libssl_ctx_id, pool_size, max_concurrent_request); + + const auto id = g_http2_next_context_id++; + g_http2_contexts[id] = {libnet_mem_id, libssl_ctx_id, pool_size, max_concurrent_request}; + + return id; +} + +static int KYTY_SYSV_ABI Http2CreateTemplate(int lib_http2_ctx_id, const char* user_agent, + int http_ver, int is_auto_proxy_conf) { + PRINT_NAME(); + + LOGF("\t lib_http2_ctx_id = %d\n", lib_http2_ctx_id); + LOGF("\t user_agent = %s\n", user_agent != nullptr ? user_agent : "(null)"); + LOGF("\t http_ver = %d\n", http_ver); + LOGF("\t is_auto_proxy_conf = %d\n", is_auto_proxy_conf); + + if (g_http2_contexts.find(lib_http2_ctx_id) == g_http2_contexts.end()) { + return HTTP2_ERROR_INVALID_ID; + } + + const auto tmpl_id = g_http2_next_template_id++; + g_http2_templates[tmpl_id] = {lib_http2_ctx_id, user_agent != nullptr ? user_agent : "", + http_ver, is_auto_proxy_conf}; + + return tmpl_id; +} + +static int KYTY_SYSV_ABI Http2DeleteTemplate(int tmpl_id) { + PRINT_NAME(); + + LOGF("\t tmpl_id = %d\n", tmpl_id); + + auto tmpl = g_http2_templates.find(tmpl_id); + if (tmpl == g_http2_templates.end()) { + return HTTP2_ERROR_INVALID_ID; + } + + for (auto request = g_http2_requests.begin(); request != g_http2_requests.end();) { + if (request->second.tmpl_id == tmpl_id) { + request = g_http2_requests.erase(request); + } else { + ++request; + } + } + + g_http2_templates.erase(tmpl); + + return 0; +} + +static int KYTY_SYSV_ABI Http2Term(int lib_http2_ctx_id) { + PRINT_NAME(); + + LOGF("\t lib_http2_ctx_id = %d\n", lib_http2_ctx_id); + + auto context = g_http2_contexts.find(lib_http2_ctx_id); + if (context == g_http2_contexts.end()) { + return HTTP2_ERROR_INVALID_ID; + } + + for (auto tmpl = g_http2_templates.begin(); tmpl != g_http2_templates.end();) { + if (tmpl->second.context_id == lib_http2_ctx_id) { + const auto tmpl_id = tmpl->first; + for (auto request = g_http2_requests.begin(); request != g_http2_requests.end();) { + if (request->second.tmpl_id == tmpl_id) { + request = g_http2_requests.erase(request); + } else { + ++request; + } + } + tmpl = g_http2_templates.erase(tmpl); + } else { + ++tmpl; + } + } + + g_http2_contexts.erase(context); + + return 0; +} + +static int KYTY_SYSV_ABI Http2CreateRequestWithURL(int tmpl_id, const char* method, const char* url, + uint64_t content_length) { + PRINT_NAME(); + + LOGF("\t tmpl_id = %d\n", tmpl_id); + LOGF("\t method = %s\n", method != nullptr ? method : "(null)"); + LOGF("\t url = %s\n", url != nullptr ? url : "(null)"); + LOGF("\t content_length = %" PRIu64 "\n", content_length); + + if (method == nullptr || url == nullptr) { + return HTTP2_ERROR_NULL_POINTER; + } + + if (g_http2_templates.find(tmpl_id) == g_http2_templates.end()) { + return HTTP2_ERROR_INVALID_ID; + } + + const auto req_id = g_http2_next_request_id++; + g_http2_requests[req_id] = {tmpl_id, method, url, content_length}; + + return req_id; +} + +static int KYTY_SYSV_ABI Http2AddRequestHeader(int id, const char* name, const char* value, + uint32_t mode) { + PRINT_NAME(); + + LOGF("\t id = %d\n", id); + LOGF("\t name = %s\n", name != nullptr ? name : "(null)"); + LOGF("\t value = %s\n", value != nullptr ? value : "(null)"); + LOGF("\t mode = %" PRIu32 "\n", mode); + + if (name == nullptr || value == nullptr) { + return HTTP2_ERROR_NULL_POINTER; + } + + auto request = g_http2_requests.find(id); + if (request == g_http2_requests.end()) { + return HTTP2_ERROR_INVALID_ID; + } + + request->second.headers.emplace_back(name, value); + + return 0; +} + +static int KYTY_SYSV_ABI Http2SetRequestContentLength(int id, uint64_t content_length) { + PRINT_NAME(); + + LOGF("\t id = %d\n", id); + LOGF("\t content_length = %" PRIu64 "\n", content_length); + + auto request = g_http2_requests.find(id); + if (request == g_http2_requests.end()) { + return HTTP2_ERROR_INVALID_ID; + } + + request->second.content_length = content_length; + + return 0; +} + +static int KYTY_SYSV_ABI Http2SetAuthEnabled(int id, int is_enable) { + PRINT_NAME(); + + LOGF("\t id = %d\n", id); + LOGF("\t is_enable = %d\n", is_enable); + + auto* options = GetHttp2Options(id); + if (options == nullptr) { + return HTTP2_ERROR_INVALID_ID; + } + + options->auth_enabled = is_enable != 0; + + return 0; +} + +static int KYTY_SYSV_ABI Http2SetAutoRedirect(int id, int enable) { + PRINT_NAME(); + + LOGF("\t id = %d\n", id); + LOGF("\t enable = %d\n", enable); + + auto* options = GetHttp2Options(id); + if (options == nullptr) { + return HTTP2_ERROR_INVALID_ID; + } + + options->auto_redirect = enable != 0; + + return 0; +} + +static int KYTY_SYSV_ABI Http2SetInflateGZIPEnabled(int id, int enable) { + PRINT_NAME(); + + LOGF("\t id = %d\n", id); + LOGF("\t enable = %d\n", enable); + + auto* options = GetHttp2Options(id); + if (options == nullptr) { + return HTTP2_ERROR_INVALID_ID; + } + + options->inflate_gzip = enable != 0; + + return 0; +} + +static int KYTY_SYSV_ABI Http2SslEnableOption(int id, uint32_t ssl_flags) { + PRINT_NAME(); + + LOGF("\t id = %d\n", id); + LOGF("\t ssl_flags = 0x%08" PRIx32 "\n", ssl_flags); + + auto* options = GetHttp2Options(id); + if (options == nullptr) { + return HTTP2_ERROR_INVALID_ID; + } + + options->ssl_options |= ssl_flags; + + return 0; +} + +static int KYTY_SYSV_ABI Http2SslDisableOption(int id, uint32_t ssl_flags) { + PRINT_NAME(); + + LOGF("\t id = %d\n", id); + LOGF("\t ssl_flags = 0x%08" PRIx32 "\n", ssl_flags); + + auto* options = GetHttp2Options(id); + if (options == nullptr) { + return HTTP2_ERROR_INVALID_ID; + } + + options->ssl_options &= ~ssl_flags; + + return 0; +} + +static int KYTY_SYSV_ABI Http2SetSslCallback(int id, void* cb_func, void* user_arg) { + PRINT_NAME(); + + LOGF("\t id = %d\n", id); + LOGF("\t cb_func = 0x%016" PRIx64 "\n", reinterpret_cast(cb_func)); + LOGF("\t user_arg = 0x%016" PRIx64 "\n", reinterpret_cast(user_arg)); + + auto* options = GetHttp2Options(id); + if (options == nullptr) { + return HTTP2_ERROR_INVALID_ID; + } + + options->ssl_callback = cb_func; + options->ssl_callback_arg = user_arg; + + return 0; +} + +static int KYTY_SYSV_ABI Http2SetRedirectCallback(int id, void* cb_func, void* user_arg) { + PRINT_NAME(); + + LOGF("\t id = %d\n", id); + LOGF("\t cb_func = 0x%016" PRIx64 "\n", reinterpret_cast(cb_func)); + LOGF("\t user_arg = 0x%016" PRIx64 "\n", reinterpret_cast(user_arg)); + + auto* options = GetHttp2Options(id); + if (options == nullptr) { + return HTTP2_ERROR_INVALID_ID; + } + + options->redirect_callback = cb_func; + options->redirect_user_arg = user_arg; + + return 0; +} + +static int KYTY_SYSV_ABI Http2SetConnectTimeOut(int id, uint32_t usec) { + PRINT_NAME(); + + LOGF("\t id = %d\n", id); + LOGF("\t usec = %" PRIu32 "\n", usec); + + auto* options = GetHttp2Options(id); + if (options == nullptr) { + return HTTP2_ERROR_INVALID_ID; + } + + options->connect_timeout_us = usec; + + return 0; +} + +static int KYTY_SYSV_ABI Http2SetConnectionWaitTimeOut(int id, uint32_t usec) { + PRINT_NAME(); + + LOGF("\t id = %d\n", id); + LOGF("\t usec = %" PRIu32 "\n", usec); + + auto* options = GetHttp2Options(id); + if (options == nullptr) { + return HTTP2_ERROR_INVALID_ID; + } + + options->connection_wait_timeout_us = usec; + + return 0; +} + +static int KYTY_SYSV_ABI Http2SetResolveTimeOut(int id, uint32_t usec) { + PRINT_NAME(); + + LOGF("\t id = %d\n", id); + LOGF("\t usec = %" PRIu32 "\n", usec); + + auto* options = GetHttp2Options(id); + if (options == nullptr) { + return HTTP2_ERROR_INVALID_ID; + } + + options->resolve_timeout_us = usec; + + return 0; +} + +static int KYTY_SYSV_ABI Http2SetTimeOut(int id, uint32_t usec) { + PRINT_NAME(); + + LOGF("\t id = %d\n", id); + LOGF("\t usec = %" PRIu32 "\n", usec); + + auto* options = GetHttp2Options(id); + if (options == nullptr) { + return HTTP2_ERROR_INVALID_ID; + } + + options->timeout_us = usec; + + return 0; +} + +static int KYTY_SYSV_ABI Http2SetSendTimeOut(int id, uint32_t usec) { + PRINT_NAME(); + + LOGF("\t id = %d\n", id); + LOGF("\t usec = %" PRIu32 "\n", usec); + + auto* options = GetHttp2Options(id); + if (options == nullptr) { + return HTTP2_ERROR_INVALID_ID; + } + + options->send_timeout_us = usec; + + return 0; +} + +static int KYTY_SYSV_ABI Http2SetRecvTimeOut(int id, uint32_t usec) { + PRINT_NAME(); + + LOGF("\t id = %d\n", id); + LOGF("\t usec = %" PRIu32 "\n", usec); + + auto* options = GetHttp2Options(id); + if (options == nullptr) { + return HTTP2_ERROR_INVALID_ID; + } + + options->recv_timeout_us = usec; + + return 0; +} + +static int KYTY_SYSV_ABI Http2SendRequest(int req_id, const void* post_data, size_t size) { + PRINT_NAME(); + + LOGF("\t req_id = %d\n", req_id); + LOGF("\t post_data = 0x%016" PRIx64 "\n", reinterpret_cast(post_data)); + LOGF("\t size = %" PRIu64 "\n", size); + + if (post_data == nullptr && size != 0) { + return HTTP2_ERROR_NULL_POINTER; + } + + auto request = g_http2_requests.find(req_id); + if (request == g_http2_requests.end()) { + return HTTP2_ERROR_INVALID_ID; + } + + request->second.sent = true; + + return 0; +} + +static int KYTY_SYSV_ABI Http2SendRequestAsync(int req_id, const void* post_data, size_t size, + void* /*kqueue_option*/, void* /*option*/) { + PRINT_NAME(); + + LOGF("\t req_id = %d\n", req_id); + LOGF("\t post_data = 0x%016" PRIx64 "\n", reinterpret_cast(post_data)); + LOGF("\t size = %" PRIu64 "\n", size); + + if (post_data == nullptr && size != 0) { + return HTTP2_ERROR_NULL_POINTER; + } + + auto request = g_http2_requests.find(req_id); + if (request == g_http2_requests.end()) { + return HTTP2_ERROR_INVALID_ID; + } + + request->second.sent = true; + request->second.async_result = 0; + request->second.async_event = 0; + + return 0; +} + +static int KYTY_SYSV_ABI Http2WaitAsync(int req_id, Http2AsyncResult* result, uint32_t* /*timeout*/, + void* /*option*/) { + PRINT_NAME(); + + LOGF("\t req_id = %d\n", req_id); + LOGF("\t result = 0x%016" PRIx64 "\n", reinterpret_cast(result)); + + if (result == nullptr) { + return HTTP2_ERROR_NULL_POINTER; + } + + auto request = g_http2_requests.find(req_id); + if (request == g_http2_requests.end()) { + return HTTP2_ERROR_INVALID_ID; + } + + request->second.sent = true; + *result = {}; + result->event_type = request->second.async_event; + result->req_id = req_id; + result->result = request->second.async_result; + + return 0; +} + +static int KYTY_SYSV_ABI Http2GetStatusCode(int req_id, int* status_code) { + PRINT_NAME(); + + LOGF("\t req_id = %d\n", req_id); + LOGF("\t status_code = 0x%016" PRIx64 "\n", reinterpret_cast(status_code)); + + if (status_code == nullptr) { + return HTTP2_ERROR_NULL_POINTER; + } + + auto request = g_http2_requests.find(req_id); + if (request == g_http2_requests.end()) { + return HTTP2_ERROR_INVALID_ID; + } + + *status_code = request->second.status_code; + + return 0; +} + +static int KYTY_SYSV_ABI Http2GetResponseContentLength(int req_id, int* result, + uint64_t* content_length) { + PRINT_NAME(); + + LOGF("\t req_id = %d\n", req_id); + LOGF("\t result = 0x%016" PRIx64 "\n", reinterpret_cast(result)); + LOGF("\t content_length = 0x%016" PRIx64 "\n", reinterpret_cast(content_length)); + + if (result == nullptr || content_length == nullptr) { + return HTTP2_ERROR_NULL_POINTER; + } + + auto request = g_http2_requests.find(req_id); + if (request == g_http2_requests.end()) { + return HTTP2_ERROR_INVALID_ID; + } + + *result = 0; // SCE_HTTP2_CONTENTLEN_EXIST + *content_length = request->second.response_body.size(); + + return 0; +} + +static int KYTY_SYSV_ABI Http2GetAllResponseHeaders(int req_id, char** header, + size_t* header_size) { + PRINT_NAME(); + + LOGF("\t req_id = %d\n", req_id); + LOGF("\t header = 0x%016" PRIx64 "\n", reinterpret_cast(header)); + LOGF("\t header_size = 0x%016" PRIx64 "\n", reinterpret_cast(header_size)); + + if (header == nullptr || header_size == nullptr) { + return HTTP2_ERROR_NULL_POINTER; + } + + auto request = g_http2_requests.find(req_id); + if (request == g_http2_requests.end()) { + return HTTP2_ERROR_INVALID_ID; + } + + *header = const_cast(request->second.response_headers.c_str()); + *header_size = request->second.response_headers.size(); + + return 0; +} + +static int KYTY_SYSV_ABI Http2ReadData(int req_id, void* data, size_t size) { + PRINT_NAME(); + + LOGF("\t req_id = %d\n", req_id); + LOGF("\t data = 0x%016" PRIx64 "\n", reinterpret_cast(data)); + LOGF("\t size = %" PRIu64 "\n", size); + + if (data == nullptr && size != 0) { + return HTTP2_ERROR_NULL_POINTER; + } + + auto request = g_http2_requests.find(req_id); + if (request == g_http2_requests.end()) { + return HTTP2_ERROR_INVALID_ID; + } + + const auto& body = request->second.response_body; + const auto remaining = + request->second.read_offset < body.size() ? body.size() - request->second.read_offset : 0; + const auto to_copy = size < remaining ? size : remaining; + + if (to_copy != 0) { + std::memcpy(data, body.data() + request->second.read_offset, to_copy); + request->second.read_offset += to_copy; + } + + return static_cast(to_copy); +} + +static int KYTY_SYSV_ABI Http2ReadDataAsync(int req_id, void* data, size_t size, + void* /*kqueue_option*/, void* /*option*/) { + PRINT_NAME(); + + LOGF("\t req_id = %d\n", req_id); + LOGF("\t data = 0x%016" PRIx64 "\n", reinterpret_cast(data)); + LOGF("\t size = %" PRIu64 "\n", size); + + if (data == nullptr && size != 0) { + return HTTP2_ERROR_NULL_POINTER; + } + + auto request = g_http2_requests.find(req_id); + if (request == g_http2_requests.end()) { + return HTTP2_ERROR_INVALID_ID; + } + + const auto& body = request->second.response_body; + const auto remaining = + request->second.read_offset < body.size() ? body.size() - request->second.read_offset : 0; + const auto to_copy = size < remaining ? size : remaining; + + if (to_copy != 0) { + std::memcpy(data, body.data() + request->second.read_offset, to_copy); + request->second.read_offset += to_copy; + } + + request->second.async_result = static_cast(to_copy); + request->second.async_event = 1; + + return 0; +} + +static int KYTY_SYSV_ABI Http2DeleteRequest(int req_id) { + PRINT_NAME(); + + LOGF("\t req_id = %d\n", req_id); + + auto request = g_http2_requests.find(req_id); + if (request == g_http2_requests.end()) { + return HTTP2_ERROR_INVALID_ID; + } + + g_http2_requests.erase(request); + + return 0; +} + +LIB_DEFINE(InitNet_1_Http2) { + LIB_FUNC("3JCe3lCbQ8A", LibHttp2::Http2Init); + LIB_FUNC("YiBUtz-pGkc", LibHttp2::Http2Term); + LIB_FUNC("+wCt7fCijgk", LibHttp2::Http2CreateTemplate); + LIB_FUNC("pDom5-078DA", LibHttp2::Http2DeleteTemplate); + LIB_FUNC("mmyOCxQMVYQ", LibHttp2::Http2CreateRequestWithURL); + LIB_FUNC("nrPfOE8TQu0", LibHttp2::Http2AddRequestHeader); + LIB_FUNC("FSAFOzi0FpM", LibHttp2::Http2SetRequestContentLength); + LIB_FUNC("jjFahkBPCYs", LibHttp2::Http2SetAuthEnabled); + LIB_FUNC("b9AvoIaOuHI", LibHttp2::Http2SetAutoRedirect); + LIB_FUNC("uRosf8GQbHQ", LibHttp2::Http2SetInflateGZIPEnabled); + LIB_FUNC("B37SruheQ5Y", LibHttp2::Http2SslDisableOption); + LIB_FUNC("EWcwMpbr5F8", LibHttp2::Http2SslEnableOption); + LIB_FUNC("BJgi0CH7al4", LibHttp2::Http2SetRedirectCallback); + LIB_FUNC("izvHhqgDt44", LibHttp2::Http2SetRecvTimeOut); + LIB_FUNC("XPtW45xiLHk", LibHttp2::Http2SetSendTimeOut); + LIB_FUNC("-HIO4VT87v8", LibHttp2::Http2SetConnectTimeOut); + LIB_FUNC("n8hMLe31OPA", LibHttp2::Http2SetConnectionWaitTimeOut); + LIB_FUNC("ACjtE27aErY", LibHttp2::Http2SetResolveTimeOut); + LIB_FUNC("VYMxTcBqSE0", LibHttp2::Http2SetTimeOut); + LIB_FUNC("YrWX+DhPHQY", LibHttp2::Http2SetSslCallback); + LIB_FUNC("rbqZig38AT8", LibHttp2::Http2SendRequest); + LIB_FUNC("A+NVAFu4eCg", LibHttp2::Http2SendRequestAsync); + LIB_FUNC("o0DBQpFE13o", LibHttp2::Http2GetResponseContentLength); + LIB_FUNC("9XYJwCf3lEA", LibHttp2::Http2GetStatusCode); + LIB_FUNC("-rdXUi2XW90", LibHttp2::Http2GetAllResponseHeaders); + LIB_FUNC("QygCNNmbGss", LibHttp2::Http2ReadData); + LIB_FUNC("bGN-6zbo7ms", LibHttp2::Http2ReadDataAsync); + LIB_FUNC("c8D9qIjo8EY", LibHttp2::Http2DeleteRequest); + LIB_FUNC("MOp-AUhdfi8", LibHttp2::Http2WaitAsync); +} + +} // namespace LibHttp2 + +namespace LibNetCtl { + +LIB_VERSION("NetCtl", 1, "NetCtl", 1, 1); + +namespace NetCtl = Network::NetCtl; + +LIB_DEFINE(InitNet_1_NetCtl) { + LIB_FUNC("gky0+oaNM4k", NetCtl::NetCtlInit); + LIB_FUNC("JO4yuTuMoKI", NetCtl::NetCtlGetNatInfo); + LIB_FUNC("iQw3iQPhvUQ", NetCtl::NetCtlCheckCallback); + LIB_FUNC("uBPlr0lbuiI", NetCtl::NetCtlGetState); + LIB_FUNC("+lxqIKeU9UY", NetCtl::NetCtlGetStateV6); + LIB_FUNC("UJ+Z7Q+4ck0", NetCtl::NetCtlRegisterCallback); + LIB_FUNC("Rqm2OnZMCz0", NetCtl::NetCtlUnregisterCallback); + LIB_FUNC("0cBgduPRR+M", NetCtl::NetCtlGetResult); + LIB_FUNC("obuxdTiwkF8", NetCtl::NetCtlGetInfo); + LIB_FUNC("Z4wwCFiBELQ", NetCtl::NetCtlTerm); +} + +} // namespace LibNetCtl + +namespace LibNpManager { + +LIB_VERSION("NpManager", 1, "NpManager", 1, 1); + +namespace NpManager = Network::NpManager; + +LIB_DEFINE(InitNet_1_NpManager) { + LIB_FUNC("3Zl8BePTh9Y", NpManager::NpCheckCallback); + LIB_FUNC("Ec63y59l9tw", NpManager::NpSetNpTitleId); + LIB_FUNC("A2CQ3kgSopQ", NpManager::NpSetContentRestriction); + LIB_FUNC("VfRSmPmj8Q8", NpManager::NpRegisterStateCallback); + LIB_FUNC("qQJfO8HAiaY", NpManager::NpRegisterStateCallback); + LIB_FUNC("M3wFXbYQtAA", NpManager::NpUnregisterStateCallback); + LIB_FUNC("uFJpaKNBAj4", NpManager::NpRegisterGamePresenceCallback); + LIB_FUNC("GImICnh+boA", NpManager::NpRegisterPlusEventCallback); + LIB_FUNC("+yqjab2fUJA", NpManager::NpRegisterPremiumEventCallback); + LIB_FUNC("hw5KNqAAels", NpManager::NpRegisterNpReachabilityStateCallback); + LIB_FUNC("p-o74CnoNzY", NpManager::NpGetNpId); + LIB_FUNC("XDncXQIJUSk", NpManager::NpGetOnlineId); + LIB_FUNC("rbknaUjpqWo", NpManager::NpGetAccountIdA); + LIB_FUNC("JT+t00a3TxA", NpManager::NpGetAccountCountryA); + LIB_FUNC("+4DegjBqV1g", NpManager::NpGetAccountAge); + LIB_FUNC("GpLQDNKICac", NpManager::NpCreateRequest); + LIB_FUNC("eiqMCt9UshI", NpManager::NpCreateAsyncRequest); + LIB_FUNC("S7QTn72PrDw", NpManager::NpDeleteRequest); + LIB_FUNC("OzKvTvg3ZYU", NpManager::NpAbortRequest); + LIB_FUNC("2rsFmlGWleQ", NpManager::NpCheckNpAvailability); + LIB_FUNC("KfGZg2y73oM", NpManager::NpCheckNpReachability); + LIB_FUNC("uqcPJLWL08M", NpManager::NpPollAsync); + LIB_FUNC("O80NrhUOPGY", NpManager::NpCheckPremium); + LIB_FUNC("eQH7nWPcAgc", NpManager::NpGetState); + LIB_FUNC("e-ZuhGEoeC4", NpManager::NpGetNpReachabilityState); +} + +} // namespace LibNpManager + +namespace LibNpSessionSignaling { + +LIB_VERSION("NpSessionSignaling", 1, "NpSessionSignaling", 1, 1); + +static int KYTY_SYSV_ABI NpSessionSignalingInitialize(void* param) { + PRINT_NAME(); + + LOGF("\t param = 0x%016" PRIx64 "\n", reinterpret_cast(param)); + + return 0; +} + +LIB_DEFINE(InitNet_1_NpSessionSignaling) { + LIB_FUNC("ysmw6J-P8Ak", NpSessionSignalingInitialize); +} + +} // namespace LibNpSessionSignaling + +namespace LibNpEntitlementAccess { + +LIB_VERSION("NpEntitlementAccess", 1, "NpEntitlementAccess", 1, 1); + +constexpr int NP_ENTITLEMENT_ACCESS_ERROR_PARAMETER = -2122514430; /* 0x817D0002 */ +constexpr int NP_ENTITLEMENT_ACCESS_ERROR_NO_ENTITLEMENT = -2122514425; /* 0x817D0007 */ +constexpr uint32_t NP_ENTITLEMENT_ACCESS_SKU_FLAG_FULL = 3; + +struct NpEntitlementAccessInitParam { + char reserved[32]; +}; + +struct NpEntitlementAccessBootParam { + char reserved[32]; +}; + +struct NpUnifiedEntitlementLabel { + char data[17]; + char padding[3]; +}; + +struct NpEntitlementAccessAddcontEntitlementInfo { + NpUnifiedEntitlementLabel entitlement_label; + uint32_t package_type; + uint32_t download_status; +}; + +static constexpr NpEntitlementAccessAddcontEntitlementInfo NP_ENTITLEMENT_ACCESS_ADDON_LIST[] = { + {{{"85y-je"}, {}}, 3, 4}, // GTA V hash 0xf4315381 + {{{"5d5c48"}, {}}, 3, 4}, // GTA V hash 0x961c34b0 + {{{"_mtqu6"}, {}}, 3, 4}, // GTA V hash 0x9cd1bcad +}; + +static int KYTY_SYSV_ABI NpEntitlementAccessInitialize( + const NpEntitlementAccessInitParam* init_param, NpEntitlementAccessBootParam* boot_param) { + PRINT_NAME(); + + if (init_param == nullptr || boot_param == nullptr) { + return NP_ENTITLEMENT_ACCESS_ERROR_PARAMETER; + } + + memset(boot_param, 0, sizeof(*boot_param)); + + return 0; +} + +static int KYTY_SYSV_ABI NpEntitlementAccessGetSkuFlag(uint32_t* sku_flag) { + PRINT_NAME(); + + if (sku_flag == nullptr) { + return NP_ENTITLEMENT_ACCESS_ERROR_PARAMETER; + } + + *sku_flag = NP_ENTITLEMENT_ACCESS_SKU_FLAG_FULL; + return 0; +} + +static int KYTY_SYSV_ABI NpEntitlementAccessGetAddcontEntitlementInfoList( + uint32_t service_label, NpEntitlementAccessAddcontEntitlementInfo* list, uint32_t list_num, + uint32_t* hit_num) { + PRINT_NAME(); + + LOGF("\t service_label = %" PRIu32 "\n", service_label); + LOGF("\t list = 0x%016" PRIx64 "\n", reinterpret_cast(list)); + LOGF("\t list_num = %" PRIu32 "\n", list_num); + LOGF("\t hit_num = 0x%016" PRIx64 "\n", reinterpret_cast(hit_num)); + + if (hit_num == nullptr || (list == nullptr && list_num != 0)) { + return NP_ENTITLEMENT_ACCESS_ERROR_PARAMETER; + } + + *hit_num = static_cast(sizeof(NP_ENTITLEMENT_ACCESS_ADDON_LIST) / + sizeof(NP_ENTITLEMENT_ACCESS_ADDON_LIST[0])); + + if (list != nullptr && list_num != 0) { + memset(list, 0, sizeof(*list) * list_num); + + const auto copy_num = (list_num < *hit_num ? list_num : *hit_num); + for (uint32_t i = 0; i < copy_num; i++) { + list[i] = NP_ENTITLEMENT_ACCESS_ADDON_LIST[i]; + } + } + + return 0; +} + +static int KYTY_SYSV_ABI NpEntitlementAccessGetAddcontEntitlementInfo( + uint32_t service_label, const NpUnifiedEntitlementLabel* entitlement_label, + NpEntitlementAccessAddcontEntitlementInfo* info) { + PRINT_NAME(); + + LOGF("\t service_label = %" PRIu32 "\n", service_label); + LOGF("\t entitlement_label = 0x%016" PRIx64 "\n", + reinterpret_cast(entitlement_label)); + LOGF("\t info = 0x%016" PRIx64 "\n", reinterpret_cast(info)); + + if (entitlement_label == nullptr || info == nullptr) { + return NP_ENTITLEMENT_ACCESS_ERROR_PARAMETER; + } + + memset(info, 0, sizeof(*info)); + + for (const auto& entitlement: NP_ENTITLEMENT_ACCESS_ADDON_LIST) { + if (strncmp(entitlement_label->data, entitlement.entitlement_label.data, + sizeof(entitlement_label->data)) == 0) { + *info = entitlement; + return 0; + } + } + + return NP_ENTITLEMENT_ACCESS_ERROR_NO_ENTITLEMENT; +} + +LIB_DEFINE(InitNet_1_NpEntitlementAccess) { + LIB_FUNC("jO8DM8oyego", LibNpEntitlementAccess::NpEntitlementAccessInitialize); + LIB_FUNC("lPDO62PpJIA", LibNpEntitlementAccess::NpEntitlementAccessGetSkuFlag); + LIB_FUNC("TFyU+KFBv54", + LibNpEntitlementAccess::NpEntitlementAccessGetAddcontEntitlementInfoList); + LIB_FUNC("xddD23+8TfQ", LibNpEntitlementAccess::NpEntitlementAccessGetAddcontEntitlementInfo); +} + +} // namespace LibNpEntitlementAccess + +namespace LibNpAuth { + +LIB_VERSION("NpAuth", 1, "NpAuth", 1, 1); + +constexpr int NP_AUTH_MAX_REQUEST_NUM = 16; +constexpr int NP_AUTH_REQUEST_ID_OFFSET = 0x10000000; +constexpr int NP_AUTH_ERROR_INVALID_ARGUMENT = -2141912319; /* 0x80550301 */ +constexpr int NP_AUTH_ERROR_ABORTED = -2141912316; /* 0x80550304 */ +constexpr int NP_AUTH_ERROR_REQUEST_MAX = -2141912315; /* 0x80550305 */ +constexpr int NP_AUTH_ERROR_REQUEST_NOT_FOUND = -2141912314; /* 0x80550306 */ +constexpr int NP_AUTH_ERROR_INVALID_ID = -2141912313; /* 0x80550307 */ +constexpr int NP_ERROR_SIGNED_OUT = -2141913082; /* 0x80550006 */ + +enum class NpAuthRequestState { + None = 0, + Ready, + Aborted, + Complete, +}; + +struct NpAuthRequest { + NpAuthRequestState state = NpAuthRequestState::None; + bool async = false; + int result = 0; +}; + +static std::mutex g_np_auth_request_mutex; +static int g_np_auth_active_requests = 0; +static std::vector g_np_auth_requests; + +static int np_auth_request_index(int req_id) { + return req_id - NP_AUTH_REQUEST_ID_OFFSET - 1; +} + +static NpAuthRequest* np_auth_find_request(int req_id) { + const int index = np_auth_request_index(req_id); + if (index < 0 || g_np_auth_active_requests == 0 || + static_cast(index) >= g_np_auth_requests.size() || + g_np_auth_requests[index].state == NpAuthRequestState::None) { + return nullptr; + } + + return &g_np_auth_requests[index]; +} + +static int np_auth_create_request(bool async) { + PRINT_NAME(); + + LOGF("\t async = %d\n", async ? 1 : 0); + + std::scoped_lock lock(g_np_auth_request_mutex); + + if (g_np_auth_active_requests >= NP_AUTH_MAX_REQUEST_NUM) { + return NP_AUTH_ERROR_REQUEST_MAX; + } + + size_t index = 0; + for (; index < g_np_auth_requests.size(); index++) { + if (g_np_auth_requests[index].state == NpAuthRequestState::None) { + g_np_auth_requests[index] = {NpAuthRequestState::Ready, async, 0}; + break; + } + } + + if (index == g_np_auth_requests.size()) { + g_np_auth_requests.push_back({NpAuthRequestState::Ready, async, 0}); + } + + g_np_auth_active_requests++; + + return static_cast(index) + NP_AUTH_REQUEST_ID_OFFSET + 1; +} + +static int KYTY_SYSV_ABI NpAuthCreateRequest() { + return np_auth_create_request(false); +} + +static int KYTY_SYSV_ABI NpAuthCreateAsyncRequest(const void* param) { + PRINT_NAME(); + + LOGF("\t param = 0x%016" PRIx64 "\n", reinterpret_cast(param)); + + return np_auth_create_request(true); +} + +static int KYTY_SYSV_ABI NpAuthDeleteRequest(int req_id) { + PRINT_NAME(); + + LOGF("\t req_id = %d\n", req_id); + + std::scoped_lock lock(g_np_auth_request_mutex); + + auto* request = np_auth_find_request(req_id); + if (request == nullptr) { + return NP_AUTH_ERROR_REQUEST_NOT_FOUND; + } + + request->state = NpAuthRequestState::None; + request->async = false; + request->result = 0; + g_np_auth_active_requests--; + + return 0; +} + +static int KYTY_SYSV_ABI NpAuthAbortRequest(int req_id) { + PRINT_NAME(); + + LOGF("\t req_id = %d\n", req_id); + + std::scoped_lock lock(g_np_auth_request_mutex); + + auto* request = np_auth_find_request(req_id); + if (request == nullptr) { + return NP_AUTH_ERROR_REQUEST_NOT_FOUND; + } + + if (request->state != NpAuthRequestState::Complete) { + request->state = NpAuthRequestState::Aborted; + } + + return 0; +} + +static int np_auth_wait_or_poll_async(int req_id, int* result) { + if (result == nullptr) { + return NP_AUTH_ERROR_INVALID_ARGUMENT; + } + + std::scoped_lock lock(g_np_auth_request_mutex); + + auto* request = np_auth_find_request(req_id); + if (request == nullptr) { + return NP_AUTH_ERROR_REQUEST_NOT_FOUND; + } + + if (!request->async || request->state == NpAuthRequestState::Ready) { + return NP_AUTH_ERROR_INVALID_ID; + } + + *result = request->result; + return 0; +} + +static int np_auth_complete_signed_out(NpAuthRequest* request) { + if (request->state == NpAuthRequestState::Complete) { + request->result = NP_AUTH_ERROR_INVALID_ARGUMENT; + return NP_AUTH_ERROR_INVALID_ARGUMENT; + } + if (request->state == NpAuthRequestState::Aborted) { + request->result = NP_AUTH_ERROR_ABORTED; + return NP_AUTH_ERROR_ABORTED; + } + + request->state = NpAuthRequestState::Complete; + request->result = NP_ERROR_SIGNED_OUT; + + return request->async ? 0 : NP_ERROR_SIGNED_OUT; +} + +static int KYTY_SYSV_ABI NpAuthWaitAsync(int req_id, int* result) { + PRINT_NAME(); + + LOGF("\t req_id = %d\n", req_id); + LOGF("\t result = 0x%016" PRIx64 "\n", reinterpret_cast(result)); + + return np_auth_wait_or_poll_async(req_id, result); +} + +static int KYTY_SYSV_ABI NpAuthPollAsync(int req_id, int* result) { + PRINT_NAME(); + + LOGF("\t req_id = %d\n", req_id); + LOGF("\t result = 0x%016" PRIx64 "\n", reinterpret_cast(result)); + + return np_auth_wait_or_poll_async(req_id, result); +} + +static int KYTY_SYSV_ABI NpAuthGetAuthorizationCodeV3(int req_id, const void* param, + void* auth_code, int* issuer_id) { + PRINT_NAME(); + + LOGF("\t req_id = %d\n", req_id); + LOGF("\t param = 0x%016" PRIx64 "\n", reinterpret_cast(param)); + LOGF("\t auth_code = 0x%016" PRIx64 "\n", reinterpret_cast(auth_code)); + LOGF("\t issuer_id = 0x%016" PRIx64 "\n", reinterpret_cast(issuer_id)); + + if (auth_code != nullptr) { + memset(auth_code, 0, 136); + } + if (issuer_id != nullptr) { + *issuer_id = 0; + } + + std::scoped_lock lock(g_np_auth_request_mutex); + auto* request = np_auth_find_request(req_id); + if (request == nullptr) { + return NP_AUTH_ERROR_REQUEST_NOT_FOUND; + } + if (request->state == NpAuthRequestState::Aborted) { + // request->result = 0; + // return 0; + request->result = NP_AUTH_ERROR_ABORTED; + return NP_AUTH_ERROR_ABORTED; + } + // request->state = NpAuthRequestState::Complete; + // request->result = 0; + + // return 0; + return np_auth_complete_signed_out(request); +} + +static int KYTY_SYSV_ABI NpAuthGetIdTokenV3(int req_id, const void* param, void* id_token) { + PRINT_NAME(); + + LOGF("\t req_id = %d\n", req_id); + LOGF("\t param = 0x%016" PRIx64 "\n", reinterpret_cast(param)); + LOGF("\t id_token = 0x%016" PRIx64 "\n", reinterpret_cast(id_token)); + + if (id_token != nullptr) { + memset(id_token, 0, 4104); + } + + std::scoped_lock lock(g_np_auth_request_mutex); + auto* request = np_auth_find_request(req_id); + if (request == nullptr) { + return NP_AUTH_ERROR_REQUEST_NOT_FOUND; + } + if (request->state == NpAuthRequestState::Aborted) { + // request->result = 0; + // return 0; + request->result = NP_AUTH_ERROR_ABORTED; + return NP_AUTH_ERROR_ABORTED; + } + // request->state = NpAuthRequestState::Complete; + // request->result = 0; + + // return 0; + return np_auth_complete_signed_out(request); +} + +LIB_DEFINE(InitNet_1_NpAuth) { + LIB_FUNC("6bwFkosYRQg", LibNpAuth::NpAuthCreateRequest); + LIB_FUNC("N+mr7GjTvr8", LibNpAuth::NpAuthCreateAsyncRequest); + LIB_FUNC("H8wG9Bk-nPc", LibNpAuth::NpAuthDeleteRequest); + LIB_FUNC("cE7wIsqXdZ8", LibNpAuth::NpAuthAbortRequest); + LIB_FUNC("SK-S7daqJSE", LibNpAuth::NpAuthWaitAsync); + LIB_FUNC("gjSyfzSsDcE", LibNpAuth::NpAuthPollAsync); + LIB_FUNC("KI4dHLlTNl0", LibNpAuth::NpAuthGetAuthorizationCodeV3); + LIB_FUNC("RdsFVsgSpZY", LibNpAuth::NpAuthGetIdTokenV3); +} + +} // namespace LibNpAuth + +namespace LibNpTrophy2 { + +LIB_VERSION("NpTrophy2", 1, "NpTrophy2", 1, 1); + +constexpr int NP_TROPHY2_ERROR_ICON_FILE_NOT_FOUND = -2141898479; /* 0x80553911 */ + +struct NpTrophy2Progress { + int32_t type; + uint8_t reserved[4]; + uint64_t value; +}; + +struct NpTrophy2GameDetails { + uint32_t num_groups; + uint32_t num_trophies; + uint32_t num_platinum; + uint32_t num_gold; + uint32_t num_silver; + uint32_t num_bronze; + char title[128]; +}; + +struct NpTrophy2GameData { + uint32_t unlocked_trophies; + uint32_t unlocked_platinum; + uint32_t unlocked_gold; + uint32_t unlocked_silver; + uint32_t unlocked_bronze; + uint32_t progress_percentage; +}; + +struct NpTrophy2GroupDetails { + int32_t group_id; + uint32_t num_trophies; + uint32_t num_platinum; + uint32_t num_gold; + uint32_t num_silver; + uint32_t num_bronze; + char title[128]; +}; + +struct NpTrophy2GroupData { + int32_t group_id; + uint32_t unlocked_trophies; + uint32_t unlocked_platinum; + uint32_t unlocked_gold; + uint32_t unlocked_silver; + uint32_t unlocked_bronze; + uint32_t progress_percentage; + uint8_t reserved[4]; +}; + +struct NpTrophy2Details { + int32_t trophy_id; + int32_t trophy_grade; + int32_t group_id; + bool hidden; + bool has_reward; + uint8_t reserved2[2]; + NpTrophy2Progress target; + char name[128]; + char description[1024]; + char reward[128]; +}; + +struct NpTrophy2Data { + int32_t trophy_id; + bool unlocked; + uint8_t reserved[3]; + NpTrophy2Progress progress; + uint64_t timestamp_tick; +}; + +static_assert(sizeof(NpTrophy2GameDetails) == 152); +static_assert(sizeof(NpTrophy2GameData) == 24); +static_assert(sizeof(NpTrophy2GroupDetails) == 152); +static_assert(sizeof(NpTrophy2GroupData) == 32); +static_assert(sizeof(NpTrophy2Details) == 1312); +static_assert(sizeof(NpTrophy2Data) == 32); + +static void NpTrophy2FillTitle(char* dst, size_t dst_size, const char* src) { + if (dst == nullptr || dst_size == 0) { + return; + } + + std::strncpy(dst, src, dst_size - 1); + dst[dst_size - 1] = '\0'; +} + +static int KYTY_SYSV_ABI NpTrophy2CreateHandle(int* handle) { + PRINT_NAME(); + + if (handle != nullptr) { + *handle = 1; + } + + return 0; +} + +static int KYTY_SYSV_ABI NpTrophy2CreateContext(int* context, int user_id, uint32_t service_label, + uint64_t options) { + PRINT_NAME(); + + if (context != nullptr) { + *context = 1; + } + + LOGF("\t user_id = %d\n" + "\t service_label = %u\n" + "\t options = 0x%016" PRIx64 "\n", + user_id, service_label, options); + + return 0; +} + +static int KYTY_SYSV_ABI NpTrophy2RegisterContext(int context, int handle, uint64_t options) { + PRINT_NAME(); + + LOGF("\t context = %d\n" + "\t handle = %d\n" + "\t options = 0x%016" PRIx64 "\n", + context, handle, options); + + return 0; +} + +static int KYTY_SYSV_ABI NpTrophy2GetGameInfo(int context, int handle, + NpTrophy2GameDetails* details, + NpTrophy2GameData* data) { + PRINT_NAME(); + + LOGF("\t context = %d\n" + "\t handle = %d\n" + "\t details = 0x%016" PRIx64 "\n" + "\t data = 0x%016" PRIx64 "\n", + context, handle, reinterpret_cast(details), reinterpret_cast(data)); + + if (details != nullptr) { + std::memset(details, 0, sizeof(*details)); + details->num_trophies = 1; + details->num_bronze = 1; + NpTrophy2FillTitle(details->title, sizeof(details->title), "Kyty"); + } + if (data != nullptr) { + std::memset(data, 0, sizeof(*data)); + } + + return 0; +} + +static int KYTY_SYSV_ABI NpTrophy2GetGroupInfo(int context, int handle, int group_id, + NpTrophy2GroupDetails* details, + NpTrophy2GroupData* data) { + PRINT_NAME(); + + LOGF("\t context = %d\n" + "\t handle = %d\n" + "\t group_id = %d\n" + "\t details = 0x%016" PRIx64 "\n" + "\t data = 0x%016" PRIx64 "\n", + context, handle, group_id, reinterpret_cast(details), + reinterpret_cast(data)); + + const auto normalized_group_id = (group_id < 0 ? 0 : group_id); + + if (details != nullptr) { + std::memset(details, 0, sizeof(*details)); + details->group_id = normalized_group_id; + details->num_trophies = 1; + details->num_bronze = 1; + NpTrophy2FillTitle(details->title, sizeof(details->title), "Base Game"); + } + if (data != nullptr) { + std::memset(data, 0, sizeof(*data)); + data->group_id = normalized_group_id; + } + + return 0; +} + +static int KYTY_SYSV_ABI NpTrophy2GetGroupInfoArray(int context, int handle, uint32_t offset, + uint32_t limit, + NpTrophy2GroupDetails* details_array, + NpTrophy2GroupData* data_array, + uint32_t* count) { + PRINT_NAME(); + + LOGF("\t context = %d\n" + "\t handle = %d\n" + "\t offset = %" PRIu32 "\n" + "\t limit = %" PRIu32 "\n" + "\t details_array = 0x%016" PRIx64 "\n" + "\t data_array = 0x%016" PRIx64 "\n" + "\t count = 0x%016" PRIx64 "\n", + context, handle, offset, limit, reinterpret_cast(details_array), + reinterpret_cast(data_array), reinterpret_cast(count)); + + const uint32_t out_count = (offset == 0 && limit != 0 ? 1u : 0u); + + if (count != nullptr) { + *count = out_count; + } + if (out_count != 0 && details_array != nullptr) { + std::memset(details_array, 0, sizeof(*details_array)); + details_array->group_id = 0; + details_array->num_trophies = 1; + details_array->num_bronze = 1; + NpTrophy2FillTitle(details_array->title, sizeof(details_array->title), "Base Game"); + } + if (out_count != 0 && data_array != nullptr) { + std::memset(data_array, 0, sizeof(*data_array)); + data_array->group_id = 0; + } + + return 0; +} + +static int KYTY_SYSV_ABI NpTrophy2GetTrophyInfo(int context, int handle, int trophy_id, + NpTrophy2Details* details, NpTrophy2Data* data) { + PRINT_NAME(); + + LOGF("\t context = %d\n" + "\t handle = %d\n" + "\t trophy_id = %d\n" + "\t details = 0x%016" PRIx64 "\n" + "\t data = 0x%016" PRIx64 "\n", + context, handle, trophy_id, reinterpret_cast(details), + reinterpret_cast(data)); + + if (details != nullptr) { + std::memset(details, 0, sizeof(*details)); + details->trophy_id = trophy_id; + details->trophy_grade = 4; + details->group_id = 0; + details->target.type = 0; + details->target.value = 0; + NpTrophy2FillTitle(details->name, sizeof(details->name), "Trophy"); + NpTrophy2FillTitle(details->description, sizeof(details->description), "Trophy"); + } + if (data != nullptr) { + std::memset(data, 0, sizeof(*data)); + data->trophy_id = trophy_id; + } + + return 0; +} + +static int KYTY_SYSV_ABI NpTrophy2GetTrophyInfoArray(int context, int handle, uint32_t offset, + uint32_t limit, + NpTrophy2Details* details_array, + NpTrophy2Data* data_array, uint32_t* count) { + PRINT_NAME(); + + LOGF("\t context = %d\n" + "\t handle = %d\n" + "\t offset = %" PRIu32 "\n" + "\t limit = %" PRIu32 "\n" + "\t details_array = 0x%016" PRIx64 "\n" + "\t data_array = 0x%016" PRIx64 "\n" + "\t count = 0x%016" PRIx64 "\n", + context, handle, offset, limit, reinterpret_cast(details_array), + reinterpret_cast(data_array), reinterpret_cast(count)); + + const uint32_t out_count = (offset == 0 && limit != 0 ? 1u : 0u); + + if (count != nullptr) { + *count = out_count; + } + if (out_count != 0 && details_array != nullptr) { + std::memset(details_array, 0, sizeof(*details_array)); + details_array->trophy_id = 0; + details_array->trophy_grade = 4; + details_array->group_id = 0; + NpTrophy2FillTitle(details_array->name, sizeof(details_array->name), "Trophy"); + NpTrophy2FillTitle(details_array->description, sizeof(details_array->description), + "Trophy"); + } + if (out_count != 0 && data_array != nullptr) { + std::memset(data_array, 0, sizeof(*data_array)); + data_array->trophy_id = 0; + } + + return 0; +} + +static int KYTY_SYSV_ABI NpTrophy2GetGameIcon(int context, int handle, void* buffer, size_t* size) { + PRINT_NAME(); + + LOGF("\t context = %d\n" + "\t handle = %d\n" + "\t buffer = 0x%016" PRIx64 "\n" + "\t size = 0x%016" PRIx64 "\n", + context, handle, reinterpret_cast(buffer), reinterpret_cast(size)); + + if (size != nullptr) { + *size = 0; + } + + return NP_TROPHY2_ERROR_ICON_FILE_NOT_FOUND; +} + +static int KYTY_SYSV_ABI NpTrophy2GetGroupIcon(int context, int handle, int group_id, void* buffer, + size_t* size) { + PRINT_NAME(); + + LOGF("\t context = %d\n" + "\t handle = %d\n" + "\t group_id = %d\n" + "\t buffer = 0x%016" PRIx64 "\n" + "\t size = 0x%016" PRIx64 "\n", + context, handle, group_id, reinterpret_cast(buffer), + reinterpret_cast(size)); + + if (size != nullptr) { + *size = 0; + } + + return NP_TROPHY2_ERROR_ICON_FILE_NOT_FOUND; +} + +static int KYTY_SYSV_ABI NpTrophy2GetTrophyIcon(int context, int handle, int trophy_id, + void* buffer, size_t* size) { + PRINT_NAME(); + + LOGF("\t context = %d\n" + "\t handle = %d\n" + "\t trophy_id = %d\n" + "\t buffer = 0x%016" PRIx64 "\n" + "\t size = 0x%016" PRIx64 "\n", + context, handle, trophy_id, reinterpret_cast(buffer), + reinterpret_cast(size)); + + if (size != nullptr) { + *size = 0; + } + + return NP_TROPHY2_ERROR_ICON_FILE_NOT_FOUND; +} + +static int KYTY_SYSV_ABI NpTrophy2RegisterUnlockCallback(void* callback, void* userdata) { + PRINT_NAME(); + + LOGF("\t callback = 0x%016" PRIx64 "\n" + "\t userdata = 0x%016" PRIx64 "\n", + reinterpret_cast(callback), reinterpret_cast(userdata)); + + return 0; +} + +static int KYTY_SYSV_ABI NpTrophy2AbortHandle(int handle) { + PRINT_NAME(); + + LOGF("\t handle = %d\n", handle); + + return 0; +} + +static int KYTY_SYSV_ABI NpTrophy2DestroyHandle(int handle) { + PRINT_NAME(); + + LOGF("\t handle = %d\n", handle); + + return 0; +} + +static int KYTY_SYSV_ABI NpTrophy2DestroyContext(int context) { + PRINT_NAME(); + + LOGF("\t context = %d\n", context); + + return 0; +} + +LIB_DEFINE(InitNet_1_NpTrophy2) { + LIB_FUNC("Bagshr7OQ6Q", LibNpTrophy2::NpTrophy2CreateContext); + LIB_FUNC("Gz1rmUZpROM", LibNpTrophy2::NpTrophy2CreateHandle); + LIB_FUNC("bIDov3wBu5Q", LibNpTrophy2::NpTrophy2RegisterContext); + LIB_FUNC("4IzqhhUQ3nk", LibNpTrophy2::NpTrophy2GetGameInfo); + LIB_FUNC("DoZWauG8mu0", LibNpTrophy2::NpTrophy2GetGroupInfo); + LIB_FUNC("+PDSI6WgPRc", LibNpTrophy2::NpTrophy2GetGroupInfoArray); + LIB_FUNC("EwNylPdWUTM", LibNpTrophy2::NpTrophy2GetTrophyInfo); + LIB_FUNC("y3zHpdZO6ME", LibNpTrophy2::NpTrophy2GetTrophyInfoArray); + LIB_FUNC("2QgUy+xJqS0", LibNpTrophy2::NpTrophy2GetGameIcon); + LIB_FUNC("6IjXJUy6ZnA", LibNpTrophy2::NpTrophy2GetGroupIcon); + LIB_FUNC("-9LLVU0uvs8", LibNpTrophy2::NpTrophy2GetTrophyIcon); + LIB_FUNC("sUXGfNMalIo", LibNpTrophy2::NpTrophy2RegisterUnlockCallback); + LIB_FUNC("fYapWA9xVmA", LibNpTrophy2::NpTrophy2AbortHandle); + LIB_FUNC("d8P11CI40KE", LibNpTrophy2::NpTrophy2DestroyHandle); + LIB_FUNC("sysY2FHYff4", LibNpTrophy2::NpTrophy2DestroyContext); +} + +} // namespace LibNpTrophy2 + +namespace LibNpUniversalDataSystem { + +LIB_VERSION("NpUniversalDataSystem", 1, "NpUniversalDataSystem", 1, 1); + +constexpr int NP_UNIVERSAL_DATA_SYSTEM_ERROR_INVALID_ARGUMENT = -2141900542; /* 0x80553102 */ + +struct NpUniversalDataSystemInitParam { + size_t size; + size_t pool_size; +}; + +struct NpUniversalDataSystemMemoryStat { + size_t pool_size; + size_t max_inuse_size; + size_t current_inuse_size; +}; + +struct NpUniversalDataSystemEvent {}; + +struct NpUniversalDataSystemEventPropertyObject {}; + +struct NpUniversalDataSystemEventPropertyArray {}; + +struct NpUniversalDataSystemStorageStat { + size_t in_events; + size_t out_events; + size_t lost_events; + size_t max_inuse_size; + size_t current_events; + size_t current_inuse_size; + size_t current_free_size; +}; + +static int KYTY_SYSV_ABI +NpUniversalDataSystemInitialize(const NpUniversalDataSystemInitParam* param) { + PRINT_NAME(); + + if (param == nullptr) { + return NP_UNIVERSAL_DATA_SYSTEM_ERROR_INVALID_ARGUMENT; + } + + LOGF("\t size = %" PRIu64 "\n" + "\t pool_size = %" PRIu64 "\n", + static_cast(param->size), static_cast(param->pool_size)); + + return 0; +} + +static int KYTY_SYSV_ABI NpUniversalDataSystemCreateContext(int* context, int user_id, + uint32_t service_label, + uint64_t options) { + PRINT_NAME(); + + if (context != nullptr) { + *context = 1; + } + + LOGF("\t user_id = %d\n" + "\t service_label = %u\n" + "\t options = 0x%016" PRIx64 "\n", + user_id, service_label, options); + + return 0; +} + +static int KYTY_SYSV_ABI NpUniversalDataSystemCreateHandle(int* handle) { + PRINT_NAME(); + + if (handle == nullptr) { + return NP_UNIVERSAL_DATA_SYSTEM_ERROR_INVALID_ARGUMENT; + } + + if (handle != nullptr) { + *handle = 1; + } + + return 0; +} + +static int KYTY_SYSV_ABI NpUniversalDataSystemDestroyHandle(int handle) { + PRINT_NAME(); + + LOGF("\t handle = %d\n", handle); + + return 0; +} + +static int KYTY_SYSV_ABI NpUniversalDataSystemAbortHandle(int handle) { + PRINT_NAME(); + + LOGF("\t handle = %d\n", handle); + + return 0; +} + +static int KYTY_SYSV_ABI NpUniversalDataSystemCreateEvent( + const char* event_name, const NpUniversalDataSystemEventPropertyObject* prop, + NpUniversalDataSystemEvent** new_event, NpUniversalDataSystemEventPropertyObject** prop_ptr) { + PRINT_NAME(); + + if (event_name == nullptr || new_event == nullptr) { + return NP_UNIVERSAL_DATA_SYSTEM_ERROR_INVALID_ARGUMENT; + } + + *new_event = new NpUniversalDataSystemEvent; + if (prop_ptr != nullptr) { + *prop_ptr = (prop != nullptr ? const_cast(prop) + : new NpUniversalDataSystemEventPropertyObject); + } + + LOGF("\t event_name = %s\n" + "\t prop = 0x%016" PRIx64 "\n" + "\t new_event = 0x%016" PRIx64 "\n" + "\t prop_ptr = 0x%016" PRIx64 "\n", + event_name != nullptr ? event_name : "", reinterpret_cast(prop), + reinterpret_cast(*new_event), + prop_ptr != nullptr ? reinterpret_cast(*prop_ptr) : 0); + + return 0; +} + +static int KYTY_SYSV_ABI NpUniversalDataSystemPostEvent(int context, int handle, const void* event, + uint64_t options) { + PRINT_NAME(); + + LOGF("\t context = %d\n" + "\t handle = %d\n" + "\t event = 0x%016" PRIx64 "\n" + "\t options = 0x%016" PRIx64 "\n", + context, handle, reinterpret_cast(event), options); + + return 0; +} + +static int KYTY_SYSV_ABI +NpUniversalDataSystemEventEstimateSize(const NpUniversalDataSystemEvent* event, size_t* size) { + PRINT_NAME(); + + LOGF("\t event = 0x%016" PRIx64 "\n" + "\t size = 0x%016" PRIx64 "\n", + reinterpret_cast(event), reinterpret_cast(size)); + + if (event == nullptr || size == nullptr) { + return NP_UNIVERSAL_DATA_SYSTEM_ERROR_INVALID_ARGUMENT; + } + + *size = 3; + + return 0; +} + +static int KYTY_SYSV_ABI NpUniversalDataSystemEventToString(const NpUniversalDataSystemEvent* event, + char* buf, size_t buf_size, + size_t* string_size) { + PRINT_NAME(); + + LOGF("\t event = 0x%016" PRIx64 "\n" + "\t buf = 0x%016" PRIx64 "\n" + "\t buf_size = %" PRIu64 "\n" + "\t string_size = 0x%016" PRIx64 "\n", + reinterpret_cast(event), reinterpret_cast(buf), + static_cast(buf_size), reinterpret_cast(string_size)); + + if (event == nullptr) { + return NP_UNIVERSAL_DATA_SYSTEM_ERROR_INVALID_ARGUMENT; + } + + const char* json = "{}"; + if (string_size != nullptr) { + *string_size = std::strlen(json) + 1; + } + if (buf != nullptr && buf_size > 0) { + std::snprintf(buf, buf_size, "%s", json); + } + + return 0; +} + +static int KYTY_SYSV_ABI NpUniversalDataSystemDestroyEvent(NpUniversalDataSystemEvent* event) { + PRINT_NAME(); + + LOGF("\t event = 0x%016" PRIx64 "\n", reinterpret_cast(event)); + + delete event; + + return 0; +} + +static int KYTY_SYSV_ABI NpUniversalDataSystemRegisterContext(int context, int handle, + uint64_t options) { + PRINT_NAME(); + + LOGF("\t context = %d\n" + "\t handle = %d\n" + "\t options = 0x%016" PRIx64 "\n", + context, handle, options); + + return 0; +} + +static int KYTY_SYSV_ABI NpUniversalDataSystemDestroyContext(int context) { + PRINT_NAME(); + + LOGF("\t context = %d\n", context); + + return 0; +} + +static int KYTY_SYSV_ABI NpUniversalDataSystemGetMemoryStat(NpUniversalDataSystemMemoryStat* stat) { + PRINT_NAME(); + + if (stat == nullptr) { + return NP_UNIVERSAL_DATA_SYSTEM_ERROR_INVALID_ARGUMENT; + } + + *stat = {}; + + return 0; +} + +static int KYTY_SYSV_ABI NpUniversalDataSystemCreateEventPropertyObject( + NpUniversalDataSystemEventPropertyObject** new_object) { + PRINT_NAME(); + + if (new_object == nullptr) { + return NP_UNIVERSAL_DATA_SYSTEM_ERROR_INVALID_ARGUMENT; + } + + *new_object = new NpUniversalDataSystemEventPropertyObject; + + LOGF("\t new_object = 0x%016" PRIx64 "\n", reinterpret_cast(*new_object)); + + return 0; +} + +static int KYTY_SYSV_ABI +NpUniversalDataSystemDestroyEventPropertyObject(NpUniversalDataSystemEventPropertyObject* object) { + PRINT_NAME(); + + LOGF("\t object = 0x%016" PRIx64 "\n", reinterpret_cast(object)); + + delete object; + + return 0; +} + +static int KYTY_SYSV_ABI NpUniversalDataSystemEventPropertyObjectSetString( + NpUniversalDataSystemEventPropertyObject* object, const char* key, const char* value) { + PRINT_NAME(); + + LOGF("\t object = 0x%016" PRIx64 "\n" + "\t key = %s\n" + "\t value = %s\n", + reinterpret_cast(object), key != nullptr ? key : "", + value != nullptr ? value : ""); + + if (object == nullptr || key == nullptr || value == nullptr) { + return NP_UNIVERSAL_DATA_SYSTEM_ERROR_INVALID_ARGUMENT; + } + + return 0; +} + +static int KYTY_SYSV_ABI NpUniversalDataSystemEventPropertyObjectSetInt32( + NpUniversalDataSystemEventPropertyObject* object, const char* key, int32_t value) { + PRINT_NAME(); + + LOGF("\t object = 0x%016" PRIx64 "\n" + "\t key = %s\n" + "\t value = %" PRId32 "\n", + reinterpret_cast(object), key != nullptr ? key : "", value); + + if (object == nullptr || key == nullptr) { + return NP_UNIVERSAL_DATA_SYSTEM_ERROR_INVALID_ARGUMENT; + } + + return 0; +} + +static int KYTY_SYSV_ABI NpUniversalDataSystemEventPropertyObjectSetUInt32( + NpUniversalDataSystemEventPropertyObject* object, const char* key, uint32_t value) { + PRINT_NAME(); + + if (object == nullptr || key == nullptr) { + return NP_UNIVERSAL_DATA_SYSTEM_ERROR_INVALID_ARGUMENT; + } + + LOGF("\t object = 0x%016" PRIx64 "\n" + "\t key = %s\n" + "\t value = %" PRIu32 "\n", + reinterpret_cast(object), key, value); + + return 0; +} + +static int KYTY_SYSV_ABI NpUniversalDataSystemEventPropertyObjectSetInt64( + NpUniversalDataSystemEventPropertyObject* object, const char* key, int64_t value) { + PRINT_NAME(); + + if (object == nullptr || key == nullptr) { + return NP_UNIVERSAL_DATA_SYSTEM_ERROR_INVALID_ARGUMENT; + } + + LOGF("\t object = 0x%016" PRIx64 "\n" + "\t key = %s\n" + "\t value = %" PRId64 "\n", + reinterpret_cast(object), key, value); + + return 0; +} + +static int KYTY_SYSV_ABI NpUniversalDataSystemEventPropertyObjectSetUInt64( + NpUniversalDataSystemEventPropertyObject* object, const char* key, uint64_t value) { + PRINT_NAME(); + + if (object == nullptr || key == nullptr) { + return NP_UNIVERSAL_DATA_SYSTEM_ERROR_INVALID_ARGUMENT; + } + + LOGF("\t object = 0x%016" PRIx64 "\n" + "\t key = %s\n" + "\t value = %" PRIu64 "\n", + reinterpret_cast(object), key, value); + + return 0; +} + +static int KYTY_SYSV_ABI NpUniversalDataSystemEventPropertyObjectSetFloat32( + NpUniversalDataSystemEventPropertyObject* object, const char* key, float value) { + PRINT_NAME(); + + if (object == nullptr || key == nullptr) { + return NP_UNIVERSAL_DATA_SYSTEM_ERROR_INVALID_ARGUMENT; + } + + LOGF("\t object = 0x%016" PRIx64 "\n" + "\t key = %s\n" + "\t value = %f\n", + reinterpret_cast(object), key, static_cast(value)); + + return 0; +} + +static int KYTY_SYSV_ABI NpUniversalDataSystemEventPropertyObjectSetFloat64( + NpUniversalDataSystemEventPropertyObject* object, const char* key, double value) { + PRINT_NAME(); + + if (object == nullptr || key == nullptr) { + return NP_UNIVERSAL_DATA_SYSTEM_ERROR_INVALID_ARGUMENT; + } + + LOGF("\t object = 0x%016" PRIx64 "\n" + "\t key = %s\n" + "\t value = %f\n", + reinterpret_cast(object), key, value); + + return 0; +} + +static int KYTY_SYSV_ABI NpUniversalDataSystemEventPropertyObjectSetBool( + NpUniversalDataSystemEventPropertyObject* object, const char* key, bool value) { + PRINT_NAME(); + + if (object == nullptr || key == nullptr) { + return NP_UNIVERSAL_DATA_SYSTEM_ERROR_INVALID_ARGUMENT; + } + + LOGF("\t object = 0x%016" PRIx64 "\n" + "\t key = %s\n" + "\t value = %d\n", + reinterpret_cast(object), key, value ? 1 : 0); + + return 0; +} + +static int KYTY_SYSV_ABI NpUniversalDataSystemEventPropertyObjectSetBinary( + NpUniversalDataSystemEventPropertyObject* object, const char* key, const void* value, + size_t value_size) { + PRINT_NAME(); + + if (object == nullptr || key == nullptr || (value == nullptr && value_size != 0)) { + return NP_UNIVERSAL_DATA_SYSTEM_ERROR_INVALID_ARGUMENT; + } + + LOGF("\t object = 0x%016" PRIx64 "\n" + "\t key = %s\n" + "\t value = 0x%016" PRIx64 "\n" + "\t value_size = %" PRIu64 "\n", + reinterpret_cast(object), key, reinterpret_cast(value), + static_cast(value_size)); + + return 0; +} + +static int KYTY_SYSV_ABI NpUniversalDataSystemEventPropertyObjectSetObject( + NpUniversalDataSystemEventPropertyObject* object, const char* key, + const NpUniversalDataSystemEventPropertyObject* value, + NpUniversalDataSystemEventPropertyObject** value_ptr) { + PRINT_NAME(); + + if (object == nullptr || key == nullptr) { + return NP_UNIVERSAL_DATA_SYSTEM_ERROR_INVALID_ARGUMENT; + } + + if (value_ptr != nullptr) { + *value_ptr = + (value != nullptr ? const_cast(value) + : new NpUniversalDataSystemEventPropertyObject); + } + + LOGF("\t object = 0x%016" PRIx64 "\n" + "\t key = %s\n" + "\t value = 0x%016" PRIx64 "\n" + "\t value_ptr = 0x%016" PRIx64 "\n", + reinterpret_cast(object), key, reinterpret_cast(value), + value_ptr != nullptr ? reinterpret_cast(*value_ptr) : 0); + + return 0; +} + +static int KYTY_SYSV_ABI NpUniversalDataSystemEventPropertyObjectSetArray( + NpUniversalDataSystemEventPropertyObject* object, const char* key, + const NpUniversalDataSystemEventPropertyArray* value, + NpUniversalDataSystemEventPropertyArray** value_ptr) { + PRINT_NAME(); + + LOGF("\t object = 0x%016" PRIx64 "\n" + "\t key = %s\n" + "\t value = 0x%016" PRIx64 "\n" + "\t value_ptr = 0x%016" PRIx64 "\n", + reinterpret_cast(object), key != nullptr ? key : "", + reinterpret_cast(value), reinterpret_cast(value_ptr)); + + if (object == nullptr || key == nullptr) { + return NP_UNIVERSAL_DATA_SYSTEM_ERROR_INVALID_ARGUMENT; + } + + if (value_ptr != nullptr) { + *value_ptr = (value != nullptr ? const_cast(value) + : new NpUniversalDataSystemEventPropertyArray); + } + + return 0; +} + +static int KYTY_SYSV_ABI +NpUniversalDataSystemCreateEventPropertyArray(NpUniversalDataSystemEventPropertyArray** new_array) { + PRINT_NAME(); + + if (new_array == nullptr) { + return NP_UNIVERSAL_DATA_SYSTEM_ERROR_INVALID_ARGUMENT; + } + + *new_array = new NpUniversalDataSystemEventPropertyArray; + + LOGF("\t new_array = 0x%016" PRIx64 "\n", reinterpret_cast(*new_array)); + + return 0; +} + +static int KYTY_SYSV_ABI +NpUniversalDataSystemDestroyEventPropertyArray(NpUniversalDataSystemEventPropertyArray* array) { + PRINT_NAME(); + + LOGF("\t array = 0x%016" PRIx64 "\n", reinterpret_cast(array)); + + delete array; + + return 0; +} + +static int KYTY_SYSV_ABI NpUniversalDataSystemEventPropertyArraySetString( + NpUniversalDataSystemEventPropertyArray* array, const char* value) { + PRINT_NAME(); + + LOGF("\t array = 0x%016" PRIx64 "\n" + "\t value = %s\n", + reinterpret_cast(array), value != nullptr ? value : ""); + + if (array == nullptr || value == nullptr) { + return NP_UNIVERSAL_DATA_SYSTEM_ERROR_INVALID_ARGUMENT; + } + + return 0; +} + +static int KYTY_SYSV_ABI NpUniversalDataSystemEventPropertyArraySetInt32( + NpUniversalDataSystemEventPropertyArray* array, int32_t value) { + PRINT_NAME(); + + LOGF("\t array = 0x%016" PRIx64 "\n" + "\t value = %" PRId32 "\n", + reinterpret_cast(array), value); + + return (array != nullptr ? 0 : NP_UNIVERSAL_DATA_SYSTEM_ERROR_INVALID_ARGUMENT); +} + +static int KYTY_SYSV_ABI NpUniversalDataSystemEventPropertyArraySetUInt32( + NpUniversalDataSystemEventPropertyArray* array, uint32_t value) { + PRINT_NAME(); + + LOGF("\t array = 0x%016" PRIx64 "\n" + "\t value = %" PRIu32 "\n", + reinterpret_cast(array), value); + + return (array != nullptr ? 0 : NP_UNIVERSAL_DATA_SYSTEM_ERROR_INVALID_ARGUMENT); +} + +static int KYTY_SYSV_ABI NpUniversalDataSystemEventPropertyArraySetInt64( + NpUniversalDataSystemEventPropertyArray* array, int64_t value) { + PRINT_NAME(); + + LOGF("\t array = 0x%016" PRIx64 "\n" + "\t value = %" PRId64 "\n", + reinterpret_cast(array), value); + + return (array != nullptr ? 0 : NP_UNIVERSAL_DATA_SYSTEM_ERROR_INVALID_ARGUMENT); +} + +static int KYTY_SYSV_ABI NpUniversalDataSystemEventPropertyArraySetUInt64( + NpUniversalDataSystemEventPropertyArray* array, uint64_t value) { + PRINT_NAME(); + + LOGF("\t array = 0x%016" PRIx64 "\n" + "\t value = %" PRIu64 "\n", + reinterpret_cast(array), value); + + return (array != nullptr ? 0 : NP_UNIVERSAL_DATA_SYSTEM_ERROR_INVALID_ARGUMENT); +} + +static int KYTY_SYSV_ABI NpUniversalDataSystemEventPropertyArraySetFloat32( + NpUniversalDataSystemEventPropertyArray* array, float value) { + PRINT_NAME(); + + LOGF("\t array = 0x%016" PRIx64 "\n" + "\t value = %f\n", + reinterpret_cast(array), static_cast(value)); + + return (array != nullptr ? 0 : NP_UNIVERSAL_DATA_SYSTEM_ERROR_INVALID_ARGUMENT); +} + +static int KYTY_SYSV_ABI NpUniversalDataSystemEventPropertyArraySetFloat64( + NpUniversalDataSystemEventPropertyArray* array, double value) { + PRINT_NAME(); + + LOGF("\t array = 0x%016" PRIx64 "\n" + "\t value = %f\n", + reinterpret_cast(array), value); + + return (array != nullptr ? 0 : NP_UNIVERSAL_DATA_SYSTEM_ERROR_INVALID_ARGUMENT); +} + +static int KYTY_SYSV_ABI NpUniversalDataSystemEventPropertyArraySetBool( + NpUniversalDataSystemEventPropertyArray* array, bool value) { + PRINT_NAME(); + + LOGF("\t array = 0x%016" PRIx64 "\n" + "\t value = %d\n", + reinterpret_cast(array), value ? 1 : 0); + + return (array != nullptr ? 0 : NP_UNIVERSAL_DATA_SYSTEM_ERROR_INVALID_ARGUMENT); +} + +static int KYTY_SYSV_ABI NpUniversalDataSystemEventPropertyArraySetBinary( + NpUniversalDataSystemEventPropertyArray* array, const void* value, size_t value_size) { + PRINT_NAME(); + + LOGF("\t array = 0x%016" PRIx64 "\n" + "\t value = 0x%016" PRIx64 "\n" + "\t value_size = %" PRIu64 "\n", + reinterpret_cast(array), reinterpret_cast(value), + static_cast(value_size)); + + if (array == nullptr || (value == nullptr && value_size != 0)) { + return NP_UNIVERSAL_DATA_SYSTEM_ERROR_INVALID_ARGUMENT; + } + + return 0; +} + +static int KYTY_SYSV_ABI NpUniversalDataSystemEventPropertyArraySetObject( + NpUniversalDataSystemEventPropertyArray* array, + const NpUniversalDataSystemEventPropertyObject* value, + NpUniversalDataSystemEventPropertyObject** value_ptr) { + PRINT_NAME(); + + if (array == nullptr) { + return NP_UNIVERSAL_DATA_SYSTEM_ERROR_INVALID_ARGUMENT; + } + + if (value_ptr != nullptr) { + *value_ptr = + (value != nullptr ? const_cast(value) + : new NpUniversalDataSystemEventPropertyObject); + } + + LOGF("\t array = 0x%016" PRIx64 "\n" + "\t value = 0x%016" PRIx64 "\n" + "\t value_ptr = 0x%016" PRIx64 "\n", + reinterpret_cast(array), reinterpret_cast(value), + value_ptr != nullptr ? reinterpret_cast(*value_ptr) : 0); + + return 0; +} + +static int KYTY_SYSV_ABI NpUniversalDataSystemEventPropertyArraySetArray( + NpUniversalDataSystemEventPropertyArray* array, + const NpUniversalDataSystemEventPropertyArray* value, + NpUniversalDataSystemEventPropertyArray** value_ptr) { + PRINT_NAME(); + + if (array == nullptr) { + return NP_UNIVERSAL_DATA_SYSTEM_ERROR_INVALID_ARGUMENT; + } + + if (value_ptr != nullptr) { + *value_ptr = (value != nullptr ? const_cast(value) + : new NpUniversalDataSystemEventPropertyArray); + } + + LOGF("\t array = 0x%016" PRIx64 "\n" + "\t value = 0x%016" PRIx64 "\n" + "\t value_ptr = 0x%016" PRIx64 "\n", + reinterpret_cast(array), reinterpret_cast(value), + value_ptr != nullptr ? reinterpret_cast(*value_ptr) : 0); + + return 0; +} + +static int KYTY_SYSV_ABI +NpUniversalDataSystemGetStorageStat(int context, NpUniversalDataSystemStorageStat* stat) { + PRINT_NAME(); + + LOGF("\t context = %d\n" + "\t stat = 0x%016" PRIx64 "\n", + context, reinterpret_cast(stat)); + + if (stat == nullptr) { + return NP_UNIVERSAL_DATA_SYSTEM_ERROR_INVALID_ARGUMENT; + } + + *stat = {}; + + return 0; +} + +static int KYTY_SYSV_ABI NpUniversalDataSystemTerminate() { + PRINT_NAME(); + + return 0; +} + +LIB_DEFINE(InitNet_1_NpUniversalDataSystem) { + LIB_FUNC("sjaobBgqeB4", LibNpUniversalDataSystem::NpUniversalDataSystemInitialize); + LIB_FUNC("5zBnau1uIEo", LibNpUniversalDataSystem::NpUniversalDataSystemCreateContext); + LIB_FUNC("hT0IAEvN+M0", LibNpUniversalDataSystem::NpUniversalDataSystemCreateHandle); + LIB_FUNC("p+GcLqwpL9M", LibNpUniversalDataSystem::NpUniversalDataSystemCreateEvent); + LIB_FUNC("CzkKf7ahIyU", LibNpUniversalDataSystem::NpUniversalDataSystemPostEvent); + LIB_FUNC("AUIHb7jUX3I", LibNpUniversalDataSystem::NpUniversalDataSystemDestroyHandle); + LIB_FUNC("jZCqWFgMehE", LibNpUniversalDataSystem::NpUniversalDataSystemAbortHandle); + LIB_FUNC("wB7IWzGp2v0", LibNpUniversalDataSystem::NpUniversalDataSystemDestroyContext); + LIB_FUNC("su7jW3VDDb4", LibNpUniversalDataSystem::NpUniversalDataSystemGetMemoryStat); + LIB_FUNC("+s14jq-KGYw", LibNpUniversalDataSystem::NpUniversalDataSystemEventEstimateSize); + LIB_FUNC("vj6CQGWtEBg", LibNpUniversalDataSystem::NpUniversalDataSystemEventToString); + LIB_FUNC("wG+84pnNIuo", LibNpUniversalDataSystem::NpUniversalDataSystemDestroyEvent); + LIB_FUNC("tpFJ8LIKvPw", LibNpUniversalDataSystem::NpUniversalDataSystemRegisterContext); + LIB_FUNC("s6W4Zl4Slgk", + LibNpUniversalDataSystem::NpUniversalDataSystemCreateEventPropertyObject); + LIB_FUNC("kKUH0Viib3c", + LibNpUniversalDataSystem::NpUniversalDataSystemDestroyEventPropertyObject); + LIB_FUNC("MfDb+4Nln64", + LibNpUniversalDataSystem::NpUniversalDataSystemEventPropertyObjectSetString); + LIB_FUNC("YE4dbtbz6OE", + LibNpUniversalDataSystem::NpUniversalDataSystemEventPropertyObjectSetInt32); + LIB_FUNC("AzD4irAcKE4", + LibNpUniversalDataSystem::NpUniversalDataSystemEventPropertyObjectSetUInt32); + LIB_FUNC("56QLTqx911s", + LibNpUniversalDataSystem::NpUniversalDataSystemEventPropertyObjectSetInt64); + LIB_FUNC("xvsP5Yz6FmY", + LibNpUniversalDataSystem::NpUniversalDataSystemEventPropertyObjectSetUInt64); + LIB_FUNC("lbPlT4+QVcE", + LibNpUniversalDataSystem::NpUniversalDataSystemEventPropertyObjectSetFloat32); + LIB_FUNC("4Fu8tHW+u-k", + LibNpUniversalDataSystem::NpUniversalDataSystemEventPropertyObjectSetFloat64); + LIB_FUNC("Fidd8vWgyVE", + LibNpUniversalDataSystem::NpUniversalDataSystemEventPropertyObjectSetBool); + LIB_FUNC("wAcxBDLHj1M", + LibNpUniversalDataSystem::NpUniversalDataSystemEventPropertyObjectSetBinary); + LIB_FUNC("74ASEqxSnkM", + LibNpUniversalDataSystem::NpUniversalDataSystemEventPropertyObjectSetObject); + LIB_FUNC("Wxbg5x3pTXA", + LibNpUniversalDataSystem::NpUniversalDataSystemEventPropertyObjectSetArray); + LIB_FUNC("Hm7qubT3b70", + LibNpUniversalDataSystem::NpUniversalDataSystemCreateEventPropertyArray); + LIB_FUNC("W-0xwY0ZMjw", + LibNpUniversalDataSystem::NpUniversalDataSystemDestroyEventPropertyArray); + LIB_FUNC("4llLk7YJRTE", + LibNpUniversalDataSystem::NpUniversalDataSystemEventPropertyArraySetString); + LIB_FUNC("BypQuF113-k", + LibNpUniversalDataSystem::NpUniversalDataSystemEventPropertyArraySetInt32); + LIB_FUNC("yMi0xAOpmXM", + LibNpUniversalDataSystem::NpUniversalDataSystemEventPropertyArraySetUInt32); + LIB_FUNC("viVXAwmmYrY", + LibNpUniversalDataSystem::NpUniversalDataSystemEventPropertyArraySetInt64); + LIB_FUNC("Qo9qR7v5zO4", + LibNpUniversalDataSystem::NpUniversalDataSystemEventPropertyArraySetUInt64); + LIB_FUNC("JmgwKm96Lq4", + LibNpUniversalDataSystem::NpUniversalDataSystemEventPropertyArraySetFloat32); + LIB_FUNC("sbSYZLR5AiE", + LibNpUniversalDataSystem::NpUniversalDataSystemEventPropertyArraySetFloat64); + LIB_FUNC("0+l4QSWCM4E", + LibNpUniversalDataSystem::NpUniversalDataSystemEventPropertyArraySetBool); + LIB_FUNC("IEdUCV9j2Cw", + LibNpUniversalDataSystem::NpUniversalDataSystemEventPropertyArraySetBinary); + LIB_FUNC("XY14n3jNIpE", + LibNpUniversalDataSystem::NpUniversalDataSystemEventPropertyArraySetObject); + LIB_FUNC("rdi9BAfDLq8", + LibNpUniversalDataSystem::NpUniversalDataSystemEventPropertyArraySetArray); + LIB_FUNC("KmN62tT4U8A", LibNpUniversalDataSystem::NpUniversalDataSystemGetStorageStat); + LIB_FUNC("47UAEuQl+iI", LibNpUniversalDataSystem::NpUniversalDataSystemTerminate); +} + +} // namespace LibNpUniversalDataSystem + +namespace LibCes { + +LIB_VERSION("Ces", 1, "Ces", 1, 1); + +static const uint8_t* KYTY_SYSV_ABI CesRefersUcsProfileCp1252() { + PRINT_NAME(); + + static const uint8_t profile = 0; + return &profile; +} + +static uint32_t CesDecodeUtf8(const uint8_t* utf8, uint32_t utf8max, uint32_t* utf8_len) { + if (utf8 == nullptr || utf8max == 0) { + if (utf8_len != nullptr) { + *utf8_len = 0; + } + return 0xfffd; + } + + const uint8_t c0 = utf8[0]; + if (c0 < 0x80) { + if (utf8_len != nullptr) { + *utf8_len = 1; + } + return c0; + } + if ((c0 & 0xe0) == 0xc0 && utf8max >= 2 && (utf8[1] & 0xc0) == 0x80) { + if (utf8_len != nullptr) { + *utf8_len = 2; + } + return ((c0 & 0x1f) << 6u) | (utf8[1] & 0x3fu); + } + if ((c0 & 0xf0) == 0xe0 && utf8max >= 3 && (utf8[1] & 0xc0) == 0x80 && + (utf8[2] & 0xc0) == 0x80) { + if (utf8_len != nullptr) { + *utf8_len = 3; + } + return ((c0 & 0x0f) << 12u) | ((utf8[1] & 0x3fu) << 6u) | (utf8[2] & 0x3fu); + } + if ((c0 & 0xf8) == 0xf0 && utf8max >= 4 && (utf8[1] & 0xc0) == 0x80 && + (utf8[2] & 0xc0) == 0x80 && (utf8[3] & 0xc0) == 0x80) { + if (utf8_len != nullptr) { + *utf8_len = 4; + } + return ((c0 & 0x07) << 18u) | ((utf8[1] & 0x3fu) << 12u) | ((utf8[2] & 0x3fu) << 6u) | + (utf8[3] & 0x3fu); + } + + if (utf8_len != nullptr) { + *utf8_len = 1; + } + return 0xfffd; +} + +struct Cp1252Mapping { + uint8_t cp1252; + uint32_t unicode; +}; + +static constexpr Cp1252Mapping CP1252_EXTENDED_MAP[] = { + {0x80, 0x20ac}, {0x82, 0x201a}, {0x83, 0x0192}, {0x84, 0x201e}, {0x85, 0x2026}, {0x86, 0x2020}, + {0x87, 0x2021}, {0x88, 0x02c6}, {0x89, 0x2030}, {0x8a, 0x0160}, {0x8b, 0x2039}, {0x8c, 0x0152}, + {0x8e, 0x017d}, {0x91, 0x2018}, {0x92, 0x2019}, {0x93, 0x201c}, {0x94, 0x201d}, {0x95, 0x2022}, + {0x96, 0x2013}, {0x97, 0x2014}, {0x98, 0x02dc}, {0x99, 0x2122}, {0x9a, 0x0161}, {0x9b, 0x203a}, + {0x9c, 0x0153}, {0x9e, 0x017e}, {0x9f, 0x0178}, +}; + +static uint8_t CesUnicodeToCp1252(uint32_t code) { + if (code <= 0x7f || (code >= 0xa0 && code <= 0xff)) { + return static_cast(code); + } + + for (const auto& map: CP1252_EXTENDED_MAP) { + if (map.unicode == code) { + return map.cp1252; + } + } + return '?'; +} + +static uint32_t CesCp1252ToUnicode(uint8_t sbc) { + if (sbc <= 0x7f || sbc >= 0xa0) { + return sbc; + } + + for (const auto& map: CP1252_EXTENDED_MAP) { + if (map.cp1252 == sbc) { + return map.unicode; + } + } + return sbc; +} + +static uint32_t CesEncodeUtf8(uint32_t code, uint8_t* utf8, uint32_t utf8max) { + uint8_t tmp[4] {}; + uint32_t len = 0; + + if (code <= 0x7f) { + tmp[0] = static_cast(code); + len = 1; + } else if (code <= 0x7ff) { + tmp[0] = static_cast(0xc0u | ((code >> 6u) & 0x1fu)); + tmp[1] = static_cast(0x80u | (code & 0x3fu)); + len = 2; + } else if (code <= 0xffff) { + tmp[0] = static_cast(0xe0u | ((code >> 12u) & 0x0fu)); + tmp[1] = static_cast(0x80u | ((code >> 6u) & 0x3fu)); + tmp[2] = static_cast(0x80u | (code & 0x3fu)); + len = 3; + } else { + tmp[0] = static_cast(0xf0u | ((code >> 18u) & 0x07u)); + tmp[1] = static_cast(0x80u | ((code >> 12u) & 0x3fu)); + tmp[2] = static_cast(0x80u | ((code >> 6u) & 0x3fu)); + tmp[3] = static_cast(0x80u | (code & 0x3fu)); + len = 4; + } + + if (utf8 != nullptr && utf8max >= len) { + std::memcpy(utf8, tmp, len); + } + + return len; +} + +static int KYTY_SYSV_ABI CesUtf8ToSbc(const uint8_t* utf8, uint32_t utf8max, uint32_t* utf8_len, + const uint8_t* profile, uint8_t* sbc) { + PRINT_NAME(); + + if (sbc == nullptr || profile == nullptr) { + return -1; + } + + uint32_t local_len = 0; + const auto code = CesDecodeUtf8(utf8, utf8max, &local_len); + if (utf8_len != nullptr) { + *utf8_len = local_len; + } + *sbc = CesUnicodeToCp1252(code); + + return 0; +} + +static int KYTY_SYSV_ABI CesSbcToUtf8(const uint8_t* profile, uint8_t sbc, uint8_t* utf8, + uint32_t utf8max, uint32_t* utf8_len) { + PRINT_NAME(); + + if (profile == nullptr || utf8 == nullptr) { + return -1; + } + + const auto code = CesCp1252ToUnicode(sbc); + const auto len = CesEncodeUtf8(code, utf8, utf8max); + if (utf8_len != nullptr) { + *utf8_len = len; + } + + return utf8max >= len ? 0 : -1; +} + +static int KYTY_SYSV_ABI CesStub(uint64_t arg0, uint64_t arg1, uint64_t arg2, uint64_t arg3) { + PRINT_NAME(); + + (void)arg0; + (void)arg1; + (void)arg2; + (void)arg3; + + return 0; +} + +LIB_DEFINE(InitNet_1_Ces) { + LIB_FUNC("ZiDCxUUGbec", LibCes::CesStub); + LIB_FUNC("538bRGc6Zo8", LibCes::CesStub); + LIB_FUNC("LPzYZ+FR0BI", LibCes::CesRefersUcsProfileCp1252); + LIB_FUNC("3Q1gOWWarcw", LibCes::CesUtf8ToSbc); + LIB_FUNC("xTd54EEL1Ao", LibCes::CesSbcToUtf8); +} + +} // namespace LibCes + +namespace LibNpGameIntent { + +LIB_VERSION("NpGameIntent", 1, "NpGameIntent", 1, 1); + +constexpr int NP_GAME_INTENT_ERROR_INVALID_ARGUMENT = -2141898748; /* 0x80553804 */ +constexpr int NP_GAME_INTENT_ERROR_INTENT_NOT_FOUND = -2141898746; /* 0x80553806 */ +constexpr int NP_GAME_INTENT_ERROR_VALUE_NOT_FOUND = -2141898745; /* 0x80553807 */ +constexpr int NP_GAME_INTENT_USER_ID_INVALID = -1; +constexpr size_t NP_GAME_INTENT_TYPE_MAX_SIZE = 33; +constexpr size_t NP_GAME_INTENT_DATA_MAX_SIZE = 16 * 1024 + 1; + +struct NpGameIntentData { + uint8_t data[NP_GAME_INTENT_DATA_MAX_SIZE]; + uint8_t padding[7]; +}; + +struct NpGameIntentInfo { + size_t size; + int32_t user_id; + char intent_type[NP_GAME_INTENT_TYPE_MAX_SIZE]; + uint8_t padding[7]; + uint8_t reserved[256]; + NpGameIntentData intent_data; +}; + +static_assert(sizeof(NpGameIntentData) == 16392); +static_assert(sizeof(NpGameIntentInfo) == 16704); + +static int KYTY_SYSV_ABI NpGameIntentInitialize(const void* init_param) { + PRINT_NAME(); + + LOGF("\t init_param = 0x%016" PRIx64 "\n", reinterpret_cast(init_param)); + + return 0; +} + +static int KYTY_SYSV_ABI NpGameIntentTerminate() { + PRINT_NAME(); + + return 0; +} + +static int KYTY_SYSV_ABI NpGameIntentReceiveIntent(NpGameIntentInfo* intent_info) { + PRINT_NAME(); + + LOGF("\t intent_info = 0x%016" PRIx64 "\n", reinterpret_cast(intent_info)); + + if (intent_info == nullptr) { + return NP_GAME_INTENT_ERROR_INVALID_ARGUMENT; + } + + intent_info->user_id = NP_GAME_INTENT_USER_ID_INVALID; + std::memset(intent_info->intent_type, 0, sizeof(intent_info->intent_type)); + std::memset(&intent_info->intent_data, 0, sizeof(intent_info->intent_data)); + + return NP_GAME_INTENT_ERROR_INTENT_NOT_FOUND; +} + +static int KYTY_SYSV_ABI NpGameIntentGetPropertyValueString(const NpGameIntentData* intent_data, + const char* key, char* value_buf, + size_t buf_size) { + PRINT_NAME(); + + LOGF("\t intent_data = 0x%016" PRIx64 "\n" + "\t key = %s\n" + "\t value_buf = 0x%016" PRIx64 "\n" + "\t buf_size = 0x%016" PRIx64 "\n", + reinterpret_cast(intent_data), key != nullptr ? key : "(null)", + reinterpret_cast(value_buf), buf_size); + + if (intent_data == nullptr || key == nullptr || value_buf == nullptr || buf_size == 0) { + return NP_GAME_INTENT_ERROR_INVALID_ARGUMENT; + } + + value_buf[0] = '\0'; + return NP_GAME_INTENT_ERROR_VALUE_NOT_FOUND; +} + +LIB_DEFINE(InitNet_1_NpGameIntent) { + LIB_FUNC("m87BHxt-H60", LibNpGameIntent::NpGameIntentInitialize); + LIB_FUNC("0HBYxYAjmf0", LibNpGameIntent::NpGameIntentTerminate); + LIB_FUNC("jEIXUAr9XE8", LibNpGameIntent::NpGameIntentReceiveIntent); + LIB_FUNC("rPl0INNc-M8", LibNpGameIntent::NpGameIntentGetPropertyValueString); +} + +} // namespace LibNpGameIntent + +namespace LibNpWebApi2 { + +LIB_VERSION("NpWebApi2", 1, "NpWebApi2", 1, 1); + +constexpr int NP_WEBAPI2_ERROR_INVALID_ARGUMENT = -2141899774; /* 0x80553402 */ +constexpr int NP_WEBAPI2_ERROR_REQUEST_NOT_FOUND = -2141899770; /* 0x80553406 */ +constexpr int NP_WEBAPI2_ERROR_NOT_SIGNED_IN = -2141899769; /* 0x80553407 */ + +struct NpWebApi2ResponseInformationOption { + int32_t http_status; + char* error_object; + size_t error_object_size; + size_t response_data_size; +}; + +struct NpWebApi2Request { + std::string api_group; + std::string path; + std::string method; + std::string response; + size_t read_offset = 0; +}; + +static std::map& NpWebApi2Requests() { + static std::map requests; + return requests; +} + +[[maybe_unused]] static std::string NpWebApi2MakeResponse(const NpWebApi2Request& request) { + if (request.api_group.find("sessionManager") != std::string::npos || + request.path.find("sessions") != std::string::npos || + request.path.find("Sessions") != std::string::npos) { + return R"({"gameSessions":[],"playerSessions":[]})"; + } + + return "{}"; +} + +static int KYTY_SYSV_ABI NpWebApi2Initialize(int lib_http_ctx_id, size_t pool_size) { + PRINT_NAME(); + + LOGF("\t lib_http_ctx_id = %d\n" + "\t pool_size = %" PRIu64 "\n", + lib_http_ctx_id, pool_size); + + static int id = 0; + + return ++id; +} + +static int KYTY_SYSV_ABI NpWebApi2PushEventCreateHandle(int lib_ctx_id) { + PRINT_NAME(); + + LOGF("\t lib_ctx_id = %d\n", lib_ctx_id); + + static int handle = 0; + + return ++handle; +} + +static int KYTY_SYSV_ABI NpWebApi2CreateUserContext(int lib_ctx_id, int user_id) { + PRINT_NAME(); + + LOGF("\t lib_ctx_id = %d\n" + "\t user_id = %d\n", + lib_ctx_id, user_id); + + static int user_context = 1000; + + return ++user_context; +} + +static int KYTY_SYSV_ABI NpWebApi2DeleteUserContext(int user_context_id) { + PRINT_NAME(); + + LOGF("\t user_context_id = %d\n", user_context_id); + + return 0; +} + +static int KYTY_SYSV_ABI NpWebApi2CreateRequest(int user_context_id, const char* api_group, + const char* path, const char* method, + const void* content_parameter, + int64_t* request_id) { + PRINT_NAME(); + + LOGF("\t user_context_id = %d\n", user_context_id); + LOGF("\t api_group = %s\n", api_group != nullptr ? api_group : "(null)"); + LOGF("\t path = %s\n", path != nullptr ? path : "(null)"); + LOGF("\t method = %s\n", method != nullptr ? method : "(null)"); + LOGF("\t content_parameter = 0x%016" PRIx64 "\n", + reinterpret_cast(content_parameter)); + LOGF("\t request_id = 0x%016" PRIx64 "\n", reinterpret_cast(request_id)); + + static int64_t next_request_id = 1000; + + if (request_id != nullptr) { + *request_id = next_request_id++; + NpWebApi2Request request {}; + request.api_group = (api_group != nullptr ? api_group : ""); + request.path = (path != nullptr ? path : ""); + request.method = (method != nullptr ? method : ""); + NpWebApi2Requests()[*request_id] = request; + } + + return 0; +} + +static int KYTY_SYSV_ABI NpWebApi2AbortRequest(int64_t request_id) { + PRINT_NAME(); + + LOGF("\t request_id = %" PRIi64 "\n", request_id); + + return (NpWebApi2Requests().find(request_id) == NpWebApi2Requests().end() + ? NP_WEBAPI2_ERROR_REQUEST_NOT_FOUND + : 0); +} + +static int KYTY_SYSV_ABI +NpWebApi2SendRequest(int64_t request_id, const void* data, size_t data_size, + NpWebApi2ResponseInformationOption* response_info_option) { + PRINT_NAME(); + + LOGF("\t request_id = %" PRIi64 "\n", request_id); + LOGF("\t data = 0x%016" PRIx64 "\n", reinterpret_cast(data)); + LOGF("\t data_size = %" PRIu64 "\n", data_size); + LOGF("\t response_info_option = 0x%016" PRIx64 "\n", + reinterpret_cast(response_info_option)); + + auto request = NpWebApi2Requests().find(request_id); + if (request == NpWebApi2Requests().end()) { + return NP_WEBAPI2_ERROR_REQUEST_NOT_FOUND; + } + + // request->second.response = NpWebApi2MakeResponse(request->second); + request->second.response.clear(); + request->second.read_offset = 0; + + if (response_info_option != nullptr) { + // response_info_option->http_status = 200; + response_info_option->http_status = 0; + response_info_option->response_data_size = request->second.response.size(); + if (response_info_option->error_object != nullptr && + response_info_option->error_object_size > 0) { + response_info_option->error_object[0] = '\0'; + } + } + + // return 0; + return NP_WEBAPI2_ERROR_NOT_SIGNED_IN; +} + +static int KYTY_SYSV_ABI NpWebApi2ReadData(int64_t request_id, void* data, size_t size) { + PRINT_NAME(); + + LOGF("\t request_id = %" PRIi64 "\n", request_id); + LOGF("\t data = 0x%016" PRIx64 "\n", reinterpret_cast(data)); + LOGF("\t size = %" PRIu64 "\n", size); + + if (data == nullptr || size == 0) { + return NP_WEBAPI2_ERROR_INVALID_ARGUMENT; + } + + auto request = NpWebApi2Requests().find(request_id); + if (request == NpWebApi2Requests().end()) { + return NP_WEBAPI2_ERROR_REQUEST_NOT_FOUND; + } + + const auto remaining = request->second.response.size() - request->second.read_offset; + const auto to_copy = (remaining < size ? remaining : size); + if (to_copy > 0) { + std::memcpy(data, request->second.response.data() + request->second.read_offset, to_copy); + request->second.read_offset += to_copy; + } + + return static_cast(to_copy); +} + +static int KYTY_SYSV_ABI NpWebApi2DeleteRequest(int64_t request_id) { + PRINT_NAME(); + + LOGF("\t request_id = %" PRIi64 "\n", request_id); + + NpWebApi2Requests().erase(request_id); + + return 0; +} + +static int KYTY_SYSV_ABI NpWebApi2AddHttpRequestHeader(int64_t request_id, const char* field_name, + const char* value) { + PRINT_NAME(); + + LOGF("\t request_id = %" PRIi64 "\n", request_id); + LOGF("\t field_name = %s\n", field_name != nullptr ? field_name : "(null)"); + LOGF("\t value = %s\n", value != nullptr ? value : "(null)"); + + return 0; +} + +static int KYTY_SYSV_ABI NpWebApi2GetHttpResponseHeaderValueLength(int64_t request_id, + const char* field_name, + size_t* value_length) { + PRINT_NAME(); + + LOGF("\t request_id = %" PRIi64 "\n", request_id); + LOGF("\t field_name = %s\n", field_name != nullptr ? field_name : "(null)"); + LOGF("\t value_length = 0x%016" PRIx64 "\n", reinterpret_cast(value_length)); + + if (field_name == nullptr || value_length == nullptr) { + return NP_WEBAPI2_ERROR_INVALID_ARGUMENT; + } + if (NpWebApi2Requests().find(request_id) == NpWebApi2Requests().end()) { + return NP_WEBAPI2_ERROR_REQUEST_NOT_FOUND; + } + + *value_length = (std::strcmp(field_name, "Content-Type") == 0 || + std::strcmp(field_name, "content-type") == 0 + ? std::strlen("application/json; charset=utf-8") + 1 + : 0); + + return 0; +} + +static int KYTY_SYSV_ABI NpWebApi2GetHttpResponseHeaderValue(int64_t request_id, + const char* field_name, char* value, + size_t value_size) { + PRINT_NAME(); + + LOGF("\t request_id = %" PRIi64 "\n", request_id); + LOGF("\t field_name = %s\n", field_name != nullptr ? field_name : "(null)"); + LOGF("\t value = 0x%016" PRIx64 "\n", reinterpret_cast(value)); + LOGF("\t value_size = %" PRIu64 "\n", value_size); + + if (field_name == nullptr || value == nullptr || value_size == 0) { + return NP_WEBAPI2_ERROR_INVALID_ARGUMENT; + } + if (NpWebApi2Requests().find(request_id) == NpWebApi2Requests().end()) { + return NP_WEBAPI2_ERROR_REQUEST_NOT_FOUND; + } + + const char* header = ""; + if (std::strcmp(field_name, "Content-Type") == 0 || + std::strcmp(field_name, "content-type") == 0) { + header = "application/json; charset=utf-8"; + } + + const auto copy_size = + std::strlen(header) < value_size - 1 ? std::strlen(header) : value_size - 1; + std::memcpy(value, header, copy_size); + value[copy_size] = '\0'; + + return 0; +} + +static int KYTY_SYSV_ABI NpWebApi2PushEventDeleteHandle(int lib_ctx_id, int handle_id) { + PRINT_NAME(); + + LOGF("\t lib_ctx_id = %d\n", lib_ctx_id); + LOGF("\t handle_id = %d\n", handle_id); + + return 0; +} + +static int KYTY_SYSV_ABI NpWebApi2PushEventDeletePushContext(int user_context_id, + const void* push_context_id) { + PRINT_NAME(); + + LOGF("\t user_context_id = %d\n", user_context_id); + LOGF("\t push_context_id = 0x%016" PRIx64 "\n", reinterpret_cast(push_context_id)); + + return 0; +} + +static int KYTY_SYSV_ABI NpWebApi2PushEventCreateFilter(int lib_ctx_id, int handle_id, + const char* np_service_name, + uint32_t np_service_label, + const void* filter_param, + size_t filter_param_num) { + PRINT_NAME(); + + LOGF("\t lib_ctx_id = %d\n", lib_ctx_id); + LOGF("\t handle_id = %d\n", handle_id); + LOGF("\t np_service_name = %s\n", np_service_name != nullptr ? np_service_name : "(null)"); + LOGF("\t np_service_label = %" PRIu32 "\n", np_service_label); + LOGF("\t filter_param = 0x%016" PRIx64 "\n", reinterpret_cast(filter_param)); + LOGF("\t filter_param_num = %" PRIu64 "\n", filter_param_num); + + static int filter_id = 1; + + return filter_id++; +} + +static int KYTY_SYSV_ABI NpWebApi2PushEventRegisterCallback(int user_context_id, int filter_id, + void* callback, void* user_arg) { + PRINT_NAME(); + + LOGF("\t user_context_id = %d\n", user_context_id); + LOGF("\t filter_id = %d\n", filter_id); + LOGF("\t callback = 0x%016" PRIx64 "\n", reinterpret_cast(callback)); + LOGF("\t user_arg = 0x%016" PRIx64 "\n", reinterpret_cast(user_arg)); + + static int callback_id = 1; + + return callback_id++; +} + +static void KYTY_SYSV_ABI NpWebApi2CheckTimeout() { + // Timeout processing is an internal maintenance tick in Prospero. Requests + // complete synchronously in this implementation, so there is no pending + // timeout state to advance. +} + +static int KYTY_SYSV_ABI NpWebApi2Terminate(int lib_ctx_id) { + PRINT_NAME(); + + LOGF("\t lib_ctx_id = %d\n", lib_ctx_id); + + return 0; +} + +LIB_DEFINE(InitNet_1_NpWebApi2) { + LIB_FUNC("+o9816YQhqQ", LibNpWebApi2::NpWebApi2Initialize); + LIB_FUNC("WV1GwM32NgY", LibNpWebApi2::NpWebApi2PushEventCreateHandle); + LIB_FUNC("sk54bi6FtYM", LibNpWebApi2::NpWebApi2CreateUserContext); + LIB_FUNC("9X9+cneTGUU", LibNpWebApi2::NpWebApi2DeleteUserContext); + LIB_FUNC("3EI-OSJ65Xc", LibNpWebApi2::NpWebApi2CreateRequest); + LIB_FUNC("lQOCF84lvzw", LibNpWebApi2::NpWebApi2SendRequest); + LIB_FUNC("OOY9+ObfKec", LibNpWebApi2::NpWebApi2ReadData); + LIB_FUNC("zpiPsH7dbFQ", LibNpWebApi2::NpWebApi2AbortRequest); + LIB_FUNC("vvzWO-DvG1s", LibNpWebApi2::NpWebApi2DeleteRequest); + LIB_FUNC("egOOvrnF6mI", LibNpWebApi2::NpWebApi2AddHttpRequestHeader); + LIB_FUNC("HwP3aM+c85c", LibNpWebApi2::NpWebApi2GetHttpResponseHeaderValueLength); + LIB_FUNC("hksbskNToEA", LibNpWebApi2::NpWebApi2GetHttpResponseHeaderValue); + LIB_FUNC("fIATVMo4Y1w", LibNpWebApi2::NpWebApi2PushEventDeleteHandle); + LIB_FUNC("QafxeZM3WK4", LibNpWebApi2::NpWebApi2PushEventDeletePushContext); + LIB_FUNC("MsaFhR+lPE4", LibNpWebApi2::NpWebApi2PushEventCreateFilter); + LIB_FUNC("fY3QqeNkF8k", LibNpWebApi2::NpWebApi2PushEventRegisterCallback); + LIB_FUNC("3Tt9zL3tkoc", LibNpWebApi2::NpWebApi2CheckTimeout); + LIB_FUNC("bEvXpcEk200", LibNpWebApi2::NpWebApi2Terminate); +} + +} // namespace LibNpWebApi2 + +namespace LibGameUpdate { + +LIB_VERSION("GameUpdate", 1, "GameUpdate", 1, 1); + +constexpr int GAME_UPDATE_ERROR_NOT_INITIALIZED = static_cast(0x80412801); +constexpr int GAME_UPDATE_ERROR_INVALID_ARG = static_cast(0x80412803); +constexpr int GAME_UPDATE_ERROR_INVALID_SIZE = static_cast(0x80412804); +constexpr int GAME_UPDATE_ERROR_REQUEST_NOT_FOUND = static_cast(0x80412805); + +struct GameUpdateCheckParam { + size_t size; + uint32_t option; + uint32_t reserved[9]; +}; + +struct GameUpdateCheckResult { + size_t size; + bool found; + bool addcont_found; + char padding[2]; + char content_version[11]; + char padding2[1]; + uint32_t reserved[6]; +}; + +struct GameUpdateAddcontVersionInfo { + size_t size; + bool found; + char content_version[11]; + uint32_t reserved[6]; +}; + +static bool g_game_update_initialized = false; +static int g_game_update_next_request = 1; +static std::map g_game_update_requests; + +static int KYTY_SYSV_ABI GameUpdateInitialize() { + PRINT_NAME(); + + g_game_update_initialized = true; + + return 0; +} + +static int KYTY_SYSV_ABI GameUpdateTerminate() { + PRINT_NAME(); + + g_game_update_initialized = false; + g_game_update_requests.clear(); + + return 0; +} + +static int KYTY_SYSV_ABI GameUpdateCreateRequest() { + PRINT_NAME(); + + if (!g_game_update_initialized) { + return GAME_UPDATE_ERROR_NOT_INITIALIZED; + } + + const int request_id = g_game_update_next_request++; + g_game_update_requests[request_id] = true; + + return request_id; +} + +static int KYTY_SYSV_ABI GameUpdateCheck(int request_id, const GameUpdateCheckParam* param, + GameUpdateCheckResult* result) { + PRINT_NAME(); + + LOGF("\t request_id = %d\n" + "\t param = 0x%016" PRIx64 "\n" + "\t result = 0x%016" PRIx64 "\n", + request_id, reinterpret_cast(param), reinterpret_cast(result)); + + if (!g_game_update_initialized) { + return GAME_UPDATE_ERROR_NOT_INITIALIZED; + } + if (g_game_update_requests.find(request_id) == g_game_update_requests.end()) { + return GAME_UPDATE_ERROR_REQUEST_NOT_FOUND; + } + if (param == nullptr || result == nullptr) { + return GAME_UPDATE_ERROR_INVALID_ARG; + } + if (param->size < sizeof(GameUpdateCheckParam) || + result->size < sizeof(GameUpdateCheckResult)) { + return GAME_UPDATE_ERROR_INVALID_SIZE; + } + + const auto result_size = result->size; + *result = {}; + result->size = result_size; + result->found = false; + result->addcont_found = false; + + return 0; +} + +static int KYTY_SYSV_ABI GameUpdateAbortRequest(int request_id) { + PRINT_NAME(); + + LOGF("\t request_id = %d\n", request_id); + + if (!g_game_update_initialized) { + return GAME_UPDATE_ERROR_NOT_INITIALIZED; + } + if (g_game_update_requests.find(request_id) == g_game_update_requests.end()) { + return GAME_UPDATE_ERROR_REQUEST_NOT_FOUND; + } + + return 0; +} + +static int KYTY_SYSV_ABI GameUpdateDeleteRequest(int request_id) { + PRINT_NAME(); + + LOGF("\t request_id = %d\n", request_id); + + if (!g_game_update_initialized) { + return GAME_UPDATE_ERROR_NOT_INITIALIZED; + } + if (g_game_update_requests.erase(request_id) == 0) { + return GAME_UPDATE_ERROR_REQUEST_NOT_FOUND; + } + + return 0; +} + +static int KYTY_SYSV_ABI GameUpdateGetAddcontLatestVersion(uint32_t service_label, + const void* entitlement_label, + GameUpdateAddcontVersionInfo* info) { + PRINT_NAME(); + + LOGF("\t service_label = %" PRIu32 "\n" + "\t entitlement_label = 0x%016" PRIx64 "\n" + "\t info = 0x%016" PRIx64 "\n", + service_label, reinterpret_cast(entitlement_label), + reinterpret_cast(info)); + + if (!g_game_update_initialized) { + return GAME_UPDATE_ERROR_NOT_INITIALIZED; + } + if (info == nullptr) { + return GAME_UPDATE_ERROR_INVALID_ARG; + } + if (info->size < sizeof(GameUpdateAddcontVersionInfo)) { + return GAME_UPDATE_ERROR_INVALID_SIZE; + } + + const auto info_size = info->size; + *info = {}; + info->size = info_size; + info->found = false; + + return 0; +} + +LIB_DEFINE(InitNet_1_GameUpdate) { + LIB_FUNC("YJtKLttI9fM", LibGameUpdate::GameUpdateInitialize); + LIB_FUNC("NSH-C-OmoNI", LibGameUpdate::GameUpdateTerminate); + LIB_FUNC("UvcvKaFvupA", LibGameUpdate::GameUpdateCreateRequest); + LIB_FUNC("LYVV9z8+owM", LibGameUpdate::GameUpdateCheck); + LIB_FUNC("d1CNGEOaK28", LibGameUpdate::GameUpdateAbortRequest); + LIB_FUNC("bcCyjHN5sn0", LibGameUpdate::GameUpdateDeleteRequest); + LIB_FUNC("0g0+Oq9xcI0", LibGameUpdate::GameUpdateGetAddcontLatestVersion); +} + +} // namespace LibGameUpdate + +namespace LibShare { + +LIB_VERSION("Share", 1, "Share", 1, 1); + +static int KYTY_SYSV_ABI ShareInitialize(size_t heap_size, int thread_priority, + uint64_t affinity_mask) { + PRINT_NAME(); + + LOGF("\t heap_size = %" PRIu64 "\n" + "\t thread_priority = %d\n" + "\t affinity_mask = 0x%016" PRIx64 "\n", + heap_size, thread_priority, affinity_mask); + + return 0; +} + +static int KYTY_SYSV_ABI ShareFeaturePermit(uint32_t feature_flags) { + PRINT_NAME(); + + LOGF("\t feature_flags = 0x%08" PRIx32 "\n", feature_flags); + + return 0; +} + +static int KYTY_SYSV_ABI ShareTerminate() { + PRINT_NAME(); + + return 0; +} + +LIB_DEFINE(InitNet_1_Share) { + LIB_FUNC("nBDD66kiFW8", LibShare::ShareInitialize); + LIB_FUNC("YBiIdcDPrxs", LibShare::ShareFeaturePermit); + LIB_FUNC("0IL1keINExQ", LibShare::ShareTerminate); +} + +} // namespace LibShare + +namespace LibJson2 { +LIB_DEFINE(InitNet_1_Json2); +} // namespace LibJson2 + +namespace LibRandom { + +LIB_VERSION("Random", 1, "Random", 1, 1); + +constexpr int RANDOM_ERROR_INVALID = 0x817c0016; +constexpr auto RANDOM_MAX_SIZE = 64u; + +static int KYTY_SYSV_ABI RandomGetRandomNumber(void* buf, size_t size) { + PRINT_NAME(); + + if ((buf == nullptr && size != 0) || size > RANDOM_MAX_SIZE) { + return RANDOM_ERROR_INVALID; + } + + static uint32_t seed = 0x9e3779b9u; + auto* out = static_cast(buf); + for (size_t i = 0; i < size; i++) { + seed = seed * 1664525u + 1013904223u; + out[i] = static_cast(seed >> 24u); + } + + return 0; +} + +LIB_DEFINE(InitPlatform_1_Random) { + LIB_FUNC("PI7jIZj4pcE", LibRandom::RandomGetRandomNumber); +} + +} // namespace LibRandom + +namespace LibIme { + +LIB_VERSION("Ime", 1, "Ime", 1, 1); + +static int KYTY_SYSV_ABI ImeKeyboardOpen(int user_id, const void* param) { + PRINT_NAME(); + + LOGF("\t user_id = %d\n" + "\t param = 0x%016" PRIx64 "\n", + user_id, reinterpret_cast(param)); + + return 0; +} + +static int KYTY_SYSV_ABI ImeKeyboardClose(int user_id) { + PRINT_NAME(); + + LOGF("\t user_id = %d\n", user_id); + + return 0; +} + +static int KYTY_SYSV_ABI ImeKeyboardGetResourceId(int user_id, void* resource_id_array) { + PRINT_NAME(); + + LOGF("\t user_id = %d\n" + "\t resource_id_array = 0x%016" PRIx64 "\n", + user_id, reinterpret_cast(resource_id_array)); + + return 0; +} + +static int KYTY_SYSV_ABI ImeKeyboardGetInfo(uint32_t resource_id, void* info) { + PRINT_NAME(); + + LOGF("\t resource_id = 0x%08" PRIx32 "\n" + "\t info = 0x%016" PRIx64 "\n", + resource_id, reinterpret_cast(info)); + + return 0; +} + +static int KYTY_SYSV_ABI ImeKeyboardSetMode(int user_id, uint32_t mode) { + PRINT_NAME(); + + LOGF("\t user_id = %d\n" + "\t mode = 0x%08" PRIx32 "\n", + user_id, mode); + + return 0; +} + +static int KYTY_SYSV_ABI ImeUpdate(void* handler) { + PRINT_NAME(); + + // LOGF("\t handler = 0x%016" PRIx64 "\n", reinterpret_cast(handler)); + + return 0; +} + +LIB_DEFINE(InitPlatform_1_Ime) { + LIB_FUNC("eaFXjfJv3xs", LibIme::ImeKeyboardOpen); + LIB_FUNC("PMVehSlfZ94", LibIme::ImeKeyboardClose); + LIB_FUNC("dKadqZFgKKQ", LibIme::ImeKeyboardGetResourceId); + LIB_FUNC("VkqLPArfFdc", LibIme::ImeKeyboardGetInfo); + LIB_FUNC("ua+13Hk9kKs", LibIme::ImeKeyboardSetMode); + LIB_FUNC("-4GCfYdNF1s", LibIme::ImeUpdate); +} + +} // namespace LibIme + +namespace LibRemoteplay { + +LIB_VERSION("Remoteplay", 1, "Remoteplay", 1, 1); + +static int KYTY_SYSV_ABI RemoteplayInitialize(void* heap, size_t heap_size) { + PRINT_NAME(); + + LOGF("\t heap = 0x%016" PRIx64 "\n" + "\t heap_size = %" PRIu64 "\n", + reinterpret_cast(heap), static_cast(heap_size)); + + return 0; +} + +static int KYTY_SYSV_ABI RemoteplayTerminate() { + PRINT_NAME(); + + return 0; +} + +static int KYTY_SYSV_ABI RemoteplayGetConnectionStatus(int user_id, int* status) { + PRINT_NAME(); + + LOGF("\t user_id = %d\n", user_id); + + if (status == nullptr) { + return -1; + } + + *status = 0; + + return 0; +} + +LIB_DEFINE(InitPlatform_1_Remoteplay) { + LIB_FUNC("k1SwgkMSOM8", LibRemoteplay::RemoteplayInitialize); + LIB_FUNC("BOwybKVa3Do", LibRemoteplay::RemoteplayTerminate); + LIB_FUNC("g3PNjYKWqnQ", LibRemoteplay::RemoteplayGetConnectionStatus); +} + +} // namespace LibRemoteplay + +namespace LibWebBrowserDialog { + +LIB_VERSION("WebBrowserDialog", 1, "WebBrowserDialog", 1, 1); + +static int KYTY_SYSV_ABI WebBrowserDialogInitialize() { + PRINT_NAME(); + + return 0; +} + +static int KYTY_SYSV_ABI WebBrowserDialogTerminate() { + PRINT_NAME(); + + return 0; +} + +LIB_DEFINE(InitPlatform_1_WebBrowserDialog) { + LIB_FUNC("jqb7HntFQFc", LibWebBrowserDialog::WebBrowserDialogInitialize); + LIB_FUNC("ocHtyBwHfys", LibWebBrowserDialog::WebBrowserDialogTerminate); +} + +} // namespace LibWebBrowserDialog + +namespace LibGameLiveStreaming { + +LIB_VERSION("GameLiveStreaming", 1, "GameLiveStreaming", 1, 1); + +static int KYTY_SYSV_ABI GameLiveStreamingInitialize(size_t heap_size) { + PRINT_NAME(); + + LOGF("\t heap_size = %" PRIu64 "\n", static_cast(heap_size)); + + return 0; +} + +static int KYTY_SYSV_ABI GameLiveStreamingTerminate() { + PRINT_NAME(); + + return 0; +} + +LIB_DEFINE(InitPlatform_1_GameLiveStreaming) { + LIB_FUNC("kvYEw2lBndk", LibGameLiveStreaming::GameLiveStreamingInitialize); + LIB_FUNC("9yK6Fk8mKOQ", LibGameLiveStreaming::GameLiveStreamingTerminate); +} + +} // namespace LibGameLiveStreaming + +namespace LibSharePlay { + +LIB_VERSION("SharePlay", 1, "SharePlay", 1, 1); + +static int KYTY_SYSV_ABI SharePlayInitialize(void* heap, size_t heap_size) { + PRINT_NAME(); + + LOGF("\t heap = 0x%016" PRIx64 "\n" + "\t heap_size = %" PRIu64 "\n", + reinterpret_cast(heap), static_cast(heap_size)); + + return 0; +} + +static int KYTY_SYSV_ABI SharePlayTerminate() { + PRINT_NAME(); + + return 0; +} + +LIB_DEFINE(InitPlatform_1_SharePlay) { + LIB_FUNC("isruqthpYcw", LibSharePlay::SharePlayInitialize); + LIB_FUNC("UaLjloJinow", LibSharePlay::SharePlayTerminate); +} + +} // namespace LibSharePlay + +LIB_DEFINE(InitPlatform_1) { + LibRandom::InitPlatform_1_Random(s); + LibIme::InitPlatform_1_Ime(s); + LibRemoteplay::InitPlatform_1_Remoteplay(s); + LibWebBrowserDialog::InitPlatform_1_WebBrowserDialog(s); + LibGameLiveStreaming::InitPlatform_1_GameLiveStreaming(s); + LibSharePlay::InitPlatform_1_SharePlay(s); +} + +LIB_DEFINE(InitNet_1) { + LibNet::InitNet_1_Net(s); + LibSsl::InitNet_1_Ssl(s); + LibHttp::InitNet_1_Http(s); + LibHttp2::InitNet_1_Http2(s); + LibNetCtl::InitNet_1_NetCtl(s); + LibNpManager::InitNet_1_NpManager(s); + LibNpSessionSignaling::InitNet_1_NpSessionSignaling(s); + LibNpEntitlementAccess::InitNet_1_NpEntitlementAccess(s); + LibNpAuth::InitNet_1_NpAuth(s); + LibNpTrophy2::InitNet_1_NpTrophy2(s); + LibNpUniversalDataSystem::InitNet_1_NpUniversalDataSystem(s); + LibCes::InitNet_1_Ces(s); + LibNpGameIntent::InitNet_1_NpGameIntent(s); + LibNpWebApi2::InitNet_1_NpWebApi2(s); + LibGameUpdate::InitNet_1_GameUpdate(s); + LibShare::InitNet_1_Share(s); + LibJson2::InitNet_1_Json2(s); +} + +} // namespace Libs diff --git a/src/libs/libPad.cpp b/src/libs/libPad.cpp new file mode 100644 index 000000000..b48a5a5f6 --- /dev/null +++ b/src/libs/libPad.cpp @@ -0,0 +1,444 @@ +#include "common/abi.h" +#include "common/logging/log.h" +#include "libs/controller.h" +#include "libs/errno.h" +#include "libs/libs.h" +#include "libs/padData.h" +#include "loader/symbolDatabase.h" + +#include + +namespace Libs { + +LIB_VERSION("Pad", 1, "Pad", 1, 1); + +static int KYTY_SYSV_ABI PadSetVibrationMode(int handle, int mode) { + PRINT_NAME(); + + LOGF("\t handle = %d\n" + "\t mode = %d\n", + handle, mode); + + return 0; +} + +struct PadTriggerEffectStateInformation { + int32_t state[2]; +}; + +static int KYTY_SYSV_ABI PadGetTriggerEffectState(int handle, + PadTriggerEffectStateInformation* info) { + PRINT_NAME(); + + LOGF("\t handle = %d\n", handle); + + if (info == nullptr) { + return -2137653243; /* 0x80960005 */ + } + + info->state[0] = 0; + info->state[1] = 0; + + return 0; +} + +static int KYTY_SYSV_ABI PadSetTriggerEffect(int handle, const void* param) { + PRINT_NAME(); + + LOGF("\t handle = %d\n" + "\t param = 0x%016" PRIx64 "\n", + handle, reinterpret_cast(param)); + + if (handle != 1) { + return -2137915389; /* 0x80920003 */ + } + if (param == nullptr) { + return -2137915391; /* 0x80920001 */ + } + + return OK; +} + +static int KYTY_SYSV_ABI PadSetVibrationTriggerEffectWeakWhileEmbeddedMicInUse(bool enabled) { + PRINT_NAME(); + + LOGF("\t enabled = %d\n", enabled ? 1 : 0); + + return OK; +} + +struct PadDeviceClassExtendedInformation { + int32_t device_class; + uint8_t reserved[4]; + uint8_t class_data[12]; +}; + +struct PadDeviceClassData { + int32_t device_class; + bool data_valid; + uint8_t reserved[3]; + union { + struct { + float steering_wheel_angle; + uint16_t steering_wheel; + uint16_t accelerator_pedal; + uint16_t brake_pedal; + uint16_t clutch_pedal; + uint16_t hand_brake; + uint8_t gear; + uint8_t reserved[1]; + } steering_wheel; + struct { + uint8_t data_len; + uint8_t reserved[3]; + uint8_t data[12]; + } others; + uint8_t raw[16]; + } class_data; +}; + +static_assert(sizeof(PadDeviceClassData) == 24); + +static int KYTY_SYSV_ABI +PadDeviceClassGetExtendedInformation(int handle, PadDeviceClassExtendedInformation* info) { + PRINT_NAME(); + + constexpr int pad_error_invalid_handle = -2137915389; /* 0x80920003 */ + constexpr int pad_error_invalid_arg = -2137915391; /* 0x80920001 */ + + if (handle != 1) { + return pad_error_invalid_handle; + } + if (info == nullptr) { + return pad_error_invalid_arg; + } + + std::memset(info, 0, sizeof(*info)); + + return OK; +} + +static int KYTY_SYSV_ABI PadDeviceClassParseData(int handle, const Controller::PadData* data, + PadDeviceClassData* class_data) { + PRINT_NAME(); + + constexpr int pad_error_invalid_handle = -2137915389; /* 0x80920003 */ + constexpr int pad_error_invalid_arg = -2137915391; /* 0x80920001 */ + + LOGF("\t handle = %d\n" + "\t data = 0x%016" PRIx64 "\n" + "\t class_data = 0x%016" PRIx64 "\n", + handle, reinterpret_cast(data), reinterpret_cast(class_data)); + + if (handle != 1) { + return pad_error_invalid_handle; + } + if (data == nullptr || class_data == nullptr) { + return pad_error_invalid_arg; + } + + std::memset(class_data, 0, sizeof(*class_data)); + class_data->device_class = 0; // SCE_PAD_DEVICE_CLASS_STANDARD + class_data->data_valid = data->connected; + + if (data->device_unique_data_len > 0) { + const auto data_len = + (data->device_unique_data_len > 12 ? 12 : data->device_unique_data_len); + class_data->device_class = -1; // SCE_PAD_DEVICE_CLASS_INVALID/unsupported device payload + class_data->class_data.others.data_len = data_len; + std::memcpy(class_data->class_data.others.data, data->device_unique_data, data_len); + } + + return OK; +} + +static int KYTY_SYSV_ABI PadSetTiltCorrectionState(int handle, bool enabled) { + PRINT_NAME(); + + LOGF("\t handle = %d\n" + "\t enabled = %d\n", + handle, enabled ? 1 : 0); + + return 0; +} + +static int KYTY_SYSV_ABI PadClose(int handle) { + PRINT_NAME(); + + LOGF("\t handle = %d\n", handle); + + return OK; +} + +static int KYTY_SYSV_ABI PadUnknownN3kSX62fgNo(uint64_t arg0, uint64_t arg1, uint64_t arg2, + uint64_t arg3, uint64_t arg4, uint64_t arg5) { + PRINT_NAME(); + + LOGF("\t argc-style args = (0x%016" PRIx64 ", 0x%016" PRIx64 ", 0x%016" PRIx64 ", 0x%016" PRIx64 + ", 0x%016" PRIx64 ", 0x%016" PRIx64 ")\n", + arg0, arg1, arg2, arg3, arg4, arg5); + + if (arg0 >= 0x10000 && arg3 > 0 && arg3 <= 0x1000) { + std::memset(reinterpret_cast(arg0), 0, static_cast(arg3)); + } + + return OK; +} + +LIB_DEFINE(InitPad_1) { + PRINT_NAME_ENABLE(true); + + LIB_FUNC("hv1luiJrqQM", Controller::PadInit); + LIB_FUNC("xk0AcarP3V4", Controller::PadOpen); + LIB_FUNC("u1GRHp+oWoY", Controller::PadGetHandle); + LIB_FUNC("AcslpN1jHR8", PadDeviceClassGetExtendedInformation); + LIB_FUNC("IHPqcbc0zCA", PadDeviceClassParseData); + LIB_FUNC("clVvL4ZDntw", Controller::PadSetMotionSensorState); + LIB_FUNC("r44mAxdSG+U", Controller::PadSetAngularVelocityDeadbandState); + LIB_FUNC("rIZnR6eSpvk", Controller::PadResetOrientation); + LIB_FUNC("vDLMoJLde8I", PadSetTiltCorrectionState); + LIB_FUNC("gjP9-KQzoUk", Controller::PadGetControllerInformation); + LIB_FUNC("2JgFB2n9oUM", PadSetTriggerEffect); + LIB_FUNC("YndgXqQVV7c", Controller::PadReadState); + LIB_FUNC("q1cHNfGycLI", Controller::PadRead); + LIB_FUNC("yFVnOdGxvZY", Controller::PadSetVibration); + LIB_FUNC("W2G-yoyMF5U", PadSetVibrationMode); + LIB_FUNC("Yq0zOH7YNOM", PadSetVibrationTriggerEffectWeakWhileEmbeddedMicInUse); + LIB_FUNC("znaWI0gpuo8", PadGetTriggerEffectState); + LIB_FUNC("DscD1i9HX1w", Controller::PadResetLightBar); + LIB_FUNC("RR4novUEENY", Controller::PadSetLightBar); + LIB_FUNC("6ncge5+l5Qs", PadClose); + LIB_FUNC("n3kSX62fgNo", PadUnknownN3kSX62fgNo); +} + +namespace LibMouse { + +LIB_VERSION("Mouse", 1, "Mouse", 1, 1); + +namespace Mouse { + +constexpr int MOUSE_ERROR_INVALID_ARG = -2132869119; /* 0x80DF0001 */ +constexpr int MOUSE_ERROR_INVALID_HANDLE = -2132869117; /* 0x80DF0003 */ + +struct MouseData { + uint64_t timestamp; + bool connected; + uint8_t padding[3]; + uint32_t buttons; + int32_t x_axis; + int32_t y_axis; + int32_t wheel; + int32_t tilt; + uint8_t reserved[8]; +}; + +static int KYTY_SYSV_ABI MouseInit() { + PRINT_NAME(); + + return OK; +} + +static int KYTY_SYSV_ABI MouseOpen(int user_id, int32_t type, int32_t index, const void* param) { + PRINT_NAME(); + + LOGF("\t user_id = %d\n" + "\t type = %" PRId32 "\n" + "\t index = %" PRId32 "\n" + "\t param = 0x%016" PRIx64 "\n", + user_id, type, index, reinterpret_cast(param)); + + if (type != 0 || index < 0 || index >= 2) { + return MOUSE_ERROR_INVALID_ARG; + } + + return 1; +} + +static int KYTY_SYSV_ABI MouseClose(int32_t handle) { + PRINT_NAME(); + + LOGF("\t handle = %" PRId32 "\n", handle); + + if (handle <= 0) { + return MOUSE_ERROR_INVALID_HANDLE; + } + + return OK; +} + +static int KYTY_SYSV_ABI MouseRead(int32_t handle, MouseData* data, int32_t num) { + PRINT_NAME(); + + LOGF("\t handle = %" PRId32 "\n" + "\t data = 0x%016" PRIx64 "\n" + "\t num = %" PRId32 "\n", + handle, reinterpret_cast(data), num); + + if (handle <= 0) { + return MOUSE_ERROR_INVALID_HANDLE; + } + if (data == nullptr || num <= 0) { + return MOUSE_ERROR_INVALID_ARG; + } + + std::memset(data, 0, sizeof(MouseData) * static_cast(num)); + + return 0; +} + +} // namespace Mouse + +LIB_DEFINE(InitMouse_1) { + LIB_FUNC("cAnT0Rw-IwU", Mouse::MouseClose); + LIB_FUNC("Qs0wWulgl7U", Mouse::MouseInit); + LIB_FUNC("RaqxZIf6DvE", Mouse::MouseOpen); + LIB_FUNC("x8qnXqh-tiM", Mouse::MouseRead); +} + +} // namespace LibMouse + +namespace LibKeyboard { + +LIB_VERSION("Keyboard", 1, "Keyboard", 1, 1); + +namespace Keyboard { + +constexpr int KEYBOARD_ERROR_INVALID_ARG = -2133196799; /* 0x80da0001 */ +constexpr int KEYBOARD_ERROR_INVALID_HANDLE = -2133196797; /* 0x80da0003 */ +constexpr int KEYBOARD_MAX_KEYCODES = 16; +constexpr int KEYBOARD_MAX_DATA_NUM = 16; +constexpr int KEYBOARD_MAPPING_101 = 0; +constexpr int KEYBOARD_MAPPING_106 = 1; + +struct KeyboardData { + uint64_t timestamp; + bool intercepted; + uint8_t reserve1[7]; + bool connected; + int32_t length; + uint32_t led; + uint32_t modifier_key; + uint16_t key_code[KEYBOARD_MAX_KEYCODES]; + uint8_t reserve2[32]; +}; + +struct KeyboardCharData { + bool processed; + int32_t length; + uint16_t char_code; + uint8_t reserve[8]; +}; + +static int KYTY_SYSV_ABI KeyboardInit() { + PRINT_NAME(); + + return OK; +} + +static int KYTY_SYSV_ABI KeyboardOpen(int user_id, int32_t type, int32_t index, const void* param) { + PRINT_NAME(); + + LOGF("\t user_id = %d\n" + "\t type = %" PRId32 "\n" + "\t index = %" PRId32 "\n" + "\t param = 0x%016" PRIx64 "\n", + user_id, type, index, reinterpret_cast(param)); + + if (type != 0 || index < 0 || index >= 2) { + return KEYBOARD_ERROR_INVALID_ARG; + } + + return 1; +} + +static int KYTY_SYSV_ABI KeyboardClose(int32_t handle) { + PRINT_NAME(); + + LOGF("\t handle = %" PRId32 "\n", handle); + + if (handle <= 0) { + return KEYBOARD_ERROR_INVALID_HANDLE; + } + + return OK; +} + +static int KYTY_SYSV_ABI KeyboardRead(int32_t handle, KeyboardData* data, int32_t num) { + PRINT_NAME(); + + LOGF("\t handle = %" PRId32 "\n" + "\t data = 0x%016" PRIx64 "\n" + "\t num = %" PRId32 "\n", + handle, reinterpret_cast(data), num); + + if (handle <= 0) { + return KEYBOARD_ERROR_INVALID_HANDLE; + } + if (data == nullptr || num <= 0 || num > KEYBOARD_MAX_DATA_NUM) { + return KEYBOARD_ERROR_INVALID_ARG; + } + + std::memset(data, 0, sizeof(KeyboardData) * static_cast(num)); + + return 0; +} + +static int KYTY_SYSV_ABI KeyboardReadState(int32_t handle, KeyboardData* data) { + PRINT_NAME(); + + LOGF("\t handle = %" PRId32 "\n" + "\t data = 0x%016" PRIx64 "\n", + handle, reinterpret_cast(data)); + + if (handle <= 0) { + return KEYBOARD_ERROR_INVALID_HANDLE; + } + if (data == nullptr) { + return KEYBOARD_ERROR_INVALID_ARG; + } + + std::memset(data, 0, sizeof(KeyboardData)); + + return 0; +} + +static int KYTY_SYSV_ABI KeyboardGetKey2Char(int32_t handle, int32_t arrange, uint32_t led, + uint32_t modifier_key, uint16_t key_code, + KeyboardCharData* char_data) { + PRINT_NAME(); + + LOGF("\t handle = %" PRId32 "\n" + "\t arrange = %" PRId32 "\n" + "\t led = %" PRIu32 "\n" + "\t modifier_key = %" PRIu32 "\n" + "\t key_code = %" PRIu16 "\n" + "\t char_data = 0x%016" PRIx64 "\n", + handle, arrange, led, modifier_key, key_code, reinterpret_cast(char_data)); + + if (handle <= 0) { + return KEYBOARD_ERROR_INVALID_HANDLE; + } + if (char_data == nullptr || + (arrange != KEYBOARD_MAPPING_101 && arrange != KEYBOARD_MAPPING_106)) { + return KEYBOARD_ERROR_INVALID_ARG; + } + + std::memset(char_data, 0, sizeof(KeyboardCharData)); + + return OK; +} + +} // namespace Keyboard + +LIB_DEFINE(InitKeyboard_1) { + LIB_FUNC("wadT3QBCGY0", Keyboard::KeyboardInit); + LIB_FUNC("HJ+KnEHcaxI", Keyboard::KeyboardOpen); + LIB_FUNC("0LWei+c7RNc", Keyboard::KeyboardClose); + LIB_FUNC("6HpE68bzX6M", Keyboard::KeyboardReadState); + LIB_FUNC("xybbGMCr738", Keyboard::KeyboardRead); + LIB_FUNC("yO9JwdRhtSA", Keyboard::KeyboardGetKey2Char); +} + +} // namespace LibKeyboard + +} // namespace Libs diff --git a/src/libs/libPlayGo.cpp b/src/libs/libPlayGo.cpp new file mode 100644 index 000000000..b7c4346b8 --- /dev/null +++ b/src/libs/libPlayGo.cpp @@ -0,0 +1,458 @@ +#include "common/abi.h" +#include "common/assert.h" +#include "common/common.h" +#include "common/logging/log.h" +#include "common/stringUtils.h" +#include "libs/errno.h" +#include "libs/libs.h" +#include "loader/symbolDatabase.h" +#include "loader/systemContent.h" + +#include + +namespace Libs { + +LIB_VERSION("PlayGo", 1, "PlayGo", 1, 0); + +namespace PlayGo { + +static constexpr int PLAYGO_HANDLE = 1; +static constexpr int8_t PLAYGO_LOCUS_NOT_DOWNLOADED = 0; +static constexpr int8_t PLAYGO_LOCUS_LOCAL_SLOW = 2; +static constexpr int8_t PLAYGO_LOCUS_LOCAL_FAST = 3; +static constexpr int32_t PLAYGO_INSTALL_SPEED_FULL = 2; +static constexpr int32_t PLAYGO_OPTIONAL_TYPE_LANGUAGE = 0; +static constexpr int32_t PLAYGO_OPTIONAL_TYPE_SCENARIO = 1; +static constexpr uint64_t PLAYGO_LANGUAGE_MASK_ALL = 0xffffffffffffffffull; +static constexpr uint64_t PLAYGO_SCENARIO_MASK_ALL = 0x1full; + +static uint32_t g_chunks_num = 0; + +struct PlayGoInitParams { + const void* buf_addr; + uint32_t buf_size; + uint32_t reserved; +}; + +struct PlayGoToDo { + uint16_t chunk_id; + int8_t locus; + int8_t reserved; +}; + +struct PlayGoProgress { + uint64_t progress_size; + uint64_t total_size; +}; + +union PlayGoOptionalChunk { + uint64_t bitmask; + uint64_t languages; + uint64_t scenarios; +}; + +static bool ensure_chunks_loaded() { + if (g_chunks_num != 0) { + return true; + } + if (!Loader::SystemContentGetChunksNum(&g_chunks_num)) { + LOGF("Warning: assume that chunks count is 1\n"); + g_chunks_num = 1; + } + return g_chunks_num != 0; +} + +static bool is_valid_chunk(uint16_t chunk_id) { + return ensure_chunks_loaded() && chunk_id < g_chunks_num; +} + +static bool is_valid_locus(int8_t locus) { + return locus == PLAYGO_LOCUS_NOT_DOWNLOADED || locus == PLAYGO_LOCUS_LOCAL_SLOW || + locus == PLAYGO_LOCUS_LOCAL_FAST; +} + +static int validate_handle(int handle) { + return (handle == PLAYGO_HANDLE ? OK : PLAYGO_ERROR_BAD_HANDLE); +} + +static int validate_optional_type(int32_t type) { + return (type == PLAYGO_OPTIONAL_TYPE_LANGUAGE || type == PLAYGO_OPTIONAL_TYPE_SCENARIO + ? OK + : PLAYGO_ERROR_BAD_OPTIONAL_TYPE); +} + +static void set_optional_chunk(int32_t type, PlayGoOptionalChunk* option) { + EXIT_IF(option == nullptr); + + option->bitmask = (type == PLAYGO_OPTIONAL_TYPE_LANGUAGE ? PLAYGO_LANGUAGE_MASK_ALL + : PLAYGO_SCENARIO_MASK_ALL); +} + +int KYTY_SYSV_ABI PlayGoInitialize(const PlayGoInitParams* init) { + PRINT_NAME(); + + if (init == nullptr || init->buf_addr == nullptr) { + return PLAYGO_ERROR_BAD_POINTER; + } + if (init->buf_size < 2u * 1024u * 1024u) { + return PLAYGO_ERROR_BAD_SIZE; + } + + LOGF("\t buf_addr = %016" PRIx64 "\n" + "\t buf_size = %" PRIu32 "\n" + "\t reserved = %" PRId32 "\n", + reinterpret_cast(init->buf_addr), init->buf_size, init->reserved); + + return OK; +} + +int KYTY_SYSV_ABI PlayGoTerminate() { + PRINT_NAME(); + + return OK; +} + +int KYTY_SYSV_ABI PlayGoOpen(int* out_handle, const void* param) { + PRINT_NAME(); + + if (out_handle == nullptr) { + return PLAYGO_ERROR_BAD_POINTER; + } + if (param != nullptr) { + return PLAYGO_ERROR_INVALID_ARGUMENT; + } + + *out_handle = PLAYGO_HANDLE; + + ensure_chunks_loaded(); + + return OK; +} + +int KYTY_SYSV_ABI PlayGoClose(int handle) { + PRINT_NAME(); + + if (auto ret = validate_handle(handle); ret != OK) { + return ret; + } + + return OK; +} + +int KYTY_SYSV_ABI PlayGoGetLocus(int handle, const uint16_t* chunk_ids, uint32_t number_of_entries, + int8_t* out_loci) { + PRINT_NAME(); + + LOGF("\t handle = %d\n", handle); + + if (auto ret = validate_handle(handle); ret != OK) { + return ret; + } + if (chunk_ids == nullptr || out_loci == nullptr) { + return PLAYGO_ERROR_BAD_POINTER; + } + if (number_of_entries == 0) { + return PLAYGO_ERROR_BAD_SIZE; + } + ensure_chunks_loaded(); + + for (uint32_t i = 0; i < number_of_entries; i++) { + LOGF("\t chunk_ids[%u] = %" PRIu16 "\n", i, chunk_ids[i]); + + if (is_valid_chunk(chunk_ids[i])) { + out_loci[i] = PLAYGO_LOCUS_LOCAL_FAST; + } else { + return PLAYGO_ERROR_BAD_CHUNK_ID; + } + } + + return OK; +} + +int KYTY_SYSV_ABI PlayGoGetInstallSpeed(int handle, int32_t* out_speed) { + PRINT_NAME(); + + LOGF("\t handle = %d\n", handle); + + if (auto ret = validate_handle(handle); ret != OK) { + return ret; + } + if (out_speed == nullptr) { + return PLAYGO_ERROR_BAD_POINTER; + } + + *out_speed = PLAYGO_INSTALL_SPEED_FULL; + + return OK; +} + +int KYTY_SYSV_ABI PlayGoSetInstallSpeed(int handle, int32_t speed) { + PRINT_NAME(); + + LOGF("\t handle = %d\n" + "\t speed = %" PRId32 "\n", + handle, speed); + + if (auto ret = validate_handle(handle); ret != OK) { + return ret; + } + if (speed < 0 || speed > 2) { + return PLAYGO_ERROR_BAD_SPEED; + } + + return OK; +} + +int KYTY_SYSV_ABI PlayGoGetChunkId(int handle, uint16_t* out_chunk_id_list, + uint32_t number_of_entries, uint32_t* out_entries) { + PRINT_NAME(); + + LOGF("\t handle = %d\n" + "\t out_chunk_id_list = 0x%016" PRIx64 "\n" + "\t number_of_entries = %" PRIu32 "\n" + "\t out_entries = 0x%016" PRIx64 "\n", + handle, reinterpret_cast(out_chunk_id_list), number_of_entries, + reinterpret_cast(out_entries)); + + if (auto ret = validate_handle(handle); ret != OK) { + return ret; + } + if (out_entries == nullptr) { + return PLAYGO_ERROR_BAD_POINTER; + } + if (number_of_entries != 0 && out_chunk_id_list == nullptr) { + return PLAYGO_ERROR_BAD_POINTER; + } + ensure_chunks_loaded(); + + const uint32_t entries = std::min(number_of_entries, g_chunks_num); + for (uint32_t i = 0; i < entries; i++) { + out_chunk_id_list[i] = static_cast(i); + } + *out_entries = entries; + + return OK; +} + +int KYTY_SYSV_ABI PlayGoGetToDoList(int handle, PlayGoToDo* out_todo_list, + uint32_t number_of_entries, uint32_t* out_entries) { + PRINT_NAME(); + + LOGF("\t handle = %d\n" + "\t out_todo_list = 0x%016" PRIx64 "\n" + "\t number_of_entries = %" PRIu32 "\n" + "\t out_entries = 0x%016" PRIx64 "\n", + handle, reinterpret_cast(out_todo_list), number_of_entries, + reinterpret_cast(out_entries)); + + if (auto ret = validate_handle(handle); ret != OK) { + return ret; + } + if (out_entries == nullptr) { + return PLAYGO_ERROR_BAD_POINTER; + } + if (number_of_entries != 0 && out_todo_list == nullptr) { + return PLAYGO_ERROR_BAD_POINTER; + } + *out_entries = 0; + + return OK; +} + +int KYTY_SYSV_ABI PlayGoSetToDoList(int handle, const PlayGoToDo* todo_list, + uint32_t number_of_entries) { + PRINT_NAME(); + + LOGF("\t handle = %d\n" + "\t todo_list = 0x%016" PRIx64 "\n" + "\t number_of_entries = %" PRIu32 "\n", + handle, reinterpret_cast(todo_list), number_of_entries); + + if (auto ret = validate_handle(handle); ret != OK) { + return ret; + } + if (todo_list == nullptr) { + return PLAYGO_ERROR_BAD_POINTER; + } + if (number_of_entries == 0) { + return PLAYGO_ERROR_BAD_SIZE; + } + for (uint32_t i = 0; i < number_of_entries; i++) { + if (!is_valid_chunk(todo_list[i].chunk_id)) { + return PLAYGO_ERROR_BAD_CHUNK_ID; + } + if (!is_valid_locus(todo_list[i].locus)) { + return PLAYGO_ERROR_BAD_LOCUS; + } + } + + return OK; +} + +int KYTY_SYSV_ABI PlayGoPrefetch(int handle, const uint16_t* chunk_ids, uint32_t number_of_entries, + int8_t minimum_locus) { + PRINT_NAME(); + + LOGF("\t handle = %d\n" + "\t chunk_ids = 0x%016" PRIx64 "\n" + "\t number_of_entries = %" PRIu32 "\n" + "\t minimum_locus = %" PRId8 "\n", + handle, reinterpret_cast(chunk_ids), number_of_entries, minimum_locus); + + if (auto ret = validate_handle(handle); ret != OK) { + return ret; + } + if (chunk_ids == nullptr) { + return PLAYGO_ERROR_BAD_POINTER; + } + if (number_of_entries == 0) { + return PLAYGO_ERROR_BAD_SIZE; + } + if (!is_valid_locus(minimum_locus)) { + return PLAYGO_ERROR_BAD_LOCUS; + } + for (uint32_t i = 0; i < number_of_entries; i++) { + if (!is_valid_chunk(chunk_ids[i])) { + return PLAYGO_ERROR_BAD_CHUNK_ID; + } + } + + return OK; +} + +int KYTY_SYSV_ABI PlayGoGetEta(int handle, const uint16_t* chunk_ids, uint32_t number_of_entries, + int64_t* out_eta) { + PRINT_NAME(); + + if (auto ret = validate_handle(handle); ret != OK) { + return ret; + } + if (chunk_ids == nullptr || out_eta == nullptr) { + return PLAYGO_ERROR_BAD_POINTER; + } + if (number_of_entries == 0) { + return PLAYGO_ERROR_BAD_SIZE; + } + for (uint32_t i = 0; i < number_of_entries; i++) { + if (!is_valid_chunk(chunk_ids[i])) { + return PLAYGO_ERROR_BAD_CHUNK_ID; + } + } + *out_eta = 0; + + return OK; +} + +int KYTY_SYSV_ABI PlayGoGetLanguageMask(int handle, uint64_t* out_language_mask) { + PRINT_NAME(); + + if (auto ret = validate_handle(handle); ret != OK) { + return ret; + } + if (out_language_mask == nullptr) { + return PLAYGO_ERROR_BAD_POINTER; + } + + *out_language_mask = PLAYGO_LANGUAGE_MASK_ALL; + + return OK; +} + +int KYTY_SYSV_ABI PlayGoGetProgress(int handle, const uint16_t* chunk_ids, + uint32_t number_of_entries, PlayGoProgress* out_progress) { + PRINT_NAME(); + + if (auto ret = validate_handle(handle); ret != OK) { + return ret; + } + if (chunk_ids == nullptr || out_progress == nullptr) { + return PLAYGO_ERROR_BAD_POINTER; + } + if (number_of_entries == 0) { + return PLAYGO_ERROR_BAD_SIZE; + } + for (uint32_t i = 0; i < number_of_entries; i++) { + if (!is_valid_chunk(chunk_ids[i])) { + return PLAYGO_ERROR_BAD_CHUNK_ID; + } + } + + out_progress->total_size = number_of_entries; + out_progress->progress_size = number_of_entries; + + return OK; +} + +int KYTY_SYSV_ABI PlayGoGetOptionalChunk(int handle, int32_t type, PlayGoOptionalChunk* option) { + PRINT_NAME(); + + if (auto ret = validate_handle(handle); ret != OK) { + return ret; + } + if (option == nullptr) { + return PLAYGO_ERROR_BAD_POINTER; + } + if (auto ret = validate_optional_type(type); ret != OK) { + return ret; + } + + set_optional_chunk(type, option); + + return OK; +} + +int KYTY_SYSV_ABI PlayGoPrefetchOptionalChunk(int handle, int32_t type, + const PlayGoOptionalChunk* option) { + PRINT_NAME(); + + if (auto ret = validate_handle(handle); ret != OK) { + return ret; + } + if (option == nullptr) { + return PLAYGO_ERROR_BAD_POINTER; + } + if (auto ret = validate_optional_type(type); ret != OK) { + return ret; + } + + return OK; +} + +int KYTY_SYSV_ABI PlayGoGetInstallChunkId(int handle, uint16_t* out_chunk_id_list, + uint32_t number_of_entries, uint32_t* out_entries) { + PRINT_NAME(); + + return PlayGoGetChunkId(handle, out_chunk_id_list, number_of_entries, out_entries); +} + +int KYTY_SYSV_ABI PlayGoGetSupportedOptionalChunk(int handle, int32_t type, + PlayGoOptionalChunk* option) { + PRINT_NAME(); + + return PlayGoGetOptionalChunk(handle, type, option); +} + +} // namespace PlayGo + +LIB_DEFINE(InitPlayGo_1) { + LIB_FUNC("ts6GlZOKRrE", PlayGo::PlayGoInitialize); + LIB_FUNC("MPe0EeBGM-E", PlayGo::PlayGoTerminate); + LIB_FUNC("M1Gma1ocrGE", PlayGo::PlayGoOpen); + LIB_FUNC("Uco1I0dlDi8", PlayGo::PlayGoClose); + LIB_FUNC("uWIYLFkkwqk", PlayGo::PlayGoGetLocus); + LIB_FUNC("rvBSfTimejE", PlayGo::PlayGoGetInstallSpeed); + LIB_FUNC("4AAcTU9R3XM", PlayGo::PlayGoSetInstallSpeed); + LIB_FUNC("73fF1MFU8hA", PlayGo::PlayGoGetChunkId); + LIB_FUNC("Nn7zKwnA5q0", PlayGo::PlayGoGetToDoList); + LIB_FUNC("gUPGiOQ1tmQ", PlayGo::PlayGoSetToDoList); + LIB_FUNC("-Q1-u1a7p0g", PlayGo::PlayGoPrefetch); + LIB_FUNC("v6EZ-YWRdMs", PlayGo::PlayGoGetEta); + LIB_FUNC("3OMbYZBaa50", PlayGo::PlayGoGetLanguageMask); + LIB_FUNC("-RJWNMK3fC8", PlayGo::PlayGoGetProgress); + LIB_FUNC("g4AZyxpSAlA", PlayGo::PlayGoGetOptionalChunk); + LIB_FUNC("HVAa744ecdw", PlayGo::PlayGoPrefetchOptionalChunk); + LIB_FUNC("8-e7E989rCU", PlayGo::PlayGoGetInstallChunkId); + LIB_FUNC("IfiN+-oeVWI", PlayGo::PlayGoGetSupportedOptionalChunk); +} + +} // namespace Libs diff --git a/src/libs/libPngDec.cpp b/src/libs/libPngDec.cpp new file mode 100644 index 000000000..2582ee8fa --- /dev/null +++ b/src/libs/libPngDec.cpp @@ -0,0 +1,381 @@ +#include "common/abi.h" +#include "common/logging/log.h" +#include "libs/libs.h" +#include "loader/symbolDatabase.h" +#include "stb_image.h" + +#include +#include +#include + +namespace Libs { + +LIB_VERSION("PngDec", 1, "PngDec", 1, 1); + +namespace PngDec { + +constexpr int32_t PNG_DEC_ERROR_INVALID_ADDR = -2140602367; // 0x80690001 +constexpr int32_t PNG_DEC_ERROR_INVALID_SIZE = -2140602366; // 0x80690002 +constexpr int32_t PNG_DEC_ERROR_INVALID_PARAM = -2140602365; // 0x80690003 +constexpr int32_t PNG_DEC_ERROR_INVALID_HANDLE = -2140602364; // 0x80690004 +constexpr int32_t PNG_DEC_ERROR_INVALID_WORK_MEMORY = -2140602363; // 0x80690005 +constexpr int32_t PNG_DEC_ERROR_INVALID_DATA = -2140602352; // 0x80690010 +constexpr int32_t PNG_DEC_ERROR_DECODE_ERROR = -2140602350; // 0x80690012 + +constexpr uint32_t PNG_DEC_ATTRIBUTE_BIT_DEPTH_16 = 1; +constexpr uint16_t PNG_DEC_COLOR_SPACE_GRAYSCALE = 2; +constexpr uint16_t PNG_DEC_COLOR_SPACE_RGB = 3; +constexpr uint16_t PNG_DEC_COLOR_SPACE_CLUT = 4; +constexpr uint16_t PNG_DEC_COLOR_SPACE_GRAYSCALE_ALPHA = 18; +constexpr uint16_t PNG_DEC_COLOR_SPACE_RGBA = 19; +constexpr uint16_t PNG_DEC_PIXEL_FORMAT_R8G8B8A8 = 0; +constexpr uint16_t PNG_DEC_PIXEL_FORMAT_B8G8R8A8 = 1; +constexpr uint32_t PNG_DEC_IMAGE_FLAG_ADAM7_INTERLACE = 1; +constexpr uint32_t PNG_DEC_IMAGE_FLAG_TRNS_CHUNK_EXIST = 2; + +struct PngDecCreateParam { + uint32_t this_size; + uint32_t attribute; + uint32_t max_image_width; +}; + +struct PngDecDecodeParam { + const void* png_mem_addr; + void* image_mem_addr; + uint32_t png_mem_size; + uint32_t image_mem_size; + uint16_t pixel_format; + uint16_t alpha_value; + uint32_t image_pitch; +}; + +struct PngDecParseParam { + const void* png_mem_addr; + uint32_t png_mem_size; + uint32_t reserved0; +}; + +struct PngDecImageInfo { + uint32_t image_width; + uint32_t image_height; + uint16_t color_space; + uint16_t bit_depth; + uint32_t image_flag; +}; + +struct PngDecContext { + uint64_t magic; +}; + +struct PngHeaderInfo { + uint32_t width; + uint32_t height; + uint16_t color_space; + uint16_t bit_depth; + uint32_t image_flag; +}; + +constexpr uint64_t PNG_DEC_CONTEXT_MAGIC = 0x4b595459504e4744ull; // KYTYPNGD + +static uint32_t ReadBe32(const uint8_t* ptr) { + return (static_cast(ptr[0]) << 24u) | (static_cast(ptr[1]) << 16u) | + (static_cast(ptr[2]) << 8u) | static_cast(ptr[3]); +} + +static bool AddOverflow(uint32_t a, uint32_t b, uint32_t* out) { + *out = a + b; + return *out < a; +} + +static bool ParsePngHeader(const void* data, uint32_t size, PngHeaderInfo* info) { + if (data == nullptr || info == nullptr) { + return false; + } + + const auto* bytes = static_cast(data); + constexpr uint8_t png_signature[] = {0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a}; + + if (size < 33u || std::memcmp(bytes, png_signature, sizeof(png_signature)) != 0) { + return false; + } + + if (ReadBe32(bytes + 8u) != 13u || std::memcmp(bytes + 12u, "IHDR", 4u) != 0) { + return false; + } + + const auto color_type = bytes[25u]; + + switch (color_type) { + case 0: info->color_space = PNG_DEC_COLOR_SPACE_GRAYSCALE; break; + case 2: info->color_space = PNG_DEC_COLOR_SPACE_RGB; break; + case 3: info->color_space = PNG_DEC_COLOR_SPACE_CLUT; break; + case 4: info->color_space = PNG_DEC_COLOR_SPACE_GRAYSCALE_ALPHA; break; + case 6: info->color_space = PNG_DEC_COLOR_SPACE_RGBA; break; + default: return false; + } + + info->width = ReadBe32(bytes + 16u); + info->height = ReadBe32(bytes + 20u); + info->bit_depth = bytes[24u]; + info->image_flag = (bytes[28u] == 1u ? PNG_DEC_IMAGE_FLAG_ADAM7_INTERLACE : 0u); + + if (info->width == 0 || info->height == 0) { + return false; + } + + uint32_t offset = 33u; + while (offset + 12u <= size) { + const uint32_t length = ReadBe32(bytes + offset); + const auto* type = bytes + offset + 4u; + + uint32_t chunk_end = 0; + if (AddOverflow(offset, 12u, &chunk_end) || AddOverflow(chunk_end, length, &chunk_end) || + chunk_end > size) { + return false; + } + + if (std::memcmp(type, "tRNS", 4u) == 0) { + info->image_flag |= PNG_DEC_IMAGE_FLAG_TRNS_CHUNK_EXIST; + } + + if (std::memcmp(type, "IDAT", 4u) == 0 || std::memcmp(type, "IEND", 4u) == 0) { + break; + } + + offset = chunk_end; + } + + return true; +} + +static void FillImageInfo(const PngHeaderInfo& header, PngDecImageInfo* image_info) { + if (image_info != nullptr) { + image_info->image_width = header.width; + image_info->image_height = header.height; + image_info->color_space = header.color_space; + image_info->bit_depth = header.bit_depth; + image_info->image_flag = header.image_flag; + } +} + +static bool PngHasSourceAlpha(const PngHeaderInfo& header) { + return header.color_space == PNG_DEC_COLOR_SPACE_RGBA || + header.color_space == PNG_DEC_COLOR_SPACE_GRAYSCALE_ALPHA || + (header.image_flag & PNG_DEC_IMAGE_FLAG_TRNS_CHUNK_EXIST) != 0; +} + +static int32_t KYTY_SYSV_ABI PngDecQueryMemorySize(const PngDecCreateParam* param) { + PRINT_NAME(); + + if (param == nullptr) { + return PNG_DEC_ERROR_INVALID_PARAM; + } + + LOGF("\t this_size = %" PRIu32 "\n", param->this_size); + LOGF("\t attribute = %" PRIu32 "\n", param->attribute); + LOGF("\t max_image_width = %" PRIu32 "\n", param->max_image_width); + + if (param->attribute > PNG_DEC_ATTRIBUTE_BIT_DEPTH_16) { + return PNG_DEC_ERROR_INVALID_PARAM; + } + + if (param->max_image_width == 0) { + return PNG_DEC_ERROR_INVALID_SIZE; + } + + return sizeof(PngDecContext); +} + +static int32_t KYTY_SYSV_ABI PngDecCreate(const PngDecCreateParam* param, void* memory_address, + uint32_t memory_size, void** handle) { + PRINT_NAME(); + + if (param == nullptr || handle == nullptr) { + return PNG_DEC_ERROR_INVALID_PARAM; + } + + LOGF("\t memory_address = %p\n", memory_address); + LOGF("\t memory_size = %" PRIu32 "\n", memory_size); + + if (param->attribute > PNG_DEC_ATTRIBUTE_BIT_DEPTH_16) { + return PNG_DEC_ERROR_INVALID_PARAM; + } + + if (param->max_image_width == 0) { + return PNG_DEC_ERROR_INVALID_SIZE; + } + + if (memory_address == nullptr) { + return PNG_DEC_ERROR_INVALID_ADDR; + } + + if (memory_size < sizeof(PngDecContext)) { + return PNG_DEC_ERROR_INVALID_WORK_MEMORY; + } + + auto* ctx = static_cast(memory_address); + ctx->magic = PNG_DEC_CONTEXT_MAGIC; + *handle = ctx; + + return 0; +} + +static int32_t KYTY_SYSV_ABI PngDecDecode(void* handle, const PngDecDecodeParam* param, + PngDecImageInfo* image_info) { + PRINT_NAME(); + + if (handle == nullptr) { + return PNG_DEC_ERROR_INVALID_HANDLE; + } + + const auto* ctx = static_cast(handle); + if (ctx->magic != PNG_DEC_CONTEXT_MAGIC) { + return PNG_DEC_ERROR_INVALID_HANDLE; + } + + if (param == nullptr) { + return PNG_DEC_ERROR_INVALID_PARAM; + } + + if (param->png_mem_addr == nullptr || param->image_mem_addr == nullptr) { + return PNG_DEC_ERROR_INVALID_ADDR; + } + + if (param->pixel_format != PNG_DEC_PIXEL_FORMAT_R8G8B8A8 && + param->pixel_format != PNG_DEC_PIXEL_FORMAT_B8G8R8A8) { + return PNG_DEC_ERROR_INVALID_PARAM; + } + + PngHeaderInfo header {}; + if (!ParsePngHeader(param->png_mem_addr, param->png_mem_size, &header)) { + return PNG_DEC_ERROR_INVALID_DATA; + } + + FillImageInfo(header, image_info); + + const uint32_t min_pitch = header.width * 4u; + const uint32_t pitch = (param->image_pitch == 0 ? min_pitch : param->image_pitch); + const uint64_t min_size = static_cast(pitch) * (header.height - 1u) + min_pitch; + + if (pitch < min_pitch || min_size > param->image_mem_size) { + return PNG_DEC_ERROR_INVALID_SIZE; + } + + LOGF("\t png_mem_addr = %p\n", param->png_mem_addr); + LOGF("\t image_mem_addr = %p\n", param->image_mem_addr); + LOGF("\t png_mem_size = %" PRIu32 "\n", param->png_mem_size); + LOGF("\t image_mem_size = %" PRIu32 "\n", param->image_mem_size); + LOGF("\t pixel_format = %" PRIu16 "\n", param->pixel_format); + LOGF("\t alpha_value = %" PRIu16 "\n", param->alpha_value); + LOGF("\t image_pitch = %" PRIu32 "\n", param->image_pitch); + + int width = 0; + int height = 0; + int components = 0; + auto* rgba = stbi_load_from_memory(static_cast(param->png_mem_addr), + static_cast(param->png_mem_size), &width, &height, + &components, 4); + + if (rgba == nullptr || width <= 0 || height <= 0 || + static_cast(width) != header.width || + static_cast(height) != header.height) { + if (rgba != nullptr) { + stbi_image_free(rgba); + } + return PNG_DEC_ERROR_DECODE_ERROR; + } + + const bool apply_alpha_value = !PngHasSourceAlpha(header); + auto* dst_row = static_cast(param->image_mem_addr); + + for (uint32_t y = 0; y < header.height; y++) { + const auto* src = rgba + static_cast(y) * min_pitch; + auto* dst = dst_row; + + for (uint32_t x = 0; x < header.width; x++) { + dst[0] = src[0]; + dst[1] = src[1]; + dst[2] = src[2]; + dst[3] = (apply_alpha_value + ? static_cast(std::min(param->alpha_value, 255u)) + : src[3]); + + if (param->pixel_format == PNG_DEC_PIXEL_FORMAT_B8G8R8A8) { + std::swap(dst[0], dst[2]); + } + + src += 4; + dst += 4; + } + + dst_row += pitch; + } + + stbi_image_free(rgba); + + return (header.width > 32767u || header.height > 32767u + ? 0 + : static_cast((header.width << 16u) | header.height)); +} + +static int32_t KYTY_SYSV_ABI PngDecDelete(void* handle) { + PRINT_NAME(); + + if (handle == nullptr) { + return PNG_DEC_ERROR_INVALID_HANDLE; + } + + auto* ctx = static_cast(handle); + if (ctx->magic != PNG_DEC_CONTEXT_MAGIC) { + return PNG_DEC_ERROR_INVALID_HANDLE; + } + + ctx->magic = 0; + return 0; +} + +static int32_t KYTY_SYSV_ABI PngDecParseHeader(const PngDecParseParam* param, + PngDecImageInfo* image_info) { + PRINT_NAME(); + + if (param == nullptr || image_info == nullptr) { + return PNG_DEC_ERROR_INVALID_PARAM; + } + + if (param->png_mem_addr == nullptr) { + return PNG_DEC_ERROR_INVALID_ADDR; + } + + if (param->reserved0 != 0) { + return PNG_DEC_ERROR_INVALID_PARAM; + } + + PngHeaderInfo header {}; + if (!ParsePngHeader(param->png_mem_addr, param->png_mem_size, &header)) { + return PNG_DEC_ERROR_INVALID_DATA; + } + + FillImageInfo(header, image_info); + + LOGF("\t image_width = %" PRIu32 "\n", image_info->image_width); + LOGF("\t image_height = %" PRIu32 "\n", image_info->image_height); + LOGF("\t color_space = %" PRIu16 "\n", image_info->color_space); + LOGF("\t bit_depth = %" PRIu16 "\n", image_info->bit_depth); + LOGF("\t image_flag = 0x%08" PRIx32 "\n", image_info->image_flag); + + return 0; +} + +} // namespace PngDec + +LIB_DEFINE(InitPngDec_1) { + PRINT_NAME_ENABLE(true); + + LIB_FUNC("-6srIGbLTIU", PngDec::PngDecQueryMemorySize); + LIB_FUNC("m0uW+8pFyaw", PngDec::PngDecCreate); + LIB_FUNC("WC216DD3El4", PngDec::PngDecDecode); + LIB_FUNC("QbD+eENEwo8", PngDec::PngDecDelete); + LIB_FUNC("U6h4e5JRPaQ", PngDec::PngDecParseHeader); +} + +} // namespace Libs diff --git a/src/libs/libPsml.cpp b/src/libs/libPsml.cpp new file mode 100644 index 000000000..6c9ef71be --- /dev/null +++ b/src/libs/libPsml.cpp @@ -0,0 +1,353 @@ +#include "libs/errno.h" +#include "libs/libs.h" +#include "loader/symbolDatabase.h" + +#include +#include + +namespace Libs { + +namespace LibPsml { + +LIB_VERSION("Psml", 1, "Psml", 1, 1); + +namespace Psml { + +constexpr int Error(uint32_t value) { + return static_cast(value); +} + +constexpr int PSML_ERROR_NOT_INITIALIZED = Error(0x8a810001u); +constexpr int PSML_ERROR_INVALID_OBJECT = Error(0x8a810005u); +constexpr int PSML_ERROR_INVALID_POINTER = Error(0x8a810009u); +constexpr int PSML_ERROR_INVALID_VALUE = Error(0x8a81000du); +constexpr int PSML_ERROR_NULL_OBJECT = Error(0x8a810014u); + +constexpr uint32_t SHARED_RESOURCES_MAGIC = 0xa9c4; +constexpr uint32_t CONTEXT_MAGIC = 0x9231; + +constexpr uint64_t MAIN_MEMORY_BLOCK_SIZE = 0x200000; +constexpr uint64_t MAIN_MEMORY_ALIGNMENT = 0x200000; +constexpr uint64_t EXTRA_VA_BYTES = 0x600000; + +struct MainMemoryRequirements { + uint64_t block_size; + uint64_t alignment; + uint64_t block_count; +}; + +struct MainMemoryParameters { + uint32_t type; + uint32_t reserved; +}; + +struct DirectMemoryBlock { + uint64_t address; + uint64_t size; +}; + +struct SharedResourcesInitParameters { + uint32_t type; + uint32_t reserved; + const DirectMemoryBlock* blocks; + uint64_t block_count; + uint64_t virtual_address_start; +}; + +static_assert(sizeof(MainMemoryRequirements) == 24); +static_assert(sizeof(DirectMemoryBlock) == 16); + +static bool g_initialized = false; +static uint32_t g_supported_modes = 0x3; +static std::atomic g_shared_ref_count = 0; + +static uint64_t BaseVaBytes(uint32_t type) { + // The native implementation indexes this table before adding one 6 MiB range. + switch (type) { + case 0: return 0x06200000; + case 1: return 0x18200000; + case 2: return 0x12200000; + default: return 0; + } +} + +static bool IsSupportedType(uint32_t type) { + switch (type) { + case 0: return true; + case 1: return (g_supported_modes & 0x1u) != 0; + case 2: return (g_supported_modes & 0x2u) != 0; + default: return false; + } +} + +static uint64_t RequiredBlockCount(uint32_t type) { + const auto base = BaseVaBytes(type); + if (base == 0) { + return 0; + } + return (base + EXTRA_VA_BYTES) / MAIN_MEMORY_BLOCK_SIZE; +} + +static int CheckInitialized() { + return (g_initialized ? OK : PSML_ERROR_NOT_INITIALIZED); +} + +static void MarkSharedResources(void* resources, const SharedResourcesInitParameters& params) { + auto* bytes = static_cast(resources); + + *reinterpret_cast(bytes) = SHARED_RESOURCES_MAGIC; + *reinterpret_cast(bytes + 0x08) = params.blocks; + *reinterpret_cast(bytes + 0x18) = params.block_count; + *reinterpret_cast(bytes + 0x20) = params.type; + *reinterpret_cast(bytes + 0x28) = params.virtual_address_start; +} + +static bool HasKnownObjectMagic(const void* object) { + if (object == nullptr) { + return false; + } + const auto magic = *static_cast(object); + return magic == SHARED_RESOURCES_MAGIC || magic == CONTEXT_MAGIC; +} + +static int KYTY_SYSV_ABI PsmlInitialize() { + PRINT_NAME(); + + g_initialized = true; + g_supported_modes = 0x3; + g_shared_ref_count = 0; + + return OK; +} + +static int KYTY_SYSV_ABI PsmlGetMainMemoryRequirements(MainMemoryRequirements* out, + const MainMemoryParameters* params) { + PRINT_NAME(); + + const auto init_error = CheckInitialized(); + if (init_error != OK) { + return init_error; + } + if (out == nullptr || params == nullptr) { + return PSML_ERROR_INVALID_POINTER; + } + if (!IsSupportedType(params->type)) { + return PSML_ERROR_INVALID_VALUE; + } + + out->block_size = MAIN_MEMORY_BLOCK_SIZE; + out->alignment = MAIN_MEMORY_ALIGNMENT; + out->block_count = RequiredBlockCount(params->type); + + return OK; +} + +static int KYTY_SYSV_ABI +PsmlSharedResourcesInitialize(void* resources, const SharedResourcesInitParameters* params) { + PRINT_NAME(); + + const auto init_error = CheckInitialized(); + if (init_error != OK) { + return init_error; + } + if (resources == nullptr || params == nullptr || params->blocks == nullptr) { + return PSML_ERROR_INVALID_POINTER; + } + if (params->reserved != 0 || !IsSupportedType(params->type) || + params->block_count < RequiredBlockCount(params->type)) { + return PSML_ERROR_INVALID_VALUE; + } + + MarkSharedResources(resources, *params); + g_shared_ref_count.fetch_add(1, std::memory_order_relaxed); + + return OK; +} + +static int KYTY_SYSV_ABI PsmlSharedResourcesFinalize(void* resources) { + PRINT_NAME(); + + const auto init_error = CheckInitialized(); + if (init_error != OK) { + return init_error; + } + if (resources == nullptr) { + return PSML_ERROR_NULL_OBJECT; + } + + g_shared_ref_count.fetch_sub(1, std::memory_order_relaxed); + + return OK; +} + +static int KYTY_SYSV_ABI PsmlGetContextMemoryRequirements(MainMemoryRequirements* out, + uint64_t /*params*/) { + PRINT_NAME(); + + const auto init_error = CheckInitialized(); + if (init_error != OK) { + return init_error; + } + if (out == nullptr) { + return PSML_ERROR_INVALID_POINTER; + } + + out->block_size = MAIN_MEMORY_BLOCK_SIZE; + out->alignment = MAIN_MEMORY_ALIGNMENT; + out->block_count = 1; + + return OK; +} + +static int KYTY_SYSV_ABI PsmlContextInitialize(void* context, const void* params) { + PRINT_NAME(); + + const auto init_error = CheckInitialized(); + if (init_error != OK) { + return init_error; + } + if (context == nullptr || params == nullptr) { + return PSML_ERROR_INVALID_POINTER; + } + + auto* bytes = static_cast(context); + auto* params_bytes = static_cast(params); + auto* shared_resources = *reinterpret_cast(params_bytes + 0x08); + if (shared_resources == nullptr) { + return PSML_ERROR_INVALID_POINTER; + } + + *reinterpret_cast(bytes) = CONTEXT_MAGIC; + *reinterpret_cast(bytes + 0x360) = shared_resources; + *reinterpret_cast(bytes + 0x368) = 0; + + return OK; +} + +static int KYTY_SYSV_ABI PsmlContextFinalize(void* context) { + PRINT_NAME(); + + const auto init_error = CheckInitialized(); + if (init_error != OK) { + return init_error; + } + if (!HasKnownObjectMagic(context)) { + return PSML_ERROR_INVALID_OBJECT; + } + + *static_cast(context) = 0; + + return OK; +} + +static int KYTY_SYSV_ABI PsmlGetWorkAreaSize(const void* object, uint32_t* out_size) { + PRINT_NAME(); + + const auto init_error = CheckInitialized(); + if (init_error != OK) { + return init_error; + } + if (!HasKnownObjectMagic(object)) { + return PSML_ERROR_INVALID_OBJECT; + } + if (out_size == nullptr) { + return PSML_ERROR_INVALID_POINTER; + } + + *out_size = 0x600; + + return OK; +} + +static int KYTY_SYSV_ABI PsmlDispatch(void* context, uint64_t command, uint64_t params) { + PRINT_NAME(); + + const auto init_error = CheckInitialized(); + if (init_error != OK) { + return init_error; + } + if (!HasKnownObjectMagic(context)) { + return PSML_ERROR_INVALID_OBJECT; + } + if (command == 0 || params == 0) { + return PSML_ERROR_INVALID_POINTER; + } + + return OK; +} + +static int KYTY_SYSV_ABI PsmlGetProgress(const void* object, float* out_progress, + uint32_t /*frame_index*/) { + PRINT_NAME(); + + const auto init_error = CheckInitialized(); + if (init_error != OK) { + return init_error; + } + if (!HasKnownObjectMagic(object)) { + return PSML_ERROR_INVALID_OBJECT; + } + if (out_progress == nullptr) { + return PSML_ERROR_INVALID_POINTER; + } + + *out_progress = 0.0f; + + return OK; +} + +static int KYTY_SYSV_ABI PsmlRequestCapture(const void* object) { + PRINT_NAME(); + + const auto init_error = CheckInitialized(); + if (init_error != OK) { + return init_error; + } + if (!HasKnownObjectMagic(object)) { + return PSML_ERROR_INVALID_OBJECT; + } + + return OK; +} + +static int KYTY_SYSV_ABI PsmlValidateObject(const void* object) { + PRINT_NAME(); + + const auto init_error = CheckInitialized(); + if (init_error != OK) { + return init_error; + } + if (!HasKnownObjectMagic(object)) { + return PSML_ERROR_INVALID_OBJECT; + } + + return OK; +} + +} // namespace Psml + +LIB_DEFINE(InitPsml_1) { + LIB_FUNC("3WVD91e12ZQ", Psml::PsmlInitialize); + LIB_FUNC("+2KpvixvL6E", Psml::PsmlGetMainMemoryRequirements); + LIB_FUNC("eWoKNeB6V-k", Psml::PsmlSharedResourcesInitialize); + LIB_FUNC("jEevBXmagOQ", Psml::PsmlSharedResourcesFinalize); + LIB_FUNC("EO9YQXmJEN8", Psml::PsmlGetContextMemoryRequirements); + LIB_FUNC("IMlj247LdTo", Psml::PsmlGetContextMemoryRequirements); + LIB_FUNC("sjTyYheKTrU", Psml::PsmlGetContextMemoryRequirements); + LIB_FUNC("ArakEpzsZo0", Psml::PsmlGetContextMemoryRequirements); + LIB_FUNC("VGjrQa-WqdU", Psml::PsmlGetContextMemoryRequirements); + LIB_FUNC("2ecEbQaf9VU", Psml::PsmlContextInitialize); + LIB_FUNC("vUk2pWMx3KQ", Psml::PsmlContextInitialize); + LIB_FUNC("gxv3i+MTEzU", Psml::PsmlContextInitialize); + LIB_FUNC("JaLBe0P3jSU", Psml::PsmlContextFinalize); + LIB_FUNC("AHalTX9wFZY", Psml::PsmlGetWorkAreaSize); + LIB_FUNC("RUNLFro+qok", Psml::PsmlDispatch); + LIB_FUNC("GHna9-DvnUk", Psml::PsmlGetProgress); + LIB_FUNC("GJY0MvuTcs8", Psml::PsmlRequestCapture); + LIB_FUNC("LXq+6mIxpCw", Psml::PsmlValidateObject); + LIB_FUNC("FSGaTQze0UY", Psml::PsmlValidateObject); +} + +} // namespace LibPsml + +} // namespace Libs diff --git a/src/libs/libRtc.cpp b/src/libs/libRtc.cpp new file mode 100644 index 000000000..dc55f6654 --- /dev/null +++ b/src/libs/libRtc.cpp @@ -0,0 +1,594 @@ +#include "common/abi.h" +#include "common/dateTime.h" +#include "libs/errno.h" +#include "libs/libs.h" +#include "loader/symbolDatabase.h" + +#include +#include +#include + +namespace Libs { + +namespace LibRtc { + +LIB_VERSION("Rtc", 1, "Rtc", 1, 1); + +namespace Rtc { + +constexpr int RTC_ERROR_DATETIME_UNINITIALIZED = 0x7ffef9fe; +constexpr int RTC_ERROR_INVALID_POINTER = -2135621630; /* 0x80B50002 */ +constexpr int RTC_ERROR_INVALID_VALUE = -2135621629; /* 0x80B50003 */ +constexpr int RTC_ERROR_INVALID_YEAR = -2135621624; /* 0x80B50008 */ +constexpr int RTC_ERROR_INVALID_MONTH = -2135621623; /* 0x80B50009 */ +constexpr int RTC_ERROR_INVALID_DAY = -2135621622; /* 0x80B5000A */ +constexpr int RTC_ERROR_INVALID_HOUR = -2135621621; /* 0x80B5000B */ +constexpr int RTC_ERROR_INVALID_MINUTE = -2135621620; /* 0x80B5000C */ +constexpr int RTC_ERROR_INVALID_SECOND = -2135621619; /* 0x80B5000D */ +constexpr int RTC_ERROR_INVALID_MICROSECOND = -2135621618; /* 0x80B5000E */ + +constexpr uint64_t RTC_UNIX_EPOCH_TICKS = 0xdcbffeff2bc000ull; +constexpr uint64_t RTC_WIN32_FILETIME_EPOCH_TICKS = 0xb36168b6a58000ull; + +struct RtcDateTime { + uint16_t year; + uint16_t month; + uint16_t day; + uint16_t hour; + uint16_t minute; + uint16_t second; + uint32_t microsecond; +}; + +static_assert(sizeof(RtcDateTime) == 16); + +struct RtcTick { + uint64_t tick; +}; + +static_assert(sizeof(RtcTick) == 8); + +static int KYTY_SYSV_ABI RtcCheckValid(const RtcDateTime* time) { + if (time == nullptr) { + return RTC_ERROR_INVALID_POINTER; + } + if (time->year == 0 || time->year > 9999) { + return RTC_ERROR_INVALID_YEAR; + } + if (time->month == 0 || time->month > 12) { + return RTC_ERROR_INVALID_MONTH; + } + if (time->day == 0 || !Common::Date::IsValid(time->year, time->month, time->day)) { + return RTC_ERROR_INVALID_DAY; + } + if (time->hour >= 24) { + return RTC_ERROR_INVALID_HOUR; + } + if (time->minute >= 60) { + return RTC_ERROR_INVALID_MINUTE; + } + if (time->second >= 60) { + return RTC_ERROR_INVALID_SECOND; + } + if (time->microsecond >= 1000000) { + return RTC_ERROR_INVALID_MICROSECOND; + } + + return OK; +} + +static int RtcGetDaysInMonth(int year, int month) { + if (year <= 0 || year > 9999) { + return RTC_ERROR_INVALID_YEAR; + } + if (month <= 0 || month > 12) { + return RTC_ERROR_INVALID_MONTH; + } + + if (month == 2 && Common::Date::IsLeapYear(year)) { + return 29; + } + + return Common::Date::DaysInMonth(month); +} + +static int KYTY_SYSV_ABI RtcIsLeapYear(int year) { + PRINT_NAME(); + + if (year <= 0 || year > 9999) { + return RTC_ERROR_INVALID_YEAR; + } + + return Common::Date::IsLeapYear(year) ? 1 : 0; +} + +static int KYTY_SYSV_ABI RtcGetDayOfWeek(int year, int month, int day) { + PRINT_NAME(); + + const auto days = RtcGetDaysInMonth(year, month); + if (days < 0) { + return days; + } + if (day <= 0 || day > days) { + return RTC_ERROR_INVALID_DAY; + } + + const auto dow = Common::Date(year, month, day).DayOfWeek(); + return (dow == 7 ? 0 : dow); +} + +static int KYTY_SYSV_ABI RtcGetDaysInMonthExport(int year, int month) { + PRINT_NAME(); + + return RtcGetDaysInMonth(year, month); +} + +static int KYTY_SYSV_ABI RtcGetTickResolution() { + PRINT_NAME(); + + return 1000000; +} + +static int KYTY_SYSV_ABI RtcGetCurrentClockLocalTime(RtcDateTime* time) { + PRINT_NAME(); + + if (time == nullptr) { + return RTC_ERROR_DATETIME_UNINITIALIZED; + } + + const auto now = Common::DateTime::FromSystem(); + const auto date = now.GetDate(); + const auto tod = now.GetTime(); + + time->year = static_cast(date.Year()); + time->month = static_cast(date.Month()); + time->day = static_cast(date.Day()); + time->hour = static_cast(tod.Hour24()); + time->minute = static_cast(tod.Minute()); + time->second = static_cast(tod.Second()); + time->microsecond = static_cast(tod.Msec() * 1000); + + return OK; +} + +static int KYTY_SYSV_ABI RtcSetTick(RtcDateTime* time, const RtcTick* tick) { + PRINT_NAME(); + + if (time == nullptr || tick == nullptr) { + return RTC_ERROR_INVALID_POINTER; + } + + auto days = tick->tick / 86400000000ull; + auto usec = tick->tick % 86400000000ull; + + days += 307; + + auto j = (days << 2) - 1; + auto ly = j / 146097; + + j -= 146097 * ly; + auto ld = j >> 2; + + j = ((ld << 2) + 3) / 1461; + ld = (((ld << 2) + 7) - 1461 * j) >> 2; + + auto lm = (5 * ld - 3) / 153; + ld = (5 * ld + 2 - 153 * lm) / 5; + ly = 100 * ly + j; + + if (lm < 10) { + lm += 3; + } else { + lm -= 9; + ly++; + } + + if (ly == 0 || ly > 9999) { + return RTC_ERROR_INVALID_VALUE; + } + + time->year = static_cast(ly); + time->month = static_cast(lm); + time->day = static_cast(ld); + + time->hour = static_cast(usec / 3600000000ull); + usec %= 3600000000ull; + time->minute = static_cast(usec / 60000000ull); + usec %= 60000000ull; + time->second = static_cast(usec / 1000000ull); + usec %= 1000000ull; + time->microsecond = static_cast(usec); + + return RtcCheckValid(time); +} + +static int KYTY_SYSV_ABI RtcGetTick(const RtcDateTime* time, RtcTick* tick) { + PRINT_NAME(); + + if (time == nullptr || tick == nullptr) { + return RTC_ERROR_INVALID_POINTER; + } + + const auto valid = RtcCheckValid(time); + if (valid != OK) { + return valid; + } + + uint64_t year = time->year; + uint64_t month = time->month; + + if (month > 2) { + month -= 3; + } else { + month += 9; + year -= 1; + } + + const auto c = year / 100; + const auto ya = year - 100 * c; + + auto days = ((146097 * c) >> 2) + ((1461 * ya) >> 2) + (153 * month + 2) / 5 + time->day; + days -= 307; + days *= 86400000000ull; + + const auto usec = static_cast(time->hour) * 3600000000ull + + static_cast(time->minute) * 60000000ull + + static_cast(time->second) * 1000000ull + time->microsecond; + + tick->tick = days + usec; + + return OK; +} + +static int KYTY_SYSV_ABI RtcGetCurrentTick(RtcTick* tick) { + PRINT_NAME(); + + if (tick == nullptr) { + return RTC_ERROR_DATETIME_UNINITIALIZED; + } + + const auto now = Common::DateTime::FromSystemUTC(); + const auto date = now.GetDate(); + const auto tod = now.GetTime(); + + RtcDateTime time {}; + time.year = static_cast(date.Year()); + time.month = static_cast(date.Month()); + time.day = static_cast(date.Day()); + time.hour = static_cast(tod.Hour24()); + time.minute = static_cast(tod.Minute()); + time.second = static_cast(tod.Second()); + time.microsecond = static_cast(tod.Msec() * 1000); + + return RtcGetTick(&time, tick); +} + +static int KYTY_SYSV_ABI RtcGetCurrentNetworkTick(RtcTick* tick) { + PRINT_NAME(); + + return RtcGetCurrentTick(tick); +} + +static int KYTY_SYSV_ABI RtcGetCurrentClock(RtcDateTime* time, int time_zone_minutes) { + PRINT_NAME(); + + if (time == nullptr) { + return RTC_ERROR_DATETIME_UNINITIALIZED; + } + + RtcTick tick {}; + auto ret = RtcGetCurrentTick(&tick); + if (ret != OK) { + return ret; + } + + tick.tick += static_cast(time_zone_minutes) * 60000000ll; + + return RtcSetTick(time, &tick); +} + +static int KYTY_SYSV_ABI RtcConvertUtcToLocalTime(const RtcTick* utc, RtcTick* local_time) { + PRINT_NAME(); + + if (utc == nullptr || local_time == nullptr) { + return RTC_ERROR_INVALID_POINTER; + } + + // Keep host-independent behavior for now; Kyty stores RTC ticks in UTC. + *local_time = *utc; + + return OK; +} + +static int KYTY_SYSV_ABI RtcConvertLocalTimeToUtc(const RtcTick* local_time, RtcTick* utc) { + PRINT_NAME(); + + if (local_time == nullptr || utc == nullptr) { + return RTC_ERROR_INVALID_POINTER; + } + + // Keep host-independent behavior for now; Kyty stores RTC ticks in UTC. + *utc = *local_time; + + return OK; +} + +static int KYTY_SYSV_ABI RtcFormatRFC3339(char* date_time, const RtcTick* utc, + int time_zone_minutes) { + PRINT_NAME(); + + if (date_time == nullptr) { + return RTC_ERROR_INVALID_POINTER; + } + + RtcTick tick {}; + if (utc == nullptr) { + auto ret = RtcGetCurrentTick(&tick); + if (ret != OK) { + return ret; + } + } else { + tick = *utc; + } + + const auto offset_ticks = static_cast(time_zone_minutes) * 60000000ll; + if (offset_ticks < 0 && tick.tick < static_cast(-offset_ticks)) { + return RTC_ERROR_INVALID_VALUE; + } + tick.tick = static_cast(static_cast(tick.tick) + offset_ticks); + + RtcDateTime time {}; + auto ret = RtcSetTick(&time, &tick); + if (ret != OK) { + return ret; + } + + char zone[8] {}; + if (time_zone_minutes == 0) { + std::snprintf(zone, sizeof(zone), "Z"); + } else { + const auto sign = (time_zone_minutes < 0 ? '-' : '+'); + auto minutes = (time_zone_minutes < 0 ? -time_zone_minutes : time_zone_minutes); + std::snprintf(zone, sizeof(zone), "%c%02d:%02d", sign, minutes / 60, minutes % 60); + } + + std::snprintf(date_time, 32, "%04u-%02u-%02uT%02u:%02u:%02u.%02u%s", + static_cast(time.year), static_cast(time.month), + static_cast(time.day), static_cast(time.hour), + static_cast(time.minute), static_cast(time.second), + static_cast(time.microsecond / 10000), zone); + + return OK; +} + +static int KYTY_SYSV_ABI RtcParseRFC3339(RtcTick* utc, const char* date_time) { + PRINT_NAME(); + + if (utc == nullptr || date_time == nullptr) { + return RTC_ERROR_INVALID_POINTER; + } + + RtcDateTime time {}; + int parsed = 0; + int count = std::sscanf(date_time, "%hu-%hu-%huT%hu:%hu:%hu%n", &time.year, &time.month, + &time.day, &time.hour, &time.minute, &time.second, &parsed); + if (count != 6) { + return RTC_ERROR_INVALID_VALUE; + } + + const char* pos = date_time + parsed; + if (*pos == '.') { + pos++; + uint32_t microsecond = 0; + int digits = 0; + while (*pos >= '0' && *pos <= '9') { + if (digits < 6) { + microsecond = microsecond * 10 + static_cast(*pos - '0'); + } + digits++; + pos++; + } + while (digits > 0 && digits < 6) { + microsecond *= 10; + digits++; + } + time.microsecond = microsecond; + } + + auto ret = RtcGetTick(&time, utc); + if (ret != OK) { + return ret; + } + + if (*pos == 'Z' || *pos == '\0') { + return OK; + } + if (*pos != '+' && *pos != '-') { + return RTC_ERROR_INVALID_VALUE; + } + + int zone_hour = 0; + int zone_min = 0; + if (std::sscanf(pos + 1, "%02d:%02d", &zone_hour, &zone_min) != 2) { + return RTC_ERROR_INVALID_VALUE; + } + + int zone_minutes = zone_hour * 60 + zone_min; + if (*pos == '-') { + zone_minutes = -zone_minutes; + } + + const auto offset_ticks = static_cast(zone_minutes) * 60000000ll; + if (offset_ticks > 0 && utc->tick < static_cast(offset_ticks)) { + return RTC_ERROR_INVALID_VALUE; + } + utc->tick = static_cast(static_cast(utc->tick) - offset_ticks); + + return OK; +} + +static int KYTY_SYSV_ABI RtcGetTime_t(const RtcDateTime* time, int64_t* seconds) { + PRINT_NAME(); + + if (time == nullptr || seconds == nullptr) { + return RTC_ERROR_INVALID_POINTER; + } + + RtcTick tick {}; + auto ret = RtcGetTick(time, &tick); + if (ret != OK) { + return ret; + } + + *seconds = (tick.tick < RTC_UNIX_EPOCH_TICKS + ? 0 + : static_cast((tick.tick - RTC_UNIX_EPOCH_TICKS) / 1000000ull)); + + return OK; +} + +static int KYTY_SYSV_ABI RtcGetWin32FileTime(const RtcDateTime* time, uint64_t* win32_time) { + PRINT_NAME(); + + if (time == nullptr || win32_time == nullptr) { + return RTC_ERROR_INVALID_POINTER; + } + + RtcTick tick {}; + auto ret = RtcGetTick(time, &tick); + if (ret != OK) { + return ret; + } + + *win32_time = (tick.tick < RTC_WIN32_FILETIME_EPOCH_TICKS + ? 0 + : (tick.tick - RTC_WIN32_FILETIME_EPOCH_TICKS) * 10ull); + + return OK; +} + +static int KYTY_SYSV_ABI RtcSetWin32FileTime(RtcDateTime* time, uint64_t win32_time) { + PRINT_NAME(); + + if (time == nullptr) { + return RTC_ERROR_INVALID_POINTER; + } + + if (win32_time / 10ull > + std::numeric_limits::max() - RTC_WIN32_FILETIME_EPOCH_TICKS) { + return RTC_ERROR_INVALID_VALUE; + } + + RtcTick tick {}; + tick.tick = win32_time / 10ull + RTC_WIN32_FILETIME_EPOCH_TICKS; + + return RtcSetTick(time, &tick); +} + +static int KYTY_SYSV_ABI RtcSetTime_t(RtcDateTime* time, int64_t seconds) { + PRINT_NAME(); + + if (time == nullptr) { + return RTC_ERROR_INVALID_POINTER; + } + if (seconds < 0) { + return RTC_ERROR_INVALID_VALUE; + } + + const auto sec = static_cast(seconds); + if (sec > (std::numeric_limits::max() - RTC_UNIX_EPOCH_TICKS) / 1000000ull) { + return RTC_ERROR_INVALID_VALUE; + } + + RtcTick tick {}; + tick.tick = sec * 1000000ull + RTC_UNIX_EPOCH_TICKS; + + return RtcSetTick(time, &tick); +} + +static int KYTY_SYSV_ABI RtcTickAddTicks(RtcTick* dst, const RtcTick* src, int64_t ticks) { + PRINT_NAME(); + + if (dst == nullptr || src == nullptr) { + return RTC_ERROR_INVALID_POINTER; + } + + dst->tick = src->tick + ticks; + + return OK; +} + +static int KYTY_SYSV_ABI RtcTickAddMicroseconds(RtcTick* dst, const RtcTick* src, int64_t usec) { + PRINT_NAME(); + + return RtcTickAddTicks(dst, src, usec); +} + +static int KYTY_SYSV_ABI RtcTickAddSeconds(RtcTick* dst, const RtcTick* src, int64_t seconds) { + PRINT_NAME(); + + return RtcTickAddTicks(dst, src, seconds * 1000000ll); +} + +static int KYTY_SYSV_ABI RtcTickAddMinutes(RtcTick* dst, const RtcTick* src, int64_t minutes) { + PRINT_NAME(); + + return RtcTickAddTicks(dst, src, minutes * 60000000ll); +} + +static int KYTY_SYSV_ABI RtcTickAddHours(RtcTick* dst, const RtcTick* src, int32_t hours) { + PRINT_NAME(); + + return RtcTickAddTicks(dst, src, static_cast(hours) * 3600000000ll); +} + +static int KYTY_SYSV_ABI RtcTickAddDays(RtcTick* dst, const RtcTick* src, int32_t days) { + PRINT_NAME(); + + return RtcTickAddTicks(dst, src, static_cast(days) * 86400000000ll); +} + +static int KYTY_SYSV_ABI RtcTickAddWeeks(RtcTick* dst, const RtcTick* src, int32_t weeks) { + PRINT_NAME(); + + return RtcTickAddTicks(dst, src, static_cast(weeks) * 604800000000ll); +} + +} // namespace Rtc + +LIB_DEFINE(InitRtc_1_Rtc) { + LIB_FUNC("lPEBYdVX0XQ", Rtc::RtcCheckValid); + LIB_FUNC("8Yr143yEnRo", Rtc::RtcConvertLocalTimeToUtc); + LIB_FUNC("M1TvFst-jrM", Rtc::RtcConvertUtcToLocalTime); + LIB_FUNC("WJ3rqFwymew", Rtc::RtcFormatRFC3339); + LIB_FUNC("8lfvnRMqwEM", Rtc::RtcGetCurrentClock); + LIB_FUNC("ZPD1YOKI+Kw", Rtc::RtcGetCurrentClockLocalTime); + LIB_FUNC("18B2NS1y9UU", Rtc::RtcGetCurrentTick); + LIB_FUNC("zO9UL3qIINQ", Rtc::RtcGetCurrentNetworkTick); + LIB_FUNC("CyIK-i4XdgQ", Rtc::RtcGetDayOfWeek); + LIB_FUNC("Ug8pCwQvh0c", Rtc::RtcIsLeapYear); + LIB_FUNC("3O7Ln8AqJ1o", Rtc::RtcGetDaysInMonthExport); + LIB_FUNC("8w-H19ip48I", Rtc::RtcGetTick); + LIB_FUNC("jMNwqYr4R-k", Rtc::RtcGetTickResolution); + LIB_FUNC("BtqmpTRXHgk", Rtc::RtcGetTime_t); + LIB_FUNC("jfRO0uTjtzA", Rtc::RtcGetWin32FileTime); + LIB_FUNC("99bMGglFW3I", Rtc::RtcParseRFC3339); + LIB_FUNC("n5JiAJXsbcs", Rtc::RtcSetWin32FileTime); + LIB_FUNC("ueega6v3GUw", Rtc::RtcSetTick); + LIB_FUNC("bDEVVP4bTjQ", Rtc::RtcSetTime_t); + LIB_FUNC("NR1J0N7L2xY", Rtc::RtcTickAddDays); + LIB_FUNC("MDc5cd8HfCA", Rtc::RtcTickAddHours); + LIB_FUNC("XPIiw58C+GM", Rtc::RtcTickAddMicroseconds); + LIB_FUNC("mn-tf4QiFzk", Rtc::RtcTickAddMinutes); + LIB_FUNC("07O525HgICs", Rtc::RtcTickAddSeconds); + LIB_FUNC("AqVMssr52Rc", Rtc::RtcTickAddTicks); + LIB_FUNC("gI4t194c2W8", Rtc::RtcTickAddWeeks); +} + +} // namespace LibRtc + +LIB_DEFINE(InitRtc_1) { + LibRtc::InitRtc_1_Rtc(s); +} + +} // namespace Libs diff --git a/src/libs/libRudp.cpp b/src/libs/libRudp.cpp new file mode 100644 index 000000000..8a4c59af5 --- /dev/null +++ b/src/libs/libRudp.cpp @@ -0,0 +1,65 @@ +#include "common/abi.h" +#include "common/logging/log.h" +#include "libs/errno.h" +#include "libs/libs.h" +#include "loader/symbolDatabase.h" + +#include +#include + +namespace Libs { + +LIB_VERSION("Rudp", 1, "Rudp", 1, 1); + +namespace Rudp { + +using RudpEventHandler = void (*)(int ctx_id, int event_id, int error_code, void* arg); + +static RudpEventHandler g_event_handler = nullptr; +static void* g_event_arg = nullptr; +static bool g_initialized = false; + +static KYTY_SYSV_ABI int RudpInit(void* mem_pool, int mem_pool_size) { + PRINT_NAME(); + + LOGF("\t mem_pool = 0x%016" PRIx64 "\n" + "\t mem_pool_size = %d\n", + reinterpret_cast(mem_pool), mem_pool_size); + + g_initialized = true; + + return OK; +} + +static KYTY_SYSV_ABI int RudpEnableInternalIOThread(uint32_t stack_size, uint32_t priority) { + PRINT_NAME(); + + LOGF("\t stack_size = %" PRIu32 "\n" + "\t priority = %" PRIu32 "\n", + stack_size, priority); + + return OK; +} + +static KYTY_SYSV_ABI int RudpSetEventHandler(RudpEventHandler handler, void* arg) { + PRINT_NAME(); + + LOGF("\t handler = 0x%016" PRIx64 "\n" + "\t arg = 0x%016" PRIx64 "\n", + reinterpret_cast(handler), reinterpret_cast(arg)); + + g_event_handler = handler; + g_event_arg = arg; + + return OK; +} + +} // namespace Rudp + +LIB_DEFINE(InitRudp_1) { + LIB_FUNC("amuBfI-AQc4", Rudp::RudpInit); + LIB_FUNC("6PBNpsgyaxw", Rudp::RudpEnableInternalIOThread); + LIB_FUNC("SUEVes8gvmw", Rudp::RudpSetEventHandler); +} + +} // namespace Libs diff --git a/src/libs/libSaveData.cpp b/src/libs/libSaveData.cpp new file mode 100644 index 000000000..a52502d55 --- /dev/null +++ b/src/libs/libSaveData.cpp @@ -0,0 +1,916 @@ +#include "common/abi.h" +#include "common/assert.h" +#include "common/common.h" +#include "common/file.h" +#include "common/logging/log.h" +#include "common/stringUtils.h" +#include "kernel/fileSystem.h" +#include "libs/errno.h" +#include "libs/libs.h" +#include "loader/symbolDatabase.h" +#include "loader/systemContent.h" + +#include +#include +#include +#include + +namespace Libs { + +LIB_VERSION("SaveData", 1, "SaveData", 1, 1); + +namespace SaveData { + +// TODO(): specify dir at launcher +static constexpr char SAVE_DATA_DIR[] = "_SaveData"; +static constexpr char SAVE_DATA_POINT[] = "/savedata0"; +static constexpr uint64_t SAVE_DATA_BLOCKS_MAX = 16384; + +struct SceSaveDataDirName { + char data[32]; +}; + +struct SceSaveDataTitleId { + char data[10]; + char padding[6]; +}; + +struct SceSaveDataFingerprint { + char data[65]; + char padding[15]; +}; + +struct SaveDataParam; + +enum class SaveDataSortKey : uint32_t { + DirName = 0, + UserParam = 1, + Blocks = 2, + Mtime = 3, + FreeBlocks = 5, +}; + +enum class SaveDataSortOrder : uint32_t { + Ascent = 0, + Descent = 1, +}; + +struct SaveDataDirNameSearchCond { + int32_t user_id; + int32_t pad; + const SceSaveDataTitleId* title_id; + const SceSaveDataDirName* dir_name; + SaveDataSortKey key; + SaveDataSortOrder order; + uint8_t reserved[32]; +}; + +struct SaveDataSearchInfo { + uint64_t blocks; + uint64_t free_blocks; + uint8_t reserved[32]; +}; + +struct SaveDataDirNameSearchResult { + uint32_t hit_num; + int32_t pad; + SceSaveDataDirName* dir_names; + uint32_t dir_names_num; + uint32_t set_num; + SaveDataParam* params; + SaveDataSearchInfo* infos; + uint8_t reserved[12]; + int32_t pad2; +}; + +struct SaveDataMountPoint { + char data[16]; +}; + +struct SaveDataMount3 { + int user_id; + int pad; + const SceSaveDataDirName* dir_name; + uint64_t blocks; + uint64_t system_blocks; + uint32_t mount_mode; + int pad2; + int32_t resource; + uint8_t reserved[32]; +}; + +struct SaveDataMountResult { + SaveDataMountPoint mount_point; + uint64_t required_blocks; + uint32_t unused; + uint32_t mount_status; + uint8_t reserved[28]; + int pad; +}; + +struct SaveDataParam { + char title[128]; + char sub_title[128]; + char detail[1024]; + uint32_t user_param; + int pad; + int64_t mtime; + uint8_t reserved[32]; +}; + +struct SaveDataMountInfo { + uint64_t blocks; + uint64_t free_blocks; + uint8_t reserved[32]; +}; + +struct SaveDataIcon { + void* buf; + size_t buf_size; + size_t data_size; + uint8_t reserved[32]; +}; + +struct SaveDataMemoryData { + void* buf; + size_t buf_size; + int64_t offset; + uint8_t reserved[40]; +}; + +struct SaveDataMemoryGet2 { + int32_t user_id; + uint8_t padding[4]; + SaveDataMemoryData* data; + SaveDataParam* param; + SaveDataIcon* icon; + uint32_t slot_id; + uint8_t reserved[28]; +}; + +struct SaveDataMemorySetup2 { + uint32_t option; + int32_t user_id; + size_t memory_size; + size_t icon_memory_size; + const SaveDataParam* init_param; + const SaveDataIcon* init_icon; + uint32_t slot_id; + uint8_t reserved[20]; +}; + +struct SaveDataMemorySetupResult { + size_t existed_memory_size; + uint8_t reserved[16]; +}; + +struct SaveDataMemorySet2 { + int32_t user_id; + uint8_t padding[4]; + const SaveDataMemoryData* data; + const SaveDataParam* param; + const SaveDataIcon* icon; + uint32_t data_num; + uint32_t slot_id; + uint8_t reserved[24]; +}; + +struct SaveDataTransferringMount { + int32_t user_id; + const SceSaveDataTitleId* title_id; + const SceSaveDataDirName* dir_name; + const void* fingerprint; + uint8_t reserved[32]; +}; + +struct SaveDataPrepareParam { + int32_t resource; + uint32_t prepare_mode; + uint8_t reserved[32]; +}; + +struct SaveDataCommitParam { + int32_t resource; + uint32_t commit_mode; + uint8_t reserved[32]; +}; + +struct SaveDataDelete { + int32_t user_id; + int32_t pad; + const SceSaveDataTitleId* title_id; + const SceSaveDataDirName* dir_name; + uint32_t unused; + uint8_t reserved[32]; + int32_t pad2; +}; + +struct SaveDataEvent { + uint32_t type; + int32_t error_code; + int32_t user_id; + uint8_t padding[4]; + SceSaveDataTitleId title_id; + SceSaveDataDirName dir_name; + uint8_t reserved[40]; +}; + +struct SaveDataBackup { + int32_t user_id; + int32_t pad; + const SceSaveDataTitleId* title_id; + const SceSaveDataDirName* dir_name; + const void* fingerprint; + uint8_t reserved[32]; +}; + +static constexpr uint32_t SAVE_DATA_UMOUNT_MODE_BACKUP_ASYNC = (1u << 16u); +static constexpr uint32_t SAVE_DATA_COMMIT_MODE_BACKUP_ASYNC = 1u; + +static constexpr uint32_t SAVE_DATA_EVENT_TYPE_UMOUNT_BACKUP_END = 1u; +static constexpr uint32_t SAVE_DATA_EVENT_TYPE_BACKUP_END = 2u; +static constexpr uint32_t SAVE_DATA_EVENT_TYPE_COMMIT_BACKUP_END = 4u; + +static std::vector g_save_data_memory(0x10000); +static int32_t g_next_transaction_resource = 1; +static std::deque g_save_data_events; + +static std::string get_title_id() { + std::string title_id; + if (!Loader::SystemContentParamSfoGetString("TITLE_ID", &title_id) || title_id.empty()) { + title_id = "UNKNOWN"; + } + + return title_id; +} + +static void queue_save_data_event(uint32_t type, int32_t user_id, + const SceSaveDataTitleId* title_id, + const SceSaveDataDirName* dir_name, int32_t error_code = OK) { + SaveDataEvent event = {}; + event.type = type; + event.error_code = error_code; + event.user_id = user_id; + if (title_id != nullptr) { + std::memcpy(&event.title_id, title_id, sizeof(event.title_id)); + } + if (dir_name != nullptr) { + std::memcpy(&event.dir_name, dir_name, sizeof(event.dir_name)); + } + + g_save_data_events.push_back(event); + if (g_save_data_events.size() > 20) { + g_save_data_events.pop_front(); + } +} + +static bool dir_name_match(const char* str, const char* pattern) { + while (*str != '\0' && *pattern != '\0') { + if (*pattern == '%') { + for (const char* s = str;; s++) { + if (dir_name_match(s, pattern + 1)) { + return true; + } + if (*s == '\0') { + break; + } + } + return false; + } + if (*pattern == '_') { + str++; + pattern++; + continue; + } + if (*pattern != *str) { + return false; + } + str++; + pattern++; + } + return *str == '\0' && *pattern == '\0'; +} + +int KYTY_SYSV_ABI SaveDataInitialize3(const void* /*init*/) { + PRINT_NAME(); + + // EXIT_NOT_IMPLEMENTED(init != nullptr); + + return OK; +} + +int KYTY_SYSV_ABI SaveDataTerminate() { + PRINT_NAME(); + + g_save_data_events.clear(); + + return OK; +} + +int KYTY_SYSV_ABI SaveDataCreateTransactionResource(uint32_t size) { + PRINT_NAME(); + + LOGF("\t size = %" PRIu32 "\n", size); + + return g_next_transaction_resource++; +} + +int KYTY_SYSV_ABI SaveDataDeleteTransactionResource(int32_t resource) { + PRINT_NAME(); + + LOGF("\t resource = %" PRId32 "\n", resource); + + return OK; +} + +int KYTY_SYSV_ABI SaveDataDirNameSearch(const SaveDataDirNameSearchCond* cond, + SaveDataDirNameSearchResult* result) { + PRINT_NAME(); + + if (cond == nullptr || result == nullptr || + static_cast(cond->key) > static_cast(SaveDataSortKey::FreeBlocks) || + static_cast(cond->order) > static_cast(SaveDataSortOrder::Descent)) { + return SAVE_DATA_ERROR_PARAMETER; + } + + LOGF("\t user_id = %d\n" + "\t title_id = %s\n" + "\t dir_name = %s\n" + "\t key = %" PRIu32 "\n" + "\t order = %" PRIu32 "\n" + "\t dir_names_num = %" PRIu32 "\n", + cond->user_id, cond->title_id != nullptr ? cond->title_id->data : "", + cond->dir_name != nullptr ? cond->dir_name->data : "", + static_cast(cond->key), static_cast(cond->order), + result->dir_names_num); + + result->hit_num = 0; + result->pad = 0; + result->set_num = 0; + std::memset(result->reserved, 0, sizeof(result->reserved)); + result->pad2 = 0; + + std::vector dir_list; + std::string root = + Common::FixDirectorySlash((std::string(SAVE_DATA_DIR) + "/" + get_title_id())); + + if (Common::File::IsDirectoryExisting(root)) { + for (const auto& entry: Common::File::GetDirEntries(root)) { + if (!entry.is_file && !Common::StartsWith(entry.name, "sce_")) { + if (cond->dir_name == nullptr || cond->dir_name->data[0] == '\0' || + dir_name_match(Common::ToLower(entry.name).c_str(), + Common::ToLower(std::string(cond->dir_name->data)).c_str())) { + dir_list.push_back(entry.name); + } + } + } + } + + std::sort(dir_list.begin(), dir_list.end(), [](const std::string& a, const std::string& b) { + return std::strcmp(a.c_str(), b.c_str()) < 0; + }); + + if (cond->order == SaveDataSortOrder::Descent) { + std::vector reversed; + for (size_t i = dir_list.size(); i > 0; i--) { + reversed.push_back(dir_list[i - 1]); + } + dir_list = reversed; + } + + auto max_count = + (result->dir_names_num < dir_list.size() ? result->dir_names_num : dir_list.size()); + + result->hit_num = static_cast(dir_list.size()); + result->set_num = static_cast(max_count); + + for (size_t i = 0; i < max_count; i++) { + std::snprintf(result->dir_names[i].data, sizeof(result->dir_names[i].data), "%s", + dir_list[i].c_str()); + if (result->params != nullptr) { + result->params[i] = {}; + } + if (result->infos != nullptr) { + result->infos[i] = {}; + result->infos[i].blocks = SAVE_DATA_BLOCKS_MAX; + result->infos[i].free_blocks = SAVE_DATA_BLOCKS_MAX; + } + } + + return OK; +} + +int KYTY_SYSV_ABI SaveDataMount3(const SaveDataMount3* mount, SaveDataMountResult* mount_result) { + PRINT_NAME(); + + EXIT_NOT_IMPLEMENTED(mount == nullptr); + EXIT_NOT_IMPLEMENTED(mount_result == nullptr); + EXIT_NOT_IMPLEMENTED(mount->dir_name == nullptr); + + LOGF("\t user_id = %d\n" + "\t dir_name = %s\n" + "\t blocks = %" PRIu64 "\n" + "\t system_blocks = %" PRIu64 "\n" + "\t mount_mode = %" PRIu32 "\n" + "\t resource = %" PRId32 "\n", + mount->user_id, mount->dir_name->data, mount->blocks, mount->system_blocks, + mount->mount_mode, mount->resource); + + *mount_result = {}; + + std::string mount_dir = std::string(SAVE_DATA_DIR) + "/" + get_title_id() + "/" + + std::string(mount->dir_name->data); + std::string mount_point = SAVE_DATA_POINT; + bool create = ((mount->mount_mode & 4u) != 0); + bool create2 = ((mount->mount_mode & 32u) != 0); + bool open = (!create && !create2 && ((mount->mount_mode & 3u) != 0)); + + if (!create && !create2 && !open) { + EXIT("unknown mount mode: %u", mount->mount_mode); + } + + if (open && !Common::File::IsDirectoryExisting(mount_dir)) { + return SAVE_DATA_ERROR_NOT_FOUND; + } + + if (create && Common::File::IsDirectoryExisting(mount_dir)) { + return SAVE_DATA_ERROR_EXISTS; + } + + bool created = false; + if ((create || create2) && !Common::File::IsDirectoryExisting(mount_dir)) { + Common::File::CreateDirectories(mount_dir); + created = true; + + EXIT_NOT_IMPLEMENTED((!Common::File::IsDirectoryExisting(mount_dir))); + } + + LibKernel::FileSystem::Mount(mount_dir, mount_point); + + int s = snprintf(mount_result->mount_point.data, 16, "%s", mount_point.c_str()); + + EXIT_NOT_IMPLEMENTED(s >= 16); + + mount_result->required_blocks = 0; + mount_result->mount_status = (created ? 1 : 0); + + return OK; +} + +int KYTY_SYSV_ABI SaveDataSetupSaveDataMemory2(const SaveDataMemorySetup2* setup_param, + SaveDataMemorySetupResult* result) { + PRINT_NAME(); + + if (setup_param == nullptr) { + return SAVE_DATA_ERROR_PARAMETER; + } + + LOGF("\t option = 0x%08" PRIx32 "\n" + "\t user_id = %" PRId32 "\n" + "\t memory_size = %" PRIu64 "\n" + "\t icon_memory_size = %" PRIu64 "\n" + "\t slot_id = %" PRIu32 "\n", + setup_param->option, setup_param->user_id, static_cast(setup_param->memory_size), + static_cast(setup_param->icon_memory_size), setup_param->slot_id); + + if (setup_param->memory_size > g_save_data_memory.size()) { + g_save_data_memory.resize(setup_param->memory_size); + } + + if (result != nullptr) { + *result = {}; + result->existed_memory_size = g_save_data_memory.size(); + } + + return OK; +} + +int KYTY_SYSV_ABI SaveDataGetSaveDataMemory2(SaveDataMemoryGet2* get_param) { + PRINT_NAME(); + + if (get_param == nullptr) { + return SAVE_DATA_ERROR_PARAMETER; + } + + LOGF("\t user_id = %" PRId32 "\n" + "\t data = 0x%016" PRIx64 "\n" + "\t param = 0x%016" PRIx64 "\n" + "\t icon = 0x%016" PRIx64 "\n" + "\t slot_id = %" PRIu32 "\n", + get_param->user_id, reinterpret_cast(get_param->data), + reinterpret_cast(get_param->param), reinterpret_cast(get_param->icon), + get_param->slot_id); + + if (get_param->data != nullptr) { + auto* data = get_param->data; + if (data->buf == nullptr || data->offset < 0) { + return SAVE_DATA_ERROR_PARAMETER; + } + + const auto offset = static_cast(data->offset); + if (offset + data->buf_size > g_save_data_memory.size()) { + g_save_data_memory.resize(offset + data->buf_size); + } + std::memcpy(data->buf, g_save_data_memory.data() + offset, data->buf_size); + } + if (get_param->param != nullptr) { + std::memset(get_param->param, 0, sizeof(*get_param->param)); + } + if (get_param->icon != nullptr) { + get_param->icon->data_size = 0; + } + + return OK; +} + +int KYTY_SYSV_ABI SaveDataSetSaveDataMemory2(const SaveDataMemorySet2* set_param) { + PRINT_NAME(); + + if (set_param == nullptr) { + return SAVE_DATA_ERROR_PARAMETER; + } + + LOGF("\t user_id = %" PRId32 "\n" + "\t data = 0x%016" PRIx64 "\n" + "\t param = 0x%016" PRIx64 "\n" + "\t icon = 0x%016" PRIx64 "\n" + "\t data_num = %" PRIu32 "\n" + "\t slot_id = %" PRIu32 "\n", + set_param->user_id, reinterpret_cast(set_param->data), + reinterpret_cast(set_param->param), reinterpret_cast(set_param->icon), + set_param->data_num, set_param->slot_id); + + const uint32_t data_num = (set_param->data_num == 0 ? 1 : set_param->data_num); + if (set_param->data != nullptr) { + for (uint32_t i = 0; i < data_num; i++) { + const auto& data = set_param->data[i]; + if (data.buf == nullptr || data.offset < 0) { + return SAVE_DATA_ERROR_PARAMETER; + } + + const auto offset = static_cast(data.offset); + if (offset + data.buf_size > g_save_data_memory.size()) { + g_save_data_memory.resize(offset + data.buf_size); + } + std::memcpy(g_save_data_memory.data() + offset, data.buf, data.buf_size); + } + } + + return OK; +} + +int KYTY_SYSV_ABI SaveDataTransferringMount(const SaveDataTransferringMount* mount, + SaveDataMountResult* mount_result) { + PRINT_NAME(); + + if (mount == nullptr || mount_result == nullptr || mount->dir_name == nullptr) { + return SAVE_DATA_ERROR_PARAMETER; + } + + LOGF("\t user_id = %" PRId32 "\n" + "\t title_id = %s\n" + "\t dir_name = %s\n", + mount->user_id, mount->title_id != nullptr ? mount->title_id->data : "", + mount->dir_name->data); + + *mount_result = {}; + + std::string mount_dir = std::string(SAVE_DATA_DIR) + "/" + get_title_id() + "/" + + std::string(mount->dir_name->data); + std::string mount_point = SAVE_DATA_POINT; + + if (!Common::File::IsDirectoryExisting(mount_dir)) { + Common::File::CreateDirectories(mount_dir); + } + + LibKernel::FileSystem::Mount(mount_dir, mount_point); + + int s = snprintf(mount_result->mount_point.data, 16, "%s", mount_point.c_str()); + + EXIT_NOT_IMPLEMENTED(s >= 16); + + mount_result->required_blocks = 0; + mount_result->mount_status = 1; + + return OK; +} + +int KYTY_SYSV_ABI SaveDataUmount2(uint32_t mode, const SaveDataMountPoint* mount_point) { + PRINT_NAME(); + + if (mount_point == nullptr) { + return SAVE_DATA_ERROR_PARAMETER; + } + + LOGF("\t mode = %" PRIu32 "\n" + "\t mount_point = %s\n", + mode, mount_point->data); + + if ((mode & SAVE_DATA_UMOUNT_MODE_BACKUP_ASYNC) != 0) { + queue_save_data_event(SAVE_DATA_EVENT_TYPE_UMOUNT_BACKUP_END, 0, nullptr, nullptr); + } + + LibKernel::FileSystem::Umount(std::string(mount_point->data)); + + return OK; +} + +int KYTY_SYSV_ABI SaveDataPrepare(const SaveDataMountPoint* mount_point, + const SaveDataPrepareParam* param) { + PRINT_NAME(); + + if (mount_point == nullptr || param == nullptr) { + return SAVE_DATA_ERROR_PARAMETER; + } + + LOGF("\t mount_point = %s\n" + "\t resource = %" PRId32 "\n" + "\t prepare_mode = %" PRIu32 "\n", + mount_point->data, param->resource, param->prepare_mode); + + return OK; +} + +int KYTY_SYSV_ABI SaveDataCommit(const SaveDataCommitParam* param) { + PRINT_NAME(); + + if (param == nullptr) { + return SAVE_DATA_ERROR_PARAMETER; + } + + LOGF("\t resource = %" PRId32 "\n" + "\t commit_mode = %" PRIu32 "\n", + param->resource, param->commit_mode); + + if ((param->commit_mode & SAVE_DATA_COMMIT_MODE_BACKUP_ASYNC) != 0) { + queue_save_data_event(SAVE_DATA_EVENT_TYPE_COMMIT_BACKUP_END, 0, nullptr, nullptr); + } + + return OK; +} + +int KYTY_SYSV_ABI SaveDataDelete(const SaveDataDelete* del) { + PRINT_NAME(); + + if (del == nullptr || del->dir_name == nullptr) { + return SAVE_DATA_ERROR_PARAMETER; + } + + LOGF("\t user_id = %" PRId32 "\n" + "\t title_id = %s\n" + "\t dir_name = %s\n", + del->user_id, del->title_id != nullptr ? del->title_id->data : "", + del->dir_name->data); + + std::string dir = + std::string(SAVE_DATA_DIR) + "/" + get_title_id() + "/" + std::string(del->dir_name->data); + if (Common::File::IsDirectoryExisting(dir)) { + Common::File::DeleteDirectory(dir); + } + + return OK; +} + +int KYTY_SYSV_ABI SaveDataGetParam(const SaveDataMountPoint* mount_point, uint32_t param_type, + void* param_buf, size_t param_buf_size, size_t* got_size) { + PRINT_NAME(); + + if (mount_point == nullptr || param_buf == nullptr) { + return SAVE_DATA_ERROR_PARAMETER; + } + + LOGF("\t mount_point = %s\n" + "\t param_type = %u\n" + "\t param_buf_size = %" PRIu64 "\n", + mount_point->data, param_type, param_buf_size); + + if (param_type == 0) { + if (param_buf_size < sizeof(SaveDataParam)) { + return SAVE_DATA_ERROR_PARAMETER; + } + std::memset(param_buf, 0, sizeof(SaveDataParam)); + if (got_size != nullptr) { + *got_size = sizeof(SaveDataParam); + } + return OK; + } + + std::memset(param_buf, 0, param_buf_size); + if (got_size != nullptr) { + *got_size = param_buf_size; + } + + return OK; +} + +int KYTY_SYSV_ABI SaveDataLoadIcon(const SaveDataMountPoint* mount_point, SaveDataIcon* icon) { + PRINT_NAME(); + + if (mount_point == nullptr || icon == nullptr) { + return SAVE_DATA_ERROR_PARAMETER; + } + + LOGF("\t mount_point = %s\n" + "\t buf = %016" PRIx64 "\n" + "\t buf_size = %" PRIu64 "\n", + mount_point->data, reinterpret_cast(icon->buf), icon->buf_size); + + icon->data_size = 0; + + return OK; +} + +int KYTY_SYSV_ABI SaveDataSaveIconByPath(const SaveDataMountPoint* mount_point, const char* path) { + PRINT_NAME(); + + if (mount_point == nullptr || path == nullptr) { + return SAVE_DATA_ERROR_PARAMETER; + } + + LOGF("\t mount_point = %s\n" + "\t path = %s\n", + mount_point->data, path); + + return OK; +} + +int KYTY_SYSV_ABI SaveDataSyncSaveDataMemory(const void* sync_param) { + PRINT_NAME(); + + LOGF("\t sync_param = 0x%016" PRIx64 "\n", reinterpret_cast(sync_param)); + + return OK; +} + +int KYTY_SYSV_ABI SaveDataGetEventResult(const void* event_param, SaveDataEvent* event) { + PRINT_NAME(); + + LOGF("\t event_param = 0x%016" PRIx64 "\n" + "\t event = 0x%016" PRIx64 "\n", + reinterpret_cast(event_param), reinterpret_cast(event)); + + if (event == nullptr) { + return SAVE_DATA_ERROR_PARAMETER; + } + + if (g_save_data_events.empty()) { + return SAVE_DATA_ERROR_NOT_FOUND; + } + + *event = g_save_data_events.front(); + g_save_data_events.pop_front(); + + LOGF("\t type = %" PRIu32 "\n" + "\t error_code = %" PRId32 "\n" + "\t user_id = %" PRId32 "\n" + "\t title_id = %.10s\n" + "\t dir_name = %.32s\n", + event->type, event->error_code, event->user_id, event->title_id.data, + event->dir_name.data); + + return OK; +} + +int KYTY_SYSV_ABI SaveDataBackup(const SaveDataBackup* backup) { + PRINT_NAME(); + + LOGF("\t backup = 0x%016" PRIx64 "\n", reinterpret_cast(backup)); + + if (backup == nullptr || backup->dir_name == nullptr) { + return SAVE_DATA_ERROR_PARAMETER; + } + + LOGF("\t user_id = %" PRId32 "\n" + "\t title_id = %.10s\n" + "\t dir_name = %.32s\n", + backup->user_id, backup->title_id != nullptr ? backup->title_id->data : "", + backup->dir_name->data); + + queue_save_data_event(SAVE_DATA_EVENT_TYPE_BACKUP_END, backup->user_id, backup->title_id, + backup->dir_name); + + return OK; +} + +int KYTY_SYSV_ABI SaveDataSetParam(const SaveDataMountPoint* mount_point, uint32_t param_type, + const void* param_buf, size_t param_buf_size) { + PRINT_NAME(); + + EXIT_NOT_IMPLEMENTED(mount_point == nullptr); + + LOGF("\t mount_point = %s\n" + "\t param_type = %u\n" + "\t param_buf_size = %" PRIu64 "\n", + mount_point->data, param_type, param_buf_size); + + if (param_type == 0) { + const auto* p = static_cast(param_buf); + + LOGF("\t title = %s\n" + "\t sub_title = %s\n" + "\t detail = %s\n" + "\t user_param = %u\n", + p->title, p->sub_title, p->detail, p->user_param); + } else { + LOGF("\t unsupported param_type, accepting as no-op\n"); + } + + return OK; +} + +int KYTY_SYSV_ABI SaveDataGetMountInfo(const SaveDataMountPoint* mount_point, + SaveDataMountInfo* info) { + PRINT_NAME(); + + EXIT_NOT_IMPLEMENTED(mount_point == nullptr); + EXIT_NOT_IMPLEMENTED(info == nullptr); + + *info = {}; + + info->blocks = SAVE_DATA_BLOCKS_MAX; + info->free_blocks = SAVE_DATA_BLOCKS_MAX; + + return OK; +} + +int KYTY_SYSV_ABI SaveDataSaveIcon(const SaveDataMountPoint* mount_point, + const SaveDataIcon* icon) { + EXIT_NOT_IMPLEMENTED(mount_point == nullptr); + EXIT_NOT_IMPLEMENTED(icon == nullptr); + + LOGF("\t buf = %016" PRIx64 "\n" + "\t buf_size = %" PRIu64 "\n" + "\t data_size = %" PRIu64 "\n", + reinterpret_cast(icon->buf), icon->buf_size, icon->data_size); + + return OK; +} + +} // namespace SaveData + +namespace LibSaveDataNative { + +LIB_VERSION("SaveData_native", 1, "SaveData_native", 1, 1); + +LIB_DEFINE(InitSaveDataNative_1) { + LIB_FUNC("TywrFKCoLGY", ::Libs::SaveData::SaveDataInitialize3); + LIB_FUNC("dyIhnXq-0SM", ::Libs::SaveData::SaveDataDirNameSearch); + LIB_FUNC("PHnuI4LhuRk", ::Libs::SaveData::SaveDataDirNameSearch); + LIB_FUNC("ZP4e7rlzOUk", ::Libs::SaveData::SaveDataMount3); + LIB_FUNC("gjRZNnw0JPE", ::Libs::SaveData::SaveDataCreateTransactionResource); + LIB_FUNC("lJUQuaKqoKY", ::Libs::SaveData::SaveDataDeleteTransactionResource); + LIB_FUNC("sDCBrmc61XU", ::Libs::SaveData::SaveDataPrepare); + LIB_FUNC("ie7qhZ4X0Cc", ::Libs::SaveData::SaveDataCommit); + LIB_FUNC("oQySEUfgXRA", ::Libs::SaveData::SaveDataSetupSaveDataMemory2); + LIB_FUNC("QwOO7vegnV8", ::Libs::SaveData::SaveDataGetSaveDataMemory2); + LIB_FUNC("cduy9v4YmT4", ::Libs::SaveData::SaveDataSetSaveDataMemory2); + LIB_FUNC("wiT9jeC7xPw", ::Libs::SaveData::SaveDataSyncSaveDataMemory); + LIB_FUNC("j8xKtiFj0SY", ::Libs::SaveData::SaveDataGetEventResult); + LIB_FUNC("WAzWTZm1H+I", ::Libs::SaveData::SaveDataTransferringMount); + LIB_FUNC("RjMlsR8EXrw", ::Libs::SaveData::SaveDataTransferringMount); + LIB_FUNC("z1JA8-iJt3k", ::Libs::SaveData::SaveDataBackup); + LIB_FUNC("uW4vfTwMQVo", ::Libs::SaveData::SaveDataUmount2); + LIB_FUNC("S1GkePI17zQ", ::Libs::SaveData::SaveDataDelete); + LIB_FUNC("85zul--eGXs", ::Libs::SaveData::SaveDataSetParam); + LIB_FUNC("XgvSuIdnMlw", ::Libs::SaveData::SaveDataGetParam); + LIB_FUNC("65VH0Qaaz6s", ::Libs::SaveData::SaveDataGetMountInfo); + LIB_FUNC("c88Yy54Mx0w", ::Libs::SaveData::SaveDataSaveIcon); + LIB_FUNC("Z7z6HXWORJY", ::Libs::SaveData::SaveDataSaveIconByPath); + LIB_FUNC("cGjO3wM3V28", ::Libs::SaveData::SaveDataLoadIcon); + LIB_FUNC("X4MYzukPc3g", ::Libs::SaveData::SaveDataDirNameSearch); + LIB_FUNC("yKDy8S5yLA0", ::Libs::SaveData::SaveDataTerminate); +} + +} // namespace LibSaveDataNative + +LIB_DEFINE(InitSaveData_1) { + LIB_FUNC("TywrFKCoLGY", SaveData::SaveDataInitialize3); + LIB_FUNC("dyIhnXq-0SM", SaveData::SaveDataDirNameSearch); + LIB_FUNC("ZP4e7rlzOUk", SaveData::SaveDataMount3); + LIB_FUNC("gjRZNnw0JPE", SaveData::SaveDataCreateTransactionResource); + LIB_FUNC("lJUQuaKqoKY", SaveData::SaveDataDeleteTransactionResource); + LIB_FUNC("sDCBrmc61XU", SaveData::SaveDataPrepare); + LIB_FUNC("ie7qhZ4X0Cc", SaveData::SaveDataCommit); + LIB_FUNC("oQySEUfgXRA", SaveData::SaveDataSetupSaveDataMemory2); + LIB_FUNC("QwOO7vegnV8", SaveData::SaveDataGetSaveDataMemory2); + LIB_FUNC("cduy9v4YmT4", SaveData::SaveDataSetSaveDataMemory2); + LIB_FUNC("wiT9jeC7xPw", SaveData::SaveDataSyncSaveDataMemory); + LIB_FUNC("j8xKtiFj0SY", SaveData::SaveDataGetEventResult); + LIB_FUNC("WAzWTZm1H+I", SaveData::SaveDataTransferringMount); + LIB_FUNC("RjMlsR8EXrw", SaveData::SaveDataTransferringMount); + LIB_FUNC("z1JA8-iJt3k", SaveData::SaveDataBackup); + LIB_FUNC("uW4vfTwMQVo", SaveData::SaveDataUmount2); + LIB_FUNC("S1GkePI17zQ", SaveData::SaveDataDelete); + LIB_FUNC("85zul--eGXs", SaveData::SaveDataSetParam); + LIB_FUNC("XgvSuIdnMlw", SaveData::SaveDataGetParam); + LIB_FUNC("65VH0Qaaz6s", SaveData::SaveDataGetMountInfo); + LIB_FUNC("c88Yy54Mx0w", SaveData::SaveDataSaveIcon); + LIB_FUNC("Z7z6HXWORJY", SaveData::SaveDataSaveIconByPath); + LIB_FUNC("cGjO3wM3V28", SaveData::SaveDataLoadIcon); + LIB_FUNC("X4MYzukPc3g", SaveData::SaveDataDirNameSearch); + LIB_FUNC("yKDy8S5yLA0", SaveData::SaveDataTerminate); + + LibSaveDataNative::InitSaveDataNative_1(s); +} + +} // namespace Libs diff --git a/src/libs/libShare.cpp b/src/libs/libShare.cpp new file mode 100644 index 000000000..a38efbeef --- /dev/null +++ b/src/libs/libShare.cpp @@ -0,0 +1,245 @@ +#include "common/abi.h" +#include "common/logging/log.h" +#include "common/stringUtils.h" +#include "libs/errno.h" +#include "libs/libs.h" +#include "loader/symbolDatabase.h" + +#include +#include + +namespace Libs { + +LIB_VERSION("Share", 1, "Share", 1, 1); + +namespace Share { + +constexpr int SHARE_ERROR_INVALID_PARAM = -2120876030; /* 0x81960002 */ +constexpr int SHARE_ERROR_NOT_SUPPORTED = -2120876025; /* 0x81960007 */ +constexpr int SHARE_REQUEST_ID_INVALID = -1; + +struct ShareCurrentRecordingStatus { + uint32_t recording_2k_status; + uint32_t recording_4k_status; + uint8_t reserved[8]; +}; + +union ShareCurrentStatus { + ShareCurrentRecordingStatus recording_status; + uint8_t reserved[16]; +}; + +static bool share_feature_flag_valid(uint32_t feature_flags) { + constexpr uint32_t share_feature_flag_all = 0xffffffffu; + return feature_flags != 0 && (feature_flags & share_feature_flag_all) == feature_flags; +} + +static KYTY_SYSV_ABI int ShareCaptureScreenshot(const void* param, int32_t* req_id) { + PRINT_NAME(); + + LOGF("\t param = 0x%016" PRIx64 "\n" + "\t req_id = 0x%016" PRIx64 "\n", + reinterpret_cast(param), reinterpret_cast(req_id)); + + if (req_id != nullptr) { + *req_id = SHARE_REQUEST_ID_INVALID; + } + + return SHARE_ERROR_NOT_SUPPORTED; +} + +static KYTY_SYSV_ABI int ShareCaptureScreenshotExtended(const void* extended_param, + int32_t* req_id) { + PRINT_NAME(); + + LOGF("\t extended_param = 0x%016" PRIx64 "\n" + "\t req_id = 0x%016" PRIx64 "\n", + reinterpret_cast(extended_param), reinterpret_cast(req_id)); + + if (req_id != nullptr) { + *req_id = SHARE_REQUEST_ID_INVALID; + } + + return SHARE_ERROR_NOT_SUPPORTED; +} + +static KYTY_SYSV_ABI int ShareCaptureVideoClip(const void* param, int32_t* req_id) { + PRINT_NAME(); + + LOGF("\t param = 0x%016" PRIx64 "\n" + "\t req_id = 0x%016" PRIx64 "\n", + reinterpret_cast(param), reinterpret_cast(req_id)); + + if (req_id != nullptr) { + *req_id = SHARE_REQUEST_ID_INVALID; + } + + return SHARE_ERROR_NOT_SUPPORTED; +} + +static KYTY_SYSV_ABI int ShareCaptureVideoClipExtended(const void* extended_param, + int32_t* req_id) { + PRINT_NAME(); + + LOGF("\t extended_param = 0x%016" PRIx64 "\n" + "\t req_id = 0x%016" PRIx64 "\n", + reinterpret_cast(extended_param), reinterpret_cast(req_id)); + + if (req_id != nullptr) { + *req_id = SHARE_REQUEST_ID_INVALID; + } + + return SHARE_ERROR_NOT_SUPPORTED; +} + +static KYTY_SYSV_ABI int ShareUnregisterContentEventCallback(void* callback) { + PRINT_NAME(); + + LOGF("\t callback = 0x%016" PRIx64 "\n", reinterpret_cast(callback)); + + return OK; +} + +static KYTY_SYSV_ABI int ShareOpenMenuForContent(const void* content_id) { + PRINT_NAME(); + + LOGF("\t content_id = 0x%016" PRIx64 "\n", reinterpret_cast(content_id)); + + return SHARE_ERROR_NOT_SUPPORTED; +} + +static KYTY_SYSV_ABI int ShareSetScreenshotOverlayImage(const char* file_path, int32_t margin_x, + int32_t margin_y, int32_t origin) { + PRINT_NAME(); + + LOGF("\t file_path = %s\n" + "\t margin_x = %d\n" + "\t margin_y = %d\n" + "\t origin = %d\n", + (file_path != nullptr ? file_path : ""), margin_x, margin_y, origin); + + return OK; +} + +static KYTY_SYSV_ABI int ShareSetCaptureSource(uint32_t feature_flags, const void* tap_point) { + PRINT_NAME(); + + LOGF("\t feature_flags = 0x%08" PRIx32 "\n" + "\t tap_point = 0x%016" PRIx64 "\n", + feature_flags, reinterpret_cast(tap_point)); + + if (!share_feature_flag_valid(feature_flags)) { + return SHARE_ERROR_INVALID_PARAM; + } + + return OK; +} + +static KYTY_SYSV_ABI int ShareSetContentParam(const char* content_param) { + PRINT_NAME(); + + LOGF("\t content_param = %s\n", (content_param != nullptr ? content_param : "")); + + if (content_param == nullptr) { + return SHARE_ERROR_INVALID_PARAM; + } + + return OK; +} + +static KYTY_SYSV_ABI int ShareSetContentParamForApplicationTitle(const char* application_title) { + PRINT_NAME(); + + LOGF("\t application_title = %s\n", + (application_title != nullptr ? application_title : "")); + + if (application_title == nullptr) { + return SHARE_ERROR_INVALID_PARAM; + } + + return OK; +} + +static KYTY_SYSV_ABI int ShareRegisterContentEventCallback(void* callback, void* user_data) { + PRINT_NAME(); + + LOGF("\t callback = 0x%016" PRIx64 "\n" + "\t user_data = 0x%016" PRIx64 "\n", + reinterpret_cast(callback), reinterpret_cast(user_data)); + + return OK; +} + +static KYTY_SYSV_ABI int ShareGetCurrentStatus(uint32_t feature_flag, ShareCurrentStatus* status) { + PRINT_NAME(); + + LOGF("\t feature_flag = 0x%08" PRIx32 "\n" + "\t status = 0x%016" PRIx64 "\n", + feature_flag, reinterpret_cast(status)); + + if (!share_feature_flag_valid(feature_flag) || status == nullptr) { + return SHARE_ERROR_INVALID_PARAM; + } + + std::memset(status, 0, sizeof(*status)); + return OK; +} + +static KYTY_SYSV_ABI int ShareGetRunningStatus(uint32_t* feature_flags) { + PRINT_NAME(); + + LOGF("\t feature_flags = 0x%016" PRIx64 "\n", reinterpret_cast(feature_flags)); + + if (feature_flags == nullptr) { + return SHARE_ERROR_INVALID_PARAM; + } + + *feature_flags = 0; + return OK; +} + +static KYTY_SYSV_ABI int ShareFeaturePermit(uint32_t feature_flags) { + PRINT_NAME(); + + LOGF("\t feature_flags = 0x%08" PRIx32 "\n", feature_flags); + + if (!share_feature_flag_valid(feature_flags)) { + return SHARE_ERROR_INVALID_PARAM; + } + + return OK; +} + +static KYTY_SYSV_ABI int ShareFeatureProhibit(uint32_t feature_flags) { + PRINT_NAME(); + + LOGF("\t feature_flags = 0x%08" PRIx32 "\n", feature_flags); + + if (!share_feature_flag_valid(feature_flags)) { + return SHARE_ERROR_INVALID_PARAM; + } + + return OK; +} + +} // namespace Share + +LIB_DEFINE(InitShare_1) { + LIB_FUNC("4jt8pMDudgk", Share::ShareCaptureVideoClip); + LIB_FUNC("5wjxESwX68I", Share::ShareFeatureProhibit); + LIB_FUNC("7QZtURYnXG4", Share::ShareSetContentParam); + LIB_FUNC("8qAJ0Jd58-Q", Share::ShareOpenMenuForContent); + LIB_FUNC("AcDNpEpoT9U", Share::ShareCaptureVideoClipExtended); + LIB_FUNC("ErH6tKS7fzE", Share::ShareCaptureScreenshot); + LIB_FUNC("GQTObcITIXI", Share::ShareCaptureScreenshotExtended); + LIB_FUNC("KnsfHKmZqFA", Share::ShareUnregisterContentEventCallback); + LIB_FUNC("ORspsWDXPps", Share::ShareSetContentParamForApplicationTitle); + LIB_FUNC("QNop2YAtIDE", Share::ShareGetCurrentStatus); + LIB_FUNC("Sygnk9dr5WQ", Share::ShareRegisterContentEventCallback); + LIB_FUNC("T64o-315wbg", Share::ShareSetScreenshotOverlayImage); + LIB_FUNC("YBiIdcDPrxs", Share::ShareFeaturePermit); + LIB_FUNC("crFxyW3HdK0", Share::ShareGetRunningStatus); + LIB_FUNC("kCurUZVFqcI", Share::ShareSetCaptureSource); +} + +} // namespace Libs diff --git a/src/libs/libSysmodule.cpp b/src/libs/libSysmodule.cpp new file mode 100644 index 000000000..ac032dbf2 --- /dev/null +++ b/src/libs/libSysmodule.cpp @@ -0,0 +1,75 @@ +#include "common/abi.h" +#include "common/assert.h" +#include "common/common.h" +#include "common/logging/log.h" +#include "common/stringUtils.h" +#include "libs/libs.h" +#include "loader/symbolDatabase.h" + +namespace Libs { + +LIB_VERSION("Sysmodule", 1, "Sysmodule", 1, 1); + +namespace LibKernel { +struct ModuleInfoForUnwind; +int KYTY_SYSV_ABI KernelGetModuleInfoForUnwind(uint64_t addr, int flags, ModuleInfoForUnwind* info); +} // namespace LibKernel + +namespace Sysmodule { + +static KYTY_SYSV_ABI int SysmoduleGetModuleInfoForUnwind(uint64_t addr, int flags, + LibKernel::ModuleInfoForUnwind* info) { + return LibKernel::KernelGetModuleInfoForUnwind(addr, flags, info); +} + +static KYTY_SYSV_ABI int SysmoduleLoadModule(uint16_t id) { + PRINT_NAME(); + + LOGF("\t id = %d\n", static_cast(id)); + + return 0; +} + +static KYTY_SYSV_ABI int SysmoduleUnloadModule(uint16_t id) { + PRINT_NAME(); + + LOGF("\t id = %d\n", static_cast(id)); + + return 0; +} + +static KYTY_SYSV_ABI int SysmoduleLoadModuleInternalWithArg(uint16_t id, int arg1, int arg2, + int arg3, int* ret) { + PRINT_NAME(); + + LOGF("\t id = %d\n", static_cast(id)); + + EXIT_IF(arg1 != 0); + EXIT_IF(arg2 != 0); + EXIT_IF(arg3 != 0); + EXIT_IF(ret == nullptr); + + *ret = 0; + + return 0; +} + +static KYTY_SYSV_ABI int SysmoduleIsLoaded(uint16_t id) { + PRINT_NAME(); + + LOGF("\t id = %d\n", static_cast(id)); + + return 0; +} + +} // namespace Sysmodule + +LIB_DEFINE(InitSysmodule_1) { + LIB_FUNC("4fU5yvOkVG4", Sysmodule::SysmoduleGetModuleInfoForUnwind); + LIB_FUNC("eR2bZFAAU0Q", Sysmodule::SysmoduleUnloadModule); + LIB_FUNC("hHrGoGoNf+s", Sysmodule::SysmoduleLoadModuleInternalWithArg); + LIB_FUNC("g8cM39EUZ6o", Sysmodule::SysmoduleLoadModule); + LIB_FUNC("fMP5NHUOaMk", Sysmodule::SysmoduleIsLoaded); +} + +} // namespace Libs diff --git a/src/libs/libSystemService.cpp b/src/libs/libSystemService.cpp new file mode 100644 index 000000000..3f1537907 --- /dev/null +++ b/src/libs/libSystemService.cpp @@ -0,0 +1,569 @@ +#include "common/abi.h" +#include "common/assert.h" +#include "common/common.h" +#include "common/logging/log.h" +#include "common/stringUtils.h" +#include "libs/errno.h" +#include "libs/libs.h" +#include "loader/symbolDatabase.h" + +#include + +namespace Libs { + +LIB_VERSION("SystemService", 1, "SystemService", 1, 1); + +namespace SystemService { + +[[maybe_unused]] constexpr int PARAM_ID_LANG = 1; +[[maybe_unused]] constexpr int PARAM_ID_DATE_FORMAT = 2; +[[maybe_unused]] constexpr int PARAM_ID_TIME_FORMAT = 3; +[[maybe_unused]] constexpr int PARAM_ID_TIME_ZONE = 4; +[[maybe_unused]] constexpr int PARAM_ID_SUMMERTIME = 5; +[[maybe_unused]] constexpr int PARAM_ID_SYSTEM_NAME = 6; +[[maybe_unused]] constexpr int PARAM_ID_GAME_PARENTAL_LEVEL = 7; +[[maybe_unused]] constexpr int PARAM_ID_ENTER_BUTTON_ASSIGN = 1000; + +[[maybe_unused]] constexpr int PARAM_LANG_JAPANESE = 0; +[[maybe_unused]] constexpr int PARAM_LANG_ENGLISH_US = 1; +[[maybe_unused]] constexpr int PARAM_LANG_FRENCH = 2; +[[maybe_unused]] constexpr int PARAM_LANG_SPANISH = 3; +[[maybe_unused]] constexpr int PARAM_LANG_GERMAN = 4; +[[maybe_unused]] constexpr int PARAM_LANG_ITALIAN = 5; +[[maybe_unused]] constexpr int PARAM_LANG_DUTCH = 6; +[[maybe_unused]] constexpr int PARAM_LANG_PORTUGUESE_PT = 7; +[[maybe_unused]] constexpr int PARAM_LANG_RUSSIAN = 8; +[[maybe_unused]] constexpr int PARAM_LANG_KOREAN = 9; +[[maybe_unused]] constexpr int PARAM_LANG_CHINESE_T = 10; +[[maybe_unused]] constexpr int PARAM_LANG_CHINESE_S = 11; +[[maybe_unused]] constexpr int PARAM_LANG_FINNISH = 12; +[[maybe_unused]] constexpr int PARAM_LANG_SWEDISH = 13; +[[maybe_unused]] constexpr int PARAM_LANG_DANISH = 14; +[[maybe_unused]] constexpr int PARAM_LANG_NORWEGIAN = 15; +[[maybe_unused]] constexpr int PARAM_LANG_POLISH = 16; +[[maybe_unused]] constexpr int PARAM_LANG_PORTUGUESE_BR = 17; +[[maybe_unused]] constexpr int PARAM_LANG_ENGLISH_GB = 18; +[[maybe_unused]] constexpr int PARAM_LANG_TURKISH = 19; +[[maybe_unused]] constexpr int PARAM_LANG_SPANISH_LA = 20; +[[maybe_unused]] constexpr int PARAM_LANG_ARABIC = 21; +[[maybe_unused]] constexpr int PARAM_LANG_FRENCH_CA = 22; +[[maybe_unused]] constexpr int PARAM_LANG_CZECH = 23; +[[maybe_unused]] constexpr int PARAM_LANG_HUNGARIAN = 24; +[[maybe_unused]] constexpr int PARAM_LANG_GREEK = 25; +[[maybe_unused]] constexpr int PARAM_LANG_ROMANIAN = 26; +[[maybe_unused]] constexpr int PARAM_LANG_THAI = 27; +[[maybe_unused]] constexpr int PARAM_LANG_VIETNAMESE = 28; +[[maybe_unused]] constexpr int PARAM_LANG_INDONESIAN = 29; + +[[maybe_unused]] constexpr int PARAM_DATE_FORMAT_YYYYMMDD = 0; +[[maybe_unused]] constexpr int PARAM_DATE_FORMAT_DDMMYYYY = 1; +[[maybe_unused]] constexpr int PARAM_DATE_FORMAT_MMDDYYYY = 2; + +[[maybe_unused]] constexpr int PARAM_TIME_FORMAT_12HOUR = 0; +[[maybe_unused]] constexpr int PARAM_TIME_FORMAT_24HOUR = 1; + +[[maybe_unused]] constexpr int MAX_SYSTEM_NAME_LENGTH = 65; + +[[maybe_unused]] constexpr int PARAM_GAME_PARENTAL_OFF = 0; +[[maybe_unused]] constexpr int PARAM_GAME_PARENTAL_LEVEL01 = 1; +[[maybe_unused]] constexpr int PARAM_GAME_PARENTAL_LEVEL02 = 2; +[[maybe_unused]] constexpr int PARAM_GAME_PARENTAL_LEVEL03 = 3; +[[maybe_unused]] constexpr int PARAM_GAME_PARENTAL_LEVEL04 = 4; +[[maybe_unused]] constexpr int PARAM_GAME_PARENTAL_LEVEL05 = 5; +[[maybe_unused]] constexpr int PARAM_GAME_PARENTAL_LEVEL06 = 6; +[[maybe_unused]] constexpr int PARAM_GAME_PARENTAL_LEVEL07 = 7; +[[maybe_unused]] constexpr int PARAM_GAME_PARENTAL_LEVEL08 = 8; +[[maybe_unused]] constexpr int PARAM_GAME_PARENTAL_LEVEL09 = 9; +[[maybe_unused]] constexpr int PARAM_GAME_PARENTAL_LEVEL10 = 10; +[[maybe_unused]] constexpr int PARAM_GAME_PARENTAL_LEVEL11 = 11; + +[[maybe_unused]] constexpr int PARAM_ENTER_BUTTON_ASSIGN_CIRCLE = 0; +[[maybe_unused]] constexpr int PARAM_ENTER_BUTTON_ASSIGN_CROSS = 1; + +struct SystemServiceStatus { + int32_t event_num = 0; + bool is_system_ui_overlaid = false; + bool is_in_background_execution = false; + bool is_vr_play_area_overlaid = false; + uint8_t reserved[127] = {}; +}; + +struct SystemServiceEvent { + int32_t event_type; + uint8_t data[8192]; +}; + +struct SystemServiceDisplaySafeAreaInfo { + float ratio; + uint8_t reserved[128]; +}; + +struct SystemServiceHdrToneMapLuminance { + float max_full_frame_tone_map_luminance; + float max_tone_map_luminance; + float min_tone_map_luminance; +}; + +static int KYTY_SYSV_ABI SystemServiceHideSplashScreen() { + PRINT_NAME(); + + return OK; +} + +static int KYTY_SYSV_ABI SystemServiceParamGetInt(int param_id, int* value) { + PRINT_NAME(); + + if (value == nullptr) { + return SYSTEM_SERVICE_ERROR_PARAMETER; + } + + int v = 0; + + switch (param_id) { + case PARAM_ID_LANG: v = PARAM_LANG_ENGLISH_US; break; + case PARAM_ID_DATE_FORMAT: v = PARAM_DATE_FORMAT_DDMMYYYY; break; + case PARAM_ID_TIME_FORMAT: v = PARAM_TIME_FORMAT_24HOUR; break; + case PARAM_ID_TIME_ZONE: v = +180; break; + case PARAM_ID_SUMMERTIME: v = 0; break; + case PARAM_ID_GAME_PARENTAL_LEVEL: v = PARAM_GAME_PARENTAL_OFF; break; + case PARAM_ID_ENTER_BUTTON_ASSIGN: v = PARAM_ENTER_BUTTON_ASSIGN_CROSS; break; + default: EXIT("unknown param_id: %d\n", param_id); + } + + LOGF(" %d = %d\n", param_id, v); + + *value = v; + + return OK; +} + +static int KYTY_SYSV_ABI SystemServiceParamGetString(int param_id, char* buf, size_t buf_size) { + PRINT_NAME(); + + if (buf == nullptr || buf_size == 0) { + return SYSTEM_SERVICE_ERROR_PARAMETER; + } + + const char* value = nullptr; + switch (param_id) { + case PARAM_ID_SYSTEM_NAME: value = "Kyty"; break; + default: EXIT("unknown string param_id: %d\n", param_id); + } + + const auto len = std::strlen(value); + if (len + 1 > buf_size) { + return SYSTEM_SERVICE_ERROR_PARAMETER; + } + + std::memcpy(buf, value, len + 1); + LOGF(" %d = %s\n", param_id, value); + + return OK; +} + +static int KYTY_SYSV_ABI SystemServiceReceiveEvent(SystemServiceEvent* event) { + PRINT_NAME(); + + if (event == nullptr) { + return SYSTEM_SERVICE_ERROR_PARAMETER; + } + + event->event_type = -1; + std::memset(event->data, 0, sizeof(event->data)); + + return SYSTEM_SERVICE_ERROR_NO_EVENT; +} + +static int KYTY_SYSV_ABI SystemServiceGetStatus(SystemServiceStatus* status) { + PRINT_NAME(); + + if (status == nullptr) { + return SYSTEM_SERVICE_ERROR_PARAMETER; + } + + *status = SystemServiceStatus(); + + return OK; +} + +static int KYTY_SYSV_ABI +SystemServiceGetDisplaySafeAreaInfo(SystemServiceDisplaySafeAreaInfo* info) { + PRINT_NAME(); + + if (info == nullptr) { + return SYSTEM_SERVICE_ERROR_PARAMETER; + } + + info->ratio = 1.0f; + + return OK; +} + +static int KYTY_SYSV_ABI +SystemServiceGetHdrToneMapLuminance(SystemServiceHdrToneMapLuminance* luminance) { + PRINT_NAME(); + + if (luminance == nullptr) { + return SYSTEM_SERVICE_ERROR_PARAMETER; + } + + luminance->max_full_frame_tone_map_luminance = 80.0f; + luminance->max_tone_map_luminance = 1000.0f; + luminance->min_tone_map_luminance = 0.0f; + + return OK; +} + +static int KYTY_SYSV_ABI SystemServiceGetNoticeScreenSkipFlag(bool* value) { + PRINT_NAME(); + + if (value == nullptr) { + return SYSTEM_SERVICE_ERROR_PARAMETER; + } + + *value = false; + + return OK; +} + +static int KYTY_SYSV_ABI SystemServiceDisableNoticeScreenSkipFlagAutoSet() { + PRINT_NAME(); + + return OK; +} + +static int KYTY_SYSV_ABI SystemServiceSetNoticeScreenSkipFlag() { + PRINT_NAME(); + + return OK; +} + +static int KYTY_SYSV_ABI SystemServicePowerTick() { + PRINT_NAME(); + + return OK; +} + +static int KYTY_SYSV_ABI SystemServiceReportAbnormalTermination(const void* info) { + PRINT_NAME(); + + LOGF("\t info = 0x%016" PRIx64 "\n", reinterpret_cast(info)); + + return OK; +} + +} // namespace SystemService + +namespace SystemGesture { + +LIB_VERSION("SystemGesture", 1, "SystemGesture", 1, 1); + +constexpr int SYSTEM_GESTURE_INPUT_TOUCH_PAD = 0; + +constexpr int SYSTEM_GESTURE_ERROR_INVALID_ARGUMENT = -2138505215; /* 0x80890001 */ +constexpr int SYSTEM_GESTURE_ERROR_EVENT_DATA_NOT_FOUND = -2138505211; /* 0x80890005 */ +constexpr int SYSTEM_GESTURE_ERROR_INVALID_HANDLE = -2138505210; /* 0x80890006 */ + +constexpr int32_t SYSTEM_GESTURE_HANDLE = 1; + +struct SystemGestureVector2 { + float x; + float y; +}; + +struct SystemGesturePrimitiveTouchEvent { + int32_t event_state; + uint16_t primitive_id; + uint8_t is_updated; + uint8_t reserved0; + SystemGestureVector2 pressed_position; + SystemGestureVector2 current_position; + SystemGestureVector2 delta_vector; + uint64_t delta_time; + uint64_t elapsed_time; + uint8_t reserve[32]; +}; + +struct SystemGestureRectangle { + float x; + float y; + float width; + float height; + uint8_t reserve[8]; +}; + +struct SystemGestureTouchRecognizer { + uint64_t reserve[361]; +}; + +struct SystemGestureTouchRecognizerInformation { + int32_t gesture_type; + SystemGestureRectangle rectangle; + uint64_t updated_time; + uint8_t reserve[256]; +}; + +struct SystemGestureTouchEvent { + uint8_t reserve[168]; +}; + +static bool SystemGestureIsValidHandle(int32_t gesture_handle) { + return gesture_handle == SYSTEM_GESTURE_HANDLE; +} + +static int KYTY_SYSV_ABI SystemGestureInitializePrimitiveTouchRecognizer(const void*) { + return OK; +} + +static int KYTY_SYSV_ABI SystemGestureFinalizePrimitiveTouchRecognizer() { + return OK; +} + +static int32_t KYTY_SYSV_ABI SystemGestureOpen(int32_t input_type, const void*) { + if (input_type != SYSTEM_GESTURE_INPUT_TOUCH_PAD) { + return SYSTEM_GESTURE_ERROR_INVALID_ARGUMENT; + } + + return SYSTEM_GESTURE_HANDLE; +} + +static int KYTY_SYSV_ABI SystemGestureClose(int32_t gesture_handle) { + return SystemGestureIsValidHandle(gesture_handle) ? OK : SYSTEM_GESTURE_ERROR_INVALID_HANDLE; +} + +static int KYTY_SYSV_ABI SystemGestureResetPrimitiveTouchRecognizer(int32_t gesture_handle) { + return SystemGestureIsValidHandle(gesture_handle) ? OK : SYSTEM_GESTURE_ERROR_INVALID_HANDLE; +} + +static int KYTY_SYSV_ABI SystemGestureUpdatePrimitiveTouchRecognizer(int32_t gesture_handle, + const void*) { + return SystemGestureIsValidHandle(gesture_handle) ? OK : SYSTEM_GESTURE_ERROR_INVALID_HANDLE; +} + +static int KYTY_SYSV_ABI SystemGestureGetPrimitiveTouchEvents( + int32_t gesture_handle, SystemGesturePrimitiveTouchEvent* event_buffer, + uint32_t capacity_of_buffer, uint32_t* number_of_event) { + if (!SystemGestureIsValidHandle(gesture_handle)) { + return SYSTEM_GESTURE_ERROR_INVALID_HANDLE; + } + if (number_of_event == nullptr) { + return SYSTEM_GESTURE_ERROR_INVALID_ARGUMENT; + } + + *number_of_event = 0; + if (event_buffer != nullptr && capacity_of_buffer != 0) { + std::memset(event_buffer, 0, sizeof(SystemGesturePrimitiveTouchEvent)); + } + + return OK; +} + +static int KYTY_SYSV_ABI SystemGestureGetPrimitiveTouchEventsCount(int32_t gesture_handle) { + return SystemGestureIsValidHandle(gesture_handle) ? 0 : SYSTEM_GESTURE_ERROR_INVALID_HANDLE; +} + +static int KYTY_SYSV_ABI SystemGestureGetPrimitiveTouchEventByIndex( + int32_t gesture_handle, uint32_t, SystemGesturePrimitiveTouchEvent* event) { + if (!SystemGestureIsValidHandle(gesture_handle)) { + return SYSTEM_GESTURE_ERROR_INVALID_HANDLE; + } + if (event != nullptr) { + std::memset(event, 0, sizeof(SystemGesturePrimitiveTouchEvent)); + } + return SYSTEM_GESTURE_ERROR_EVENT_DATA_NOT_FOUND; +} + +static int KYTY_SYSV_ABI SystemGestureGetPrimitiveTouchEventByPrimitiveID( + int32_t gesture_handle, uint16_t, SystemGesturePrimitiveTouchEvent* event) { + if (!SystemGestureIsValidHandle(gesture_handle)) { + return SYSTEM_GESTURE_ERROR_INVALID_HANDLE; + } + if (event != nullptr) { + std::memset(event, 0, sizeof(SystemGesturePrimitiveTouchEvent)); + } + return SYSTEM_GESTURE_ERROR_EVENT_DATA_NOT_FOUND; +} + +static int KYTY_SYSV_ABI +SystemGestureCreateTouchRecognizer(int32_t gesture_handle, SystemGestureTouchRecognizer* recognizer, + int32_t, const SystemGestureRectangle*, const void*) { + if (!SystemGestureIsValidHandle(gesture_handle)) { + return SYSTEM_GESTURE_ERROR_INVALID_HANDLE; + } + if (recognizer == nullptr) { + return SYSTEM_GESTURE_ERROR_INVALID_ARGUMENT; + } + + std::memset(recognizer, 0, sizeof(SystemGestureTouchRecognizer)); + + return OK; +} + +static int KYTY_SYSV_ABI SystemGestureAppendTouchRecognizer( + int32_t gesture_handle, SystemGestureTouchRecognizer* recognizer) { + if (!SystemGestureIsValidHandle(gesture_handle)) { + return SYSTEM_GESTURE_ERROR_INVALID_HANDLE; + } + return (recognizer != nullptr ? OK : SYSTEM_GESTURE_ERROR_INVALID_ARGUMENT); +} + +static int KYTY_SYSV_ABI SystemGestureRemoveTouchRecognizer( + int32_t gesture_handle, SystemGestureTouchRecognizer* recognizer) { + if (!SystemGestureIsValidHandle(gesture_handle)) { + return SYSTEM_GESTURE_ERROR_INVALID_HANDLE; + } + return (recognizer != nullptr ? OK : SYSTEM_GESTURE_ERROR_INVALID_ARGUMENT); +} + +static int KYTY_SYSV_ABI SystemGestureResetTouchRecognizer( + int32_t gesture_handle, SystemGestureTouchRecognizer* recognizer) { + if (!SystemGestureIsValidHandle(gesture_handle)) { + return SYSTEM_GESTURE_ERROR_INVALID_HANDLE; + } + if (recognizer != nullptr) { + std::memset(recognizer, 0, sizeof(SystemGestureTouchRecognizer)); + } + return (recognizer != nullptr ? OK : SYSTEM_GESTURE_ERROR_INVALID_ARGUMENT); +} + +static int KYTY_SYSV_ABI SystemGestureGetTouchRecognizerInformation( + int32_t gesture_handle, const SystemGestureTouchRecognizer* recognizer, + SystemGestureTouchRecognizerInformation* information) { + if (!SystemGestureIsValidHandle(gesture_handle)) { + return SYSTEM_GESTURE_ERROR_INVALID_HANDLE; + } + if (recognizer == nullptr || information == nullptr) { + return SYSTEM_GESTURE_ERROR_INVALID_ARGUMENT; + } + + std::memset(information, 0, sizeof(SystemGestureTouchRecognizerInformation)); + + return OK; +} + +static int KYTY_SYSV_ABI SystemGestureUpdateTouchRecognizer( + int32_t gesture_handle, SystemGestureTouchRecognizer* recognizer) { + if (!SystemGestureIsValidHandle(gesture_handle)) { + return SYSTEM_GESTURE_ERROR_INVALID_HANDLE; + } + return (recognizer != nullptr ? OK : SYSTEM_GESTURE_ERROR_INVALID_ARGUMENT); +} + +static int KYTY_SYSV_ABI SystemGestureUpdateAllTouchRecognizer(int32_t gesture_handle) { + return SystemGestureIsValidHandle(gesture_handle) ? OK : SYSTEM_GESTURE_ERROR_INVALID_HANDLE; +} + +static int KYTY_SYSV_ABI SystemGestureUpdateTouchRecognizerRectangle( + int32_t gesture_handle, SystemGestureTouchRecognizer* recognizer, + const SystemGestureRectangle* rectangle) { + if (!SystemGestureIsValidHandle(gesture_handle)) { + return SYSTEM_GESTURE_ERROR_INVALID_HANDLE; + } + if (recognizer == nullptr || rectangle == nullptr) { + return SYSTEM_GESTURE_ERROR_INVALID_ARGUMENT; + } + return OK; +} + +static int KYTY_SYSV_ABI SystemGestureGetTouchEvents(int32_t gesture_handle, + const SystemGestureTouchRecognizer* recognizer, + SystemGestureTouchEvent* event_buffer, + uint32_t capacity_of_buffer, + uint32_t* number_of_event) { + if (!SystemGestureIsValidHandle(gesture_handle)) { + return SYSTEM_GESTURE_ERROR_INVALID_HANDLE; + } + if (recognizer == nullptr || number_of_event == nullptr) { + return SYSTEM_GESTURE_ERROR_INVALID_ARGUMENT; + } + + *number_of_event = 0; + if (event_buffer != nullptr && capacity_of_buffer != 0) { + std::memset(event_buffer, 0, sizeof(SystemGestureTouchEvent)); + } + + return OK; +} + +static int KYTY_SYSV_ABI SystemGestureGetTouchEventsCount( + int32_t gesture_handle, const SystemGestureTouchRecognizer* recognizer) { + if (!SystemGestureIsValidHandle(gesture_handle)) { + return SYSTEM_GESTURE_ERROR_INVALID_HANDLE; + } + return (recognizer != nullptr ? 0 : SYSTEM_GESTURE_ERROR_INVALID_ARGUMENT); +} + +static int KYTY_SYSV_ABI SystemGestureGetTouchEventByIndex( + int32_t gesture_handle, const SystemGestureTouchRecognizer* recognizer, uint32_t, + SystemGestureTouchEvent* event) { + if (!SystemGestureIsValidHandle(gesture_handle)) { + return SYSTEM_GESTURE_ERROR_INVALID_HANDLE; + } + if (recognizer == nullptr) { + return SYSTEM_GESTURE_ERROR_INVALID_ARGUMENT; + } + if (event != nullptr) { + std::memset(event, 0, sizeof(SystemGestureTouchEvent)); + } + return SYSTEM_GESTURE_ERROR_EVENT_DATA_NOT_FOUND; +} + +static int KYTY_SYSV_ABI SystemGestureGetTouchEventByEventID( + int32_t gesture_handle, const SystemGestureTouchRecognizer* recognizer, uint32_t, + SystemGestureTouchEvent* event) { + if (!SystemGestureIsValidHandle(gesture_handle)) { + return SYSTEM_GESTURE_ERROR_INVALID_HANDLE; + } + if (recognizer == nullptr) { + return SYSTEM_GESTURE_ERROR_INVALID_ARGUMENT; + } + if (event != nullptr) { + std::memset(event, 0, sizeof(SystemGestureTouchEvent)); + } + return SYSTEM_GESTURE_ERROR_EVENT_DATA_NOT_FOUND; +} + +LIB_DEFINE(InitSystemGesture_1) { + LIB_FUNC("qpo-mEOwje0", SystemGesture::SystemGestureOpen); + LIB_FUNC("j4yXIA2jJ68", SystemGesture::SystemGestureClose); + LIB_FUNC("3pcAvmwKCvM", SystemGesture::SystemGestureInitializePrimitiveTouchRecognizer); + LIB_FUNC("3QYCmMlOlCY", SystemGesture::SystemGestureFinalizePrimitiveTouchRecognizer); + LIB_FUNC("o11J529VaAE", SystemGesture::SystemGestureResetPrimitiveTouchRecognizer); + LIB_FUNC("GgFMb22sbbI", SystemGesture::SystemGestureUpdatePrimitiveTouchRecognizer); + LIB_FUNC("L8YmemOeSNY", SystemGesture::SystemGestureGetPrimitiveTouchEvents); + LIB_FUNC("JhwByySf9FY", SystemGesture::SystemGestureGetPrimitiveTouchEventsCount); + LIB_FUNC("KAeP0+cQPVU", SystemGesture::SystemGestureGetPrimitiveTouchEventByIndex); + LIB_FUNC("yBaQ0h9m1NM", SystemGesture::SystemGestureGetPrimitiveTouchEventByPrimitiveID); + LIB_FUNC("FWF8zkhr854", SystemGesture::SystemGestureCreateTouchRecognizer); + LIB_FUNC("1MMK0W-kMgA", SystemGesture::SystemGestureAppendTouchRecognizer); + LIB_FUNC("ELvBVG-LKT0", SystemGesture::SystemGestureRemoveTouchRecognizer); + LIB_FUNC("oBuH3zFWYIg", SystemGesture::SystemGestureResetTouchRecognizer); + LIB_FUNC("0KrW5eMnrwY", SystemGesture::SystemGestureGetTouchRecognizerInformation); + LIB_FUNC("j4h82CQWENo", SystemGesture::SystemGestureUpdateTouchRecognizer); + LIB_FUNC("wPJGwI2RM2I", SystemGesture::SystemGestureUpdateAllTouchRecognizer); + LIB_FUNC("4WOA1eTx3V8", SystemGesture::SystemGestureUpdateTouchRecognizerRectangle); + LIB_FUNC("fLTseA7XiWY", SystemGesture::SystemGestureGetTouchEvents); + LIB_FUNC("h8uongcBNVs", SystemGesture::SystemGestureGetTouchEventsCount); + LIB_FUNC("TSKvgSz5ChU", SystemGesture::SystemGestureGetTouchEventByIndex); + LIB_FUNC("lpsXm7tzeoc", SystemGesture::SystemGestureGetTouchEventByEventID); +} + +} // namespace SystemGesture + +LIB_DEFINE(InitSystemService_1) { + LIB_FUNC("Vo5V8KAwCmk", SystemService::SystemServiceHideSplashScreen); + LIB_FUNC("fZo48un7LK4", SystemService::SystemServiceParamGetInt); + LIB_FUNC("SsC-m-S9JTA", SystemService::SystemServiceParamGetString); + LIB_FUNC("656LMQSrg6U", SystemService::SystemServiceReceiveEvent); + LIB_FUNC("rPo6tV8D9bM", SystemService::SystemServiceGetStatus); + LIB_FUNC("1n37q1Bvc5Y", SystemService::SystemServiceGetDisplaySafeAreaInfo); + LIB_FUNC("mPpPxv5CZt4", SystemService::SystemServiceGetHdrToneMapLuminance); + LIB_FUNC("3RQ5aQfnstU", SystemService::SystemServiceGetNoticeScreenSkipFlag); + LIB_FUNC("8Lo6Zv94aho", SystemService::SystemServiceDisableNoticeScreenSkipFlagAutoSet); + LIB_FUNC("Q3utJvma4Mo", SystemService::SystemServiceSetNoticeScreenSkipFlag); + LIB_FUNC("XbbJC3E+L5M", SystemService::SystemServicePowerTick); + LIB_FUNC("3s8cHiCBKBE", SystemService::SystemServiceReportAbnormalTermination); + SystemGesture::InitSystemGesture_1(s); +} + +} // namespace Libs diff --git a/src/libs/libUlt.cpp b/src/libs/libUlt.cpp new file mode 100644 index 000000000..30d81e11f --- /dev/null +++ b/src/libs/libUlt.cpp @@ -0,0 +1,566 @@ +#include "common/assert.h" +#include "common/logging/log.h" +#include "kernel/pthread.h" +#include "libs/errno.h" +#include "libs/libs.h" +#include "loader/symbolDatabase.h" + +#include +#include +#include +#include +#include +#include + +namespace Libs { + +namespace LibUlt { + +LIB_VERSION("Ult", 1, "Ult", 1, 1); + +constexpr int ULT_ERROR_NULL = -2139029503; // 0x80810001 +constexpr int ULT_ERROR_INVALID = -2139029500; // 0x80810004 +constexpr int ULT_ERROR_STATE = -2139029498; // 0x80810006 +constexpr int ULT_ERROR_AGAIN = -2139029496; // 0x80810008 + +struct UltMutexOptParam { + uint32_t reserved_header[2]; + uint32_t attribute; + uint32_t reserved0; +}; + +struct UltUlthreadRuntimeOptParam { + uint8_t bytes[128]; +}; + +struct UltMutexState { + std::recursive_mutex mutex; + uint32_t attribute = 0; +}; + +struct UltResourcePoolState { + uint32_t num_threads = 0; + uint32_t num_sync_objects = 0; + void* work_area = nullptr; +}; + +struct UltQueueDataResourcePoolState { + uint32_t num_data = 0; + uint64_t data_size = 0; + uint32_t num_queue_object = 0; + void* waiting_pool = nullptr; + void* work_area = nullptr; +}; + +struct UltQueueState { + std::mutex mutex; + std::deque> items; + uint64_t data_size = 0; + uint32_t capacity = 0; + void* waiting_pool = nullptr; + void* data_pool = nullptr; +}; + +struct UltRuntimeState { + uint32_t max_num_ulthread = 0; + uint32_t num_worker_thread = 0; + void* work_area = nullptr; +}; + +using UltUlthreadEntry = KYTY_SYSV_ABI int32_t (*)(uint64_t); + +struct UltUlthreadState { + UltUlthreadEntry entry = nullptr; + uint64_t arg = 0; + LibKernel::Pthread thread = nullptr; +}; + +static std::mutex g_ult_mutex; +static std::unordered_map> g_ult_mutexes; +static std::unordered_map g_ult_resource_pools; +static std::unordered_map g_ult_queue_data_pools; +static std::unordered_map> g_ult_queues; +static std::unordered_map g_ult_runtimes; +static std::unordered_map> g_ult_ulthreads; + +static uint64_t ult_align_up(uint64_t value, uint64_t alignment) { + return (value + alignment - 1u) & ~(alignment - 1u); +} + +static int UltQueueGetState(void* queue, std::shared_ptr* state) { + EXIT_IF(state == nullptr); + + std::scoped_lock lock(g_ult_mutex); + auto it = g_ult_queues.find(queue); + if (it == g_ult_queues.end()) { + return (queue == nullptr ? ULT_ERROR_NULL : ULT_ERROR_STATE); + } + + *state = it->second; + return OK; +} + +static int KYTY_SYSV_ABI UltInitialize() { + PRINT_NAME(); + + return OK; +} + +static int KYTY_SYSV_ABI UltFinalize() { + PRINT_NAME(); + + std::scoped_lock lock(g_ult_mutex); + g_ult_mutexes.clear(); + g_ult_resource_pools.clear(); + g_ult_queue_data_pools.clear(); + g_ult_queues.clear(); + g_ult_runtimes.clear(); + g_ult_ulthreads.clear(); + + return OK; +} + +static int KYTY_SYSV_ABI UltUlthreadRuntimeOptParamInitialize(UltUlthreadRuntimeOptParam* opt_param, + uint32_t build_version) { + PRINT_NAME(); + + if (opt_param == nullptr) { + return ULT_ERROR_NULL; + } + + LOGF("\t opt_param = 0x%016" PRIx64 "\n" + "\t build_version = 0x%08" PRIx32 "\n", + reinterpret_cast(opt_param), build_version); + + std::memset(opt_param, 0, sizeof(*opt_param)); + + return OK; +} + +static uint64_t KYTY_SYSV_ABI UltUlthreadRuntimeGetWorkAreaSize(uint32_t max_num_ulthread, + uint32_t num_worker_thread) { + PRINT_NAME(); + + return ult_align_up(static_cast(max_num_ulthread) * 256u + + static_cast(num_worker_thread) * 16u * 1024u, + 8u); +} + +static int KYTY_SYSV_ABI UltUlthreadRuntimeCreate(void* runtime, const char* name, + uint32_t max_num_ulthread, + uint32_t num_worker_thread, void* work_area, + const void* opt_param, uint32_t build_version) { + PRINT_NAME(); + + if (runtime == nullptr) { + return ULT_ERROR_NULL; + } + + LOGF("\t runtime = 0x%016" PRIx64 "\n" + "\t name = 0x%016" PRIx64 "\n" + "\t max_num_ulthread = %" PRIu32 "\n" + "\t num_worker_thread = %" PRIu32 "\n" + "\t work_area = 0x%016" PRIx64 "\n" + "\t opt_param = 0x%016" PRIx64 "\n" + "\t build_version = 0x%08" PRIx32 "\n", + reinterpret_cast(runtime), reinterpret_cast(name), max_num_ulthread, + num_worker_thread, reinterpret_cast(work_area), + reinterpret_cast(opt_param), build_version); + + std::memset(runtime, 0, 4096); + + std::scoped_lock lock(g_ult_mutex); + g_ult_runtimes[runtime] = {max_num_ulthread, num_worker_thread, work_area}; + + return OK; +} + +static KYTY_SYSV_ABI void* UltUlthreadRun(void* arg) { + auto* state = static_cast(arg); + return reinterpret_cast(static_cast(state->entry(state->arg))); +} + +static int KYTY_SYSV_ABI UltUlthreadCreate(void* ulthread, const char* name, UltUlthreadEntry entry, + uint64_t arg, void* context, uint64_t size_context, + void* runtime, const void* opt_param, + uint32_t build_version) { + PRINT_NAME(); + + LOGF("\t ulthread = 0x%016" PRIx64 "\n" + "\t name = 0x%016" PRIx64 "\n" + "\t entry = 0x%016" PRIx64 "\n" + "\t arg = 0x%016" PRIx64 "\n" + "\t context = 0x%016" PRIx64 "\n" + "\t size_context = 0x%016" PRIx64 "\n" + "\t runtime = 0x%016" PRIx64 "\n" + "\t opt_param = 0x%016" PRIx64 "\n" + "\t build_version = 0x%08" PRIx32 "\n", + reinterpret_cast(ulthread), reinterpret_cast(name), + reinterpret_cast(entry), arg, reinterpret_cast(context), size_context, + reinterpret_cast(runtime), reinterpret_cast(opt_param), build_version); + + if (ulthread == nullptr || entry == nullptr || context == nullptr || runtime == nullptr) { + return ULT_ERROR_NULL; + } + + auto state = std::make_shared(); + state->entry = entry; + state->arg = arg; + + { + std::scoped_lock lock(g_ult_mutex); + if (g_ult_runtimes.find(runtime) == g_ult_runtimes.end()) { + return ULT_ERROR_STATE; + } + if (g_ult_ulthreads.find(ulthread) != g_ult_ulthreads.end()) { + return ULT_ERROR_STATE; + } + std::memset(ulthread, 0, 512); + g_ult_ulthreads[ulthread] = state; + } + + const auto result = LibKernel::PthreadCreate(&state->thread, nullptr, UltUlthreadRun, + state.get(), name != nullptr ? name : ""); + if (result != OK) { + std::scoped_lock lock(g_ult_mutex); + g_ult_ulthreads.erase(ulthread); + return ULT_ERROR_AGAIN; + } + + return OK; +} + +static int KYTY_SYSV_ABI UltUlthreadJoin(void* ulthread, int32_t* status) { + PRINT_NAME(); + + LOGF("\t ulthread = 0x%016" PRIx64 "\n" + "\t status = 0x%016" PRIx64 "\n", + reinterpret_cast(ulthread), reinterpret_cast(status)); + + if (ulthread == nullptr) { + return ULT_ERROR_NULL; + } + + std::shared_ptr state; + { + std::scoped_lock lock(g_ult_mutex); + auto it = g_ult_ulthreads.find(ulthread); + if (it == g_ult_ulthreads.end()) { + return ULT_ERROR_STATE; + } + state = it->second; + } + + void* result = nullptr; + if (LibKernel::PthreadJoin(state->thread, &result) != OK) { + return ULT_ERROR_STATE; + } + if (status != nullptr) { + *status = static_cast(reinterpret_cast(result)); + } + + std::scoped_lock lock(g_ult_mutex); + g_ult_ulthreads.erase(ulthread); + return OK; +} + +static uint64_t KYTY_SYSV_ABI +UltWaitingQueueResourcePoolGetWorkAreaSize(uint32_t num_threads, uint32_t num_sync_objects) { + PRINT_NAME(); + + return ult_align_up(static_cast(num_threads + num_sync_objects) * 256u, 8u); +} + +static int KYTY_SYSV_ABI UltWaitingQueueResourcePoolCreate(void* pool, const char* name, + uint32_t num_threads, + uint32_t num_sync_objects, + void* work_area, const void* opt_param, + uint32_t build_version) { + PRINT_NAME(); + + if (pool == nullptr) { + return ULT_ERROR_NULL; + } + + LOGF("\t pool = 0x%016" PRIx64 "\n" + "\t name = 0x%016" PRIx64 "\n" + "\t num_threads = %" PRIu32 "\n" + "\t num_sync_objects = %" PRIu32 "\n" + "\t work_area = 0x%016" PRIx64 "\n" + "\t opt_param = 0x%016" PRIx64 "\n" + "\t build_version = 0x%08" PRIx32 "\n", + reinterpret_cast(pool), reinterpret_cast(name), num_threads, + num_sync_objects, reinterpret_cast(work_area), + reinterpret_cast(opt_param), build_version); + + std::memset(pool, 0, 256); + + std::scoped_lock lock(g_ult_mutex); + g_ult_resource_pools[pool] = {num_threads, num_sync_objects, work_area}; + + return OK; +} + +static uint64_t KYTY_SYSV_ABI UltQueueDataResourcePoolGetWorkAreaSize(uint32_t num_data, + uint64_t data_size, + uint32_t num_queue_object) { + PRINT_NAME(); + + LOGF("\t num_data = %" PRIu32 "\n" + "\t data_size = 0x%016" PRIx64 "\n" + "\t num_queue_object = %" PRIu32 "\n", + num_data, data_size, num_queue_object); + + const uint64_t data_area = static_cast(num_data) * ult_align_up(data_size, 8u); + const uint64_t queue_area = static_cast(num_queue_object) * 512u; + + return ult_align_up(data_area + queue_area, 8u); +} + +static int KYTY_SYSV_ABI UltQueueDataResourcePoolCreate(void* pool, const char* name, + uint32_t num_data, uint64_t data_size, + uint32_t num_queue_object, + void* waiting_queue_resource_pool, + void* work_area, const void* opt_param, + uint32_t build_version) { + PRINT_NAME(); + + if (pool == nullptr) { + return ULT_ERROR_NULL; + } + + LOGF("\t pool = 0x%016" PRIx64 "\n" + "\t name = 0x%016" PRIx64 "\n" + "\t num_data = %" PRIu32 "\n" + "\t data_size = 0x%016" PRIx64 "\n" + "\t num_queue_object = %" PRIu32 "\n" + "\t waiting_queue_resource_pool = 0x%016" PRIx64 "\n" + "\t work_area = 0x%016" PRIx64 "\n" + "\t opt_param = 0x%016" PRIx64 "\n" + "\t build_version = 0x%08" PRIx32 "\n", + reinterpret_cast(pool), reinterpret_cast(name), num_data, data_size, + num_queue_object, reinterpret_cast(waiting_queue_resource_pool), + reinterpret_cast(work_area), reinterpret_cast(opt_param), + build_version); + + std::scoped_lock lock(g_ult_mutex); + if (waiting_queue_resource_pool != nullptr && + g_ult_resource_pools.find(waiting_queue_resource_pool) == g_ult_resource_pools.end()) { + return ULT_ERROR_INVALID; + } + + std::memset(pool, 0, 512); + g_ult_queue_data_pools[pool] = {num_data, data_size, num_queue_object, + waiting_queue_resource_pool, work_area}; + + return OK; +} + +static int KYTY_SYSV_ABI UltQueueCreate(void* queue, const char* name, uint64_t data_size, + void* waiting_queue_resource_pool, + void* queue_data_resource_pool, const void* opt_param, + uint32_t build_version) { + PRINT_NAME(); + + if (queue == nullptr) { + return ULT_ERROR_NULL; + } + + LOGF("\t queue = 0x%016" PRIx64 "\n" + "\t name = 0x%016" PRIx64 "\n" + "\t data_size = 0x%016" PRIx64 "\n" + "\t waiting_queue_resource_pool = 0x%016" PRIx64 "\n" + "\t queue_data_resource_pool = 0x%016" PRIx64 "\n" + "\t opt_param = 0x%016" PRIx64 "\n" + "\t build_version = 0x%08" PRIx32 "\n", + reinterpret_cast(queue), reinterpret_cast(name), data_size, + reinterpret_cast(waiting_queue_resource_pool), + reinterpret_cast(queue_data_resource_pool), + reinterpret_cast(opt_param), build_version); + + auto state = std::make_shared(); + state->data_size = data_size; + state->waiting_pool = waiting_queue_resource_pool; + state->data_pool = queue_data_resource_pool; + + std::scoped_lock lock(g_ult_mutex); + auto data_pool = g_ult_queue_data_pools.find(queue_data_resource_pool); + if (data_pool == g_ult_queue_data_pools.end() || + (waiting_queue_resource_pool != nullptr && + g_ult_resource_pools.find(waiting_queue_resource_pool) == g_ult_resource_pools.end())) { + return ULT_ERROR_INVALID; + } + state->capacity = data_pool->second.num_data; + + std::memset(queue, 0, 512); + g_ult_queues[queue] = std::move(state); + + return OK; +} + +static int KYTY_SYSV_ABI UltQueuePush(void* queue, const void* data) { + PRINT_NAME(); + + std::shared_ptr state; + if (auto ret = UltQueueGetState(queue, &state); ret != OK) { + return ret; + } + + if (data == nullptr && state->data_size != 0) { + return ULT_ERROR_NULL; + } + + std::scoped_lock lock(state->mutex); + if (state->capacity != 0 && state->items.size() >= state->capacity) { + static std::atomic log_count {0}; + if (log_count.fetch_add(1) < 32) { + LOGF("\t queue full, dropping pushed item: queue=0x%016" PRIx64 " capacity=%" PRIu32 + " data_size=0x%016" PRIx64 "\n", + reinterpret_cast(queue), state->capacity, state->data_size); + } + return OK; + } + + auto& item = state->items.emplace_back(static_cast(state->data_size)); + if (!item.empty()) { + std::memcpy(item.data(), data, item.size()); + } + + return OK; +} + +static int KYTY_SYSV_ABI UltQueueTryPop(void* queue, void* data) { + PRINT_NAME(); + + std::shared_ptr state; + if (auto ret = UltQueueGetState(queue, &state); ret != OK) { + return ret; + } + + if (data == nullptr && state->data_size != 0) { + return ULT_ERROR_NULL; + } + + std::scoped_lock lock(state->mutex); + if (state->items.empty()) { + return ULT_ERROR_AGAIN; + } + + auto item = std::move(state->items.front()); + state->items.pop_front(); + if (!item.empty()) { + std::memcpy(data, item.data(), item.size()); + } + + return OK; +} + +static int KYTY_SYSV_ABI UltMutexOptParamInitialize(UltMutexOptParam* opt_param, + uint32_t build_version) { + PRINT_NAME(); + + if (opt_param == nullptr) { + return ULT_ERROR_NULL; + } + + LOGF("\t opt_param = 0x%016" PRIx64 "\n" + "\t build_version = 0x%08" PRIx32 "\n", + reinterpret_cast(opt_param), build_version); + + std::memset(opt_param, 0, sizeof(*opt_param)); + + return OK; +} + +static int KYTY_SYSV_ABI UltMutexCreate(void* mutex, const char* name, + void* waiting_queue_resource_pool, + const UltMutexOptParam* opt_param, uint32_t build_version) { + PRINT_NAME(); + + if (mutex == nullptr) { + return ULT_ERROR_NULL; + } + + auto state = std::make_shared(); + state->attribute = (opt_param != nullptr ? opt_param->attribute : 0); + + LOGF("\t mutex = 0x%016" PRIx64 "\n" + "\t name = 0x%016" PRIx64 "\n" + "\t waiting_queue_resource_pool = 0x%016" PRIx64 "\n" + "\t attribute = 0x%08" PRIx32 "\n" + "\t build_version = 0x%08" PRIx32 "\n", + reinterpret_cast(mutex), reinterpret_cast(name), + reinterpret_cast(waiting_queue_resource_pool), state->attribute, build_version); + + std::memset(mutex, 0, 256); + + std::scoped_lock lock(g_ult_mutex); + if (waiting_queue_resource_pool != nullptr && + g_ult_resource_pools.find(waiting_queue_resource_pool) == g_ult_resource_pools.end()) { + return ULT_ERROR_INVALID; + } + g_ult_mutexes[mutex] = state; + + return OK; +} + +static int KYTY_SYSV_ABI UltMutexLock(void* mutex) { + PRINT_NAME(); + + std::shared_ptr state; + { + std::scoped_lock lock(g_ult_mutex); + auto it = g_ult_mutexes.find(mutex); + if (it == g_ult_mutexes.end()) { + return (mutex == nullptr ? ULT_ERROR_NULL : ULT_ERROR_STATE); + } + state = it->second; + } + + state->mutex.lock(); + + return OK; +} + +static int KYTY_SYSV_ABI UltMutexUnlock(void* mutex) { + PRINT_NAME(); + + std::shared_ptr state; + { + std::scoped_lock lock(g_ult_mutex); + auto it = g_ult_mutexes.find(mutex); + if (it == g_ult_mutexes.end()) { + return (mutex == nullptr ? ULT_ERROR_NULL : ULT_ERROR_STATE); + } + state = it->second; + } + + state->mutex.unlock(); + + return OK; +} + +LIB_DEFINE(InitUlt_1) { + LIB_FUNC("hZIg1EWGsHM", UltInitialize); + LIB_FUNC("d-kSG2fLrvI", UltFinalize); + LIB_FUNC("V2u3WLrwh64", UltUlthreadRuntimeOptParamInitialize); + LIB_FUNC("grs2pbc2awM", UltUlthreadRuntimeGetWorkAreaSize); + LIB_FUNC("jw9FkZBXo-g", UltUlthreadRuntimeCreate); + LIB_FUNC("znI3q8S7KQ4", UltUlthreadCreate); + LIB_FUNC("gCeAI57LGgI", UltUlthreadJoin); + LIB_FUNC("WIWV1Qd7PFU", UltWaitingQueueResourcePoolGetWorkAreaSize); + LIB_FUNC("YiHujOG9vXY", UltWaitingQueueResourcePoolCreate); + LIB_FUNC("evj9YPkS8s4", UltQueueDataResourcePoolGetWorkAreaSize); + LIB_FUNC("TFHm6-N6vks", UltQueueDataResourcePoolCreate); + LIB_FUNC("9Y5keOvb6ok", UltQueueCreate); + LIB_FUNC("dUwpX3e5NDE", UltQueuePush); + LIB_FUNC("uZz3ci7XYqc", UltQueueTryPop); + LIB_FUNC("1+8t9aHLiz8", UltMutexOptParamInitialize); + LIB_FUNC("mmt8Sa6tL6c", UltMutexCreate); + LIB_FUNC("8hEGkR1pfr8", UltMutexLock); + LIB_FUNC("h0XebKiMBtk", UltMutexUnlock); +} + +} // namespace LibUlt + +} // namespace Libs diff --git a/src/libs/libUserService.cpp b/src/libs/libUserService.cpp new file mode 100644 index 000000000..cfcb83e43 --- /dev/null +++ b/src/libs/libUserService.cpp @@ -0,0 +1,279 @@ +#include "common/abi.h" +#include "common/assert.h" +#include "common/stringUtils.h" +#include "libs/errno.h" +#include "libs/libs.h" +#include "loader/symbolDatabase.h" + +#include +#include +#include + +namespace Libs { + +LIB_VERSION("UserService", 1, "UserService", 1, 1); + +namespace UserService { + +struct UserServiceLoginUserIdList { + int user_id[4]; +}; + +enum UserServiceEventType { UserServiceEventTypeLogin, UserServiceEventTypeLogout }; + +struct SceUserServiceEvent { + UserServiceEventType event_type; + int user_id; +}; + +struct UserServiceGamePresets { + size_t this_size; + uint32_t difficulty; + uint32_t priority; + uint32_t invert_vertical_view_for_1st_person_view; + uint32_t invert_horizontal_view_for_1st_person_view; + uint32_t invert_vertical_view_for_3rd_person_view; + uint32_t invert_horizontal_view_for_3rd_person_view; + uint32_t display_sub_titles; + uint32_t audio_language; +}; + +static_assert(sizeof(UserServiceGamePresets) == 40); + +static KYTY_SYSV_ABI int UserServiceInitialize(const void* /*params*/) { + PRINT_NAME(); + + return OK; +} + +static KYTY_SYSV_ABI int UserServiceInitialize2() { + PRINT_NAME(); + + return OK; +} + +static KYTY_SYSV_ABI int UserServiceGetInitialUser(int* user_id) { + PRINT_NAME(); + + EXIT_NOT_IMPLEMENTED(user_id == nullptr); + + *user_id = 1000; + + return OK; +} + +static KYTY_SYSV_ABI int UserServiceGetEvent(SceUserServiceEvent* event) { + PRINT_NAME(); + + EXIT_NOT_IMPLEMENTED(event == nullptr); + + static bool logged_in = false; + + if (!logged_in) { + logged_in = true; + event->event_type = UserServiceEventTypeLogin; + event->user_id = 1000; + return OK; + } + + return USER_SERVICE_ERROR_NO_EVENT; +} + +static KYTY_SYSV_ABI int UserServiceGetLoginUserIdList(UserServiceLoginUserIdList* user_id_list) { + PRINT_NAME(); + + EXIT_NOT_IMPLEMENTED(user_id_list == nullptr); + + user_id_list->user_id[0] = 1000; + user_id_list->user_id[1] = -1; + user_id_list->user_id[2] = -1; + user_id_list->user_id[3] = -1; + + return OK; +} + +static KYTY_SYSV_ABI int UserServiceGetUserName(int user_id, char* name, size_t size) { + EXIT_NOT_IMPLEMENTED(user_id != 1000); + EXIT_NOT_IMPLEMENTED(size < 5); + + int s = snprintf(name, size, "%s", "Kyty"); + + EXIT_NOT_IMPLEMENTED(static_cast(s) >= size); + + return OK; +} + +static KYTY_SYSV_ABI int UserServiceGetUserNumber(int user_id, int32_t* number) { + PRINT_NAME(); + + if (number == nullptr) { + return USER_SERVICE_ERROR_INVALID_ARGUMENT; + } + if (user_id != 1000) { + return USER_SERVICE_ERROR_NOT_LOGGED_IN; + } + + *number = 1; + + return OK; +} + +static KYTY_SYSV_ABI int UserServiceGetGamePresets(int user_id, UserServiceGamePresets* presets) { + PRINT_NAME(); + + if (presets == nullptr) { + return USER_SERVICE_ERROR_INVALID_ARGUMENT; + } + if (user_id != 1000) { + return USER_SERVICE_ERROR_NOT_LOGGED_IN; + } + + auto this_size = presets->this_size; + if (this_size == 0 || this_size > sizeof(UserServiceGamePresets)) { + this_size = sizeof(UserServiceGamePresets); + } + + std::memset(presets, 0, this_size); + presets->this_size = sizeof(UserServiceGamePresets); + + return OK; +} + +static KYTY_SYSV_ABI int UserServiceGetAccessibilityVibration(int user_id, int32_t* vibration) { + PRINT_NAME(); + + if (vibration == nullptr) { + return USER_SERVICE_ERROR_INVALID_ARGUMENT; + } + if (user_id != 1000) { + return USER_SERVICE_ERROR_NOT_LOGGED_IN; + } + + *vibration = 1; + + return OK; +} + +static KYTY_SYSV_ABI int UserServiceGetAccessibilityTriggerEffect(int user_id, + int32_t* trigger_effect) { + PRINT_NAME(); + + if (trigger_effect == nullptr) { + return USER_SERVICE_ERROR_INVALID_ARGUMENT; + } + if (user_id != 1000) { + return USER_SERVICE_ERROR_NOT_LOGGED_IN; + } + + *trigger_effect = 1; + + return OK; +} + +static KYTY_SYSV_ABI int UserServiceGetAgeLevel(int user_id, uint32_t* age_level) { + PRINT_NAME(); + + if (age_level == nullptr) { + return USER_SERVICE_ERROR_INVALID_ARGUMENT; + } + if (user_id != 1000) { + return USER_SERVICE_ERROR_NOT_LOGGED_IN; + } + + *age_level = 9; + + return OK; +} + +static KYTY_SYSV_ABI int UserServiceGetAccessibilityChatTranscription(int user_id, + int32_t* chat_transcription) { + PRINT_NAME(); + + if (chat_transcription == nullptr) { + return USER_SERVICE_ERROR_INVALID_ARGUMENT; + } + if (user_id != 1000) { + return USER_SERVICE_ERROR_NOT_LOGGED_IN; + } + + *chat_transcription = 0; + + return OK; +} + +static KYTY_SYSV_ABI int +UserServiceGetAccessibilityPressAndHoldDelay(int user_id, int32_t* press_and_hold_delay) { + PRINT_NAME(); + + if (press_and_hold_delay == nullptr) { + return USER_SERVICE_ERROR_INVALID_ARGUMENT; + } + if (user_id != 1000) { + return USER_SERVICE_ERROR_NOT_LOGGED_IN; + } + + *press_and_hold_delay = 0; + + return OK; +} + +static KYTY_SYSV_ABI int UserServiceGetAccessibilityZoomEnabled(int user_id, + int32_t* zoom_enabled) { + PRINT_NAME(); + + if (zoom_enabled == nullptr) { + return USER_SERVICE_ERROR_INVALID_ARGUMENT; + } + if (user_id != 1000) { + return USER_SERVICE_ERROR_NOT_LOGGED_IN; + } + + *zoom_enabled = 0; + + return OK; +} + +static KYTY_SYSV_ABI int UserServiceGetAccessibilityZoomFollowFocus(int user_id, + int32_t* zoom_follow_focus) { + PRINT_NAME(); + + if (zoom_follow_focus == nullptr) { + return USER_SERVICE_ERROR_INVALID_ARGUMENT; + } + if (user_id != 1000) { + return USER_SERVICE_ERROR_NOT_LOGGED_IN; + } + + *zoom_follow_focus = 0; + + return OK; +} + +static KYTY_SYSV_ABI int UserServicePlatformPrivacyWs1Stub(uint64_t, uint64_t, uint64_t, uint64_t, + uint64_t, uint64_t) { + return OK; +} + +} // namespace UserService + +LIB_DEFINE(InitUserService_1) { + LIB_FUNC("j3YMu1MVNNo", UserService::UserServiceInitialize); + LIB_FUNC("az-0R6eviZ0", UserService::UserServiceInitialize2); + LIB_FUNC("CdWp0oHWGr0", UserService::UserServiceGetInitialUser); + LIB_FUNC("yH17Q6NWtVg", UserService::UserServiceGetEvent); + LIB_FUNC("fPhymKNvK-A", UserService::UserServiceGetLoginUserIdList); + LIB_FUNC("1xxcMiGu2fo", UserService::UserServiceGetUserName); + LIB_FUNC("bwFjS+bX9mA", UserService::UserServiceGetUserNumber); + LIB_FUNC("qbwy0Ub8b3M", UserService::UserServiceGetUserNumber); + LIB_FUNC("-sD02mFDBh4", UserService::UserServiceGetGamePresets); + LIB_FUNC("woNpu+45RLk", UserService::UserServiceGetAgeLevel); + LIB_FUNC("rnEhHqG-4xo", UserService::UserServiceGetAccessibilityChatTranscription); + LIB_FUNC("ZKJtxdgvzwg", UserService::UserServiceGetAccessibilityPressAndHoldDelay); + LIB_FUNC("qWYHOFwqCxY", UserService::UserServiceGetAccessibilityVibration); + LIB_FUNC("-3Y5GO+-i78", UserService::UserServiceGetAccessibilityTriggerEffect); + LIB_FUNC("hD-H81EN9Vg", UserService::UserServiceGetAccessibilityZoomEnabled); + LIB_FUNC("O6IW1-Dwm-w", UserService::UserServiceGetAccessibilityZoomFollowFocus); + LIB_FUNC("D-CzAxQL0XI", UserService::UserServicePlatformPrivacyWs1Stub); +} + +} // namespace Libs diff --git a/src/libs/libVideoDec2.cpp b/src/libs/libVideoDec2.cpp new file mode 100644 index 000000000..80cfeb33d --- /dev/null +++ b/src/libs/libVideoDec2.cpp @@ -0,0 +1,455 @@ +#include "common/abi.h" +#include "libs/errno.h" +#include "libs/libs.h" +#include "loader/symbolDatabase.h" + +#include +#include +#include +#include + +namespace Libs { + +LIB_VERSION("Videodec2", 1, "Videodec2", 1, 1); + +namespace VideoDec2 { + +constexpr int32_t VIDEODEC2_ERROR_STRUCT_SIZE = -2128805631; // 0x811d0101 +constexpr int32_t VIDEODEC2_ERROR_ARGUMENT_POINTER = -2128805630; // 0x811d0102 +constexpr int32_t VIDEODEC2_ERROR_DECODER_INSTANCE = -2128805629; // 0x811d0103 +constexpr int32_t VIDEODEC2_ERROR_MEMORY_SIZE = -2128805628; // 0x811d0104 +constexpr int32_t VIDEODEC2_ERROR_MEMORY_POINTER = -2128805627; // 0x811d0105 +constexpr int32_t VIDEODEC2_ERROR_FRAME_BUFFER_SIZE = -2128805626; // 0x811d0106 +constexpr int32_t VIDEODEC2_ERROR_FRAME_BUFFER_POINTER = -2128805625; // 0x811d0107 +constexpr int32_t VIDEODEC2_ERROR_CONFIG_INFO = -2128805376; // 0x811d0200 +constexpr int32_t VIDEODEC2_ERROR_COMPUTE_PIPE_ID = -2128805375; // 0x811d0201 +constexpr int32_t VIDEODEC2_ERROR_COMPUTE_QUEUE_ID = -2128805374; // 0x811d0202 +constexpr int32_t VIDEODEC2_ERROR_RESOURCE_TYPE = -2128805373; // 0x811d0203 +constexpr int32_t VIDEODEC2_ERROR_INPUT_QUEUE_DEPTH = -2128805370; // 0x811d0206 +constexpr int32_t VIDEODEC2_ERROR_DPB_FRAME_COUNT = -2128805367; // 0x811d0209 +constexpr int32_t VIDEODEC2_ERROR_FRAME_WIDTH_HEIGHT = -2128805366; // 0x811d020a + +constexpr uint32_t VIDEODEC2_RESOURCE_TYPE_COMPUTE = 1; +constexpr size_t VIDEODEC2_MIN_MEMORY_SIZE = 16ull * 1024ull * 1024ull; +constexpr uint32_t VIDEODEC2_FRAME_FORMAT_DEFAULT = 0; + +using Videodec2Decoder = void*; +using Videodec2ComputeQueue = void*; + +struct Videodec2DecoderConfigInfo { + size_t this_size; + uint32_t resource_type; + uint32_t codec_type; + uint32_t profile; + uint32_t max_level; + int32_t max_frame_width; + int32_t max_frame_height; + int32_t max_dpb_frame_count; + uint32_t decode_input_queue_depth; + Videodec2ComputeQueue compute_queue; + uint64_t cpu_affinity_mask; + int32_t cpu_thread_priority; + bool optimize_progressive_video; + bool check_memory_type; + uint8_t reserved0; + uint8_t reserved1; + void* extra_config_info; +}; + +struct Videodec2DecoderMemoryInfo { + size_t this_size; + size_t cpu_memory_size; + void* cpu_memory; + size_t gpu_memory_size; + void* gpu_memory; + size_t cpu_gpu_memory_size; + void* cpu_gpu_memory; + size_t max_frame_buffer_size; + uint32_t frame_buffer_alignment; + uint32_t reserved0; +}; + +struct Videodec2InputData { + size_t this_size; + void* au_data; + size_t au_size; + uint64_t pts_data; + uint64_t dts_data; + uint64_t attached_data; +}; + +struct Videodec2OutputInfo { + size_t this_size; + bool is_valid; + bool is_error_frame; + uint8_t picture_count; + bool is_discarded_frame; + uint32_t codec_type; + uint32_t frame_width; + uint32_t frame_pitch; + uint32_t frame_height; + void* frame_buffer; + size_t frame_buffer_size; + uint32_t frame_format; + uint32_t frame_pitch_in_bytes; +}; + +struct Videodec2FrameBuffer { + size_t this_size; + void* frame_buffer; + size_t frame_buffer_size; + bool is_accepted; +}; + +struct Videodec2ComputeMemoryInfo { + size_t this_size; + size_t cpu_gpu_memory_size; + void* cpu_gpu_memory; +}; + +struct Videodec2ComputeConfigInfo { + size_t this_size; + uint16_t compute_pipe_id; + uint16_t compute_queue_id; + bool check_memory_type; + uint8_t reserved0; + uint16_t reserved1; +}; + +struct DecoderState { + uint64_t magic; + uint32_t codec_type; +}; + +static_assert(sizeof(Videodec2ComputeMemoryInfo) == 24); +static_assert(sizeof(Videodec2ComputeConfigInfo) == 16); +static_assert(sizeof(Videodec2DecoderConfigInfo) == 72); +static_assert(sizeof(Videodec2DecoderMemoryInfo) == 72); +static_assert(sizeof(Videodec2InputData) == 48); +static_assert(sizeof(Videodec2OutputInfo) == 56); +static_assert(sizeof(Videodec2FrameBuffer) == 32); + +constexpr uint64_t DECODER_MAGIC = 0x4b59545956444543ull; // KYTYVDEC + +static std::mutex g_decoder_mutex; +static std::unordered_set g_decoders; + +static bool IsOutputInfoSizeValid(size_t this_size) { + return this_size == sizeof(Videodec2OutputInfo) || + (this_size | 8u) == sizeof(Videodec2OutputInfo); +} + +static DecoderState* GetDecoder(Videodec2Decoder decoder) { + std::scoped_lock lock(g_decoder_mutex); + return g_decoders.contains(decoder) ? static_cast(decoder) : nullptr; +} + +static void FillNoPictureOutput(const Videodec2FrameBuffer* frame_buffer, + Videodec2OutputInfo* output_info, uint32_t codec_type) { + output_info->is_valid = false; + output_info->is_error_frame = false; + output_info->picture_count = 0; + output_info->is_discarded_frame = false; + output_info->codec_type = codec_type; + output_info->frame_width = 0; + output_info->frame_pitch = 0; + output_info->frame_height = 0; + output_info->frame_buffer = frame_buffer != nullptr ? frame_buffer->frame_buffer : nullptr; + output_info->frame_buffer_size = frame_buffer != nullptr ? frame_buffer->frame_buffer_size : 0; + output_info->frame_format = VIDEODEC2_FRAME_FORMAT_DEFAULT; + output_info->frame_pitch_in_bytes = 0; +} + +static int32_t ValidateDecoderConfig(const Videodec2DecoderConfigInfo* config) { + if (config->resource_type != VIDEODEC2_RESOURCE_TYPE_COMPUTE) { + return VIDEODEC2_ERROR_RESOURCE_TYPE; + } + + if (config->reserved0 != 0 || config->reserved1 != 0) { + return VIDEODEC2_ERROR_CONFIG_INFO; + } + + if (config->decode_input_queue_depth == 0) { + return VIDEODEC2_ERROR_INPUT_QUEUE_DEPTH; + } + + if (config->max_dpb_frame_count < -1 || config->max_dpb_frame_count == 0) { + return VIDEODEC2_ERROR_DPB_FRAME_COUNT; + } + + if (config->max_frame_width < -1 || config->max_frame_height < -1 || + config->max_frame_width == 0 || config->max_frame_height == 0) { + return VIDEODEC2_ERROR_FRAME_WIDTH_HEIGHT; + } + + if (config->compute_queue == nullptr) { + return VIDEODEC2_ERROR_CONFIG_INFO; + } + + return OK; +} + +static int32_t KYTY_SYSV_ABI +QueryComputeMemoryInfo(Videodec2ComputeMemoryInfo* compute_memory_info) { + PRINT_NAME(); + + if (compute_memory_info == nullptr) { + return VIDEODEC2_ERROR_ARGUMENT_POINTER; + } + + if (compute_memory_info->this_size != sizeof(Videodec2ComputeMemoryInfo)) { + return VIDEODEC2_ERROR_STRUCT_SIZE; + } + + compute_memory_info->cpu_gpu_memory_size = VIDEODEC2_MIN_MEMORY_SIZE; + compute_memory_info->cpu_gpu_memory = nullptr; + + return OK; +} + +static int32_t KYTY_SYSV_ABI AllocateComputeQueue( + const Videodec2ComputeConfigInfo* compute_config_info, + const Videodec2ComputeMemoryInfo* compute_memory_info, Videodec2ComputeQueue* compute_queue) { + PRINT_NAME(); + + if (compute_config_info == nullptr || compute_memory_info == nullptr || + compute_queue == nullptr) { + return VIDEODEC2_ERROR_ARGUMENT_POINTER; + } + + if (compute_config_info->this_size != sizeof(Videodec2ComputeConfigInfo) || + compute_memory_info->this_size != sizeof(Videodec2ComputeMemoryInfo)) { + return VIDEODEC2_ERROR_STRUCT_SIZE; + } + + if (compute_config_info->reserved0 != 0 || compute_config_info->reserved1 != 0) { + return VIDEODEC2_ERROR_CONFIG_INFO; + } + + if (compute_config_info->compute_pipe_id > 4) { + return VIDEODEC2_ERROR_COMPUTE_PIPE_ID; + } + + if (compute_config_info->compute_queue_id > 7) { + return VIDEODEC2_ERROR_COMPUTE_QUEUE_ID; + } + + if (compute_memory_info->cpu_gpu_memory_size < VIDEODEC2_MIN_MEMORY_SIZE) { + return VIDEODEC2_ERROR_MEMORY_SIZE; + } + + if (compute_memory_info->cpu_gpu_memory == nullptr) { + return VIDEODEC2_ERROR_MEMORY_POINTER; + } + + *compute_queue = compute_memory_info->cpu_gpu_memory; + + return OK; +} + +static int32_t KYTY_SYSV_ABI ReleaseComputeQueue(Videodec2ComputeQueue compute_queue) { + PRINT_NAME(); + + return compute_queue != nullptr ? OK : VIDEODEC2_ERROR_COMPUTE_QUEUE_ID; +} + +static int32_t KYTY_SYSV_ABI QueryDecoderMemoryInfo(const Videodec2DecoderConfigInfo* config, + Videodec2DecoderMemoryInfo* memory_info) { + PRINT_NAME(); + + if (config == nullptr || memory_info == nullptr) { + return VIDEODEC2_ERROR_ARGUMENT_POINTER; + } + + if (config->this_size != sizeof(Videodec2DecoderConfigInfo) || + memory_info->this_size != sizeof(Videodec2DecoderMemoryInfo)) { + return VIDEODEC2_ERROR_STRUCT_SIZE; + } + + const auto validation_result = ValidateDecoderConfig(config); + if (validation_result != OK) { + return validation_result; + } + + memory_info->cpu_memory_size = VIDEODEC2_MIN_MEMORY_SIZE; + memory_info->cpu_memory = nullptr; + memory_info->gpu_memory_size = VIDEODEC2_MIN_MEMORY_SIZE; + memory_info->gpu_memory = nullptr; + memory_info->cpu_gpu_memory_size = VIDEODEC2_MIN_MEMORY_SIZE; + memory_info->cpu_gpu_memory = nullptr; + memory_info->max_frame_buffer_size = VIDEODEC2_MIN_MEMORY_SIZE; + memory_info->frame_buffer_alignment = 0x100; + memory_info->reserved0 = 0; + + return OK; +} + +static int32_t KYTY_SYSV_ABI CreateDecoder(const Videodec2DecoderConfigInfo* config, + const Videodec2DecoderMemoryInfo* memory_info, + Videodec2Decoder* decoder) { + PRINT_NAME(); + + if (config == nullptr || memory_info == nullptr || decoder == nullptr) { + return VIDEODEC2_ERROR_ARGUMENT_POINTER; + } + + if (config->this_size != sizeof(Videodec2DecoderConfigInfo) || + memory_info->this_size != sizeof(Videodec2DecoderMemoryInfo)) { + return VIDEODEC2_ERROR_STRUCT_SIZE; + } + + const auto validation_result = ValidateDecoderConfig(config); + if (validation_result != OK) { + return validation_result; + } + + if (memory_info->cpu_memory_size < VIDEODEC2_MIN_MEMORY_SIZE || + memory_info->gpu_memory_size < VIDEODEC2_MIN_MEMORY_SIZE || + memory_info->cpu_gpu_memory_size < VIDEODEC2_MIN_MEMORY_SIZE || + memory_info->max_frame_buffer_size < VIDEODEC2_MIN_MEMORY_SIZE) { + return VIDEODEC2_ERROR_MEMORY_SIZE; + } + + if (memory_info->cpu_memory == nullptr || memory_info->gpu_memory == nullptr || + memory_info->cpu_gpu_memory == nullptr) { + return VIDEODEC2_ERROR_MEMORY_POINTER; + } + + auto* state = new DecoderState {}; + state->magic = DECODER_MAGIC; + state->codec_type = config->codec_type; + + { + std::scoped_lock lock(g_decoder_mutex); + g_decoders.insert(state); + } + + *decoder = state; + + return OK; +} + +static int32_t KYTY_SYSV_ABI DeleteDecoder(Videodec2Decoder decoder) { + PRINT_NAME(); + + DecoderState* state = nullptr; + { + std::scoped_lock lock(g_decoder_mutex); + auto it = g_decoders.find(decoder); + if (it == g_decoders.end()) { + return VIDEODEC2_ERROR_DECODER_INSTANCE; + } + state = static_cast(*it); + g_decoders.erase(it); + } + + delete state; + + return OK; +} + +static int32_t KYTY_SYSV_ABI Decode(Videodec2Decoder decoder, const Videodec2InputData* input_data, + Videodec2FrameBuffer* frame_buffer, + Videodec2OutputInfo* output_info) { + PRINT_NAME(); + + const auto* state = GetDecoder(decoder); + if (state == nullptr || state->magic != DECODER_MAGIC) { + return VIDEODEC2_ERROR_DECODER_INSTANCE; + } + + if (input_data == nullptr || frame_buffer == nullptr || output_info == nullptr) { + return VIDEODEC2_ERROR_ARGUMENT_POINTER; + } + + if (input_data->this_size != sizeof(Videodec2InputData) || + frame_buffer->this_size != sizeof(Videodec2FrameBuffer) || + !IsOutputInfoSizeValid(output_info->this_size)) { + return VIDEODEC2_ERROR_STRUCT_SIZE; + } + + if (input_data->au_size != 0 && input_data->au_data == nullptr) { + return VIDEODEC2_ERROR_ARGUMENT_POINTER; + } + + if (frame_buffer->frame_buffer_size == 0) { + return VIDEODEC2_ERROR_FRAME_BUFFER_SIZE; + } + + if (frame_buffer->frame_buffer == nullptr) { + return VIDEODEC2_ERROR_FRAME_BUFFER_POINTER; + } + + frame_buffer->is_accepted = false; + FillNoPictureOutput(frame_buffer, output_info, state->codec_type); + + return OK; +} + +static int32_t KYTY_SYSV_ABI Flush(Videodec2Decoder decoder, Videodec2FrameBuffer* frame_buffer, + Videodec2OutputInfo* output_info) { + PRINT_NAME(); + + const auto* state = GetDecoder(decoder); + if (state == nullptr || state->magic != DECODER_MAGIC) { + return VIDEODEC2_ERROR_DECODER_INSTANCE; + } + + if (frame_buffer == nullptr || output_info == nullptr) { + return VIDEODEC2_ERROR_ARGUMENT_POINTER; + } + + if (frame_buffer->this_size != sizeof(Videodec2FrameBuffer) || + !IsOutputInfoSizeValid(output_info->this_size)) { + return VIDEODEC2_ERROR_STRUCT_SIZE; + } + + frame_buffer->is_accepted = false; + FillNoPictureOutput(frame_buffer, output_info, state->codec_type); + + return OK; +} + +static int32_t KYTY_SYSV_ABI Reset(Videodec2Decoder decoder) { + PRINT_NAME(); + + const auto* state = GetDecoder(decoder); + return state != nullptr && state->magic == DECODER_MAGIC ? OK + : VIDEODEC2_ERROR_DECODER_INSTANCE; +} + +static int32_t KYTY_SYSV_ABI GetPictureInfo(const Videodec2OutputInfo* output_info, + void* /*first_picture_info*/, + void* /*second_picture_info*/) { + PRINT_NAME(); + + if (output_info == nullptr) { + return VIDEODEC2_ERROR_ARGUMENT_POINTER; + } + + if (!IsOutputInfoSizeValid(output_info->this_size)) { + return VIDEODEC2_ERROR_STRUCT_SIZE; + } + + return OK; +} + +LIB_DEFINE(InitVideoDec2_1) { + PRINT_NAME_ENABLE(true); + + LIB_FUNC("RnDibcGCPKw", QueryComputeMemoryInfo); + LIB_FUNC("eD+X2SmxUt4", AllocateComputeQueue); + LIB_FUNC("UvtA3FAiF4Y", ReleaseComputeQueue); + LIB_FUNC("qqMCwlULR+E", QueryDecoderMemoryInfo); + LIB_FUNC("CNNRoRYd8XI", CreateDecoder); + LIB_FUNC("jwImxXRGSKA", DeleteDecoder); + LIB_FUNC("852F5+q6+iM", Decode); + LIB_FUNC("l1hXwscLuCY", Flush); + LIB_FUNC("wJXikG6QFN8", Reset); + LIB_FUNC("NtXRa3dRzU0", GetPictureInfo); + LIB_FUNC("kjrLbcyhEiw", GetPictureInfo); +} + +} // namespace VideoDec2 + +} // namespace Libs diff --git a/src/libs/libVideoOut.cpp b/src/libs/libVideoOut.cpp new file mode 100644 index 000000000..554e2cf83 --- /dev/null +++ b/src/libs/libVideoOut.cpp @@ -0,0 +1,54 @@ +#include "common/abi.h" +#include "graphics/presentation/videoOut.h" +#include "libs/libs.h" +#include "loader/symbolDatabase.h" + +namespace Libs { + +namespace LibGen5 { + +LIB_VERSION("VideoOut", 1, "VideoOut", 1, 1); + +LIB_DEFINE(InitVideoOut_1) { + PRINT_NAME_ENABLE(true); + + LIB_FUNC("Up36PTk687E", VideoOut::VideoOutOpen); + LIB_FUNC("uquVH4-Du78", VideoOut::VideoOutClose); + LIB_FUNC("PjS5uASwcV8", VideoOut::VideoOutSetBufferAttribute2); + LIB_FUNC("rKBUtgRrtbk", VideoOut::VideoOutRegisterBuffers2); + LIB_FUNC("HuViW4HnrOw", VideoOut::VideoOutSubmitChangeBufferAttribute2); + LIB_FUNC("N5KDtkIjjJ4", VideoOut::VideoOutUnregisterBuffers); + LIB_FUNC("CBiu4mCE1DA", VideoOut::VideoOutSetFlipRate); + LIB_FUNC("HXzjK9yI30k", VideoOut::VideoOutAddFlipEvent); + LIB_FUNC("Xru92wHJRmg", VideoOut::VideoOutAddVblankEvent); + LIB_FUNC("keipklF0pMY", VideoOut::VideoOutAddPreVblankStartEvent); + LIB_FUNC("-Ozn0F1AFRg", VideoOut::VideoOutDeleteFlipEvent); + LIB_FUNC("oNOQn3knW6s", VideoOut::VideoOutDeleteVblankEvent); + LIB_FUNC("elWQ9vERF-Q", VideoOut::VideoOutDeletePreVblankStartEvent); + LIB_FUNC("U46NwOiJpys", VideoOut::VideoOutSubmitFlip); + LIB_FUNC("SbU3dwp80lQ", VideoOut::VideoOutGetFlipStatus); + LIB_FUNC("zgXifHT9ErY", VideoOut::VideoOutIsFlipPending); + LIB_FUNC("1FZBKy8HeNU", VideoOut::VideoOutGetVblankStatus); + LIB_FUNC("MTxxrOCeSig", VideoOut::VideoOutSetWindowModeMargins); + LIB_FUNC("kmSe30JTs+E", VideoOut::VideoOutAddOutputModeEvent); + LIB_FUNC("U2JJtSqNKZI", VideoOut::VideoOutGetEventId); + LIB_FUNC("rWUTcKdkUzQ", VideoOut::VideoOutGetEventData); + LIB_FUNC("Mt4QHHkxkOc", VideoOut::VideoOutGetEventCount); + LIB_FUNC("j6RaAUlaLv0", VideoOut::VideoOutWaitVblank); + LIB_FUNC("utPrVdxio-8", VideoOut::VideoOutGetOutputStatus); + LIB_FUNC("+I4K03i3EL0", VideoOut::VideoOutInitializeOutputOptions); + LIB_FUNC("Nv8c-Kb+DUM", VideoOut::VideoOutIsOutputSupported); + LIB_FUNC("w0hLuNarQxY", VideoOut::VideoOutConfigureOutput); + LIB_FUNC("eb-gvTYQcoY", VideoOut::VideoOutLatencyControlWaitBeforeInput); + LIB_FUNC("MCJ8SkzsQxY", VideoOut::VideoOutLatencyMeasureSetStartPoint); + LIB_FUNC("DYhhWbJSeRg", VideoOut::VideoOutColorSettingsSetGamma); + LIB_FUNC("pv9CI5VC+R0", VideoOut::VideoOutAdjustColor); +} + +} // namespace LibGen5 + +LIB_DEFINE(InitVideoOut_1) { + LibGen5::InitVideoOut_1(s); +} + +} // namespace Libs diff --git a/src/libs/libs.cpp b/src/libs/libs.cpp new file mode 100644 index 000000000..a3610bca6 --- /dev/null +++ b/src/libs/libs.cpp @@ -0,0 +1,237 @@ +#include "libs/libs.h" + +#include "common/logging/log.h" +#include "libs/errno.h" +#include "loader/symbolDatabase.h" + +namespace Libs { + +namespace LibcInternal { +LIB_DEFINE(InitLibcInternal_1); +} // namespace LibcInternal + +namespace LibContentDelete { +LIB_DEFINE(InitContentDelete_1); +} // namespace LibContentDelete + +namespace LibContentExport { +LIB_DEFINE(InitContentExport_1); +} // namespace LibContentExport + +namespace LibContentSearch { +LIB_DEFINE(InitContentSearch_1); +} // namespace LibContentSearch + +namespace VideoDec2 { +LIB_DEFINE(InitVideoDec2_1); +} // namespace VideoDec2 + +namespace LibMouse { +LIB_DEFINE(InitMouse_1); +} // namespace LibMouse + +namespace LibKeyboard { +LIB_DEFINE(InitKeyboard_1); +} // namespace LibKeyboard + +namespace LibUlt { +LIB_DEFINE(InitUlt_1); +} // namespace LibUlt + +namespace LibPsml { +LIB_DEFINE(InitPsml_1); +} // namespace LibPsml + +LIB_DEFINE(InitLibC_1); +LIB_DEFINE(InitAppContent_1); +LIB_DEFINE(InitAmpr_1); +LIB_DEFINE(InitAudio_1); +LIB_DEFINE(InitCoredump_1); +LIB_DEFINE(InitDbgAddressSanitizer_1); +LIB_DEFINE(InitDebug_1); +LIB_DEFINE(InitDialog_1); +LIB_DEFINE(InitFiber_1); +LIB_DEFINE(InitFont_1); +LIB_DEFINE(InitFontFt_1); +LIB_DEFINE(InitGraphicsDriver_1); +LIB_DEFINE(InitLibKernel_1); +LIB_DEFINE(InitNet_1); +LIB_DEFINE(InitPad_1); +LIB_DEFINE(InitPlayGo_1); +LIB_DEFINE(InitPngDec_1); +LIB_DEFINE(InitPlatform_1); +LIB_DEFINE(InitRudp_1); +LIB_DEFINE(InitRtc_1); +LIB_DEFINE(InitSaveData_1); +LIB_DEFINE(InitShare_1); +LIB_DEFINE(InitSysmodule_1); +LIB_DEFINE(InitSystemService_1); +LIB_DEFINE(InitUserService_1); +LIB_DEFINE(InitVideoOut_1); + +void InitAll(Loader::SymbolDatabase* s) { + LIB_LOAD(InitAudio_1); + LIB_LOAD(InitAmpr_1); + LIB_LOAD(InitAppContent_1); + LIB_LOAD(InitCoredump_1); + LIB_LOAD(LibContentDelete::InitContentDelete_1); + LIB_LOAD(LibContentExport::InitContentExport_1); + LIB_LOAD(LibContentSearch::InitContentSearch_1); + LIB_LOAD(InitLibC_1); + LIB_LOAD(InitDbgAddressSanitizer_1); + LIB_LOAD(InitDebug_1); + LIB_LOAD(InitDialog_1); + LIB_LOAD(InitFiber_1); + LIB_LOAD(InitFont_1); + LIB_LOAD(InitFontFt_1); + LIB_LOAD(InitGraphicsDriver_1); + LIB_LOAD(InitLibKernel_1); + LIB_LOAD(LibMouse::InitMouse_1); + LIB_LOAD(LibKeyboard::InitKeyboard_1); + LIB_LOAD(InitNet_1); + LIB_LOAD(InitPad_1); + LIB_LOAD(InitPlayGo_1); + LIB_LOAD(LibPsml::InitPsml_1); + LIB_LOAD(InitPngDec_1); + LIB_LOAD(InitPlatform_1); + LIB_LOAD(InitRudp_1); + LIB_LOAD(InitRtc_1); + LIB_LOAD(InitSaveData_1); + LIB_LOAD(InitShare_1); + LIB_LOAD(InitSysmodule_1); + LIB_LOAD(InitSystemService_1); + LIB_LOAD(LibUlt::InitUlt_1); + LIB_LOAD(InitUserService_1); + LIB_LOAD(VideoDec2::InitVideoDec2_1); + LIB_LOAD(InitVideoOut_1); +} + +namespace LibContentExport { + +LIB_VERSION("ContentExport", 1, "ContentExport", 1, 1); + +namespace ContentExport { + +constexpr int CONTENT_EXPORT_ERROR_INVALID_PARAM = -2137182186; /* 0x809D3016 */ + +struct ContentExportInitParam2 { + void* malloc_func; + void* free_func; + void* user_data; + size_t buffer_size; + int64_t reserved0; + int64_t reserved1; +}; + +static bool g_initialized = false; + +static int KYTY_SYSV_ABI ContentExportInit2(const ContentExportInitParam2* init_param) { + PRINT_NAME(); + + LOGF("\t init_param = 0x%016" PRIx64 "\n", reinterpret_cast(init_param)); + + if (init_param == nullptr) { + return CONTENT_EXPORT_ERROR_INVALID_PARAM; + } + + LOGF("\t malloc_func = 0x%016" PRIx64 "\n" + "\t free_func = 0x%016" PRIx64 "\n" + "\t user_data = 0x%016" PRIx64 "\n" + "\t buffer_size = %" PRIu64 "\n", + reinterpret_cast(init_param->malloc_func), + reinterpret_cast(init_param->free_func), + reinterpret_cast(init_param->user_data), + static_cast(init_param->buffer_size)); + + g_initialized = true; + + return OK; +} + +} // namespace ContentExport + +LIB_DEFINE(InitContentExport_1) { + LIB_FUNC("0GnN4QCgIfs", ContentExport::ContentExportInit2); +} + +} // namespace LibContentExport + +namespace LibContentSearch { + +LIB_VERSION("ContentSearch", 1, "ContentSearch", 1, 0); + +namespace ContentSearch { + +constexpr int CONTENT_SEARCH_ERROR_INVALID_PARAM = -2137190397; /* 0x809D1003 */ + +struct ContentSearchInitParam { + size_t memory_size; +}; + +static bool g_initialized = false; + +static int KYTY_SYSV_ABI ContentSearchInit(const ContentSearchInitParam* init_param) { + PRINT_NAME(); + + LOGF("\t init_param = 0x%016" PRIx64 "\n", reinterpret_cast(init_param)); + + if (init_param == nullptr) { + return CONTENT_SEARCH_ERROR_INVALID_PARAM; + } + + LOGF("\t memory_size = %" PRIu64 "\n", static_cast(init_param->memory_size)); + + g_initialized = true; + + return OK; +} + +} // namespace ContentSearch + +LIB_DEFINE(InitContentSearch_1) { + LIB_FUNC("dPj4ZtRcIWk", ContentSearch::ContentSearchInit); +} + +} // namespace LibContentSearch + +namespace LibContentDelete { + +LIB_VERSION("ContentDelete", 1, "ContentDelete", 1, 1); + +namespace ContentDelete { + +constexpr int CONTENT_DELETE_ERROR_INVALID_PARAM = -2137174015; /* 0x809D5001 */ + +struct ContentDeleteInitParam { + char reserved1[4]; + size_t heap_size; + char reserved2[32]; +}; + +static bool g_initialized = false; + +static int KYTY_SYSV_ABI ContentDeleteInitialize(const ContentDeleteInitParam* init_param) { + PRINT_NAME(); + + LOGF("\t init_param = 0x%016" PRIx64 "\n", reinterpret_cast(init_param)); + + if (init_param == nullptr) { + return CONTENT_DELETE_ERROR_INVALID_PARAM; + } + + LOGF("\t heap_size = %" PRIu64 "\n", static_cast(init_param->heap_size)); + + g_initialized = true; + + return OK; +} + +} // namespace ContentDelete + +LIB_DEFINE(InitContentDelete_1) { + LIB_FUNC("zoxb0wEChEM", ContentDelete::ContentDeleteInitialize); +} + +} // namespace LibContentDelete + +} // namespace Libs diff --git a/src/libs/libs.h b/src/libs/libs.h new file mode 100644 index 000000000..dbd82f1d9 --- /dev/null +++ b/src/libs/libs.h @@ -0,0 +1,78 @@ +#ifndef EMULATOR_INCLUDE_EMULATOR_LIBS_LIBS_H_ +#define EMULATOR_INCLUDE_EMULATOR_LIBS_LIBS_H_ + +#include "common/abi.h" +#include "common/logging/log.h" +#include "common/stringUtils.h" +#include "common/threads.h" +#include "loader/timer.h" // IWYU pragma: keep + +// NOLINTNEXTLINE(cppcoreguidelines-macro-usage) +#define PRINT_NAME_ENABLED g_print_name + +// NOLINTNEXTLINE(cppcoreguidelines-macro-usage) +#define PRINT_NAME_ENABLE(flag) PRINT_NAME_ENABLED = flag; + +// NOLINTNEXTLINE(cppcoreguidelines-macro-usage) +#define LIB_DEFINE(name) void name(Loader::SymbolDatabase* s) +// NOLINTNEXTLINE(cppcoreguidelines-macro-usage) +#define LIB_NAME(l, m) \ + [[maybe_unused]] static thread_local bool PRINT_NAME_ENABLED = false; \ + static constexpr char g_library[] = l; \ + static constexpr char g_module[] = m; +// NOLINTNEXTLINE(cppcoreguidelines-macro-usage) +#define LIB_VERSION(l, lv, m, mv1, mv2) \ + LIB_NAME(l, m); \ + static constexpr int g_library_version = lv; \ + static constexpr int g_module_version_major = mv1; \ + static constexpr int g_module_version_minor = mv2; +// NOLINTNEXTLINE(cppcoreguidelines-macro-usage) +#define LIB_USING(n) \ + using n::g_library; \ + using n::g_library_version; \ + using n::g_module; \ + using n::g_module_version_major; \ + using n::g_module_version_minor; +// NOLINTNEXTLINE(cppcoreguidelines-macro-usage) +#define LIB_LOAD(name) name(s) +// NOLINTNEXTLINE(cppcoreguidelines-macro-usage) +#define LIB_ADD(n, f, t) \ + { \ + Loader::SymbolResolve sr {}; \ + sr.name = n; \ + sr.library = g_library; \ + sr.library_version = g_library_version; \ + sr.module = g_module; \ + sr.module_version_major = g_module_version_major; \ + sr.module_version_minor = g_module_version_minor; \ + sr.type = t; \ + auto func = reinterpret_cast(f); \ + const char* dbg_name = "" #f; \ + s->Add(sr, func, dbg_name); \ + } +// NOLINTNEXTLINE(cppcoreguidelines-macro-usage) +#define LIB_OBJECT(n, f) LIB_ADD(n, f, Loader::SymbolType::Object) +// NOLINTNEXTLINE(cppcoreguidelines-macro-usage) +#define LIB_FUNC(n, f) LIB_ADD(n, f, Loader::SymbolType::Func) + +// NOLINTNEXTLINE(cppcoreguidelines-macro-usage) +#define PRINT_NAME() \ + if (PRINT_NAME_ENABLED) { \ + if (Log::GetDirection() != Log::Direction::Silent) { \ + const auto print_name_time = Loader::Timer::GetTime().ToString("HH24:MI:SS.FFF"); \ + LOGF_COLOR(Log::Color::Cyan, "[%d][%s] %s::%s::%s()\n", \ + Common::Thread::GetThreadIdUnique(), print_name_time.c_str(), g_library, \ + g_module, __func__); \ + } \ + } + +namespace Loader { +class SymbolDatabase; +} // namespace Loader + +namespace Libs { + +void InitAll(Loader::SymbolDatabase* s); + +} // namespace Libs +#endif /* EMULATOR_INCLUDE_EMULATOR_LIBS_LIBS_H_ */ diff --git a/src/libs/network.cpp b/src/libs/network.cpp new file mode 100644 index 000000000..dacbd3421 --- /dev/null +++ b/src/libs/network.cpp @@ -0,0 +1,3727 @@ +#if defined(_WIN32) +#ifndef WIN32_LEAN_AND_MEAN +#define WIN32_LEAN_AND_MEAN +#endif +#ifndef NOMINMAX +#define NOMINMAX +#endif +#include +#include +#include +#include +#ifdef s_addr +#undef s_addr +#endif +#else +#include +#endif + +#include "common/assert.h" +#include "common/byteBuffer.h" +#include "common/common.h" +#include "common/logging/log.h" +#include "common/stringUtils.h" +#include "common/threads.h" +#include "kernel/pthread.h" +#include "libs/errno.h" +#include "libs/libs.h" +#include "libs/network.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace Libs::Network { + +class Network { +public: + class Id { + public: + static constexpr int MAX_ID = 65536; + + enum class Type : uint32_t { + Invalid = 0, + Http = 1, + Ssl = 2, + Template = 3, + Connection = 4, + Request = 5, + }; + + explicit Id(int id) + : m_id(static_cast(id) & 0xffffu), m_type(static_cast(id) >> 16u) {} + [[nodiscard]] int ToInt() const { + return static_cast(m_id + (static_cast(m_type) << 16u)); + } + [[nodiscard]] bool IsValid() const { return GetType() != Type::Invalid; } + [[nodiscard]] Type GetType() const { + switch (m_type) { + case static_cast(Type::Http): return Type::Http; + case static_cast(Type::Ssl): return Type::Ssl; + case static_cast(Type::Template): return Type::Template; + case static_cast(Type::Connection): return Type::Connection; + case static_cast(Type::Request): return Type::Request; + default: return Type::Invalid; + } + } + + friend class Network; + + private: + Id() = default; + static Id Invalid() { return {}; } + static Id Create(int net_id, Type type) { + Id r; + r.m_id = net_id; + r.m_type = static_cast(type); + return r; + } + [[nodiscard]] int GetId() const { return static_cast(m_id); } + + uint32_t m_id = 0; + uint32_t m_type = static_cast(Type::Invalid); + }; + + using HttpsCallback = KYTY_SYSV_ABI int (*)(int, unsigned int, void* const*, int, void*); + + Network() = default; + virtual ~Network() = default; + + KYTY_CLASS_NO_COPY(Network); + + int PoolCreate(const char* name, int size); + bool PoolDestroy(int memid); + int ResolverCreate(const char* name, int memid); + bool ResolverValid(int rid); + + Id SslInit(uint64_t pool_size); + bool SslTerm(Id ssl_ctx_id); + bool SslValid(Id ssl_ctx_id); + + Id HttpInit(int memid, Id ssl_ctx_id, uint64_t pool_size); + bool HttpTerm(Id http_ctx_id); + Id HttpCreateTemplate(Id http_ctx_id, const char* user_agent, int http_ver, + bool is_auto_proxy_conf); + bool HttpDeleteTemplate(Id tmpl_id); + bool HttpSetNonblock(Id id, bool enable); + bool HttpsSetSslCallback(Id id, HttpsCallback cbfunc, void* user_arg); + bool HttpsSetMinSslVersion(Id id, uint32_t ssl_version); + bool HttpsDisableOption(Id id, uint32_t ssl_flags); + bool HttpAddRequestHeader(Id id, const char* name, const char* value, bool add); + bool HttpValid(Id http_ctx_id); + bool HttpValidTemplate(Id tmpl_id); + bool HttpValidConnection(Id conn_id); + bool HttpValidRequest(Id req_id); + Id HttpCreateConnection(Id tmpl_id, const char* server_name, const char* scheme, uint16_t port, + bool enable_keep_alive); + Id HttpCreateConnectionWithURL(Id tmpl_id, const char* url, bool enable_keep_alive); + bool HttpDeleteConnection(Id conn_id); + Id HttpCreateRequestWithURL2(Id conn_id, const char* method, const char* url, + uint64_t content_length); + bool HttpSetRequestContentLength(Id req_id, uint64_t content_length); + bool HttpDeleteRequest(Id req_id); + bool HttpSetResolveTimeOut(Id id, uint32_t usec); + bool HttpSetResolveRetry(Id id, int32_t retry); + bool HttpSetConnectTimeOut(Id id, uint32_t usec); + bool HttpSetSendTimeOut(Id id, uint32_t usec); + bool HttpSetRecvTimeOut(Id id, uint32_t usec); + bool HttpSetAutoRedirect(Id id, int enable); + bool HttpSetAuthEnabled(Id id, int enable); + bool HttpMarkRequestSent(Id req_id, int result); + bool HttpGetRequestResponse(Id req_id, int* send_result, int* status_code, const char** headers, + size_t* headers_size, uint64_t* content_length); + +private: + struct Pool { + bool used = false; + std::string name; + int size = 0; + }; + + struct Ssl { + bool used = false; + uint64_t size = 0; + }; + + struct Resolver { + bool used = false; + std::string name; + int memid = 0; + }; + + struct Http { + bool used = false; + uint64_t size = 0; + int memid = 0; + int ssl_ctx_id = 0; + }; + + struct HttpHeader { + std::string name; + std::string value; + }; + + struct HttpBase { + std::vector headers; + bool used = false; + bool nonblock = false; + bool auto_redirect = true; + bool auth_enabled = true; + HttpsCallback ssl_cbfunc = nullptr; + void* ssl_user_arg = nullptr; + uint32_t ssl_flags = 0xA7; + uint32_t min_ssl_version = 0; + int http_ctx_id = 0; + uint32_t resolve_timeout = 1'000000; + int32_t resolve_retry = 4; + uint32_t connect_timeout = 30'000000; + uint32_t send_timeout = 120'000000; + uint32_t recv_timeout = 120'000000; + }; + + struct HttpTemplate: public HttpBase { + std::string user_agent; + int http_ver = 0; + bool is_auto_proxy_conf = true; + }; + + struct HttpConnection: public HttpTemplate { + explicit HttpConnection(const HttpTemplate& tmpl): HttpTemplate(tmpl) {} + // int tmpl_id = 0; + std::string url; + bool enable_keep_alive = false; + }; + + struct HttpRequest: public HttpConnection { + explicit HttpRequest(HttpConnection& conn): HttpConnection(conn) {} + // int conn_id = 0; + std::string method; + std::string url; + uint64_t content_length = 0; + bool send_attempted = false; + int send_result = HTTP_ERROR_BEFORE_SEND; + int status_code = 0; + std::string response_headers; + }; + + HttpBase* FindHttpBase(Id id, bool include_request); + + static constexpr int POOLS_MAX = 32; + static constexpr int RESOLVERS_MAX = 32; + static constexpr int SSL_MAX = 32; + static constexpr int HTTP_MAX = 32; + + Common::Mutex m_mutex; + Pool m_pools[POOLS_MAX]; + Resolver m_resolvers[RESOLVERS_MAX]; + Ssl m_ssl[SSL_MAX]; + Http m_http[HTTP_MAX]; + std::vector m_templates; + std::vector m_connections; + std::vector m_requests; +}; + +static Network* g_net = nullptr; + +KYTY_SUBSYSTEM_INIT(Network) { + EXIT_IF(g_net != nullptr); + + g_net = new Network; +} + +KYTY_SUBSYSTEM_UNEXPECTED_SHUTDOWN(Network) {} + +KYTY_SUBSYSTEM_DESTROY(Network) {} + +int Network::PoolCreate(const char* name, int size) { + Common::LockGuard lock(m_mutex); + + for (int id = 0; id < POOLS_MAX; id++) { + if (!m_pools[id].used) { + m_pools[id].used = true; + m_pools[id].size = size; + m_pools[id].name = std::string(name); + + return id; + } + } + + return -1; +} + +bool Network::PoolDestroy(int memid) { + Common::LockGuard lock(m_mutex); + + if (memid >= 0 && memid < POOLS_MAX && m_pools[memid].used) { + m_pools[memid].used = false; + + return true; + } + + return false; +} + +int Network::ResolverCreate(const char* name, int memid) { + Common::LockGuard lock(m_mutex); + + if (memid < 0 || memid >= POOLS_MAX || !m_pools[memid].used) { + return -1; + } + + for (int id = 0; id < RESOLVERS_MAX; id++) { + if (!m_resolvers[id].used) { + m_resolvers[id].used = true; + m_resolvers[id].name = std::string(name); + m_resolvers[id].memid = memid; + + return id; + } + } + + return -1; +} + +bool Network::ResolverValid(int rid) { + Common::LockGuard lock(m_mutex); + + return rid >= 0 && rid < RESOLVERS_MAX && m_resolvers[rid].used; +} + +Network::Id Network::SslInit(uint64_t pool_size) { + Common::LockGuard lock(m_mutex); + + for (int id = 0; id < SSL_MAX; id++) { + if (!m_ssl[id].used) { + m_ssl[id].used = true; + m_ssl[id].size = pool_size; + + return Id::Create(id, Id::Type::Ssl); + } + } + + return Id::Invalid(); +} + +bool Network::SslTerm(Id ssl_ctx_id) { + Common::LockGuard lock(m_mutex); + + if (ssl_ctx_id.GetType() == Id::Type::Ssl && ssl_ctx_id.GetId() >= 0 && + ssl_ctx_id.GetId() < SSL_MAX && m_ssl[ssl_ctx_id.GetId()].used) { + m_ssl[ssl_ctx_id.GetId()].used = false; + + return true; + } + + return false; +} + +bool Network::SslValid(Id ssl_ctx_id) { + Common::LockGuard lock(m_mutex); + + return ssl_ctx_id.GetType() == Id::Type::Ssl && ssl_ctx_id.GetId() >= 0 && + ssl_ctx_id.GetId() < SSL_MAX && m_ssl[ssl_ctx_id.GetId()].used; +} + +Network::Id Network::HttpInit(int memid, Id ssl_ctx_id, uint64_t pool_size) { + Common::LockGuard lock(m_mutex); + + if (ssl_ctx_id.GetType() == Id::Type::Ssl && ssl_ctx_id.GetId() >= 0 && + ssl_ctx_id.GetId() < SSL_MAX && m_ssl[ssl_ctx_id.GetId()].used && memid >= 0 && + memid < POOLS_MAX && m_pools[memid].used) { + for (int id = 0; id < HTTP_MAX; id++) { + if (!m_http[id].used) { + m_http[id].used = true; + m_http[id].size = pool_size; + m_http[id].ssl_ctx_id = ssl_ctx_id.GetId(); + m_http[id].memid = memid; + + return Id::Create(id, Id::Type::Http); + } + } + } + + return Id::Invalid(); +} + +bool Network::HttpValid(Id http_ctx_id) { + Common::LockGuard lock(m_mutex); + + return (http_ctx_id.GetType() == Id::Type::Http && http_ctx_id.GetId() >= 0 && + http_ctx_id.GetId() < HTTP_MAX && m_http[http_ctx_id.GetId()].used); +} + +bool Network::HttpValidTemplate(Id tmpl_id) { + Common::LockGuard lock(m_mutex); + + return (tmpl_id.GetType() == Id::Type::Template && tmpl_id.GetId() >= 0 && + static_cast(tmpl_id.GetId()) < m_templates.size() && + m_templates[tmpl_id.GetId()].used); +} + +bool Network::HttpValidConnection(Id conn_id) { + Common::LockGuard lock(m_mutex); + + return (conn_id.GetType() == Id::Type::Connection && conn_id.GetId() >= 0 && + static_cast(conn_id.GetId()) < m_connections.size() && + m_connections[conn_id.GetId()].used); +} + +bool Network::HttpValidRequest(Id req_id) { + Common::LockGuard lock(m_mutex); + + return (req_id.GetType() == Id::Type::Request && req_id.GetId() >= 0 && + static_cast(req_id.GetId()) < m_requests.size() && + m_requests[req_id.GetId()].used); +} + +Network::HttpBase* Network::FindHttpBase(Id id, bool include_request) { + const auto index = id.GetId(); + + switch (id.GetType()) { + case Id::Type::Template: + if (index >= 0 && static_cast(index) < m_templates.size() && + m_templates[index].used) { + return &m_templates[index]; + } + break; + case Id::Type::Connection: + if (index >= 0 && static_cast(index) < m_connections.size() && + m_connections[index].used) { + return &m_connections[index]; + } + break; + case Id::Type::Request: + if (include_request && index >= 0 && static_cast(index) < m_requests.size() && + m_requests[index].used) { + return &m_requests[index]; + } + break; + default: break; + } + + return nullptr; +} + +bool Network::HttpTerm(Id http_ctx_id) { + Common::LockGuard lock(m_mutex); + + if (HttpValid(http_ctx_id)) { + m_http[http_ctx_id.GetId()].used = false; + + return true; + } + + return false; +} + +Network::Id Network::HttpCreateTemplate(Id http_ctx_id, const char* user_agent, int http_ver, + bool is_auto_proxy_conf) { + Common::LockGuard lock(m_mutex); + + if (HttpValid(http_ctx_id)) { + HttpTemplate tn {}; + tn.used = true; + tn.http_ver = http_ver; + tn.user_agent = std::string(user_agent); + tn.is_auto_proxy_conf = is_auto_proxy_conf; + tn.http_ctx_id = http_ctx_id.GetId(); + tn.nonblock = false; + + int index = 0; + for (auto& t: m_templates) { + if (!t.used) { + t = tn; + return Id::Create(index, Id::Type::Template); + } + index++; + } + + if (index < Id::MAX_ID) { + m_templates.push_back(tn); + return Id::Create(index, Id::Type::Template); + } + } + + return Id::Invalid(); +} + +Network::Id Network::HttpCreateConnectionWithURL(Id tmpl_id, const char* url, + bool enable_keep_alive) { + Common::LockGuard lock(m_mutex); + + if (url != nullptr && url[0] != '\0' && HttpValidTemplate(tmpl_id)) { + HttpConnection cn(m_templates[tmpl_id.GetId()]); + cn.used = true; + cn.enable_keep_alive = enable_keep_alive; + cn.url = std::string(url); + // cn.tmpl_id = tmpl_id.ToInt(); + + int index = 0; + for (auto& t: m_connections) { + if (!t.used) { + t = cn; + return Id::Create(index, Id::Type::Connection); + } + index++; + } + + if (index < Id::MAX_ID) { + m_connections.push_back(cn); + return Id::Create(index, Id::Type::Connection); + } + } + + return Id::Invalid(); +} + +Network::Id Network::HttpCreateConnection(Id tmpl_id, const char* server_name, const char* scheme, + uint16_t port, bool enable_keep_alive) { + Common::LockGuard lock(m_mutex); + + if (server_name != nullptr && scheme != nullptr && HttpValidTemplate(tmpl_id)) { + HttpConnection cn(m_templates[tmpl_id.GetId()]); + cn.used = true; + cn.enable_keep_alive = enable_keep_alive; + cn.url = std::string(scheme) + "://" + std::string(server_name); + if (port != 0) { + cn.url = cn.url + fmt::format(":{}", static_cast(port)); + } + + int index = 0; + for (auto& t: m_connections) { + if (!t.used) { + t = cn; + return Id::Create(index, Id::Type::Connection); + } + index++; + } + + if (index < Id::MAX_ID) { + m_connections.push_back(cn); + return Id::Create(index, Id::Type::Connection); + } + } + + return Id::Invalid(); +} + +bool Network::HttpDeleteConnection(Id conn_id) { + Common::LockGuard lock(m_mutex); + + if (HttpValidConnection(conn_id)) { + m_connections[conn_id.GetId()].used = false; + + return true; + } + + return false; +} + +Network::Id Network::HttpCreateRequestWithURL2(Id conn_id, const char* method, const char* url, + uint64_t content_length) { + Common::LockGuard lock(m_mutex); + + if (method != nullptr && url != nullptr && url[0] != '\0' && HttpValidConnection(conn_id)) { + HttpRequest cn(m_connections[conn_id.GetId()]); + cn.used = true; + cn.method = std::string(method); + cn.url = std::string(url); + // cn.conn_id = conn_id.ToInt(); + cn.content_length = content_length; + + int index = 0; + for (auto& t: m_requests) { + if (!t.used) { + t = cn; + return Id::Create(index, Id::Type::Request); + } + index++; + } + + if (index < Id::MAX_ID) { + m_requests.push_back(cn); + return Id::Create(index, Id::Type::Request); + } + } + + return Id::Invalid(); +} + +bool Network::HttpDeleteRequest(Id req_id) { + Common::LockGuard lock(m_mutex); + + if (HttpValidRequest(req_id)) { + m_requests[req_id.GetId()].used = false; + + return true; + } + + return false; +} + +bool Network::HttpSetRequestContentLength(Id req_id, uint64_t content_length) { + Common::LockGuard lock(m_mutex); + + if (HttpValidRequest(req_id)) { + m_requests[req_id.GetId()].content_length = content_length; + return true; + } + + return false; +} + +bool Network::HttpMarkRequestSent(Id req_id, int result) { + Common::LockGuard lock(m_mutex); + + if (req_id.GetType() == Id::Type::Request && req_id.GetId() >= 0 && + static_cast(req_id.GetId()) < m_requests.size() && + m_requests[req_id.GetId()].used) { + auto& request = m_requests[req_id.GetId()]; + request.send_attempted = true; + request.send_result = result; + request.status_code = (result == OK ? 200 : 0); + request.response_headers.clear(); + return true; + } + + return false; +} + +bool Network::HttpGetRequestResponse(Id req_id, int* send_result, int* status_code, + const char** headers, size_t* headers_size, + uint64_t* content_length) { + Common::LockGuard lock(m_mutex); + + if (req_id.GetType() != Id::Type::Request || req_id.GetId() < 0 || + static_cast(req_id.GetId()) >= m_requests.size() || + !m_requests[req_id.GetId()].used) { + return false; + } + + const auto& request = m_requests[req_id.GetId()]; + + if (send_result != nullptr) { + *send_result = (request.send_attempted ? request.send_result : HTTP_ERROR_BEFORE_SEND); + } + if (status_code != nullptr) { + *status_code = request.status_code; + } + if (headers != nullptr) { + *headers = request.response_headers.c_str(); + } + if (headers_size != nullptr) { + *headers_size = request.response_headers.size(); + } + if (content_length != nullptr) { + *content_length = 0; + } + + return true; +} + +bool Network::HttpDeleteTemplate(Id tmpl_id) { + Common::LockGuard lock(m_mutex); + + if (HttpValidTemplate(tmpl_id)) { + m_templates[tmpl_id.GetId()].used = false; + + return true; + } + + return false; +} + +bool Network::HttpSetNonblock(Id id, bool enable) { + Common::LockGuard lock(m_mutex); + + HttpBase* base = FindHttpBase(id, true); + + if (base != nullptr) { + base->nonblock = enable; + return true; + } + + return false; +} + +bool Network::HttpsSetSslCallback(Id id, HttpsCallback cbfunc, void* user_arg) { + Common::LockGuard lock(m_mutex); + + HttpBase* base = FindHttpBase(id, true); + + if (base != nullptr) { + base->ssl_cbfunc = cbfunc; + base->ssl_user_arg = user_arg; + return true; + } + + return false; +} + +bool Network::HttpsSetMinSslVersion(Id id, uint32_t ssl_version) { + Common::LockGuard lock(m_mutex); + + HttpBase* base = FindHttpBase(id, true); + + if (base != nullptr) { + base->min_ssl_version = ssl_version; + return true; + } + + return false; +} + +bool Network::HttpsDisableOption(Id id, uint32_t ssl_flags) { + Common::LockGuard lock(m_mutex); + + HttpBase* base = FindHttpBase(id, true); + + if (base != nullptr) { + base->ssl_flags &= ~ssl_flags; + return true; + } + + return false; +} + +bool Network::HttpAddRequestHeader(Id id, const char* name, const char* value, bool add) { + Common::LockGuard lock(m_mutex); + + HttpBase* base = FindHttpBase(id, true); + + if (base != nullptr) { + HttpHeader nh({std::string(name), std::string(value)}); + if (add) { + base->headers.push_back(nh); + } else { + for (auto& h: base->headers) { + if (h.name == nh.name) { + h.value = nh.value; + } + } + } + return true; + } + + return false; +} + +bool Network::HttpSetResolveTimeOut(Id id, uint32_t usec) { + Common::LockGuard lock(m_mutex); + + HttpBase* base = FindHttpBase(id, false); + + if (base != nullptr) { + base->resolve_timeout = usec; + return true; + } + + return false; +} + +bool Network::HttpSetResolveRetry(Id id, int32_t retry) { + Common::LockGuard lock(m_mutex); + + HttpBase* base = FindHttpBase(id, false); + + if (base != nullptr) { + base->resolve_retry = retry; + return true; + } + + return false; +} + +bool Network::HttpSetConnectTimeOut(Id id, uint32_t usec) { + Common::LockGuard lock(m_mutex); + + HttpBase* base = FindHttpBase(id, true); + + if (base != nullptr) { + base->connect_timeout = usec; + return true; + } + + return false; +} + +bool Network::HttpSetSendTimeOut(Id id, uint32_t usec) { + Common::LockGuard lock(m_mutex); + + HttpBase* base = FindHttpBase(id, true); + + if (base != nullptr) { + base->send_timeout = usec; + return true; + } + + return false; +} + +bool Network::HttpSetRecvTimeOut(Id id, uint32_t usec) { + Common::LockGuard lock(m_mutex); + + HttpBase* base = FindHttpBase(id, true); + + if (base != nullptr) { + base->recv_timeout = usec; + return true; + } + + return false; +} + +bool Network::HttpSetAutoRedirect(Id id, int enable) { + Common::LockGuard lock(m_mutex); + + HttpBase* base = FindHttpBase(id, true); + + if (base != nullptr) { + base->auto_redirect = (enable != 0); + return true; + } + + return false; +} + +bool Network::HttpSetAuthEnabled(Id id, int enable) { + Common::LockGuard lock(m_mutex); + + HttpBase* base = FindHttpBase(id, true); + + if (base != nullptr) { + base->auth_enabled = (enable != 0); + return true; + } + + return false; +} + +namespace Net { + +LIB_NAME("Net", "Net"); + +struct NetEtherAddr { + uint8_t data[6] = {0}; +}; + +struct SocketSlot { + bool used = false; +#if defined(_WIN32) + SOCKET socket = INVALID_SOCKET; +#else + int socket = -1; +#endif +}; + +struct NetTimeval { + int64_t tv_sec; + int64_t tv_usec; +}; + +static constexpr int SOCKET_FD_MIN = 128; +static constexpr int SOCKET_FD_MAX = 1024; +static Common::Mutex g_socket_mutex; +static std::array g_sockets; +static std::atomic_bool g_winsock_initialized = false; + +constexpr int EPOLL_ID_BASE = SOCKET_FD_MAX; +constexpr uint32_t EPOLL_IN = 0x00000001; +constexpr uint32_t EPOLL_OUT = 0x00000002; +constexpr uint32_t EPOLL_ERR = 0x00000008; + +struct EpollRegistration { + int id = -1; + NetEpollEvent event {}; +}; + +struct EpollSlot { + bool used = false; + std::string name; + std::vector registrations; +}; + +static std::mutex g_epoll_mutex; +static std::condition_variable g_epoll_changed; +static std::array g_epolls; + +static EpollSlot* GetEpollSlot(int eid) { + const auto index = eid - EPOLL_ID_BASE; + if (index < 0 || index >= static_cast(g_epolls.size())) { + return nullptr; + } + auto& slot = g_epolls[static_cast(index)]; + return slot.used ? &slot : nullptr; +} + +static void RemoveSocketFromEpolls(int id) { + std::lock_guard lock(g_epoll_mutex); + for (auto& epoll: g_epolls) { + if (!epoll.used) { + continue; + } + std::erase_if(epoll.registrations, + [id](const auto& registration) { return registration.id == id; }); + } + g_epoll_changed.notify_all(); +} + +static bool EnsureSocketBackend() { +#if defined(_WIN32) + if (!g_winsock_initialized.load()) { + WSADATA data {}; + if (WSAStartup(MAKEWORD(2, 2), &data) != 0) { + return false; + } + g_winsock_initialized = true; + } +#endif + return true; +} + +static int SetPosixSocketError(int error) { + int posix_error = Posix::POSIX_EIO; +#if defined(_WIN32) + switch (error) { + case WSAEACCES: posix_error = Posix::POSIX_EACCES; break; + case WSAEADDRINUSE: posix_error = Posix::POSIX_EADDRINUSE; break; + case WSAEADDRNOTAVAIL: posix_error = Posix::POSIX_EADDRNOTAVAIL; break; + case WSAEAFNOSUPPORT: posix_error = Posix::POSIX_EAFNOSUPPORT; break; + case WSAEFAULT: posix_error = Posix::POSIX_EFAULT; break; + case WSAEINTR: posix_error = Posix::POSIX_EINTR; break; + case WSAEINVAL: posix_error = Posix::POSIX_EINVAL; break; + case WSAEISCONN: posix_error = Posix::POSIX_EISCONN; break; + case WSAEMFILE: posix_error = Posix::POSIX_EMFILE; break; + case WSAEMSGSIZE: posix_error = Posix::POSIX_EMSGSIZE; break; + case WSAENOBUFS: posix_error = Posix::POSIX_ENOBUFS; break; + case WSAENETDOWN: posix_error = Posix::POSIX_ENETDOWN; break; + case WSAENETRESET: posix_error = Posix::POSIX_ENETRESET; break; + case WSAENETUNREACH: posix_error = Posix::POSIX_ENETUNREACH; break; + case WSAENOTCONN: posix_error = Posix::POSIX_ENOTCONN; break; + case WSAENOTSOCK: posix_error = Posix::POSIX_ENOTSOCK; break; + case WSAEOPNOTSUPP: posix_error = Posix::POSIX_EOPNOTSUPP; break; + case WSAEPROTONOSUPPORT: posix_error = Posix::POSIX_EPROTONOSUPPORT; break; + case WSAESHUTDOWN: posix_error = Posix::POSIX_ESHUTDOWN; break; + case WSAETIMEDOUT: posix_error = Posix::POSIX_ETIMEDOUT; break; + case WSAEWOULDBLOCK: posix_error = Posix::POSIX_EWOULDBLOCK; break; + case WSAECONNABORTED: posix_error = Posix::POSIX_ECONNABORTED; break; + case WSAECONNREFUSED: posix_error = Posix::POSIX_ECONNREFUSED; break; + case WSAECONNRESET: posix_error = Posix::POSIX_ECONNRESET; break; + case WSAEDESTADDRREQ: posix_error = Posix::POSIX_EDESTADDRREQ; break; + case WSAEHOSTUNREACH: posix_error = Posix::POSIX_EHOSTUNREACH; break; + case WSAEINPROGRESS: posix_error = Posix::POSIX_EINPROGRESS; break; + case WSAEALREADY: posix_error = Posix::POSIX_EALREADY; break; + default: break; + } +#else + posix_error = error; +#endif + *Posix::GetErrorAddr() = posix_error; + return -1; +} + +static int SetPosixSocketError() { +#if defined(_WIN32) + return SetPosixSocketError(WSAGetLastError()); +#else + return SetPosixSocketError(errno); +#endif +} + +static int ConvertFamily(int family) { + switch (family) { + case 2: return AF_INET; + case 28: return AF_INET6; + default: return -1; + } +} + +static int ConvertSocketOptionLevel(int level) { + return (level == 0xffff ? SOL_SOCKET : level); +} + +static int ConvertMessageFlags(int flags) { +#if defined(_WIN32) + constexpr int guest_msg_peek = 0x00000002; + constexpr int guest_msg_dontroute = 0x00000004; + constexpr int guest_msg_waitall = 0x00000040; + constexpr int guest_msg_dontwait = 0x00000080; + constexpr int guest_msg_nosignal = 0x00020000; + + int host_flags = 0; + if ((flags & guest_msg_peek) != 0) { + host_flags |= MSG_PEEK; + } + if ((flags & guest_msg_dontroute) != 0) { + host_flags |= MSG_DONTROUTE; + } + if ((flags & guest_msg_waitall) != 0) { + host_flags |= MSG_WAITALL; + } + + flags &= ~(guest_msg_peek | guest_msg_dontroute | guest_msg_waitall | guest_msg_dontwait | + guest_msg_nosignal); + if (flags != 0) { + *Posix::GetErrorAddr() = Posix::POSIX_EOPNOTSUPP; + return -1; + } + + return host_flags; +#else + return flags; +#endif +} + +#if defined(_WIN32) +static int ConvertGuestSockaddr(const void* addr, uint32_t addrlen, sockaddr_storage* out, + int* out_len) { + EXIT_IF(out == nullptr); + EXIT_IF(out_len == nullptr); + + if (addr == nullptr) { + *Posix::GetErrorAddr() = Posix::POSIX_EFAULT; + return -1; + } + + const auto* bytes = static_cast(addr); + if (addrlen < 2) { + *Posix::GetErrorAddr() = Posix::POSIX_EINVAL; + return -1; + } + + const int family = ConvertFamily(bytes[1]); + if (family != AF_INET) { + *Posix::GetErrorAddr() = (family < 0 ? Posix::POSIX_EAFNOSUPPORT : Posix::POSIX_EOPNOTSUPP); + return -1; + } + + if (addrlen < 8) { + *Posix::GetErrorAddr() = Posix::POSIX_EINVAL; + return -1; + } + + std::memset(out, 0, sizeof(*out)); + auto* in = reinterpret_cast(out); + in->sin_family = AF_INET; + std::memcpy(&in->sin_port, bytes + 2, sizeof(in->sin_port)); + std::memcpy(&in->sin_addr, bytes + 4, sizeof(in->sin_addr)); + *out_len = sizeof(sockaddr_in); + return 0; +} + +static int ConvertHostSockaddr(const sockaddr_storage* addr, int addrlen, void* out, + uint32_t* out_len) { + EXIT_IF(addr == nullptr); + EXIT_IF(out_len == nullptr); + + if (out == nullptr) { + *Posix::GetErrorAddr() = Posix::POSIX_EFAULT; + return -1; + } + + if (addr->ss_family != AF_INET || addrlen < static_cast(sizeof(sockaddr_in))) { + *Posix::GetErrorAddr() = Posix::POSIX_EOPNOTSUPP; + return -1; + } + + static constexpr uint32_t guest_addrlen = 16; + if (*out_len < guest_addrlen) { + *out_len = guest_addrlen; + *Posix::GetErrorAddr() = Posix::POSIX_EINVAL; + return -1; + } + + auto* bytes = static_cast(out); + const auto* in = reinterpret_cast(addr); + std::memset(bytes, 0, guest_addrlen); + bytes[0] = static_cast(guest_addrlen); + bytes[1] = 2; + std::memcpy(bytes + 2, &in->sin_port, sizeof(in->sin_port)); + std::memcpy(bytes + 4, &in->sin_addr, sizeof(in->sin_addr)); + *out_len = guest_addrlen; + return 0; +} +#endif + +static bool GetSocketBackend(int guest_fd, SOCKET* out) { + EXIT_IF(out == nullptr); + + if (guest_fd < 0 || guest_fd >= SOCKET_FD_MAX) { + return false; + } + + Common::LockGuard lock(g_socket_mutex); + if (!g_sockets[static_cast(guest_fd)].used) { + return false; + } + + *out = g_sockets[static_cast(guest_fd)].socket; + return true; +} + +bool KYTY_SYSV_ABI IsSocket(int s) { + if (s < 0 || s >= SOCKET_FD_MAX) { + return false; + } + + Common::LockGuard lock(g_socket_mutex); + return g_sockets[static_cast(s)].used; +} + +static bool TakeSocketBackend(int guest_fd, SOCKET* out) { + EXIT_IF(out == nullptr); + + if (guest_fd < 0 || guest_fd >= SOCKET_FD_MAX) { + return false; + } + + Common::LockGuard lock(g_socket_mutex); + auto& slot = g_sockets[static_cast(guest_fd)]; + if (!slot.used) { + return false; + } + + *out = slot.socket; + slot = {}; + return true; +} + +static int AllocSocketFd(SOCKET socket) { + Common::LockGuard lock(g_socket_mutex); + for (int fd = SOCKET_FD_MIN; fd < SOCKET_FD_MAX; fd++) { + auto& slot = g_sockets[static_cast(fd)]; + if (!slot.used) { + slot.used = true; + slot.socket = socket; + return fd; + } + } + + return -1; +} + +static bool GuestFdIsSet(const void* fds, int fd) { + if (fds == nullptr || fd < 0 || fd >= SOCKET_FD_MAX) { + return false; + } + + const auto* words = static_cast(fds); + return (words[fd / 64] & (uint64_t {1} << (fd % 64))) != 0; +} + +static void GuestFdZero(void* fds, int nfds) { + if (fds == nullptr || nfds <= 0) { + return; + } + + std::memset(fds, 0, static_cast((nfds + 63) / 64) * sizeof(uint64_t)); +} + +static void GuestFdSet(void* fds, int fd) { + if (fds == nullptr || fd < 0 || fd >= SOCKET_FD_MAX) { + return; + } + + auto* words = static_cast(fds); + words[fd / 64] |= (uint64_t {1} << (fd % 64)); +} + +int KYTY_SYSV_ABI NetInit() { + PRINT_NAME(); + + EnsureSocketBackend(); + + return OK; +} + +int KYTY_SYSV_ABI NetTerm() { + PRINT_NAME(); + + return OK; +} + +int KYTY_SYSV_ABI NetPoolCreate(const char* name, int size, int flags) { + PRINT_NAME(); + + LOGF("\t name = %s\n" + "\t size = %d\n" + "\t flags = %d\n", + name, size, flags); + + EXIT_IF(g_net == nullptr); + + EXIT_NOT_IMPLEMENTED(flags != 0); + EXIT_NOT_IMPLEMENTED(size == 0); + + int id = g_net->PoolCreate(name, size); + + if (id < 0) { + return NET_ERROR_ENFILE; + } + + return id; +} + +int KYTY_SYSV_ABI NetPoolDestroy(int memid) { + PRINT_NAME(); + + EXIT_IF(g_net == nullptr); + + if (!g_net->PoolDestroy(memid)) { + return NET_ERROR_EBADF; + } + + return OK; +} + +int KYTY_SYSV_ABI NetResolverCreate(const char* name, int memid, int flags) { + PRINT_NAME(); + + LOGF("\t name = %s\n" + "\t memid = %d\n" + "\t flags = %d\n", + name != nullptr ? name : "", memid, flags); + + if (name == nullptr || flags != 0) { + return NET_ERROR_EINVAL; + } + + EXIT_IF(g_net == nullptr); + + const int id = g_net->ResolverCreate(name, memid); + if (id < 0) { + return NET_ERROR_RESOLVER_ENOSPACE; + } + + return id; +} + +int KYTY_SYSV_ABI NetResolverStartNtoa(int rid, const char* hostname, void* addr, int timeout, + int retry, int flags) { + PRINT_NAME(); + + LOGF("\t rid = %d\n" + "\t hostname = %s\n" + "\t timeout = %d\n" + "\t retry = %d\n" + "\t flags = 0x%08x\n", + rid, hostname != nullptr ? hostname : "", timeout, retry, flags); + + (void)timeout; + (void)retry; + + if (hostname == nullptr || addr == nullptr || (flags & ~0x00010000) != 0) { + return NET_ERROR_EINVAL; + } + + EXIT_IF(g_net == nullptr); + if (!g_net->ResolverValid(rid)) { + return NET_ERROR_EBADF; + } + + if ((flags & 0x00010000) == 0 && NetInetPton(2, hostname, addr) == 1) { + return OK; + } + +#if defined(_WIN32) + if (!EnsureSocketBackend()) { + return NET_ERROR_ENETDOWN; + } + + addrinfo hints {}; + hints.ai_family = AF_INET; + hints.ai_socktype = SOCK_STREAM; + + addrinfo* result = nullptr; + const int ret = getaddrinfo(hostname, nullptr, &hints, &result); + if (ret != 0 || result == nullptr) { + if (result != nullptr) { + freeaddrinfo(result); + } + return ret == EAI_NONAME ? NET_ERROR_RESOLVER_ENOHOST : NET_ERROR_RESOLVER_EINTERNAL; + } + + for (auto* ai = result; ai != nullptr; ai = ai->ai_next) { + if (ai->ai_family == AF_INET && ai->ai_addr != nullptr && + ai->ai_addrlen >= sizeof(sockaddr_in)) { + const auto* in = reinterpret_cast(ai->ai_addr); + std::memcpy(addr, &in->sin_addr, sizeof(in->sin_addr)); + freeaddrinfo(result); + return OK; + } + } + + freeaddrinfo(result); + return NET_ERROR_RESOLVER_ENORECORD; +#else + return NET_ERROR_RESOLVER_ENOTIMPLEMENTED; +#endif +} + +int KYTY_SYSV_ABI NetInetPton(int af, const char* src, void* dst) { + PRINT_NAME(); + + if (af != 2) { + return NET_ERROR_EAFNOSUPPORT; + } + if (src == nullptr || dst == nullptr) { + return NET_ERROR_EINVAL; + } + + LOGF("\t src = %.16s\n", src); + + uint32_t octets[4] = {}; + char extra = '\0'; + if (sscanf(src, "%u.%u.%u.%u%c", &octets[0], &octets[1], &octets[2], &octets[3], &extra) != 4) { + return 0; + } + for (auto octet: octets) { + if (octet > 255) { + return 0; + } + } + + auto* out = static_cast(dst); + out[0] = static_cast(octets[0]); + out[1] = static_cast(octets[1]); + out[2] = static_cast(octets[2]); + out[3] = static_cast(octets[3]); + + return 1; +} + +const char* KYTY_SYSV_ABI NetInetNtop(int af, const void* src, char* dst, uint32_t size) { + PRINT_NAME(); + + if (src == nullptr || dst == nullptr || size == 0) { + return nullptr; + } + + if (af != 2 && af != 28) { + return nullptr; + } + +#if defined(_WIN32) + const int win_af = (af == 28 ? AF_INET6 : AF_INET); + if (::InetNtopA(win_af, const_cast(src), dst, size) == nullptr) { + return nullptr; + } +#else + if (::inet_ntop(af, src, dst, size) == nullptr) { + return nullptr; + } +#endif + + return dst; +} + +int KYTY_SYSV_ABI NetEtherNtostr(const NetEtherAddr* n, char* str, size_t len) { + PRINT_NAME(); + + NetEtherAddr zero {}; + + EXIT_NOT_IMPLEMENTED(len != 18); + EXIT_NOT_IMPLEMENTED(n == nullptr); + EXIT_NOT_IMPLEMENTED(str == nullptr); + EXIT_NOT_IMPLEMENTED(memcmp(n->data, zero.data, sizeof(zero.data)) != 0); + + strcpy(str, "00:00:00:00:00:00"); // NOLINT + + return OK; +} + +int KYTY_SYSV_ABI NetGetMacAddress(NetEtherAddr* addr, int flags) { + PRINT_NAME(); + + EXIT_NOT_IMPLEMENTED(addr == nullptr); + EXIT_NOT_IMPLEMENTED(flags != 0); + + memset(addr->data, 0, sizeof(addr->data)); + + return OK; +} + +int KYTY_SYSV_ABI NetGetSockInfo(int s, void* info, int n, int flags) { + PRINT_NAME(); + + LOGF("\t s = %d\n" + "\t info = 0x%016" PRIx64 "\n" + "\t n = %d\n" + "\t flags = %d\n", + s, reinterpret_cast(info), n, flags); + + return OK; +} + +int KYTY_SYSV_ABI EpollCreate(const char* name, int flags) { + if (name == nullptr || flags != 0) { + return SetPosixSocketError(Posix::POSIX_EINVAL); + } + + std::lock_guard lock(g_epoll_mutex); + for (size_t index = 0; index < g_epolls.size(); index++) { + auto& slot = g_epolls[index]; + if (!slot.used) { + slot.used = true; + slot.name = name; + slot.registrations.clear(); + return EPOLL_ID_BASE + static_cast(index); + } + } + + return SetPosixSocketError(Posix::POSIX_EMFILE); +} + +int KYTY_SYSV_ABI EpollControl(int eid, int op, int id, const NetEpollEvent* event) { + constexpr int EPOLL_CTL_ADD = 1; + constexpr int EPOLL_CTL_MOD = 2; + constexpr int EPOLL_CTL_DEL = 3; + + if ((op == EPOLL_CTL_ADD || op == EPOLL_CTL_MOD) && event == nullptr) { + return SetPosixSocketError(Posix::POSIX_EINVAL); + } + if (op == EPOLL_CTL_DEL && event != nullptr) { + return SetPosixSocketError(Posix::POSIX_EINVAL); + } + if (op < EPOLL_CTL_ADD || op > EPOLL_CTL_DEL) { + return SetPosixSocketError(Posix::POSIX_EINVAL); + } + if (!IsSocket(id)) { + return SetPosixSocketError(Posix::POSIX_EBADF); + } + + std::lock_guard lock(g_epoll_mutex); + auto* slot = GetEpollSlot(eid); + if (slot == nullptr) { + return SetPosixSocketError(Posix::POSIX_EBADF); + } + + auto registration = std::find_if(slot->registrations.begin(), slot->registrations.end(), + [id](const auto& value) { return value.id == id; }); + switch (op) { + case EPOLL_CTL_ADD: + if (registration != slot->registrations.end()) { + return SetPosixSocketError(Posix::POSIX_EEXIST); + } + slot->registrations.push_back({id, *event}); + break; + case EPOLL_CTL_MOD: + if (registration == slot->registrations.end()) { + return SetPosixSocketError(Posix::POSIX_ENOENT); + } + registration->event = *event; + break; + case EPOLL_CTL_DEL: + if (registration == slot->registrations.end()) { + return SetPosixSocketError(Posix::POSIX_ENOENT); + } + slot->registrations.erase(registration); + break; + default: break; + } + + g_epoll_changed.notify_all(); + return 0; +} + +int KYTY_SYSV_ABI EpollWait(int eid, NetEpollEvent* events, int maxevents, int timeout) { + if (events == nullptr) { + return SetPosixSocketError(Posix::POSIX_EFAULT); + } + if (maxevents <= 0 || timeout < -1) { + return SetPosixSocketError(Posix::POSIX_EINVAL); + } + + std::vector registrations; + { + std::unique_lock lock(g_epoll_mutex); + auto* slot = GetEpollSlot(eid); + if (slot == nullptr) { + return SetPosixSocketError(Posix::POSIX_EBADF); + } + + if (slot->registrations.empty() && timeout != 0) { + auto ready = [eid] { + auto* current = GetEpollSlot(eid); + return current == nullptr || !current->registrations.empty(); + }; + if (timeout < 0) { + g_epoll_changed.wait(lock, ready); + } else { + g_epoll_changed.wait_for(lock, std::chrono::microseconds(timeout), ready); + } + slot = GetEpollSlot(eid); + if (slot == nullptr) { + return SetPosixSocketError(Posix::POSIX_EBADF); + } + } + registrations = slot->registrations; + } + + if (registrations.empty()) { + return 0; + } + +#if defined(_WIN32) + struct HostRegistration { + EpollRegistration guest; + SOCKET socket = INVALID_SOCKET; + }; + + fd_set host_read {}; + fd_set host_write {}; + fd_set host_except {}; + FD_ZERO(&host_read); + FD_ZERO(&host_write); + FD_ZERO(&host_except); + + std::vector host_registrations; + host_registrations.reserve(registrations.size()); + for (const auto& registration: registrations) { + SOCKET socket = INVALID_SOCKET; + if (!GetSocketBackend(registration.id, &socket)) { + continue; + } + if (host_registrations.size() >= FD_SETSIZE) { + return SetPosixSocketError(Posix::POSIX_EINVAL); + } + if ((registration.event.events & EPOLL_IN) != 0) { + FD_SET(socket, &host_read); + } + if ((registration.event.events & EPOLL_OUT) != 0) { + FD_SET(socket, &host_write); + } + FD_SET(socket, &host_except); + host_registrations.push_back({registration, socket}); + } + + if (host_registrations.empty()) { + return 0; + } + + timeval host_timeout {}; + timeval* host_timeout_ptr = nullptr; + if (timeout >= 0) { + host_timeout.tv_sec = timeout / 1000000; + host_timeout.tv_usec = timeout % 1000000; + host_timeout_ptr = &host_timeout; + } + + const int result = ::select(0, &host_read, &host_write, &host_except, host_timeout_ptr); + if (result == SOCKET_ERROR) { + return SetPosixSocketError(); + } + if (result == 0) { + return 0; + } + + int count = 0; + for (const auto& registration: host_registrations) { + uint32_t ready = 0; + if (FD_ISSET(registration.socket, &host_read)) { + ready |= EPOLL_IN; + } + if (FD_ISSET(registration.socket, &host_write)) { + ready |= EPOLL_OUT; + } + if (FD_ISSET(registration.socket, &host_except)) { + ready |= EPOLL_ERR; + } + if (ready == 0) { + continue; + } + + auto& output = events[count++]; + output = registration.guest.event; + output.events = ready; + output.reserved = 0; + output.ident = static_cast(registration.guest.id); + if (count == maxevents) { + break; + } + } + return count; +#else + (void)timeout; + return SetPosixSocketError(Posix::POSIX_ENOSYS); +#endif +} + +int KYTY_SYSV_ABI EpollDestroy(int eid) { + std::lock_guard lock(g_epoll_mutex); + auto* slot = GetEpollSlot(eid); + if (slot == nullptr) { + return SetPosixSocketError(Posix::POSIX_EBADF); + } + + slot->used = false; + slot->name.clear(); + slot->registrations.clear(); + g_epoll_changed.notify_all(); + return 0; +} + +int KYTY_SYSV_ABI SocketClose(int s) { + PRINT_NAME(); + + LOGF("\t s = %d\n", s); + + SOCKET socket = INVALID_SOCKET; + if (!TakeSocketBackend(s, &socket)) { + return NET_ERROR_EBADF; + } + RemoveSocketFromEpolls(s); + +#if defined(_WIN32) + if (closesocket(socket) == SOCKET_ERROR) { + return NET_ERROR_EBADF; + } + + return OK; +#else + return NET_ERROR_ENOSYS; +#endif +} + +int KYTY_SYSV_ABI Socket(int family, int type, int protocol) { + PRINT_NAME(); + + LOGF("\t family = %d\n" + "\t type = %d\n" + "\t protocol = %d\n", + family, type, protocol); + + if (!EnsureSocketBackend()) { + return SetPosixSocketError(Posix::POSIX_ENETDOWN); + } + + const int host_family = ConvertFamily(family); + if (host_family < 0) { + *Posix::GetErrorAddr() = Posix::POSIX_EAFNOSUPPORT; + return -1; + } + +#if defined(_WIN32) + SOCKET socket = ::socket(host_family, type, protocol); + if (socket == INVALID_SOCKET) { + return SetPosixSocketError(); + } + + const int fd = AllocSocketFd(socket); + if (fd < 0) { + closesocket(socket); + *Posix::GetErrorAddr() = Posix::POSIX_EMFILE; + return -1; + } + + LOGF("\t fd = %d\n", fd); + return fd; +#else + *Posix::GetErrorAddr() = Posix::POSIX_ENOSYS; + return -1; +#endif +} + +int KYTY_SYSV_ABI Bind(int s, const void* addr, uint32_t addrlen) { + PRINT_NAME(); + + LOGF("\t s = %d\n" + "\t addr = 0x%016" PRIx64 "\n" + "\t addrlen = %" PRIu32 "\n", + s, reinterpret_cast(addr), addrlen); + + SOCKET socket = INVALID_SOCKET; + if (addr == nullptr || !GetSocketBackend(s, &socket)) { + *Posix::GetErrorAddr() = (addr == nullptr ? Posix::POSIX_EFAULT : Posix::POSIX_EBADF); + return -1; + } + +#if defined(_WIN32) + sockaddr_storage host_addr {}; + int host_addrlen = 0; + if (ConvertGuestSockaddr(addr, addrlen, &host_addr, &host_addrlen) != 0) { + return -1; + } + + if (::bind(socket, reinterpret_cast(&host_addr), host_addrlen) == + SOCKET_ERROR) { + return SetPosixSocketError(); + } + + return 0; +#else + *Posix::GetErrorAddr() = Posix::POSIX_ENOSYS; + return -1; +#endif +} + +int KYTY_SYSV_ABI Connect(int s, const void* addr, uint32_t addrlen) { + PRINT_NAME(); + + LOGF("\t s = %d\n" + "\t addr = 0x%016" PRIx64 "\n" + "\t addrlen = %" PRIu32 "\n", + s, reinterpret_cast(addr), addrlen); + + SOCKET socket = INVALID_SOCKET; + if (addr == nullptr || !GetSocketBackend(s, &socket)) { + *Posix::GetErrorAddr() = (addr == nullptr ? Posix::POSIX_EFAULT : Posix::POSIX_EBADF); + return -1; + } + +#if defined(_WIN32) + sockaddr_storage host_addr {}; + int host_addrlen = 0; + if (ConvertGuestSockaddr(addr, addrlen, &host_addr, &host_addrlen) != 0) { + return -1; + } + + if (::connect(socket, reinterpret_cast(&host_addr), host_addrlen) == + SOCKET_ERROR) { + return SetPosixSocketError(); + } + + return 0; +#else + *Posix::GetErrorAddr() = Posix::POSIX_ENOSYS; + return -1; +#endif +} + +int KYTY_SYSV_ABI Listen(int s, int backlog) { + PRINT_NAME(); + + LOGF("\t s = %d\n" + "\t backlog = %d\n", + s, backlog); + + SOCKET socket = INVALID_SOCKET; + if (!GetSocketBackend(s, &socket)) { + *Posix::GetErrorAddr() = Posix::POSIX_EBADF; + return -1; + } + +#if defined(_WIN32) + if (::listen(socket, backlog) == SOCKET_ERROR) { + return SetPosixSocketError(); + } + + return 0; +#else + *Posix::GetErrorAddr() = Posix::POSIX_ENOSYS; + return -1; +#endif +} + +int KYTY_SYSV_ABI Accept(int s, void* addr, uint32_t* addrlen) { + PRINT_NAME(); + + LOGF("\t s = %d\n" + "\t addr = 0x%016" PRIx64 "\n" + "\t addrlen = 0x%016" PRIx64 "\n", + s, reinterpret_cast(addr), reinterpret_cast(addrlen)); + + SOCKET socket = INVALID_SOCKET; + if (!GetSocketBackend(s, &socket)) { + *Posix::GetErrorAddr() = Posix::POSIX_EBADF; + return -1; + } + +#if defined(_WIN32) + sockaddr_storage host_addr {}; + int host_addrlen = sizeof(host_addr); + SOCKET accepted = ::accept(socket, reinterpret_cast(&host_addr), &host_addrlen); + if (accepted == INVALID_SOCKET) { + return SetPosixSocketError(); + } + + const int fd = AllocSocketFd(accepted); + if (fd < 0) { + closesocket(accepted); + *Posix::GetErrorAddr() = Posix::POSIX_EMFILE; + return -1; + } + + if (addr != nullptr || addrlen != nullptr) { + if (addr == nullptr || addrlen == nullptr) { + SocketClose(fd); + *Posix::GetErrorAddr() = Posix::POSIX_EFAULT; + return -1; + } + if (ConvertHostSockaddr(&host_addr, host_addrlen, addr, addrlen) != 0) { + SocketClose(fd); + return -1; + } + } + + return fd; +#else + *Posix::GetErrorAddr() = Posix::POSIX_ENOSYS; + return -1; +#endif +} + +int KYTY_SYSV_ABI Shutdown(int s, int how) { + PRINT_NAME(); + + LOGF("\t s = %d\n" + "\t how = %d\n", + s, how); + + SOCKET socket = INVALID_SOCKET; + if (!GetSocketBackend(s, &socket)) { + *Posix::GetErrorAddr() = Posix::POSIX_EBADF; + return -1; + } + + if (how < 0 || how > 2) { + *Posix::GetErrorAddr() = Posix::POSIX_EINVAL; + return -1; + } + +#if defined(_WIN32) + if (::shutdown(socket, how) == SOCKET_ERROR) { + return SetPosixSocketError(); + } + + return 0; +#else + *Posix::GetErrorAddr() = Posix::POSIX_ENOSYS; + return -1; +#endif +} + +int KYTY_SYSV_ABI Getsockname(int s, void* addr, uint32_t* addrlen) { + PRINT_NAME(); + + LOGF("\t s = %d\n" + "\t addr = 0x%016" PRIx64 "\n" + "\t addrlen = 0x%016" PRIx64 "\n", + s, reinterpret_cast(addr), reinterpret_cast(addrlen)); + + SOCKET socket = INVALID_SOCKET; + const bool socket_ok = GetSocketBackend(s, &socket); + if (addr == nullptr || addrlen == nullptr || !socket_ok) { + *Posix::GetErrorAddr() = (!socket_ok ? Posix::POSIX_EBADF : Posix::POSIX_EFAULT); + return -1; + } + +#if defined(_WIN32) + sockaddr_storage host_addr {}; + int host_addrlen = sizeof(host_addr); + if (::getsockname(socket, reinterpret_cast(&host_addr), &host_addrlen) == + SOCKET_ERROR) { + return SetPosixSocketError(); + } + + return ConvertHostSockaddr(&host_addr, host_addrlen, addr, addrlen); +#else + *Posix::GetErrorAddr() = Posix::POSIX_ENOSYS; + return -1; +#endif +} + +int KYTY_SYSV_ABI Getsockopt(int s, int level, int optname, void* optval, uint32_t* optlen) { + PRINT_NAME(); + + LOGF("\t s = %d\n" + "\t level = 0x%08" PRIx32 "\n" + "\t optname = 0x%08" PRIx32 "\n", + s, static_cast(level), static_cast(optname)); + + SOCKET socket = INVALID_SOCKET; + const bool socket_ok = GetSocketBackend(s, &socket); + if (optval == nullptr || optlen == nullptr || !socket_ok) { + *Posix::GetErrorAddr() = (!socket_ok ? Posix::POSIX_EBADF : Posix::POSIX_EFAULT); + return -1; + } + +#if defined(_WIN32) + int len = static_cast(*optlen); + if (::getsockopt(socket, ConvertSocketOptionLevel(level), optname, static_cast(optval), + &len) == SOCKET_ERROR) { + return SetPosixSocketError(); + } + *optlen = static_cast(len); + return 0; +#else + *Posix::GetErrorAddr() = Posix::POSIX_ENOSYS; + return -1; +#endif +} + +int KYTY_SYSV_ABI Setsockopt(int s, int level, int optname, const void* optval, uint32_t optlen) { + PRINT_NAME(); + + LOGF("\t s = %d\n" + "\t level = 0x%08" PRIx32 "\n" + "\t optname = 0x%08" PRIx32 "\n" + "\t optlen = %" PRIu32 "\n", + s, static_cast(level), static_cast(optname), optlen); + + SOCKET socket = INVALID_SOCKET; + if (optval == nullptr || !GetSocketBackend(s, &socket)) { + *Posix::GetErrorAddr() = (optval == nullptr ? Posix::POSIX_EFAULT : Posix::POSIX_EBADF); + return -1; + } + +#if defined(_WIN32) + constexpr int ORBIS_SO_NBIO = 0x1200; + if (ConvertSocketOptionLevel(level) == SOL_SOCKET && optname == ORBIS_SO_NBIO && + optlen >= sizeof(int)) { + u_long enabled = (*static_cast(optval) != 0 ? 1 : 0); + if (ioctlsocket(socket, FIONBIO, &enabled) == SOCKET_ERROR) { + return SetPosixSocketError(); + } + return 0; + } + + if (::setsockopt(socket, ConvertSocketOptionLevel(level), optname, + static_cast(optval), static_cast(optlen)) == SOCKET_ERROR) { + return SetPosixSocketError(); + } + + return 0; +#else + *Posix::GetErrorAddr() = Posix::POSIX_ENOSYS; + return -1; +#endif +} + +int64_t KYTY_SYSV_ABI Send(int s, const void* buf, uint64_t len, int flags) { + PRINT_NAME(); + + LOGF("\t s = %d\n" + "\t buf = 0x%016" PRIx64 "\n" + "\t len = %" PRIu64 "\n" + "\t flags = 0x%08" PRIx32 "\n", + s, reinterpret_cast(buf), len, static_cast(flags)); + + SOCKET socket = INVALID_SOCKET; + if (buf == nullptr || !GetSocketBackend(s, &socket)) { + *Posix::GetErrorAddr() = (buf == nullptr ? Posix::POSIX_EFAULT : Posix::POSIX_EBADF); + return -1; + } + +#if defined(_WIN32) + const int host_flags = ConvertMessageFlags(flags); + if (host_flags < 0) { + return -1; + } + + const int host_len = static_cast(len > 0x7fffffffu ? 0x7fffffffu : len); + const int result = ::send(socket, static_cast(buf), host_len, host_flags); + if (result == SOCKET_ERROR) { + return SetPosixSocketError(); + } + + return result; +#else + *Posix::GetErrorAddr() = Posix::POSIX_ENOSYS; + return -1; +#endif +} + +int64_t KYTY_SYSV_ABI Recv(int s, void* buf, uint64_t len, int flags) { + PRINT_NAME(); + + LOGF("\t s = %d\n" + "\t buf = 0x%016" PRIx64 "\n" + "\t len = %" PRIu64 "\n" + "\t flags = 0x%08" PRIx32 "\n", + s, reinterpret_cast(buf), len, static_cast(flags)); + + SOCKET socket = INVALID_SOCKET; + if (buf == nullptr || !GetSocketBackend(s, &socket)) { + *Posix::GetErrorAddr() = (buf == nullptr ? Posix::POSIX_EFAULT : Posix::POSIX_EBADF); + return -1; + } + +#if defined(_WIN32) + const int host_flags = ConvertMessageFlags(flags); + if (host_flags < 0) { + return -1; + } + + const int host_len = static_cast(len > 0x7fffffffu ? 0x7fffffffu : len); + const int result = ::recv(socket, static_cast(buf), host_len, host_flags); + if (result == SOCKET_ERROR) { + return SetPosixSocketError(); + } + + return result; +#else + *Posix::GetErrorAddr() = Posix::POSIX_ENOSYS; + return -1; +#endif +} + +int KYTY_SYSV_ABI Select(int nfds, void* readfds, void* writefds, void* exceptfds, + const void* timeout) { + PRINT_NAME(); + + if (nfds < 0 || nfds > SOCKET_FD_MAX) { + *Posix::GetErrorAddr() = Posix::POSIX_EINVAL; + return -1; + } + + static std::atomic_uint32_t select_log_count = 0; + const bool log_select = select_log_count.fetch_add(1, std::memory_order_relaxed) < 64; + + if (log_select) { + const auto read0 = (readfds != nullptr ? *static_cast(readfds) : 0); + const auto write0 = (writefds != nullptr ? *static_cast(writefds) : 0); + const auto except0 = (exceptfds != nullptr ? *static_cast(exceptfds) : 0); + const auto* guest_timeout = static_cast(timeout); + LOGF("\t nfds = %d\n" + "\t read = 0x%016" PRIx64 " bits0=0x%016" PRIx64 "\n" + "\t write = 0x%016" PRIx64 " bits0=0x%016" PRIx64 "\n" + "\t except = 0x%016" PRIx64 " bits0=0x%016" PRIx64 "\n" + "\t timeout = 0x%016" PRIx64, + nfds, reinterpret_cast(readfds), read0, reinterpret_cast(writefds), + write0, reinterpret_cast(exceptfds), except0, + reinterpret_cast(timeout)); + if (guest_timeout != nullptr) { + LOGF(" sec=%" PRId64 " usec=%" PRId64, guest_timeout->tv_sec, guest_timeout->tv_usec); + } + LOGF("\n"); + } + +#if defined(_WIN32) + fd_set host_read {}; + fd_set host_write {}; + fd_set host_except {}; + FD_ZERO(&host_read); + FD_ZERO(&host_write); + FD_ZERO(&host_except); + + std::array read_map {}; + std::array write_map {}; + std::array except_map {}; + int read_count = 0; + int write_count = 0; + int except_count = 0; + + for (int fd = 0; fd < nfds; fd++) { + SOCKET socket = INVALID_SOCKET; + if (!GetSocketBackend(fd, &socket)) { + continue; + } + if (GuestFdIsSet(readfds, fd) && read_count < FD_SETSIZE) { + FD_SET(socket, &host_read); + read_map[static_cast(read_count++)] = fd; + } + if (GuestFdIsSet(writefds, fd) && write_count < FD_SETSIZE) { + FD_SET(socket, &host_write); + write_map[static_cast(write_count++)] = fd; + } + if (GuestFdIsSet(exceptfds, fd) && except_count < FD_SETSIZE) { + FD_SET(socket, &host_except); + except_map[static_cast(except_count++)] = fd; + } + } + + timeval host_timeout {}; + timeval* host_timeout_ptr = nullptr; + if (timeout != nullptr) { + const auto* guest_timeout = static_cast(timeout); + host_timeout.tv_sec = static_cast(guest_timeout->tv_sec); + host_timeout.tv_usec = static_cast(guest_timeout->tv_usec); + host_timeout_ptr = &host_timeout; + } + + fd_set* read_ptr = (read_count != 0 ? &host_read : nullptr); + fd_set* write_ptr = (write_count != 0 ? &host_write : nullptr); + fd_set* except_ptr = (except_count != 0 ? &host_except : nullptr); + if (read_ptr == nullptr && write_ptr == nullptr && except_ptr == nullptr) { + if (host_timeout_ptr != nullptr) { + const auto sleep_ms = + static_cast(host_timeout.tv_sec * 1000 + host_timeout.tv_usec / 1000); + Sleep(sleep_ms); + } + GuestFdZero(readfds, nfds); + GuestFdZero(writefds, nfds); + GuestFdZero(exceptfds, nfds); + return 0; + } + + const int result = ::select(0, read_ptr, write_ptr, except_ptr, host_timeout_ptr); + if (result == SOCKET_ERROR) { + return SetPosixSocketError(); + } + + GuestFdZero(readfds, nfds); + GuestFdZero(writefds, nfds); + GuestFdZero(exceptfds, nfds); + for (int i = 0; i < read_count; i++) { + SOCKET socket = INVALID_SOCKET; + if (GetSocketBackend(read_map[static_cast(i)], &socket) && + FD_ISSET(socket, &host_read)) { + GuestFdSet(readfds, read_map[static_cast(i)]); + } + } + for (int i = 0; i < write_count; i++) { + SOCKET socket = INVALID_SOCKET; + if (GetSocketBackend(write_map[static_cast(i)], &socket) && + FD_ISSET(socket, &host_write)) { + GuestFdSet(writefds, write_map[static_cast(i)]); + } + } + for (int i = 0; i < except_count; i++) { + SOCKET socket = INVALID_SOCKET; + if (GetSocketBackend(except_map[static_cast(i)], &socket) && + FD_ISSET(socket, &host_except)) { + GuestFdSet(exceptfds, except_map[static_cast(i)]); + } + } + + return result; +#else + *Posix::GetErrorAddr() = Posix::POSIX_ENOSYS; + return -1; +#endif +} + +} // namespace Net + +namespace Ssl { + +LIB_NAME("Ssl", "Ssl"); + +struct SslData { + char* ptr = nullptr; + size_t size = 0; +}; + +struct SslCaCerts { + SslData* cert_data = nullptr; + size_t cert_data_num = 0; + void* pool = nullptr; +}; + +int KYTY_SYSV_ABI SslInit(uint64_t pool_size) { + PRINT_NAME(); + + LOGF("\t size = %" PRIu64 "\n", pool_size); + + EXIT_IF(g_net == nullptr); + + EXIT_NOT_IMPLEMENTED(pool_size == 0); + + auto id = g_net->SslInit(pool_size); + + if (!id.IsValid()) { + return SSL_ERROR_OUT_OF_SIZE; + } + + return id.ToInt(); +} + +int KYTY_SYSV_ABI SslTerm(int ssl_ctx_id) { + PRINT_NAME(); + + EXIT_IF(g_net == nullptr); + + if (!g_net->SslTerm(Network::Id(ssl_ctx_id))) { + return SSL_ERROR_INVALID_ID; + } + + return OK; +} + +int KYTY_SYSV_ABI SslGetCaCerts(int ssl_ctx_id, void* ca_certs) { + PRINT_NAME(); + + LOGF("\t ssl_ctx_id = %d\n", ssl_ctx_id); + + if (ca_certs == nullptr) { + return SSL_ERROR_INVALID_ARG; + } + + EXIT_IF(g_net == nullptr); + + if (!g_net->SslValid(Network::Id(ssl_ctx_id))) { + return SSL_ERROR_INVALID_ID; + } + + auto* certs = static_cast(ca_certs); + certs->cert_data = nullptr; + certs->cert_data_num = 0; + certs->pool = nullptr; + + return SSL_ERROR_NOT_FOUND; +} + +int KYTY_SYSV_ABI SslFreeCaCerts(int ssl_ctx_id, void* ca_certs) { + PRINT_NAME(); + + LOGF("\t ssl_ctx_id = %d\n", ssl_ctx_id); + + if (ca_certs == nullptr) { + return SSL_ERROR_INVALID_ARG; + } + + EXIT_IF(g_net == nullptr); + + if (!g_net->SslValid(Network::Id(ssl_ctx_id))) { + return SSL_ERROR_INVALID_ID; + } + + auto* certs = static_cast(ca_certs); + certs->cert_data = nullptr; + certs->cert_data_num = 0; + certs->pool = nullptr; + + return OK; +} + +} // namespace Ssl + +namespace Http { + +struct HttpEpoll { + Network::Id http_ctx_id = Network::Id(0); + Network::Id request_id = Network::Id(0); + void* user_arg = nullptr; +}; + +struct HttpNBEvent { + uint32_t events = 0; + uint32_t event_detail = 0; + int id = 0; + void* user_arg = nullptr; +}; + +LIB_NAME("Http", "Http"); + +static const char* HttpMethodToString(int method) { + switch (method) { + case 0: return "GET"; + case 1: return "POST"; + case 2: return "HEAD"; + case 3: return "OPTIONS"; + case 4: return "PUT"; + case 5: return "DELETE"; + case 6: return "TRACE"; + case 7: return "CONNECT"; + default: return nullptr; + } +} + +int KYTY_SYSV_ABI HttpInit(int memid, int ssl_ctx_id, uint64_t pool_size) { + PRINT_NAME(); + + LOGF("\t memid = %d\n" + "\t ssl_ctx_id = %d\n" + "\t size = %" PRIu64 "\n", + memid, ssl_ctx_id, pool_size); + + EXIT_IF(g_net == nullptr); + + EXIT_NOT_IMPLEMENTED(pool_size == 0); + + auto id = g_net->HttpInit(memid, Network::Id(ssl_ctx_id), pool_size); + + if (!id.IsValid()) { + return HTTP_ERROR_OUT_OF_MEMORY; + } + + return id.ToInt(); +} + +int KYTY_SYSV_ABI HttpTerm(int http_ctx_id) { + PRINT_NAME(); + + EXIT_IF(g_net == nullptr); + + if (!g_net->HttpTerm(Network::Id(http_ctx_id))) { + return HTTP_ERROR_INVALID_ID; + } + + return OK; +} + +int KYTY_SYSV_ABI HttpCreateTemplate(int http_ctx_id, const char* user_agent, int http_ver, + int is_auto_proxy_conf) { + PRINT_NAME(); + + LOGF("\t http_ctx_id = %d\n" + "\t user_agent = %s\n" + "\t http_ver = %d\n" + "\t is_auto_proxy_conf = %d\n", + http_ctx_id, user_agent, http_ver, is_auto_proxy_conf); + + EXIT_IF(g_net == nullptr); + + auto id = g_net->HttpCreateTemplate(Network::Id(http_ctx_id), user_agent, http_ver, + is_auto_proxy_conf != 0); + + if (!id.IsValid()) { + return HTTP_ERROR_OUT_OF_MEMORY; + } + + return id.ToInt(); +} + +int KYTY_SYSV_ABI HttpDeleteTemplate(int tmpl_id) { + PRINT_NAME(); + + EXIT_IF(g_net == nullptr); + + if (!g_net->HttpDeleteTemplate(Network::Id(tmpl_id))) { + return HTTP_ERROR_INVALID_ID; + } + + return OK; +} + +int KYTY_SYSV_ABI HttpSetNonblock(int id, int enable) { + PRINT_NAME(); + + LOGF("\t id = %d\n" + "\t enable = %d\n", + id, enable); + + if (!g_net->HttpSetNonblock(Network::Id(id), enable != 0)) { + return HTTP_ERROR_INVALID_ID; + } + + return OK; +} + +int KYTY_SYSV_ABI HttpsSetSslCallback(int id, HttpsCallback cbfunc, void* user_arg) { + PRINT_NAME(); + + LOGF("\t id = %d\n", id); + + if (!g_net->HttpsSetSslCallback(Network::Id(id), cbfunc, user_arg)) { + return HTTP_ERROR_INVALID_ID; + } + + return OK; +} + +int KYTY_SYSV_ABI HttpsSetMinSslVersion(int id, uint32_t ssl_version) { + PRINT_NAME(); + + LOGF("\t id = %d\n" + "\t ssl_version = %u\n", + id, ssl_version); + + if (!g_net->HttpsSetMinSslVersion(Network::Id(id), ssl_version)) { + return HTTP_ERROR_INVALID_ID; + } + + return OK; +} + +int KYTY_SYSV_ABI HttpsDisableOption(int id, uint32_t ssl_flags) { + PRINT_NAME(); + + LOGF("\t id = %d\n" + "\t ssl_flags = %u\n", + id, ssl_flags); + + if (!g_net->HttpsDisableOption(Network::Id(id), ssl_flags)) { + return HTTP_ERROR_INVALID_ID; + } + + return OK; +} + +int KYTY_SYSV_ABI HttpSetResolveTimeOut(int id, uint32_t usec) { + PRINT_NAME(); + + LOGF("\t id = %d\n" + "\t usec = %u\n", + id, usec); + + if (!g_net->HttpSetResolveTimeOut(Network::Id(id), usec)) { + return HTTP_ERROR_INVALID_ID; + } + + return OK; +} + +int KYTY_SYSV_ABI HttpSetResolveRetry(int id, int32_t retry) { + PRINT_NAME(); + + LOGF("\t id = %d\n" + "\t retry = %d\n", + id, retry); + + if (!g_net->HttpSetResolveRetry(Network::Id(id), retry)) { + return HTTP_ERROR_INVALID_ID; + } + + return OK; +} + +int KYTY_SYSV_ABI HttpSetConnectTimeOut(int id, uint32_t usec) { + PRINT_NAME(); + + LOGF("\t id = %d\n" + "\t usec = %u\n", + id, usec); + + if (!g_net->HttpSetConnectTimeOut(Network::Id(id), usec)) { + return HTTP_ERROR_INVALID_ID; + } + + return OK; +} + +int KYTY_SYSV_ABI HttpSetSendTimeOut(int id, uint32_t usec) { + PRINT_NAME(); + + LOGF("\t id = %d\n" + "\t usec = %u\n", + id, usec); + + if (!g_net->HttpSetSendTimeOut(Network::Id(id), usec)) { + return HTTP_ERROR_INVALID_ID; + } + + return OK; +} + +int KYTY_SYSV_ABI HttpSetRecvTimeOut(int id, uint32_t usec) { + PRINT_NAME(); + + LOGF("\t id = %d\n" + "\t usec = %u\n", + id, usec); + + if (!g_net->HttpSetRecvTimeOut(Network::Id(id), usec)) { + return HTTP_ERROR_INVALID_ID; + } + + return OK; +} + +int KYTY_SYSV_ABI HttpSetAutoRedirect(int id, int enable) { + PRINT_NAME(); + + LOGF("\t id = %d\n" + "\t enable = %d\n", + id, enable); + + if (!g_net->HttpSetAutoRedirect(Network::Id(id), enable)) { + return HTTP_ERROR_INVALID_ID; + } + + return OK; +} + +int KYTY_SYSV_ABI HttpSetAuthEnabled(int id, int enable) { + PRINT_NAME(); + + LOGF("\t id = %d\n" + "\t enable = %d\n", + id, enable); + + if (!g_net->HttpSetAuthEnabled(Network::Id(id), enable)) { + return HTTP_ERROR_INVALID_ID; + } + + return OK; +} + +int KYTY_SYSV_ABI HttpAddRequestHeader(int id, const char* name, const char* value, uint32_t mode) { + PRINT_NAME(); + + LOGF("\t id = %d\n" + "\t name = %s\n" + "\t value = %s\n" + "\t mode = %u\n", + id, name, value, mode); + + EXIT_NOT_IMPLEMENTED(mode != 0 && mode != 1); + + if (!g_net->HttpAddRequestHeader(Network::Id(id), name, value, mode == 1)) { + return HTTP_ERROR_INVALID_ID; + } + + return OK; +} + +int KYTY_SYSV_ABI HttpCreateEpoll(int http_ctx_id, HttpEpollHandle* eh) { + PRINT_NAME(); + + LOGF("\t http_ctx_id = %d\n", http_ctx_id); + + EXIT_IF(g_net == nullptr); + + EXIT_NOT_IMPLEMENTED(eh == nullptr); + + EXIT_NOT_IMPLEMENTED(!g_net->HttpValid(Network::Id(http_ctx_id))); + + *eh = new HttpEpoll; + + (*eh)->http_ctx_id = Network::Id(http_ctx_id); + + return OK; +} + +int KYTY_SYSV_ABI HttpDestroyEpoll(int http_ctx_id, HttpEpollHandle eh) { + PRINT_NAME(); + + LOGF("\t http_ctx_id = %d\n", http_ctx_id); + + EXIT_IF(g_net == nullptr); + + EXIT_NOT_IMPLEMENTED(eh == nullptr); + + EXIT_NOT_IMPLEMENTED(!g_net->HttpValid(Network::Id(http_ctx_id))); + + delete eh; + + return OK; +} + +int KYTY_SYSV_ABI HttpSetEpoll(int id, HttpEpollHandle eh, void* user_arg) { + PRINT_NAME(); + + LOGF("\t id = %d\n", id); + + EXIT_NOT_IMPLEMENTED(eh == nullptr); + + EXIT_NOT_IMPLEMENTED(!g_net->HttpValidRequest(Network::Id(id))); + + eh->request_id = Network::Id(id); + eh->user_arg = user_arg; + + return OK; +} + +int KYTY_SYSV_ABI HttpUnsetEpoll(int id) { + PRINT_NAME(); + + LOGF("\t id = %d\n", id); + + EXIT_NOT_IMPLEMENTED(!g_net->HttpValidRequest(Network::Id(id))); + + return OK; +} + +int KYTY_SYSV_ABI HttpSendRequest(int request_id, const void* /*post_data*/, size_t /*size*/) { + PRINT_NAME(); + + LOGF("\t request_id = %d\n", request_id); + + EXIT_IF(g_net == nullptr); + + if (!g_net->HttpMarkRequestSent(Network::Id(request_id), HTTP_ERROR_TIMEOUT)) { + return HTTP_ERROR_INVALID_ID; + } + + return HTTP_ERROR_TIMEOUT; +} + +int KYTY_SYSV_ABI HttpAbortRequest(int request_id) { + PRINT_NAME(); + + LOGF("\t request_id = %d\n", request_id); + + EXIT_IF(g_net == nullptr); + + if (!g_net->HttpValidRequest(Network::Id(request_id))) { + return HTTP_ERROR_INVALID_ID; + } + + return OK; +} + +int KYTY_SYSV_ABI HttpWaitRequest(HttpEpollHandle eh, HttpNBEvent* nbev, int maxevents, + int timeout) { + PRINT_NAME(); + + LOGF("\t eh = 0x%016" PRIx64 "\n" + "\t nbev = 0x%016" PRIx64 "\n" + "\t maxevents = %d\n" + "\t timeout = %d\n", + reinterpret_cast(eh), reinterpret_cast(nbev), maxevents, timeout); + + EXIT_IF(g_net == nullptr); + + if (eh == nullptr || maxevents < 0 || (maxevents > 0 && nbev == nullptr)) { + return HTTP_ERROR_INVALID_VALUE; + } + + return 0; +} + +int KYTY_SYSV_ABI HttpGetStatusCode(int request_id, int* status_code) { + PRINT_NAME(); + + LOGF("\t request_id = %d\n" + "\t status_code = 0x%016" PRIx64 "\n", + request_id, reinterpret_cast(status_code)); + + if (status_code == nullptr) { + return HTTP_ERROR_INVALID_VALUE; + } + + *status_code = 0; + + EXIT_IF(g_net == nullptr); + + int send_result = HTTP_ERROR_BEFORE_SEND; + if (!g_net->HttpGetRequestResponse(Network::Id(request_id), &send_result, status_code, nullptr, + nullptr, nullptr)) { + return HTTP_ERROR_INVALID_ID; + } + + return send_result; +} + +int KYTY_SYSV_ABI HttpGetAllResponseHeaders(int request_id, char** header, size_t* header_size) { + PRINT_NAME(); + + LOGF("\t request_id = %d\n" + "\t header = 0x%016" PRIx64 "\n" + "\t header_size = 0x%016" PRIx64 "\n", + request_id, reinterpret_cast(header), reinterpret_cast(header_size)); + + if (header == nullptr || header_size == nullptr) { + return HTTP_ERROR_INVALID_VALUE; + } + + *header = nullptr; + *header_size = 0; + + EXIT_IF(g_net == nullptr); + + int send_result = HTTP_ERROR_BEFORE_SEND; + const char* headers = nullptr; + size_t headers_size = 0; + if (!g_net->HttpGetRequestResponse(Network::Id(request_id), &send_result, nullptr, &headers, + &headers_size, nullptr)) { + return HTTP_ERROR_INVALID_ID; + } + + *header = const_cast(headers); + *header_size = headers_size; + + return send_result; +} + +int KYTY_SYSV_ABI HttpGetResponseContentLength(int request_id, int* result, + uint64_t* content_length) { + PRINT_NAME(); + + LOGF("\t request_id = %d\n" + "\t result = 0x%016" PRIx64 "\n" + "\t content_length = 0x%016" PRIx64 "\n", + request_id, reinterpret_cast(result), + reinterpret_cast(content_length)); + + if (result == nullptr || content_length == nullptr) { + return HTTP_ERROR_INVALID_VALUE; + } + + *result = 0; + *content_length = 0; + + EXIT_IF(g_net == nullptr); + + int send_result = HTTP_ERROR_BEFORE_SEND; + if (!g_net->HttpGetRequestResponse(Network::Id(request_id), &send_result, nullptr, nullptr, + nullptr, content_length)) { + return HTTP_ERROR_INVALID_ID; + } + + *result = (send_result == OK ? 0 : -1); + return send_result; +} + +int KYTY_SYSV_ABI HttpCreateConnection(int tmpl_id, const char* server_name, const char* scheme, + uint16_t port, int enable_keep_alive) { + PRINT_NAME(); + + LOGF("\t tmpl_id = %d\n" + "\t server_name = %s\n" + "\t scheme = %s\n" + "\t port = %u\n" + "\t enable_keep_alive = %d\n", + tmpl_id, server_name, scheme, static_cast(port), enable_keep_alive); + + EXIT_IF(g_net == nullptr); + + if (server_name == nullptr || scheme == nullptr) { + return HTTP_ERROR_INVALID_VALUE; + } + + auto id = g_net->HttpCreateConnection(Network::Id(tmpl_id), server_name, scheme, port, + enable_keep_alive != 0); + + if (!id.IsValid()) { + return HTTP_ERROR_OUT_OF_MEMORY; + } + + return id.ToInt(); +} + +int KYTY_SYSV_ABI HttpCreateConnectionWithURL(int tmpl_id, const char* url, int enable_keep_alive) { + PRINT_NAME(); + + LOGF("\t tmpl_id = %d\n" + "\t url = %s\n" + "\t enable_keep_alive = %d\n", + tmpl_id, url, enable_keep_alive); + + EXIT_IF(g_net == nullptr); + + if (url == nullptr || url[0] == '\0') { + return HTTP_ERROR_INVALID_URL; + } + + auto id = g_net->HttpCreateConnectionWithURL(Network::Id(tmpl_id), url, enable_keep_alive != 0); + + if (!id.IsValid()) { + return HTTP_ERROR_OUT_OF_MEMORY; + } + + return id.ToInt(); +} + +int KYTY_SYSV_ABI HttpDeleteConnection(int conn_id) { + PRINT_NAME(); + + LOGF("\t conn_id = %d\n", conn_id); + + EXIT_IF(g_net == nullptr); + + if (!g_net->HttpDeleteConnection(Network::Id(conn_id))) { + return HTTP_ERROR_INVALID_ID; + } + + return OK; +} + +int KYTY_SYSV_ABI HttpCreateRequest(int conn_id, int method, const char* path, + uint64_t content_length) { + PRINT_NAME(); + + LOGF("\t conn_id = %d\n" + "\t method = %d\n" + "\t path = %s\n" + "\t content_length = %" PRIu64 "\n", + conn_id, method, path, content_length); + + EXIT_IF(g_net == nullptr); + + auto method_name = HttpMethodToString(method); + if (method_name == nullptr) { + return HTTP_ERROR_UNKNOWN_METHOD; + } + if (path == nullptr) { + return HTTP_ERROR_INVALID_VALUE; + } + + auto id = + g_net->HttpCreateRequestWithURL2(Network::Id(conn_id), method_name, path, content_length); + + if (!id.IsValid()) { + return HTTP_ERROR_OUT_OF_MEMORY; + } + + return id.ToInt(); +} + +int KYTY_SYSV_ABI HttpCreateRequestWithURL2(int conn_id, const char* method, const char* url, + uint64_t content_length) { + PRINT_NAME(); + + LOGF("\t conn_id = %d\n" + "\t url = %s\n" + "\t method = %s\n" + "\t content_length = %" PRIu64 "\n", + conn_id, url, method, content_length); + + EXIT_IF(g_net == nullptr); + + if (method == nullptr) { + return HTTP_ERROR_INVALID_VALUE; + } + if (url == nullptr || url[0] == '\0') { + return HTTP_ERROR_INVALID_URL; + } + + auto id = g_net->HttpCreateRequestWithURL2(Network::Id(conn_id), method, url, content_length); + + if (!id.IsValid()) { + return HTTP_ERROR_OUT_OF_MEMORY; + } + + return id.ToInt(); +} + +int KYTY_SYSV_ABI HttpDeleteRequest(int req_id) { + PRINT_NAME(); + + LOGF("\t req_id = %d\n", req_id); + + EXIT_IF(g_net == nullptr); + + if (!g_net->HttpDeleteRequest(Network::Id(req_id))) { + return HTTP_ERROR_INVALID_ID; + } + + return OK; +} + +int KYTY_SYSV_ABI HttpSetRequestContentLength(int request_id, uint64_t content_length) { + PRINT_NAME(); + + LOGF("\t request_id = %d\n" + "\t content_length = %" PRIu64 "\n", + request_id, content_length); + + EXIT_IF(g_net == nullptr); + + if (!g_net->HttpSetRequestContentLength(Network::Id(request_id), content_length)) { + return HTTP_ERROR_INVALID_ID; + } + + return OK; +} + +} // namespace Http + +namespace NetCtl { + +LIB_NAME("NetCtl", "NetCtl"); + +constexpr int NET_CTL_STATE_DISCONNECTED = 0; +constexpr int NET_CTL_STATE_IPOBTAINED = 3; +constexpr int NET_CTL_DEVICE_WIRED = 0; +constexpr int NET_CTL_DEVICE_WIRELESS = 1; +constexpr int NET_CTL_LINK_DISCONNECTED = 0; +constexpr int NET_CTL_LINK_CONNECTED = 1; +constexpr int NET_CTL_EVENT_DISCONNECTED = 1; +constexpr int NET_CTL_EVENT_IPOBTAINED = 3; +constexpr int NET_CTL_IP_DHCP = 0; +constexpr int NET_CTL_HTTP_PROXY_OFF = 0; +constexpr int NET_CTL_CALLBACK_MAX = 8; +constexpr int NET_CTL_ERROR_CALLBACK_MAX = -2143215357; /* 0x80412103 */ +constexpr int NET_CTL_ERROR_ID_NOT_FOUND = -2143215356; /* 0x80412104 */ +constexpr int NET_CTL_ERROR_INVALID_ID = -2143215355; /* 0x80412105 */ +constexpr int NET_CTL_ERROR_INVALID_CODE = -2143215354; /* 0x80412106 */ +constexpr int NET_CTL_ERROR_INVALID_ADDR = -2143215353; /* 0x80412107 */ +constexpr int NET_CTL_ERROR_NOT_CONNECTED = -2143215352; /* 0x80412108 */ +constexpr int NET_CTL_ERROR_INVALID_TYPE = -2143215345; /* 0x8041210f */ + +struct NetInAddr { + uint32_t s_addr = 0; +}; + +struct NetEtherAddr { + uint8_t data[6]; +}; + +struct NetCtlNatInfo { + unsigned int size = sizeof(NetCtlNatInfo); + int stunStatus = 0; + int natType = 0; + NetInAddr mappedAddr; +}; + +union NetCtlInfo { + uint32_t device; + NetEtherAddr ether_addr; + uint32_t mtu; + uint32_t link; + NetEtherAddr bssid; + char ssid[32 + 1]; + uint32_t wifi_security; + int32_t rssi_dbm; + uint8_t rssi_percentage; + uint8_t channel; + uint32_t ip_config; + char dhcp_hostname[255 + 1]; + char pppoe_auth_name[127 + 1]; + char ip_address[16]; + char netmask[16]; + char default_route[16]; + char primary_dns[16]; + char secondary_dns[16]; + uint32_t http_proxy_config; + char http_proxy_server[255 + 1]; + uint16_t http_proxy_port; +}; + +struct NetCtlCallbackSlot { + NetCtlCallback func = nullptr; + void* arg = nullptr; + int last_event = 0; +}; + +static Common::Mutex g_net_ctl_callbacks_mutex; +static std::array g_net_ctl_callbacks; +static std::atomic_bool g_net_ctl_connected = false; +static std::atomic_bool g_net_ctl_status_initialized = false; + +struct HostNetworkInfo { + bool connected = false; + bool wireless = false; + NetEtherAddr ether_addr {}; + char ip_address[16] {}; + char netmask[16] {}; + char default_route[16] {}; + char primary_dns[16] {}; + char secondary_dns[16] {}; +}; + +static void CopyIpv4String(char* dst, size_t dst_size, const sockaddr* addr) { +#if defined(_WIN32) + if (dst == nullptr || dst_size == 0 || addr == nullptr || addr->sa_family != AF_INET) { + return; + } + + const auto* in = reinterpret_cast(addr); + inet_ntop(AF_INET, &in->sin_addr, dst, static_cast(dst_size)); +#else + (void)dst; + (void)dst_size; + (void)addr; +#endif +} + +static void CopyIpv4Netmask(char* dst, size_t dst_size, uint8_t prefix_len) { +#if defined(_WIN32) + if (dst == nullptr || dst_size == 0 || prefix_len > 32) { + return; + } + + const uint32_t mask = (prefix_len == 0 ? 0 : (0xffffffffu << (32u - prefix_len))); + in_addr in {}; + in.S_un.S_addr = htonl(mask); + inet_ntop(AF_INET, &in, dst, static_cast(dst_size)); +#else + (void)dst; + (void)dst_size; + (void)prefix_len; +#endif +} + +static HostNetworkInfo QueryHostNetworkInfo() { + HostNetworkInfo info; +#if defined(_WIN32) + ULONG flags = GAA_FLAG_SKIP_ANYCAST | GAA_FLAG_SKIP_MULTICAST; + ULONG size = 0; + auto ret = GetAdaptersAddresses(AF_UNSPEC, flags, nullptr, nullptr, &size); + if (ret != ERROR_BUFFER_OVERFLOW || size == 0) { + return info; + } + + std::vector buffer(size); + auto* adapters = reinterpret_cast(buffer.data()); + ret = GetAdaptersAddresses(AF_UNSPEC, flags, nullptr, adapters, &size); + if (ret != NO_ERROR) { + return info; + } + + for (auto* adapter = adapters; adapter != nullptr; adapter = adapter->Next) { + if (adapter->OperStatus != IfOperStatusUp || adapter->IfType == IF_TYPE_SOFTWARE_LOOPBACK || + adapter->IfType == IF_TYPE_TUNNEL) { + continue; + } + + for (auto* unicast = adapter->FirstUnicastAddress; unicast != nullptr; + unicast = unicast->Next) { + const auto* addr = unicast->Address.lpSockaddr; + if (addr == nullptr) { + continue; + } + if (addr->sa_family == AF_INET) { + const auto* in = reinterpret_cast(addr); + const auto ip = ntohl(in->sin_addr.S_un.S_addr); + if ((ip & 0xff000000u) != 0x7f000000u && ip != 0) { + info.connected = true; + info.wireless = (adapter->IfType == IF_TYPE_IEEE80211); + if (adapter->PhysicalAddressLength >= sizeof(info.ether_addr.data)) { + std::memcpy(info.ether_addr.data, adapter->PhysicalAddress, + sizeof(info.ether_addr.data)); + } + CopyIpv4String(info.ip_address, sizeof(info.ip_address), addr); + CopyIpv4Netmask(info.netmask, sizeof(info.netmask), + unicast->OnLinkPrefixLength); + if (adapter->FirstGatewayAddress != nullptr) { + CopyIpv4String(info.default_route, sizeof(info.default_route), + adapter->FirstGatewayAddress->Address.lpSockaddr); + } + int dns_index = 0; + for (auto* dns = adapter->FirstDnsServerAddress; dns != nullptr; + dns = dns->Next) { + if (dns->Address.lpSockaddr != nullptr && + dns->Address.lpSockaddr->sa_family == AF_INET) { + CopyIpv4String(dns_index == 0 ? info.primary_dns : info.secondary_dns, + dns_index == 0 ? sizeof(info.primary_dns) + : sizeof(info.secondary_dns), + dns->Address.lpSockaddr); + dns_index++; + if (dns_index >= 2) { + break; + } + } + } + return info; + } + } else if (addr->sa_family == AF_INET6) { + const auto* in6 = reinterpret_cast(addr); + if (!IN6_IS_ADDR_LOOPBACK(&in6->sin6_addr) && + !IN6_IS_ADDR_UNSPECIFIED(&in6->sin6_addr)) { + info.connected = true; + } + } + } + } + + return info; +#else + info.connected = true; + return info; +#endif +} + +[[maybe_unused]] static bool HostNetworkConnected() { + return QueryHostNetworkInfo().connected; +} + +static bool NetCtlConnected() { + if (!g_net_ctl_status_initialized.load()) { + // g_net_ctl_connected = HostNetworkConnected(); + g_net_ctl_connected = false; + g_net_ctl_status_initialized = true; + // LOGF("\t host network connected = %s\n", (g_net_ctl_connected.load() ? "true" : + // "false")); + LOGF("\t host network connected = false (forced offline)\n"); + } + + return g_net_ctl_connected.load(); +} + +int KYTY_SYSV_ABI NetCtlInit() { + PRINT_NAME(); + + // g_net_ctl_connected = HostNetworkConnected(); + g_net_ctl_connected = false; + g_net_ctl_status_initialized = true; + // LOGF("\t host network connected = %s\n", (g_net_ctl_connected.load() ? "true" : "false")); + LOGF("\t host network connected = false (forced offline)\n"); + + return OK; +} + +void KYTY_SYSV_ABI NetCtlTerm() { + PRINT_NAME(); + + Common::LockGuard lock(g_net_ctl_callbacks_mutex); + + for (auto& cb: g_net_ctl_callbacks) { + cb = {}; + } +} + +int KYTY_SYSV_ABI NetCtlGetNatInfo(NetCtlNatInfo* nat_info) { + PRINT_NAME(); + + EXIT_NOT_IMPLEMENTED(nat_info == nullptr); + EXIT_NOT_IMPLEMENTED(nat_info->size != sizeof(NetCtlNatInfo)); + + nat_info->stunStatus = 0; + nat_info->natType = 0; + nat_info->mappedAddr.s_addr = 0; + + return OK; +} + +int KYTY_SYSV_ABI NetCtlCheckCallback() { + PRINT_NAME(); + + Common::LockGuard lock(g_net_ctl_callbacks_mutex); + const int event = (NetCtlConnected() ? NET_CTL_EVENT_IPOBTAINED : NET_CTL_EVENT_DISCONNECTED); + for (auto& cb: g_net_ctl_callbacks) { + if (cb.func != nullptr && cb.last_event != event) { + cb.func(event, cb.arg); + cb.last_event = event; + } + } + + return OK; +} + +int KYTY_SYSV_ABI NetCtlGetState(int* state) { + PRINT_NAME(); + + if (state == nullptr) { + return NET_CTL_ERROR_INVALID_ADDR; + } + + *state = (NetCtlConnected() ? NET_CTL_STATE_IPOBTAINED : NET_CTL_STATE_DISCONNECTED); + + return OK; +} + +int KYTY_SYSV_ABI NetCtlGetStateV6(int* state) { + PRINT_NAME(); + + if (state == nullptr) { + return NET_CTL_ERROR_INVALID_ADDR; + } + + *state = (NetCtlConnected() ? NET_CTL_STATE_IPOBTAINED : NET_CTL_STATE_DISCONNECTED); + + return OK; +} + +int KYTY_SYSV_ABI NetCtlRegisterCallback(NetCtlCallback func, void* arg, int* cid) { + PRINT_NAME(); + + if (func == nullptr || cid == nullptr) { + return NET_CTL_ERROR_INVALID_ADDR; + } + + Common::LockGuard lock(g_net_ctl_callbacks_mutex); + + for (int i = 0; i < NET_CTL_CALLBACK_MAX; i++) { + if (g_net_ctl_callbacks[static_cast(i)].func == nullptr) { + g_net_ctl_callbacks[static_cast(i)].func = func; + g_net_ctl_callbacks[static_cast(i)].arg = arg; + g_net_ctl_callbacks[static_cast(i)].last_event = 0; + *cid = i; + return OK; + } + } + + return NET_CTL_ERROR_CALLBACK_MAX; +} + +int KYTY_SYSV_ABI NetCtlUnregisterCallback(int cid) { + PRINT_NAME(); + + if (cid < 0 || cid >= NET_CTL_CALLBACK_MAX) { + return NET_CTL_ERROR_INVALID_ID; + } + + Common::LockGuard lock(g_net_ctl_callbacks_mutex); + + auto& cb = g_net_ctl_callbacks[static_cast(cid)]; + + if (cb.func == nullptr) { + return NET_CTL_ERROR_ID_NOT_FOUND; + } + + cb = {}; + + return OK; +} + +int KYTY_SYSV_ABI NetCtlGetResult(int event_type, int* error_code) { + PRINT_NAME(); + + if (error_code == nullptr) { + return NET_CTL_ERROR_INVALID_ADDR; + } + + LOGF("\t event_type = %d\n", event_type); + + switch (event_type) { + case NET_CTL_EVENT_DISCONNECTED: + case 2: + case NET_CTL_EVENT_IPOBTAINED: *error_code = OK; break; + default: return NET_CTL_ERROR_INVALID_TYPE; + } + + return OK; +} + +int KYTY_SYSV_ABI NetCtlGetInfo(int code, NetCtlInfo* info) { + PRINT_NAME(); + + EXIT_NOT_IMPLEMENTED(info == nullptr); + + memset(info, 0, sizeof(NetCtlInfo)); + + // Online/host-backed info responses are left below for testing, but disabled + // to preserve the disconnected-console behavior. + return NET_CTL_ERROR_NOT_CONNECTED; + + switch (code) { + case 1: + info->device = + (QueryHostNetworkInfo().wireless ? NET_CTL_DEVICE_WIRELESS : NET_CTL_DEVICE_WIRED); + break; + case 2: info->ether_addr = QueryHostNetworkInfo().ether_addr; break; + case 3: info->mtu = 1500; break; + case 4: + info->link = (NetCtlConnected() ? NET_CTL_LINK_CONNECTED : NET_CTL_LINK_DISCONNECTED); + break; + case 5: break; + case 6: break; + case 7: info->wifi_security = 0; break; + case 8: info->rssi_dbm = 0; break; + case 9: info->rssi_percentage = 0; break; + case 10: info->channel = 0; break; + case 11: info->ip_config = NET_CTL_IP_DHCP; break; + case 12: break; + case 13: break; + case 14: + std::strncpy(info->ip_address, QueryHostNetworkInfo().ip_address, + sizeof(info->ip_address) - 1); + break; + case 15: + std::strncpy(info->netmask, QueryHostNetworkInfo().netmask, sizeof(info->netmask) - 1); + break; + case 16: + std::strncpy(info->default_route, QueryHostNetworkInfo().default_route, + sizeof(info->default_route) - 1); + break; + case 17: + std::strncpy(info->primary_dns, QueryHostNetworkInfo().primary_dns, + sizeof(info->primary_dns) - 1); + break; + case 18: + std::strncpy(info->secondary_dns, QueryHostNetworkInfo().secondary_dns, + sizeof(info->secondary_dns) - 1); + break; + case 19: info->http_proxy_config = NET_CTL_HTTP_PROXY_OFF; break; + case 20: break; + case 21: info->http_proxy_port = 0; break; + default: LOGF("\t unknown NetCtl info code: %d\n", code); return NET_CTL_ERROR_INVALID_CODE; + } + + return OK; +} + +} // namespace NetCtl + +namespace NpManager { + +LIB_NAME("NpManager", "NpManager"); + +struct NpTitleId { + char id[12 + 1]; + uint8_t padding[3]; +}; + +struct NpTitleSecret { + uint8_t data[128]; +}; + +struct NpCountryCode { + char data[2]; + char term; + char padding[1]; +}; + +struct NpAgeRestriction { + NpCountryCode country_code; + int8_t age; + uint8_t padding[3]; +}; + +struct NpContentRestriction { + size_t size; + int8_t default_age_restriction; + char padding[3]; + int32_t age_restriction_count; + const NpAgeRestriction* age_restriction; +}; + +struct NpOnlineId { + char data[16]; + char term; + char dummy[3]; +}; + +struct NpId { + NpOnlineId handle; + uint8_t opt[8]; + uint8_t reserved[8]; +}; + +struct NpCreateAsyncRequestParameter { + size_t size; + LibKernel::KernelCpumask cpu_affinity_mask; + int thread_priority; + uint8_t padding[4]; +}; + +struct NpCheckPremiumParameter { + size_t size; + int user_id; + char padding[4]; + uint64_t features; + uint8_t reserved[32]; +}; + +struct NpCheckPremiumResult { + bool authorized; + uint8_t reserved[32]; +}; + +constexpr int np_error_invalid_argument = -2141913085; /* 0x80550003 */ +constexpr int np_error_signed_out = -2141913082; /* 0x80550006 */ +constexpr int np_error_invalid_size = -2141913071; /* 0x80550011 */ +constexpr int np_error_aborted = -2141913070; /* 0x80550012 */ +constexpr int np_error_request_max = -2141913069; /* 0x80550013 */ +constexpr int np_error_request_not_found = -2141913068; /* 0x80550014 */ +constexpr int np_error_invalid_id = -2141913067; /* 0x80550015 */ +constexpr int np_request_max = 128; + +enum class NpRequestState { + Free, + Ready, + Aborted, + Complete, +}; + +struct NpRequest { + NpRequestState state = NpRequestState::Free; + bool async = false; + int result = OK; +}; + +static std::mutex g_np_request_mutex; +static std::vector g_np_requests; + +static int np_create_request(bool async) { + std::lock_guard lock(g_np_request_mutex); + + for (size_t i = 0; i < g_np_requests.size(); i++) { + auto& request = g_np_requests[i]; + if (request.state == NpRequestState::Free) { + request.state = NpRequestState::Ready; + request.async = async; + request.result = OK; + return static_cast(i + 1); + } + } + + if (g_np_requests.size() >= np_request_max) { + return np_error_request_max; + } + + g_np_requests.push_back({NpRequestState::Ready, async, OK}); + return static_cast(g_np_requests.size()); +} + +static NpRequest* np_get_request_locked(int req_id) { + if (req_id <= 0) { + return nullptr; + } + + const auto index = static_cast(req_id - 1); + if (index >= g_np_requests.size() || g_np_requests[index].state == NpRequestState::Free) { + return nullptr; + } + + return &g_np_requests[index]; +} + +static int np_complete_signed_out_locked(NpRequest* request) { + if (request->state == NpRequestState::Complete) { + request->result = np_error_invalid_argument; + return np_error_invalid_argument; + } + if (request->state == NpRequestState::Aborted) { + request->result = np_error_aborted; + return np_error_aborted; + } + + request->state = NpRequestState::Complete; + request->result = np_error_signed_out; + + return (request->async ? OK : np_error_signed_out); +} + +int KYTY_SYSV_ABI NpCheckCallback() { + PRINT_NAME(); + + return OK; +} + +int KYTY_SYSV_ABI NpSetNpTitleId(const NpTitleId* title_id, const NpTitleSecret* title_secret) { + PRINT_NAME(); + + EXIT_NOT_IMPLEMENTED(title_id == nullptr); + EXIT_NOT_IMPLEMENTED(title_secret == nullptr); + + LOGF("\t title_id = %.12s\n" + "\t title_secret = %s\n", + title_id->id, Common::HexFromBin(Common::ByteBuffer(title_secret->data, 128)).c_str()); + + return OK; +} + +int KYTY_SYSV_ABI NpSetContentRestriction(const NpContentRestriction* restriction) { + PRINT_NAME(); + + EXIT_NOT_IMPLEMENTED(restriction == nullptr); + EXIT_NOT_IMPLEMENTED(restriction->size != sizeof(NpContentRestriction)); + + LOGF("\t default_age_restriction = %" PRIi8 "\n" + "\t age_restriction_count = %" PRIi32 "\n", + restriction->default_age_restriction, restriction->age_restriction_count); + + for (int i = 0; i < restriction->age_restriction_count; i++) { + LOGF("\t age_restriction[%d].age = %" PRIi8 "\n" + "\t age_restriction[%d].country_code.data = %.2s\n", + i, restriction->age_restriction[i].age, i, + restriction->age_restriction[i].country_code.data); + } + + return OK; +} + +int KYTY_SYSV_ABI NpRegisterStateCallback(void* /*callback*/, void* /*userdata*/) { + PRINT_NAME(); + + return OK; +} + +int KYTY_SYSV_ABI NpUnregisterStateCallback() { + PRINT_NAME(); + + return OK; +} + +void KYTY_SYSV_ABI NpRegisterGamePresenceCallback(void* /*callback*/, void* /*userdata*/) { + PRINT_NAME(); +} + +int KYTY_SYSV_ABI NpRegisterPlusEventCallback(void* /*callback*/, void* /*userdata*/) { + PRINT_NAME(); + + return OK; +} + +int KYTY_SYSV_ABI NpRegisterPremiumEventCallback(void* /*callback*/, void* /*userdata*/) { + PRINT_NAME(); + + return OK; +} + +int KYTY_SYSV_ABI NpRegisterNpReachabilityStateCallback(void* /*callback*/, void* /*userdata*/) { + PRINT_NAME(); + + return OK; +} + +int KYTY_SYSV_ABI NpGetNpId(int user_id, NpId* np_id) { + PRINT_NAME(); + + LOGF("\t user_id = %d\n", user_id); + + EXIT_NOT_IMPLEMENTED(np_id == nullptr); + + // int s = snprintf(np_id->handle.data, 16, "Kyty"); + // EXIT_NOT_IMPLEMENTED(s >= 16); + // np_id->handle.term = 0; + std::memset(np_id, 0, sizeof(*np_id)); + + // return OK; + return np_error_signed_out; +} + +int KYTY_SYSV_ABI NpGetOnlineId(int user_id, NpOnlineId* online_id) { + PRINT_NAME(); + + LOGF("\t user_id = %d\n", user_id); + + EXIT_NOT_IMPLEMENTED(online_id == nullptr); + + // int s = snprintf(online_id->data, 16, "Kyty"); + // EXIT_NOT_IMPLEMENTED(s >= 16); + // online_id->term = 0; + std::memset(online_id, 0, sizeof(*online_id)); + + // return OK; + return np_error_signed_out; +} + +int KYTY_SYSV_ABI NpGetAccountIdA(int user_id, uint64_t* account_id) { + PRINT_NAME(); + + LOGF("\t user_id = %d\n", user_id); + + EXIT_NOT_IMPLEMENTED(account_id == nullptr); + + // *account_id = 0x00000000feedfaceull; + *account_id = 0; + + // return OK; + return np_error_signed_out; +} + +int KYTY_SYSV_ABI NpGetAccountCountryA(int user_id, void* country_code) { + PRINT_NAME(); + + LOGF("\t user_id = %d\n", user_id); + + EXIT_NOT_IMPLEMENTED(country_code == nullptr); + + auto* code = static_cast(country_code); + std::memset(code, 0, sizeof(*code)); + // code->data[0] = ''; + // code->data[1] = 'S'; + + // return OK; + return np_error_signed_out; +} + +int KYTY_SYSV_ABI NpGetAccountAge(int req_id, int user_id, uint8_t* age) { + PRINT_NAME(); + + LOGF("\t req_id = %d\n", req_id); + LOGF("\t user_id = %d\n", user_id); + LOGF("\t age = 0x%016" PRIx64 "\n", reinterpret_cast(age)); + + if (req_id <= 0 || age == nullptr) { + return np_error_invalid_argument; + } + + // *age = 21; + *age = 0; + + // return OK; + return np_error_signed_out; +} + +int KYTY_SYSV_ABI NpCreateRequest() { + PRINT_NAME(); + + const auto req_id = np_create_request(false); + + LOGF("\t req_id = %d\n", req_id); + + return req_id; +} + +int KYTY_SYSV_ABI NpCreateAsyncRequest(const NpCreateAsyncRequestParameter* param) { + PRINT_NAME(); + + if (param == nullptr) { + return np_error_invalid_argument; + } + + if (param->size < sizeof(NpCreateAsyncRequestParameter)) { + return np_error_invalid_size; + } + + LOGF("\t size = %" PRIu64 "\n" + "\t cpu_affinity_mask = %" PRIu64 "\n" + "\t thread_priority = %d\n", + param->size, param->cpu_affinity_mask, param->thread_priority); + + const auto req_id = np_create_request(true); + + LOGF("\t req_id = %d\n", req_id); + + return req_id; +} + +int KYTY_SYSV_ABI NpDeleteRequest(int req_id) { + PRINT_NAME(); + + LOGF("\t req_id = %d\n", req_id); + + std::lock_guard lock(g_np_request_mutex); + + auto* request = np_get_request_locked(req_id); + if (request == nullptr) { + return np_error_request_not_found; + } + + request->state = NpRequestState::Free; + request->async = false; + request->result = OK; + + return OK; +} + +int KYTY_SYSV_ABI NpAbortRequest(int req_id) { + PRINT_NAME(); + + LOGF("\t req_id = %d\n", req_id); + + std::lock_guard lock(g_np_request_mutex); + + auto* request = np_get_request_locked(req_id); + if (request == nullptr) { + return np_error_request_not_found; + } + + request->state = NpRequestState::Aborted; + request->result = np_error_aborted; + + return OK; +} + +int KYTY_SYSV_ABI NpCheckNpAvailability(int req_id, const char* user, void* result) { + PRINT_NAME(); + + // EXIT_NOT_IMPLEMENTED(req_id <= 0); + // EXIT_NOT_IMPLEMENTED(user == nullptr); + // EXIT_NOT_IMPLEMENTED(result != nullptr); + + LOGF("\t req_id = %d\n" + "\t user = %s\n", + req_id, user != nullptr ? user : "(null)"); + + if (req_id <= 0 || user == nullptr) { + return np_error_invalid_argument; + } + + if (result != nullptr) { + std::memset(result, 0, sizeof(int)); + } + + std::lock_guard lock(g_np_request_mutex); + + auto* request = np_get_request_locked(req_id); + if (request == nullptr) { + return np_error_request_not_found; + } + + // return OK; + return np_complete_signed_out_locked(request); +} + +int KYTY_SYSV_ABI NpCheckNpReachability(int req_id, int user_id) { + PRINT_NAME(); + + // EXIT_NOT_IMPLEMENTED(req_id <= 0); + + LOGF("\t req_id = %d\n", req_id); + LOGF("\t user_id = %d\n", user_id); + + if (req_id <= 0) { + return np_error_invalid_argument; + } + + std::lock_guard lock(g_np_request_mutex); + + auto* request = np_get_request_locked(req_id); + if (request == nullptr) { + return np_error_request_not_found; + } + + // return OK; + return np_complete_signed_out_locked(request); +} + +int KYTY_SYSV_ABI NpPollAsync(int req_id, int* result) { + PRINT_NAME(); + + if (result == nullptr) { + return np_error_invalid_argument; + } + + LOGF("\t req_id = %d\n", req_id); + + std::lock_guard lock(g_np_request_mutex); + + auto* request = np_get_request_locked(req_id); + if (request == nullptr) { + return np_error_request_not_found; + } + + if (!request->async || request->state == NpRequestState::Ready) { + return np_error_invalid_id; + } + + *result = request->result; + + // return request->state == NpRequestState::Aborted ? np_error_aborted : OK; + return OK; +} + +int KYTY_SYSV_ABI NpCheckPremium(int req_id, const NpCheckPremiumParameter* param, + NpCheckPremiumResult* result) { + PRINT_NAME(); + + LOGF("\t req_id = %d\n", req_id); + LOGF("\t param = 0x%016" PRIx64 "\n", reinterpret_cast(param)); + LOGF("\t result = 0x%016" PRIx64 "\n", reinterpret_cast(result)); + + if (param == nullptr || result == nullptr) { + return np_error_invalid_argument; + } + + if (param->size < sizeof(NpCheckPremiumParameter)) { + return np_error_invalid_size; + } + + LOGF("\t size = %" PRIu64 "\n" + "\t user_id = %d\n" + "\t features = 0x%016" PRIx64 "\n", + param->size, param->user_id, param->features); + + std::lock_guard lock(g_np_request_mutex); + + auto* request = np_get_request_locked(req_id); + if (request == nullptr) { + return np_error_request_not_found; + } + + if (request->state == NpRequestState::Complete) { + request->result = np_error_invalid_argument; + return np_error_invalid_argument; + } + + if (request->state == NpRequestState::Aborted) { + return np_error_aborted; + } + + std::memset(result, 0, sizeof(*result)); + // result->authorized = true; + + // request->state = NpRequestState::Complete; + // request->result = OK; + + // return OK; + return np_complete_signed_out_locked(request); +} + +int KYTY_SYSV_ABI NpGetState(int user_id, uint32_t* state) { + PRINT_NAME(); + + EXIT_NOT_IMPLEMENTED(state == nullptr); + + LOGF("\t user_id = %d\n", user_id); + + *state = 1; // Signed out + + return OK; +} + +int KYTY_SYSV_ABI NpGetNpReachabilityState(int user_id, uint32_t* state) { + PRINT_NAME(); + + constexpr int np_error_invalid_argument = -2141913085; /* 0x80550003 */ + + if (state == nullptr) { + return np_error_invalid_argument; + } + + LOGF("\t user_id = %d\n", user_id); + + // *state = 2; // SCE_NP_REACHABILITY_STATE_REACHABLE + *state = 0; // SCE_NP_REACHABILITY_STATE_UNAVAILABLE + + return OK; +} + +} // namespace NpManager + +} // namespace Libs::Network diff --git a/src/libs/network.h b/src/libs/network.h new file mode 100644 index 000000000..91da65055 --- /dev/null +++ b/src/libs/network.h @@ -0,0 +1,188 @@ +#ifndef EMULATOR_INCLUDE_EMULATOR_NETWORK_H_ +#define EMULATOR_INCLUDE_EMULATOR_NETWORK_H_ + +#include "common/abi.h" +#include "common/common.h" +#include "common/subsystems.h" + +namespace Libs::Network { + +KYTY_SUBSYSTEM_DEFINE(Network); + +namespace Net { + +struct NetEtherAddr; + +union NetEpollData { + void* ptr; + uint32_t u32; + int fd; + uint64_t u64; +}; + +struct NetEpollEvent { + uint32_t events; + uint32_t reserved; + uint64_t ident; + NetEpollData data; +}; + +static_assert(sizeof(NetEpollData) == 8); +static_assert(sizeof(NetEpollEvent) == 24); + +int KYTY_SYSV_ABI NetInit(); +int KYTY_SYSV_ABI NetTerm(); +int KYTY_SYSV_ABI NetPoolCreate(const char* name, int size, int flags); +int KYTY_SYSV_ABI NetPoolDestroy(int memid); +int KYTY_SYSV_ABI NetResolverCreate(const char* name, int memid, int flags); +int KYTY_SYSV_ABI NetResolverStartNtoa(int rid, const char* hostname, void* addr, int timeout, + int retry, int flags); +int KYTY_SYSV_ABI NetInetPton(int af, const char* src, void* dst); +const char* KYTY_SYSV_ABI NetInetNtop(int af, const void* src, char* dst, uint32_t size); +int KYTY_SYSV_ABI NetEtherNtostr(const NetEtherAddr* n, char* str, size_t len); +int KYTY_SYSV_ABI NetGetMacAddress(NetEtherAddr* addr, int flags); +int KYTY_SYSV_ABI NetGetSockInfo(int s, void* info, int n, int flags); +int KYTY_SYSV_ABI EpollCreate(const char* name, int flags); +int KYTY_SYSV_ABI EpollControl(int eid, int op, int id, const NetEpollEvent* event); +int KYTY_SYSV_ABI EpollWait(int eid, NetEpollEvent* events, int maxevents, int timeout); +int KYTY_SYSV_ABI EpollDestroy(int eid); +bool KYTY_SYSV_ABI IsSocket(int s); +int KYTY_SYSV_ABI SocketClose(int s); +int KYTY_SYSV_ABI Socket(int family, int type, int protocol); +int KYTY_SYSV_ABI Bind(int s, const void* addr, uint32_t addrlen); +int KYTY_SYSV_ABI Connect(int s, const void* addr, uint32_t addrlen); +int KYTY_SYSV_ABI Listen(int s, int backlog); +int KYTY_SYSV_ABI Accept(int s, void* addr, uint32_t* addrlen); +int KYTY_SYSV_ABI Shutdown(int s, int how); +int KYTY_SYSV_ABI Getsockname(int s, void* addr, uint32_t* addrlen); +int KYTY_SYSV_ABI Getsockopt(int s, int level, int optname, void* optval, uint32_t* optlen); +int KYTY_SYSV_ABI Setsockopt(int s, int level, int optname, const void* optval, uint32_t optlen); +int KYTY_SYSV_ABI Select(int nfds, void* readfds, void* writefds, void* exceptfds, + const void* timeout); +int64_t KYTY_SYSV_ABI Send(int s, const void* buf, uint64_t len, int flags); +int64_t KYTY_SYSV_ABI Recv(int s, void* buf, uint64_t len, int flags); + +} // namespace Net + +namespace Ssl { + +int KYTY_SYSV_ABI SslInit(uint64_t pool_size); +int KYTY_SYSV_ABI SslTerm(int ssl_ctx_id); +int KYTY_SYSV_ABI SslGetCaCerts(int ssl_ctx_id, void* ca_certs); +int KYTY_SYSV_ABI SslFreeCaCerts(int ssl_ctx_id, void* ca_certs); + +} // namespace Ssl + +namespace Http { + +struct HttpEpoll; +struct HttpNBEvent; + +using HttpEpollHandle = HttpEpoll*; + +using HttpsCallback = KYTY_SYSV_ABI int (*)(int, unsigned int, void* const*, int, void*); + +int KYTY_SYSV_ABI HttpInit(int memid, int ssl_ctx_id, uint64_t pool_size); +int KYTY_SYSV_ABI HttpTerm(int http_ctx_id); +int KYTY_SYSV_ABI HttpCreateTemplate(int http_ctx_id, const char* user_agent, int http_ver, + int is_auto_proxy_conf); +int KYTY_SYSV_ABI HttpDeleteTemplate(int tmpl_id); +int KYTY_SYSV_ABI HttpSetNonblock(int id, int enable); +int KYTY_SYSV_ABI HttpCreateEpoll(int http_ctx_id, HttpEpollHandle* eh); +int KYTY_SYSV_ABI HttpDestroyEpoll(int http_ctx_id, HttpEpollHandle eh); +int KYTY_SYSV_ABI HttpCreateConnection(int tmpl_id, const char* server_name, const char* scheme, + uint16_t port, int enable_keep_alive); +int KYTY_SYSV_ABI HttpCreateConnectionWithURL(int tmpl_id, const char* url, int enable_keep_alive); +int KYTY_SYSV_ABI HttpDeleteConnection(int conn_id); +int KYTY_SYSV_ABI HttpCreateRequest(int conn_id, int method, const char* path, + uint64_t content_length); +int KYTY_SYSV_ABI HttpCreateRequestWithURL2(int conn_id, const char* method, const char* url, + uint64_t content_length); +int KYTY_SYSV_ABI HttpSetRequestContentLength(int request_id, uint64_t content_length); +int KYTY_SYSV_ABI HttpDeleteRequest(int req_id); +int KYTY_SYSV_ABI HttpAddRequestHeader(int id, const char* name, const char* value, uint32_t mode); +int KYTY_SYSV_ABI HttpSetEpoll(int id, HttpEpollHandle eh, void* user_arg); +int KYTY_SYSV_ABI HttpUnsetEpoll(int id); +int KYTY_SYSV_ABI HttpSendRequest(int request_id, const void* post_data, size_t size); +int KYTY_SYSV_ABI HttpAbortRequest(int request_id); +int KYTY_SYSV_ABI HttpWaitRequest(HttpEpollHandle eh, HttpNBEvent* nbev, int maxevents, + int timeout); +int KYTY_SYSV_ABI HttpGetStatusCode(int request_id, int* status_code); +int KYTY_SYSV_ABI HttpGetAllResponseHeaders(int request_id, char** header, size_t* header_size); +int KYTY_SYSV_ABI HttpGetResponseContentLength(int request_id, int* result, + uint64_t* content_length); +int KYTY_SYSV_ABI HttpsSetSslCallback(int id, HttpsCallback cbfunc, void* user_arg); +int KYTY_SYSV_ABI HttpsSetMinSslVersion(int id, uint32_t ssl_version); +int KYTY_SYSV_ABI HttpsDisableOption(int id, uint32_t ssl_flags); +int KYTY_SYSV_ABI HttpSetResolveTimeOut(int id, uint32_t usec); +int KYTY_SYSV_ABI HttpSetResolveRetry(int id, int32_t retry); +int KYTY_SYSV_ABI HttpSetConnectTimeOut(int id, uint32_t usec); +int KYTY_SYSV_ABI HttpSetSendTimeOut(int id, uint32_t usec); +int KYTY_SYSV_ABI HttpSetRecvTimeOut(int id, uint32_t usec); +int KYTY_SYSV_ABI HttpSetAutoRedirect(int id, int enable); +int KYTY_SYSV_ABI HttpSetAuthEnabled(int id, int enable); + +} // namespace Http + +namespace NetCtl { + +struct NetCtlNatInfo; +union NetCtlInfo; + +using NetCtlCallback = KYTY_SYSV_ABI void (*)(int, void*); + +int KYTY_SYSV_ABI NetCtlInit(); +void KYTY_SYSV_ABI NetCtlTerm(); +int KYTY_SYSV_ABI NetCtlGetNatInfo(NetCtlNatInfo* nat_info); +int KYTY_SYSV_ABI NetCtlCheckCallback(); +int KYTY_SYSV_ABI NetCtlGetState(int* state); +int KYTY_SYSV_ABI NetCtlGetStateV6(int* state); +int KYTY_SYSV_ABI NetCtlRegisterCallback(NetCtlCallback func, void* arg, int* cid); +int KYTY_SYSV_ABI NetCtlUnregisterCallback(int cid); +int KYTY_SYSV_ABI NetCtlGetResult(int event_type, int* error_code); +int KYTY_SYSV_ABI NetCtlGetInfo(int code, NetCtlInfo* info); + +} // namespace NetCtl + +namespace NpManager { + +struct NpTitleId; +struct NpTitleSecret; +struct NpContentRestriction; +struct NpId; +struct NpOnlineId; +struct NpCreateAsyncRequestParameter; +struct NpCheckPremiumParameter; +struct NpCheckPremiumResult; + +int KYTY_SYSV_ABI NpCheckCallback(); +int KYTY_SYSV_ABI NpSetNpTitleId(const NpTitleId* title_id, const NpTitleSecret* title_secret); +int KYTY_SYSV_ABI NpSetContentRestriction(const NpContentRestriction* restriction); +int KYTY_SYSV_ABI NpRegisterStateCallback(void* callback, void* userdata); +int KYTY_SYSV_ABI NpUnregisterStateCallback(); +void KYTY_SYSV_ABI NpRegisterGamePresenceCallback(void* callback, void* userdata); +int KYTY_SYSV_ABI NpRegisterPlusEventCallback(void* callback, void* userdata); +int KYTY_SYSV_ABI NpRegisterPremiumEventCallback(void* callback, void* userdata); +int KYTY_SYSV_ABI NpRegisterNpReachabilityStateCallback(void* callback, void* userdata); +int KYTY_SYSV_ABI NpGetNpId(int user_id, NpId* np_id); +int KYTY_SYSV_ABI NpGetOnlineId(int user_id, NpOnlineId* online_id); +int KYTY_SYSV_ABI NpGetAccountIdA(int user_id, uint64_t* account_id); +int KYTY_SYSV_ABI NpGetAccountCountryA(int user_id, void* country_code); +int KYTY_SYSV_ABI NpGetAccountAge(int req_id, int user_id, uint8_t* age); +int KYTY_SYSV_ABI NpCreateRequest(); +int KYTY_SYSV_ABI NpCreateAsyncRequest(const NpCreateAsyncRequestParameter* param); +int KYTY_SYSV_ABI NpDeleteRequest(int req_id); +int KYTY_SYSV_ABI NpAbortRequest(int req_id); +int KYTY_SYSV_ABI NpCheckNpAvailability(int req_id, const char* user, void* result); +int KYTY_SYSV_ABI NpCheckNpReachability(int req_id, int user_id); +int KYTY_SYSV_ABI NpPollAsync(int req_id, int* result); +int KYTY_SYSV_ABI NpCheckPremium(int req_id, const NpCheckPremiumParameter* param, + NpCheckPremiumResult* result); +int KYTY_SYSV_ABI NpGetState(int user_id, uint32_t* state); +int KYTY_SYSV_ABI NpGetNpReachabilityState(int user_id, uint32_t* state); + +} // namespace NpManager + +} // namespace Libs::Network + +#endif /* EMULATOR_INCLUDE_EMULATOR_NETWORK_H_ */ diff --git a/src/libs/padData.h b/src/libs/padData.h new file mode 100644 index 000000000..2924dfac1 --- /dev/null +++ b/src/libs/padData.h @@ -0,0 +1,54 @@ +#ifndef EMULATOR_INCLUDE_EMULATOR_LIBS_PADDATA_H_ +#define EMULATOR_INCLUDE_EMULATOR_LIBS_PADDATA_H_ + +#include "common/common.h" + +namespace Libs::Controller { + +struct PadData { + uint32_t buttons; + uint8_t left_stick_x; + uint8_t left_stick_y; + uint8_t right_stick_x; + uint8_t right_stick_y; + uint8_t analog_buttons_l2; + uint8_t analog_buttons_r2; + uint8_t padding[2]; + float orientation_x; + float orientation_y; + float orientation_z; + float orientation_w; + float acceleration_x; + float acceleration_y; + float acceleration_z; + float angular_velocity_x; + float angular_velocity_y; + float angular_velocity_z; + uint8_t touch_data_touch_num; + uint8_t touch_data_reserve[3]; + uint32_t touch_data_reserve1; + uint16_t touch_data_touch0_x; + uint16_t touch_data_touch0_y; + uint8_t touch_data_touch0_id; + uint8_t touch_data_touch0_reserve[3]; + uint16_t touch_data_touch1_x; + uint16_t touch_data_touch1_y; + uint8_t touch_data_touch1_id; + uint8_t touch_data_touch1_reserve[3]; + bool connected; + uint64_t timestamp; + uint32_t extension_unit_data_extension_unit_id; + uint8_t extension_unit_data_reserve[1]; + uint8_t extension_unit_data_data_length; + uint8_t extension_unit_data_data[10]; + uint8_t connected_count; + uint8_t reserve[2]; + uint8_t device_unique_data_len; + uint8_t device_unique_data[12]; +}; + +static_assert(sizeof(PadData) == 120); + +} // namespace Libs::Controller + +#endif /* EMULATOR_INCLUDE_EMULATOR_LIBS_PADDATA_H_ */ diff --git a/src/libs/vaContext.h b/src/libs/vaContext.h new file mode 100644 index 000000000..399d0cc06 --- /dev/null +++ b/src/libs/vaContext.h @@ -0,0 +1,209 @@ +#ifndef EMULATOR_INCLUDE_EMULATOR_LIBS_VACONTEXT_H_ +#define EMULATOR_INCLUDE_EMULATOR_LIBS_VACONTEXT_H_ + +#include +#include + +// NOLINTNEXTLINE(cppcoreguidelines-macro-usage) +#define VA_ARGS \ + uint64_t rdi, uint64_t rsi, uint64_t rdx, uint64_t rcx, uint64_t r8, uint64_t r9, \ + uint64_t overflow_arg_area, __m128 xmm0, __m128 xmm1, __m128 xmm2, __m128 xmm3, \ + __m128 xmm4, __m128 xmm5, __m128 xmm6, __m128 xmm7, ... + +// NOLINTNEXTLINE(cppcoreguidelines-macro-usage) +#define VA_CONTEXT(ctx) \ + alignas(16) VaContext ctx; \ + (ctx).reg_save_area.gp[0] = rdi; \ + (ctx).reg_save_area.gp[1] = rsi; \ + (ctx).reg_save_area.gp[2] = rdx; \ + (ctx).reg_save_area.gp[3] = rcx; \ + (ctx).reg_save_area.gp[4] = r8; \ + (ctx).reg_save_area.gp[5] = r9; \ + (ctx).reg_save_area.fp[0] = xmm0; \ + (ctx).reg_save_area.fp[1] = xmm1; \ + (ctx).reg_save_area.fp[2] = xmm2; \ + (ctx).reg_save_area.fp[3] = xmm3; \ + (ctx).reg_save_area.fp[4] = xmm4; \ + (ctx).reg_save_area.fp[5] = xmm5; \ + (ctx).reg_save_area.fp[6] = xmm6; \ + (ctx).reg_save_area.fp[7] = xmm7; \ + (ctx).va_list.reg_save_area = &(ctx).reg_save_area; \ + (ctx).va_list.gp_offset = offsetof(VaRegSave, gp); \ + (ctx).va_list.fp_offset = offsetof(VaRegSave, fp); \ + (ctx).va_list.overflow_arg_area = &overflow_arg_area; + +namespace Libs { + +#pragma pack(1) + +struct VaList { + uint32_t gp_offset; + uint32_t fp_offset; + void* overflow_arg_area; + void* reg_save_area; +}; + +// typedef float __m128 __attribute__((__vector_size__(16), __aligned__(16))); + +struct VaRegSave { + uint64_t gp[6]; + __m128 fp[8]; +}; + +struct VaContext { + VaRegSave reg_save_area; + VaList va_list; +}; + +struct VaCharX16 { + char x[16]; +}; + +struct VaShortX8 { + short x[8]; // NOLINT(google-runtime-int) +}; + +struct VaIntX4 { + int x[4]; +}; + +struct VaFloatX4 { + float x[4]; +}; + +#pragma pack() + +template +T VaArg_overflow_arg_area(VaList* l) { + auto ptr = ((reinterpret_cast(l->overflow_arg_area) + (Align - 1)) & ~(Align - 1)); + auto* addr = reinterpret_cast(ptr); + l->overflow_arg_area = reinterpret_cast(ptr + Size); + return *addr; +} + +template +T VaArg_reg_save_area_gp(VaList* l) { + auto* addr = reinterpret_cast(static_cast(l->reg_save_area) + l->gp_offset); + l->gp_offset += Size; + return *addr; +} + +template +T VaArg_reg_save_area_fp(VaList* l) { + auto* addr = reinterpret_cast(static_cast(l->reg_save_area) + l->fp_offset); + l->fp_offset += Size; + return *addr; +} + +template <> +inline VaFloatX4 VaArg_reg_save_area_fp(VaList* l) { + auto* addr = + reinterpret_cast(static_cast(l->reg_save_area) + l->fp_offset); + l->fp_offset += 32; + VaFloatX4 ret = {{addr[0].x[0], addr[0].x[1], addr[1].x[0], addr[1].x[1]}}; + return ret; +} + +inline int VaArg_int(VaList* l) { + if (l->gp_offset <= 40) { + return VaArg_reg_save_area_gp(l); + } + return VaArg_overflow_arg_area(l); +} + +inline double VaArg_double(VaList* l) { + if (l->fp_offset <= 160) { + return VaArg_reg_save_area_fp(l); + } + return VaArg_overflow_arg_area(l); +} + +inline long double VaArg_long_double(VaList* l) { + return VaArg_overflow_arg_area(l); +} + +/*inline wint_t VaArg_wint_t(VaList* l) +{ + if (l->gp_offset <= 40) + { + return VaArg_reg_save_area_gp(l); + } + return VaArg_overflow_arg_area(l); +}*/ + +inline VaCharX16 VaArg_char_x16(VaList* l) { + if (l->gp_offset <= 32) { + return VaArg_reg_save_area_gp(l); + } + return VaArg_overflow_arg_area(l); +} + +inline long VaArg_long(VaList* l) // NOLINT(google-runtime-int) +{ + if (l->gp_offset <= 40) { + return VaArg_reg_save_area_gp(l); // NOLINT(google-runtime-int) + } + return VaArg_overflow_arg_area(l); // NOLINT(google-runtime-int) +} + +inline intmax_t VaArg_intmax_t(VaList* l) { + if (l->gp_offset <= 40) { + return VaArg_reg_save_area_gp(l); + } + return VaArg_overflow_arg_area(l); +} + +inline long long VaArg_long_long(VaList* l) // NOLINT(google-runtime-int) +{ + if (l->gp_offset <= 40) { + return VaArg_reg_save_area_gp(l); // NOLINT(google-runtime-int) + } + return VaArg_overflow_arg_area(l); // NOLINT(google-runtime-int) +} + +inline ptrdiff_t VaArg_ptrdiff_t(VaList* l) { + if (l->gp_offset <= 40) { + return VaArg_reg_save_area_gp(l); + } + return VaArg_overflow_arg_area(l); +} + +inline size_t VaArg_size_t(VaList* l) { + if (l->gp_offset <= 40) { + return VaArg_reg_save_area_gp(l); + } + return VaArg_overflow_arg_area(l); +} + +inline VaShortX8 VaArg_ShortX8(VaList* l) { + if (l->gp_offset <= 32) { + return VaArg_reg_save_area_gp(l); + } + return VaArg_overflow_arg_area(l); +} + +inline VaIntX4 VaArg_IntX4(VaList* l) { + if (l->gp_offset <= 32) { + return VaArg_reg_save_area_gp(l); + } + return VaArg_overflow_arg_area(l); +} + +inline VaFloatX4 VaArg_FloatX4(VaList* l) { + if (l->fp_offset <= 144) { + return VaArg_reg_save_area_fp(l); + } + return VaArg_overflow_arg_area(l); +} + +template +T* VaArg_ptr(VaList* l) { + if (l->gp_offset <= 40) { + return VaArg_reg_save_area_gp(l); + } + return VaArg_overflow_arg_area(l); +} + +} // namespace Libs + +#endif /* EMULATOR_INCLUDE_EMULATOR_LIBS_VACONTEXT_H_ */ diff --git a/src/loader/elf.cpp b/src/loader/elf.cpp new file mode 100644 index 000000000..df243654d --- /dev/null +++ b/src/loader/elf.cpp @@ -0,0 +1,646 @@ +#include "loader/elf.h" + +#include "common/assert.h" +#include "common/file.h" +#include "common/logging/log.h" + +#include +#include + +namespace Loader { + +static std::unique_ptr LoadSelf(Common::File& f) { + if (f.Remaining() < sizeof(SelfHeader)) { + return nullptr; + } + + auto self = std::make_unique(); + + f.Read(self.get(), sizeof(SelfHeader)); + + return self; +} + +static std::unique_ptr LoadSelfSegments(Common::File& f, uint16_t num) { + auto segs = std::make_unique(num); + + f.Read(segs.get(), sizeof(SelfSegment) * num); + + return segs; +} + +static std::unique_ptr LoadEhdr64(Common::File& f) { + if (f.Remaining() < sizeof(Elf64_Ehdr)) { + return nullptr; + } + + auto ehdr = std::make_unique(); + + f.Read(ehdr.get(), sizeof(Elf64_Ehdr)); + + return ehdr; +} + +static void SaveEhdr64(Common::File& f, const Elf64_Ehdr* ehdr) { + EXIT_IF(ehdr == nullptr); + + uint32_t bytes_written = 0; + + f.Write(ehdr, sizeof(Elf64_Ehdr), &bytes_written); + + EXIT_IF(bytes_written == 0); +} + +static std::unique_ptr LoadPhdr64(Common::File& f, uint64_t offset, Elf64_Half num) { + auto phdr = std::make_unique(num); + + f.Seek(offset); + f.Read(phdr.get(), sizeof(Elf64_Phdr) * num); + + return phdr; +} + +static void SavePhdr64(Common::File& f, uint64_t offset, Elf64_Half num, const Elf64_Phdr* phdr) { + EXIT_IF(phdr == nullptr); + + uint32_t bytes_written = 0; + + f.Seek(offset); + f.Write(phdr, sizeof(Elf64_Phdr) * num, &bytes_written); + + EXIT_IF(bytes_written == 0); +} + +static std::unique_ptr LoadShdr64(Common::File& f, uint64_t offset, Elf64_Half num) { + if (num == 0) { + return nullptr; + } + + auto shdr = std::make_unique(num); + + f.Seek(offset); + f.Read(shdr.get(), sizeof(Elf64_Shdr) * num); + + return shdr; +} + +static void SaveShdr64(Common::File& f, uint64_t offset, Elf64_Half num, const Elf64_Shdr* shdr) { + if (num == 0) { + return; + } + + EXIT_IF(shdr == nullptr); + + uint32_t bytes_written = 0; + + f.Seek(offset); + f.Write(shdr, sizeof(Elf64_Shdr) * num, &bytes_written); + + EXIT_IF(bytes_written == 0); +} + +static std::unique_ptr LoadDynamic64(Elf64* f, uint64_t offset, uint64_t size) { + auto dynamic_data = std::make_unique(size); + + // f.Seek(offset); + // f.Read(dynamic_data, size); + + f->LoadSegment(reinterpret_cast(dynamic_data.get()), offset, size); + + return dynamic_data; +} + +static std::unique_ptr LoadStrTable(Common::File& f, uint64_t offset, uint32_t size) { + if (size == 0) { + return nullptr; + } + + auto str_table = std::make_unique(size); + f.Seek(offset); + f.Read(str_table.get(), size); + return str_table; +} + +static void DbgPrintEhdr64(Elf64_Ehdr* ehdr, Common::File& f) { + f.Printf("ehdr->e_ident = "); + for (auto i: ehdr->e_ident) { + f.Printf("%02x", i); + } + f.Printf("\n"); + + f.Printf("ehdr->e_type = 0x%04" PRIx16 "\n", ehdr->e_type); + f.Printf("ehdr->e_machine = 0x%04" PRIx16 "\n", ehdr->e_machine); + f.Printf("ehdr->e_version = 0x%08" PRIx32 "\n", ehdr->e_version); + + f.Printf("ehdr->e_entry = 0x%016" PRIx64 "\n", ehdr->e_entry); + f.Printf("ehdr->e_phoff = 0x%016" PRIx64 "\n", ehdr->e_phoff); + f.Printf("ehdr->e_shoff = 0x%016" PRIx64 "\n", ehdr->e_shoff); + f.Printf("ehdr->e_flags = 0x%08" PRIx32 "\n", ehdr->e_flags); + f.Printf("ehdr->e_ehsize = 0x%04" PRIx16 "\n", ehdr->e_ehsize); + f.Printf("ehdr->e_phentsize = 0x%04" PRIx16 "\n", ehdr->e_phentsize); + f.Printf("ehdr->e_phnum = %" PRIu16 "\n", ehdr->e_phnum); + f.Printf("ehdr->e_shentsize = 0x%04" PRIx16 "\n", ehdr->e_shentsize); + f.Printf("ehdr->e_shnum = %" PRIu16 "\n", ehdr->e_shnum); + f.Printf("ehdr->e_shstrndx = %" PRIu16 "\n", ehdr->e_shstrndx); +} + +static void DbgPrintPhdr64(Elf64_Phdr* phdr, Common::File& f) { + f.Printf("phdr->p_type = 0x%08" PRIx32 "\n", phdr->p_type); + f.Printf("phdr->p_flags = 0x%08" PRIx32 "\n", phdr->p_flags); + f.Printf("phdr->p_offset = 0x%016" PRIx64 "\n", phdr->p_offset); + f.Printf("phdr->p_vaddr = 0x%016" PRIx64 "\n", phdr->p_vaddr); + f.Printf("phdr->p_paddr = 0x%016" PRIx64 "\n", phdr->p_paddr); + f.Printf("phdr->p_filesz = 0x%016" PRIx64 "\n", phdr->p_filesz); + f.Printf("phdr->p_memsz = 0x%016" PRIx64 "\n", phdr->p_memsz); + f.Printf("phdr->p_align = 0x%016" PRIx64 "\n", phdr->p_align); +} + +static void DbgPrintShdr64(Elf64_Shdr* shdr, Common::File& f) { + f.Printf("shdr->sh_name = %d\n", shdr->sh_name); + f.Printf("shdr->sh_type = 0x%08" PRIx32 "\n", shdr->sh_type); + f.Printf("shdr->sh_flags = 0x%016" PRIx64 "\n", shdr->sh_flags); + f.Printf("shdr->sh_addr = 0x%016" PRIx64 "\n", shdr->sh_addr); + f.Printf("shdr->sh_offset = 0x%016" PRIx64 "\n", shdr->sh_offset); + f.Printf("shdr->sh_size = 0x%016" PRIx64 "\n", shdr->sh_size); + f.Printf("shdr->sh_link = %" PRId32 "\n", shdr->sh_link); + f.Printf("shdr->sh_info = 0x%08" PRIx32 "\n", shdr->sh_info); + f.Printf("shdr->sh_addralign = 0x%016" PRIx64 "\n", shdr->sh_addralign); + f.Printf("shdr->sh_entsize = 0x%016" PRIx64 "\n", shdr->sh_entsize); +} + +// NOLINTNEXTLINE(cppcoreguidelines-macro-usage) +#define DBG_NAME(tag) \ + case tag: name = #tag; break; + +static void DbgPrintDynamic64(const Elf64_Dyn* dyn, Common::File& f) { + const char* name = "Unknown"; + switch (dyn->d_tag) { + DBG_NAME(DT_OS_HASH) + DBG_NAME(DT_HASH) + DBG_NAME(DT_OS_STRTAB) + DBG_NAME(DT_OS_STRSZ) + DBG_NAME(DT_STRTAB) + DBG_NAME(DT_STRSZ) + DBG_NAME(DT_OS_SYMTAB) + DBG_NAME(DT_SYMTAB) + DBG_NAME(DT_OS_HASHSZ) + DBG_NAME(DT_OS_SYMTABSZ) + DBG_NAME(DT_INIT) + DBG_NAME(DT_FINI) + DBG_NAME(DT_OS_PLTGOT) + DBG_NAME(DT_PLTGOT) + DBG_NAME(DT_OS_JMPREL) + DBG_NAME(DT_JMPREL) + DBG_NAME(DT_OS_PLTRELSZ) + DBG_NAME(DT_PLTRELSZ) + DBG_NAME(DT_OS_PLTREL) + DBG_NAME(DT_PLTREL) + DBG_NAME(DT_OS_RELA) + DBG_NAME(DT_RELA) + DBG_NAME(DT_OS_RELASZ) + DBG_NAME(DT_RELASZ) + DBG_NAME(DT_OS_RELAENT) + DBG_NAME(DT_RELAENT) + DBG_NAME(DT_INIT_ARRAY) + DBG_NAME(DT_INIT_ARRAYSZ) + DBG_NAME(DT_FINI_ARRAY) + DBG_NAME(DT_FINI_ARRAYSZ) + DBG_NAME(DT_PREINIT_ARRAY) + DBG_NAME(DT_PREINIT_ARRAYSZ) + DBG_NAME(DT_OS_SYMENT) + DBG_NAME(DT_SYMENT) + DBG_NAME(DT_DEBUG) + DBG_NAME(DT_TEXTREL) + DBG_NAME(DT_FLAGS) + DBG_NAME(DT_NEEDED) + DBG_NAME(DT_OS_NEEDED_MODULE) + DBG_NAME(DT_OS_NEEDED_MODULE_1) + DBG_NAME(DT_OS_IMPORT_LIB) + DBG_NAME(DT_OS_IMPORT_LIB_1) + DBG_NAME(DT_OS_IMPORT_LIB_ATTR) + DBG_NAME(DT_OS_FINGERPRINT) + DBG_NAME(DT_OS_ORIGINAL_FILENAME) + DBG_NAME(DT_OS_ORIGINAL_FILENAME_1) + DBG_NAME(DT_OS_MODULE_INFO) + DBG_NAME(DT_OS_MODULE_INFO_1) + DBG_NAME(DT_OS_MODULE_ATTR) + DBG_NAME(DT_SONAME) + DBG_NAME(DT_OS_EXPORT_LIB) + DBG_NAME(DT_OS_EXPORT_LIB_1) + DBG_NAME(DT_OS_EXPORT_LIB_ATTR) + DBG_NAME(DT_RELACOUNT) + DBG_NAME(DT_NULL) + } + f.Printf("d_tag = 0x%016" PRIx64 ", d_val = 0x%016" PRIx64 ", name = %s\n", dyn->d_tag, + dyn->d_un.d_val, name); +} + +Elf64::~Elf64() { + Clear(); +} + +void Elf64::LoadSegment(uint64_t vaddr, uint64_t file_offset, uint64_t size) { + EXIT_IF(m_f == nullptr); + + if (m_self != nullptr) { + EXIT_IF(m_self_segments == nullptr); + EXIT_IF(m_phdr == nullptr); + + for (uint16_t i = 0; i < m_self->segments_num; i++) { + const auto& seg = m_self_segments[i]; + if ((seg.type & 0x800u) != 0) { + auto phdr_id = ((seg.type >> 20u) & 0xFFFu); + + const auto& phdr = m_phdr[phdr_id]; + + if (file_offset >= phdr.p_offset && file_offset < phdr.p_offset + phdr.p_filesz) { + EXIT_NOT_IMPLEMENTED(seg.decompressed_size != phdr.p_filesz); + EXIT_NOT_IMPLEMENTED(seg.compressed_size != seg.decompressed_size); + + auto offset = file_offset - phdr.p_offset; + + EXIT_NOT_IMPLEMENTED(offset + size > seg.decompressed_size); + + m_f->Seek(offset + seg.offset); + m_f->Read(reinterpret_cast(static_cast(vaddr)), size); + + return; + } + } + } + + if (m_f->Size() - m_self->file_size == size) { + m_f->Seek(m_self->file_size); + m_f->Read(reinterpret_cast(static_cast(vaddr)), size); + + return; + } + + EXIT("missing self segment\n"); + } else { + m_f->Seek(file_offset); + m_f->Read(reinterpret_cast(static_cast(vaddr)), size); + } +} + +const Elf64_Dyn* Elf64::GetDynValue(Elf64_Sxword tag) const { + for (const auto* dyn = GetDynamic(); dyn->d_tag != DT_NULL; dyn++) { + if (dyn->d_tag == tag) { + return dyn; + } + } + return nullptr; +} + +std::vector Elf64::GetDynList(Elf64_Sxword tag) const { + std::vector ret; + for (const auto* dyn = GetDynamic(); dyn->d_tag != DT_NULL; dyn++) { + if (dyn->d_tag == tag) { + ret.push_back(dyn); + } + } + return ret; +} + +bool Elf64::IsShared() const { + return (m_ehdr->e_type == ET_DYNAMIC); +} + +bool Elf64::IsNextGen() const { + return (m_ehdr->e_ident[EI_ABIVERSION] == 2); +} + +const char* Elf64::GetSectionName(int index) const { + if (m_ehdr == nullptr || m_shdr == nullptr || m_str_table == nullptr || index < 0 || + index >= m_ehdr->e_shnum) { + return nullptr; + } + + const auto name_offset = m_shdr[index].sh_name; + if (name_offset >= m_str_table_size) { + return nullptr; + } + + const auto remaining = m_str_table_size - name_offset; + if (std::memchr(m_str_table.get() + name_offset, '\0', remaining) == nullptr) { + return nullptr; + } + + return m_str_table.get() + name_offset; +} + +void Elf64::Clear() { + if (m_f != nullptr) { + m_f->Close(); + } + + m_f.reset(); + m_self.reset(); + m_ehdr.reset(); + m_self_segments.reset(); + m_phdr.reset(); + m_shdr.reset(); + m_str_table.reset(); + m_str_table_size = 0; + m_dynamic.reset(); + m_dynamic_data.reset(); +} + +void Elf64::DbgDump(const std::string& folder) { + auto folder_str = Common::FixDirectorySlash(folder); + + Common::File::CreateDirectories(folder_str); + + for (uint16_t i = 0; i < m_ehdr->e_phnum; i++) { + if (m_phdr[i].p_filesz == 0u) { + continue; + } + + char str[512]; + int s = snprintf(str, 512, "phdr_%03d", i); + EXIT_NOT_IMPLEMENTED(s >= 512); + + Common::File fout; + fout.Create(folder_str + str); + + auto buf = std::make_unique(static_cast(m_phdr[i].p_filesz)); + + // m_f->Seek(m_phdr[i].p_offset); + // m_f->Read(buf, static_cast(m_phdr[i].p_filesz)); + + LoadSegment(reinterpret_cast(buf.get()), m_phdr[i].p_offset, m_phdr[i].p_filesz); + + fout.Write(buf.get(), static_cast(m_phdr[i].p_filesz)); + + fout.Close(); + } + + for (uint16_t i = 0; i < m_ehdr->e_shnum; i++) { + if (m_shdr[i].sh_size == 0u) { + continue; + } + + char str[512]; + int s = snprintf(str, 512, "shdr_%03d", i); + EXIT_NOT_IMPLEMENTED(s >= 512); + + Common::File fout; + fout.Create(folder_str + str); + + auto buf = std::make_unique(static_cast(m_shdr[i].sh_size)); + + m_f->Seek(m_shdr[i].sh_offset); + m_f->Read(buf.get(), static_cast(m_shdr[i].sh_size)); + fout.Write(buf.get(), static_cast(m_shdr[i].sh_size)); + + fout.Close(); + } + + Common::File fout; + + fout.Create(folder_str + "ehdr.txt"); + DbgPrintEhdr64(m_ehdr.get(), fout); + fout.Close(); + + fout.Create(folder_str + "phdr.txt"); + for (uint16_t i = 0; i < m_ehdr->e_phnum; i++) { + fout.Printf("--- phdr [%d] ---\n", i); + DbgPrintPhdr64(m_phdr.get() + i, fout); + } + fout.Close(); + + fout.Create(folder_str + "shdr.txt"); + for (uint16_t i = 0; i < m_ehdr->e_shnum; i++) { + const char* section_name = GetSectionName(i); + fout.Printf("--- shdr [%d] %s ---\n", i, section_name != nullptr ? section_name : ""); + DbgPrintShdr64(m_shdr.get() + i, fout); + } + fout.Close(); + + fout.Create(folder_str + "dynamic.txt"); + for (const auto* dyn = GetDynamic(); dyn->d_tag != DT_NULL; dyn++) { + DbgPrintDynamic64(dyn, fout); + } + fout.Close(); +} + +uint64_t Elf64::GetEntry() { + return m_ehdr->e_entry; +} + +bool Elf64::IsSelf() const { + if (m_f == nullptr || m_f->IsInvalid()) { + return false; + } + + if (m_self == nullptr) { + return false; + } + + const bool known_magic = (m_self->ident[0] == 0x4f && m_self->ident[1] == 0x15 && + m_self->ident[2] == 0x3d && m_self->ident[3] == 0x1d) || + (m_self->ident[0] == 0x54 && m_self->ident[1] == 0x14 && + m_self->ident[2] == 0xf5 && m_self->ident[3] == 0xee); + if (!known_magic) { + return false; + } + + const bool known_ident_tail = + (m_self->ident[4] == 0x00 && m_self->ident[5] == 0x01 && m_self->ident[6] == 0x01 && + m_self->ident[7] == 0x12 && m_self->ident[8] == 0x01 && m_self->ident[9] == 0x01 && + m_self->ident[10] == 0x00 && m_self->ident[11] == 0x00 && m_self->unknown == 0x22) || + (m_self->ident[4] == 0x10 && m_self->ident[5] == 0x01 && m_self->ident[6] == 0x01 && + m_self->ident[7] == 0x12 && m_self->ident[8] == 0x01 && m_self->ident[9] == 0x01 && + m_self->ident[10] == 0x00 && m_self->ident[11] == 0x10 && m_self->unknown == 0x32); + + if (!known_ident_tail) { + LOGF("Unknown SELF file\n"); + return false; + } + + return true; +} + +bool Elf64::IsValid() const { + if (m_f == nullptr || m_f->IsInvalid()) { + return false; + } + + if (m_ehdr == nullptr) { + return false; + } + + if (m_ehdr->e_ident[EI_MAG0] != '\x7f' || m_ehdr->e_ident[EI_MAG1] != 'E' || + m_ehdr->e_ident[EI_MAG2] != 'L' || m_ehdr->e_ident[EI_MAG3] != 'F') { + LOGF("Not an ELF file\n"); + return false; + } + + if (m_ehdr->e_ident[EI_CLASS] != ELFCLASS64) { + LOGF("ehdr->e_ident[EI_CLASS] (0x%x) != ELFCLASS64\n", m_ehdr->e_ident[EI_CLASS]); + return false; + } + + if (m_ehdr->e_ident[EI_DATA] != ELFDATA2LSB) { + LOGF("ehdr->e_ident[EI_DATA] (0x%x) != ELFDATA2LSB\n", m_ehdr->e_ident[EI_DATA]); + return false; + } + + if (m_ehdr->e_ident[EI_VERSION] != EV_CURRENT) { + LOGF("ehdr->e_ident[EI_VERSION] != EV_CURRENT\n"); + return false; + } + + if (m_ehdr->e_ident[EI_OSABI] != ELFOSABI_FREEBSD) { + LOGF("ehdr->e_ident[EI_OSABI] (0x%x) != ELFOSABI_FREEBSD\n", m_ehdr->e_ident[EI_OSABI]); + return false; + } + + if (m_ehdr->e_ident[EI_ABIVERSION] != 0 && m_ehdr->e_ident[EI_ABIVERSION] != 2) { + LOGF("ehdr->e_ident[EI_ABIVERSION] (0x%x) != (0 or 2)\n", m_ehdr->e_ident[EI_ABIVERSION]); + return false; + } + + if (m_ehdr->e_type != ET_DYNEXEC && m_ehdr->e_type != ET_DYNAMIC) { + LOGF("ehdr->e_type (%04x) != ET_DYNEXEC && m_ehdr->e_type != ET_DYNAMIC\n", m_ehdr->e_type); + return false; + } + + if (m_ehdr->e_machine != EM_X86_64) { + LOGF("ehdr->e_machine (%04x) != EM_X86_64\n", m_ehdr->e_machine); + return false; + } + + if (m_ehdr->e_version != EV_CURRENT) { + LOGF("ehdr->e_version != EV_CURRENT\n"); + return false; + } + + if (m_ehdr->e_phentsize != sizeof(Elf64_Phdr)) { + LOGF("ehdr->e_phentsize != sizeof(Elf64_Phdr)\n"); + return false; + } + + if (m_ehdr->e_shentsize > 0 && m_ehdr->e_shentsize != sizeof(Elf64_Shdr)) { + LOGF("ehdr->e_shentsize (%d) != sizeof(Elf64_Shdr)\n", m_ehdr->e_shentsize); + return false; + } + + return true; +} + +void Elf64::Open(const std::filesystem::path& file_name) { + Clear(); + + m_f = std::make_unique(); + m_f->Open(file_name, Common::File::Mode::Read); + + if (m_f->IsInvalid()) { + EXIT("Can't open %s\n", Common::PathToString(file_name).c_str()); + } + + m_self = LoadSelf(*m_f); + + if (!IsSelf()) { + m_self.reset(); + m_f->Seek(0); + } else { + m_self_segments = LoadSelfSegments(*m_f, m_self->segments_num); + } + + auto ehdr_pos = m_f->Tell(); + + m_ehdr = LoadEhdr64(*m_f); + + if (!IsValid()) { + m_ehdr.reset(); + } + + if (m_ehdr != nullptr /*&& m_self == nullptr*/) { + m_phdr = LoadPhdr64(*m_f, ehdr_pos + m_ehdr->e_phoff, m_ehdr->e_phnum); + if (m_self == nullptr) { + m_shdr = LoadShdr64(*m_f, ehdr_pos + m_ehdr->e_shoff, m_ehdr->e_shnum); + + if (m_shdr != nullptr) { + if (m_ehdr->e_shstrndx < m_ehdr->e_shnum) { + m_str_table_size = static_cast(m_shdr[m_ehdr->e_shstrndx].sh_size); + m_str_table = + LoadStrTable(*m_f, m_shdr[m_ehdr->e_shstrndx].sh_offset, m_str_table_size); + } + } + } else if (m_ehdr->e_shnum != 0) { + LOGF("SELF: skipping ELF section table: shoff=0x%016" PRIx64 ", shnum=%" PRIu16 "\n", + m_ehdr->e_shoff, m_ehdr->e_shnum); + } + + for (Elf64_Half i = 0; i < m_ehdr->e_phnum; i++) { + switch (m_phdr[i].p_type) { + case PT_DYNAMIC: + m_dynamic = LoadDynamic64(this, m_phdr[i].p_offset, m_phdr[i].p_filesz); + break; + case PT_OS_DYNLIBDATA: + m_dynamic_data = LoadDynamic64(this, m_phdr[i].p_offset, m_phdr[i].p_filesz); + break; + default: break; + } + } + } +} + +void Elf64::Save(const std::filesystem::path& file_name) { + EXIT_IF(!IsValid()); + + if (IsValid()) { + Common::File f; + f.Create(file_name); + + if (f.IsInvalid()) { + EXIT("Can't create %s\n", Common::PathToString(file_name).c_str()); + } + + SaveEhdr64(f, m_ehdr.get()); + + SavePhdr64(f, m_ehdr->e_phoff, m_ehdr->e_phnum, m_phdr.get()); + SaveShdr64(f, m_ehdr->e_shoff, m_ehdr->e_shnum, m_shdr.get()); + + for (uint16_t i = 0; i < m_ehdr->e_phnum; i++) { + if (m_phdr[i].p_filesz == 0u) { + continue; + } + + auto buf = std::make_unique(static_cast(m_phdr[i].p_filesz)); + + LoadSegment(reinterpret_cast(buf.get()), m_phdr[i].p_offset, + m_phdr[i].p_filesz); + + uint32_t bytes_written = 0; + + f.Seek(m_phdr[i].p_offset); + f.Write(buf.get(), static_cast(m_phdr[i].p_filesz), &bytes_written); + + EXIT_IF(bytes_written == 0); + } + + for (uint16_t i = 0; i < m_ehdr->e_shnum; i++) { + if (m_shdr[i].sh_size == 0u) { + continue; + } + + auto buf = std::make_unique(static_cast(m_shdr[i].sh_size)); + + m_f->Seek(m_shdr[i].sh_offset); + m_f->Read(buf.get(), static_cast(m_shdr[i].sh_size)); + + uint32_t bytes_written = 0; + + f.Seek(m_shdr[i].sh_offset); + f.Write(buf.get(), static_cast(m_shdr[i].sh_size), &bytes_written); + + EXIT_IF(bytes_written == 0); + } + + f.Close(); + } +} + +} // namespace Loader diff --git a/src/loader/elf.h b/src/loader/elf.h new file mode 100644 index 000000000..3680620ff --- /dev/null +++ b/src/loader/elf.h @@ -0,0 +1,308 @@ +#ifndef EMULATOR_INCLUDE_EMULATOR_LOADER_ELF_H_ +#define EMULATOR_INCLUDE_EMULATOR_LOADER_ELF_H_ + +#include "common/abi.h" +#include "common/common.h" +#include "common/stringUtils.h" + +#include +#include +#include + +namespace Common { +class File; +} // namespace Common + +namespace Loader { + +using Elf64_Addr = uint64_t; // Unsigned program address +using Elf64_Off = uint64_t; // Unsigned file offset +using Elf64_Half = uint16_t; // Unsigned medium integer +using Elf64_Word = uint32_t; // Unsigned integer +using Elf64_Sword = int32_t; // Signed integer +using Elf64_Xword = uint64_t; // Unsigned long integer +using Elf64_Sxword = int64_t; // Signed long integer + +constexpr int EI_MAG0 = 0; +constexpr int EI_MAG1 = 1; +constexpr int EI_MAG2 = 2; +constexpr int EI_MAG3 = 3; +constexpr int EI_CLASS = 4; +constexpr int EI_DATA = 5; +constexpr int EI_VERSION = 6; +constexpr int EI_OSABI = 7; +constexpr int EI_ABIVERSION = 8; +constexpr int EI_PAD = 9; +constexpr int EI_NIDENT = 16; + +constexpr char ELFCLASS64 = 2; // 64-bit objects + +constexpr char ELFDATA2LSB = 1; // Object file data structures are little-endian + +constexpr Elf64_Half EM_X86_64 = 62; /* AMD x86-64 architecture */ + +constexpr int EV_CURRENT = 1; + +constexpr char ELFOSABI_FREEBSD = 9; // FreeBSD operating system + +constexpr Elf64_Half ET_DYNEXEC = 0xfe10; // Executable file +constexpr Elf64_Half ET_DYNAMIC = 0xfe18; // Shared + +// #define SHT_PROGBITS 1 + +constexpr Elf64_Word PT_LOAD = 1; +constexpr Elf64_Word PT_DYNAMIC = 2; +constexpr Elf64_Word PT_TLS = 7; +constexpr Elf64_Word PT_OS_DYNLIBDATA = 0x61000000; +constexpr Elf64_Word PT_OS_PROCPARAM = 0x61000001; +constexpr Elf64_Word PT_OS_RELRO = 0x61000010; + +constexpr Elf64_Word PF_X = 0x1; // Execute +constexpr Elf64_Word PF_W = 0x2; // Write +constexpr Elf64_Word PF_R = 0x4; // Read + +constexpr Elf64_Sxword DT_DEBUG = 0x00000015; +constexpr Elf64_Sxword DT_FINI = 0x0000000d; +constexpr Elf64_Sxword DT_FINI_ARRAY = 0x0000001a; +constexpr Elf64_Sxword DT_FINI_ARRAYSZ = 0x0000001c; +constexpr Elf64_Sxword DT_FLAGS = 0x0000001e; +constexpr Elf64_Sxword DT_INIT = 0x0000000c; +constexpr Elf64_Sxword DT_INIT_ARRAY = 0x00000019; +constexpr Elf64_Sxword DT_INIT_ARRAYSZ = 0x0000001b; +constexpr Elf64_Sxword DT_NEEDED = 0x00000001; +constexpr Elf64_Sxword DT_OS_EXPORT_LIB = 0x61000013; +constexpr Elf64_Sxword DT_OS_EXPORT_LIB_1 = 0x61000047; +constexpr Elf64_Sxword DT_OS_EXPORT_LIB_ATTR = 0x61000017; +constexpr Elf64_Sxword DT_OS_FINGERPRINT = 0x61000007; +constexpr Elf64_Sxword DT_OS_HASH = 0x61000025; +constexpr Elf64_Sxword DT_OS_HASHSZ = 0x6100003d; +constexpr Elf64_Sxword DT_OS_IMPORT_LIB = 0x61000015; +constexpr Elf64_Sxword DT_OS_IMPORT_LIB_1 = 0x61000049; +constexpr Elf64_Sxword DT_OS_IMPORT_LIB_ATTR = 0x61000019; +constexpr Elf64_Sxword DT_OS_JMPREL = 0x61000029; +constexpr Elf64_Sxword DT_OS_MODULE_ATTR = 0x61000011; +constexpr Elf64_Sxword DT_OS_MODULE_INFO = 0x6100000d; +constexpr Elf64_Sxword DT_OS_MODULE_INFO_1 = 0x61000043; +constexpr Elf64_Sxword DT_OS_NEEDED_MODULE = 0x6100000f; +constexpr Elf64_Sxword DT_OS_NEEDED_MODULE_1 = 0x61000045; +constexpr Elf64_Sxword DT_OS_ORIGINAL_FILENAME = 0x61000009; +constexpr Elf64_Sxword DT_OS_ORIGINAL_FILENAME_1 = 0x61000041; +constexpr Elf64_Sxword DT_OS_PLTGOT = 0x61000027; +constexpr Elf64_Sxword DT_OS_PLTREL = 0x6100002b; +constexpr Elf64_Sxword DT_OS_PLTRELSZ = 0x6100002d; +constexpr Elf64_Sxword DT_OS_RELA = 0x6100002f; +constexpr Elf64_Sxword DT_OS_RELAENT = 0x61000033; +constexpr Elf64_Sxword DT_OS_RELASZ = 0x61000031; +constexpr Elf64_Sxword DT_OS_STRSZ = 0x61000037; +constexpr Elf64_Sxword DT_OS_STRTAB = 0x61000035; +constexpr Elf64_Sxword DT_OS_SYMENT = 0x6100003b; +constexpr Elf64_Sxword DT_OS_SYMTAB = 0x61000039; +constexpr Elf64_Sxword DT_OS_SYMTABSZ = 0x6100003f; +constexpr Elf64_Sxword DT_PREINIT_ARRAY = 0x00000020; +constexpr Elf64_Sxword DT_PREINIT_ARRAYSZ = 0x00000021; +constexpr Elf64_Sxword DT_REL = 0x00000011; +constexpr Elf64_Sxword DT_RELA = 0x00000007; +constexpr Elf64_Sxword DT_SONAME = 0x0000000e; +constexpr Elf64_Sxword DT_TEXTREL = 0x00000016; + +constexpr Elf64_Sxword DT_HASH = 0x00000004; +constexpr Elf64_Sxword DT_STRTAB = 0x00000005; +constexpr Elf64_Sxword DT_STRSZ = 0x0000000a; +constexpr Elf64_Sxword DT_SYMTAB = 0x00000006; +constexpr Elf64_Sxword DT_SYMENT = 0x0000000b; +constexpr Elf64_Sxword DT_PLTGOT = 0x00000003; +constexpr Elf64_Sxword DT_PLTREL = 0x00000014; +constexpr Elf64_Sxword DT_JMPREL = 0x00000017; +constexpr Elf64_Sxword DT_PLTRELSZ = 0x00000002; +constexpr Elf64_Sxword DT_RELASZ = 0x00000008; +constexpr Elf64_Sxword DT_RELAENT = 0x00000009; +constexpr Elf64_Sxword DT_RELACOUNT = 0x6ffffff9; + +constexpr Elf64_Sxword DT_NULL = 0; + +constexpr Elf64_Word R_X86_64_64 = 1; +constexpr Elf64_Word R_X86_64_GLOB_DAT = 6; +constexpr Elf64_Word R_X86_64_JUMP_SLOT = 7; +constexpr Elf64_Word R_X86_64_RELATIVE = 8; +constexpr Elf64_Word R_X86_64_DTPMOD64 = 16; + +constexpr uint8_t STB_LOCAL = 0; +constexpr uint8_t STB_GLOBAL = 1; +constexpr uint8_t STB_WEAK = 2; + +constexpr uint8_t STT_NOTYPE = 0; +constexpr uint8_t STT_OBJECT = 1; +constexpr uint8_t STT_FUNC = 2; + +#pragma pack(1) + +struct SelfHeader { + uint8_t ident[12]; + uint16_t size1; + uint16_t size2; + uint64_t file_size; + uint16_t segments_num; + uint16_t unknown; + uint32_t pad; +}; + +struct SelfSegment { + uint64_t type; + uint64_t offset; + uint64_t compressed_size; + uint64_t decompressed_size; +}; + +struct Elf64_Ehdr // NOLINT(readability-identifier-naming) +{ + uint8_t e_ident[EI_NIDENT]; /* ELF identification */ + Elf64_Half e_type; /* Object file type */ + Elf64_Half e_machine; /* Machine type */ + Elf64_Word e_version; /* Object file version */ + Elf64_Addr e_entry; /* Entry point address */ + Elf64_Off e_phoff; /* Program header offset */ + Elf64_Off e_shoff; /* Section header offset */ + Elf64_Word e_flags; /* Processor-specific flags */ + Elf64_Half e_ehsize; /* ELF header size */ + Elf64_Half e_phentsize; /* Size of program header entry */ + Elf64_Half e_phnum; /* Number of program header entries */ + Elf64_Half e_shentsize; /* Size of section header entry */ + Elf64_Half e_shnum; /* Number of section header entries */ + Elf64_Half e_shstrndx; /* Section name string table index */ +}; + +struct Elf64_Phdr // NOLINT(readability-identifier-naming) +{ + Elf64_Word p_type; /* Type of segment */ + Elf64_Word p_flags; /* Segment attributes */ + Elf64_Off p_offset; /* Offset in file */ + Elf64_Addr p_vaddr; /* Virtual address in memory */ + Elf64_Addr p_paddr; /* Reserved */ + Elf64_Xword p_filesz; /* Size of segment in file */ + Elf64_Xword p_memsz; /* Size of segment in memory */ + Elf64_Xword p_align; /* Alignment of segment */ +}; + +struct Elf64_Shdr // NOLINT(readability-identifier-naming) +{ + Elf64_Word sh_name; /* Section name */ + Elf64_Word sh_type; /* Section type */ + Elf64_Xword sh_flags; /* Section attributes */ + Elf64_Addr sh_addr; /* Virtual address in memory */ + Elf64_Off sh_offset; /* Offset in file */ + Elf64_Xword sh_size; /* Size of section */ + Elf64_Word sh_link; /* Link to other section */ + Elf64_Word sh_info; /* Miscellaneous information */ + Elf64_Xword sh_addralign; /* Address alignment boundary */ + Elf64_Xword sh_entsize; /* Size of entries, if section has table */ +}; + +struct Elf64_Dyn // NOLINT(readability-identifier-naming) +{ + Elf64_Sxword d_tag; + union { + Elf64_Xword d_val; + Elf64_Addr d_ptr; + } d_un; +}; + +struct Elf64_Sym // NOLINT(readability-identifier-naming) +{ + [[nodiscard]] unsigned char GetBind() const { return st_info >> 4u; } + [[nodiscard]] unsigned char GetType() const { return st_info & 0xfu; } + + Elf64_Word st_name; + unsigned char st_info; + unsigned char st_other; + Elf64_Half st_shndx; + Elf64_Addr st_value; + Elf64_Xword st_size; +}; + +// struct Elf64_Rel // NOLINT(readability-identifier-naming) +//{ +// Elf64_Addr r_offset; +// Elf64_Xword r_info; +//}; + +struct Elf64_Rela // NOLINT(readability-identifier-naming) +{ + [[nodiscard]] Elf64_Word GetSymbol() const { return static_cast(r_info >> 32u); } + [[nodiscard]] Elf64_Word GetType() const { + return static_cast(r_info & 0xffffffff); + } + + Elf64_Addr r_offset; + Elf64_Xword r_info; + Elf64_Sxword r_addend; +}; + +struct tls_info_t // NOLINT(readability-identifier-naming) +{ + uint64_t addr; + uint64_t filesz; + uint64_t memsz; +}; + +#pragma pack() + +class Elf64 { +public: + Elf64() = default; + virtual ~Elf64(); + + void Open(const std::filesystem::path& file_name); + void Save(const std::filesystem::path& file_name); + + void DbgDump(const std::string& folder); + + const char* GetSectionName(int index) const; + + [[nodiscard]] bool IsSelf() const; + [[nodiscard]] bool IsValid() const; + [[nodiscard]] bool IsShared() const; + [[nodiscard]] bool IsNextGen() const; + + void LoadSegment(uint64_t vaddr, uint64_t file_offset, uint64_t size); + + uint64_t GetEntry(); + + [[nodiscard]] const Elf64_Dyn* GetDynValue(Elf64_Sxword tag) const; + [[nodiscard]] std::vector GetDynList(Elf64_Sxword tag) const; + [[nodiscard]] bool HasDynValue(Elf64_Sxword tag) const { return GetDynValue(tag) != nullptr; } + + KYTY_CLASS_NO_COPY(Elf64); + + [[nodiscard]] const Elf64_Dyn* GetDynamic() const { + return reinterpret_cast(m_dynamic.get()); + } + [[nodiscard]] const Elf64_Ehdr* GetEhdr() const { return m_ehdr.get(); } + [[nodiscard]] const Elf64_Phdr* GetPhdr() const { return m_phdr.get(); } + [[nodiscard]] const Elf64_Shdr* GetShdr() const { return m_shdr.get(); } + [[nodiscard]] char* GetStrTable() const { return m_str_table.get(); } + + template + [[nodiscard]] T GetDynamicData(uint64_t offset) const { + return (m_dynamic_data == nullptr ? nullptr + : reinterpret_cast(m_dynamic_data.get() + offset)); + } + +private: + void Clear(); + + std::unique_ptr m_f; + std::unique_ptr m_self; + std::unique_ptr m_self_segments; + std::unique_ptr m_ehdr; + std::unique_ptr m_phdr; + std::unique_ptr m_shdr; + std::unique_ptr m_dynamic; + std::unique_ptr m_dynamic_data; + std::unique_ptr m_str_table; + uint32_t m_str_table_size = 0; + // uint64_t m_base_vaddr = 0; +}; + +} // namespace Loader + +#endif /* EMULATOR_INCLUDE_EMULATOR_LOADER_ELF_H_ */ diff --git a/src/loader/jit.h b/src/loader/jit.h new file mode 100644 index 000000000..7d5a2f1fc --- /dev/null +++ b/src/loader/jit.h @@ -0,0 +1,167 @@ +#ifndef EMULATOR_INCLUDE_EMULATOR_LOADER_JIT_H_ +#define EMULATOR_INCLUDE_EMULATOR_LOADER_JIT_H_ + +#include "common/abi.h" + +namespace Loader::Jit { + +#pragma pack(1) + +struct JmpWithIndex { + void SetIndex(uint32_t index) { *reinterpret_cast(&code[1]) = index; } + + void SetFunc(void* handler) { + auto func_addr = reinterpret_cast(handler); + auto rip_addr = reinterpret_cast(&code[10]); + auto offset64 = func_addr - rip_addr; + auto offset32 = static_cast(static_cast(offset64) & 0xffffffffu); + + *reinterpret_cast(&code[6]) = offset32; + } + + static uint64_t GetSize() { return 16; } + + // 68 00 00 00 00 push + // E9 E0 FF FF FF jmp + uint8_t code[16] = {0x68, 0x00, 0x00, 0x00, 0x00, 0xE9, 0x00, 0x00, + 0x00, 0x00, 0x90, 0x90, 0x90, 0x90, 0x90, 0x90}; +}; + +struct CallPlt { + explicit CallPlt(uint32_t table_size) { + for (uint32_t index = 0; index < table_size; index++) { + auto* c = new (&code[32] + JmpWithIndex::GetSize() * index) JmpWithIndex; + c->SetIndex(index); + c->SetFunc(this); + } + } + + void SetPltGot(uint64_t vaddr) { *reinterpret_cast(&code[2]) = vaddr; } + + uint64_t GetAddr(uint32_t index) { + return reinterpret_cast(&code[32] + JmpWithIndex::GetSize() * index); + } + + static uint64_t GetSize(uint32_t table_size) { + return 32 + JmpWithIndex::GetSize() * table_size; + } + + // 0: 49 bb 88 77 66 55 44 movabs r11,0x1122334455667788 + // 7: 33 22 11 + // a: 41 ff 73 08 push QWORD PTR [r11+0x8] + // e: 41 ff 63 10 jmp QWORD PTR [r11+0x10] + uint8_t code[32] = {0x49, 0xBB, 0x88, 0x77, 0x66, 0x55, 0x44, 0x33, 0x22, 0x11, 0x41, 0xFF, + 0x73, 0x08, 0x41, 0xFF, 0x63, 0x10, 0x90, 0x90, 0x90, 0x90, 0x90, 0x90}; +}; + +struct JmpRax { + template + void SetFunc(Handler func) { + *reinterpret_cast(&code[2]) = func; + } + + // mov rax, 0x1122334455667788 + // jmp rax + uint8_t code[16] = {0x48, 0xB8, 0x88, 0x77, 0x66, 0x55, 0x44, 0x33, 0x22, 0x11, 0xFF, 0xE0}; +}; + +struct Call9 { + template + void SetFunc(Handler func) { + auto func_addr = reinterpret_cast(reinterpret_cast(func)); + auto rip_addr = reinterpret_cast(&code[5]); + auto offset64 = func_addr - rip_addr; + auto offset32 = static_cast(static_cast(offset64) & 0xffffffffu); + + *reinterpret_cast(&code[1]) = offset32; + } + + void SetOutputReg(uint8_t reg) { code[7] = 0xc0u | (reg & 7u); } + + static uint64_t GetSize() { return 9; } + + // call func + // mov rax,rax + // nop + uint8_t code[9] = {0xE8, 0x00, 0x00, 0x00, 0x00, 0x48, 0x89, 0xC0, 0x90}; +}; + +struct TlsRegStub { + template + void SetFunc(Handler func) { + auto func_addr = reinterpret_cast(reinterpret_cast(func)); + auto rip_addr = reinterpret_cast(&code[13]); + auto offset64 = func_addr - rip_addr; + auto offset32 = static_cast(static_cast(offset64) & 0xffffffffu); + + *reinterpret_cast(&code[9]) = offset32; + } + + void SetOutputReg(uint8_t reg) { code[15] = 0xc0u | (reg & 7u); } + + static uint64_t GetOffset(uint8_t reg) { + return 0x100 + static_cast(reg) * GetSize(); + } + static uint64_t GetSize() { return 32; } + + // sub rsp,0x80 + // push rax + // call safe_call + // mov ,rax + // pop rax + // add rsp,0x80 + // ret + uint8_t code[32] = {0x48, 0x81, 0xEC, 0x80, 0x00, 0x00, 0x00, 0x50, 0xE8, 0x00, 0x00, + 0x00, 0x00, 0x48, 0x89, 0xC0, 0x58, 0x48, 0x81, 0xC4, 0x80, 0x00, + 0x00, 0x00, 0xC3, 0x90, 0x90, 0x90, 0x90, 0x90, 0x90, 0x90}; +}; + +struct SafeCall { + using func_t = KYTY_MS_ABI uint8_t* (*)(); + + void SetFunc(func_t func) { *reinterpret_cast(&code[0x22]) = func; } + + static uint64_t GetSize() { return 0x1000; } + + uint8_t code[0x90] = { + /*00*/ 0x48, 0x81, 0xec, 0x80, 0x00, 0x00, 0x00, // sub rsp,0x80 + /*07*/ 0x9c, // pushfq + /*08*/ 0x51, // push rcx + /*09*/ 0x52, // push rdx + /*0a*/ 0x41, 0x50, // push r8 + /*0c*/ 0x41, 0x51, // push r9 + /*0e*/ 0x41, 0x52, // push r10 + /*10*/ 0x41, 0x53, // push r11 + /*12*/ 0x53, // push rbx + /*13*/ 0x57, // push rdi + /*14*/ 0x56, // push rsi + /*15*/ 0x48, 0x89, 0xe3, // mov rbx,rsp + /*18*/ 0x48, 0x83, 0xe4, 0xf0, // and rsp,0xfffffffffffffff0 + /*1c*/ 0x48, 0x83, 0xec, 0x20, // sub rsp,0x20 + /*20*/ 0x48, 0xb9, 0x88, 0x77, 0x66, 0x55, 0x44, + 0x33, 0x22, 0x11, // movabs rcx,0x1122334455667788 + /*2a*/ 0xff, 0xd1, // call rcx + /*2c*/ 0x48, 0x89, 0xc1, // mov rcx,rax + /*2f*/ 0x48, 0x89, 0xdc, // mov rsp,rbx + /*32*/ 0x48, 0x89, 0xc8, // mov rax,rcx + /*35*/ 0x5e, // pop rsi + /*36*/ 0x5f, // pop rdi + /*37*/ 0x5b, // pop rbx + /*38*/ 0x41, 0x5b, // pop r11 + /*3a*/ 0x41, 0x5a, // pop r10 + /*3c*/ 0x41, 0x59, // pop r9 + /*3e*/ 0x41, 0x58, // pop r8 + /*40*/ 0x5a, // pop rdx + /*41*/ 0x59, // pop rcx + /*42*/ 0x9d, // popfq + /*43*/ 0x48, 0x81, 0xc4, 0x80, 0x00, 0x00, + 0x00, // add rsp,0x80 + /*4a*/ 0xc3, // ret + }; +}; + +#pragma pack() + +} // namespace Loader::Jit + +#endif /* EMULATOR_INCLUDE_EMULATOR_LOADER_JIT_H_ */ diff --git a/src/loader/runtimeLinker.cpp b/src/loader/runtimeLinker.cpp new file mode 100644 index 000000000..526c31d00 --- /dev/null +++ b/src/loader/runtimeLinker.cpp @@ -0,0 +1,2387 @@ +#include "loader/runtimeLinker.h" + +#include "common/assert.h" +#include "common/common.h" +#include "common/emulatorConfig.h" +#include "common/file.h" +#include "common/hostException.h" +#include "common/logging/log.h" +#include "common/magicEnum.h" +#include "common/platform/sysDbg.h" +#include "common/profiler.h" +#include "common/singleton.h" +#include "common/stringUtils.h" +#include "common/threads.h" +#include "common/virtualMemory.h" +#include "graphics/host_gpu/renderer/renderContext.h" +#include "kernel/memory.h" +#include "kernel/pthread.h" +#include "loader/elf.h" +#include "loader/jit.h" +#include "loader/symbolDatabase.h" +#include "loader/x64InstructionEmulator.h" + +#include +#include +#include +#include +#include +#include +#include +#include + +#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS +#ifndef NOMINMAX +#define NOMINMAX +#endif +#include +#endif + +namespace Libs::LibKernel { +void SetProgName(const std::string& name); +} // namespace Libs::LibKernel + +namespace Loader { + +Program::Program() = default; + +Program::~Program() = default; + +static void FreeTlsBlock(ThreadLocalStorage::Block* block) { + if (block == nullptr || block->ptr == nullptr) { + return; + } + + if (block->free_func != nullptr) { + block->free_func(block->ptr); + } else if (block->vm_alloc) { + Common::VirtualMemory::Free(reinterpret_cast(block->ptr)); + } else { + delete[] block->ptr; + } + + block->ptr = nullptr; + block->free_func = nullptr; + block->vm_alloc = false; +} + +static uint64_t AlignUp(uint64_t value, uint64_t alignment) { + return alignment != 0 ? (value + alignment - 1) & ~(alignment - 1) : value; +} + +ThreadLocalStorage::~ThreadLocalStorage() { + for (auto& [_, block]: tlss) { + FreeTlsBlock(&block); + } +} + +#pragma pack(1) + +struct EntryParams { + int argc; + uint32_t pad; + const char* argv[3]; +}; + +#pragma pack() + +using atexit_func_t = KYTY_SYSV_ABI void (*)(); +using entry_func_t = KYTY_SYSV_ABI void (*)(EntryParams* params, atexit_func_t atexit_func); +using module_ini_fini_func_t = KYTY_SYSV_ABI int (*)(size_t args, const void* argp, + module_func_t func); + +enum class BindType { Unknown, Local, Global, Weak }; + +struct RelocationInfo { + bool resolved = false; + BindType bind = BindType::Unknown; + SymbolType type = SymbolType::Unknown; + uint64_t value = 0; + uint64_t vaddr = 0; + uint64_t base_vaddr = 0; + std::string name; + std::string dbg_name; + bool bind_self = false; +}; + +struct StubbedImportRecord { + uint32_t index = 0; + uint64_t patch_vaddr = 0; + uint64_t thunk_vaddr = 0; + std::string name; + SymbolType type = SymbolType::Unknown; + BindType bind = BindType::Unknown; + std::string program; +}; + +// The structure will be passed via the stack +// since the size of an object is larger than 16 bytes +struct RelocateHandlerStack { + uint64_t stack[3]; +}; + +static std::vector g_stubbed_imports; +static std::atomic_uint32_t g_unresolved_stub_call_log_count {0}; +static std::vector g_unresolved_stub_thunk_pages; +static uint64_t g_unresolved_stub_thunk_offset = 0; + +static KYTY_SYSV_ABI uint64_t ResolveImportStubWithId(uint64_t record_id); + +static uint64_t AllocateUnresolvedImportThunk(uint64_t record_id) { + constexpr uint64_t page_size = 4096; + constexpr uint64_t thunk_size = 162; + + if (g_unresolved_stub_thunk_pages.empty() || + g_unresolved_stub_thunk_offset + thunk_size > page_size) { + auto page = Common::VirtualMemory::Alloc(0, page_size, + Common::VirtualMemory::Mode::ExecuteReadWrite); + EXIT_NOT_IMPLEMENTED(page == 0); + g_unresolved_stub_thunk_pages.push_back(page); + g_unresolved_stub_thunk_offset = 0; + } + + auto* code = reinterpret_cast(g_unresolved_stub_thunk_pages.back() + + g_unresolved_stub_thunk_offset); + g_unresolved_stub_thunk_offset += thunk_size; + + const auto target = reinterpret_cast(ResolveImportStubWithId); + uint8_t bytes[thunk_size] {}; + size_t i = 0; + const auto emit = [&](uint8_t b) { bytes[i++] = b; }; + const auto emit64 = [&](uint64_t v) { + std::memcpy(bytes + i, &v, sizeof(v)); + i += sizeof(v); + }; + const auto emit32 = [&](uint32_t v) { + std::memcpy(bytes + i, &v, sizeof(v)); + i += sizeof(v); + }; + const auto save_xmm = [&](uint8_t reg, uint8_t offset) { + emit(0xf3); + emit(0x0f); + emit(0x7f); + if (offset == 0) { + emit(static_cast(0x04u | (reg << 3u))); + emit(0x24); + } else { + emit(static_cast(0x44u | (reg << 3u))); + emit(0x24); + emit(offset); + } + }; + const auto load_xmm = [&](uint8_t reg, uint8_t offset) { + emit(0xf3); + emit(0x0f); + emit(0x6f); + if (offset == 0) { + emit(static_cast(0x04u | (reg << 3u))); + emit(0x24); + } else { + emit(static_cast(0x44u | (reg << 3u))); + emit(0x24); + emit(offset); + } + }; + + emit(0x50); // push rax; preserve AL for variadic SysV calls + emit(0x57); // push rdi + emit(0x56); // push rsi + emit(0x52); // push rdx + emit(0x51); // push rcx + emit(0x41); + emit(0x50); // push r8 + emit(0x41); + emit(0x51); // push r9 + emit(0x48); + emit(0x81); + emit(0xec); + emit32(0x80); // sub rsp, 0x80 + for (uint8_t reg = 0; reg < 8; reg++) { + save_xmm(reg, static_cast(reg * 0x10u)); + } + emit(0x48); + emit(0xbf); + emit64(record_id); // mov rdi, record_id + emit(0x48); + emit(0xb8); + emit64(target); // mov rax, ResolveImportStubWithId + emit(0xff); + emit(0xd0); // call rax + emit(0x49); + emit(0x89); + emit(0xc3); // mov r11, rax + for (uint8_t reg = 0; reg < 8; reg++) { + load_xmm(reg, static_cast(reg * 0x10u)); + } + emit(0x48); + emit(0x81); + emit(0xc4); + emit32(0x80); // add rsp, 0x80 + emit(0x41); + emit(0x59); // pop r9 + emit(0x41); + emit(0x58); // pop r8 + emit(0x59); // pop rcx + emit(0x5a); // pop rdx + emit(0x5e); // pop rsi + emit(0x5f); // pop rdi + emit(0x58); // pop rax + emit(0x4d); + emit(0x85); + emit(0xdb); // test r11, r11 + emit(0x74); + emit(0x03); // jz +3 + emit(0x41); + emit(0xff); + emit(0xe3); // jmp r11 + emit(0x31); + emit(0xc0); // xor eax, eax + emit(0xc3); // ret + + EXIT_NOT_IMPLEMENTED(i != thunk_size); + std::memcpy(code, bytes, sizeof(bytes)); + Common::VirtualMemory::FlushInstructionCache(reinterpret_cast(code), thunk_size); + return reinterpret_cast(code); +} + +static uint64_t RegisterStubbedImport(uint32_t index, const Program* program, + const RelocationInfo& ri) { + const auto program_name = program != nullptr ? Common::PathToString(program->file_name) : ""; + + for (auto& record: g_stubbed_imports) { + if (record.patch_vaddr == ri.vaddr) { + record.index = index; + record.name = ri.name; + record.type = ri.type; + record.bind = ri.bind; + record.program = program_name; + return record.thunk_vaddr; + } + } + + StubbedImportRecord record {}; + record.index = index; + record.patch_vaddr = ri.vaddr; + record.name = ri.name; + record.type = ri.type; + record.bind = ri.bind; + record.program = program_name; + g_stubbed_imports.push_back(record); + const auto record_id = g_stubbed_imports.size() - 1; + const auto thunk = AllocateUnresolvedImportThunk(record_id); + g_stubbed_imports[record_id].thunk_vaddr = thunk; + return thunk; +} + +static KYTY_SYSV_ABI uint64_t ResolveImportStubWithId(uint64_t record_id) { + if (record_id < g_stubbed_imports.size()) { + auto& record = g_stubbed_imports[record_id]; + auto nid = record.name; + auto pos = Common::FindIndex(nid, "["); + if (Common::IndexValid(nid, pos)) { + nid = Common::Left(nid, pos); + } + + SymbolRecord resolved {}; + if (!nid.empty() && + Common::Singleton::Instance()->ResolveLoadedSymbolByNid(nid, record.type, + &resolved) && + resolved.vaddr != 0 && resolved.vaddr != record.thunk_vaddr) { + LOGF("Late-resolved import: %s -> %s [0x%016" PRIx64 "]\n", record.name.c_str(), + resolved.name.c_str(), resolved.vaddr); + + if (record.patch_vaddr != 0) { + *reinterpret_cast(record.patch_vaddr) = resolved.vaddr; + } + + return resolved.vaddr; + } + } + + const auto log_index = g_unresolved_stub_call_log_count.fetch_add(1); + if (log_index < 1024) { + if (record_id < g_stubbed_imports.size()) { + const auto& record = g_stubbed_imports[record_id]; + printf("Unresolved import stub called: %s\n", record.name.c_str()); + LOGF("Unresolved import stub called [%u]: patch_vaddr=0x%016" PRIx64 + " jmprela_index=%" PRIu32 " symbol=%s type=%s bind=%s program=%s\n", + log_index, record.patch_vaddr, record.index, record.name.c_str(), + Common::EnumName(record.type).c_str(), Common::EnumName(record.bind).c_str(), + record.program.c_str()); + } else { + printf("Unresolved import stub called: \n"); + LOGF("Unresolved import stub called [%u]: record_id=%" PRIu64 " symbol=\n", + log_index, record_id); + } + } + return 0; +} + +constexpr uint64_t SYSTEM_RESERVED = 0x800000000u; +constexpr uint64_t CODE_BASE_INCR = 0x010000000u; +constexpr uint64_t INVALID_OFFSET = 0x040000000u; +constexpr uint64_t CODE_BASE_OFFSET = 0x100000000u; +constexpr uint64_t INVALID_MEMORY = SYSTEM_RESERVED + INVALID_OFFSET; + +static uint64_t g_desired_base_addr = SYSTEM_RESERVED + CODE_BASE_OFFSET; +static uint64_t g_invalid_memory = 0; + +static Program* g_tls_main_program = nullptr; +static thread_local Program* g_tls_cached_main_program = nullptr; +static thread_local uint8_t* g_tls_cached_main_tcb = nullptr; + +static KYTY_SYSV_ABI void RunEntry(uint64_t addr, EntryParams* params, atexit_func_t atexit_func, + void* stack_top) { +#if defined(__x86_64__) || defined(_M_X64) + auto* func = reinterpret_cast(addr); + + if (stack_top != nullptr) { + const auto guest_rsp = + reinterpret_cast(stack_top) & ~static_cast(0x0f); + const auto guest_rbp = guest_rsp - 4u * sizeof(uint64_t); + + auto* guest_root_frame = reinterpret_cast(guest_rbp); + guest_root_frame[0] = 0; + guest_root_frame[1] = 0; + + asm volatile("pushq %%r12\n\t" + "pushq %%r13\n\t" + "movq %%rsp, %%r12\n\t" + "movq %%rbp, %%r13\n\t" + "movq %[guest_rsp], %%rsp\n\t" + "movq %[guest_rbp], %%rbp\n\t" + "callq *%[func]\n\t" + "movq %%r13, %%rbp\n\t" + "movq %%r12, %%rsp\n\t" + "popq %%r13\n\t" + "popq %%r12\n\t" + : + : [func] "r"(func), "D"(params), + "S"(atexit_func), [guest_rsp] "r"(guest_rsp), [guest_rbp] "r"(guest_rbp) + : "cc", "memory", "rax", "rcx", "rdx", "r8", "r9", "r10", "r11", "xmm0", + "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6", "xmm7", "xmm8", "xmm9", + "xmm10", "xmm11", "xmm12", "xmm13", "xmm14", "xmm15"); + return; + } + + uintptr_t guest_root_frame[2] = {}; + + asm volatile("pushq %%r12\n\t" + "pushq %%r13\n\t" + "movq %%rbp, %%r12\n\t" + "movq %[guest_rbp], %%rbp\n\t" + "callq *%[func]\n\t" + "movq %%r12, %%rbp\n\t" + "popq %%r13\n\t" + "popq %%r12\n\t" + : + : [func] "r"(func), "D"(params), + "S"(atexit_func), [guest_rbp] "r"(guest_root_frame) + : "cc", "memory", "rax", "rcx", "rdx", "r8", "r9", "r10", "r11", "xmm0", "xmm1", + "xmm2", "xmm3", "xmm4", "xmm5", "xmm6", "xmm7", "xmm8", "xmm9", "xmm10", "xmm11", + "xmm12", "xmm13", "xmm14", "xmm15"); +#else + (void)stack_top; + reinterpret_cast(addr)(params, atexit_func); +#endif +} + +static uint64_t GetAlignedSize(const Elf64_Phdr* p) { + return (p->p_align != 0 ? (p->p_memsz + (p->p_align - 1)) & ~(p->p_align - 1) : p->p_memsz); +} + +static void DbgDumpSymbols(const std::string& folder, Elf64_Sym* symbols, uint64_t size, + const char* names) { + auto folder_str = Common::FixDirectorySlash(folder); + + Common::File::CreateDirectories(folder_str); + + Common::File f; + f.Create(folder_str + "symbols.txt"); + + for (auto* sym = symbols; + reinterpret_cast(sym) < reinterpret_cast(symbols) + size; sym++) { + f.Printf("----\n"); + f.Printf("st_name = %" PRIu32 ", %s\n", sym->st_name, names + sym->st_name); + f.Printf("st_info = 0x%02" PRIx8 "\n", sym->st_info); + f.Printf("st_other = 0x%02" PRIx8 "\n", sym->st_other); + f.Printf("st_shndx = 0x%04" PRIx16 "\n", sym->st_shndx); + f.Printf("st_value = 0x%016" PRIx64 "\n", sym->st_value); + f.Printf("st_size = %" PRIu64 "\n", sym->st_size); + } + + f.Close(); +} + +static void DbgDumpRela(const std::string& folder, Elf64_Rela* records, uint64_t size, + const char* /*names*/, const char* file_name) { + auto folder_str = Common::FixDirectorySlash(folder); + + Common::File::CreateDirectories(folder_str); + + Common::File f; + f.Create(folder_str + file_name); + + for (auto* r = records; + reinterpret_cast(r) < reinterpret_cast(records) + size; r++) { + f.Printf("----\n" + "r_offset = 0x%016" PRIx64 "\n" + "r_info = 0x%016" PRIx64 "\n" + "r_addend = %" PRId64 "\n", + r->r_offset, r->r_info, r->r_addend); + } + + f.Close(); +} + +static Common::VirtualMemory::Mode GetMode(Elf64_Word flags) { + switch (flags) { + case PF_R: return Common::VirtualMemory::Mode::Read; + case PF_W: return Common::VirtualMemory::Mode::Write; + case PF_R | PF_W: return Common::VirtualMemory::Mode::ReadWrite; + case PF_X: return Common::VirtualMemory::Mode::Execute; + case PF_X | PF_R: return Common::VirtualMemory::Mode::ExecuteRead; + case PF_X | PF_W: return Common::VirtualMemory::Mode::ExecuteWrite; + case PF_X | PF_W | PF_R: return Common::VirtualMemory::Mode::ExecuteReadWrite; + + default: return Common::VirtualMemory::Mode::NoAccess; + } +} + +struct FrameS { + FrameS* next; + uintptr_t ret_addr; +}; + +static void KYTY_SYSV_ABI StackwalkX86(uint64_t rbp, void** stack, int* depth, uintptr_t stack_addr, + size_t stack_size, uintptr_t code_addr, size_t code_size) { + auto* frame = reinterpret_cast(rbp); + + int d = *depth; + int i = 0; + + for (; i < d; i++) { + if (!(reinterpret_cast(frame) >= stack_addr && + reinterpret_cast(frame) < stack_addr + stack_size)) { + break; + } + + if (!(frame->ret_addr >= code_addr && frame->ret_addr < code_addr + code_size)) { + break; + } + + stack[i] = reinterpret_cast(frame->ret_addr); + + frame = frame->next; + } + + *depth = i; +} + +static void KYTY_SYSV_ABI SysStackWalkX86(uint64_t rbp, uint64_t rsp, void** stack, int* depth) { + if (rsp == 0 || rbp < rsp) { + *depth = 0; + return; + } + + StackwalkX86(rbp, stack, depth, rsp, 1024u * 1024u, SYSTEM_RESERVED + CODE_BASE_OFFSET, + g_desired_base_addr - (SYSTEM_RESERVED + CODE_BASE_OFFSET)); +} + +void KYTY_SYSV_ABI SysStackWalkX86(uint64_t rbp, void** stack, int* depth) { + SysStackWalkX86(rbp, rbp, stack, depth); +} + +static bool KytyExceptionHandler(const Common::HostException::ExceptionInfo& exception_info) { + const auto* info = &exception_info; + + if (info->type == Common::HostException::ExceptionType::IllegalInstruction && + Loader::X64InstructionEmulator::TryEmulate(info->native_context)) { + return true; + } + + if (info->type == Common::HostException::ExceptionType::AccessViolation) { + using CoreAccess = Common::HostException::AccessViolationType; + using GpuAccess = Libs::Graphics::PageFaultAccess; + const auto access = [&]() { + switch (info->access_violation_type) { + case CoreAccess::Read: return GpuAccess::Read; + case CoreAccess::Write: return GpuAccess::Write; + case CoreAccess::Execute: return GpuAccess::Execute; + case CoreAccess::Unknown: + EXIT("unknown access type for page fault at 0x%016" PRIx64 "\n", + info->access_violation_vaddr); + } + EXIT("invalid access type for page fault at 0x%016" PRIx64 "\n", + info->access_violation_vaddr); + }(); + if (Libs::Graphics::g_render_ctx != nullptr && + Libs::Graphics::g_render_ctx->GetGpuResources()->HandleFault( + access, info->access_violation_vaddr)) { + return true; + } + + if (Libs::LibKernel::Memory::KernelHandleReservedRangeAccessViolation( + info->access_violation_vaddr)) { + return true; + } + } + + LOGF("kyty_exception_handler: %016" PRIx64 "\n", info->exception_address); +#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS + HMODULE owner_module = nullptr; + if (GetModuleHandleExA(GET_MODULE_HANDLE_EX_FLAG_FROM_ADDRESS | + GET_MODULE_HANDLE_EX_FLAG_UNCHANGED_REFCOUNT, + reinterpret_cast(info->exception_address), &owner_module) != 0 && + owner_module != nullptr) { + char module_name[MAX_PATH] = {}; + if (GetModuleFileNameA(owner_module, module_name, MAX_PATH) != 0) { + LOGF("exception module: %s\n", module_name); + } + } +#endif + if (info->exception_address != 0) { +#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS + MEMORY_BASIC_INFORMATION mem_info = {}; + auto* dump_ptr = reinterpret_cast(info->exception_address - 32); + if (VirtualQuery(dump_ptr, &mem_info, sizeof(mem_info)) != 0 && + mem_info.State == MEM_COMMIT && mem_info.Protect != PAGE_NOACCESS && + (mem_info.Protect & PAGE_GUARD) == 0) { + const auto dump_start = reinterpret_cast(dump_ptr); + const auto region_end = + reinterpret_cast(mem_info.BaseAddress) + mem_info.RegionSize; + const auto dump_size = + (dump_start + 64 <= region_end ? 64u + : static_cast(region_end - dump_start)); + LOGF("code-32:"); + for (uint32_t i = 0; i < dump_size; i++) { + LOGF(" %02" PRIx32, static_cast(dump_ptr[i])); + } + LOGF("\n"); + } else { + LOGF("code-32: unavailable\n"); + } +#else + LOGF("code-32:"); + for (uint64_t i = 0; i < 64; i++) { + LOGF(" %02" PRIx32, static_cast(*reinterpret_cast( + info->exception_address + i - 32))); + } + LOGF("\n"); +#endif + } else { + LOGF("code: unavailable\n"); + } + LOGF("exception: type=%s, av_type=%s, av_addr=%016" PRIx64 ", native_code=%08" PRIx32 "\n", + Common::EnumName(info->type).c_str(), + Common::EnumName(info->access_violation_type).c_str(), info->access_violation_vaddr, + info->native_code); + LOGF("regs: rax=%016" PRIx64 " rbx=%016" PRIx64 " rcx=%016" PRIx64 " rdx=%016" PRIx64 "\n", + info->rax, info->rbx, info->rcx, info->rdx); + LOGF("regs: rsi=%016" PRIx64 " rdi=%016" PRIx64 " rbp=%016" PRIx64 " rsp=%016" PRIx64 "\n", + info->rsi, info->rdi, info->rbp, info->rsp); + LOGF("regs: r8 =%016" PRIx64 " r9 =%016" PRIx64 " r10=%016" PRIx64 " r11=%016" PRIx64 "\n", + info->r8, info->r9, info->r10, info->r11); + LOGF("regs: r12=%016" PRIx64 " r13=%016" PRIx64 " r14=%016" PRIx64 " r15=%016" PRIx64 "\n", + info->r12, info->r13, info->r14, info->r15); + + auto is_readable_range = [](uint64_t addr, uint64_t size) { + if (addr == 0 || size == 0) { + return false; + } +#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS + uint64_t current = addr; + uint64_t end = addr + size; + if (end < addr) { + return false; + } + while (current < end) { + MEMORY_BASIC_INFORMATION mbi {}; + if (VirtualQuery(reinterpret_cast(current), &mbi, sizeof(mbi)) == 0 || + mbi.State != MEM_COMMIT || (mbi.Protect & (PAGE_NOACCESS | PAGE_GUARD)) != 0) { + return false; + } + const auto region_end = reinterpret_cast(mbi.BaseAddress) + mbi.RegionSize; + if (region_end <= current) { + return false; + } + current = std::min(region_end, end); + } +#else + (void)addr; + (void)size; +#endif + return true; + }; + + if (is_readable_range(info->rsp, 16u * sizeof(uint64_t))) { + auto* stack = reinterpret_cast(info->rsp); + LOGF("stack:"); + for (uint64_t i = 0; i < 16; i++) { + LOGF(" [%02" PRIu64 "]=%016" PRIx64, i, stack[i]); + } + LOGF("\n"); + } else { + LOGF("stack: unavailable\n"); + } + + auto dump_guest_code = [](const char* name, uint64_t addr) { + auto* p = Common::Singleton::Instance()->FindProgramByAddr(addr); + if (p == nullptr || addr < p->base_vaddr) { + return; + } + +#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS + MEMORY_BASIC_INFORMATION mbi {}; + auto* dump_ptr = reinterpret_cast(addr >= 16 ? addr - 16 : addr); + if (VirtualQuery(dump_ptr, &mbi, sizeof(mbi)) == 0 || mbi.State != MEM_COMMIT || + (mbi.Protect & (PAGE_NOACCESS | PAGE_GUARD)) != 0) { + return; + } + const auto dump_start = reinterpret_cast(dump_ptr); + const auto region_end = reinterpret_cast(mbi.BaseAddress) + mbi.RegionSize; + const auto dump_size = + (dump_start + 32 <= region_end ? 32u : static_cast(region_end - dump_start)); +#else + auto* dump_ptr = reinterpret_cast(addr >= 16 ? addr - 16 : addr); + const auto dump_size = 32u; +#endif + + LOGF("%s code: addr=%016" PRIx64 ", off=%016" PRIx64 ", module=%s:", name, addr, + addr - p->base_vaddr, + Common::FilenameWithoutDirectory(Common::PathToGenericString(p->file_name)).c_str()); + for (uint32_t i = 0; i < dump_size; i++) { + LOGF(" %02" PRIx32, static_cast(dump_ptr[i])); + } + LOGF("\n"); + }; + + dump_guest_code("guest rax[0]", info->rax); + dump_guest_code("guest rbx[0]", info->rbx); + dump_guest_code("guest rcx[0]", info->rcx); + dump_guest_code("guest rsi[0]", info->rsi); + if (info->rsp != 0) { + auto* stack = reinterpret_cast(info->rsp); + for (uint64_t i = 0; i < 16; i++) { + char name[32] {}; + std::snprintf(name, sizeof(name), "stack[%" PRIu64 "]", i); + dump_guest_code(name, stack[i]); + } + } + + if (info->type == Common::HostException::ExceptionType::AccessViolation) { + if (info->rbp != 0) { + void* stack[20]; + int depth = 20; + SysStackWalkX86(info->rbp, info->rsp, stack, &depth); + + LOGF("Stack trace [thread = %d]:\n", Common::Thread::GetThreadIdUnique()); + for (int i = 0; i < depth; i++) { + auto vaddr = reinterpret_cast(stack[i]); + auto* p = + Common::Singleton::Instance()->FindProgramByAddr(vaddr); + LOGF("[%d] %016" PRIx64 ", off=%016" PRIx64 ", %s\n", i, vaddr, + (p == nullptr ? 0 : vaddr - p->base_vaddr), + (p == nullptr ? "???" + : Common::FilenameWithoutDirectory( + Common::PathToGenericString(p->file_name)) + .c_str())); + } + } + + auto dump_guest_qwords = [&is_readable_range](const char* name, uint64_t addr) { + if (addr == 0) { + LOGF("%s = 0\n", name); + return; + } + +#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS + MEMORY_BASIC_INFORMATION mbi {}; + if (VirtualQuery(reinterpret_cast(addr), &mbi, sizeof(mbi)) == 0 || + mbi.State != MEM_COMMIT || (mbi.Protect & (PAGE_NOACCESS | PAGE_GUARD)) != 0) { + LOGF("%s = %016" PRIx64 " (unmapped)\n", name, addr); + return; + } +#endif + + if (!is_readable_range(addr, 8u * sizeof(uint64_t))) { + LOGF("%s = %016" PRIx64 " (unmapped)\n", name, addr); + return; + } + + auto* q = reinterpret_cast(addr); + LOGF("%s = %016" PRIx64 ": %016" PRIx64 " %016" PRIx64 " %016" PRIx64 " %016" PRIx64 + " %016" PRIx64 " %016" PRIx64 " %016" PRIx64 " %016" PRIx64 "\n", + name, addr, q[0], q[1], q[2], q[3], q[4], q[5], q[6], q[7]); + }; + + dump_guest_qwords("guest rbx", info->rbx); + dump_guest_qwords("guest rax", info->rax); + dump_guest_qwords("guest rcx", info->rcx); + dump_guest_qwords("guest rsi", info->rsi); + dump_guest_qwords("guest rdi", info->rdi); + dump_guest_qwords("guest r8 ", info->r8); + dump_guest_qwords("guest r9 ", info->r9); + dump_guest_qwords("guest r10", info->r10); + dump_guest_qwords("guest r11", info->r11); + dump_guest_qwords("guest r12", info->r12); + dump_guest_qwords("guest r13", info->r13); + dump_guest_qwords("guest r14", info->r14); + dump_guest_qwords("guest r15", info->r15); + + if (info->exception_address == 0x000000090064364e && info->rbx != 0) { + auto* local = reinterpret_cast(info->rbx); + dump_guest_qwords("vorbis obj", local[0]); + dump_guest_qwords("vorbis len", info->rcx); + } + + EXIT("Access violation: %s [%016" PRIx64 "] %s\n", + Common::EnumName(info->access_violation_type).c_str(), info->access_violation_vaddr, + (info->access_violation_vaddr == g_invalid_memory ? "(Unpatched object)" : "")); + return false; + } + + EXIT("Unknown exception!!! (%08" PRIx32 ")", info->native_code); + return false; +} + +static void EncodeId64(uint16_t in_id, std::string* out_id) { + static const char* str = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+-"; + if (in_id < 0x40u) { + *out_id += str[in_id]; + } else { + if (in_id < 0x1000u) { + *out_id += str[static_cast(in_id >> 6u) & 0x3fu]; + *out_id += str[in_id & 0x3fu]; + } else { + *out_id += str[static_cast(in_id >> 12u) & 0x3fu]; + *out_id += str[static_cast(in_id >> 6u) & 0x3fu]; + *out_id += str[in_id & 0x3fu]; + } + } +} + +template +static void GetDynDataOs(Elf64* elf, T* out, Elf64_Sxword tag) { + if (const auto* dyn = elf->GetDynValue(tag); dyn != nullptr) { + *out = elf->GetDynamicData(dyn->d_un.d_ptr); + } +} + +template +static void GetDynData(Elf64* elf, uint64_t base_vaddr, T* out, Elf64_Sxword tag) { + if (const auto* dyn = elf->GetDynValue(tag); dyn != nullptr) { + *out = reinterpret_cast(base_vaddr + dyn->d_un.d_ptr); + } +} + +template +static void GetDynValue(Elf64* elf, T* out, Elf64_Sxword tag) { + if (const auto* dyn = elf->GetDynValue(tag); dyn != nullptr) { + *out = dyn->d_un.d_val; + } +} + +template +static void GetDynValues(Elf64* elf, T* out, Elf64_Sxword tag) { + for (const auto* dyn: elf->GetDynList(tag)) { + out->push_back(dyn->d_un.d_val); + } +} + +template +static void GetDynPtr(Elf64* elf, T* out, Elf64_Sxword tag) { + if (const auto* dyn = elf->GetDynValue(tag); dyn != nullptr) { + *out = dyn->d_un.d_ptr; + } +} + +static void KYTY_SYSV_ABI ProgramExitHandler() { + Common::Singleton::Instance()->StopAllModules(); + + LOGF("exit!!!\n"); +} + +template +static void GetDynModules(Elf64* elf, T* out, const char* names, Elf64_Sxword tag) { + std::vector needed_modules; + GetDynValues(elf, &needed_modules, tag); + for (auto need: needed_modules) { + ModuleId id {}; + // id.id = static_cast((need >> 48u) & 0xffffu); + EncodeId64(static_cast((need >> 48u) & 0xffffu), &id.id); + id.version_major = static_cast((need >> 40u) & 0xffu); + id.version_minor = static_cast((need >> 32u) & 0xffu); + id.name = names + (need & 0xffffffff); + out->push_back(id); + } +} + +template +static void GetDynLibs(Elf64* elf, T* out, const char* names, Elf64_Sxword tag) { + std::vector needed_modules; + GetDynValues(elf, &needed_modules, tag); + for (auto need: needed_modules) { + LibraryId id {}; + // id.id = static_cast((need >> 48u) & 0xffffu); + EncodeId64(static_cast((need >> 48u) & 0xffffu), &id.id); + id.version = static_cast((need >> 32u) & 0xffffu); + id.name = names + (need & 0xffffffff); + out->push_back(id); + } +} + +static RelocationInfo GetRelocationInfo(Elf64_Rela* r, Program* program) { + KYTY_PROFILER_FUNCTION(); + + // KYTY_PROFILER_BLOCK("1"); + + RelocationInfo ret; + // SymbolRecord sr {}; + + // KYTY_PROFILER_END_BLOCK; + + // KYTY_PROFILER_BLOCK("2"); + + auto type = r->GetType(); + auto symbol = r->GetSymbol(); + Elf64_Sxword addend = r->r_addend; + auto* symbols = program->dynamic_info->symbol_table; + auto* names = program->dynamic_info->str_table; + ret.base_vaddr = program->base_vaddr; + ret.vaddr = ret.base_vaddr + r->r_offset; + ret.bind_self = false; + + // KYTY_PROFILER_END_BLOCK; + + // KYTY_PROFILER_BLOCK("3"); + + switch (type) { + case R_X86_64_GLOB_DAT: + case R_X86_64_JUMP_SLOT: addend = 0; [[fallthrough]]; + case R_X86_64_64: { + auto sym = symbols[symbol]; + auto bind = sym.GetBind(); + auto sym_type = sym.GetType(); + uint64_t symbol_vaddr = 0; + SymbolRecord sr {}; + switch (sym_type) { + case STT_NOTYPE: ret.type = SymbolType::NoType; break; + case STT_FUNC: ret.type = SymbolType::Func; break; + case STT_OBJECT: ret.type = SymbolType::Object; break; + default: EXIT("unknown symbol type: %d\n", (int)sym_type); + } + switch (bind) { + case STB_LOCAL: + symbol_vaddr = ret.base_vaddr + sym.st_value; + ret.bind = BindType::Local; + break; + case STB_GLOBAL: ret.bind = BindType::Global; [[fallthrough]]; + case STB_WEAK: { + ret.bind = (ret.bind == BindType::Unknown ? BindType::Weak : ret.bind); + ret.name = names + sym.st_name; + program->rt->Resolve(ret.name, ret.type, program, &sr, &ret.bind_self); + symbol_vaddr = sr.vaddr; + } break; + default: EXIT("unknown bind: %d\n", (int)bind); + } + ret.resolved = (symbol_vaddr != 0); + ret.value = (ret.resolved ? symbol_vaddr + addend : 0); + ret.name = sr.name; + ret.dbg_name = sr.dbg_name; + } break; + case R_X86_64_RELATIVE: + ret.value = ret.base_vaddr + addend; + ret.resolved = true; + break; + case R_X86_64_DTPMOD64: + ret.value = reinterpret_cast(program); + ret.resolved = true; + ret.type = SymbolType::TlsModule; + ret.bind = BindType::Local; + ret.dbg_name = Common::PathToString(program->file_name); + break; + default: EXIT("unknown type: %d\n", (int)type); + } + + // KYTY_PROFILER_END_BLOCK; + + return ret; +} + +static void RelocateRecord(uint32_t index, Elf64_Rela* r, Program* program, bool jmprela_table, + bool imports_only, std::vector* unresolved) { + KYTY_PROFILER_FUNCTION(); + + auto ri = GetRelocationInfo(r, program); + + if (imports_only && + (ri.bind_self || (ri.bind != BindType::Global && ri.bind != BindType::Weak))) { + return; + } + + [[maybe_unused]] bool patched = false; + bool stubbed_import = false; + bool stubbed_func = false; + + // KYTY_PROFILER_BLOCK("patch"); + + if (ri.resolved) { + patched = Common::VirtualMemory::PatchReplace(ri.vaddr, ri.value); + } else { + uint64_t value = 0; + bool weak = (ri.bind == BindType::Weak || !program->fail_if_global_not_resolved); + if (ri.type == SymbolType::Object && weak) { + value = g_invalid_memory; + } else if (ri.type == SymbolType::Func && jmprela_table && weak) { + value = RegisterStubbedImport(index, program, ri); + stubbed_import = true; + stubbed_func = true; + } else if (ri.type == SymbolType::Func && !jmprela_table && weak) { + value = RegisterStubbedImport(index, program, ri); + stubbed_func = true; + } else if (ri.type == SymbolType::NoType && weak) { + value = RuntimeLinker::ReadFromElf(program, ri.vaddr) + ri.base_vaddr; + } + + if (value != 0) { + patched = Common::VirtualMemory::PatchReplace(ri.vaddr, value); + } else { + auto dbg_str = fmt::format("[{:016x}] <- {:016x}, {}, {}, {}, {}", ri.vaddr, ri.value, + ri.name.c_str(), Common::EnumName(ri.type).c_str(), + Common::EnumName(ri.bind).c_str(), ri.dbg_name.c_str()); + + if (unresolved != nullptr) { + unresolved->push_back(dbg_str); + } else { + EXIT("Can't resolve: %s\n", dbg_str.c_str()); + } + + if (ri.type == SymbolType::Object) { + value = g_invalid_memory; + } else if (ri.type == SymbolType::Func || ri.type == SymbolType::NoType) { + value = RegisterStubbedImport(index, program, ri); + stubbed_func = true; + if (jmprela_table) { + stubbed_import = true; + } + } + + if (value != 0) { + patched = Common::VirtualMemory::PatchReplace(ri.vaddr, value); + } + } + } + + // KYTY_PROFILER_END_BLOCK; + + if (patched && stubbed_import) { + const auto thunk = RegisterStubbedImport(index, program, ri); + LOGF("Relocate: unresolved PLT import patched to stub [%u] [%016" PRIx64 "] <- %016" PRIx64 + ", %s, %s, %s, %s\n", + index, ri.vaddr, thunk, ri.name.c_str(), Common::EnumName(ri.type).c_str(), + Common::EnumName(ri.bind).c_str(), Common::PathToString(program->file_name).c_str()); + } else if (patched && stubbed_func) { + const auto thunk = RegisterStubbedImport(index, program, ri); + LOGF("Relocate: unresolved non-PLT function patched to stub [%u] [%016" PRIx64 + "] <- %016" PRIx64 ", %s, %s, %s, %s\n", + index, ri.vaddr, thunk, ri.name.c_str(), Common::EnumName(ri.type).c_str(), + Common::EnumName(ri.bind).c_str(), Common::PathToString(program->file_name).c_str()); + } + + if (program->dbg_print_reloc) { + if (/* !dbg_str.ContainsStr("libc_") && */ patched && !ri.bind_self && + (ri.bind == BindType::Global || ri.bind == BindType::Weak || + ri.type == SymbolType::TlsModule)) { + auto dbg_str = fmt::format("[{:016x}] <- {:016x}, {}, {}, {}, {}", ri.vaddr, ri.value, + ri.name.c_str(), Common::EnumName(ri.type).c_str(), + Common::EnumName(ri.bind).c_str(), ri.dbg_name.c_str()); + + LOGF("Relocate: %s\n", dbg_str.c_str()); + } + } +} + +static void RelocateRecords(Elf64_Rela* records, uint64_t size, Program* program, + bool jmprela_table, bool imports_only, + std::vector* unresolved) { + KYTY_PROFILER_FUNCTION(); + + uint32_t index = 0; + for (auto* r = records; + reinterpret_cast(r) < reinterpret_cast(records) + size; r++, index++) { + RelocateRecord(index, r, program, jmprela_table, imports_only, unresolved); + } +} + +__attribute__((naked)) static KYTY_SYSV_ABI void RelocateHandlerReturnStub() { + asm volatile("addq $8, %rsp\n\t" + "retq\n"); +} + +static KYTY_SYSV_ABI uint64_t RelocateHandler(RelocateHandlerStack s) { + auto* stack = s.stack; + auto* program = reinterpret_cast(stack[-1]); + auto rel_index = stack[0]; + std::string name = ""; + + if (program != nullptr && program->dynamic_info != nullptr && + program->dynamic_info->jmprela_table != nullptr) { + auto ri = GetRelocationInfo(program->dynamic_info->jmprela_table + rel_index, program); + + name = ri.name.c_str(); + } + + // Restore return address (for stack trace) + stack[-1] = reinterpret_cast(RelocateHandlerReturnStub); + + LOGF("=== Stubbed function, returning OK ===\n[%d]\t%s\n", Common::Thread::GetThreadIdUnique(), + name.c_str()); + return 0; +} + +static KYTY_MS_ABI uint8_t* TlsMainGetAddr() { + EXIT_IF(g_tls_main_program == nullptr); + + if (g_tls_cached_main_program == g_tls_main_program && g_tls_cached_main_tcb != nullptr) { + return g_tls_cached_main_tcb; + } + + g_tls_cached_main_program = g_tls_main_program; + g_tls_cached_main_tcb = + RuntimeLinker::TlsGetAddr(g_tls_main_program) + g_tls_main_program->tls.tcb_offset; + return g_tls_cached_main_tcb; +} + +static void PatchProgram(Program* program, uint64_t address, uint64_t size) { + EXIT_IF(program == nullptr); + EXIT_IF(program->elf == nullptr); + + if (size >= 12) { + // Replace guest stack-canary/errno stores through fs:[0x28] with nops. + // Windows x64 cannot host guest FS directly, and an unpatched shared-library access faults + // at address 0x28. + const uint8_t fs_store_pattern[8] = {0x64, 0xc7, 0x04, 0x25, 0x28, 0x00, 0x00, 0x00}; + auto* start_ptr = reinterpret_cast(address); + auto* end_ptr = start_ptr + size - 12; + + for (auto* ptr = start_ptr; ptr <= end_ptr; ptr++) { + if (memcmp(ptr, fs_store_pattern, sizeof(fs_store_pattern)) == 0) { + LOGF("Patch fs:[0x28] store at addr: [%016" PRIx64 "]\n", + reinterpret_cast(ptr)); + if (ptr + 16 < start_ptr + size && ptr[12] == 0xcd && ptr[13] == 0x45 && + ptr[14] == 0x90 && ptr[15] == 0x0f && ptr[16] == 0x0b) { + ptr[0] = 0x5d; // pop rbp + ptr[1] = 0xc3; // ret + std::memset(ptr + 2, 0x90, 15); + } else { + std::memset(ptr, 0x90, 12); + } + } + } + } + + if (!program->elf->IsShared() && program->tls.handler_vaddr != 0) { + // Replace: + // 66 66 66 + // mov , qword ptr fs:[0x00] + // with: + // call + // mov ,rax + // nop ... + const uint8_t tls_pattern[5] = {0x64, 0x48, 0x8B, 0x00, 0x25}; + + EXIT_IF(Jit::Call9::GetSize() != 9); + + auto* start_ptr = reinterpret_cast(address); + auto* end_ptr = start_ptr + size - Jit::Call9::GetSize(); + + for (auto* ptr = start_ptr; ptr <= end_ptr; ptr++) { + auto* inst_ptr = ptr; + size_t prefix_count = 0; + while (prefix_count < 3 && inst_ptr < start_ptr + size && *inst_ptr == 0x66) { + inst_ptr++; + prefix_count++; + } + + const size_t inst_size = prefix_count + Jit::Call9::GetSize(); + if (inst_ptr + Jit::Call9::GetSize() > start_ptr + size) { + break; + } + + const uint8_t modrm = inst_ptr[3]; + if (memcmp(inst_ptr, tls_pattern, 3) == 0 && (modrm & 0xc7u) == 0x04u && + inst_ptr[4] == tls_pattern[4] && + *reinterpret_cast(inst_ptr + 5) == 0) { + LOGF("Patch tls at addr: [%016" PRIx64 "]\n", reinterpret_cast(ptr)); + + const auto reg = (modrm >> 3u) & 7u; + EXIT_NOT_IMPLEMENTED(reg == 4u); + + auto* code = new (ptr) Jit::Call9; + code->SetFunc(reg == 0 + ? program->tls.handler_vaddr + : program->tls.handler_vaddr + Jit::TlsRegStub::GetOffset(reg)); + if (inst_size > Jit::Call9::GetSize()) { + std::memset(ptr + Jit::Call9::GetSize(), 0x90, + inst_size - Jit::Call9::GetSize()); + } + ptr += inst_size - 1; + } + } + } +} + +uint64_t RuntimeLinker::GetEntry() { + // EXIT_NOT_IMPLEMENTED(!Common::Thread::IsMainThread()); + + Common::LockGuard lock(m_mutex); + + for (const auto* p: m_programs) { + if (p->elf != nullptr && !p->elf->IsShared()) { + return p->elf->GetEntry() + p->base_vaddr; + } + } + return 0; +} + +uint64_t RuntimeLinker::GetProcParam() { + // EXIT_NOT_IMPLEMENTED(!Common::Thread::IsMainThread()); + + Common::LockGuard lock(m_mutex); + + for (const auto* p: m_programs) { + if (p->elf != nullptr && !p->elf->IsShared()) { + return p->proc_param_vaddr; + } + } + return 0; +} + +void RuntimeLinker::DbgDump(const std::string& folder) { + KYTY_PROFILER_FUNCTION(); + + EXIT_NOT_IMPLEMENTED(!Common::Thread::IsMainThread()); + + Common::LockGuard lock(m_mutex); + + for (const auto* p: m_programs) { + auto folder_str = Common::FixDirectorySlash(folder); + folder_str += Common::FilenameWithoutDirectory(Common::PathToGenericString(p->file_name)); + + EXIT_IF(p->elf == nullptr); + + p->elf->DbgDump(folder_str); + + if (p->dynamic_info != nullptr) { + EXIT_NOT_IMPLEMENTED(p->dynamic_info->symbol_table_entry_size != 0 && + p->dynamic_info->symbol_table_entry_size != sizeof(Elf64_Sym)); + EXIT_NOT_IMPLEMENTED(p->dynamic_info->rela_table_entry_size != 0 && + p->dynamic_info->rela_table_entry_size != sizeof(Elf64_Rela)); + // EXIT_NOT_IMPLEMENTED(p->dynamic_info->jmprela_table == nullptr); + // EXIT_NOT_IMPLEMENTED(p->dynamic_info->rela_table == nullptr); + // EXIT_NOT_IMPLEMENTED(p->dynamic_info->symbol_table == nullptr); + + if (p->dynamic_info->symbol_table != nullptr) { + DbgDumpSymbols(folder_str, p->dynamic_info->symbol_table, + p->dynamic_info->symbol_table_total_size, + p->dynamic_info->str_table); + } + if (p->dynamic_info->jmprela_table != nullptr) { + DbgDumpRela(folder_str, p->dynamic_info->jmprela_table, + p->dynamic_info->jmprela_table_size, p->dynamic_info->str_table, + "jmprela_table.txt"); + } + if (p->dynamic_info->rela_table != nullptr) { + DbgDumpRela(folder_str, p->dynamic_info->rela_table, + p->dynamic_info->rela_table_total_size, p->dynamic_info->str_table, + "rela_table.txt"); + } + } + + if (p->export_symbols != nullptr) { + p->export_symbols->DbgDump(folder_str, "export_symbols.txt"); + } + if (p->import_symbols != nullptr) { + p->import_symbols->DbgDump(folder_str, "import_symbols.txt"); + } + } +} + +void RuntimeLinker::RelocateAll() { + // EXIT_NOT_IMPLEMENTED(!Common::Thread::IsMainThread()); + + Common::LockGuard lock(m_mutex); + + for (auto* p: m_programs) { + Relocate(p); + } + + m_relocated = true; +} + +void RuntimeLinker::RelocateProgram(Program* program) { + Common::LockGuard lock(m_mutex); + + EXIT_IF(program == nullptr); + EXIT_IF(std::find(m_programs.begin(), m_programs.end(), program) == m_programs.end()); + + Relocate(program); +} + +void RuntimeLinker::UnloadProgram(Program* program) { + // EXIT_NOT_IMPLEMENTED(!Common::Thread::IsMainThread()); + + Common::LockGuard lock(m_mutex); + + if (auto it = std::find(m_programs.begin(), m_programs.end(), program); + it != m_programs.end()) { + DeleteProgram(*it); + m_programs.erase(it); + } else { + EXIT("program not found"); + } + + if (m_relocated) { + RelocateAll(); + } +} + +RuntimeLinker::RuntimeLinker(): m_symbols(std::make_unique()) { + EXIT_NOT_IMPLEMENTED(!Common::Thread::IsMainThread()); +} + +RuntimeLinker::~RuntimeLinker() { + Clear(); +} + +Program* RuntimeLinker::LoadProgram(const std::filesystem::path& elf_name) { + KYTY_PROFILER_FUNCTION(); + + Common::LockGuard lock(m_mutex); + + static int32_t id_seq = 0; + + LOGF("Loading: %s\n", Common::PathToString(elf_name).c_str()); + + auto program_owner = std::make_unique(); + auto* program = program_owner.get(); + + program->rt = this; + program->file_name = elf_name; + program->unique_id = ++id_seq; + + program->elf = std::make_unique(); + program->elf->Open(elf_name); + + if (program->elf->IsValid()) { + LoadProgramToMemory(program); + ParseProgramDynamicInfo(program); + CreateSymbolDatabase(program); + } else { + EXIT("elf is not valid: %s\n", Common::PathToString(elf_name).c_str()); + } + + m_programs.push_back(program_owner.release()); + + if (!program->elf->IsShared()) { + program->fail_if_global_not_resolved = false; + Libs::LibKernel::SetProgName(elf_name.filename().string()); + } + + if (Common::EndsWith(Common::ToLower(Common::DirectoryWithoutFilename( + Common::PathToGenericString(elf_name))), + "_module/")) { + program->fail_if_global_not_resolved = false; + } + + return program; +} + +void RuntimeLinker::SaveMainProgram(const std::filesystem::path& elf_name) { + EXIT_NOT_IMPLEMENTED(!Common::Thread::IsMainThread()); + + Common::LockGuard lock(m_mutex); + + for (const auto* p: m_programs) { + EXIT_IF(p->elf == nullptr); + + if (!p->elf->IsShared()) { + p->elf->Save(elf_name); + break; + } + } +} + +void RuntimeLinker::SaveProgram(Program* program, const std::filesystem::path& elf_name) { + EXIT_NOT_IMPLEMENTED(!Common::Thread::IsMainThread()); + + Common::LockGuard lock(m_mutex); + + if (auto it = std::find(m_programs.begin(), m_programs.end(), program); + it != m_programs.end()) { + EXIT_IF((*it)->elf == nullptr); + + (*it)->elf->Save(elf_name); + } else { + EXIT("program not found"); + } +} + +void RuntimeLinker::Execute() { + KYTY_PROFILER_THREAD("Thread_Main"); + + Libs::LibKernel::PthreadInitSelfForMainThread(); + auto* main_stack_top = Libs::LibKernel::PthreadCreateMainGuestStack(); + +#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS + // Guest code has no Windows stack probes and may jump over the guard page. Module + // initializers execute on the host stack too, so grow it before calling any guest code. + size_t expanded_size = 0; + while (expanded_size < static_cast(768) * 1024) { + sys_dbg_stack_info_t stack {}; + SysStackUsage(stack); + *reinterpret_cast(stack.guard_addr) = 0; + expanded_size += stack.guard_size; + } +#endif + + PreloadAdjacentPrograms(); + RelocateAll(); + StartAllModules(); + + LOGF_COLOR(Log::Color::BrightYellow, "---\n--- Execute: %s\n---\n", "Main"); + + if (auto entry = GetEntry(); entry != 0) { + auto* params = reinterpret_cast( + (reinterpret_cast(main_stack_top) - 0x100u) & ~static_cast(0x0f)); + std::memset(params, 0, sizeof(EntryParams)); + params->argc = 1; + params->argv[0] = "KytyEmu"; + + LOGF("stack_addr = %" PRIx64 "\n", reinterpret_cast(params)); + + RunEntry(entry, params, ProgramExitHandler, + reinterpret_cast(reinterpret_cast(params) - 0x1000u)); + } +} + +void RuntimeLinker::Clear() { + // EXIT_NOT_IMPLEMENTED(!Common::Thread::IsMainThread()); + + Common::LockGuard lock(m_mutex); + + for (auto* p: m_programs) { + DeleteProgram(p); + } + m_programs.clear(); + m_symbols.reset(); + m_relocated = false; +} + +void RuntimeLinker::Resolve(const std::string& name, SymbolType type, Program* program, + SymbolRecord* out_info, bool* bind_self) { + KYTY_PROFILER_FUNCTION(); + + Common::LockGuard lock(m_mutex); + + EXIT_IF(out_info == nullptr); + + auto ids = Common::Split(name, '#'); + + if (bind_self != nullptr) { + *bind_self = false; + } + + if (ids.size() == 3) { + const LibraryId* l = FindLibrary(*program, ids.at(1)); + const ModuleId* m = FindModule(*program, ids.at(2)); + + auto resolve_by_nid = [this, type](const std::string& nid, SymbolRecord* out) -> bool { + EXIT_IF(out == nullptr); + + if (m_symbols != nullptr) { + if (const auto* rec = m_symbols->FindByNid(nid, type); rec != nullptr) { + *out = *rec; + return true; + } + } + + for (auto* p: m_programs) { + if (p != nullptr && p->export_symbols != nullptr) { + if (const auto* rec = p->export_symbols->FindByNid(nid, type); rec != nullptr) { + *out = *rec; + return true; + } + } + } + + return false; + }; + + if (l != nullptr && m != nullptr) { + SymbolResolve sr {}; + sr.name = ids.at(0); + sr.library = l->name; + sr.library_version = l->version; + sr.module = m->name; + sr.module_version_major = m->version_major; + sr.module_version_minor = m->version_minor; + sr.type = type; + + const SymbolRecord* rec = nullptr; + + if (m_symbols != nullptr) { + rec = m_symbols->Find(sr); + } + + if (rec == nullptr) { + if (auto* p = FindProgram(*m, *l); p != nullptr && p->export_symbols != nullptr) { + rec = p->export_symbols->Find(sr); + if (bind_self != nullptr) { + *bind_self = (p == program); + } + } + } + + if (rec == nullptr) { + if (resolve_by_nid(sr.name, out_info)) { + LOGF("PS5 NID fallback: %s -> %s\n", sr.name.c_str(), out_info->name.c_str()); + return; + } + } + + if (rec != nullptr) { + //*out_vaddr = rec->vaddr; + *out_info = *rec; + } else { + out_info->vaddr = 0; + out_info->name = SymbolDatabase::GenerateName(sr); + out_info->dbg_name = ""; + } + } else { + if (resolve_by_nid(ids.at(0), out_info)) { + LOGF("PS5 NID fallback: %s -> %s (missing lib/module metadata)\n", + ids.at(0).c_str(), out_info->name.c_str()); + return; + } + + EXIT("l == nullptr || m == nullptr"); + } + } else { + out_info->vaddr = 0; + out_info->name = name; + out_info->dbg_name = ""; + } +} + +bool RuntimeLinker::ResolveLoadedSymbolByNid(const std::string& nid, SymbolType type, + SymbolRecord* out_info) { + KYTY_PROFILER_FUNCTION(); + + Common::LockGuard lock(m_mutex); + + EXIT_IF(out_info == nullptr); + + for (auto* p: m_programs) { + if (p != nullptr && p->export_symbols != nullptr) { + if (const auto* rec = p->export_symbols->FindByNid(nid, type); rec != nullptr) { + *out_info = *rec; + return true; + } + } + } + + if (m_symbols != nullptr) { + if (const auto* rec = m_symbols->FindByNid(nid, type); rec != nullptr) { + *out_info = *rec; + return true; + } + } + + return false; +} + +uint64_t RuntimeLinker::ReadFromElf(Program* program, uint64_t vaddr) { + EXIT_IF(program == nullptr); + EXIT_IF(program->base_vaddr == 0 || program->base_size == 0); + EXIT_IF(program->elf == nullptr); + + uint64_t ret = 0; + + const auto* ehdr = program->elf->GetEhdr(); + const auto* phdr = program->elf->GetPhdr(); + + EXIT_IF(phdr == nullptr || ehdr == nullptr); + + for (Elf64_Half i = 0; i < ehdr->e_phnum; i++) { + if (phdr[i].p_memsz != 0 && (phdr[i].p_type == PT_LOAD || phdr[i].p_type == PT_OS_RELRO)) { + uint64_t segment_addr = phdr[i].p_vaddr + program->base_vaddr; + uint64_t segment_file_size = phdr[i].p_filesz; + + if (vaddr >= segment_addr && vaddr < segment_addr + segment_file_size) { + program->elf->LoadSegment(reinterpret_cast(&ret), + phdr[i].p_offset + vaddr - segment_addr, sizeof(ret)); + break; + } + } + } + + return ret; +} + +Program* RuntimeLinker::FindProgramById(int32_t id) { + Common::LockGuard lock(m_mutex); + + // Id 0 is reserved for main program + if (id == 0 && !m_programs.empty()) { + return m_programs.front(); + } + + for (auto* p: m_programs) { + if (p->unique_id == id) { + return p; + } + } + + return nullptr; +} + +Program* RuntimeLinker::FindProgramByFileName(const std::filesystem::path& elf_name) { + Common::LockGuard lock(m_mutex); + + auto fixed_name = Common::FixFilenameSlash(Common::PathToGenericString(elf_name)); + for (auto* p: m_programs) { + if (Common::EqualNoCase(Common::FixFilenameSlash(Common::PathToGenericString(p->file_name)), + fixed_name)) { + return p; + } + } + + return nullptr; +} + +Program* RuntimeLinker::FindProgramByAddr(uint64_t vaddr) { + Common::LockGuard lock(m_mutex); + + for (auto* p: m_programs) { + const auto* ehdr = p->elf->GetEhdr(); + const auto* phdr = p->elf->GetPhdr(); + + EXIT_IF(phdr == nullptr || ehdr == nullptr); + + for (Elf64_Half i = 0; i < ehdr->e_phnum; i++) { + if (phdr[i].p_memsz != 0 && + (phdr[i].p_type == PT_LOAD || phdr[i].p_type == PT_OS_RELRO)) { + uint64_t segment_addr = phdr[i].p_vaddr + p->base_vaddr; + uint64_t segment_size = GetAlignedSize(phdr + i); + + if (vaddr >= segment_addr && vaddr < segment_addr + segment_size) { + return p; + } + } + } + } + + return nullptr; +} + +void RuntimeLinker::StackTrace(uint64_t frame_ptr) { + void* stack[20]; + int depth = 20; + + SysStackWalkX86(frame_ptr, stack, &depth); + + LOGF("Stack trace [thread = %d]:\n", Common::Thread::GetThreadIdUnique()); + + for (int i = 0; i < depth; i++) { + auto vaddr = reinterpret_cast(stack[i]); + auto* p = FindProgramByAddr(vaddr); + LOGF("[%d] %016" PRIx64 ", off=%016" PRIx64 ", %s\n", i, vaddr, + (p == nullptr ? 0 : vaddr - p->base_vaddr), + (p == nullptr + ? "???" + : Common::FilenameWithoutDirectory(Common::PathToGenericString(p->file_name)) + .c_str())); + } +} + +static std::string GetProgramModuleName(const Program* program) { + EXIT_IF(program == nullptr); + + if (program->dynamic_info != nullptr && program->dynamic_info->so_name != nullptr && + program->dynamic_info->so_name[0] != '\0') { + return std::string(program->dynamic_info->so_name); + } + + return Common::FilenameWithoutDirectory(Common::PathToGenericString(program->file_name)); +} + +static bool ModuleStartDependenciesSatisfied(const Program* program, + const std::vector& programs, + const std::vector& started) { + EXIT_IF(program == nullptr); + EXIT_IF(program->dynamic_info == nullptr); + + for (const auto* needed: program->dynamic_info->needed) { + if (needed == nullptr || needed[0] == '\0') { + continue; + } + + const auto needed_name = std::string(needed); + + for (auto* dependency: programs) { + if (dependency == nullptr || dependency == program || dependency->elf == nullptr || + !dependency->elf->IsShared()) { + continue; + } + + const auto dependency_name = GetProgramModuleName(dependency); + if (Common::EqualNoCase(dependency_name, needed_name) || + Common::EqualNoCase(Common::FilenameWithoutDirectory( + Common::PathToGenericString(dependency->file_name)), + needed_name)) { + if (std::find(started.begin(), started.end(), dependency) == started.end()) { + return false; + } + break; + } + } + } + + return true; +} + +void RuntimeLinker::StartAllModules() { + Common::LockGuard lock(m_mutex); + + std::vector started; + + for (;;) { + bool progressed = false; + + for (auto* p: m_programs) { + if (p->elf->IsShared() && p->dynamic_info->init_vaddr != 0 && + std::find(started.begin(), started.end(), p) == started.end() && + ModuleStartDependenciesSatisfied(p, m_programs, started)) { + StartModule(p, 0, nullptr, nullptr); + started.push_back(p); + progressed = true; + } + } + + if (!progressed) { + break; + } + } + + for (auto* p: m_programs) { + if (p->elf->IsShared() && p->dynamic_info->init_vaddr != 0 && + std::find(started.begin(), started.end(), p) == started.end()) { + StartModule(p, 0, nullptr, nullptr); + started.push_back(p); + } + } +} + +void RuntimeLinker::StopAllModules() { + Common::LockGuard lock(m_mutex); + + for (auto* p: m_programs) { + if (p->elf->IsShared() && p->dynamic_info->fini_vaddr != 0) { + StopModule(p, 0, nullptr, nullptr); + } + } +} + +static bool IsAdjacentModuleFile(const std::string& name) { + auto lower = Common::ToLower(name); + return Common::EndsWith(lower, ".prx") || Common::EndsWith(lower, ".sprx"); +} + +static bool SkipAdjacentModuleFile(const std::string& name) { + auto lower = Common::ToLower(name); + return lower == "eboot.bin" || lower == "libkernel.prx" || lower == "libkernel_sys.prx"; +} + +void RuntimeLinker::PreloadAdjacentPrograms() { + if (m_programs.empty()) { + return; + } + + std::vector module_paths; + + auto is_loaded = [this](const std::filesystem::path& path) { + auto fixed_path = Common::FixFilenameSlash(Common::PathToGenericString(path)); + for (auto* program: m_programs) { + if (Common::EqualNoCase( + Common::FixFilenameSlash(Common::PathToGenericString(program->file_name)), + fixed_path)) { + return true; + } + } + return false; + }; + + auto add_path = [&module_paths, &is_loaded](const std::filesystem::path& path) { + if (is_loaded(path)) { + return; + } + for (const auto& p: module_paths) { + if (Common::EqualNoCase(Common::PathToGenericString(p), + Common::PathToGenericString(path))) { + return; + } + } + module_paths.push_back(path); + }; + + auto add_dir = [&add_path](const std::filesystem::path& dir) { + if (!Common::File::IsDirectoryExisting(dir)) { + return; + } + for (const auto& entry: Common::File::GetDirEntries(dir)) { + if (entry.is_file && IsAdjacentModuleFile(entry.name) && + !SkipAdjacentModuleFile(entry.name)) { + add_path(dir / entry.name); + } + } + }; + + auto root = m_programs.at(0)->file_name.parent_path(); + if (root.empty()) { + return; + } + + add_dir(root); + add_dir(root / "sce_module"); + add_dir(root / "sce_modules"); + + for (const auto& path: module_paths) { + auto* program = LoadProgram(path); + program->fail_if_global_not_resolved = false; + } +} + +int RuntimeLinker::StartModule(Program* program, size_t args, const void* argp, + module_func_t func) { + EXIT_IF(program == nullptr); + EXIT_IF(program->dynamic_info == nullptr); + EXIT_IF(program->elf == nullptr); + EXIT_IF(!program->elf->IsShared()); + + EXIT_IF(std::find(m_programs.begin(), m_programs.end(), program) == m_programs.end()); + + LOGF_COLOR(Log::Color::BrightYellow, "---\n--- Start module: %s\n---\n", + Common::PathToString(program->file_name).c_str()); + + return reinterpret_cast(program->dynamic_info->init_vaddr + + program->base_vaddr)(args, argp, func); +} + +int RuntimeLinker::StopModule(Program* program, size_t args, const void* argp, module_func_t func) { + EXIT_IF(program == nullptr); + EXIT_IF(program->dynamic_info == nullptr); + EXIT_IF(program->elf == nullptr); + EXIT_IF(!program->elf->IsShared()); + + EXIT_IF(std::find(m_programs.begin(), m_programs.end(), program) == m_programs.end()); + + LOGF_COLOR(Log::Color::BrightYellow, "---\n--- Stop module: %s\n---\n", + Common::PathToString(program->file_name).c_str()); + + int result = reinterpret_cast(program->dynamic_info->fini_vaddr + + program->base_vaddr)(args, argp, func); + + Libs::LibKernel::PthreadDeleteStaticObjects(program); + + return result; +} + +uint8_t* RuntimeLinker::TlsGetAddr(Program* program) { + EXIT_IF(program == nullptr); + + Common::LockGuard lock(program->tls.mutex); + + auto& tls = program->tls.tlss[Common::Thread::GetThreadIdUnique()]; + + if (tls.ptr == nullptr) { + constexpr uint64_t TCB_SIZE = 0x40; + constexpr uint64_t TCB_ALIGN = 0x20; + + const auto tcb_offset = + program->tls.tcb_offset != 0 ? program->tls.tcb_offset : program->tls.image_size; + const auto alloc_size = AlignUp(tcb_offset, TCB_ALIGN) + TCB_SIZE; + tls.ptr = reinterpret_cast( + Common::VirtualMemory::Alloc(0, alloc_size, Common::VirtualMemory::Mode::ReadWrite)); + tls.free_func = nullptr; + tls.vm_alloc = true; + + EXIT_IF(tls.ptr == nullptr); + + std::memset(tls.ptr, 0, alloc_size); + + if (!program->tls.init_image.empty()) { + std::memcpy(tls.ptr, program->tls.init_image.data(), program->tls.init_image.size()); + } else { + std::memcpy(tls.ptr, reinterpret_cast(program->tls.image_vaddr), + program->tls.init_size); + } + + auto* tcb = reinterpret_cast(tls.ptr + tcb_offset); + tcb[0] = reinterpret_cast(tcb); + } + + return tls.ptr; +} + +void RuntimeLinker::DeleteTls(Program* program, int thread_id) { + EXIT_IF(program == nullptr); + + if (thread_id == Common::Thread::GetThreadIdUnique() && g_tls_cached_main_program == program) { + g_tls_cached_main_program = nullptr; + g_tls_cached_main_tcb = nullptr; + } + + Common::LockGuard lock(program->tls.mutex); + + if (auto it = program->tls.tlss.find(thread_id); it != program->tls.tlss.end()) { + FreeTlsBlock(&it->second); + program->tls.tlss.erase(it); + } +} + +static uint64_t CalcBaseSize(const Elf64_Ehdr* ehdr, const Elf64_Phdr* phdr) { + uint64_t base_size = 0; + for (Elf64_Half i = 0; i < ehdr->e_phnum; i++) { + if (phdr[i].p_memsz != 0 && (phdr[i].p_type == PT_LOAD || phdr[i].p_type == PT_OS_RELRO)) { + uint64_t last_addr = phdr[i].p_vaddr + GetAlignedSize(phdr + i); + if (last_addr > base_size) { + base_size = last_addr; + } + } + } + return base_size; +} + +// NOLINTNEXTLINE(readability-function-cognitive-complexity) +void RuntimeLinker::LoadProgramToMemory(Program* program) { + KYTY_PROFILER_FUNCTION(); + + EXIT_IF(program == nullptr || program->base_vaddr != 0 || program->base_size != 0 || + program->elf == nullptr); + + // static uint64_t desired_base_addr = DESIRED_BASE_ADDR; + + bool is_shared = program->elf->IsShared(); + bool is_next_gen = program->elf->IsNextGen(); + + EXIT_NOT_IMPLEMENTED(!is_shared && !is_next_gen); + + const auto* ehdr = program->elf->GetEhdr(); + const auto* phdr = program->elf->GetPhdr(); + + EXIT_IF(phdr == nullptr || ehdr == nullptr); + + program->base_size = CalcBaseSize(ehdr, phdr); + constexpr uint64_t GUEST_PAGE_SIZE = 0x4000; + EXIT_IF(program->base_size > UINT64_MAX - (GUEST_PAGE_SIZE - 1)); + program->base_size_aligned = AlignUp(program->base_size, GUEST_PAGE_SIZE); + + uint64_t tls_handler_size = is_shared ? 0 : Jit::SafeCall::GetSize(); + EXIT_IF(tls_handler_size > UINT64_MAX - program->base_size_aligned); + program->mapped_size = program->base_size_aligned + tls_handler_size; + + program->base_vaddr = Common::VirtualMemory::Alloc( + g_desired_base_addr, program->mapped_size, Common::VirtualMemory::Mode::ExecuteReadWrite); + + if (!is_shared) { + program->tls.handler_vaddr = program->base_vaddr + program->base_size_aligned; + } + + g_desired_base_addr += CODE_BASE_INCR * (1 + program->mapped_size / CODE_BASE_INCR); + + EXIT_IF(program->base_vaddr == 0); + EXIT_IF(program->base_size_aligned < program->base_size); + Libs::LibKernel::Memory::RegisterProgramMemory( + program->base_vaddr, program->mapped_size, Common::VirtualMemory::Mode::ExecuteReadWrite, + Common::PathToString(program->file_name.filename()).c_str()); + + LOGF("base_vaddr = 0x%016" PRIx64 "\n" + "base_size = 0x%016" PRIx64 "\n" + "base_size_aligned = 0x%016" PRIx64 "\n" + "mapped_size = 0x%016" PRIx64 "\n", + program->base_vaddr, program->base_size, program->base_size_aligned, program->mapped_size); + if (!is_shared) { + LOGF("tls_handler_size = 0x%016" PRIx64 "\n", tls_handler_size); + } + + if (!Common::HostException::InstallHandler(KytyExceptionHandler)) { + EXIT("Failed to install the required vectored exception handler\n"); + } + + // program->elf->SetBaseVAddr(program->base_vaddr); + + for (Elf64_Half i = 0; i < ehdr->e_phnum; i++) { + if (phdr[i].p_memsz != 0 && (phdr[i].p_type == PT_LOAD || phdr[i].p_type == PT_OS_RELRO)) { + uint64_t segment_addr = phdr[i].p_vaddr + program->base_vaddr; + uint64_t segment_file_size = phdr[i].p_filesz; + uint64_t segment_memory_size = GetAlignedSize(phdr + i); + auto mode = GetMode(phdr[i].p_flags); + + LOGF("[%d] addr = 0x%016" PRIx64 "\n" + "[%d] file_size = %" PRIu64 "\n" + "[%d] memory_size = %" PRIu64 "\n" + "[%d] mode = %s\n", + i, segment_addr, i, segment_file_size, i, segment_memory_size, i, + Common::EnumName(mode).c_str()); + + program->elf->LoadSegment(segment_addr, phdr[i].p_offset, segment_file_size); + + bool skip_protect = (phdr[i].p_type == PT_LOAD && is_next_gen && + mode == Common::VirtualMemory::Mode::NoAccess); + + if (Common::VirtualMemory::IsExecute(mode)) { + PatchProgram(program, segment_addr, segment_memory_size); + } + + if (!skip_protect) { + if (!Common::VirtualMemory::Protect(segment_addr, segment_memory_size, mode)) { + EXIT("failed to protect ELF segment %u\n", static_cast(i)); + } + Libs::LibKernel::Memory::UpdateProgramMemoryProtection(segment_addr, + segment_memory_size, mode); + + if (Common::VirtualMemory::IsExecute(mode)) { + Common::VirtualMemory::FlushInstructionCache(segment_addr, segment_memory_size); + } + } + } + + if (phdr[i].p_type == PT_TLS) { + EXIT_IF(phdr[i].p_vaddr >= program->base_size); + + program->tls.image_vaddr = phdr[i].p_vaddr + program->base_vaddr; + program->tls.init_size = std::min(phdr[i].p_filesz, GetAlignedSize(phdr + i)); + program->tls.image_size = GetAlignedSize(phdr + i); + program->tls.tcb_offset = program->tls.image_size; + + LOGF("tls addr = 0x%016" PRIx64 "\n" + "tls init = %" PRIu64 "\n" + "tls size = %" PRIu64 "\n" + "tls offset = %" PRIu64 "\n", + program->tls.image_vaddr, program->tls.init_size, program->tls.image_size, + program->tls.tcb_offset); + } + + if (phdr[i].p_type == PT_OS_PROCPARAM) { + EXIT_IF(program->proc_param_vaddr != 0); + EXIT_IF(phdr[i].p_vaddr >= program->base_size); + + program->proc_param_vaddr = phdr[i].p_vaddr + program->base_vaddr; + } + } + + if (!is_shared) { + SetupTlsHandler(program); + } + + LOGF("entry = 0x%016" PRIx64 "\n", program->elf->GetEntry() + program->base_vaddr); +} + +void RuntimeLinker::DeleteProgram(Program* p) { + auto program = std::unique_ptr(p); + + if (program->base_vaddr != 0 || program->mapped_size != 0) { + EXIT_IF(program->base_vaddr == 0 || program->mapped_size == 0); + Libs::LibKernel::Memory::UnregisterProgramMemory(program->base_vaddr, program->mapped_size); + EXIT_IF(!Common::VirtualMemory::Free(program->base_vaddr)); + } + + if (program->custom_call_plt_vaddr != 0 || program->custom_call_plt_num != 0) { + Common::VirtualMemory::Free(program->custom_call_plt_vaddr); + } +} + +void RuntimeLinker::ParseProgramDynamicInfo(Program* program) { + KYTY_PROFILER_FUNCTION(); + + EXIT_IF(program == nullptr); + EXIT_IF(program->elf == nullptr); + EXIT_IF(program->dynamic_info != nullptr); + + program->dynamic_info = std::make_unique(); + + auto* elf = program->elf.get(); + + EXIT_NOT_IMPLEMENTED(elf->HasDynValue(DT_OS_HASH) && elf->HasDynValue(DT_HASH)); + GetDynDataOs(elf, &program->dynamic_info->hash_table, DT_OS_HASH); + GetDynData(elf, program->base_vaddr, &program->dynamic_info->hash_table, DT_HASH); + GetDynValue(elf, &program->dynamic_info->hash_table_size, DT_OS_HASHSZ); + + EXIT_NOT_IMPLEMENTED(elf->HasDynValue(DT_OS_STRTAB) && elf->HasDynValue(DT_STRTAB)); + EXIT_NOT_IMPLEMENTED(elf->HasDynValue(DT_OS_STRSZ) && elf->HasDynValue(DT_STRSZ)); + GetDynDataOs(elf, &program->dynamic_info->str_table, DT_OS_STRTAB); + GetDynData(elf, program->base_vaddr, &program->dynamic_info->str_table, DT_STRTAB); + GetDynValue(elf, &program->dynamic_info->str_table_size, DT_OS_STRSZ); + GetDynValue(elf, &program->dynamic_info->str_table_size, DT_STRSZ); + + EXIT_NOT_IMPLEMENTED(elf->HasDynValue(DT_OS_SYMTAB) && elf->HasDynValue(DT_SYMTAB)); + EXIT_NOT_IMPLEMENTED(elf->HasDynValue(DT_OS_SYMENT) && elf->HasDynValue(DT_SYMENT)); + GetDynDataOs(elf, &program->dynamic_info->symbol_table, DT_OS_SYMTAB); + GetDynData(elf, program->base_vaddr, &program->dynamic_info->symbol_table, DT_SYMTAB); + GetDynValue(elf, &program->dynamic_info->symbol_table_total_size, DT_OS_SYMTABSZ); + GetDynValue(elf, &program->dynamic_info->symbol_table_entry_size, DT_OS_SYMENT); + GetDynValue(elf, &program->dynamic_info->symbol_table_entry_size, DT_SYMENT); + + GetDynPtr(elf, &program->dynamic_info->init_vaddr, DT_INIT); + GetDynPtr(elf, &program->dynamic_info->fini_vaddr, DT_FINI); + GetDynPtr(elf, &program->dynamic_info->init_array_vaddr, DT_INIT_ARRAY); + GetDynPtr(elf, &program->dynamic_info->fini_array_vaddr, DT_FINI_ARRAY); + GetDynPtr(elf, &program->dynamic_info->preinit_array_vaddr, DT_PREINIT_ARRAY); + GetDynValue(elf, &program->dynamic_info->init_array_size, DT_INIT_ARRAYSZ); + GetDynValue(elf, &program->dynamic_info->fini_array_size, DT_FINI_ARRAYSZ); + GetDynValue(elf, &program->dynamic_info->preinit_array_size, DT_PREINIT_ARRAYSZ); + + EXIT_NOT_IMPLEMENTED(elf->HasDynValue(DT_OS_PLTGOT) && elf->HasDynValue(DT_PLTGOT)); + GetDynPtr(elf, &program->dynamic_info->pltgot_vaddr, DT_OS_PLTGOT); + GetDynPtr(elf, &program->dynamic_info->pltgot_vaddr, DT_PLTGOT); + + Elf64_Sxword jmprel_type = 0; + EXIT_NOT_IMPLEMENTED(elf->HasDynValue(DT_OS_PLTREL) && elf->HasDynValue(DT_PLTREL)); + GetDynValue(elf, &jmprel_type, DT_OS_PLTREL); + GetDynValue(elf, &jmprel_type, DT_PLTREL); + + EXIT_NOT_IMPLEMENTED(jmprel_type != DT_RELA); + if (jmprel_type == DT_RELA) { + EXIT_NOT_IMPLEMENTED(elf->HasDynValue(DT_OS_JMPREL) && elf->HasDynValue(DT_JMPREL)); + EXIT_NOT_IMPLEMENTED(elf->HasDynValue(DT_OS_PLTRELSZ) && elf->HasDynValue(DT_PLTRELSZ)); + GetDynDataOs(elf, &program->dynamic_info->jmprela_table, DT_OS_JMPREL); + GetDynData(elf, program->base_vaddr, &program->dynamic_info->jmprela_table, DT_JMPREL); + GetDynValue(elf, &program->dynamic_info->jmprela_table_size, DT_OS_PLTRELSZ); + GetDynValue(elf, &program->dynamic_info->jmprela_table_size, DT_PLTRELSZ); + } + + EXIT_NOT_IMPLEMENTED(elf->HasDynValue(DT_OS_RELA) && elf->HasDynValue(DT_RELA)); + GetDynDataOs(elf, &program->dynamic_info->rela_table, DT_OS_RELA); + GetDynData(elf, program->base_vaddr, &program->dynamic_info->rela_table, DT_RELA); + GetDynValue(elf, &program->dynamic_info->rela_table_total_size, DT_OS_RELASZ); + GetDynValue(elf, &program->dynamic_info->rela_table_total_size, DT_RELASZ); + GetDynValue(elf, &program->dynamic_info->rela_table_entry_size, DT_OS_RELAENT); + GetDynValue(elf, &program->dynamic_info->rela_table_entry_size, DT_RELAENT); + + GetDynValue(elf, &program->dynamic_info->relative_count, DT_RELACOUNT); + + GetDynValue(elf, &program->dynamic_info->debug, DT_DEBUG); + GetDynValue(elf, &program->dynamic_info->flags, DT_FLAGS); + GetDynValue(elf, &program->dynamic_info->textrel, DT_TEXTREL); + + EXIT_NOT_IMPLEMENTED(program->dynamic_info->debug != 0); + EXIT_NOT_IMPLEMENTED(program->dynamic_info->textrel != 0); + + std::vector needed; + GetDynValues(elf, &needed, DT_NEEDED); + for (auto need: needed) { + program->dynamic_info->needed.push_back(program->dynamic_info->str_table + need); + } + + uint64_t so_name = 0; + GetDynValue(elf, &so_name, DT_SONAME); + program->dynamic_info->so_name = program->dynamic_info->str_table + so_name; + + EXIT_NOT_IMPLEMENTED(elf->HasDynValue(DT_OS_NEEDED_MODULE) && + elf->HasDynValue(DT_OS_NEEDED_MODULE_1)); + EXIT_NOT_IMPLEMENTED(elf->HasDynValue(DT_OS_MODULE_INFO) && + elf->HasDynValue(DT_OS_MODULE_INFO_1)); + EXIT_NOT_IMPLEMENTED(elf->HasDynValue(DT_OS_IMPORT_LIB) && + elf->HasDynValue(DT_OS_IMPORT_LIB_1)); + EXIT_NOT_IMPLEMENTED(elf->HasDynValue(DT_OS_EXPORT_LIB) && + elf->HasDynValue(DT_OS_EXPORT_LIB_1)); + GetDynModules(elf, &program->dynamic_info->import_modules, program->dynamic_info->str_table, + DT_OS_NEEDED_MODULE); + GetDynModules(elf, &program->dynamic_info->import_modules, program->dynamic_info->str_table, + DT_OS_NEEDED_MODULE_1); + GetDynModules(elf, &program->dynamic_info->export_modules, program->dynamic_info->str_table, + DT_OS_MODULE_INFO); + GetDynModules(elf, &program->dynamic_info->export_modules, program->dynamic_info->str_table, + DT_OS_MODULE_INFO_1); + GetDynLibs(elf, &program->dynamic_info->import_libs, program->dynamic_info->str_table, + DT_OS_IMPORT_LIB); + GetDynLibs(elf, &program->dynamic_info->import_libs, program->dynamic_info->str_table, + DT_OS_IMPORT_LIB_1); + GetDynLibs(elf, &program->dynamic_info->export_libs, program->dynamic_info->str_table, + DT_OS_EXPORT_LIB); + GetDynLibs(elf, &program->dynamic_info->export_libs, program->dynamic_info->str_table, + DT_OS_EXPORT_LIB_1); +} + +static void InstallRelocateHandler(Program* program) { + KYTY_PROFILER_FUNCTION(); + + uint64_t pltgot_vaddr = program->dynamic_info->pltgot_vaddr + program->base_vaddr; + uint64_t pltgot_size = static_cast(3) * 8; + void** pltgot = reinterpret_cast(pltgot_vaddr); + + Common::VirtualMemory::Mode old_mode {}; + Common::VirtualMemory::Protect(pltgot_vaddr, pltgot_size, Common::VirtualMemory::Mode::Write, + &old_mode); + + pltgot[1] = program; + pltgot[2] = reinterpret_cast(RelocateHandler); + + Common::VirtualMemory::Protect(pltgot_vaddr, pltgot_size, old_mode); + + if (Common::VirtualMemory::IsExecute(old_mode)) { + Common::VirtualMemory::FlushInstructionCache(pltgot_vaddr, pltgot_size); + } + + // TODO(): check if this table already generated by compiler (sometimes it is missing) + if (program->custom_call_plt_vaddr == 0) { + program->custom_call_plt_num = + program->dynamic_info->jmprela_table_size / sizeof(Elf64_Rela); + auto size = Jit::CallPlt::GetSize(program->custom_call_plt_num); + program->custom_call_plt_vaddr = + Common::VirtualMemory::Alloc(SYSTEM_RESERVED, size, Common::VirtualMemory::Mode::Write); + EXIT_NOT_IMPLEMENTED(program->custom_call_plt_vaddr == 0); + auto* code = new (reinterpret_cast(program->custom_call_plt_vaddr)) + Jit::CallPlt(program->custom_call_plt_num); + code->SetPltGot(pltgot_vaddr); + Common::VirtualMemory::Protect(program->custom_call_plt_vaddr, size, + Common::VirtualMemory::Mode::Execute); + Common::VirtualMemory::FlushInstructionCache(program->custom_call_plt_vaddr, size); + } +} + +void RuntimeLinker::Relocate(Program* program) { + KYTY_PROFILER_FUNCTION(); + + EXIT_IF(program == nullptr); + + if (g_invalid_memory == 0) { + g_invalid_memory = Common::VirtualMemory::Alloc(INVALID_MEMORY, 4096, + Common::VirtualMemory::Mode::NoAccess); + EXIT_NOT_IMPLEMENTED(g_invalid_memory == 0); + } + + LOGF_COLOR(Log::Color::White, "--- Relocate program: %s ---\n", + Common::PathToString(program->file_name).c_str()); + + EXIT_NOT_IMPLEMENTED(program->dynamic_info->symbol_table_entry_size != sizeof(Elf64_Sym)); + EXIT_NOT_IMPLEMENTED(program->dynamic_info->rela_table_entry_size != sizeof(Elf64_Rela)); + EXIT_NOT_IMPLEMENTED(program->dynamic_info->jmprela_table == nullptr); + EXIT_NOT_IMPLEMENTED(program->dynamic_info->rela_table == nullptr); + EXIT_NOT_IMPLEMENTED(program->dynamic_info->symbol_table == nullptr); + EXIT_NOT_IMPLEMENTED(program->dynamic_info->pltgot_vaddr == 0); + + InstallRelocateHandler(program); + + std::vector unresolved; + const bool imports_only = program->relocated; + + RelocateRecords(program->dynamic_info->rela_table, program->dynamic_info->rela_table_total_size, + program, false, imports_only, &unresolved); + RelocateRecords(program->dynamic_info->jmprela_table, program->dynamic_info->jmprela_table_size, + program, true, imports_only, &unresolved); + program->relocated = true; + + if (program->tls.image_vaddr != 0 && program->tls.init_size != 0 && + program->tls.init_image.empty()) { + const auto* src = reinterpret_cast(program->tls.image_vaddr); + program->tls.init_image.assign(src, src + program->tls.init_size); + } + + if (!unresolved.empty()) { + LOGF("--- Stubbed unresolved imports: %zu ---\n", unresolved.size()); + for (const auto& symbol: unresolved) { + LOGF("Stubbed: %s\n", symbol.c_str()); + } + } +} + +Program* RuntimeLinker::FindProgram(const ModuleId& m, const LibraryId& l) { + Common::LockGuard lock(m_mutex); + + for (auto* p: m_programs) { + const auto& export_libs = p->dynamic_info->export_libs; + const auto& export_modules = p->dynamic_info->export_modules; + + if (std::find(export_libs.begin(), export_libs.end(), l) != export_libs.end() && + std::find(export_modules.begin(), export_modules.end(), m) != export_modules.end()) { + return p; + } + } + return nullptr; +} + +const ModuleId* RuntimeLinker::FindModule(const Program& program, const std::string& id) { + const auto& import_modules = program.dynamic_info->import_modules; + + if (auto it = std::find_if(import_modules.begin(), import_modules.end(), + [&id](const auto& module) { return module.id == id; }); + it != import_modules.end()) { + return &(*it); + } + + const auto& export_modules = program.dynamic_info->export_modules; + + if (auto it = std::find_if(export_modules.begin(), export_modules.end(), + [&id](const auto& module) { return module.id == id; }); + it != export_modules.end()) { + return &(*it); + } + + return nullptr; +} + +const LibraryId* RuntimeLinker::FindLibrary(const Program& program, const std::string& id) { + const auto& import_libs = program.dynamic_info->import_libs; + + if (auto it = std::find_if(import_libs.begin(), import_libs.end(), + [&id](const auto& lib) { return lib.id == id; }); + it != import_libs.end()) { + return &(*it); + } + + const auto& export_libs = program.dynamic_info->export_libs; + + if (auto it = std::find_if(export_libs.begin(), export_libs.end(), + [&id](const auto& lib) { return lib.id == id; }); + it != export_libs.end()) { + return &(*it); + } + + return nullptr; +} + +void RuntimeLinker::CreateSymbolDatabase(Program* program) { + KYTY_PROFILER_FUNCTION(); + + EXIT_IF(program == nullptr); + EXIT_IF(program->export_symbols != nullptr); + EXIT_IF(program->import_symbols != nullptr); + + program->export_symbols = std::make_unique(); + program->import_symbols = std::make_unique(); + + auto syms = [](Program* program, SymbolDatabase* symbols, bool is_export) { + if (program->dynamic_info->symbol_table == nullptr || + program->dynamic_info->str_table == nullptr) { + return; + } + + for (auto* sym = program->dynamic_info->symbol_table; + reinterpret_cast(sym) < + reinterpret_cast(program->dynamic_info->symbol_table) + + program->dynamic_info->symbol_table_total_size; + sym++) { + std::string id = std::string(program->dynamic_info->str_table + sym->st_name); + auto bind = sym->GetBind(); + auto type = sym->GetType(); + auto ids = Common::Split(id, '#'); + + if (ids.size() == 3) { + const auto* l = FindLibrary(*program, ids.at(1)); + const auto* m = FindModule(*program, ids.at(2)); + + if (l != nullptr && m != nullptr && (bind == STB_GLOBAL || bind == STB_WEAK) && + (type == STT_FUNC || type == STT_OBJECT || type == STT_NOTYPE) && + is_export == (sym->st_value != 0)) { + SymbolResolve sr {}; + sr.name = ids.at(0); + sr.library = l->name; + sr.library_version = l->version; + sr.module = m->name; + sr.module_version_major = m->version_major; + sr.module_version_minor = m->version_minor; + switch (type) { + case STT_NOTYPE: sr.type = SymbolType::NoType; break; + case STT_FUNC: sr.type = SymbolType::Func; break; + case STT_OBJECT: sr.type = SymbolType::Object; break; + default: sr.type = SymbolType::Unknown; break; + } + symbols->Add(sr, (is_export ? sym->st_value + program->base_vaddr : 0)); + } + } + } + }; + + syms(program, program->export_symbols.get(), true); + syms(program, program->import_symbols.get(), false); +} + +void RuntimeLinker::SetupTlsHandler(Program* program) { + EXIT_IF(program == nullptr); + EXIT_IF(g_tls_main_program != nullptr); + EXIT_IF(program->elf == nullptr); + EXIT_IF(program->elf->IsShared()); + EXIT_IF(program->tls.handler_vaddr == 0); + + g_tls_main_program = program; + + auto* code = new (reinterpret_cast(program->tls.handler_vaddr)) Jit::SafeCall; + + code->SetFunc(TlsMainGetAddr); + + for (uint8_t reg = 1; reg < 8; reg++) { + if (reg == 4) { + continue; + } + + auto* stub = new (reinterpret_cast(program->tls.handler_vaddr + + Jit::TlsRegStub::GetOffset(reg))) Jit::TlsRegStub; + stub->SetFunc(program->tls.handler_vaddr); + stub->SetOutputReg(reg); + } + + if (!Common::VirtualMemory::Protect(program->tls.handler_vaddr, Jit::SafeCall::GetSize(), + Common::VirtualMemory::Mode::Execute)) { + EXIT("failed to protect program TLS handler\n"); + } + Libs::LibKernel::Memory::UpdateProgramMemoryProtection( + program->tls.handler_vaddr, Jit::SafeCall::GetSize(), Common::VirtualMemory::Mode::Execute); + Common::VirtualMemory::FlushInstructionCache(program->tls.handler_vaddr, + Jit::SafeCall::GetSize()); +} + +void RuntimeLinker::DeleteTlss(int thread_id) { + Common::LockGuard lock(m_mutex); + + for (auto* p: m_programs) { + DeleteTls(p, thread_id); + } +} + +void RuntimeLinker::SetApplicationHeapApi(void* const api[10]) { + Common::LockGuard lock(m_mutex); + + if (api == nullptr || api[0] == nullptr || api[1] == nullptr) { + m_application_heap_malloc = nullptr; + m_application_heap_free = nullptr; + m_application_heap_posix_memalign = nullptr; + return; + } + + m_application_heap_malloc = reinterpret_cast(api[0]); + m_application_heap_free = reinterpret_cast(api[1]); + m_application_heap_posix_memalign = + reinterpret_cast(api[6]); +} + +void* RuntimeLinker::ApplicationHeapMemalign(uint64_t alignment, uint64_t size) { + Common::LockGuard lock(m_mutex); + + if (m_application_heap_posix_memalign != nullptr) { + void* ptr = nullptr; + return m_application_heap_posix_memalign(&ptr, alignment, size) == 0 ? ptr : nullptr; + } + + return nullptr; +} + +void* RuntimeLinker::ApplicationHeapMalloc(uint64_t size) { + Common::LockGuard lock(m_mutex); + + return m_application_heap_malloc != nullptr ? m_application_heap_malloc(size) : nullptr; +} + +} // namespace Loader diff --git a/src/loader/runtimeLinker.h b/src/loader/runtimeLinker.h new file mode 100644 index 000000000..5536978eb --- /dev/null +++ b/src/loader/runtimeLinker.h @@ -0,0 +1,209 @@ +#ifndef EMULATOR_INCLUDE_EMULATOR_LOADER_RUNTIMELINKER_H_ +#define EMULATOR_INCLUDE_EMULATOR_LOADER_RUNTIMELINKER_H_ + +#include "common/abi.h" +#include "common/common.h" +#include "common/stringUtils.h" +#include "common/threads.h" +#include "loader/symbolDatabase.h" + +#include +#include +#include +#include + +namespace Loader { + +class Elf64; +struct Elf64_Sym; +struct Elf64_Rela; +class RuntimeLinker; + +using module_func_t = KYTY_SYSV_ABI int (*)(size_t args, const void* argp); +using application_heap_free_func_t = KYTY_SYSV_ABI void (*)(void*); +using application_heap_malloc_func_t = KYTY_SYSV_ABI void* (*)(uint64_t); +using application_heap_posix_memalign_func_t = KYTY_SYSV_ABI int (*)(void**, uint64_t, uint64_t); + +struct ModuleId { + bool operator==(const ModuleId& other) const { + return version_major == other.version_major && version_minor == other.version_minor && + name == other.name; + } + + std::string id; + int version_major; + int version_minor; + std::string name; +}; + +struct LibraryId { + bool operator==(const LibraryId& other) const { + return version == other.version && name == other.name; + } + + std::string id; + int version; + std::string name; +}; + +struct ThreadLocalStorage { + struct Block { + uint8_t* ptr = nullptr; + application_heap_free_func_t free_func = nullptr; + bool vm_alloc = false; + }; + + ~ThreadLocalStorage(); + + uint64_t image_vaddr = 0; + uint64_t init_size = 0; + uint64_t image_size = 0; + uint64_t tcb_offset = 0; + uint64_t handler_vaddr = 0; + + std::vector init_image; + std::unordered_map tlss; + Common::Mutex mutex; +}; + +struct DynamicInfo { + void* hash_table = nullptr; + uint64_t hash_table_size = 0; + + char* str_table = nullptr; + uint64_t str_table_size = 0; + + Elf64_Sym* symbol_table = nullptr; + uint64_t symbol_table_total_size = 0; + uint64_t symbol_table_entry_size = 0; + + uint64_t init_vaddr = 0; + uint64_t fini_vaddr = 0; + uint64_t init_array_vaddr = 0; + uint64_t fini_array_vaddr = 0; + uint64_t preinit_array_vaddr = 0; + uint64_t init_array_size = 0; + uint64_t fini_array_size = 0; + uint64_t preinit_array_size = 0; + uint64_t pltgot_vaddr = 0; + + Elf64_Rela* jmprela_table = nullptr; + uint64_t jmprela_table_size = 0; + + Elf64_Rela* rela_table = nullptr; + uint64_t rela_table_total_size = 0; + uint64_t rela_table_entry_size = 0; + + uint64_t relative_count = 0; + + uint64_t debug = 0; + uint64_t textrel = 0; + uint64_t flags = 0; + + const char* so_name = nullptr; + + std::vector needed; + std::vector export_modules; + std::vector import_modules; + std::vector export_libs; + std::vector import_libs; +}; + +struct Program { + Program(); + ~Program(); + + int32_t unique_id = -1; + RuntimeLinker* rt = nullptr; + std::filesystem::path file_name; + std::unique_ptr elf; + std::unique_ptr dynamic_info; + uint64_t base_vaddr = 0; + uint64_t base_size = 0; + uint64_t base_size_aligned = 0; + uint64_t mapped_size = 0; + std::unique_ptr export_symbols; + std::unique_ptr import_symbols; + ThreadLocalStorage tls; + bool fail_if_global_not_resolved = true; + bool dbg_print_reloc = false; + bool relocated = false; + uint64_t proc_param_vaddr = 0; + uint64_t custom_call_plt_vaddr = 0; + uint32_t custom_call_plt_num = 0; +}; + +class RuntimeLinker { +public: + RuntimeLinker(); + virtual ~RuntimeLinker(); + void Clear(); + + KYTY_CLASS_NO_COPY(RuntimeLinker); + + void DbgDump(const std::string& folder); + + Program* LoadProgram(const std::filesystem::path& elf_name); + void SaveMainProgram(const std::filesystem::path& elf_name); + void SaveProgram(Program* program, const std::filesystem::path& elf_name); + void UnloadProgram(Program* program); + + [[nodiscard]] uint64_t GetEntry(); + [[nodiscard]] uint64_t GetProcParam(); + + void RelocateAll(); + void RelocateProgram(Program* program); + + void Execute(); + int StartModule(Program* program, size_t args, const void* argp, module_func_t func); + int StopModule(Program* program, size_t args, const void* argp, module_func_t func); + void StartAllModules(); + void StopAllModules(); + void DeleteTlss(int thread_id); + void SetApplicationHeapApi(void* const api[10]); + void* ApplicationHeapMalloc(uint64_t size); + void* ApplicationHeapMemalign(uint64_t alignment, uint64_t size); + + void Resolve(const std::string& name, SymbolType type, Program* program, SymbolRecord* out_info, + bool* bind_self); + bool ResolveLoadedSymbolByNid(const std::string& nid, SymbolType type, SymbolRecord* out_info); + + SymbolDatabase* Symbols() { return m_symbols.get(); } + + static uint64_t ReadFromElf(Program* program, uint64_t vaddr); + Program* FindProgramByAddr(uint64_t vaddr); + Program* FindProgramById(int32_t id); + Program* FindProgramByFileName(const std::filesystem::path& elf_name); + + static uint8_t* TlsGetAddr(Program* program); + static void DeleteTls(Program* program, int thread_id); + + void StackTrace(uint64_t frame_ptr); + +private: + static void LoadProgramToMemory(Program* program); + static void ParseProgramDynamicInfo(Program* program); + static void CreateSymbolDatabase(Program* program); + static void Relocate(Program* program); + static void DeleteProgram(Program* program); + static void SetupTlsHandler(Program* program); + void PreloadAdjacentPrograms(); + + Program* FindProgram(const ModuleId& m, const LibraryId& l); + + static const ModuleId* FindModule(const Program& program, const std::string& id); + static const LibraryId* FindLibrary(const Program& program, const std::string& id); + + std::vector m_programs; + std::unique_ptr m_symbols; + bool m_relocated = false; + Common::Mutex m_mutex; + + application_heap_malloc_func_t m_application_heap_malloc = nullptr; + application_heap_free_func_t m_application_heap_free = nullptr; + application_heap_posix_memalign_func_t m_application_heap_posix_memalign = nullptr; +}; + +} // namespace Loader + +#endif /* EMULATOR_INCLUDE_EMULATOR_LOADER_RUNTIMELINKER_H_ */ diff --git a/src/loader/symbolDatabase.cpp b/src/loader/symbolDatabase.cpp new file mode 100644 index 000000000..688a5c531 --- /dev/null +++ b/src/loader/symbolDatabase.cpp @@ -0,0 +1,94 @@ +#include "loader/symbolDatabase.h" + +#include "common/file.h" +#include "common/magicEnum.h" + +#include + +namespace Loader { + +constexpr char LIB_PREFIX[] = "libSce"; + +static std::string UpdateName(const std::string& str) { + return Common::StartsWith(str, LIB_PREFIX) ? Common::RemoveFirst(str, 6) : str; +} + +std::string SymbolDatabase::GenerateName(const SymbolResolve& s) { + auto library = UpdateName(s.library); + auto module = UpdateName(s.module); + return fmt::format("{}[{}_v{}][{}_v{}.{}][{}]", s.name.c_str(), library.c_str(), + s.library_version, module.c_str(), s.module_version_major, + s.module_version_minor, Common::EnumName(s.type).c_str()); +} + +void SymbolDatabase::Add(const SymbolResolve& s, uint64_t vaddr) { + SymbolRecord r {}; + r.name = GenerateName(s); + r.vaddr = vaddr; + m_map.insert_or_assign(r.name, m_symbols.size()); + m_symbols.push_back(r); +} + +void SymbolDatabase::Add(const SymbolResolve& s, uint64_t vaddr, const std::string& dbg_name) { + SymbolRecord r {}; + r.name = GenerateName(s); + r.vaddr = vaddr; + r.dbg_name = dbg_name; + m_map.insert_or_assign(r.name, m_symbols.size()); + m_symbols.push_back(r); +} + +void SymbolDatabase::DbgDump(const std::string& folder, const std::string& file_name) { + auto folder_str = Common::FixDirectorySlash(folder); + + Common::File::CreateDirectories(folder_str); + + Common::File f; + f.Create(folder_str + file_name); + + for (const auto& sym: m_symbols) { + f.Printf("%" PRIx64 " %s\n", sym.vaddr, sym.name.c_str()); + } + + f.Close(); +} + +const SymbolRecord* SymbolDatabase::Find(const SymbolResolve& s) const { + auto it = m_map.find(GenerateName(s)); + if (it == m_map.end()) { + return nullptr; + } + auto index = it->second; + if (index >= m_symbols.size()) { + return nullptr; + } + return &m_symbols[index]; +} + +const SymbolRecord* SymbolDatabase::FindByNid(const std::string& nid, SymbolType type) const { + auto prefix = nid + "["; + auto suffix = fmt::format("[{}]", Common::EnumName(type).c_str()); + + for (const auto& symbol: m_symbols) { + if (Common::StartsWith(symbol.name, prefix) && Common::EndsWith(symbol.name, suffix)) { + return &symbol; + } + } + + return nullptr; +} + +const SymbolRecord* SymbolDatabase::FindByName(const std::string& name, SymbolType type) const { + auto prefix = name + "["; + auto suffix = fmt::format("[{}]", Common::EnumName(type).c_str()); + + for (const auto& symbol: m_symbols) { + if (Common::StartsWith(symbol.name, prefix) && Common::EndsWith(symbol.name, suffix)) { + return &symbol; + } + } + + return nullptr; +} + +} // namespace Loader diff --git a/src/loader/symbolDatabase.h b/src/loader/symbolDatabase.h new file mode 100644 index 000000000..9c5daa20b --- /dev/null +++ b/src/loader/symbolDatabase.h @@ -0,0 +1,62 @@ +#ifndef EMULATOR_INCLUDE_EMULATOR_LOADER_SYMBOLDATABASE_H_ +#define EMULATOR_INCLUDE_EMULATOR_LOADER_SYMBOLDATABASE_H_ + +#include "common/abi.h" +#include "common/common.h" +#include "common/stringUtils.h" + +#include +#include + +namespace Loader { + +enum class SymbolType { + Unknown, + Func, + Object, + TlsModule, + NoType, +}; + +struct SymbolRecord { + std::string name; + std::string dbg_name; + uint64_t vaddr; +}; + +struct SymbolResolve { + std::string name; + std::string library; + int library_version; + std::string module; + int module_version_major; + int module_version_minor; + SymbolType type; +}; + +class SymbolDatabase { +public: + SymbolDatabase() = default; + virtual ~SymbolDatabase() = default; + + void Add(const SymbolResolve& s, uint64_t vaddr); + void Add(const SymbolResolve& s, uint64_t vaddr, const std::string& dbg_name); + + [[nodiscard]] const SymbolRecord* Find(const SymbolResolve& s) const; + [[nodiscard]] const SymbolRecord* FindByNid(const std::string& nid, SymbolType type) const; + [[nodiscard]] const SymbolRecord* FindByName(const std::string& name, SymbolType type) const; + + void DbgDump(const std::string& folder, const std::string& file_name); + + KYTY_CLASS_NO_COPY(SymbolDatabase); + + static std::string GenerateName(const SymbolResolve& s); + +private: + std::vector m_symbols; + std::unordered_map m_map; +}; + +} // namespace Loader + +#endif /* EMULATOR_INCLUDE_EMULATOR_LOADER_SYMBOLDATABASE_H_ */ diff --git a/src/loader/systemContent.cpp b/src/loader/systemContent.cpp new file mode 100644 index 000000000..efa8b4804 --- /dev/null +++ b/src/loader/systemContent.cpp @@ -0,0 +1,677 @@ +#include "loader/systemContent.h" + +#include "common/assert.h" +#include "common/byteBuffer.h" +#include "common/common.h" +#include "common/file.h" +#include "common/logging/log.h" +#include "common/singleton.h" +#include "common/threads.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace Loader { + +class Psf { +public: + Psf() = default; + virtual ~Psf(); + + KYTY_CLASS_NO_COPY(Psf); + + void Open(const std::filesystem::path& file_name); + void OpenChunkDefs(const std::filesystem::path& file_name); + void Close(); + + bool IsValid(); + + bool GetParamInt(const char* name, int32_t* value); + bool GetParamString(const char* name, std::string* value); + bool GetParamString(const char* name, char* value, size_t value_size); + uint64_t GetFlexibleMemorySize(); + + void DbgPrint(); + +private: + enum class ParamType { + Text, + Int, + }; + + struct Param { + std::string name; + ParamType type = ParamType::Text; + std::string string_value; + std::string string_utf8; + int32_t int_value = 0; + }; + + void OpenJson(const std::filesystem::path& file_name); + void AddParamString(const std::string& name, const std::string& value); + void AddParamInt(const std::string& name, int32_t value); + Param* FindParam(const char* name); + + Common::Mutex m_mutex; + bool m_opened = false; + std::vector m_params; + uint64_t m_flexible_memory_size = 0; +}; + +class PlayGo { +public: + PlayGo() = default; + virtual ~PlayGo(); + + KYTY_CLASS_NO_COPY(PlayGo); + + void Open(const std::filesystem::path& file_name); + void OpenChunkDefs(const std::filesystem::path& file_name); + void OpenChunkManifest(const std::filesystem::path& file_name); + void Close(); + + bool IsValid(); + + bool GetChunksNum(uint32_t* num); + + void DbgPrint(); + +private: + Common::File m_f; + Common::Mutex m_mutex; + bool m_opened = false; + uint16_t m_chunks_num = 0; +}; + +struct SystemContent { + std::filesystem::path psf_path; + Psf psf; + std::filesystem::path playgo_path; + PlayGo playgo; + std::filesystem::path icon_path; +}; + +Psf::~Psf() { + Close(); +} + +void Psf::Open(const std::filesystem::path& file_name) { + Common::LockGuard lock(m_mutex); + + Close(); + + OpenJson(file_name); +} + +void Psf::OpenJson(const std::filesystem::path& file_name) { + Common::File f; + + f.Open(file_name, Common::File::Mode::Read); + + if (f.IsInvalid()) { + LOGF("Can't open %s\n", Common::PathToString(file_name).c_str()); + return; + } + + auto buf = f.ReadWholeBuffer(); + if (buf.Size() == 0) { + LOGF("invalid json file: %s\n", Common::PathToString(file_name).c_str()); + f.Close(); + return; + } + + std::string default_language; + std::vector localized_languages; + std::vector localized_titles; + + const auto* json_begin = reinterpret_cast(buf.GetDataConst()); + const auto* json_end = json_begin + buf.Size(); + auto param_json = nlohmann::ordered_json::parse(json_begin, json_end, nullptr, false); + if (param_json.is_discarded() || !param_json.is_object()) { + LOGF("invalid json file: %s\n", Common::PathToString(file_name).c_str()); + f.Close(); + return; + } + + auto add_string_param = [&](const char* json_key, const char* param_key) { + auto it = param_json.find(json_key); + if (it != param_json.end() && it->is_string()) { + AddParamString(param_key, it->get_ref()); + } + }; + + auto add_int_param = [&](const char* json_key, const char* param_key) { + auto it = param_json.find(json_key); + if (it == param_json.end() || !it->is_number_integer()) { + return; + } + int64_t value = 0; + if (it->is_number_unsigned()) { + value = static_cast(it->get()); + } else { + value = it->get(); + } + AddParamInt(param_key, static_cast(value)); + }; + + add_string_param("titleId", "TITLE_ID"); + add_string_param("contentId", "CONTENT_ID"); + add_string_param("contentVersion", "APP_VER"); + add_string_param("appVersion", "APP_VER"); + add_int_param("userDefinedParam1", "USER_DEFINED_PARAM_1"); + add_int_param("userDefinedParam2", "USER_DEFINED_PARAM_2"); + add_int_param("userDefinedParam3", "USER_DEFINED_PARAM_3"); + add_int_param("userDefinedParam4", "USER_DEFINED_PARAM_4"); + + if (auto kernel = param_json.find("kernel"); + kernel != param_json.end() && kernel->is_object()) { + if (auto it = kernel->find("flexibleMemorySize"); it != kernel->end()) { + m_flexible_memory_size = it->get(); + } + } + + auto localized = param_json.find("localizedParameters"); + if (localized != param_json.end() && localized->is_object()) { + auto default_it = localized->find("defaultLanguage"); + if (default_it != localized->end() && default_it->is_string()) { + default_language = default_it->get_ref(); + } + for (auto it = localized->begin(); it != localized->end(); ++it) { + if (!it->is_object()) { + continue; + } + auto title_it = it->find("titleName"); + if (title_it != it->end() && title_it->is_string()) { + localized_languages.push_back(it.key()); + localized_titles.push_back(title_it->get_ref()); + } + } + } + + std::string title; + for (size_t i = 0; i < localized_titles.size(); i++) { + if (!default_language.empty() && localized_languages[i] == default_language) { + title = localized_titles[i]; + break; + } + } + if (title.empty()) { + for (size_t i = 0; i < localized_titles.size(); i++) { + if (localized_languages[i] == "en-US") { + title = localized_titles[i]; + break; + } + } + } + if (title.empty() && !localized_titles.empty()) { + title = localized_titles[0]; + } + if (!title.empty()) { + AddParamString("TITLE", title); + } + + f.Close(); + m_opened = true; +} + +void Psf::Close() { + Common::LockGuard lock(m_mutex); + + m_params.clear(); + m_flexible_memory_size = 0; + m_opened = false; +} + +bool Psf::IsValid() { + Common::LockGuard lock(m_mutex); + + return m_opened; +} + +void Psf::DbgPrint() { + Common::LockGuard lock(m_mutex); + + if (!m_opened) { + return; + } + + for (const auto& param: m_params) { + LOGF("%s, ", param.name.c_str()); + + switch (param.type) { + case ParamType::Text: { + auto str = Common::ReplaceChar(param.string_value, '\n', ','); + LOGF("string = %s\n", str.c_str()); + break; + } + case ParamType::Int: + LOGF("int = 0x%08" PRIx32 "\n", static_cast(param.int_value)); + break; + default: EXIT("unknown type\n"); + } + } +} + +Psf::Param* Psf::FindParam(const char* name) { + for (auto& param: m_params) { + if (param.name == name) { + return ¶m; + } + } + + return nullptr; +} + +void Psf::AddParamString(const std::string& name, const std::string& value) { + auto* param = FindParam(name.c_str()); + if (param == nullptr) { + m_params.push_back({}); + param = &m_params.back(); + param->name = name; + } + + param->type = ParamType::Text; + param->string_utf8 = value; + param->string_value = value; +} + +void Psf::AddParamInt(const std::string& name, int32_t value) { + auto* param = FindParam(name.c_str()); + if (param == nullptr) { + m_params.push_back({}); + param = &m_params.back(); + param->name = name; + } + + param->type = ParamType::Int; + param->int_value = value; +} + +bool Psf::GetParamInt(const char* name, int32_t* value) { + Common::LockGuard lock(m_mutex); + + if (!m_opened) { + return false; + } + if (auto* param = FindParam(name); param != nullptr && param->type == ParamType::Int) { + *value = param->int_value; + return true; + } + + return false; +} + +bool Psf::GetParamString(const char* name, std::string* value) { + Common::LockGuard lock(m_mutex); + + if (!m_opened) { + return false; + } + if (auto* param = FindParam(name); param != nullptr && param->type == ParamType::Text) { + *value = param->string_value; + return true; + } + + return false; +} + +bool Psf::GetParamString(const char* name, char* value, size_t value_size) { + Common::LockGuard lock(m_mutex); + + if (!m_opened) { + return false; + } + if (auto* param = FindParam(name); param != nullptr && param->type == ParamType::Text && + param->string_utf8.size() + 1 <= value_size) { + std::memcpy(value, param->string_utf8.c_str(), param->string_utf8.size() + 1); + return true; + } + + return false; +} + +uint64_t Psf::GetFlexibleMemorySize() { + Common::LockGuard lock(m_mutex); + + return m_flexible_memory_size; +} + +PlayGo::~PlayGo() { + Close(); +} + +void PlayGo::Open(const std::filesystem::path& file_name) { + Common::LockGuard lock(m_mutex); + + Close(); + + m_f.Open(file_name, Common::File::Mode::Read); + + if (m_f.IsInvalid()) { + LOGF("Can't open %s\n", Common::PathToString(file_name).c_str()); + return; + } + + uint32_t magic1 = 0; + + m_f.Read(&magic1, 4); + + if (magic1 != 0x6f676c70) { + LOGF("invalid file: magic1 = %08" PRIx32 "\n", magic1); + return; + } + + m_f.Seek(10); + m_f.Read(&m_chunks_num, 2); + + m_opened = true; +} + +static bool ParseXmlUintAttr(std::string_view tag, std::string_view attr, uint32_t& value) { + const auto attr_pos = tag.find(attr); + if (attr_pos == std::string_view::npos) { + return false; + } + + auto pos = attr_pos + attr.size(); + while (pos < tag.size() && (tag[pos] == ' ' || tag[pos] == '\t')) { + pos++; + } + if (pos >= tag.size() || tag[pos] != '=') { + return false; + } + pos++; + while (pos < tag.size() && (tag[pos] == ' ' || tag[pos] == '\t')) { + pos++; + } + if (pos >= tag.size() || (tag[pos] != '"' && tag[pos] != '\'')) { + return false; + } + + const char quote = tag[pos++]; + const auto start = pos; + while (pos < tag.size() && tag[pos] >= '0' && tag[pos] <= '9') { + pos++; + } + if (start == pos || pos >= tag.size() || tag[pos] != quote) { + return false; + } + + uint32_t parsed = 0; + const auto [ptr, ec] = std::from_chars(tag.data() + start, tag.data() + pos, parsed); + if (ec != std::errc {} || ptr != tag.data() + pos) { + return false; + } + + value = parsed; + return true; +} + +void PlayGo::OpenChunkDefs(const std::filesystem::path& file_name) { + Common::LockGuard lock(m_mutex); + + if (m_opened) { + return; + } + + Common::File f; + f.Open(file_name, Common::File::Mode::Read); + + if (f.IsInvalid()) { + LOGF("Can't open %s\n", Common::PathToString(file_name).c_str()); + return; + } + + auto buf = f.ReadWholeBuffer(); + f.Close(); + + if (buf.Size() == 0) { + LOGF("invalid file: %s\n", Common::PathToString(file_name).c_str()); + return; + } + + const std::string_view xml(reinterpret_cast(buf.GetDataConst()), buf.Size()); + + uint32_t max_chunk_id = 0; + bool found = false; + + if (const auto chunks_pos = xml.find("', chunks_pos); + if (tag_end != std::string_view::npos) { + uint32_t default_chunk = 0; + if (ParseXmlUintAttr(xml.substr(chunks_pos, tag_end - chunks_pos), "default_chunk", + default_chunk)) { + max_chunk_id = std::max(max_chunk_id, default_chunk); + found = true; + } + } + } + + size_t pos = 0; + while ((pos = xml.find("= xml.size() || !(xml[name_end] == ' ' || xml[name_end] == '\t' || + xml[name_end] == '\r' || xml[name_end] == '\n')) { + pos = name_end; + continue; + } + + const auto tag_end = xml.find('>', name_end); + if (tag_end == std::string_view::npos) { + break; + } + + uint32_t chunk_id = 0; + if (ParseXmlUintAttr(xml.substr(pos, tag_end - pos), "id", chunk_id)) { + max_chunk_id = std::max(max_chunk_id, chunk_id); + found = true; + } + + pos = tag_end + 1; + } + + if (!found || max_chunk_id >= 1000) { + LOGF("invalid playgo chunk definitions: max_chunk_id=%" PRIu32 ", path=%s\n", max_chunk_id, + Common::PathToString(file_name).c_str()); + return; + } + + m_chunks_num = static_cast(max_chunk_id + 1); + m_opened = true; +} + +void PlayGo::OpenChunkManifest(const std::filesystem::path& file_name) { + Common::LockGuard lock(m_mutex); + + if (m_opened) { + return; + } + + Common::File f; + f.Open(file_name, Common::File::Mode::Read); + + if (f.IsInvalid()) { + LOGF("Can't open %s\n", Common::PathToString(file_name).c_str()); + return; + } + + auto buf = f.ReadWholeBuffer(); + f.Close(); + + constexpr uint8_t id_field[] = {0x08, 'i', 'd', 0x00}; + bool chunks[1000] {}; + uint32_t max_chunk_id = 0; + bool found = false; + const auto* data = buf.GetDataConst(); + + for (size_t pos = 0; pos + sizeof(id_field) + sizeof(uint32_t) <= buf.Size(); pos++) { + if (std::memcmp(data + pos, id_field, sizeof(id_field)) != 0) { + continue; + } + + const auto* value = data + pos + sizeof(id_field); + const auto chunk_id = + static_cast(value[0]) | (static_cast(value[1]) << 8u) | + (static_cast(value[2]) << 16u) | (static_cast(value[3]) << 24u); + if (chunk_id < 1000) { + chunks[chunk_id] = true; + max_chunk_id = std::max(max_chunk_id, chunk_id); + found = true; + } + } + + if (!found || + !std::all_of(chunks, chunks + max_chunk_id + 1, [](bool chunk) { return chunk; })) { + LOGF("invalid chunk manifest: path=%s\n", Common::PathToString(file_name).c_str()); + return; + } + + m_chunks_num = static_cast(max_chunk_id + 1); + m_opened = true; + LOGF("PlayGo: chunks num = %" PRIu16 " (from %s)\n", m_chunks_num, + Common::PathToString(file_name).c_str()); +} + +void PlayGo::Close() { + Common::LockGuard lock(m_mutex); + + if (!m_f.IsInvalid()) { + m_f.Close(); + } + m_opened = false; + m_chunks_num = 0; +} + +bool PlayGo::IsValid() { + Common::LockGuard lock(m_mutex); + + return m_opened; +} + +bool PlayGo::GetChunksNum(uint32_t* num) { + Common::LockGuard lock(m_mutex); + + if (m_opened) { + *num = m_chunks_num; + return true; + } + return false; +} + +void PlayGo::DbgPrint() { + Common::LockGuard lock(m_mutex); + + if (!m_opened) { + return; + } + LOGF("PlayGo: chunks num = %" PRIu16 "\n", m_chunks_num); +} + +void SystemContentLoadParamSfo(const std::filesystem::path& file_name) { + auto* sc = Common::Singleton::Instance(); + + if (!Common::File::IsFileExisting(file_name)) { + EXIT("Can't find file: %s\n", Common::PathToString(file_name).c_str()); + } + + sc->psf.Open(file_name); + + if (!sc->psf.IsValid()) { + EXIT("invalid file: %s\n", Common::PathToString(file_name).c_str()); + } + + sc->psf_path = file_name; + sc->psf.DbgPrint(); + + sc->icon_path = file_name.parent_path() / "icon0.png"; + + if (!Common::File::IsFileExisting(sc->icon_path)) { + sc->icon_path.clear(); + } + + sc->playgo_path = file_name.parent_path() / "playgo-chunk.dat"; + sc->playgo.Open(sc->playgo_path); + if (!sc->playgo.IsValid()) { + sc->playgo_path = file_name.parent_path().parent_path() / "playgo-chunkdefs.xml"; + sc->playgo.OpenChunkDefs(sc->playgo_path); + } + if (!sc->playgo.IsValid()) { + sc->playgo_path = file_name.parent_path().parent_path() / "Data" / "chunkmanifest"; + sc->playgo.OpenChunkManifest(sc->playgo_path); + } + + if (sc->playgo.IsValid()) { + sc->playgo.DbgPrint(); + } +} + +bool SystemContentParamSfoGetInt(const char* name, int32_t* value) { + if (name == nullptr || value == nullptr) { + return false; + } + + auto* sc = Common::Singleton::Instance(); + + return sc->psf.GetParamInt(name, value); +} + +bool SystemContentParamSfoGetString(const char* name, std::string* value) { + if (name == nullptr || value == nullptr) { + return false; + } + + auto* sc = Common::Singleton::Instance(); + + return sc->psf.GetParamString(name, value); +} + +bool SystemContentParamSfoGetString(const char* name, char* value, size_t value_size) { + if (name == nullptr || value == nullptr) { + return false; + } + + auto* sc = Common::Singleton::Instance(); + + return sc->psf.GetParamString(name, value, value_size); +} + +uint64_t SystemContentGetFlexibleMemorySize() { + auto* sc = Common::Singleton::Instance(); + + return sc->psf.GetFlexibleMemorySize(); +} + +bool SystemContentGetIconPath(std::string* path) { + if (path == nullptr) { + return false; + } + + auto* sc = Common::Singleton::Instance(); + + if (sc->icon_path.empty()) { + return false; + } + + *path = Common::PathToString(sc->icon_path); + + return true; +} + +bool SystemContentGetChunksNum(uint32_t* num) { + if (num == nullptr) { + return false; + } + + auto* sc = Common::Singleton::Instance(); + + return sc->playgo.GetChunksNum(num); +} + +} // namespace Loader diff --git a/src/loader/systemContent.h b/src/loader/systemContent.h new file mode 100644 index 000000000..f3216f1f6 --- /dev/null +++ b/src/loader/systemContent.h @@ -0,0 +1,23 @@ +#ifndef EMULATOR_INCLUDE_EMULATOR_LOADER_SYSTEMCONTENT_H_ +#define EMULATOR_INCLUDE_EMULATOR_LOADER_SYSTEMCONTENT_H_ + +#include "common/abi.h" +#include "common/common.h" +#include "common/stringUtils.h" + +#include +#include + +namespace Loader { + +void SystemContentLoadParamSfo(const std::filesystem::path& file_name); +bool SystemContentParamSfoGetInt(const char* name, int32_t* value); +bool SystemContentParamSfoGetString(const char* name, std::string* value); +bool SystemContentParamSfoGetString(const char* name, char* value, size_t value_size); +uint64_t SystemContentGetFlexibleMemorySize(); +bool SystemContentGetIconPath(std::string* path); +bool SystemContentGetChunksNum(uint32_t* num); + +} // namespace Loader + +#endif /* EMULATOR_INCLUDE_EMULATOR_LOADER_SYSTEMCONTENT_H_ */ diff --git a/src/loader/timer.cpp b/src/loader/timer.cpp new file mode 100644 index 000000000..994e86595 --- /dev/null +++ b/src/loader/timer.cpp @@ -0,0 +1,40 @@ +#include "common/timer.h" + +#include "common/abi.h" +#include "common/dateTime.h" +#include "common/subsystems.h" +#include "loader/timer.h" + +namespace Loader::Timer { + +static Common::Timer g_timer; + +KYTY_SUBSYSTEM_INIT(Timer) { + Start(); +} + +KYTY_SUBSYSTEM_UNEXPECTED_SHUTDOWN(Timer) {} + +KYTY_SUBSYSTEM_DESTROY(Timer) {} + +void Start() { + g_timer.Start(); +} + +double GetTimeMs() { + return g_timer.GetTimeMs(); +} + +Common::Time GetTime() { + return Common::Time(static_cast(GetTimeMs())); +} + +uint64_t GetCounter() { + return g_timer.GetTicks(); +} + +uint64_t GetFrequency() { + return g_timer.GetFrequency(); +} + +} // namespace Loader::Timer diff --git a/src/loader/timer.h b/src/loader/timer.h new file mode 100644 index 000000000..3097c3ef2 --- /dev/null +++ b/src/loader/timer.h @@ -0,0 +1,21 @@ +#ifndef EMULATOR_INCLUDE_EMULATOR_LOADER_TIMER_H_ +#define EMULATOR_INCLUDE_EMULATOR_LOADER_TIMER_H_ + +#include "common/abi.h" +#include "common/common.h" +#include "common/dateTime.h" +#include "common/subsystems.h" + +namespace Loader::Timer { + +KYTY_SUBSYSTEM_DEFINE(Timer); + +void Start(); +double GetTimeMs(); +Common::Time GetTime(); +uint64_t GetCounter(); +uint64_t GetFrequency(); + +} // namespace Loader::Timer + +#endif /* EMULATOR_INCLUDE_EMULATOR_LOADER_TIMER_H_ */ diff --git a/src/loader/x64InstructionEmulator.cpp b/src/loader/x64InstructionEmulator.cpp new file mode 100644 index 000000000..7bb8aa2a5 --- /dev/null +++ b/src/loader/x64InstructionEmulator.cpp @@ -0,0 +1,155 @@ +#include "loader/x64InstructionEmulator.h" + +#include "common/common.h" + +#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS +#include // IWYU pragma: keep +#endif + +namespace Loader::X64InstructionEmulator { + +#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS + +static M128A* GetContextXmm(PCONTEXT context, uint8_t index) { + if (context == nullptr || index >= 16) { + return nullptr; + } + + return &context->Xmm0 + index; +} + +static uint64_t ExtractBitField(uint64_t value, uint32_t length, uint32_t index) { + length &= 0x3fu; + index &= 0x3fu; + + if (length == 0) { + length = 64; + } + + if (index >= 64) { + return 0; + } + + auto available = 64u - index; + if (length > available) { + length = available; + } + + const uint64_t mask = (length == 64 ? UINT64_MAX : ((uint64_t {1} << length) - 1u)); + return (value >> index) & mask; +} + +static uint64_t InsertBitField(uint64_t dst, uint64_t src, uint32_t length, uint32_t index) { + length &= 0x3fu; + index &= 0x3fu; + + if (length == 0) { + length = 64; + } + + if (index >= 64) { + return dst; + } + + auto available = 64u - index; + if (length > available) { + length = available; + } + + const uint64_t mask = (length == 64 ? UINT64_MAX : ((uint64_t {1} << length) - 1u)); + const uint64_t shifted = (index == 0 ? mask : (mask << index)); + const uint64_t src_shifted = (src & mask) << index; + + return (dst & ~shifted) | src_shifted; +} + +static bool TryEmulateSse4a(PCONTEXT context) { + if (context == nullptr) { + return false; + } + + const auto* rip = reinterpret_cast(context->Rip); + + const uint8_t prefix = rip[0]; + if (prefix != 0x66 && prefix != 0xf2) { + return false; + } + + size_t offset = 1; + uint8_t rex = 0; + if ((rip[offset] & 0xf0u) == 0x40u) { + rex = rip[offset]; + offset++; + } + + if (rip[offset] != 0x0f || rip[offset + 1] != 0x78) { + return false; + } + + auto modrm = rip[offset + 2]; + if ((modrm & 0xc0u) != 0xc0u) { + return false; + } + + const uint8_t reg = ((modrm >> 3u) & 0x07u) | ((rex & 0x04u) << 1u); + const uint8_t rm = (modrm & 0x07u) | ((rex & 0x01u) << 3u); + const uint8_t length = rip[offset + 3]; + const uint8_t index = rip[offset + 4]; + + // AMD SSE4a immediate-form EXTRQ/INSERTQ. PS5 code can execute these natively on AMD hardware, + // while Intel hosts raise an illegal-instruction exception. + if (prefix == 0x66) { + auto* dst = GetContextXmm(context, rm); + if (dst == nullptr) { + return false; + } + + dst->Low = ExtractBitField(dst->Low, length, index); + dst->High = 0; + context->Rip += offset + 5; + return true; + } + + auto* dst = GetContextXmm(context, reg); + auto* src = GetContextXmm(context, rm); + if (dst == nullptr || src == nullptr) { + return false; + } + + dst->Low = InsertBitField(dst->Low, src->Low, length, index); + context->Rip += offset + 5; + return true; +} + +static bool TryEmulateMonitorxMwaitx(PCONTEXT context) { + if (context == nullptr) { + return false; + } + + const auto* rip = reinterpret_cast(context->Rip); + if (rip[0] != 0x0f || rip[1] != 0x01 || (rip[2] != 0xfa && rip[2] != 0xfb)) { + return false; + } + + // AMD MONITORX/MWAITX are used by PS5 code in wait loops. Intel hosts can raise an illegal- + // instruction exception, so approximate them as a no-op/yield pair. + if (rip[2] == 0xfb) { + SwitchToThread(); + } + context->Rip += 3; + return true; +} + +#endif + +bool TryEmulate(void* native_context) { +#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS + auto* context = static_cast(native_context); + return TryEmulateMonitorxMwaitx(context) || TryEmulateSse4a(context); +#else + (void)native_context; + return false; +#endif +} + +} // namespace Loader::X64InstructionEmulator diff --git a/src/loader/x64InstructionEmulator.h b/src/loader/x64InstructionEmulator.h new file mode 100644 index 000000000..7fddd779a --- /dev/null +++ b/src/loader/x64InstructionEmulator.h @@ -0,0 +1,10 @@ +#ifndef KYTY_LOADER_X64_INSTRUCTION_EMULATOR_H_ +#define KYTY_LOADER_X64_INSTRUCTION_EMULATOR_H_ + +namespace Loader::X64InstructionEmulator { + +[[nodiscard]] bool TryEmulate(void* native_context); + +} // namespace Loader::X64InstructionEmulator + +#endif /* KYTY_LOADER_X64_INSTRUCTION_EMULATOR_H_ */ diff --git a/src/main.cpp b/src/main.cpp new file mode 100644 index 000000000..3cba5ae80 --- /dev/null +++ b/src/main.cpp @@ -0,0 +1,274 @@ +#include "common/common.h" +#include "common/commonSubsystem.h" +#include "common/dateTime.h" +#include "common/debug.h" +#include "common/file.h" +#include "common/magicEnum.h" +#include "common/platform/sysDbg.h" +#include "common/stringUtils.h" +#include "common/threads.h" +#include "emulator.h" +#include "kytyGitVersion.h" + +#include +#include + +using namespace Common; +using namespace Emulator; + +static std::string GetBuildString() { + Date date = Date::FromMacros(std::string(__DATE__)); + +#if KYTY_BUILD == KYTY_BUILD_DEBUG + std::string type = "Debug"; +#elif KYTY_BUILD == KYTY_BUILD_RELEASE + std::string type = "Release"; +#else + std::string type = "????"; +#endif + + std::string compiler = + Debug::GetCompiler() + "-" + Debug::GetLinker() + "-" + Debug::GetBitness(); + + std::string str = + fmt::format("{}, {}, ver = {}, git = {}, date = {}", type.c_str(), compiler.c_str(), + KYTY_VERSION, KYTY_GIT_VERSION, date.ToString().c_str()); + + return str; +} + +static void PrintUsage() { + ::printf("%s\n", GetBuildString().c_str()); + ::printf("kyty_emulator --game [options]\n\n"); + ::printf("Options:\n"); + ::printf(" --game Game directory or ELF to load.\n"); + ::printf(" --screen-width Window width. Default: 1280.\n"); + ::printf(" --screen-height Window height. Default: 720.\n"); + ::printf(" --vblank-frequency Virtual vblank frequency. Default: 60.\n"); + ::printf(" --vulkan-validation Enable Vulkan validation.\n"); + ::printf(" --shader-validation Enable shader validation.\n"); + ::printf(" --shader-optimization-type None, Size, or Performance.\n"); + ::printf(" --shader-log-direction Silent, Console, or File.\n"); + ::printf(" --shader-log-folder Shader log output folder.\n"); + ::printf(" --command-buffer-dump Enable command buffer dumps.\n"); + ::printf(" --command-buffer-dump-folder Command buffer dump folder.\n"); + ::printf(" --graphics-debug-dump Enable graphics debug dumps.\n"); + ::printf(" --printf-direction Silent, Console, or File.\n"); + ::printf(" --printf-output-file Guest printf output file.\n"); + ::printf(" --profiler-direction None or Network.\n"); + ::printf(" --spirv-debug-printf Enable SPIR-V debug printf.\n"); + ::printf(" --pipeline-dump Enable pipeline dumps.\n"); + ::printf(" --pipeline-dump-folder Pipeline dump folder.\n"); + ::printf(" --ngg-rectlist-draw Draw rect-list auto draws using the NGG " + "4-vertex path.\n"); + ::printf(" --rd Enable RenderDoc capture.\n"); +} + +static bool NextArg(int argc, char* argv[], int& index, std::string& out) { + if (index + 1 >= argc) { + return false; + } + + index++; + out = argv[index]; + return true; +} + +static bool ParseBool(const std::string& value, bool& out) { + if (Common::EqualNoCase(value, "true") || value == "1" || Common::EqualNoCase(value, "yes") || + Common::EqualNoCase(value, "on")) { + out = true; + return true; + } + + if (Common::EqualNoCase(value, "false") || value == "0" || Common::EqualNoCase(value, "no") || + Common::EqualNoCase(value, "off")) { + out = false; + return true; + } + + return false; +} + +template +static bool ParseEnum(const std::string& value, E& out) { + auto enum_value = magic_enum::enum_cast(value.c_str()); + if (!enum_value.has_value()) { + return false; + } + + out = enum_value.value(); + return true; +} + +static bool ParseArgs(int argc, char* argv[], RunOptions& options, bool& show_help) { + show_help = false; + + for (int i = 1; i < argc; i++) { + std::string arg = std::string(argv[i]); + std::string value; + + if (arg == "--help" || arg == "-h") { + show_help = true; + continue; + } + + if (arg == "--rd") { + options.config.renderdoc_enabled = true; + continue; + } + + if (!Common::StartsWith(arg, "--")) { + ::printf("game input must be provided with --game\n"); + return false; + } + + if (!NextArg(argc, argv, i, value)) { + ::printf("missing value for %s\n", arg.c_str()); + return false; + } + + if (arg == "--game") { + if (!options.app0_dir.empty()) { + ::printf("--game can only be specified once\n"); + return false; + } + + value = Common::FixFilenameSlash(value); + if (Common::File::IsDirectoryExisting(value)) { + options.app0_dir = value; + options.elf = "/app0/eboot.bin"; + } else if (Common::File::IsFileExisting(value)) { + options.app0_dir = Common::DirectoryWithoutFilename(value); + if (options.app0_dir.empty()) { + options.app0_dir = "."; + } + options.elf = "/app0/" + Common::FilenameWithoutDirectory(value); + } else { + ::printf("--game must point to an existing directory or ELF: %s\n", value.c_str()); + return false; + } + } else if (arg == "--screen-width") { + options.config.screen_width = static_cast(Common::ToInt32(value)); + } else if (arg == "--screen-height") { + options.config.screen_height = static_cast(Common::ToInt32(value)); + } else if (arg == "--vblank-frequency") { + const int32_t vblank_frequency = Common::ToInt32(value); + options.config.vblank_frequency = + static_cast(vblank_frequency < 0 ? 0 : vblank_frequency); + } else if (arg == "--vulkan-validation") { + if (!ParseBool(value, options.config.vulkan_validation_enabled)) { + ::printf("invalid boolean for %s: %s\n", arg.c_str(), value.c_str()); + return false; + } + } else if (arg == "--shader-validation") { + if (!ParseBool(value, options.config.shader_validation_enabled)) { + ::printf("invalid boolean for %s: %s\n", arg.c_str(), value.c_str()); + return false; + } + } else if (arg == "--shader-optimization-type") { + if (!ParseEnum(value, options.config.shader_optimization_type)) { + ::printf("invalid shader optimization type: %s\n", value.c_str()); + return false; + } + } else if (arg == "--shader-log-direction") { + if (!ParseEnum(value, options.config.shader_log_direction)) { + ::printf("invalid shader log direction: %s\n", value.c_str()); + return false; + } + } else if (arg == "--shader-log-folder") { + options.config.shader_log_folder = value; + } else if (arg == "--command-buffer-dump") { + if (!ParseBool(value, options.config.command_buffer_dump_enabled)) { + ::printf("invalid boolean for %s: %s\n", arg.c_str(), value.c_str()); + return false; + } + } else if (arg == "--command-buffer-dump-folder") { + options.config.command_buffer_dump_folder = value; + } else if (arg == "--graphics-debug-dump") { + if (!ParseBool(value, options.config.graphics_debug_dump_enabled)) { + ::printf("invalid boolean for %s: %s\n", arg.c_str(), value.c_str()); + return false; + } + } else if (arg == "--printf-direction") { + if (!ParseEnum(value, options.config.printf_direction)) { + ::printf("invalid printf direction: %s\n", value.c_str()); + return false; + } + } else if (arg == "--printf-output-file") { + options.config.printf_output_file = value; + } else if (arg == "--profiler-direction") { + if (!ParseEnum(value, options.config.profiler_direction)) { + ::printf("invalid profiler direction: %s\n", value.c_str()); + return false; + } + } else if (arg == "--spirv-debug-printf") { + if (!ParseBool(value, options.config.spirv_debug_printf_enabled)) { + ::printf("invalid boolean for %s: %s\n", arg.c_str(), value.c_str()); + return false; + } + } else if (arg == "--pipeline-dump") { + if (!ParseBool(value, options.config.pipeline_dump_enabled)) { + ::printf("invalid boolean for %s: %s\n", arg.c_str(), value.c_str()); + return false; + } + } else if (arg == "--pipeline-dump-folder") { + options.config.pipeline_dump_folder = value; + } else if (arg == "--ngg-rectlist-draw") { + if (!ParseBool(value, options.config.ngg_rectlist_draw_enabled)) { + ::printf("invalid boolean for %s: %s\n", arg.c_str(), value.c_str()); + return false; + } + } else { + ::printf("unknown option: %s\n", arg.c_str()); + return false; + } + } + + return show_help || (!options.app0_dir.empty() && !options.elf.empty()); +} + +int main(int argc, char* argv[]) { + auto& slist = *SubsystemsList::Instance(); + + slist.SetArgs(argc, argv); + + auto* core = CommonSubsystem::Instance(); + auto* threads = ThreadsSubsystem::Instance(); + + slist.Add(core, {}); + slist.Add(threads, {core}); + + if (!slist.InitAll(false)) { + ::printf("Failed to initialize '%s' subsystem: %s\n", slist.GetFailName(), + slist.GetFailMsg()); + return 1; + } + + RunOptions options; + bool show_help = false; + + if (argc < 2) { + PrintUsage(); + slist.DestroyAll(false); + return 0; + } + + if (!ParseArgs(argc, argv, options, show_help)) { + PrintUsage(); + slist.DestroyAll(false); + return 1; + } + + if (show_help) { + PrintUsage(); + slist.DestroyAll(false); + return 0; + } + + Run(options); + + slist.DestroyAll(false); + + return 0; +} diff --git a/src/utils.cmake b/src/utils.cmake new file mode 100644 index 000000000..778d54f07 --- /dev/null +++ b/src/utils.cmake @@ -0,0 +1,131 @@ +function(include_what_you_use target dirs) + if (NOT LINUX AND CLANG AND ("${target}" IN_LIST KYTY_IWYU)) + find_program (CLANG_IWYU_EXE NAMES "include-what-you-use") + if (CLANG_IWYU_EXE) + set_target_properties(${target} PROPERTIES CXX_INCLUDE_WHAT_YOU_USE "${CLANG_IWYU_EXE};-Xiwyu;--cxx17ns;-Qunused-arguments;-Werror") + endif() + endif() +endfunction() + +function(include_what_you_use_with_mappings target dirs mappings) + if (NOT LINUX AND CLANG AND ("${target}" IN_LIST KYTY_IWYU)) + find_program (CLANG_IWYU_EXE NAMES "include-what-you-use") + if (CLANG_IWYU_EXE) + foreach(map ${mappings}) + list(APPEND mapdirs ";-Xiwyu;--mapping_file=${map}") + endforeach() + set_target_properties(${target} PROPERTIES CXX_INCLUDE_WHAT_YOU_USE "${CLANG_IWYU_EXE};${mapdirs};-Xiwyu;--cxx17ns;-Qunused-arguments;-Werror") + endif() + endif() +endfunction() + +function(clang_tidy_check target config headers dirs) + if (CLANG AND ("${target}" IN_LIST KYTY_CLANG_TYDY) AND NOT KYTY_CLANG_CL) + find_program (CLANG_TIDY_EXE NAMES "clang-tidy") + if (CLANG_TIDY_EXE) + set(std_arg "-extra-arg=-std=c++${CMAKE_CXX_STANDARD}") + foreach(dir ${dirs}) + list(APPEND incdirs "-extra-arg=-I${dir}") + endforeach() + foreach(header ${headers}) + list(APPEND filter "(${header}.*)") + endforeach() + string(REPLACE ";" "|" filter "${filter}") + if ("${config}" STREQUAL "") + set_target_properties(${target} PROPERTIES CXX_CLANG_TIDY "${CLANG_TIDY_EXE};-warnings-as-errors=*;-header-filter=${filter};${std_arg};${incdirs}") + else() + set_target_properties(${target} PROPERTIES CXX_CLANG_TIDY "${CLANG_TIDY_EXE};-config=${config};-warnings-as-errors=*;-header-filter=${filter};${std_arg};${incdirs}") + endif() + endif() + endif() +endfunction() + +function(clang_tidy_fix target config headers dirs) + if (CLANG AND ("${target}" IN_LIST KYTY_CLANG_TYDY)) + find_program (CLANG_TIDY_EXE NAMES "clang-tidy") + if (CLANG_TIDY_EXE) + set(std_arg "-extra-arg=-std=c++${CMAKE_CXX_STANDARD}") + foreach(dir ${dirs}) + list(APPEND incdirs "-extra-arg=-I${dir}") + endforeach() + foreach(header ${headers}) + list(APPEND filter "(${header}.*)") + endforeach() + string(REPLACE ";" "|" filter "${filter}") + if ("${config}" STREQUAL "") + set_target_properties(${target} PROPERTIES CXX_CLANG_TIDY "${CLANG_TIDY_EXE};-fix;-format-style=file;-header-filter=${filter};${std_arg};${incdirs}") + else() + set_target_properties(${target} PROPERTIES CXX_CLANG_TIDY "${CLANG_TIDY_EXE};-fix;-format-style=file;-config=${config};-header-filter=${filter};${std_arg};${incdirs}") + endif() + endif() + endif() +endfunction() + +macro(config_compiler_and_linker) + +set(KYTY_WARNINGS_ARE_ERRORS OFF) + +set(KYTY_C_FLAGS "") +set(KYTY_CPP_FLAGS "") + +SET(CMAKE_NINJA_FORCE_RESPONSE_FILE 1 CACHE INTERNAL "") + +if(KYTY_CLANG_CL) + if(CMAKE_CXX_FLAGS MATCHES "/W[0-4]") + string(REGEX REPLACE "/W[0-4]" "/W3" CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS}") + else() + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /W3") + endif() + + string(REGEX REPLACE "/MD" "/MT" CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG}") + string(REGEX REPLACE "/MD" "/MT" CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE}") + string(REGEX REPLACE "/MD" "/MT" CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS}") + string(REGEX REPLACE "/MD" "/MT" CMAKE_C_FLAGS_DEBUG "${CMAKE_C_FLAGS_DEBUG}") + string(REGEX REPLACE "/MD" "/MT" CMAKE_C_FLAGS_RELEASE "${CMAKE_C_FLAGS_RELEASE}") + string(REGEX REPLACE "/MD" "/MT" CMAKE_C_FLAGS "${CMAKE_C_FLAGS}") + + set(KYTY_CPP_FLAGS "${KYTY_CPP_FLAGS} /utf-8 /Oy- /wd4244 /wd4305 /wd4800 /wd4345") + + add_link_options("$<$:/OPT:NOREF>") + + if(KYTY_WARNINGS_ARE_ERRORS) + #set(KYTY_CPP_FLAGS "${KYTY_CPP_FLAGS} /WX") + add_compile_options(/WX) + endif() + + add_compile_options("$<$:/O2>") + add_compile_options("$<$:/DNDEBUG>") + add_compile_options("$<$:/O2>") + add_compile_options("$<$:/DNDEBUG>") + + set(KYTY_C_FLAGS "${KYTY_CPP_FLAGS}") + +elseif(CLANG OR GCC) + + if (CLANG) + set(KYTY_CPP_FLAGS "${KYTY_CPP_FLAGS} -fno-rtti -fno-exceptions -fcolor-diagnostics -finput-charset=UTF-8 -fexec-charset=UTF-8 -g -fno-strict-aliasing -fno-omit-frame-pointer -Wall -fmessage-length=0") + if (WIN32) + set(KYTY_CPP_FLAGS "${KYTY_CPP_FLAGS} -static") + endif() + else() + set(KYTY_CPP_FLAGS "${KYTY_CPP_FLAGS} -fno-exceptions -fdiagnostics-color=always -finput-charset=UTF-8 -fexec-charset=UTF-8 -static-libgcc -static-libstdc++ -g -fno-strict-aliasing -fno-omit-frame-pointer -Wall -Wno-unused-value -fmessage-length=0") + endif() + + if(KYTY_WARNINGS_ARE_ERRORS) + #set(KYTY_CPP_FLAGS "${KYTY_CPP_FLAGS} -Werror") + add_compile_options(-Werror) + endif() + + add_link_options("-g") + + unset(CMAKE_CXX_STANDARD_LIBRARIES CACHE) + unset(CMAKE_C_STANDARD_LIBRARIES CACHE) + + set(KYTY_C_FLAGS "${KYTY_CPP_FLAGS}") + +endif() + +set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${KYTY_C_FLAGS}") +set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${KYTY_CPP_FLAGS}") + +endmacro() diff --git a/tests/MemoryTrackerTests.cpp b/tests/MemoryTrackerTests.cpp new file mode 100644 index 000000000..adf44a073 --- /dev/null +++ b/tests/MemoryTrackerTests.cpp @@ -0,0 +1,1042 @@ +#include "graphics/host_gpu/memoryTracker.h" +#include "graphics/host_gpu/rangeSet.h" +#include "common/assert.h" + +#include +#include +#include +#include +#include +#include +#include +#include + +#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS +#ifndef NOMINMAX +#define NOMINMAX +#endif +#include +#undef min +#undef max +#endif + +namespace { + +using Libs::Graphics::DirtySource; +using Libs::Graphics::CpuFaultAction; +using Libs::Graphics::MemoryTracker; +using Libs::Graphics::PageFaultAccess; +using Libs::Graphics::PageFaultPhase; +using Libs::Graphics::PageManager; +using Libs::Graphics::PageWatchMode; +using Libs::Graphics::RegionManager; +using Libs::Graphics::RangeSet; + +void Check(bool value, const char *text) { + if (!value) { + std::fprintf(stderr, "MemoryTrackerTests: failed: %s\n", text); + std::abort(); + } +} + +#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS +bool IsWritable(const void *address) { + MEMORY_BASIC_INFORMATION info{}; + Check(VirtualQuery(address, &info, sizeof(info)) != 0, "VirtualQuery failed"); + return info.Protect == PAGE_READWRITE; +} + +uint32_t Protection(const void *address) { + MEMORY_BASIC_INFORMATION info{}; + Check(VirtualQuery(address, &info, sizeof(info)) != 0, "VirtualQuery failed"); + return info.Protect; +} + +class SharedPage final { + public: + SharedPage(uintptr_t address, uint64_t size) { + mapping_ = CreateFileMappingA(INVALID_HANDLE_VALUE, nullptr, PAGE_READWRITE, + static_cast(size >> 32u), + static_cast(size), nullptr); + Check(mapping_ != nullptr, "CreateFileMapping failed"); + guest = static_cast(MapViewOfFileEx( + mapping_, FILE_MAP_ALL_ACCESS, 0, 0, size, + reinterpret_cast(address))); + Check(guest == reinterpret_cast(address), + "fixed shared view failed"); + backing = static_cast( + MapViewOfFile(mapping_, FILE_MAP_ALL_ACCESS, 0, 0, size)); + Check(backing != nullptr, "shared backing view failed"); + } + + ~SharedPage() { + Check(UnmapViewOfFile(backing) != 0, "shared backing unmap failed"); + Check(UnmapViewOfFile(guest) != 0, "shared guest unmap failed"); + Check(CloseHandle(mapping_) != 0, "shared mapping close failed"); + } + + SharedPage(const SharedPage &) = delete; + SharedPage &operator=(const SharedPage &) = delete; + + uint8_t *guest = nullptr; + uint8_t *backing = nullptr; + + private: + HANDLE mapping_ = nullptr; +}; + +bool DummyFault(void *, PageFaultAccess, uint64_t, uint64_t, PageFaultPhase) noexcept { + return true; +} + +struct TrackerHarness { + static bool Fault(void *context, PageFaultAccess access, uint64_t vaddr, uint64_t size, + PageFaultPhase phase) noexcept { + auto *self = static_cast(context); + if (self == nullptr || self->target == nullptr) { + EXIT("memory-tracker test fault has no target\n"); + } + return self->discard_virtual + ? self->target->InvalidateVirtualGpuWrite(access, vaddr, size, phase) + : self->target->InvalidateRegion(vaddr, size, phase); + } + + explicit TrackerHarness( + PageWatchMode gpu_watch_mode = PageWatchMode::ReadWrite) + : page_manager(Fault, this), tracker(page_manager, gpu_watch_mode) { + target = &tracker; + } + + MemoryTracker *target = nullptr; + bool discard_virtual = false; + PageManager page_manager; + MemoryTracker tracker; +}; + +struct SharedTrackerHarness { + static bool Fault(void *context, PageFaultAccess, uint64_t vaddr, uint64_t size, + PageFaultPhase phase) noexcept { + auto *self = static_cast(context); + if (self == nullptr) { + EXIT("shared memory-tracker test fault has no harness\n"); + } + const bool first = self->first.InvalidateRegion(vaddr, size, phase); + const bool second = self->second.InvalidateRegion(vaddr, size, phase); + return first || second; + } + + SharedTrackerHarness() + : page_manager(Fault, this), first(page_manager), second(page_manager) {} + + PageManager page_manager; + MemoryTracker first; + MemoryTracker second; +}; + +struct SharedMetadataImageHarness { + static bool Fault(void *context, PageFaultAccess access, uint64_t vaddr, + uint64_t size, PageFaultPhase phase) noexcept { + auto *self = static_cast(context); + if (self == nullptr || + (access != PageFaultAccess::Read && access != PageFaultAccess::Write)) { + EXIT("shared metadata/image test received an invalid fault\n"); + } + const bool metadata = access == PageFaultAccess::Write && + self->metadata.InvalidateVirtualGpuWrite( + access, vaddr, size, phase); + const bool image = self->image.InvalidateRegion(vaddr, size, phase); + return metadata || image; + } + + SharedMetadataImageHarness() + : page_manager(Fault, this), image(page_manager), + metadata(page_manager, PageWatchMode::Write) {} + + PageManager page_manager; + MemoryTracker image; + MemoryTracker metadata; +}; + +struct SplitTrackerHarness { + static bool Fault(void *context, PageFaultAccess access, uint64_t vaddr, + uint64_t size, PageFaultPhase phase) noexcept { + auto *self = static_cast(context); + if (self == nullptr) { + EXIT("split memory-tracker test fault has no harness\n"); + } + if (phase == PageFaultPhase::Release) { + return true; + } + const bool buffer = self->buffer.InvalidateRegion(vaddr, size, phase); + const bool metadata = self->metadata.InvalidateVirtualGpuWrite( + access, vaddr, size, phase); + const bool image = self->image.InvalidateRegion(vaddr, size, phase); + if (static_cast(buffer) + static_cast(metadata) + + static_cast(image) != + 1) { + EXIT("split memory-tracker fault matched multiple owners\n"); + } + return true; + } + + SplitTrackerHarness() + : page_manager(Fault, this), buffer(page_manager), image(page_manager), + metadata(page_manager, PageWatchMode::Write) {} + + PageManager page_manager; + MemoryTracker buffer; + MemoryTracker image; + MemoryTracker metadata; +}; + +struct DownloadTrackerHarness { + static bool Fault(void *context, PageFaultAccess access, uint64_t vaddr, + uint64_t size, PageFaultPhase phase) noexcept { + auto *self = static_cast(context); + if (self == nullptr) { + EXIT("download memory-tracker test fault has no harness\n"); + } + if (phase == PageFaultPhase::Invalidate) { + if (self->pending_access != PageFaultAccess::Unknown) { + EXIT("download memory-tracker test has an overlapping request\n"); + } + const auto action = self->tracker.BeginCpuFault(vaddr, size); + if (action == CpuFaultAction::Download) { + self->pending_access = access; + } + return action != CpuFaultAction::Untracked; + } + if (phase == PageFaultPhase::Release) { + return true; + } + const bool downloaded = self->pending_access != PageFaultAccess::Unknown; + if (downloaded) { + if (self->pending_access != access || self->download_data.empty()) { + EXIT("download memory-tracker test has invalid completion state\n"); + } + if (self->backing == nullptr || + self->download_address < self->guest_address) { + EXIT("download memory-tracker test has no backing alias\n"); + } + std::memcpy(self->backing + self->download_address - self->guest_address, + self->download_data.data(), self->download_data.size()); + } + const bool completed = + self->tracker.CompleteCpuFault(vaddr, size, access, downloaded); + self->pending_access = PageFaultAccess::Unknown; + return completed; + } + + DownloadTrackerHarness() : page_manager(Fault, this), tracker(page_manager) {} + + PageFaultAccess pending_access = PageFaultAccess::Unknown; + uint64_t download_address = 0; + uint64_t guest_address = 0; + uint8_t *backing = nullptr; + std::vector download_data; + PageManager page_manager; + MemoryTracker tracker; +}; + +std::atomic g_native_page_manager{nullptr}; +std::atomic_bool g_native_fault_entered{false}; +std::atomic_bool g_unmap_contended{false}; + +void UnmapContended() noexcept { + g_unmap_contended.store(true, std::memory_order_release); +} + +LONG CALLBACK NativeTrackerFaultHandler(EXCEPTION_POINTERS *exception) { + if (exception == nullptr || exception->ExceptionRecord == nullptr || + exception->ExceptionRecord->ExceptionCode != EXCEPTION_ACCESS_VIOLATION) { + return EXCEPTION_CONTINUE_SEARCH; + } + const auto operation = exception->ExceptionRecord->ExceptionInformation[0]; + const auto access = operation == 0 ? PageFaultAccess::Read + : operation == 1 ? PageFaultAccess::Write + : operation == 8 ? PageFaultAccess::Execute + : PageFaultAccess::Unknown; + auto *page_manager = g_native_page_manager.load(std::memory_order_acquire); + if (page_manager == nullptr) { + return EXCEPTION_CONTINUE_SEARCH; + } + g_native_fault_entered.store(true, std::memory_order_release); + return page_manager->HandleFault( + access, exception->ExceptionRecord->ExceptionInformation[1]) + ? EXCEPTION_CONTINUE_EXECUTION + : EXCEPTION_CONTINUE_SEARCH; +} + +void TestPendingFaultBlocksUploadConsumption() { + constexpr uintptr_t base = 0x0000000200010000ull; + constexpr uint64_t region_size = 4ull * 1024ull * 1024ull; + PageManager page_manager(DummyFault, nullptr); + const auto page_size = page_manager.GetPageSize(); + auto *memory = static_cast( + VirtualAlloc(reinterpret_cast(base), page_size, + MEM_RESERVE | MEM_COMMIT, PAGE_READWRITE)); + Check(memory == reinterpret_cast(base), "fixed VirtualAlloc failed"); + const auto address = reinterpret_cast(memory); + page_manager.OnGpuMap(address, page_size); + RegionManager region(page_manager, address & ~(region_size - 1)); + + Libs::Graphics::RegionBits changed; + { + std::scoped_lock lock(region.lock); + region.Track(address, page_size); + changed = region.ForEachModifiedRange( + address, page_size, [](uint64_t, uint64_t) noexcept {}); + changed = region.ChangeState(address, page_size); + Check(region.BeginCpuFault(address, 1) == CpuFaultAction::Continue, + "fault ownership rejected an already CPU-dirty page"); + } + page_manager.UpdatePageWatchers(false, address, page_size); + + uint32_t ranges = 0; + { + std::scoped_lock lock(region.lock); + const auto pending_change = + region.ForEachModifiedRange( + address, page_size, [&](uint64_t, uint64_t) noexcept { ranges++; }); + Check(ranges == 0 && pending_change.none() && + region.IsModified(address - region.GetCpuAddr(), + page_size), + "pending fault page was consumed by upload"); + Check(region.CompleteCpuFault(address, 1, PageFaultAccess::Write, false), + "fault completion was not recorded"); + changed = region.ForEachModifiedRange( + address, page_size, [&](uint64_t, uint64_t) noexcept { ranges++; }); + } + Check(ranges == 1, "completed fault page was not available to upload"); + { + std::scoped_lock lock(region.lock); + changed = region.ChangeState(address, page_size); + } + region.ApplyProtection(changed, false); + page_manager.OnGpuUnmap(address, page_size); + Check(VirtualFree(memory, 0, MEM_RELEASE) != 0, "VirtualFree failed"); +} + +void TestCleanReadFaultPreservesCpuState() { + constexpr uintptr_t base = 0x0000000200010000ull; + PageManager page_manager(DummyFault, nullptr); + MemoryTracker tracker(page_manager); + const auto page_size = page_manager.GetPageSize(); + auto *memory = static_cast(VirtualAlloc( + reinterpret_cast(base), page_size, MEM_RESERVE | MEM_COMMIT, + PAGE_READWRITE)); + Check(memory == reinterpret_cast(base), "fixed VirtualAlloc failed"); + const auto address = reinterpret_cast(memory); + page_manager.OnGpuMap(address, page_size); + tracker.ForEachUploadRange(address, page_size, false, + [](uint64_t, uint64_t) noexcept {}, + []() noexcept {}); + Check(!tracker.IsRegionCpuModified(address, page_size) && + !tracker.IsRegionGpuModified(address, page_size), + "clean read-fault setup retained dirty ownership"); + Check(tracker.BeginCpuFault(address, 1, PageFaultAccess::Read) == + CpuFaultAction::Continue, + "clean tracked read fault was not accepted"); + Check(tracker.CompleteCpuFault(address, 1, PageFaultAccess::Read, false), + "clean tracked read fault did not complete"); + Check(!tracker.IsRegionCpuModified(address, page_size) && + !tracker.IsRegionGpuModified(address, page_size), + "clean read fault incorrectly transferred write ownership to the CPU"); + tracker.UntrackMemory(address, page_size); + page_manager.OnGpuUnmap(address, page_size); + Check(VirtualFree(memory, 0, MEM_RELEASE) != 0, "VirtualFree failed"); +} + +void TestGpuDownloadFaultOwnership() { + constexpr uintptr_t base = 0x0000000200010000ull; + DownloadTrackerHarness harness; + const auto page_size = harness.page_manager.GetPageSize(); + SharedPage shared(base, page_size); + auto *memory = shared.guest; + const auto address = reinterpret_cast(memory); + harness.guest_address = address; + harness.backing = shared.backing; + harness.page_manager.OnGpuMap(address, page_size); + harness.tracker.ForEachUploadRange( + address, page_size, true, [](uint64_t, uint64_t) noexcept {}, + []() noexcept {}); + harness.download_address = address + 32; + harness.download_data = {0x11, 0x22, 0x33, 0x44}; + Check(harness.page_manager.HandleFault(PageFaultAccess::Read, address + 32), + "GPU-dirty read fault was not handled"); + Check(std::memcmp(memory + 32, harness.download_data.data(), 4) == 0 && + !harness.tracker.IsRegionGpuModified(address, page_size) && + !harness.tracker.IsRegionCpuModified(address, page_size) && + Protection(memory) == PAGE_READONLY && + harness.page_manager.IsTracked(address), + "GPU readback did not leave a clean write-watched page"); + + Check(harness.page_manager.HandleFault(PageFaultAccess::Write, address + 32), + "write watcher was not preserved after GPU readback"); + Check(harness.tracker.IsRegionCpuModified(address, page_size) && IsWritable(memory), + "post-read CPU write did not claim CPU ownership"); + + harness.tracker.ForEachUploadRange( + address, page_size, true, [](uint64_t, uint64_t) noexcept {}, + []() noexcept {}); + harness.download_data = {0xaa, 0xbb, 0xcc, 0xdd}; + Check(harness.page_manager.HandleFault(PageFaultAccess::Write, address + 33), + "GPU-dirty write fault was not handled"); + Check(std::memcmp(memory + 32, harness.download_data.data(), 4) == 0 && + !harness.tracker.IsRegionGpuModified(address, page_size) && + harness.tracker.IsRegionCpuModified(address, page_size) && IsWritable(memory), + "GPU write fault did not download before granting CPU ownership"); + harness.tracker.UnmapMemory(address, page_size); +} + +void TestVirtualGpuWriteDiscard() { + constexpr uintptr_t base = 0x0000000200010000ull; + TrackerHarness harness(PageWatchMode::Write); + harness.discard_virtual = true; + const auto page_size = harness.page_manager.GetPageSize(); + auto *memory = static_cast( + VirtualAlloc(reinterpret_cast(base), page_size, + MEM_RESERVE | MEM_COMMIT, PAGE_READWRITE)); + Check(memory == reinterpret_cast(base), "fixed VirtualAlloc failed"); + const auto address = reinterpret_cast(memory); + memory[8] = 0x5a; + harness.page_manager.OnGpuMap(address, page_size); + harness.tracker.ForEachUploadRange( + address, page_size, true, [](uint64_t, uint64_t) noexcept {}, + []() noexcept {}); + Check(Protection(memory) == PAGE_READONLY && memory[8] == 0x5a && + harness.tracker.IsRegionGpuModified(address, page_size), + "virtual GPU ownership did not preserve authoritative backing reads"); + Check(harness.page_manager.HandleFault(PageFaultAccess::Write, address + 8), + "virtual GPU write fault was not discarded"); + Check(!harness.tracker.IsRegionGpuModified(address, page_size) && + harness.tracker.IsRegionCpuModified(address, page_size) && IsWritable(memory), + "virtual GPU discard did not transfer the page to CPU ownership"); + harness.tracker.UnmapMemory(address, page_size); + Check(VirtualFree(memory, 0, MEM_RELEASE) != 0, "VirtualFree failed"); +} + +void TestSameSlabTrackerArbitration() { + constexpr uintptr_t base = 0x0000000200010000ull; + SplitTrackerHarness harness; + const auto page_size = harness.page_manager.GetPageSize(); + auto *memory = static_cast(VirtualAlloc( + reinterpret_cast(base), page_size * 3, + MEM_RESERVE | MEM_COMMIT, PAGE_READWRITE)); + Check(memory == reinterpret_cast(base), "fixed VirtualAlloc failed"); + const auto address = reinterpret_cast(memory); + harness.page_manager.OnGpuMap(address, page_size * 3); + + harness.buffer.ForEachUploadRange( + address, page_size, false, [](uint64_t, uint64_t) noexcept {}, + []() noexcept {}); + harness.image.ForEachUploadRange( + address + page_size, page_size, false, + [](uint64_t, uint64_t) noexcept {}, []() noexcept {}); + harness.metadata.ForEachUploadRange( + address + page_size * 2, page_size, true, + [](uint64_t, uint64_t) noexcept {}, []() noexcept {}); + + Check(harness.buffer.BeginCpuFault(address + page_size * 2, 1) == + CpuFaultAction::Untracked && + harness.image.BeginCpuFault(address + page_size * 2, 1) == + CpuFaultAction::Untracked, + "unrelated same-slab trackers claimed the metadata page"); + Check(harness.page_manager.HandleFault(PageFaultAccess::Write, + address + page_size * 2 + 8), + "metadata write fault was not exclusively handled"); + Check(harness.page_manager.HandleFault(PageFaultAccess::Write, address + 8), + "buffer write fault was not exclusively handled"); + Check(harness.page_manager.HandleFault(PageFaultAccess::Write, + address + page_size + 8), + "image write fault was not exclusively handled"); + Check(harness.buffer.IsRegionCpuModified(address, page_size) && + harness.image.IsRegionCpuModified(address + page_size, page_size) && + harness.metadata.IsRegionCpuModified(address + page_size * 2, + page_size) && + IsWritable(memory) && IsWritable(memory + page_size) && + IsWritable(memory + page_size * 2), + "exclusive same-slab faults did not transfer exact CPU ownership"); + + harness.buffer.UntrackMemory(address, page_size); + harness.image.UntrackMemory(address + page_size, page_size); + harness.metadata.UntrackMemory(address + page_size * 2, page_size); + harness.page_manager.OnGpuUnmap(address, page_size * 3); + Check(VirtualFree(memory, 0, MEM_RELEASE) != 0, "VirtualFree failed"); +} + +void TestSharedMetadataAndImagePageFault() { + constexpr uintptr_t base = 0x0000000200010000ull; + SharedMetadataImageHarness harness; + const auto page_size = harness.page_manager.GetPageSize(); + auto *memory = static_cast(VirtualAlloc( + reinterpret_cast(base), page_size, MEM_RESERVE | MEM_COMMIT, + PAGE_READWRITE)); + Check(memory == reinterpret_cast(base), "fixed VirtualAlloc failed"); + const auto address = reinterpret_cast(memory); + harness.page_manager.OnGpuMap(address, page_size); + harness.metadata.ForEachUploadRange( + address, page_size, true, [](uint64_t, uint64_t) noexcept {}, + []() noexcept {}); + harness.image.ForEachUploadRange( + address, page_size, false, [](uint64_t, uint64_t) noexcept {}, + []() noexcept {}); + Check(Protection(memory) == PAGE_READONLY && + harness.metadata.IsRegionGpuModified(address, page_size) && + !harness.image.IsRegionCpuModified(address, page_size), + "shared metadata/image page was not write-watched"); + Check(harness.page_manager.HandleFault(PageFaultAccess::Write, address + 8), + "shared metadata/image write fault was not handled"); + Check(!harness.metadata.IsRegionGpuModified(address, page_size) && + harness.metadata.IsRegionCpuModified(address, page_size) && + harness.image.IsRegionCpuModified(address, page_size) && + IsWritable(memory), + "shared write fault did not invalidate both native trackers"); + harness.metadata.UntrackMemory(address, page_size); + harness.image.UntrackMemory(address, page_size); + harness.page_manager.OnGpuUnmap(address, page_size); + Check(VirtualFree(memory, 0, MEM_RELEASE) != 0, "VirtualFree failed"); +} + +void TestRangeSet() { + RangeSet ranges; + ranges.Add(0x1000, 0x80); + ranges.Add(0x1080, 0x80); + ranges.Add(0x1200, 0x40); + auto intersections = ranges.Intersections(0x1070, 0x1b0); + Check(intersections.size() == 2 && intersections[0].address == 0x1070 && + intersections[0].size == 0x90 && intersections[1].address == 0x1200 && + intersections[1].size == 0x20, + "range set did not merge and intersect exact byte ranges"); + ranges.Subtract(0x1040, 0x1e0); + intersections = ranges.Intersections(0x1000, 0x300); + Check(intersections.size() == 2 && intersections[0].address == 0x1000 && + intersections[0].size == 0x40 && intersections[1].address == 0x1220 && + intersections[1].size == 0x20, + "range set subtraction did not preserve both exact tails"); +} + +void TestCpuDirtyUploadAndFault() { + constexpr uintptr_t base = 0x0000000200010000ull; + TrackerHarness harness; + auto &tracker = harness.tracker; + auto &page_manager = harness.page_manager; + const auto page_size = page_manager.GetPageSize(); + auto *memory = static_cast( + VirtualAlloc(reinterpret_cast(base), page_size * 2, + MEM_RESERVE | MEM_COMMIT, PAGE_READWRITE)); + Check(memory == reinterpret_cast(base), "fixed VirtualAlloc failed"); + const auto address = reinterpret_cast(memory); + page_manager.OnGpuMap(address, page_size * 2); + Check(tracker.IsRegionCpuModified(address + 16, 32), + "new region was not CPU dirty"); + + uint32_t ranges = 0; + bool uploaded = false; + tracker.ForEachUploadRange( + address + 16, 32, false, + [&](uint64_t upload_addr, uint64_t upload_size) noexcept { + Check(upload_addr == address && upload_size == page_size, + "upload range was not page aligned"); + ranges++; + }, + [&]() noexcept { uploaded = true; }); + Check(ranges == 1 && uploaded && + !tracker.IsRegionCpuModified(address + 16, 32) && + !IsWritable(memory), + "upload did not clear CPU dirty state and arm protection"); + + Check(page_manager.HandleFault(PageFaultAccess::Write, address + 24), + "tracked CPU write fault was not handled"); + Check(tracker.IsRegionCpuModified(address + 16, 32) && IsWritable(memory), + "fault did not restore CPU dirty state and write access"); + tracker.ForEachUploadRange( + address + 16, 32, false, [](uint64_t, uint64_t) noexcept {}, + []() noexcept {}); + Check(!tracker.IsRegionCpuModified(address + 16, 32) && !IsWritable(memory), + "fault owner state did not support a balanced rearm"); + tracker.MarkRegionAsCpuModified(address + 16, 32); + Check(IsWritable(memory), + "explicit CPU dirty transition did not release the rearmed watch"); + + tracker.UnmapMemory(address, page_size * 2); + Check(VirtualFree(memory, 0, MEM_RELEASE) != 0, "VirtualFree failed"); +} + +void TestFaultDuringUploadRemainsDirty() { + constexpr uintptr_t base = 0x0000000200010000ull; + TrackerHarness harness; + auto &tracker = harness.tracker; + auto &page_manager = harness.page_manager; + const auto page_size = page_manager.GetPageSize(); + auto *memory = static_cast( + VirtualAlloc(reinterpret_cast(base), page_size, + MEM_RESERVE | MEM_COMMIT, PAGE_READWRITE)); + Check(memory == reinterpret_cast(base), "fixed VirtualAlloc failed"); + const auto address = reinterpret_cast(memory); + page_manager.OnGpuMap(address, page_size); + tracker.ForEachUploadRange( + address, page_size, false, [](uint64_t, uint64_t) noexcept {}, + [&]() noexcept { + bool handled = false; + std::thread fault([&] { + handled = page_manager.HandleFault(PageFaultAccess::Write, address); + }); + fault.join(); + Check(handled, "concurrent write racing upload was not handled"); + }); + Check(tracker.IsRegionCpuModified(address, page_size) && IsWritable(memory), + "upload completion erased a racing CPU dirty transition"); + tracker.UnmapMemory(address, page_size); + Check(VirtualFree(memory, 0, MEM_RELEASE) != 0, "VirtualFree failed"); +} + +void TestNativeStoreDuringRangeEnumeration() { + constexpr uintptr_t base = 0x0000000200010000ull; + TrackerHarness harness; + auto &tracker = harness.tracker; + auto &page_manager = harness.page_manager; + const auto page_size = page_manager.GetPageSize(); + auto *memory = static_cast( + VirtualAlloc(reinterpret_cast(base), page_size, + MEM_RESERVE | MEM_COMMIT, PAGE_READWRITE)); + Check(memory == reinterpret_cast(base), "fixed VirtualAlloc failed"); + const auto address = reinterpret_cast(memory); + page_manager.OnGpuMap(address, page_size); + + void *handler = AddVectoredExceptionHandler(1, NativeTrackerFaultHandler); + Check(handler != nullptr, "AddVectoredExceptionHandler failed"); + Check(g_native_page_manager.exchange(&page_manager, std::memory_order_acq_rel) == + nullptr, + "native page manager already installed"); + g_native_fault_entered.store(false, std::memory_order_release); + + std::thread writer; + tracker.ForEachUploadRange( + address, page_size, false, + [&](uint64_t, uint64_t) noexcept { + writer = std::thread( + [&] { *static_cast(memory) = 0x6b; }); + while (!g_native_fault_entered.load(std::memory_order_acquire)) { + std::this_thread::yield(); + } + }, + [&]() noexcept { writer.join(); }); + + Check(g_native_page_manager.exchange(nullptr, std::memory_order_acq_rel) == + &page_manager, + "native page manager publication changed"); + Check(RemoveVectoredExceptionHandler(handler) != 0, + "RemoveVectoredExceptionHandler failed"); + Check(memory[0] == 0x6b && tracker.IsRegionCpuModified(address, page_size) && + IsWritable(memory), + "native store during range enumeration was lost"); + + tracker.UnmapMemory(address, page_size); + Check(VirtualFree(memory, 0, MEM_RELEASE) != 0, "VirtualFree failed"); +} + +void TestFaultDuringDownloadSynchronization() { + constexpr uintptr_t base = 0x0000000200010000ull; + TrackerHarness harness; + auto &tracker = harness.tracker; + auto &page_manager = harness.page_manager; + const auto page_size = page_manager.GetPageSize(); + SharedPage shared(base, page_size * 3); + auto *memory = shared.guest; + const auto address = reinterpret_cast(memory); + page_manager.OnGpuMap(address, page_size * 3); + tracker.ForEachUploadRange( + address, page_size, true, [](uint64_t, uint64_t) noexcept {}, + []() noexcept {}); + tracker.ForEachUploadRange( + address + page_size * 2, page_size, true, + [](uint64_t, uint64_t) noexcept {}, []() noexcept {}); + memory[page_size] = 0x31; + + void *handler = AddVectoredExceptionHandler(1, NativeTrackerFaultHandler); + Check(handler != nullptr, "AddVectoredExceptionHandler failed"); + Check(g_native_page_manager.exchange(&page_manager, std::memory_order_acq_rel) == + nullptr, + "native page manager already installed"); + g_native_fault_entered.store(false, std::memory_order_release); + + uint32_t ranges = 0; + std::vector download(page_size, 0x5a); + std::thread writer; + { + PageManager::BackingWrite first(page_manager, address, page_size); + PageManager::BackingWrite third(page_manager, address + page_size * 2, + page_size); + tracker.ForEachDownloadRange( + address, page_size * 3, + [&](uint64_t download_address, uint64_t download_size) noexcept { + Check((download_address == address || + download_address == address + page_size * 2) && + download_size == page_size, + "download transaction reported the wrong range"); + ranges++; + if (download_address == address) { + writer = std::thread( + [&] { *static_cast(memory) = 0x7c; }); + while (!g_native_fault_entered.load(std::memory_order_acquire)) { + std::this_thread::yield(); + } + } + std::memcpy(shared.backing + download_address - address, + download.data(), download_size); + Check(Protection(reinterpret_cast(download_address)) == + PAGE_NOACCESS, + "backing alias exposed the protected guest page"); + Check(Protection(memory + page_size) == PAGE_READWRITE, + "clean middle page was reserved or protected"); + }); + Check(ranges == 2 && Protection(memory) == PAGE_NOACCESS && + Protection(memory + page_size) == PAGE_READWRITE && + Protection(memory + page_size * 2) == PAGE_NOACCESS, + "download completion released protection before reservation"); + } + writer.join(); + + Check(g_native_page_manager.exchange(nullptr, std::memory_order_acq_rel) == + &page_manager, + "native page manager publication changed"); + Check(RemoveVectoredExceptionHandler(handler) != 0, + "RemoveVectoredExceptionHandler failed"); + Check(ranges == 2, "partial download did not enumerate both dirty ranges"); + Check(memory[0] == 0x7c && memory[1] == 0x5a && + memory[page_size] == 0x31 && memory[page_size * 2] == 0x5a, + "reserved backing write raced or lost downloaded data"); + Check(!tracker.IsRegionGpuModified(address, page_size * 3) && + tracker.IsRegionCpuModified(address, page_size * 3), + "partial download left incorrect tracker ownership"); + Check(IsWritable(memory), "first dirty page did not restore write access"); + Check(IsWritable(memory + page_size), "clean page lost write access"); + Check(Protection(memory + page_size * 2) == PAGE_READONLY && + page_manager.IsTracked(address + page_size * 2), + "uncontended dirty page did not retain its clean write watch"); + + tracker.UnmapMemory(address, page_size * 3); +} + +void TestFaultAndExplicitDirtyRace() { + constexpr uintptr_t base = 0x0000000200010000ull; + TrackerHarness harness; + auto &tracker = harness.tracker; + auto &page_manager = harness.page_manager; + const auto page_size = page_manager.GetPageSize(); + auto *memory = static_cast( + VirtualAlloc(reinterpret_cast(base), page_size, + MEM_RESERVE | MEM_COMMIT, PAGE_READWRITE)); + Check(memory == reinterpret_cast(base), "fixed VirtualAlloc failed"); + const auto address = reinterpret_cast(memory); + page_manager.OnGpuMap(address, page_size); + for (uint32_t iteration = 0; iteration < 64; iteration++) { + tracker.ForEachUploadRange( + address, page_size, false, [](uint64_t, uint64_t) noexcept {}, + []() noexcept {}); + std::atomic_bool start{false}; + bool handled = false; + std::thread fault([&] { + while (!start.load(std::memory_order_acquire)) { + std::this_thread::yield(); + } + handled = page_manager.HandleFault(PageFaultAccess::Write, address); + }); + std::thread dirty([&] { + while (!start.load(std::memory_order_acquire)) { + std::this_thread::yield(); + } + tracker.MarkRegionAsCpuModified(address, page_size); + }); + start.store(true, std::memory_order_release); + fault.join(); + dirty.join(); + Check(handled && tracker.IsRegionCpuModified(address, page_size) && + IsWritable(memory), + "fault/explicit-dirty race lost dirty state or write access"); + } + tracker.UnmapMemory(address, page_size); + Check(VirtualFree(memory, 0, MEM_RELEASE) != 0, "VirtualFree failed"); +} + +void TestSharedTrackersAndConcurrentPageFaults() { + constexpr uintptr_t base = 0x0000000200010000ull; + SharedTrackerHarness harness; + const auto page_size = harness.page_manager.GetPageSize(); + auto *memory = static_cast( + VirtualAlloc(reinterpret_cast(base), page_size * 2, + MEM_RESERVE | MEM_COMMIT, PAGE_READWRITE)); + Check(memory == reinterpret_cast(base), "fixed VirtualAlloc failed"); + const auto address = reinterpret_cast(memory); + harness.page_manager.OnGpuMap(address, page_size * 2); + + for (auto *tracker : {&harness.first, &harness.second}) { + tracker->ForEachUploadRange( + address, page_size * 2, false, [](uint64_t, uint64_t) noexcept {}, + []() noexcept {}); + } + Check(!IsWritable(memory) && !IsWritable(memory + page_size), + "shared trackers did not arm both pages"); + + std::atomic_bool start{false}; + bool first_handled = false; + bool second_handled = false; + std::thread first_fault([&] { + while (!start.load(std::memory_order_acquire)) { + std::this_thread::yield(); + } + first_handled = harness.page_manager.HandleFault(PageFaultAccess::Write, + address + 8); + }); + std::thread second_fault([&] { + while (!start.load(std::memory_order_acquire)) { + std::this_thread::yield(); + } + second_handled = harness.page_manager.HandleFault( + PageFaultAccess::Write, address + page_size + 8); + }); + start.store(true, std::memory_order_release); + first_fault.join(); + second_fault.join(); + + Check(first_handled && second_handled, + "concurrent faults were not handled by shared trackers"); + for (auto *tracker : {&harness.first, &harness.second}) { + Check(tracker->IsRegionCpuModified(address, page_size * 2), + "a shared tracker lost concurrent CPU dirtiness"); + } + + harness.first.UntrackMemory(address, page_size * 2); + harness.second.UnmapMemory(address, page_size * 2); + Check(VirtualFree(memory, 0, MEM_RELEASE) != 0, "VirtualFree failed"); +} + +void TestGpuDirtyBits() { + constexpr uintptr_t base = 0x0000000200010000ull; + TrackerHarness harness; + auto &tracker = harness.tracker; + auto &page_manager = harness.page_manager; + const auto page_size = page_manager.GetPageSize(); + auto *memory = static_cast( + VirtualAlloc(reinterpret_cast(base), page_size * 2, + MEM_RESERVE | MEM_COMMIT, PAGE_READWRITE)); + Check(memory == reinterpret_cast(base), "fixed VirtualAlloc failed"); + const auto address = reinterpret_cast(memory); + page_manager.OnGpuMap(address, page_size * 2); + tracker.ForEachUploadRange( + address, page_size, true, [](uint64_t, uint64_t) noexcept {}, + []() noexcept {}); + Check(tracker.IsRegionGpuModified(address, page_size) && + !tracker.IsRegionGpuModified(address + page_size, page_size) && + Protection(memory) == PAGE_NOACCESS, + "GPU dirty state escaped the requested range"); + tracker.UnmarkRegionAsGpuModified(address, page_size); + Check(!tracker.IsRegionGpuModified(address, page_size) && + Protection(memory) == PAGE_READONLY, + "GPU dirty state did not restore write-only tracking"); + tracker.MarkRegionAsGpuModified(address, page_size); + Check(tracker.IsRegionGpuModified(address, page_size) && + Protection(memory) == PAGE_NOACCESS, + "explicit GPU dirty transition did not trap CPU access"); + tracker.UnmarkRegionAsGpuModified(address, page_size); + tracker.MarkRegionAsCpuModified(address, page_size); + tracker.UnmapMemory(address, page_size * 2); + Check(VirtualFree(memory, 0, MEM_RELEASE) != 0, "VirtualFree failed"); +} + +void TestCrossRegionUpload() { + constexpr uintptr_t base = 0x0000000200010000ull; + constexpr uint64_t region_size = 4ull * 1024ull * 1024ull; + TrackerHarness harness; + auto &tracker = harness.tracker; + auto &page_manager = harness.page_manager; + const auto page_size = page_manager.GetPageSize(); + auto *memory = static_cast( + VirtualAlloc(reinterpret_cast(base), region_size * 2, + MEM_RESERVE | MEM_COMMIT, PAGE_READWRITE)); + Check(memory == reinterpret_cast(base), "fixed VirtualAlloc failed"); + const auto address = reinterpret_cast(memory); + const auto boundary = (address + region_size - 1) & ~(region_size - 1); + page_manager.OnGpuMap(address, region_size * 2); + uint32_t ranges = 0; + tracker.ForEachUploadRange( + boundary - page_size, page_size * 2, false, + [&](uint64_t, uint64_t) noexcept { ranges++; }, []() noexcept {}); + Check(ranges == 2 && + !tracker.IsRegionCpuModified(boundary - page_size, page_size * 2) && + !IsWritable(reinterpret_cast(boundary - page_size)) && + !IsWritable(reinterpret_cast(boundary)), + "cross-region upload did not clear and protect both regions"); + tracker.MarkRegionAsCpuModified(boundary - page_size, page_size * 2); + tracker.ForEachUploadRange( + boundary - page_size, page_size * 2, true, + [](uint64_t, uint64_t) noexcept {}, []() noexcept {}); + Check(tracker.IsRegionGpuModified(boundary - page_size, page_size * 2), + "cross-region written upload did not mark GPU dirty state"); + tracker.UnmarkRegionAsGpuModified(boundary - page_size, page_size * 2); + tracker.MarkRegionAsCpuModified(boundary - page_size, page_size * 2); + tracker.UnmapMemory(address, region_size * 2); + Check(VirtualFree(memory, 0, MEM_RELEASE) != 0, "VirtualFree failed"); +} + +[[noreturn]] void RunDeathCase(const char *name) { + constexpr uintptr_t base = 0x0000000200010000ull; + TrackerHarness harness; + auto &tracker = harness.tracker; + auto &page_manager = harness.page_manager; + const auto page_size = page_manager.GetPageSize(); + if (std::strcmp(name, "unmapped") == 0) { + (void)tracker.IsRegionCpuModified(base, page_size); + } + const auto allocation_size = + std::strcmp(name, "missing-download-bytes") == 0 ? page_size * 2 + : page_size; + auto *memory = static_cast( + VirtualAlloc(reinterpret_cast(base), allocation_size, + MEM_RESERVE | MEM_COMMIT, PAGE_READWRITE)); + Check(memory == reinterpret_cast(base), "fixed VirtualAlloc failed"); + const auto address = reinterpret_cast(memory); + page_manager.OnGpuMap(address, allocation_size); + if (std::strcmp(name, "gpu-dirty-fault") == 0 || + std::strcmp(name, "gpu-dirty-read") == 0 || + std::strcmp(name, "gpu-dirty-explicit-cpu") == 0 || + std::strcmp(name, "virtual-gpu-read") == 0) { + harness.discard_virtual = std::strcmp(name, "virtual-gpu-read") == 0; + tracker.ForEachUploadRange( + address, page_size, true, [](uint64_t, uint64_t) noexcept {}, + []() noexcept {}); + if (std::strcmp(name, "gpu-dirty-explicit-cpu") == 0) { + tracker.MarkRegionAsCpuModified(address, page_size); + } else { + (void)page_manager.HandleFault( + (std::strcmp(name, "gpu-dirty-read") == 0 || + std::strcmp(name, "virtual-gpu-read") == 0) + ? PageFaultAccess::Read + : PageFaultAccess::Write, + address); + } + } else if (std::strcmp(name, "reentrant-upload") == 0) { + tracker.ForEachUploadRange( + address, page_size, true, [](uint64_t, uint64_t) noexcept {}, + [&]() noexcept { + (void)tracker.IsRegionCpuModified(address, page_size); + }); + } else if (std::strcmp(name, "writable-upload-race") == 0) { + std::atomic_bool start{false}; + std::atomic_bool entered{false}; + std::thread fault([&] { + while (!start.load(std::memory_order_acquire)) { + std::this_thread::yield(); + } + entered.store(true, std::memory_order_release); + (void)page_manager.HandleFault(PageFaultAccess::Write, address); + }); + tracker.ForEachUploadRange( + address, page_size, true, [](uint64_t, uint64_t) noexcept {}, + [&]() noexcept { + start.store(true, std::memory_order_release); + while (!entered.load(std::memory_order_acquire)) { + std::this_thread::yield(); + } + }); + fault.join(); + } else if (std::strcmp(name, "gpu-dirty-unmap-race") == 0) { + g_unmap_contended.store(false, std::memory_order_release); + MemoryTracker::SetUnmapContentionHook(UnmapContended); + std::thread unmap; + tracker.ForEachUploadRange( + address, page_size, true, [](uint64_t, uint64_t) noexcept {}, + [&]() noexcept { + unmap = std::thread( + [&] { tracker.UnmapMemory(address, page_size); }); + while (!g_unmap_contended.load(std::memory_order_acquire)) { + std::this_thread::yield(); + } + }); + unmap.join(); + } else if (std::strcmp(name, "missing-download-bytes") == 0) { + tracker.ForEachUploadRange( + address, allocation_size, true, [](uint64_t, uint64_t) noexcept {}, + []() noexcept {}); + RangeSet dirty_bytes; + dirty_bytes.Add(address, 1); + PageManager::BackingWrite backing(page_manager, address, page_size); + tracker.ForEachDownloadRange( + address, allocation_size, + [&](uint64_t dirty_address, uint64_t dirty_size) noexcept { + for (auto page = dirty_address; page < dirty_address + dirty_size; + page += page_size) { + if (dirty_bytes.Intersections(page, page_size).empty()) { + EXIT("GPU-dirty test page has no dirty byte record\n"); + } + } + }, + [](uint64_t, uint64_t) noexcept {}); + } + std::_Exit(0x7f); +} + +void TestFatalPaths() { + char path[MAX_PATH]{}; + Check(GetModuleFileNameA(nullptr, path, MAX_PATH) != 0, + "GetModuleFileName failed"); + for (const char *name : {"gpu-dirty-fault", "gpu-dirty-read", "virtual-gpu-read", + "gpu-dirty-explicit-cpu", + "unmapped", "reentrant-upload", + "writable-upload-race", "gpu-dirty-unmap-race", + "missing-download-bytes"}) { + std::string command = std::string("\"") + path + "\" --death " + name; + std::vector mutable_command(command.begin(), command.end()); + mutable_command.push_back('\0'); + STARTUPINFOA startup{sizeof(startup)}; + PROCESS_INFORMATION process{}; + Check(CreateProcessA(nullptr, mutable_command.data(), nullptr, nullptr, + FALSE, CREATE_NO_WINDOW, nullptr, nullptr, &startup, + &process) != 0, + "CreateProcess failed"); + Check(WaitForSingleObject(process.hProcess, 10000) == WAIT_OBJECT_0, + "MemoryTracker death test timed out"); + DWORD exit_code = 0; + Check(GetExitCodeProcess(process.hProcess, &exit_code) != 0 && + (exit_code == 321 || + exit_code == EXCEPTION_NONCONTINUABLE_EXCEPTION), + "MemoryTracker death path used the wrong exit"); + CloseHandle(process.hThread); + CloseHandle(process.hProcess); + } +} +#endif + +} // namespace + +int main(int argc, char **argv) { +#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS + if (argc == 3 && std::strcmp(argv[1], "--death") == 0) { + RunDeathCase(argv[2]); + } + TestCpuDirtyUploadAndFault(); + TestPendingFaultBlocksUploadConsumption(); + TestCleanReadFaultPreservesCpuState(); + TestGpuDownloadFaultOwnership(); + TestVirtualGpuWriteDiscard(); + TestSameSlabTrackerArbitration(); + TestSharedMetadataAndImagePageFault(); + TestRangeSet(); + TestGpuDirtyBits(); + TestCrossRegionUpload(); + TestFaultDuringUploadRemainsDirty(); + TestNativeStoreDuringRangeEnumeration(); + TestFaultDuringDownloadSynchronization(); + TestFaultAndExplicitDirtyRace(); + TestSharedTrackersAndConcurrentPageFaults(); + TestFatalPaths(); + std::puts("MemoryTrackerTests: all cases passed"); + return 0; +#else + (void)argc; + (void)argv; + std::fputs("MemoryTrackerTests: unsupported platform\n", stderr); + return 1; +#endif +} diff --git a/tests/PageManagerTests.cpp b/tests/PageManagerTests.cpp new file mode 100644 index 000000000..03eceb6a3 --- /dev/null +++ b/tests/PageManagerTests.cpp @@ -0,0 +1,631 @@ +#include "graphics/host_gpu/pageManager.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS +#ifndef NOMINMAX +#define NOMINMAX +#endif +#include +#undef min +#undef max +#endif + +namespace { + +using Libs::Graphics::GpuAccess; +using Libs::Graphics::PageFaultAccess; +using Libs::Graphics::PageManager; + +void Check(bool value, const char *text) { + if (!value) { + std::fprintf(stderr, "PageManagerTests: failed: %s\n", text); + std::abort(); + } +} + +#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS +bool IsWritable(const void *address) { + MEMORY_BASIC_INFORMATION info{}; + Check(VirtualQuery(address, &info, sizeof(info)) != 0, "VirtualQuery failed"); + return info.Protect == PAGE_READWRITE; +} + +uint32_t Protection(const void *address) { + MEMORY_BASIC_INFORMATION info{}; + Check(VirtualQuery(address, &info, sizeof(info)) != 0, "VirtualQuery failed"); + return info.Protect; +} + +struct FaultContext { + PageManager *manager = nullptr; + bool result = true; + bool reenter = false; + uint64_t reenter_address = 0; + bool block = false; + std::atomic_uint32_t calls{0}; + std::atomic_bool entered{false}; + std::atomic_bool release{false}; +}; + +std::atomic g_native_fault_manager{nullptr}; +std::atomic_bool g_delay_native_fault{false}; +std::atomic_bool g_native_fault_entered{false}; +std::atomic_bool g_release_native_fault{false}; + +LONG CALLBACK NativeFaultHandler(EXCEPTION_POINTERS *exception) { + if (exception == nullptr || exception->ExceptionRecord == nullptr || + exception->ExceptionRecord->ExceptionCode != EXCEPTION_ACCESS_VIOLATION) { + return EXCEPTION_CONTINUE_SEARCH; + } + const auto operation = exception->ExceptionRecord->ExceptionInformation[0]; + const auto access = operation == 0 ? PageFaultAccess::Read + : operation == 1 ? PageFaultAccess::Write + : operation == 8 ? PageFaultAccess::Execute + : PageFaultAccess::Unknown; + auto *manager = g_native_fault_manager.load(std::memory_order_acquire); + if (manager == nullptr) { + return EXCEPTION_CONTINUE_SEARCH; + } + if (g_delay_native_fault.load(std::memory_order_acquire)) { + g_native_fault_entered.store(true, std::memory_order_release); + while (!g_release_native_fault.load(std::memory_order_acquire)) { + std::this_thread::yield(); + } + } + return manager->HandleFault( + access, exception->ExceptionRecord->ExceptionInformation[1]) + ? EXCEPTION_CONTINUE_EXECUTION + : EXCEPTION_CONTINUE_SEARCH; +} + +bool InvalidateFault(void *context, Libs::Graphics::PageFaultAccess, uint64_t vaddr, uint64_t size, + Libs::Graphics::PageFaultPhase phase) noexcept { + auto *fault = static_cast(context); + Check(fault != nullptr && fault->manager != nullptr, "invalid fault context"); + if (phase != Libs::Graphics::PageFaultPhase::Invalidate) { + return true; + } + fault->calls.fetch_add(1, std::memory_order_relaxed); + if (fault->reenter) { + const auto address = + fault->reenter_address != 0 ? fault->reenter_address : vaddr; + (void)fault->manager->HandleFault(PageFaultAccess::Write, address); + } + if (fault->block) { + fault->entered.store(true, std::memory_order_release); + while (!fault->release.load(std::memory_order_acquire)) { + std::this_thread::yield(); + } + } + (void)size; + return fault->result; +} + +uint8_t *Allocate(uint64_t size, uint32_t protection = PAGE_READWRITE) { + constexpr uintptr_t test_address = 0x0000000200010000ull; + auto *memory = static_cast( + VirtualAlloc(reinterpret_cast(test_address), size, + MEM_RESERVE | MEM_COMMIT, protection)); + Check(memory == reinterpret_cast(test_address), + "fixed low VirtualAlloc failed"); + return memory; +} + +void TestWatchFaultAndUnwatch() { + FaultContext context; + PageManager manager(InvalidateFault, &context); + context.manager = &manager; + const auto page_size = manager.GetPageSize(); + auto *memory = Allocate(page_size * 2); + + manager.OnGpuMap(reinterpret_cast(memory), page_size * 2); + manager.UpdatePageWatchers(true, reinterpret_cast(memory), + page_size); + Check(manager.IsTracked(reinterpret_cast(memory)) && + !IsWritable(memory), + "watch did not protect the page"); + Check(manager.HandleFault(PageFaultAccess::Write, + reinterpret_cast(memory + 32)), + "tracked write fault was not handled"); + Check(!manager.IsTracked(reinterpret_cast(memory)) && + IsWritable(memory), + "fault invalidation did not remove the watcher"); + Check(manager.HandleFault(PageFaultAccess::Write, + reinterpret_cast(memory)), + "single delayed write fault was not coalesced"); + Check(manager.HandleFault(PageFaultAccess::Write, + reinterpret_cast(memory)), + "second delayed write fault was not coalesced"); + manager.OnGpuUnmap(reinterpret_cast(memory), page_size * 2); + Check(VirtualFree(memory, 0, MEM_RELEASE) != 0, "VirtualFree failed"); +} + +void TestSharedWatcherFault() { + FaultContext context; + PageManager manager(InvalidateFault, &context); + context.manager = &manager; + const auto page_size = manager.GetPageSize(); + auto *memory = Allocate(page_size); + const auto address = reinterpret_cast(memory); + + manager.OnGpuMap(address, page_size); + manager.UpdatePageWatchers(true, address + 8, 32); + manager.UpdatePageWatchers(true, address + 128, 64); + Check(manager.HandleFault(PageFaultAccess::Write, address + 16), + "shared-watcher fault was not handled"); + Check(!manager.IsTracked(address) && IsWritable(memory), + "fault callback did not clear every shared watcher"); + manager.OnGpuUnmap(address, page_size); + Check(VirtualFree(memory, 0, MEM_RELEASE) != 0, "VirtualFree failed"); +} + +void TestReadWriteWatcherFault() { + FaultContext context; + PageManager manager(InvalidateFault, &context); + context.manager = &manager; + const auto page_size = manager.GetPageSize(); + auto *memory = Allocate(page_size); + const auto address = reinterpret_cast(memory); + + manager.OnGpuMap(address, page_size); + manager.UpdatePageWatchers(true, address, page_size, + Libs::Graphics::PageWatchMode::Write); + manager.UpdatePageWatchers(true, address, page_size, + Libs::Graphics::PageWatchMode::ReadWrite); + manager.UpdatePageWatchers(false, address, page_size, + Libs::Graphics::PageWatchMode::Write); + Check(Protection(memory) == PAGE_NOACCESS, + "read/write watcher did not install no-access protection"); + Check(manager.HandleFault(PageFaultAccess::Read, address + 8), + "tracked read fault was not handled"); + Check(Protection(memory) == PAGE_READWRITE && !manager.IsTracked(address), + "read fault did not release every watcher"); + manager.OnGpuUnmap(address, page_size); + Check(VirtualFree(memory, 0, MEM_RELEASE) != 0, "VirtualFree failed"); +} + +void TestPermittedMappedLateFaultsResume() { + FaultContext context; + PageManager manager(InvalidateFault, &context); + context.manager = &manager; + const auto page_size = manager.GetPageSize(); + auto *memory = Allocate(page_size); + const auto address = reinterpret_cast(memory); + + manager.OnGpuMap(address, page_size); + manager.UpdatePageWatchers(true, address, page_size); + manager.UpdatePageWatchers(false, address, page_size); + Check(manager.HandleFault(PageFaultAccess::Write, address), + "first delayed mapped write was not accepted"); + Check(manager.HandleFault(PageFaultAccess::Write, address), + "second delayed mapped write was not accepted"); + Check(manager.HandleFault(PageFaultAccess::Read, address), + "delayed mapped read was not accepted on readable backing"); + DWORD old_protection = 0; + Check(VirtualProtect(memory, page_size, PAGE_READONLY, &old_protection) != 0 && + old_protection == PAGE_READWRITE, + "failed to prepare intentional read-only protection"); + Check(!manager.HandleFault(PageFaultAccess::Write, address), + "intentional read-only mapping accepted a write fault"); + Check(VirtualProtect(memory, page_size, PAGE_READWRITE, &old_protection) != 0, + "failed to restore writable protection"); + manager.OnGpuUnmap(address, page_size); + Check(VirtualFree(memory, 0, MEM_RELEASE) != 0, "VirtualFree failed"); +} + +void TestPartialMappingUnmapPreservesTokens() { + FaultContext context; + PageManager manager(InvalidateFault, &context); + context.manager = &manager; + const auto page_size = manager.GetPageSize(); + auto *memory = Allocate(page_size); + const auto address = reinterpret_cast(memory); + + manager.OnGpuMap(address, page_size); + manager.OnGpuMap(address + 8, 16); + manager.UpdatePageWatchers(true, address, page_size, + Libs::Graphics::PageWatchMode::ReadWrite); + manager.UpdatePageWatchers(false, address, page_size, + Libs::Graphics::PageWatchMode::ReadWrite); + manager.OnGpuUnmap(address + 8, 16); + Check(manager.HandleFault(PageFaultAccess::Read, address), + "partial mapping unmap erased delayed read ownership"); + Check(manager.HandleFault(PageFaultAccess::Write, address), + "partial mapping unmap erased delayed write ownership"); + manager.OnGpuUnmap(address, page_size); + Check(VirtualFree(memory, 0, MEM_RELEASE) != 0, "VirtualFree failed"); +} + +void TestNativeDelayedReadAfterModeDowngrade() { + FaultContext context; + PageManager manager(InvalidateFault, &context); + context.manager = &manager; + const auto page_size = manager.GetPageSize(); + auto *memory = Allocate(page_size); + memory[0] = 0x6d; + const auto address = reinterpret_cast(memory); + + manager.OnGpuMap(address, page_size); + manager.UpdatePageWatchers(true, address, page_size); + manager.UpdatePageWatchers(true, address, page_size, + Libs::Graphics::PageWatchMode::ReadWrite); + manager.UpdatePageWatchers(false, address, page_size); + Check(Protection(memory) == PAGE_NOACCESS, + "read/write ownership did not install no-access protection"); + + void *handler = AddVectoredExceptionHandler(1, NativeFaultHandler); + Check(handler != nullptr, "AddVectoredExceptionHandler failed"); + Check(g_native_fault_manager.exchange(&manager, std::memory_order_acq_rel) == + nullptr, + "native fault manager already installed"); + g_native_fault_entered.store(false, std::memory_order_release); + g_release_native_fault.store(false, std::memory_order_release); + g_delay_native_fault.store(true, std::memory_order_release); + + uint8_t value = 0; + std::thread reader( + [&] { value = *static_cast(memory); }); + while (!g_native_fault_entered.load(std::memory_order_acquire)) { + std::this_thread::yield(); + } + manager.UpdatePageWatchers(true, address, page_size); + manager.UpdatePageWatchers(false, address, page_size, + Libs::Graphics::PageWatchMode::ReadWrite); + Check(Protection(memory) == PAGE_READONLY, + "mode downgrade did not restore readable protection"); + manager.UpdatePageWatchers(true, address, page_size); + + context.block = true; + bool write_handled = false; + std::thread writer([&] { + write_handled = manager.HandleFault(PageFaultAccess::Write, address); + }); + while (!context.entered.load(std::memory_order_acquire)) { + std::this_thread::yield(); + } + g_release_native_fault.store(true, std::memory_order_release); + reader.join(); + context.release.store(true, std::memory_order_release); + writer.join(); + + g_delay_native_fault.store(false, std::memory_order_release); + Check(g_native_fault_manager.exchange(nullptr, std::memory_order_acq_rel) == + &manager, + "native fault manager publication changed"); + Check(RemoveVectoredExceptionHandler(handler) != 0, + "RemoveVectoredExceptionHandler failed"); + Check(value == 0x6d && write_handled && + context.calls.load(std::memory_order_relaxed) == 1, + "delayed read was not coalesced across write ownership/resolution"); + + manager.OnGpuUnmap(address, page_size); + Check(VirtualFree(memory, 0, MEM_RELEASE) != 0, "VirtualFree failed"); +} + +void TestDelayedFaultAfterExplicitUnwatch() { + FaultContext context; + PageManager manager(InvalidateFault, &context); + context.manager = &manager; + const auto page_size = manager.GetPageSize(); + auto *memory = Allocate(page_size); + const auto address = reinterpret_cast(memory); + manager.OnGpuMap(address, page_size); + manager.UpdatePageWatchers(true, address, page_size); + + // Models a store that already raised an AV before another thread published + // explicit CPU dirtiness and removed the watcher. + manager.UpdatePageWatchers(false, address, page_size); + Check(manager.HandleFault(PageFaultAccess::Write, address), + "delayed watched write was not accepted after explicit unwatch"); + Check(manager.HandleFault(PageFaultAccess::Write, address) && + context.calls.load(std::memory_order_relaxed) == 0, + "second explicit-unwatch fault was not coalesced or dispatched invalidation"); + + manager.OnGpuUnmap(address, page_size); + Check(VirtualFree(memory, 0, MEM_RELEASE) != 0, "VirtualFree failed"); +} + +void TestNativeAccessViolation() { + FaultContext context; + PageManager manager(InvalidateFault, &context); + context.manager = &manager; + const auto page_size = manager.GetPageSize(); + auto *memory = Allocate(page_size); + const auto address = reinterpret_cast(memory); + manager.OnGpuMap(address, page_size); + manager.UpdatePageWatchers(true, address, page_size); + + void *handler = AddVectoredExceptionHandler(1, NativeFaultHandler); + Check(handler != nullptr, "AddVectoredExceptionHandler failed"); + Check(g_native_fault_manager.exchange(&manager, std::memory_order_acq_rel) == + nullptr, + "native fault manager already installed"); + *static_cast(memory) = 0x5a; + Check(g_native_fault_manager.exchange(nullptr, std::memory_order_acq_rel) == + &manager, + "native fault manager publication changed"); + Check(RemoveVectoredExceptionHandler(handler) != 0, + "RemoveVectoredExceptionHandler failed"); + + Check(memory[0] == 0x5a && !manager.IsTracked(address) && + context.calls.load(std::memory_order_relaxed) == 1, + "native access violation did not invalidate and resume the store"); + manager.OnGpuUnmap(address, page_size); + Check(VirtualFree(memory, 0, MEM_RELEASE) != 0, "VirtualFree failed"); +} + +void TestInvalidLateWriteTokenIsConsumed() { + FaultContext context; + PageManager manager(InvalidateFault, &context); + context.manager = &manager; + const auto page_size = manager.GetPageSize(); + auto *memory = Allocate(page_size); + const auto address = reinterpret_cast(memory); + manager.OnGpuMap(address, page_size); + manager.UpdatePageWatchers(true, address, page_size); + Check(manager.HandleFault(PageFaultAccess::Write, address), + "initial write fault was not handled"); + DWORD old_protection = 0; + Check(VirtualProtect(memory, page_size, PAGE_READONLY, &old_protection) != + 0 && + old_protection == PAGE_READWRITE, + "failed to create invalid late-write protection state"); + Check(!manager.HandleFault(PageFaultAccess::Write, address) && + !manager.HandleFault(PageFaultAccess::Write, address), + "invalid late-write token was accepted or retained"); + Check(VirtualProtect(memory, page_size, PAGE_READWRITE, &old_protection) != 0, + "failed to restore test protection"); + manager.OnGpuUnmap(address, page_size); + Check(VirtualFree(memory, 0, MEM_RELEASE) != 0, "VirtualFree failed"); +} + +void TestCrossRegionRange() { + FaultContext context; + PageManager manager(InvalidateFault, &context); + context.manager = &manager; + const auto page_size = manager.GetPageSize(); + constexpr uint64_t region_size = 4ull * 1024ull * 1024ull; + auto *memory = Allocate(region_size * 2); + const auto base = reinterpret_cast(memory); + const auto boundary = (base + region_size - 1) & ~(region_size - 1); + Check(boundary >= base + page_size && + boundary + page_size <= base + region_size * 2, + "test allocation does not contain a region boundary"); + + manager.OnGpuMap(base, region_size * 2); + manager.UpdatePageWatchers(true, boundary - page_size, page_size * 2); + Check(!IsWritable(reinterpret_cast(boundary - page_size)) && + !IsWritable(reinterpret_cast(boundary)), + "cross-region watch did not protect both pages"); + manager.UpdatePageWatchers(false, boundary - page_size, page_size * 2); + Check(IsWritable(reinterpret_cast(boundary - page_size)) && + IsWritable(reinterpret_cast(boundary)), + "cross-region unwatch did not restore both pages"); + manager.OnGpuUnmap(base, region_size * 2); + Check(VirtualFree(memory, 0, MEM_RELEASE) != 0, "VirtualFree failed"); +} + +[[noreturn]] void RunDeathCase(const char *name) { + FaultContext context; + auto manager = std::make_unique(InvalidateFault, &context); + context.manager = manager.get(); + const auto page_size = manager->GetPageSize(); + if (std::strcmp(name, "invalid-range") == 0) { + manager->UpdatePageWatchers(true, (1ull << 40u) - 1, 2); + } else if (std::strcmp(name, "unknown-untrack") == 0) { + manager->UpdatePageWatchers(false, 0x1000, page_size); + } else { + const bool two_pages = std::strcmp(name, "cross-reentrant") == 0; + auto *memory = Allocate( + two_pages ? page_size * 2 : page_size, + std::strcmp(name, "protection") == 0 ? PAGE_READONLY : PAGE_READWRITE); + const auto address = reinterpret_cast(memory); + manager->OnGpuMap(address, two_pages ? page_size * 2 : page_size); + manager->UpdatePageWatchers(true, address, page_size); + if (two_pages) { + manager->UpdatePageWatchers(true, address + page_size, page_size); + } + if (std::strcmp(name, "destructor-watch") == 0) { + manager.reset(); + } else if (std::strcmp(name, "non-write") == 0) { + (void)manager->HandleFault(PageFaultAccess::Read, address); + } else if (std::strcmp(name, "callback-false") == 0) { + context.result = false; + (void)manager->HandleFault(PageFaultAccess::Write, address); + } else if (std::strcmp(name, "reentrant") == 0) { + context.reenter = true; + (void)manager->HandleFault(PageFaultAccess::Write, address); + } else if (std::strcmp(name, "cross-reentrant") == 0) { + context.reenter = true; + context.reenter_address = address + page_size; + (void)manager->HandleFault(PageFaultAccess::Write, address); + } else if (std::strcmp(name, "concurrent-non-write") == 0) { + context.block = true; + std::thread first( + [&] { (void)manager->HandleFault(PageFaultAccess::Write, address); }); + while (!context.entered.load(std::memory_order_acquire)) { + std::this_thread::yield(); + } + (void)manager->HandleFault(PageFaultAccess::Read, address); + first.join(); + } else if (std::strcmp(name, "watched-unmap") == 0) { + manager->OnGpuUnmap(address, page_size); + } else if (std::strcmp(name, "protection") != 0) { + std::_Exit(0x7f); + } + } + std::_Exit(0x7f); +} + +void CheckDeathCase(const char *name) { + char path[MAX_PATH]{}; + Check(GetModuleFileNameA(nullptr, path, MAX_PATH) != 0, + "GetModuleFileName failed"); + std::string command = std::string("\"") + path + "\" --death " + name; + std::vector mutable_command(command.begin(), command.end()); + mutable_command.push_back('\0'); + STARTUPINFOA startup{sizeof(startup)}; + PROCESS_INFORMATION process{}; + Check(CreateProcessA(nullptr, mutable_command.data(), nullptr, nullptr, FALSE, + CREATE_NO_WINDOW, nullptr, nullptr, &startup, + &process) != 0, + "CreateProcess failed"); + Check(WaitForSingleObject(process.hProcess, 10000) == WAIT_OBJECT_0, + "death test timed out"); + DWORD exit_code = 0; + Check( + GetExitCodeProcess(process.hProcess, &exit_code) != 0 && + (exit_code == 322 || exit_code == EXCEPTION_NONCONTINUABLE_EXCEPTION), + "death case did not use the PageManager fatal exit"); + CloseHandle(process.hThread); + CloseHandle(process.hProcess); +} + +void TestFatalPaths() { + for (const char *name : + {"invalid-range", "unknown-untrack", "destructor-watch", "non-write", + "callback-false", "reentrant", "cross-reentrant", + "concurrent-non-write", "watched-unmap", "protection"}) { + CheckDeathCase(name); + } +} + +void TestConcurrentFault() { + FaultContext context; + PageManager manager(InvalidateFault, &context); + context.manager = &manager; + context.block = true; + const auto page_size = manager.GetPageSize(); + auto *memory = Allocate(page_size); + const auto address = reinterpret_cast(memory); + manager.OnGpuMap(address, page_size); + manager.UpdatePageWatchers(true, address, page_size); + bool first_result = false; + bool second_result = false; + std::thread first([&] { + first_result = manager.HandleFault(PageFaultAccess::Write, address); + }); + while (!context.entered.load(std::memory_order_acquire)) { + std::this_thread::yield(); + } + std::thread second([&] { + second_result = manager.HandleFault(PageFaultAccess::Write, address); + }); + context.release.store(true, std::memory_order_release); + first.join(); + second.join(); + Check(first_result && second_result && + context.calls.load(std::memory_order_relaxed) == 1, + "concurrent faults dispatched invalidation more than once"); + manager.OnGpuUnmap(address, page_size); + Check(VirtualFree(memory, 0, MEM_RELEASE) != 0, "VirtualFree failed"); +} + +void TestExternalDirtyTransferDuringResolution() { + FaultContext context; + PageManager manager(InvalidateFault, &context); + context.manager = &manager; + context.block = true; + const auto page_size = manager.GetPageSize(); + auto *memory = Allocate(page_size); + const auto address = reinterpret_cast(memory); + manager.OnGpuMap(address, page_size); + manager.UpdatePageWatchers(true, address, page_size); + bool handled = false; + std::thread fault( + [&] { handled = manager.HandleFault(PageFaultAccess::Write, address); }); + while (!context.entered.load(std::memory_order_acquire)) { + std::this_thread::yield(); + } + manager.UpdatePageWatchers(false, address, page_size); + context.release.store(true, std::memory_order_release); + fault.join(); + Check(handled && !manager.IsTracked(address) && IsWritable(memory), + "external dirty transfer did not satisfy active resolution"); + manager.OnGpuUnmap(address, page_size); + Check(VirtualFree(memory, 0, MEM_RELEASE) != 0, "VirtualFree failed"); +} + +void TestMappingDoesNotRequireCpuWriteAccess() { + FaultContext context; + PageManager manager(InvalidateFault, &context); + context.manager = &manager; + const auto page_size = manager.GetPageSize(); + auto *memory = Allocate(page_size); + DWORD old_protection = 0; + Check(VirtualProtect(memory, page_size, PAGE_NOACCESS, &old_protection) != 0 && + old_protection == PAGE_READWRITE, + "failed to prepare CPU-inaccessible mapping"); + const auto address = reinterpret_cast(memory); + manager.OnGpuMap(address, page_size); + Check(manager.IsMapped(address, page_size), + "CPU-inaccessible committed range was not GPU mapped"); + manager.OnGpuUnmap(address, page_size); + Check(VirtualFree(memory, 0, MEM_RELEASE) != 0, "VirtualFree failed"); +} + +void TestGpuAccessPermissions() { + FaultContext context; + PageManager manager(InvalidateFault, &context); + context.manager = &manager; + const auto page_size = manager.GetPageSize(); + auto *memory = Allocate(page_size); + const auto address = reinterpret_cast(memory); + manager.OnGpuMap(address, page_size, GpuAccess::Read); + Check(manager.HasGpuAccess(address, page_size, GpuAccess::Read) && + !manager.HasGpuAccess(address, page_size, GpuAccess::Write), + "read-only GPU mapping granted write access"); + manager.OnGpuMap(address, page_size, GpuAccess::Write); + Check(manager.HasGpuAccess(address, page_size, GpuAccess::ReadWrite), + "overlapping GPU mappings did not combine permissions"); + manager.OnGpuUnmap(address, page_size, GpuAccess::Read); + Check(!manager.HasGpuAccess(address, page_size, GpuAccess::Read) && + manager.HasGpuAccess(address, page_size, GpuAccess::Write), + "GPU read unmap removed the wrong permission"); + manager.OnGpuUnmap(address, page_size, GpuAccess::Write); + Check(!manager.IsMapped(address, page_size), + "GPU permission mappings were not fully balanced"); + Check(VirtualFree(memory, 0, MEM_RELEASE) != 0, "VirtualFree failed"); +} +#endif + +} // namespace + +int main(int argc, char **argv) { +#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS + if (argc == 3 && std::strcmp(argv[1], "--death") == 0) { + RunDeathCase(argv[2]); + } + TestWatchFaultAndUnwatch(); + TestSharedWatcherFault(); + TestReadWriteWatcherFault(); + TestPermittedMappedLateFaultsResume(); + TestPartialMappingUnmapPreservesTokens(); + TestNativeDelayedReadAfterModeDowngrade(); + TestDelayedFaultAfterExplicitUnwatch(); + TestNativeAccessViolation(); + TestInvalidLateWriteTokenIsConsumed(); + TestCrossRegionRange(); + TestConcurrentFault(); + TestExternalDirtyTransferDuringResolution(); + TestMappingDoesNotRequireCpuWriteAccess(); + TestGpuAccessPermissions(); + TestFatalPaths(); + std::puts("PageManagerTests: all cases passed"); + return 0; +#else + (void)argc; + (void)argv; + std::fputs("PageManagerTests: unsupported platform\n", stderr); + return 1; +#endif +} diff --git a/tests/ResourceMutexTests.cpp b/tests/ResourceMutexTests.cpp new file mode 100644 index 000000000..773a3533a --- /dev/null +++ b/tests/ResourceMutexTests.cpp @@ -0,0 +1,244 @@ +#include "graphics/host_gpu/renderer/resourceMutex.h" + +#include +#include +#include +#include +#include +#include +#include +#include + +#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS +#ifndef NOMINMAX +#define NOMINMAX +#endif +#include +#undef min +#undef max +#endif + +namespace { + +using Libs::Graphics::ResourceMutex; + +void Check(bool value, const char* text) { + if (!value) { + std::fprintf(stderr, "ResourceMutexTests: failed: %s\n", text); + std::abort(); + } +} + +void YieldMany() { + for (uint32_t i = 0; i < 4096; i++) { + std::this_thread::yield(); + } +} + +void TestFaultBlocksPublisher() { + ResourceMutex mutex; + std::atomic_bool publisher_started {false}; + std::atomic_bool publisher_entered {false}; + std::thread publisher; + { + ResourceMutex::FaultScope fault(mutex); + publisher = std::thread([&] { + publisher_started.store(true, std::memory_order_release); + std::lock_guard lock(mutex); + publisher_entered.store(true, std::memory_order_release); + }); + while (!publisher_started.load(std::memory_order_acquire)) { + std::this_thread::yield(); + } + YieldMany(); + Check(!publisher_entered.load(std::memory_order_acquire), + "publisher entered during active fault transaction"); + } + publisher.join(); + Check(publisher_entered.load(std::memory_order_acquire), + "publisher did not resume after fault transaction"); +} + +void TestFaultDrainsExistingOwner() { + ResourceMutex mutex; + std::unique_lock owner(mutex); + std::atomic_bool fault_started {false}; + std::atomic_bool fault_entered {false}; + std::atomic_bool release_fault {false}; + std::atomic_bool publisher_started {false}; + std::atomic_bool publisher_entered {false}; + std::thread fault([&] { + fault_started.store(true, std::memory_order_release); + ResourceMutex::FaultScope scope(mutex); + fault_entered.store(true, std::memory_order_release); + while (!release_fault.load(std::memory_order_acquire)) { + std::this_thread::yield(); + } + }); + while (!fault_started.load(std::memory_order_acquire)) { + std::this_thread::yield(); + } + YieldMany(); + Check(!fault_entered.load(std::memory_order_acquire), + "fault transaction did not wait for existing owner"); + owner.unlock(); + while (!fault_entered.load(std::memory_order_acquire)) { + std::this_thread::yield(); + } + std::thread publisher([&] { + publisher_started.store(true, std::memory_order_release); + std::lock_guard lock(mutex); + publisher_entered.store(true, std::memory_order_release); + }); + while (!publisher_started.load(std::memory_order_acquire)) { + std::this_thread::yield(); + } + YieldMany(); + Check(!publisher_entered.load(std::memory_order_acquire), + "publisher entered before drained fault completed"); + release_fault.store(true, std::memory_order_release); + fault.join(); + publisher.join(); + Check(publisher_entered.load(std::memory_order_acquire), + "publisher did not resume after drained fault"); +} + +void TestFaultScopesSerialize() { + ResourceMutex mutex; + std::atomic_bool first_entered {false}; + std::atomic_bool second_started {false}; + std::atomic_bool second_entered {false}; + std::atomic_bool release_first {false}; + std::atomic_bool release_second {false}; + std::atomic_bool publisher_entered {false}; + std::thread first([&] { + ResourceMutex::FaultScope scope(mutex); + first_entered.store(true, std::memory_order_release); + while (!release_first.load(std::memory_order_acquire)) { + std::this_thread::yield(); + } + }); + while (!first_entered.load(std::memory_order_acquire)) { + std::this_thread::yield(); + } + std::thread second([&] { + second_started.store(true, std::memory_order_release); + ResourceMutex::FaultScope scope(mutex); + second_entered.store(true, std::memory_order_release); + while (!release_second.load(std::memory_order_acquire)) { + std::this_thread::yield(); + } + }); + while (!second_started.load(std::memory_order_acquire)) { + std::this_thread::yield(); + } + YieldMany(); + Check(!second_entered.load(std::memory_order_acquire), + "second fault transaction entered before first completed"); + release_first.store(true, std::memory_order_release); + while (!second_entered.load(std::memory_order_acquire)) { + std::this_thread::yield(); + } + std::thread publisher([&] { + std::lock_guard lock(mutex); + publisher_entered.store(true, std::memory_order_release); + }); + YieldMany(); + Check(!publisher_entered.load(std::memory_order_acquire), + "publisher entered during second fault transaction"); + release_second.store(true, std::memory_order_release); + first.join(); + second.join(); + publisher.join(); + Check(publisher_entered.load(std::memory_order_acquire), + "publisher did not resume after serialized faults"); +} + +void TestPreownedFaultKeepsResourceTransaction() { + ResourceMutex mutex; + std::atomic_bool publisher_started {false}; + std::atomic_bool publisher_entered {false}; + std::thread publisher; + std::unique_lock owner(mutex); + { + ResourceMutex::FaultScope fault(mutex); + publisher = std::thread([&] { + publisher_started.store(true, std::memory_order_release); + std::lock_guard lock(mutex); + publisher_entered.store(true, std::memory_order_release); + }); + while (!publisher_started.load(std::memory_order_acquire)) { + std::this_thread::yield(); + } + YieldMany(); + Check(!publisher_entered.load(std::memory_order_acquire), + "publisher entered during preowned fault transaction"); + } + YieldMany(); + Check(!publisher_entered.load(std::memory_order_acquire), + "preowned fault released its outer resource transaction"); + owner.unlock(); + publisher.join(); + Check(publisher_entered.load(std::memory_order_acquire), + "publisher did not resume after outer transaction"); +} + +#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS +[[noreturn]] void RunDeathCase(const char* name) { + ResourceMutex mutex; + if (std::strcmp(name, "recursive-lock") == 0) { + std::lock_guard first(mutex); + std::lock_guard second(mutex); + } else if (std::strcmp(name, "nested-fault") == 0) { + ResourceMutex::FaultScope first(mutex); + ResourceMutex::FaultScope second(mutex); + } + std::_Exit(0x7f); +} + +void CheckDeathCase(const char* name) { + char path[MAX_PATH] {}; + Check(GetModuleFileNameA(nullptr, path, MAX_PATH) != 0, "GetModuleFileName failed"); + std::string command = std::string("\"") + path + "\" --death " + name; + std::vector mutable_command(command.begin(), command.end()); + mutable_command.push_back('\0'); + STARTUPINFOA startup {sizeof(startup)}; + PROCESS_INFORMATION process {}; + Check(CreateProcessA(nullptr, mutable_command.data(), nullptr, nullptr, FALSE, CREATE_NO_WINDOW, + nullptr, nullptr, &startup, &process) != 0, + "CreateProcess failed"); + const auto wait = WaitForSingleObject(process.hProcess, 10000); + if (wait != WAIT_OBJECT_0) { + TerminateProcess(process.hProcess, 0x7e); + } + Check(wait == WAIT_OBJECT_0, "ResourceMutex death test timed out"); + DWORD exit_code = 0; + Check(GetExitCodeProcess(process.hProcess, &exit_code) != 0 && exit_code == 321, + "ResourceMutex death path used the wrong exit"); + CloseHandle(process.hThread); + CloseHandle(process.hProcess); +} +#endif + +} // namespace + +int main(int argc, char** argv) { +#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS + if (argc == 3 && std::strcmp(argv[1], "--death") == 0) { + RunDeathCase(argv[2]); + } +#else + (void)argc; + (void)argv; +#endif + TestFaultBlocksPublisher(); + TestFaultDrainsExistingOwner(); + TestFaultScopesSerialize(); + TestPreownedFaultKeepsResourceTransaction(); +#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS + CheckDeathCase("recursive-lock"); + CheckDeathCase("nested-fault"); +#endif + std::puts("ResourceMutexTests: all cases passed"); + return 0; +} diff --git a/tests/ResourceTrackingTests.cpp b/tests/ResourceTrackingTests.cpp new file mode 100644 index 000000000..a32311c1c --- /dev/null +++ b/tests/ResourceTrackingTests.cpp @@ -0,0 +1,1783 @@ +#include "graphics/shader/recompiler/BindingLayout.h" +#include "graphics/shader/recompiler/ResourceMaterialization.h" +#include "graphics/shader/recompiler/ResourceTracking.h" +#include "graphics/shader/recompiler/ScalarProvenance.h" +#include "graphics/shader/recompiler/ShaderInfoCollection.h" +#include "graphics/shader/recompiler/SrtPatcher.h" +#include "graphics/shader/recompiler/SrtWalker.h" +#include "graphics/guest_gpu/gpu_defs.h" +#include "graphics/shader/shaderBindings.h" + +#include +#include +#include +#include + +namespace { + +using namespace Libs::Graphics::ShaderRecompiler::IR; +using Libs::Graphics::ShaderComputeInputInfo; +using Libs::Graphics::ShaderPixelInputInfo; +using Libs::Graphics::ShaderTextureResource; +using Libs::Graphics::ShaderType; +using Libs::Graphics::ShaderVertexInputInfo; +namespace Prospero = Libs::Graphics::Prospero; +namespace Decoder = Libs::Graphics::ShaderRecompiler::Decoder; + +void Check(bool condition, const char *message) { + if (!condition) { + throw std::runtime_error(message); + } +} + +Operand Sgpr(uint32_t reg) { + Operand operand; + operand.kind = OperandKind::Register; + operand.reg.file = RegisterFile::Scalar; + operand.reg.index = reg; + return operand; +} + +Operand Imm(uint32_t value) { + Operand operand; + operand.kind = OperandKind::ImmediateU32; + operand.imm = value; + return operand; +} + +Instruction Move(uint32_t pc, uint32_t dst, uint32_t src) { + Instruction inst; + inst.pc = pc; + inst.op = Opcode::MoveU32; + inst.dst = Sgpr(dst); + inst.src[0] = Sgpr(src); + inst.src_count = 1; + return inst; +} + +Instruction MoveImmediate(uint32_t pc, uint32_t dst, uint32_t value) { + auto inst = Move(pc, dst, 0); + inst.src[0] = Imm(value); + return inst; +} + +Instruction BufferUse(uint32_t pc, uint32_t base, + Opcode op = Opcode::BufferLoadDword) { + Instruction inst; + inst.pc = pc; + inst.op = op; + inst.memory.kind = ResourceKind::Buffer; + inst.memory.resource = base / 4; + return inst; +} + +Instruction ImageUse(uint32_t pc, Opcode op, ResourceKind kind, + Decoder::ImageDimension dimension, uint32_t resource = 0, + uint32_t sampler = 2) { + Instruction inst; + inst.pc = pc; + inst.op = op; + inst.memory.kind = kind; + inst.memory.resource = resource; + inst.memory.sampler = sampler; + inst.memory.image_dimension = dimension; + return inst; +} + +Instruction ScalarLoad(uint32_t pc, uint32_t dst, uint32_t base, + uint32_t offset) { + Instruction inst; + inst.pc = pc; + inst.op = Opcode::SLoadDword; + inst.dst = Sgpr(dst); + inst.memory.kind = ResourceKind::ScalarBuffer; + inst.memory.resource = base; + inst.memory.offset = offset; + return inst; +} + +Instruction ScalarBufferLoad(uint32_t pc, uint32_t dst, uint32_t descriptor, + uint32_t offset) { + auto inst = ScalarLoad(pc, dst, descriptor * 4, offset); + inst.op = Opcode::SBufferLoadDword; + inst.memory.resource = descriptor; + return inst; +} + +void MakeScalarMemoryGroup(std::vector *instructions) { + const auto count = static_cast(instructions->size()); + for (uint32_t i = 0; i < count; i++) { + (*instructions)[i].memory.component_index = i; + (*instructions)[i].memory.component_count = count; + } +} + +Instruction Export(uint32_t pc, ExportTargetKind kind, uint32_t index = 0, + uint32_t en = 0xf) { + Instruction inst; + inst.pc = pc; + inst.op = Opcode::Export; + inst.export_info.kind = kind; + inst.export_info.index = index; + inst.export_info.en = en; + return inst; +} + +struct TestMemory { + uint64_t base = 0x1000; + std::array words{}; + uint32_t reads = 0; + uint32_t fail_after = UINT32_MAX; +}; + +bool ReadTestMemory(void *userdata, uint64_t address, uint32_t *value) { + auto *memory = static_cast(userdata); + if (memory == nullptr || value == nullptr || address < memory->base || + address - memory->base >= memory->words.size() * sizeof(uint32_t) || + memory->reads >= memory->fail_after) { + return false; + } + const auto index = + static_cast((address - memory->base) / sizeof(uint32_t)); + *value = memory->words[index]; + memory->reads++; + return true; +} + +const StageInput *FindInput(const ShaderInfo &info, StageInputKind kind, + uint32_t location = 0) { + for (const auto &input : info.inputs) { + if (input.kind == kind && input.location == location) { + return &input; + } + } + return nullptr; +} + +const StageOutput *FindOutput(const ShaderInfo &info, StageOutputKind kind, + uint32_t index = 0) { + for (const auto &output : info.outputs) { + if (output.kind == kind && output.index == index) { + return &output; + } + } + return nullptr; +} + +void Prepare(Program *program) { + std::string error; + if (!BuildScalarProvenance(program, &error) || + !BuildSrtPlan(program, &error) || !PatchSrtReads(program, &error) || + !TrackResources(program, &error)) { + throw std::runtime_error(error); + } +} + +void TestDenseBufferPatching() { + Program program; + program.blocks.resize(1); + auto first = BufferUse(4, 0); + first.memory.offset = 4; + first.memory.formatted = true; + auto write = BufferUse(8, 0, Opcode::BufferStoreDword); + write.memory.offset = 12; + auto atomic = BufferUse(10, 0, Opcode::AtomicAddU32); + auto &insts = program.blocks[0].instructions; + insts = {first, atomic, write, Move(12, 4, 20), + Move(16, 5, 21), Move(20, 6, 22), Move(24, 7, 23), BufferUse(28, 4)}; + + Prepare(&program); + Check(program.info.buffers.size() == 2, + "buffer sources were not densely deduplicated"); + const auto &resource = program.info.buffers[0]; + Check(resource.read && resource.written && resource.atomic && + resource.formatted && resource.max_byte_extent == 16 && + resource.first_use_pc == 4, + "buffer access facts were not merged"); + Check(insts[0].memory.resource == 0 && insts[1].memory.resource == 0 && + insts[2].memory.resource == 0 && insts[7].memory.resource == 1, + "buffer operands were not patched to dense indices"); + Check(insts[0].memory.resource_source == ScalarProvenance::Undefined && + insts[7].memory.resource_source == ScalarProvenance::Undefined, + "patched instructions retained duplicate descriptor source handles"); + std::string error; + Check(!TrackResources(&program, &error) && + error.find("already tracked") != std::string::npos, + "resource tracking was not guarded against a second patch pass"); +} + +void TestScalarAndVectorBufferAlias() { + Program program; + program.blocks.resize(1); + Instruction scalar; + scalar.pc = 4; + scalar.op = Opcode::SBufferLoadDword; + scalar.dst = Sgpr(20); + scalar.src[0] = Imm(0); + scalar.src_count = 1; + scalar.memory.kind = ResourceKind::ScalarBuffer; + scalar.memory.resource = 2; + program.blocks[0].instructions = {scalar, BufferUse(8, 8)}; + + Prepare(&program); + Check(program.info.buffers.size() == 1 && program.info.buffers[0].scalar, + "scalar and vector uses of one descriptor were split"); + Check(program.blocks[0].instructions[0].memory.resource == 0 && + program.blocks[0].instructions[1].memory.resource == 0, + "scalar/vector alias did not share one dense index"); +} + +void TestImagesAndSamplers() { + Program program; + program.blocks.resize(1); + auto sample0 = ImageUse(4, Opcode::ImageSample, ResourceKind::Image, + Decoder::ImageDimension::Dim2D); + auto sample1 = ImageUse(8, Opcode::ImageSample, ResourceKind::Image, + Decoder::ImageDimension::Dim2D); + auto volume = ImageUse(12, Opcode::ImageSample, ResourceKind::Image, + Decoder::ImageDimension::Dim3D); + auto compare = ImageUse(16, Opcode::ImageSample, ResourceKind::Image, + Decoder::ImageDimension::Dim2D); + compare.memory.image_sample_flags = Decoder::ImageSampleFlagCompare; + auto storage = ImageUse(20, Opcode::ImageStore, ResourceKind::StorageImage, + Decoder::ImageDimension::Dim2D); + auto storage_mip = storage; + storage_mip.pc = 24; + storage_mip.memory.image_has_mip = true; + auto atomic = + ImageUse(28, Opcode::AtomicAddU32, ResourceKind::StorageImageUint, + Decoder::ImageDimension::Dim2D); + program.blocks[0].instructions = {sample0, sample1, volume, compare, + storage, storage_mip, atomic}; + + Prepare(&program); + Check(program.info.images.size() == 6 && program.info.samplers.size() == 1 && + program.info.sampled_pairs.size() == 3, + "image view classes or samplers were deduplicated incorrectly"); + const auto &insts = program.blocks[0].instructions; + Check(insts[0].memory.resource == 0 && insts[1].memory.resource == 0 && + insts[2].memory.resource == 1 && insts[3].memory.resource == 2 && + insts[4].memory.resource == 3 && insts[5].memory.resource == 4 && + insts[6].memory.resource == 5 && insts[0].memory.sampler == 0 && + insts[2].memory.sampler == 0, + "image/sampler operands were not patched to dense list indices"); + Check(program.info.images[0].read && !program.info.images[0].written && + program.info.images[2].depth_compare && + program.info.images[3].written && + program.info.images[4].mip_mode == ImageMipMode::DynamicStorage && + program.info.images[5].atomic && + program.info.sampled_pairs[0].sampler == + program.info.sampled_pairs[1].sampler, + "image access facts were wrong"); +} + +void TestDynamicPhiResource() { + Program program; + program.blocks.resize(4); + program.blocks[0].successors = {1, 2}; + program.blocks[1].predecessors = {0}; + program.blocks[1].successors = {3}; + program.blocks[2].predecessors = {0}; + program.blocks[2].successors = {3}; + program.blocks[3].predecessors = {1, 2}; + Instruction left; + left.op = Opcode::MoveU32; + left.dst = Sgpr(0); + left.src[0] = Imm(1); + left.src_count = 1; + Instruction right = left; + right.pc = 4; + right.src[0] = Imm(2); + program.blocks[1].instructions = {left}; + program.blocks[2].instructions = {right}; + program.blocks[3].instructions = {BufferUse(8, 0)}; + + std::string error; + Check(BuildScalarProvenance(&program, &error) && + BuildSrtPlan(&program, &error) && PatchSrtReads(&program, &error), + error.c_str()); + const auto original_source = + program.blocks[3].instructions[0].memory.resource_source; + Check( + !TrackResources(&program, &error) && + error.find("unsupported GPU selection") != std::string::npos && + program.blocks[3].instructions[0].memory.resource_source == + original_source && + !program.resource_tracking_complete, + "control-flow descriptor was patched without an executable GPU selector"); +} + +void TestTrackingRequiresCompletedSrtPlan() { + Program program; + program.stage = ShaderType::Compute; + program.shader_hash = 0x8899; + program.blocks.resize(4); + program.blocks[0].successors = {1, 2}; + program.blocks[1].predecessors = {0}; + program.blocks[1].successors = {3}; + program.blocks[2].predecessors = {0}; + program.blocks[2].successors = {3}; + program.blocks[3].predecessors = {1, 2}; + program.blocks[1].instructions = {MoveImmediate(0, 0, 1)}; + program.blocks[2].instructions = {MoveImmediate(4, 0, 2)}; + program.blocks[3].instructions = {BufferUse(8, 0)}; + + std::string error; + Check(BuildScalarProvenance(&program, &error), error.c_str()); + Check(!program.srt_plan_complete, + "provenance unexpectedly marked the SRT plan complete"); + const auto original_source = + program.blocks[3].instructions[0].memory.resource_source; + BufferResource existing_info; + existing_info.source = 777; + program.info.buffers.push_back(existing_info); + Check(!TrackResources(&program, &error) && + error.find("SRT plan is not ready") != std::string::npos && + program.blocks[3].instructions[0].memory.resource == 0 && + program.blocks[3].instructions[0].memory.resource_source == + original_source && + program.info.buffers.size() == 1 && + program.info.buffers[0].source == 777 && + !program.resource_tracking_complete, + "tracking bypassed the SRT-plan readiness invariant or partially " + "patched the program"); +} + +void TestCyclicResourceIsRejected() { + Program program; + program.blocks.resize(2); + program.blocks[0].successors = {1}; + program.blocks[1].predecessors = {0, 1}; + program.blocks[1].successors = {1}; + Instruction increment; + increment.op = Opcode::IAddU32; + increment.dst = Sgpr(0); + increment.src[0] = Sgpr(0); + increment.src[1] = Imm(1); + increment.src_count = 2; + program.blocks[1].instructions = {BufferUse(4, 0), increment}; + std::string error; + Check(BuildScalarProvenance(&program, &error) && + BuildSrtPlan(&program, &error) && PatchSrtReads(&program, &error), + error.c_str()); + Check(!TrackResources(&program, &error) && + error.find("unsupported GPU selection") != std::string::npos && + !program.resource_tracking_complete, + "cyclic descriptor was patched without a bindless/direct path"); +} + +void TestUnknownSourceFailsWithoutPatching() { + Program program; + program.stage = ShaderType::Pixel; + program.shader_hash = 0x12345678; + program.blocks.resize(1); + auto valid = BufferUse(4, 0); + Instruction unsupported; + unsupported.op = Opcode::SelectU32; + unsupported.dst = Sgpr(4); + unsupported.src[0] = Sgpr(20); + unsupported.src_count = 1; + program.blocks[0].instructions = {valid, unsupported, BufferUse(0x44, 4)}; + + std::string error; + Check(BuildScalarProvenance(&program, &error) && + BuildSrtPlan(&program, &error) && PatchSrtReads(&program, &error), + error.c_str()); + BufferResource existing_info; + existing_info.source = 777; + program.info.buffers.push_back(existing_info); + Check(!TrackResources(&program, &error), + "unknown descriptor unexpectedly tracked"); + Check(error.find("hash=0x0000000012345678") != std::string::npos && + error.find("stage=pixel") != std::string::npos && + error.find("pc=0x00000044") != std::string::npos && + error.find("unknown value") != std::string::npos, + "unknown descriptor error lost shader context"); + Check(!program.resource_tracking_complete && + program.info.buffers.size() == 1 && + program.info.buffers[0].source == 777 && + program.blocks[0].instructions[0].memory.resource == 0 && + program.blocks[0].instructions[0].memory.resource_source != + ScalarProvenance::Undefined && + program.blocks[0].instructions[2].memory.resource == 1 && + program.blocks[0].instructions[2].memory.resource_source != + ScalarProvenance::Undefined, + "failed tracking partially patched the program"); +} + +void TestResourceLimitFailsTransactionally() { + Program program; + program.stage = ShaderType::Compute; + program.blocks.resize(1); + auto &insts = program.blocks[0].instructions; + for (uint32_t i = 0; i <= ShaderInfo::MaxBuffers; i++) { + insts.push_back(MoveImmediate(i * 8, 0, i)); + insts.push_back(BufferUse(i * 8 + 4, 0)); + } + std::string error; + Check(BuildScalarProvenance(&program, &error) && + BuildSrtPlan(&program, &error) && PatchSrtReads(&program, &error), + error.c_str()); + const auto first_source = insts[1].memory.resource_source; + Check(!TrackResources(&program, &error) && + error.find("buffer resource limit exceeded") != std::string::npos && + insts[1].memory.resource_source == first_source && + program.info.buffers.empty() && !program.resource_tracking_complete, + "resource limit failure partially patched the program"); +} + +void TestComputeShaderInfoCollection() { + Program program; + program.stage = ShaderType::Compute; + program.blocks.resize(1); + Instruction add_tid; + add_tid.op = Opcode::DsReadAddtidB32; + program.blocks[0].instructions = {add_tid, BufferUse(4, 0)}; + Prepare(&program); + const auto buffers = program.info.buffers; + const auto images = program.info.images; + const auto samplers = program.info.samplers; + const auto pairs = program.info.sampled_pairs; + + ShaderComputeInputInfo compute; + compute.group_id[1] = true; + compute.dispatch_thread_dimensions = true; + std::string error; + Check(CollectShaderInfo(&program, {.compute = &compute}, &error), + error.c_str()); + Check(program.shader_info_complete && program.info.inputs.size() == 3 && + FindInput(program.info, StageInputKind::WorkgroupId) != nullptr && + FindInput(program.info, StageInputKind::LocalInvocationId) == nullptr && + FindInput(program.info, StageInputKind::LocalInvocationIndex) != nullptr && + FindInput(program.info, StageInputKind::GlobalInvocationId) != nullptr && + program.info.buffers == buffers && program.info.images == images && + program.info.samplers == samplers && + program.info.sampled_pairs == pairs, + "compute addtid input discovery or resource preservation was wrong"); + + Program metadata_program; + metadata_program.stage = ShaderType::Compute; + metadata_program.blocks.resize(1); + Prepare(&metadata_program); + compute = {}; + compute.thread_ids_num = 2; + Check(CollectShaderInfo(&metadata_program, {.compute = &compute}, &error), + error.c_str()); + Check(FindInput(metadata_program.info, StageInputKind::LocalInvocationId) != + nullptr && + FindInput(metadata_program.info, + StageInputKind::LocalInvocationIndex) != nullptr, + "compute thread metadata did not request local invocation inputs"); +} + +void TestVertexShaderInfoCollection() { + Program program; + program.stage = ShaderType::Vertex; + program.blocks.resize(1); + Instruction attr0; + attr0.op = Opcode::LoadInputF32; + attr0.input_info.attr = 0; + attr0.input_info.chan = 2; + Instruction attr2 = attr0; + attr2.input_info.attr = 2; + attr2.input_info.chan = 0; + program.blocks[0].instructions = { + attr0, attr2, Export(8, ExportTargetKind::Position), + Export(12, ExportTargetKind::Parameter, 2)}; + Prepare(&program); + + ShaderVertexInputInfo vertex; + vertex.resources_num = 3; + std::string error; + Check(CollectShaderInfo(&program, {.vertex = &vertex}, &error), + error.c_str()); + const auto *input0 = FindInput(program.info, StageInputKind::Parameter, 0); + const auto *input2 = FindInput(program.info, StageInputKind::Parameter, 2); + Check(program.info.inputs.size() == 4 && input0 != nullptr && + input0->component_count == 3 && input2 != nullptr && + input2->component_count == 1 && + FindInput(program.info, StageInputKind::VertexIndex) != nullptr && + FindInput(program.info, StageInputKind::InstanceIndex) != nullptr && + FindOutput(program.info, StageOutputKind::Position) != nullptr && + FindOutput(program.info, StageOutputKind::Parameter, 2) != nullptr, + "vertex inputs or exports were not collected from lowered IR"); + + const auto info = program.info; + const auto resources_complete = program.resource_tracking_complete; + const auto srt_complete = program.srt_plan_complete; + const auto patching_complete = program.srt_patching_complete; + Check(!CollectShaderInfo(&program, {.vertex = &vertex}, &error) && + error.find("already collected") != std::string::npos && + program.info == info && program.shader_info_complete && + program.resource_tracking_complete == resources_complete && + program.srt_plan_complete == srt_complete && + program.srt_patching_complete == patching_complete, + "repeated shader info collection mutated the immutable interface"); +} + +void TestPixelShaderInfoCollection() { + Program program; + program.stage = ShaderType::Pixel; + program.blocks.resize(1); + program.blocks[0].instructions = { + Export(4, ExportTargetKind::Mrt, 3), + Export(8, ExportTargetKind::MrtZ, 0, 0x4), + Export(12, ExportTargetKind::Mrt, 3), + Export(16, ExportTargetKind::Mrt, 7, 0)}; + Prepare(&program); + + ShaderPixelInputInfo pixel; + pixel.ps_pos_x = true; + pixel.ps_front_face = true; + pixel.input_num = 2; + pixel.ps_depth_export_enable = true; + pixel.ps_sample_mask_export_enable = true; + std::string error; + Check(CollectShaderInfo(&program, {.pixel = &pixel}, &error), + error.c_str()); + Check(program.info.inputs.size() == 4 && + FindInput(program.info, StageInputKind::FragCoord) != nullptr && + FindInput(program.info, StageInputKind::FrontFacing) != nullptr && + FindInput(program.info, StageInputKind::Parameter, 0) != nullptr && + FindInput(program.info, StageInputKind::Parameter, 1) != nullptr && + program.info.outputs.size() == 2 && + FindOutput(program.info, StageOutputKind::Mrt, 3) != nullptr && + FindOutput(program.info, StageOutputKind::Mrt, 7) == nullptr && + FindOutput(program.info, StageOutputKind::Depth) == nullptr && + FindOutput(program.info, StageOutputKind::SampleMask) != nullptr, + "pixel inputs, disabled exports, or sample-mask-only MRTZ collection was wrong"); + + Program depth_program; + depth_program.stage = ShaderType::Pixel; + depth_program.blocks.resize(1); + depth_program.blocks[0].instructions = { + Export(4, ExportTargetKind::MrtZ, 0, 0x1)}; + Prepare(&depth_program); + Check(CollectShaderInfo(&depth_program, {.pixel = &pixel}, &error), + error.c_str()); + Check(depth_program.info.outputs.size() == 1 && + FindOutput(depth_program.info, StageOutputKind::Depth) != nullptr && + FindOutput(depth_program.info, StageOutputKind::SampleMask) == nullptr, + "depth-only MRTZ export incorrectly enabled sample-mask output"); +} + +void TestShaderInfoCollectionIsTransactional() { + Program program; + program.stage = ShaderType::Compute; + StageInput sentinel; + sentinel.kind = StageInputKind::Parameter; + sentinel.location = 9; + program.info.inputs.push_back(sentinel); + const auto info = program.info; + std::string error; + Check(!CollectShaderInfo(&program, {}, &error) && + error.find("not tracked") != std::string::npos && + program.info == info && !program.shader_info_complete, + "pre-track shader info collection mutated program state"); + + Prepare(&program); + const auto tracked_info = program.info; + const auto srt_complete = program.srt_plan_complete; + const auto patching_complete = program.srt_patching_complete; + Check(!CollectShaderInfo(&program, {}, &error) && + error.find("requires compute metadata") != std::string::npos && + program.info == tracked_info && !program.shader_info_complete && + program.resource_tracking_complete && + program.srt_plan_complete == srt_complete && + program.srt_patching_complete == patching_complete, + "missing stage metadata committed incomplete shader info"); + program.stage = ShaderType::Unknown; + Check(!CollectShaderInfo(&program, {}, &error) && + error.find("unsupported shader stage") != std::string::npos && + program.info == tracked_info && !program.shader_info_complete && + program.resource_tracking_complete && + program.srt_plan_complete == srt_complete && + program.srt_patching_complete == patching_complete, + "unsupported-stage shader info collection was not transactional"); +} + +void TestShaderInfoMetadataValidation() { + std::string error; + + Program vertex_program; + vertex_program.stage = ShaderType::Vertex; + vertex_program.blocks.resize(1); + Instruction bad_input; + bad_input.op = Opcode::LoadInputF32; + bad_input.input_info.attr = 0; + bad_input.input_info.chan = 4; + vertex_program.blocks[0].instructions = {bad_input}; + Prepare(&vertex_program); + ShaderVertexInputInfo vertex; + vertex.resources_num = 1; + const auto vertex_info = vertex_program.info; + Check(!CollectShaderInfo(&vertex_program, {.vertex = &vertex}, &error) && + error.find("vertex input reference") != std::string::npos && + vertex_program.info == vertex_info && + !vertex_program.shader_info_complete, + "out-of-range vertex channel produced immutable malformed info"); + vertex.resources_num = -1; + Check(!CollectShaderInfo(&vertex_program, {.vertex = &vertex}, &error) && + error.find("vertex resource count") != std::string::npos && + vertex_program.info == vertex_info && + !vertex_program.shader_info_complete, + "negative vertex resource count produced immutable malformed info"); + + Program pixel_program; + pixel_program.stage = ShaderType::Pixel; + pixel_program.blocks.resize(1); + Prepare(&pixel_program); + ShaderPixelInputInfo pixel; + pixel.input_num = 33; + const auto pixel_info = pixel_program.info; + Check(!CollectShaderInfo(&pixel_program, {.pixel = &pixel}, &error) && + error.find("pixel input count") != std::string::npos && + pixel_program.info == pixel_info && + !pixel_program.shader_info_complete, + "out-of-range pixel input count produced immutable malformed info"); + + Program compute_program; + compute_program.stage = ShaderType::Compute; + compute_program.blocks.resize(1); + Prepare(&compute_program); + ShaderComputeInputInfo compute; + compute.thread_ids_num = 4; + const auto compute_info = compute_program.info; + Check(!CollectShaderInfo(&compute_program, {.compute = &compute}, &error) && + error.find("thread ID count") != std::string::npos && + compute_program.info == compute_info && + !compute_program.shader_info_complete, + "out-of-range compute metadata produced immutable malformed info"); +} + +void TestTrackingRequiresSrtPatching() { + Program program; + program.blocks.resize(1); + program.blocks[0].instructions = {BufferUse(4, 0)}; + std::string error; + Check(BuildScalarProvenance(&program, &error) && + BuildSrtPlan(&program, &error), + error.c_str()); + const auto source = program.blocks[0].instructions[0].memory.resource_source; + Check(!TrackResources(&program, &error) && + error.find("SRT reads were not patched") != std::string::npos && + program.blocks[0].instructions[0].memory.resource_source == source && + !program.resource_tracking_complete, + "resource tracking bypassed SRT patch completion"); + Check(PatchSrtReads(&program, &error) && TrackResources(&program, &error), + error.c_str()); +} + +void TestDynamicSrtReadRemainsExplicit() { + Program program; + program.blocks.resize(1); + auto &insts = program.blocks[0].instructions; + for (uint32_t i = 0; i < 8; i++) { + auto load = ScalarLoad(i * 4, i, 16, i * 4); + load.src[0] = Sgpr(20); + load.src_count = 1; + insts.push_back(load); + } + insts.push_back(ImageUse(0x40, Opcode::ImageStore, + ResourceKind::StorageImage, + Decoder::ImageDimension::Dim2D)); + std::string error; + Check(BuildScalarProvenance(&program, &error) && + BuildSrtPlan(&program, &error) && PatchSrtReads(&program, &error), + error.c_str()); + Check(program.srt.reads.empty() && program.srt.dynamic_reads.size() == 8 && + program.srt_patching_complete, + "dynamic SRT offsets were incorrectly assigned fixed flat slots"); + for (uint32_t i = 0; i < 8; i++) { + Check(insts[i].op == Opcode::SLoadDword && + insts[i].memory.kind == ResourceKind::ScalarBuffer, + "dynamic SRT read was rewritten as an immediate flat load"); + } + Check(TrackResources(&program, &error), error.c_str()); +} + +void TestSrtPatchingFailureIsTransactional() { + Program program; + program.blocks.resize(1); + auto &insts = program.blocks[0].instructions; + for (uint32_t i = 0; i < 4; i++) { + insts.push_back(ScalarLoad(i * 4, i, 16, i * 4)); + } + insts.push_back(BufferUse(0x20, 0)); + std::string error; + Check(BuildScalarProvenance(&program, &error) && + BuildSrtPlan(&program, &error) && program.srt.reads.size() == 4, + error.c_str()); + insts[0].scalar_value = ScalarProvenance::Undefined; + const auto instructions = insts; + const auto provenance = program.provenance; + const auto srt = program.srt; + Check(!PatchSrtReads(&program, &error) && + error.find("no scalar-load producer") != std::string::npos && + insts == instructions && program.provenance == provenance && + program.srt == srt && !program.srt_patching_complete, + "failed SRT patching partially changed the program"); +} + +void TestScalarMemoryGroupsSnapshotOperands() { + const auto CheckGroup = [](bool buffer) { + Program program; + program.blocks.resize(1); + auto &insts = program.blocks[0].instructions; + for (uint32_t i = 0; i < 4; i++) { + insts.push_back(buffer ? ScalarBufferLoad(4, 16 + i, 4, i * 4) + : ScalarLoad(4, 16 + i, 16, i * 4)); + } + MakeScalarMemoryGroup(&insts); + insts.push_back(BufferUse(8, 16)); + + std::string error; + Check(BuildScalarProvenance(&program, &error) && + BuildSrtPlan(&program, &error) && program.srt.reads.size() == 4 && + PatchSrtReads(&program, &error), + error.c_str()); + for (uint32_t i = 0; i < 4; i++) { + Check(insts[i].op == Opcode::LoadSrtDword && insts[i].src[0].imm == i, + "overlapping scalar-memory group did not snapshot its operands"); + } + }; + CheckGroup(false); + CheckGroup(true); + + const auto CheckOffsetOverlap = [](bool buffer) { + Program program; + program.blocks.resize(1); + auto &insts = program.blocks[0].instructions; + for (uint32_t i = 0; i < 4; i++) { + auto load = buffer ? ScalarBufferLoad(4, 20 + i, 4, i * 4) + : ScalarLoad(4, 20 + i, 16, i * 4); + load.src[0] = Sgpr(20); + load.src_count = 1; + insts.push_back(load); + } + MakeScalarMemoryGroup(&insts); + insts.push_back(BufferUse(8, 20)); + + std::string error; + Check(BuildScalarProvenance(&program, &error) && + BuildSrtPlan(&program, &error) && + program.srt.dynamic_reads.size() == 4 && + PatchSrtReads(&program, &error), + error.c_str()); + for (uint32_t i = 0; i < 4; i++) { + const auto value = insts[i].scalar_value; + const auto &node = program.provenance.values[value]; + const auto offset_arg = buffer ? 4u : 2u; + Check(program.provenance.values[node.args[offset_arg]].op == + ScalarValueOp::UserData && + program.provenance.values[node.args[offset_arg]].imm == 20 && + insts[i].op == + (buffer ? Opcode::SBufferLoadDword : Opcode::SLoadDword), + "overlapping scalar-memory offset was read after a component write"); + } + }; + CheckOffsetOverlap(false); + CheckOffsetOverlap(true); +} + +void TestSrtPatchingHandlesGvnAndMoveForwarding() { + Program program; + program.blocks.resize(1); + auto &insts = program.blocks[0].instructions; + for (uint32_t copy = 0; copy < 2; copy++) { + for (uint32_t i = 0; i < 4; i++) { + insts.push_back(ScalarLoad(copy * 0x20 + i * 4, copy * 4 + i, 16, + i * 4)); + } + } + for (uint32_t i = 0; i < 4; i++) { + insts.push_back(Move(0x40 + i * 4, 8 + i, i)); + } + insts.push_back(BufferUse(0x60, 4)); + insts.push_back(BufferUse(0x64, 8)); + + std::string error; + Check(BuildScalarProvenance(&program, &error) && + BuildSrtPlan(&program, &error) && program.srt.reads.size() == 4 && + PatchSrtReads(&program, &error), + error.c_str()); + for (uint32_t i = 0; i < 8; i++) { + Check(insts[i].op == Opcode::LoadSrtDword && + insts[i].src[0].imm == i % 4, + "all producers of a GVN'd SRT read were not patched"); + } + for (uint32_t i = 8; i < 12; i++) { + Check(insts[i].op == Opcode::MoveU32, + "move forwarding was incorrectly rewritten as an SRT load"); + } +} + +void TestSrtPatchingHandlesCfgProducers() { + Program program; + program.blocks.resize(4); + program.blocks[0].successors = {1, 2}; + program.blocks[1].predecessors = {0}; + program.blocks[1].successors = {3}; + program.blocks[2].predecessors = {0}; + program.blocks[2].successors = {3}; + program.blocks[3].predecessors = {1, 2}; + for (uint32_t block = 1; block <= 2; block++) { + for (uint32_t i = 0; i < 4; i++) { + program.blocks[block].instructions.push_back( + ScalarLoad(block * 0x20 + i * 4, i, 16, i * 4)); + } + } + program.blocks[3].instructions = {BufferUse(0x60, 0)}; + + std::string error; + Check(BuildScalarProvenance(&program, &error) && + BuildSrtPlan(&program, &error) && program.srt.reads.size() == 4 && + PatchSrtReads(&program, &error), + error.c_str()); + for (uint32_t block = 1; block <= 2; block++) { + for (uint32_t i = 0; i < 4; i++) { + const auto &inst = program.blocks[block].instructions[i]; + Check(inst.op == Opcode::LoadSrtDword && inst.src[0].imm == i, + "CFG-equivalent SRT producer was not patched"); + } + } +} + +void TestSrtPatchPlanValidation() { + Program program; + program.blocks.resize(1); + auto &insts = program.blocks[0].instructions; + for (uint32_t i = 0; i < 4; i++) { + insts.push_back(ScalarLoad(i * 4, i, 16, i * 4)); + } + insts.push_back(BufferUse(0x20, 0)); + std::string error; + Check(BuildScalarProvenance(&program, &error) && + BuildSrtPlan(&program, &error), + error.c_str()); + program.srt.reads = {program.srt.reads[0], program.srt.reads[0]}; + const auto instructions = insts; + Check(!PatchSrtReads(&program, &error) && + error.find("dense value-to-offset bijection") != std::string::npos && + insts == instructions && !program.srt_patching_complete, + "duplicate SRT flat slots were accepted or partially patched"); + + Program dynamic; + dynamic.blocks.resize(1); + dynamic.blocks[0].instructions = {BufferUse(4, 0)}; + Check(BuildScalarProvenance(&dynamic, &error) && + BuildSrtPlan(&dynamic, &error), + error.c_str()); + dynamic.srt.dynamic_sources = {ScalarProvenance::Undefined}; + Check(!PatchSrtReads(&dynamic, &error) && + error.find("invalid dynamic descriptor source") != std::string::npos && + !dynamic.srt_patching_complete, + "undefined dynamic descriptor source was accepted"); +} + +void TestMaterializationSharesReadConstEvaluation() { + Program program; + program.stage = ShaderType::Pixel; + program.shader_hash = 0x10203040; + program.blocks.resize(1); + auto &insts = program.blocks[0].instructions; + for (uint32_t i = 0; i < 8; i++) { + insts.push_back(ScalarLoad(i * 4, i, 16, i * 4)); + } + for (uint32_t i = 0; i < 4; i++) { + insts.push_back(MoveImmediate(0x20 + i * 4, 8 + i, 0xa0 + i)); + } + insts.push_back(ImageUse(0x40, Opcode::ImageSample, ResourceKind::Image, + Decoder::ImageDimension::Dim2D)); + insts.push_back(ImageUse(0x44, Opcode::ImageStore, + ResourceKind::StorageImage, + Decoder::ImageDimension::Dim2D)); + Prepare(&program); + Check(program.info.images.size() == 2 && program.info.samplers.size() == 1, + "materialization test did not preserve sampled/storage view topology"); + Check(program.srt.reads.size() == 8 && program.srt_patching_complete, + "immediate descriptor reads did not produce a compact SRT patch plan"); + for (uint32_t i = 0; i < 8; i++) { + Check(insts[i].op == Opcode::LoadSrtDword && insts[i].src_count == 1 && + insts[i].src[0].kind == OperandKind::ImmediateU32 && + insts[i].src[0].imm == i && + insts[i].memory.kind == ResourceKind::None, + "immediate SRT read was not patched to its dense flat-buffer slot"); + } + std::string error; + Check(!PatchSrtReads(&program, &error) && + error.find("already patched") != std::string::npos, + "repeated SRT patching was accepted"); + + TestMemory memory; + for (uint32_t i = 0; i < memory.words.size(); i++) { + memory.words[i] = 0x100 + i; + } + std::array user_data{}; + user_data[16] = static_cast(memory.base); + user_data[17] = static_cast(memory.base >> 32u); + SrtRuntime runtime{user_data, 0, ReadTestMemory, &memory}; + ResourceSnapshot snapshot; + error.clear(); + Check(MaterializeResources(program, runtime, &snapshot, &error), + error.c_str()); + Check(snapshot.buffers.empty() && snapshot.images.size() == 2 && + snapshot.samplers.size() == 1 && + snapshot.images[0] == snapshot.images[1], + "dense runtime snapshot did not preserve resource order or aliases"); + Check(snapshot.flattened_srt.size() == 8 && + std::equal(snapshot.flattened_srt.begin(), snapshot.flattened_srt.end(), + memory.words.begin()) && + snapshot.user_data == std::vector(user_data.begin(), user_data.end()), + "runtime snapshot omitted flattened SRT or current user data"); + Check(memory.reads == 8, + "aliased image views repeated ReadConst evaluation instead of sharing it"); + for (uint32_t i = 0; i < 8; i++) { + Check(snapshot.images[0].dwords[i] == memory.words[i], + "materialized image descriptor contains the wrong dword"); + } +} + +void TestMaterializationFailureIsTransactional() { + Program program; + program.blocks.resize(1); + auto &insts = program.blocks[0].instructions; + for (uint32_t i = 0; i < 8; i++) { + insts.push_back(ScalarLoad(i * 4, i, 16, i * 4)); + } + insts.push_back(ImageUse(0x40, Opcode::ImageStore, + ResourceKind::StorageImage, + Decoder::ImageDimension::Dim2D)); + Prepare(&program); + + TestMemory memory; + memory.fail_after = 3; + std::array user_data{}; + user_data[16] = static_cast(memory.base); + user_data[17] = static_cast(memory.base >> 32u); + SrtRuntime runtime{user_data, 0, ReadTestMemory, &memory}; + ResourceSnapshot snapshot; + DescriptorValue sentinel; + sentinel.dword_count = 1; + sentinel.dwords[0] = 777; + snapshot.images.push_back(sentinel); + std::string error; + Check(!MaterializeResources(program, runtime, &snapshot, &error) && + error.find("failed at") != std::string::npos && + snapshot.images.size() == 1 && + snapshot.images[0].dwords[0] == 777 && + snapshot.images[0].dword_count == 1, + "failed runtime materialization partially replaced the prior snapshot"); +} + +void TestResourceSpecializationIsTypedAndTransactional() { + Program null_program; + null_program.stage = ShaderType::Compute; + null_program.blocks.resize(1); + null_program.blocks[0].instructions = { + ImageUse(0x10, Opcode::ImageLoad, ResourceKind::Image, + Decoder::ImageDimension::Dim3D)}; + Prepare(&null_program); + ResourceSnapshot null_snapshot; + null_snapshot.images.resize(1); + null_snapshot.images[0].dword_count = 8; + null_snapshot.images[0].dwords[1] = 0x12345600u; + null_snapshot.images[0].dwords[2] = 0x89abcdefu; + null_snapshot.images[0].dwords[3] = 0x01234567u; + std::string error; + Check(SpecializeResources(&null_program, null_snapshot, &error) && + ValidateResourceSpecialization(null_program, null_snapshot, &error) && + null_program.info.images[0].kind == ResourceKind::Image && + null_program.info.images[0].dimension == Decoder::ImageDimension::Dim3D, + "zero-base image descriptor did not preserve tracked null-image shape"); + + Program program; + program.stage = ShaderType::Compute; + program.blocks.resize(1); + program.blocks[0].instructions = { + ImageUse(0x20, Opcode::ImageStore, ResourceKind::StorageImage, + Decoder::ImageDimension::Dim3D)}; + Prepare(&program); + + ResourceSnapshot snapshot; + snapshot.images.resize(1); + snapshot.images[0].dword_count = 7; + const auto info = program.info; + const auto memory = program.blocks[0].instructions[0].memory; + error.clear(); + Check(!ValidateResourceSnapshot(program, snapshot, &error) && + error.find("image descriptor 0 has 7 dwords") != std::string::npos, + "malformed resource snapshot was accepted"); + Check(!SpecializeResources(&program, snapshot, &error) && + program.info == info && + program.blocks[0].instructions[0].memory == memory, + "failed resource specialization partially mutated the program"); + + snapshot.images[0].dword_count = 8; + snapshot.images[0].dwords[0] = 0x1000; + snapshot.images[0].dwords[1] = + Prospero::GpuEnumValue(Prospero::BufferFormat::k32UInt) << 20u; + snapshot.images[0].dwords[3] = + (Prospero::GpuEnumValue(Prospero::ImageType::kColor2D) << 28u) | 0x3acu; + Check(SpecializeResources(&program, snapshot, &error), error.c_str()); + const auto &image = program.info.images[0]; + const auto &inst = program.blocks[0].instructions[0]; + Check(image.kind == ResourceKind::StorageImageUint && + image.dimension == Decoder::ImageDimension::Dim2D && + image.storage_swizzle == 0x3acu && + inst.memory.kind == ResourceKind::StorageImageUint && + inst.memory.image_dimension == Decoder::ImageDimension::Dim2D, + "runtime descriptor shape and integer format did not specialize dense IR"); + + auto stale_swizzle = snapshot; + stale_swizzle.images[0].dwords[3] = + (stale_swizzle.images[0].dwords[3] & ~0xfffu) | + StorageImageIdentitySwizzle; + Check(!ValidateResourceSpecialization(program, stale_swizzle, &error) && + error.find("changed swizzle") != std::string::npos, + "storage image cache validation ignored a SPIR-V-baked swizzle"); + + ShaderComputeInputInfo compute; + compute.thread_ids_num = 1; + Check(CollectShaderInfo(&program, {.compute = &compute}, &error) && + AllocateBindings(&program, {}, &error), + error.c_str()); + const auto *binding = + FindBinding(program.bindings, DescriptorBindingKind::StorageUint2D); + Check(binding != nullptr && binding->resources == std::vector({0}) && + FindBinding(program.bindings, DescriptorBindingKind::Storage3D) == nullptr, + "specialized image topology did not reach the exact native binding group"); +} + +void TestRuntimeSpecializationCoversBakedBufferAndAddressFields() { + Program buffer_program; + buffer_program.stage = ShaderType::Compute; + buffer_program.blocks.resize(1); + buffer_program.blocks[0].instructions = {BufferUse(0, 0)}; + Prepare(&buffer_program); + + ResourceSnapshot buffer_snapshot; + buffer_snapshot.buffers.resize(1); + buffer_snapshot.buffers[0].dword_count = 4; + buffer_snapshot.buffers[0].dwords[1] = (16u << 16u) | (1u << 31u); + buffer_snapshot.buffers[0].dwords[3] = + (Prospero::GpuEnumValue(Prospero::BufferFormat::k32UInt) << 12u) | (2u << 21u) | + (1u << 23u); + std::string error; + Check(SpecializeResources(&buffer_program, buffer_snapshot, &error) && + ValidateResourceSpecialization(buffer_program, buffer_snapshot, &error), + error.c_str()); + Check(buffer_program.info.buffers[0].packed_stride == + (16u | (1u << 14u) | (2u << 16u) | (1u << 20u)) && + buffer_program.info.buffers[0].descriptor_format == + Prospero::GpuEnumValue(Prospero::BufferFormat::k32UInt), + "buffer specialization omitted SPIR-V-baked descriptor fields"); + auto stale_buffer = buffer_snapshot; + stale_buffer.buffers[0].dwords[1] ^= 4u << 16u; + Check(!ValidateResourceSpecialization(buffer_program, stale_buffer, &error) && + error.find("buffer descriptor 0") != std::string::npos, + "stale buffer stride reused incompatible specialized SPIR-V"); + + ShaderComputeInputInfo compute; + compute.thread_ids_num = 1; + Check(CollectShaderInfo(&buffer_program, {.compute = &compute}, &error) && + AllocateBindings(&buffer_program, {}, &error), + error.c_str()); + Check(buffer_program.bindings.user_data_registers == + std::vector({0, 1, 2, 3}), + "buffer fixture did not allocate its descriptor user-data roots"); + buffer_snapshot.user_data.resize(4); + Check(ValidateResourceSpecialization(buffer_program, buffer_snapshot, &error), + error.c_str()); + auto truncated_user_data = buffer_snapshot; + truncated_user_data.user_data.resize(3); + Check(!ValidateResourceSnapshot(buffer_program, truncated_user_data, &error) && + error.find("user SGPR 3") != std::string::npos, + "truncated cache snapshot user-data window was accepted"); + + Program based_program; + based_program.stage = ShaderType::Compute; + based_program.blocks.resize(1); + auto raw = ScalarLoad(8, 0, 16, 0); + raw.memory.offset = static_cast(-4); + raw.src[0] = Sgpr(20); + raw.src_count = 1; + based_program.blocks[0].instructions = { + MoveImmediate(0, 16, 0x1000), MoveImmediate(4, 17, 0), raw}; + Prepare(&based_program); + ResourceSnapshot based_snapshot; + Check(MaterializeResources(based_program, {}, &based_snapshot, &error) && + SpecializeResources(&based_program, based_snapshot, &error), + error.c_str()); + auto relocated = based_snapshot; + relocated.addresses[0].guest_base += 0x1000; + relocated.addresses[0].binding_base += 0x1000; + Check(ValidateResourceSpecialization(based_program, relocated, &error), + "relocated based address with identical relative bias changed specialization"); + relocated.addresses[0].binding_base++; + Check(!ValidateResourceSpecialization(based_program, relocated, &error) && + error.find("address resource 0") != std::string::npos, + "stale based-address bias reused incompatible specialized SPIR-V"); + + Program flat_program; + flat_program.stage = ShaderType::Compute; + flat_program.blocks.resize(1); + Instruction flat; + flat.op = Opcode::FlatLoadDword; + flat.memory.kind = ResourceKind::Flat; + flat_program.blocks[0].instructions = {flat}; + Prepare(&flat_program); + ResourceSnapshot flat_snapshot; + flat_snapshot.addresses = {{0x11u, 0x10u}}; + flat_snapshot.user_data = {0xdeadbeefu}; + const auto prior_flat_snapshot = flat_snapshot; + Check(!MaterializeResources(flat_program, {}, &flat_snapshot, &error) && + error.find("requires runtime guest-address translation") != std::string::npos && + flat_snapshot.addresses == prior_flat_snapshot.addresses && + flat_snapshot.user_data == prior_flat_snapshot.user_data, + "unbased flat memory without a translator did not fail transactionally"); + SrtRuntime flat_runtime; + flat_runtime.flat_memory_base = 0x100000000ull; + Check(MaterializeResources(flat_program, flat_runtime, &flat_snapshot, &error) && + SpecializeResources(&flat_program, flat_snapshot, &error), + error.c_str()); + auto stale_flat = flat_snapshot; + stale_flat.addresses[0].guest_base += 0x1000; + stale_flat.addresses[0].binding_base += 0x1000; + Check(!ValidateResourceSpecialization(flat_program, stale_flat, &error), + "unbased flat memory reused an absolute base baked into SPIR-V"); +} + +void TestTrackedProgramIsImmutable() { + Program program; + program.blocks.resize(1); + auto &insts = program.blocks[0].instructions; + for (uint32_t i = 0; i < 8; i++) { + insts.push_back(ScalarLoad(i * 4, i, 16, i * 4)); + } + for (uint32_t i = 0; i < 4; i++) { + insts.push_back(MoveImmediate(0x20 + i * 4, 8 + i, 0xa0 + i)); + insts.push_back(MoveImmediate(0x30 + i * 4, 12 + i, 0xb0 + i)); + } + insts.push_back(ImageUse(0x50, Opcode::ImageSample, ResourceKind::Image, + Decoder::ImageDimension::Dim2D)); + insts.push_back(ImageUse(0x54, Opcode::ImageStore, + ResourceKind::StorageImage, + Decoder::ImageDimension::Dim2D)); + insts.push_back(BufferUse(0x58, 12)); + Prepare(&program); + Check(program.info.buffers.size() == 1 && program.info.images.size() == 2 && + program.info.samplers.size() == 1 && + program.info.sampled_pairs.size() == 1, + "post-track immutability fixture lacks complete resource topology"); + + const auto provenance = program.provenance; + const auto srt = program.srt; + const auto info = program.info; + const auto srt_complete = program.srt_plan_complete; + const auto tracking_complete = program.resource_tracking_complete; + const auto patching_complete = program.srt_patching_complete; + std::vector memory; + memory.reserve(insts.size()); + for (const auto &inst : insts) { + memory.push_back(inst.memory); + } + + const auto CheckUnchanged = [&]() { + Check(program.provenance == provenance && program.srt == srt && + program.info == info && + program.srt_plan_complete == srt_complete && + program.srt_patching_complete == patching_complete && + program.resource_tracking_complete == tracking_complete && + insts.size() == memory.size(), + "rejected post-track pass mutated immutable program state"); + for (uint32_t i = 0; i < memory.size(); i++) { + Check(insts[i].memory == memory[i], + "rejected post-track pass mutated a dense operand or source handle"); + } + }; + + std::string error; + Check(!BuildScalarProvenance(&program, &error) && + error.find("after resource tracking") != std::string::npos, + "post-track provenance rebuild was accepted"); + CheckUnchanged(); + Check(!BuildSrtPlan(&program, &error) && + error.find("after resource tracking") != std::string::npos, + "post-track SRT rebuild was accepted"); + CheckUnchanged(); +} + +void TestNativeBindingLayout() { + Program program; + program.stage = ShaderType::Compute; + program.blocks.resize(1); + auto &insts = program.blocks[0].instructions; + insts.push_back(BufferUse(4, 48)); + insts.push_back(ImageUse(8, Opcode::ImageSample, ResourceKind::Image, + Decoder::ImageDimension::Dim2D, 0, 2)); + insts.push_back(ImageUse(12, Opcode::ImageSample, ResourceKind::Image, + Decoder::ImageDimension::Dim2DArray, 0, 2)); + insts.push_back(ImageUse(16, Opcode::ImageStore, + ResourceKind::StorageImage, + Decoder::ImageDimension::Dim3D, 4)); + insts.push_back(ImageUse(20, Opcode::ImageStore, + ResourceKind::StorageImageUint, + Decoder::ImageDimension::Dim2DArray, 6)); + Prepare(&program); + + ShaderComputeInputInfo compute; + compute.thread_ids_num = 1; + std::string error; + Check(CollectShaderInfo(&program, {.compute = &compute}, &error) && + AllocateBindings(&program, {.descriptor_set = 3}, &error), + error.c_str()); + const auto *buffers = + FindBinding(program.bindings, DescriptorBindingKind::Buffers); + const auto *sampled2d = + FindBinding(program.bindings, DescriptorBindingKind::Sampled2D); + const auto *sampled_array = + FindBinding(program.bindings, DescriptorBindingKind::Sampled2DArray); + const auto *storage3d = + FindBinding(program.bindings, DescriptorBindingKind::Storage3D); + const auto *storage_uint_array = + FindBinding(program.bindings, DescriptorBindingKind::StorageUint2DArray); + const auto *samplers = + FindBinding(program.bindings, DescriptorBindingKind::Samplers); + Check(program.bindings.descriptor_set == 3 && buffers != nullptr && + sampled2d != nullptr && sampled_array != nullptr && + storage3d != nullptr && storage_uint_array != nullptr && + samplers != nullptr && buffers->binding == 0 && + sampled2d->binding == 1 && sampled_array->binding == 2 && + storage3d->binding == 3 && storage_uint_array->binding == 4 && + samplers->binding == 5 && buffers->resources == + std::vector{0} && + sampled2d->resources == std::vector{0} && + sampled_array->resources == std::vector{1} && + storage3d->resources == std::vector{2} && + storage_uint_array->resources == std::vector{3} && + samplers->resources == std::vector{0} && + !program.bindings.user_data_registers.empty() && + program.bindings.push_constant_size != 0 && + FindBinding(program.bindings, + DescriptorBindingKind::FlattenedSrt) == nullptr && + program.binding_layout_complete, + "native binding allocator did not preserve dense typed resource groups"); +} + +void TestNativeBindingLayoutSrtAndUserDataOverflow() { + Program srt; + srt.stage = ShaderType::Compute; + srt.blocks.resize(1); + for (uint32_t i = 0; i < 4; i++) { + srt.blocks[0].instructions.push_back( + ScalarLoad(i * 4, i, 16, i * 4)); + } + srt.blocks[0].instructions.push_back(BufferUse(0x20, 0)); + Prepare(&srt); + ShaderComputeInputInfo compute; + compute.thread_ids_num = 1; + std::string error; + Check(CollectShaderInfo(&srt, {.compute = &compute}, &error) && + AllocateBindings(&srt, {}, &error), + error.c_str()); + const auto *flat = + FindBinding(srt.bindings, DescriptorBindingKind::FlattenedSrt); + Check(flat != nullptr && flat->binding == 1 && flat->resources.empty(), + "flattened SRT did not receive one native backend binding"); + + Program overflow; + overflow.stage = ShaderType::Compute; + overflow.blocks.resize(1); + for (uint32_t i = 0; i < 33; i++) { + Instruction direct; + direct.pc = i * 4; + direct.op = Opcode::MoveU32; + direct.dst.kind = OperandKind::Register; + direct.dst.reg = {RegisterFile::Vector, i}; + direct.src[0] = Sgpr(i); + direct.src_count = 1; + overflow.blocks[0].instructions.push_back(direct); + } + Check(BuildScalarProvenance(&overflow, &error) && + BuildSrtPlan(&overflow, &error) && PatchSrtReads(&overflow, &error) && + TrackResources(&overflow, &error) && + CollectShaderInfo(&overflow, {.compute = &compute}, &error) && + AllocateBindings(&overflow, {}, &error), + error.c_str()); + const auto *user_data = + FindBinding(overflow.bindings, DescriptorBindingKind::UserData); + Check(overflow.bindings.user_data_registers.size() == 33 && + overflow.bindings.user_data_registers.front() == 0 && + overflow.bindings.user_data_registers.back() == 32 && + overflow.bindings.push_constant_size == 0 && user_data != nullptr && + user_data->binding == 0, + "oversized sparse user data was not moved to a descriptor binding"); +} + +void TestNativeBindingLayoutGds() { + Program program; + program.stage = ShaderType::Compute; + program.blocks.resize(1); + Instruction append; + append.op = Opcode::DsAppend; + append.memory.kind = ResourceKind::Gds; + program.blocks[0].instructions.push_back(append); + + ShaderComputeInputInfo compute; + std::string error; + Check(BuildScalarProvenance(&program, &error) && + BuildSrtPlan(&program, &error) && PatchSrtReads(&program, &error) && + TrackResources(&program, &error) && + CollectShaderInfo(&program, {.compute = &compute}, &error) && + AllocateBindings(&program, {}, &error), + error.c_str()); + const auto *gds = FindBinding(program.bindings, DescriptorBindingKind::Gds); + Check(gds != nullptr && gds->binding == 0 && gds->resources.empty(), + "GDS append/consume did not allocate one native storage binding"); + + Program lds; + lds.stage = ShaderType::Compute; + lds.blocks.resize(1); + append.memory.kind = ResourceKind::Lds; + lds.blocks[0].instructions.push_back(append); + Check(BuildScalarProvenance(&lds, &error) && BuildSrtPlan(&lds, &error) && + PatchSrtReads(&lds, &error) && TrackResources(&lds, &error) && + CollectShaderInfo(&lds, {.compute = &compute}, &error) && + AllocateBindings(&lds, {}, &error), + error.c_str()); + Check(FindBinding(lds.bindings, DescriptorBindingKind::Gds) == nullptr, + "LDS append/consume incorrectly allocated a GDS binding"); +} + +void TestNativeBindingLayoutIsTransactional() { + Program program; + program.stage = ShaderType::Compute; + program.blocks.resize(1); + std::string error; + ShaderComputeInputInfo compute; + compute.thread_ids_num = 1; + Check(BuildScalarProvenance(&program, &error) && + BuildSrtPlan(&program, &error) && PatchSrtReads(&program, &error) && + TrackResources(&program, &error) && + CollectShaderInfo(&program, {.compute = &compute}, &error), + error.c_str()); + BindingLayout sentinel; + sentinel.descriptor_set = 99; + program.bindings = sentinel; + Check(!AllocateBindings(&program, {.push_constant_offset = 129}, &error) && + error.find("Vulkan minimum") != std::string::npos && + program.bindings == sentinel && !program.binding_layout_complete, + "failed native binding allocation partially changed the program"); + Check(!AllocateBindings(&program, {.push_constant_offset = 2}, &error) && + error.find("dword aligned") != std::string::npos && + program.bindings == sentinel && !program.binding_layout_complete, + "misaligned push-constant allocation partially changed the program"); + Check(AllocateBindings(&program, {}, &error), error.c_str()); + const auto layout = program.bindings; + Check(!AllocateBindings(&program, {}, &error) && + error.find("already allocated") != std::string::npos && + program.bindings == layout, + "repeated native binding allocation was accepted or mutated layout"); + + Program tail; + tail.stage = ShaderType::Compute; + tail.blocks.resize(1); + Instruction direct; + direct.op = Opcode::MoveU32; + direct.dst.kind = OperandKind::Register; + direct.dst.reg = {RegisterFile::Vector, 0}; + direct.src[0] = Sgpr(0); + direct.src_count = 1; + tail.blocks[0].instructions = {direct}; + Check(BuildScalarProvenance(&tail, &error) && BuildSrtPlan(&tail, &error) && + PatchSrtReads(&tail, &error) && TrackResources(&tail, &error) && + CollectShaderInfo(&tail, {.compute = &compute}, &error) && + AllocateBindings(&tail, {.push_constant_offset = 124}, &error) && + tail.bindings.push_constant_size == 4 && + FindBinding(tail.bindings, DescriptorBindingKind::UserData) == nullptr, + "valid final dword of the guaranteed push-constant range was rejected"); +} + +void TestNativeBindingLayoutTracksReachingUserData() { + Program program; + program.stage = ShaderType::Compute; + program.blocks.resize(1); + auto &insts = program.blocks[0].instructions; + insts.push_back(Move(0, 100, 4)); + Instruction forwarded; + forwarded.pc = 4; + forwarded.op = Opcode::MoveU32; + forwarded.dst.kind = OperandKind::Register; + forwarded.dst.reg = {RegisterFile::Vector, 0}; + forwarded.src[0] = Sgpr(100); + forwarded.src_count = 1; + insts.push_back(forwarded); + auto sparse = forwarded; + sparse.pc = 8; + sparse.dst.reg.index = 1; + sparse.src[0] = Sgpr(20); + insts.push_back(sparse); + insts.push_back(MoveImmediate(12, 101, 7)); + auto constant = forwarded; + constant.pc = 16; + constant.dst.reg.index = 2; + constant.src[0] = Sgpr(101); + insts.push_back(constant); + + std::string error; + ShaderComputeInputInfo compute; + compute.thread_ids_num = 1; + Check(BuildScalarProvenance(&program, &error) && + BuildSrtPlan(&program, &error) && PatchSrtReads(&program, &error) && + TrackResources(&program, &error) && + CollectShaderInfo(&program, {.compute = &compute}, &error), + error.c_str()); + auto fallback = program; + Check(AllocateBindings(&program, {}, &error) && + program.bindings.user_data_registers == + std::vector({4, 20}) && + program.bindings.push_constant_size == 8 && + FindBinding(program.bindings, DescriptorBindingKind::UserData) == + nullptr, + "native user-data map included a temporary or missed sparse roots"); + Check(AllocateBindings(&fallback, {.max_push_dwords = 1}, &error) && + fallback.bindings.user_data_registers == + std::vector({4, 20}) && + fallback.bindings.push_constant_size == 0 && + FindBinding(fallback.bindings, DescriptorBindingKind::UserData) != + nullptr, + "max push-dword limit did not move exact sparse user data to storage"); +} + +void TestNativeBindingLayoutRejectsUnknownShapeAndBadProvenance() { + ShaderComputeInputInfo compute; + compute.thread_ids_num = 1; + std::string error; + + Program image; + image.stage = ShaderType::Compute; + image.blocks.resize(1); + image.blocks[0].instructions = { + ImageUse(4, Opcode::ImageStore, ResourceKind::StorageImage, + Decoder::ImageDimension::Dim2D)}; + Prepare(&image); + Check(CollectShaderInfo(&image, {.compute = &compute}, &error), + error.c_str()); + image.info.images[0].dimension = Decoder::ImageDimension::Unknown; + image.bindings.descriptor_set = 77; + const auto image_layout = image.bindings; + Check(!AllocateBindings(&image, {}, &error) && + error.find("invalid image binding class") != std::string::npos && + image.bindings == image_layout && !image.binding_layout_complete, + "unknown image shape was defaulted or partially allocated"); + + Program provenance; + provenance.stage = ShaderType::Compute; + provenance.blocks.resize(1); + Instruction direct; + direct.op = Opcode::MoveU32; + direct.dst.kind = OperandKind::Register; + direct.dst.reg = {RegisterFile::Vector, 0}; + direct.src[0] = Sgpr(4); + direct.src_count = 1; + provenance.blocks[0].instructions = {direct}; + Check(BuildScalarProvenance(&provenance, &error) && + BuildSrtPlan(&provenance, &error) && + PatchSrtReads(&provenance, &error) && + TrackResources(&provenance, &error) && + CollectShaderInfo(&provenance, {.compute = &compute}, &error), + error.c_str()); + provenance.blocks[0].instructions[0].scalar_sources[0] = + static_cast(provenance.provenance.values.size() + 1); + provenance.bindings.descriptor_set = 88; + const auto provenance_layout = provenance.bindings; + Check(!AllocateBindings(&provenance, {}, &error) && + error.find("invalid scalar provenance reference") != + std::string::npos && + provenance.bindings == provenance_layout && + !provenance.binding_layout_complete, + "malformed per-use provenance was ignored or partially allocated"); +} + +void TestNativeBindingLayoutDynamicSrtDoesNotUseFlatBinding() { + Program program; + program.stage = ShaderType::Compute; + program.blocks.resize(1); + for (uint32_t i = 0; i < 4; i++) { + auto load = ScalarLoad(i * 4, i, 16, i * 4); + load.src[0] = Sgpr(20); + load.src_count = 1; + program.blocks[0].instructions.push_back(load); + } + program.blocks[0].instructions.push_back(BufferUse(0x20, 0)); + Prepare(&program); + ShaderComputeInputInfo compute; + compute.thread_ids_num = 1; + std::string error; + Check(program.srt.reads.empty() && program.srt.dynamic_reads.size() == 4 && + CollectShaderInfo(&program, {.compute = &compute}, &error) && + AllocateBindings(&program, {}, &error), + error.c_str()); + Check(program.bindings.user_data_registers == + std::vector({16, 17, 20}) && + FindBinding(program.bindings, + DescriptorBindingKind::FlattenedSrt) == nullptr, + "dynamic-only SRT reads were flattened or lost reaching user data"); +} + +void TestNativeBindingLayoutTracksRawScalarMemoryBase() { + Program program; + program.stage = ShaderType::Compute; + program.blocks.resize(1); + auto load = ScalarLoad(0, 0, 16, 0); + load.src[0] = Sgpr(20); + load.src_count = 1; + Instruction consume; + consume.pc = 4; + consume.op = Opcode::MoveU32; + consume.dst.kind = OperandKind::Register; + consume.dst.reg = {RegisterFile::Vector, 0}; + consume.src[0] = Sgpr(0); + consume.src_count = 1; + program.blocks[0].instructions = {load, consume}; + + std::string error; + ShaderComputeInputInfo compute; + compute.thread_ids_num = 1; + Check(BuildScalarProvenance(&program, &error) && + BuildSrtPlan(&program, &error) && program.srt.reads.empty() && + program.srt.dynamic_reads.empty() && PatchSrtReads(&program, &error) && + TrackResources(&program, &error) && + CollectShaderInfo(&program, {.compute = &compute}, &error) && + AllocateBindings(&program, {}, &error), + error.c_str()); + const auto *address_memory = + FindBinding(program.bindings, DescriptorBindingKind::AddressMemory); + Check(program.blocks[0].instructions[0].op == Opcode::SLoadDword && + program.bindings.user_data_registers == + std::vector({16, 17, 20}) && + FindBinding(program.bindings, + DescriptorBindingKind::FlattenedSrt) == nullptr && + address_memory != nullptr && + address_memory->resources == std::vector({0}) && + program.info.addresses.size() == 1 && + program.info.addresses[0].kind == ResourceKind::ScalarBuffer, + "ordinary raw scalar memory omitted its implicit base or offset roots"); +} + +void TestRawScalarMemoryTracksReachingBaseIdentity() { + Program program; + program.stage = ShaderType::Compute; + program.blocks.resize(1); + auto first = ScalarLoad(8, 0, 16, 0); + first.memory.offset = static_cast(-4); + first.src[0] = Sgpr(20); + first.src_count = 1; + auto second = ScalarLoad(20, 1, 16, 0); + second.src[0] = Sgpr(20); + second.src_count = 1; + program.blocks[0].instructions = { + MoveImmediate(0, 16, 0x1000), MoveImmediate(4, 17, 0), first, + MoveImmediate(12, 16, 0x2000), MoveImmediate(16, 17, 0), second}; + + std::string error; + Check(BuildScalarProvenance(&program, &error) && BuildSrtPlan(&program, &error) && + PatchSrtReads(&program, &error) && TrackResources(&program, &error), + error.c_str()); + Check(program.info.addresses.size() == 2 && + program.info.addresses[0].source != program.info.addresses[1].source && + program.blocks[0].instructions[2].memory.resource == 0 && + program.blocks[0].instructions[5].memory.resource == 1, + "raw scalar loads with redefined SBASE collapsed to one address resource"); + + std::array user_data{}; + user_data[20] = 4; + ResourceSnapshot snapshot; + Check(MaterializeResources(program, {user_data}, &snapshot, &error), error.c_str()); + Check(snapshot.addresses.size() == 2 && + snapshot.addresses[0].guest_base == 0x1000 && + snapshot.addresses[0].binding_base == 0x0ffc && + snapshot.addresses[1].guest_base == 0x2000 && + snapshot.addresses[1].binding_base == 0x2000, + "runtime snapshot lost per-use raw scalar base values"); +} + +void TestNativeBindingLayoutUsesExplicitFlatMemory() { + Program program; + program.stage = ShaderType::Compute; + program.blocks.resize(1); + Instruction load; + load.op = Opcode::FlatLoadDword; + load.memory.kind = ResourceKind::Flat; + program.blocks[0].instructions = {load}; + Prepare(&program); + + ShaderComputeInputInfo compute; + compute.thread_ids_num = 1; + std::string error; + Check(CollectShaderInfo(&program, {.compute = &compute}, &error) && + AllocateBindings(&program, {}, &error), + error.c_str()); + const auto *flat = + FindBinding(program.bindings, DescriptorBindingKind::AddressMemory); + Check(flat != nullptr && flat->resources == std::vector({0}) && + program.info.addresses.size() == 1 && + program.info.addresses[0].kind == ResourceKind::Flat && + FindBinding(program.bindings, DescriptorBindingKind::Buffers) == nullptr, + "flat address space was aliased to an ordinary buffer descriptor group"); + + ResourceSnapshot snapshot; + SrtRuntime runtime; + runtime.flat_memory_base = 0x1234567887654000ull; + Check(MaterializeResources(program, runtime, &snapshot, &error) && + snapshot.addresses.size() == 1 && + snapshot.addresses[0].guest_base == 0x1234567887654000ull && + snapshot.addresses[0].binding_base == 0x1234567887654000ull, + "unbased flat virtual address space lost its runtime binding base"); +} + +void TestNativeBindingLayoutHonorsUserDataCount() { + Program program; + program.stage = ShaderType::Compute; + program.user_data_count = 8; + program.blocks.resize(1); + for (uint32_t i = 0; i < 2; i++) { + Instruction direct; + direct.pc = i * 4; + direct.op = Opcode::MoveU32; + direct.dst.kind = OperandKind::Register; + direct.dst.reg = {RegisterFile::Vector, i}; + direct.src[0] = Sgpr(7 + i); + direct.src_count = 1; + program.blocks[0].instructions.push_back(direct); + } + std::string error; + ShaderComputeInputInfo compute; + compute.thread_ids_num = 1; + Check(BuildScalarProvenance(&program, &error) && + program.provenance.values[program.blocks[0].instructions[0] + .scalar_sources[0]] + .op == ScalarValueOp::UserData && + program.blocks[0].instructions[1].scalar_sources[0] == + ScalarProvenance::Unknown && + BuildSrtPlan(&program, &error) && PatchSrtReads(&program, &error) && + TrackResources(&program, &error) && + CollectShaderInfo(&program, {.compute = &compute}, &error) && + AllocateBindings(&program, {}, &error) && + program.bindings.user_data_registers == std::vector({7}), + "stage user-data count did not bound provenance roots"); + + Program invalid; + invalid.user_data_count = 65; + invalid.blocks.resize(1); + Check(!BuildScalarProvenance(&invalid, &error) && + error.find("exceeds 64") != std::string::npos, + "out-of-range stage user-data count was accepted"); +} + +void TestNativeBindingLayoutHonorsUserDataBase() { + Program program; + program.stage = ShaderType::Vertex; + program.user_data_base = 8; + program.user_data_count = 8; + program.blocks.resize(1); + program.blocks[0].instructions = { + Move(0, 20, 7), Move(4, 21, 8), Move(8, 22, 15), Move(12, 23, 16), + BufferUse(16, 8)}; + + std::string error; + Check(BuildScalarProvenance(&program, &error) && + program.blocks[0].instructions[0].scalar_sources[0] == + ScalarProvenance::Unknown && + program.provenance.values[ + program.blocks[0].instructions[1].scalar_sources[0]].op == + ScalarValueOp::UserData && + program.blocks[0].instructions[3].scalar_sources[0] == + ScalarProvenance::Unknown && + BuildSrtPlan(&program, &error), + error.c_str()); + + const std::array user_data = { + 0x11111111, 0x22222222, 0x33333333, 0x44444444, + 0, 0, 0, 0xaaaaaaaa}; + DescriptorValue descriptor; + const auto source = program.blocks[0].instructions.back().memory.resource_source; + Check(EvaluateDescriptorSource(program, source, 16, {user_data}, &descriptor, &error) && + descriptor.dwords[0] == user_data[0] && + descriptor.dwords[3] == user_data[3], + "vertex user-data base was not translated to runtime-local indices"); + + ShaderVertexInputInfo vertex; + Check(PatchSrtReads(&program, &error) && TrackResources(&program, &error) && + CollectShaderInfo(&program, {.vertex = &vertex}, &error) && + AllocateBindings(&program, {}, &error), + error.c_str()); + Check(program.bindings.user_data_registers == + std::vector({8, 9, 10, 11, 15}), + "native binding plan did not retain physical shifted user-SGPR indices"); +} + +void TestTextureNullDescriptorUsesAddressBits() { + ShaderTextureResource texture; + const uint32_t captured[8] = {0x00000000, 0xc3800000, 0x0059c09f, 0x91b00fac, + 0x00000000, 0x00700000, 0x00000000, 0x00000000}; + std::copy(std::begin(captured), std::end(captured), std::begin(texture.fields)); + Check(texture.IsNull(), + "zero-address image with populated metadata was not classified as null"); + texture.fields[0] = 1; + Check(!texture.IsNull(), "nonzero image base address was classified as null"); +} + +} // namespace + +int main() { + const char* current = "startup"; +#define RUN(test) current = #test; test() + try { + RUN(TestDenseBufferPatching); + RUN(TestScalarAndVectorBufferAlias); + RUN(TestImagesAndSamplers); + RUN(TestDynamicPhiResource); + RUN(TestTrackingRequiresCompletedSrtPlan); + RUN(TestCyclicResourceIsRejected); + RUN(TestUnknownSourceFailsWithoutPatching); + RUN(TestResourceLimitFailsTransactionally); + RUN(TestComputeShaderInfoCollection); + RUN(TestVertexShaderInfoCollection); + RUN(TestPixelShaderInfoCollection); + RUN(TestShaderInfoCollectionIsTransactional); + RUN(TestShaderInfoMetadataValidation); + RUN(TestTrackingRequiresSrtPatching); + RUN(TestDynamicSrtReadRemainsExplicit); + RUN(TestSrtPatchingFailureIsTransactional); + RUN(TestScalarMemoryGroupsSnapshotOperands); + RUN(TestSrtPatchingHandlesGvnAndMoveForwarding); + RUN(TestSrtPatchingHandlesCfgProducers); + RUN(TestSrtPatchPlanValidation); + RUN(TestMaterializationSharesReadConstEvaluation); + RUN(TestMaterializationFailureIsTransactional); + RUN(TestResourceSpecializationIsTypedAndTransactional); + RUN(TestRuntimeSpecializationCoversBakedBufferAndAddressFields); + RUN(TestTrackedProgramIsImmutable); + RUN(TestNativeBindingLayout); + RUN(TestNativeBindingLayoutSrtAndUserDataOverflow); + RUN(TestNativeBindingLayoutGds); + RUN(TestNativeBindingLayoutIsTransactional); + RUN(TestNativeBindingLayoutTracksReachingUserData); + RUN(TestNativeBindingLayoutRejectsUnknownShapeAndBadProvenance); + RUN(TestNativeBindingLayoutDynamicSrtDoesNotUseFlatBinding); + RUN(TestNativeBindingLayoutTracksRawScalarMemoryBase); + RUN(TestRawScalarMemoryTracksReachingBaseIdentity); + RUN(TestNativeBindingLayoutUsesExplicitFlatMemory); + RUN(TestNativeBindingLayoutHonorsUserDataCount); + RUN(TestNativeBindingLayoutHonorsUserDataBase); + RUN(TestTextureNullDescriptorUsesAddressBits); + std::cout << "ResourceTrackingTests: all cases passed\n"; + return 0; + } catch (const std::exception &e) { + std::cerr << "ResourceTrackingTests: " << current << " failed: " << e.what() << '\n'; + return 1; + } +#undef RUN +} diff --git a/tests/ScalarProvenanceTests.cpp b/tests/ScalarProvenanceTests.cpp new file mode 100644 index 000000000..944646e6b --- /dev/null +++ b/tests/ScalarProvenanceTests.cpp @@ -0,0 +1,1081 @@ +#include "graphics/shader/recompiler/ScalarProvenance.h" +#include "graphics/shader/recompiler/SrtWalker.h" + +#include +#include +#include + +namespace { + +using namespace Libs::Graphics::ShaderRecompiler::IR; +using Libs::Graphics::ShaderType; + +void Check(bool condition, const char* message) { + if (!condition) { + throw std::runtime_error(message); + } +} + +Operand Sgpr(uint32_t reg) { + Operand operand; + operand.kind = OperandKind::Register; + operand.reg.file = RegisterFile::Scalar; + operand.reg.index = reg; + return operand; +} + +Operand Vgpr(uint32_t reg) { + Operand operand; + operand.kind = OperandKind::Register; + operand.reg.file = RegisterFile::Vector; + operand.reg.index = reg; + return operand; +} + +Operand Imm(uint32_t value) { + Operand operand; + operand.kind = OperandKind::ImmediateU32; + operand.imm = value; + return operand; +} + +Operand M0() { + Operand operand; + operand.kind = OperandKind::Register; + operand.reg.file = RegisterFile::M0; + return operand; +} + +Instruction Move(uint32_t pc, uint32_t dst, uint32_t src) { + Instruction inst; + inst.pc = pc; + inst.op = Opcode::MoveU32; + inst.dst = Sgpr(dst); + inst.src[0] = Sgpr(src); + inst.src_count = 1; + return inst; +} + +Instruction MoveImmediate(uint32_t pc, uint32_t dst, uint32_t value) { + auto inst = Move(pc, dst, 0); + inst.src[0] = Imm(value); + return inst; +} + +Instruction Move64(uint32_t pc, uint32_t dst, uint32_t src) { + auto inst = Move(pc, dst, src); + inst.op = Opcode::MoveU64; + return inst; +} + +Instruction WriteLane(uint32_t pc, uint32_t dst, uint32_t src, uint32_t lane) { + Instruction inst; + inst.pc = pc; + inst.op = Opcode::WriteLaneU32; + inst.dst = Vgpr(dst); + inst.src[0] = Sgpr(src); + inst.src[1] = Imm(lane); + inst.src_count = 2; + return inst; +} + +Instruction ReadLane(uint32_t pc, uint32_t dst, uint32_t src, uint32_t lane) { + Instruction inst; + inst.pc = pc; + inst.op = Opcode::ReadLaneU32; + inst.dst = Sgpr(dst); + inst.src[0] = Vgpr(src); + inst.src[1] = Imm(lane); + inst.src_count = 2; + return inst; +} + +Instruction BufferUse(uint32_t pc, uint32_t base_sgpr) { + Instruction inst; + inst.pc = pc; + inst.op = Opcode::BufferLoadDword; + inst.memory.kind = ResourceKind::Buffer; + inst.memory.resource = base_sgpr / 4; + return inst; +} + +const ScalarValue& Value(const Program& program, uint32_t id) { + Check(id < program.provenance.values.size(), "invalid scalar value ID"); + return program.provenance.values[id]; +} + +bool ReadHostMemory(void*, uint64_t address, uint32_t* value) { + if (address == 0 || value == nullptr) { + return false; + } + std::memcpy(value, reinterpret_cast(address), sizeof(*value)); + return true; +} + +struct ReadCounter { + uint32_t reads = 0; +}; + +bool CountedHostMemory(void* userdata, uint64_t address, uint32_t* value) { + auto* counter = static_cast(userdata); + if (counter != nullptr) { + counter->reads++; + } + return ReadHostMemory(nullptr, address, value); +} + +void SetPointer(std::vector* user_data, uint32_t reg, const void* pointer) { + const auto address = reinterpret_cast(pointer); + (*user_data)[reg] = static_cast(address); + (*user_data)[reg + 1u] = static_cast(address >> 32u); +} + +void TestPerUseDescriptorDefinitions() { + Program program; + program.blocks.resize(1); + auto& instructions = program.blocks[0].instructions; + instructions.push_back(BufferUse(0, 0)); + for (uint32_t i = 0; i < 4; i++) { + instructions.push_back(Move(4 + i * 4, i, 20 + i)); + } + instructions.push_back(BufferUse(20, 0)); + + std::string error; + Check(BuildScalarProvenance(&program, &error), error.c_str()); + const auto* before = GetDescriptorSource(program, instructions[0].memory.resource_source); + const auto* after = GetDescriptorSource(program, instructions[5].memory.resource_source); + Check(before != nullptr && after != nullptr, "descriptor sources were not attached"); + Check(*before != *after, "descriptor source was incorrectly shared across SGPR redefinition"); + for (uint32_t i = 0; i < 4; i++) { + Check(Value(program, before->dwords[i]).op == ScalarValueOp::UserData && + Value(program, before->dwords[i]).imm == i, + "pre-copy descriptor does not use the reaching user-data definition"); + Check(Value(program, after->dwords[i]).op == ScalarValueOp::UserData && + Value(program, after->dwords[i]).imm == 20 + i, + "post-copy descriptor does not use the copied user-data definition"); + } +} + +void TestCfgPhi() { + Program program; + program.blocks.resize(4); + program.blocks[0].successors = {1, 2}; + program.blocks[1].predecessors = {0}; + program.blocks[1].successors = {3}; + program.blocks[2].predecessors = {0}; + program.blocks[2].successors = {3}; + program.blocks[3].predecessors = {1, 2}; + program.blocks[1].instructions.push_back(MoveImmediate(4, 0, 0x11111111)); + program.blocks[2].instructions.push_back(MoveImmediate(8, 0, 0x22222222)); + program.blocks[3].instructions.push_back(BufferUse(12, 0)); + + std::string error; + Check(BuildScalarProvenance(&program, &error), error.c_str()); + const auto* source = + GetDescriptorSource(program, program.blocks[3].instructions[0].memory.resource_source); + Check(source != nullptr, "merged descriptor source was not attached"); + const auto& phi = Value(program, source->dwords[0]); + Check(phi.op == ScalarValueOp::Phi && phi.phi_args.size() == 2, + "differing CFG definitions did not create a provenance phi"); + Check(Value(program, phi.phi_args[0]).op == ScalarValueOp::Constant && + Value(program, phi.phi_args[1]).op == ScalarValueOp::Constant, + "provenance phi does not reference both constant definitions"); + Check(BuildSrtPlan(&program, &error) && program.srt.dynamic_sources.size() == 1 && + program.srt.dynamic_sources[0] == + program.blocks[3].instructions[0].memory.resource_source, + "acyclic control-flow descriptor phi was not classified dynamic"); +} + +void TestDiamondReadPathsAreDynamic() { + std::array left = {0x11111111u}; + std::array right = {0x22222222u}; + Program program; + program.blocks.resize(4); + program.blocks[0].successors = {1, 2}; + program.blocks[1].predecessors = {0}; + program.blocks[1].successors = {3}; + program.blocks[2].predecessors = {0}; + program.blocks[2].successors = {3}; + program.blocks[3].predecessors = {1, 2}; + const auto load = [](uint32_t pc, uint32_t base) { + Instruction inst; + inst.pc = pc; + inst.op = Opcode::SLoadDword; + inst.dst = Sgpr(0); + inst.src[0] = Imm(0); + inst.src_count = 1; + inst.memory.kind = ResourceKind::ScalarBuffer; + inst.memory.resource = base; + return inst; + }; + program.blocks[1].instructions = {load(4, 4)}; + program.blocks[2].instructions = {load(8, 6)}; + program.blocks[3].instructions = {BufferUse(12, 0)}; + + std::string error; + Check(BuildScalarProvenance(&program, &error) && BuildSrtPlan(&program, &error), error.c_str()); + const auto source = program.blocks[3].instructions[0].memory.resource_source; + Check(program.srt.reads.empty() && program.srt.dynamic_reads.empty() && + program.srt.dynamic_sources.size() == 1 && program.srt.dynamic_sources[0] == source, + "inactive diamond read paths were incorrectly flattened"); + std::array user_data = {}; + const auto left_address = reinterpret_cast(left.data()); + const auto right_address = reinterpret_cast(right.data()); + user_data[4] = static_cast(left_address); + user_data[5] = static_cast(left_address >> 32u); + user_data[6] = static_cast(right_address); + user_data[7] = static_cast(right_address >> 32u); + ReadCounter counter; + DescriptorValue descriptor; + const SrtRuntime runtime {user_data, 0, CountedHostMemory, &counter}; + Check(!EvaluateDescriptorSource(program, source, 12, runtime, &descriptor, &error) && + counter.reads == 0 && error.find("GPU-dynamic") != std::string::npos, + "dynamic descriptor evaluation touched an inactive read path"); +} + +void TestEquivalentConstantPhiIsStatic() { + Program program; + program.blocks.resize(4); + program.blocks[0].successors = {1, 2}; + program.blocks[1].predecessors = {0}; + program.blocks[1].successors = {3}; + program.blocks[2].predecessors = {0}; + program.blocks[2].successors = {3}; + program.blocks[3].predecessors = {1, 2}; + Instruction add; + add.op = Opcode::IAddU32; + add.dst = Sgpr(0); + add.src[0] = Imm(1); + add.src[1] = Imm(1); + add.src_count = 2; + program.blocks[1].instructions = {add}; + program.blocks[2].instructions = {MoveImmediate(8, 0, 2)}; + program.blocks[3].instructions = {BufferUse(12, 0)}; + + std::string error; + Check(BuildScalarProvenance(&program, &error) && BuildSrtPlan(&program, &error), error.c_str()); + const auto source = program.blocks[3].instructions[0].memory.resource_source; + Check(program.srt.dynamic_sources.empty(), + "equivalent constant phi was unnecessarily classified dynamic"); + const std::array user_data = {}; + DescriptorValue descriptor; + const SrtRuntime runtime {user_data, 0, nullptr, nullptr}; + Check(EvaluateDescriptorSource(program, source, 12, runtime, &descriptor, &error) && + descriptor.dwords[0] == 2, + "equivalent constant phi did not evaluate as one static value"); +} + +void TestWideMoveInvalidatesAndCopiesBothDwords() { + Program program; + program.blocks.resize(1); + auto& instructions = program.blocks[0].instructions; + instructions.push_back(Move64(0, 0, 20)); + instructions.push_back(Move64(4, 2, 22)); + instructions.push_back(BufferUse(8, 0)); + + std::string error; + Check(BuildScalarProvenance(&program, &error), error.c_str()); + const auto* source = GetDescriptorSource(program, instructions.back().memory.resource_source); + Check(source != nullptr && + DescriptorSourceResolved(program, instructions.back().memory.resource_source), + "wide descriptor copy did not produce a resolved source"); + for (uint32_t i = 0; i < 4; i++) { + Check(Value(program, source->dwords[i]).op == ScalarValueOp::UserData && + Value(program, source->dwords[i]).imm == 20 + i, + "wide move did not copy both scalar dwords"); + } +} + +void TestScalarCarryChain() { + Program program; + program.blocks.resize(1); + Instruction low; + low.pc = 0; + low.op = Opcode::ScalarAddCarryU32; + low.dst = Sgpr(4); + low.src[0] = Sgpr(0); + low.src[1] = Imm(16); + low.src[2] = Imm(0); + low.src_count = 3; + Instruction high = low; + high.pc = 4; + high.dst = Sgpr(5); + high.src[0] = Sgpr(1); + high.src[1] = Imm(0); + high.src[2].kind = OperandKind::Register; + high.src[2].reg.file = RegisterFile::Scc; + high.src[2].reg.index = 0; + program.blocks[0].instructions = {low, high, BufferUse(8, 4)}; + + std::string error; + Check(BuildScalarProvenance(&program, &error), error.c_str()); + const auto* source = + GetDescriptorSource(program, program.blocks[0].instructions.back().memory.resource_source); + Check(source != nullptr, "carry-chain descriptor source was not attached"); + const auto& low_value = Value(program, source->dwords[0]); + const auto& high_value = Value(program, source->dwords[1]); + Check(low_value.op == ScalarValueOp::AddCarry && high_value.op == ScalarValueOp::AddCarry, + "scalar add/addc pair was not retained"); + Check(Value(program, high_value.args[2]).op == ScalarValueOp::Carry, + "high pointer dword does not consume the low add carry"); +} + +void TestReadConstDefinition() { + Program program; + program.blocks.resize(1); + Instruction load; + load.pc = 4; + load.op = Opcode::SLoadDword; + load.dst = Sgpr(16); + load.src[0] = Imm(8); + load.src_count = 1; + load.memory.kind = ResourceKind::ScalarBuffer; + load.memory.resource = 4; + load.memory.offset = 16; + program.blocks[0].instructions.push_back(load); + program.blocks[0].instructions.push_back(BufferUse(8, 16)); + + std::string error; + Check(BuildScalarProvenance(&program, &error), error.c_str()); + const auto* source = + GetDescriptorSource(program, program.blocks[0].instructions[1].memory.resource_source); + Check(source != nullptr, "read-constant descriptor source was not attached"); + const auto& read = Value(program, source->dwords[0]); + Check(read.op == ScalarValueOp::ReadConst && read.imm == 16, + "raw scalar load was not represented as ReadConst"); + Check(Value(program, read.args[0]).op == ScalarValueOp::UserData && + Value(program, read.args[0]).imm == 4 && Value(program, read.args[1]).imm == 5, + "ReadConst pointer does not retain its user-data roots"); + Check(Value(program, read.args[2]).op == ScalarValueOp::Constant && + Value(program, read.args[2]).imm == 8, + "ReadConst dynamic/immediate offset operand was lost"); +} + +void TestReadConstBufferAndValueNumbering() { + Program program; + program.blocks.resize(1); + Instruction load; + load.op = Opcode::SBufferLoadDword; + load.dst = Sgpr(16); + load.src[0] = Imm(3); + load.src_count = 1; + load.memory.kind = ResourceKind::ScalarBuffer; + load.memory.resource = 1; + load.memory.offset = 12; + program.blocks[0].instructions.push_back(load); + load.pc = 4; + load.dst = Sgpr(17); + program.blocks[0].instructions.push_back(load); + program.blocks[0].instructions.push_back(BufferUse(8, 16)); + + std::string error; + Check(BuildScalarProvenance(&program, &error), error.c_str()); + const auto* source = + GetDescriptorSource(program, program.blocks[0].instructions.back().memory.resource_source); + Check(source != nullptr, "ReadConstBuffer source was not attached"); + Check(source->dwords[0] == source->dwords[1], + "identical read-constant values at different PCs were not " + "value-numbered"); + const auto& read = Value(program, source->dwords[0]); + Check(read.op == ScalarValueOp::ReadConstBuffer && read.args[3] != 0 && read.args[4] != 0, + "ReadConstBuffer lost descriptor dword 3 or its offset"); +} + +void TestReadLaneDescriptorSpill() { + std::array table {}; + table[72] = 0x89abcdefu; + + Program program; + program.wave_size = 64; + program.user_data_count = 28; + program.blocks.resize(1); + for (uint32_t component = 0; component < 4; component++) { + Instruction load; + load.pc = 0x14; + load.op = Opcode::SBufferLoadDword; + load.dst = Sgpr(84 + component); + load.src[0] = Imm(0); + load.src_count = 1; + load.memory.kind = ResourceKind::ScalarBuffer; + load.memory.resource = 6; + load.memory.offset = 288 + component * 4; + load.memory.component_index = component; + load.memory.component_count = 4; + program.blocks[0].instructions.push_back(load); + } + program.blocks[0].instructions.push_back(WriteLane(0x20, 11, 86, 2)); + program.blocks[0].instructions.push_back(WriteLane(0x44, 11, 85, 1)); + program.blocks[0].instructions.push_back(WriteLane(0x60, 11, 84, 0)); + program.blocks[0].instructions.push_back(MoveImmediate(0xb0, 84, 0)); + program.blocks[0].instructions.push_back(ReadLane(0x510, 0, 11, 0)); + Instruction sample; + sample.pc = 0x868; + sample.op = Opcode::ImageSample; + sample.memory.kind = ResourceKind::Image; + sample.memory.resource = 0; + sample.memory.sampler = 4; + sample.memory.image_dimension = + Libs::Graphics::ShaderRecompiler::Decoder::ImageDimension::Dim2D; + program.blocks[0].instructions.push_back(sample); + + std::string error; + Check(BuildScalarProvenance(&program, &error) && BuildSrtPlan(&program, &error), error.c_str()); + const auto source_id = program.blocks[0].instructions.back().memory.resource_source; + const auto* source = GetDescriptorSource(program, source_id); + Check(source != nullptr && DescriptorSourceResolved(program, source_id), + "readlane descriptor spill was not resolved"); + Check(Value(program, source->dwords[0]).op == ScalarValueOp::ReadConstBuffer, + "readlane did not recover the pre-overwrite SRT value"); + std::vector user_data(28); + SetPointer(&user_data, 24, table.data()); + user_data[26] = sizeof(table); + DescriptorValue descriptor; + const SrtRuntime runtime {user_data, 0, ReadHostMemory, nullptr}; + Check(EvaluateDescriptorSource(program, source_id, sample.pc, runtime, &descriptor, &error), + error.c_str()); + Check(descriptor.dwords[0] == table[72], + "readlane descriptor spill evaluated the overwritten scalar value"); +} + +void TestReadLaneVectorOverwriteInvalidatesSpill() { + Program program; + program.wave_size = 64; + program.user_data_count = 8; + program.blocks.resize(1); + Instruction overwrite; + overwrite.pc = 4; + overwrite.op = Opcode::MoveU32; + overwrite.dst = Vgpr(11); + overwrite.src[0] = Imm(0); + overwrite.src_count = 1; + program.blocks[0].instructions = {WriteLane(0, 11, 4, 0), overwrite, + ReadLane(8, 0, 11, 0), BufferUse(12, 0)}; + + std::string error; + Check(BuildScalarProvenance(&program, &error), error.c_str()); + const auto source = program.blocks[0].instructions.back().memory.resource_source; + Check(GetDescriptorSource(program, source) != nullptr && + !DescriptorSourceResolved(program, source), + "vector overwrite left stale writelane provenance"); +} + +void TestReadLanePhi() { + Program program; + program.wave_size = 32; + program.user_data_count = 8; + program.blocks.resize(4); + program.blocks[0].successors = {1, 2}; + program.blocks[1].predecessors = {0}; + program.blocks[1].successors = {3}; + program.blocks[2].predecessors = {0}; + program.blocks[2].successors = {3}; + program.blocks[3].predecessors = {1, 2}; + program.blocks[1].instructions = {WriteLane(0, 11, 4, 0)}; + program.blocks[2].instructions = {WriteLane(4, 11, 5, 0)}; + program.blocks[3].instructions = {ReadLane(8, 0, 11, 0), BufferUse(12, 0)}; + + std::string error; + Check(BuildScalarProvenance(&program, &error), error.c_str()); + const auto* source = + GetDescriptorSource(program, program.blocks[3].instructions.back().memory.resource_source); + Check(source != nullptr && Value(program, source->dwords[0]).op == ScalarValueOp::Phi && + DescriptorSourceResolved( + program, program.blocks[3].instructions.back().memory.resource_source), + "readlane definitions were not merged across CFG predecessors"); +} + +void TestReadLaneLoopConvergence() { + Program program; + program.wave_size = 64; + program.user_data_count = 8; + program.blocks.resize(3); + program.blocks[0].successors = {1}; + program.blocks[1].predecessors = {0, 2}; + program.blocks[1].successors = {2}; + program.blocks[2].predecessors = {1}; + program.blocks[2].successors = {1}; + program.blocks[0].instructions = {WriteLane(0, 11, 4, 0)}; + program.blocks[1].instructions = {ReadLane(4, 0, 11, 0), BufferUse(8, 0)}; + + std::string error; + Check(BuildScalarProvenance(&program, &error), error.c_str()); + const auto source = program.blocks[1].instructions.back().memory.resource_source; + Check(DescriptorSourceResolved(program, source), + "unchanged writelane spill did not converge across a loop backedge"); +} + +void TestReadLaneWideAndRelativeWritesInvalidateSpill() { + const auto check_invalidated = [](Opcode op, const char* message) { + Program program; + program.wave_size = 64; + program.user_data_count = 8; + program.blocks.resize(1); + Instruction overwrite; + overwrite.pc = 4; + overwrite.op = op; + overwrite.dst = Vgpr(11); + overwrite.src[0] = Imm(0); + overwrite.src_count = 1; + program.blocks[0].instructions = {WriteLane(0, 12, 4, 0), overwrite, + ReadLane(8, 0, 12, 0), BufferUse(12, 0)}; + + std::string error; + Check(BuildScalarProvenance(&program, &error), error.c_str()); + const auto source = program.blocks[0].instructions.back().memory.resource_source; + Check(!DescriptorSourceResolved(program, source), message); + }; + + check_invalidated(Opcode::UMadU64U32, + "64-bit vector destination left stale high-half lane provenance"); + check_invalidated(Opcode::MoveRelDestU32, + "relative vector destination left stale lane provenance"); +} + +void TestReadLaneModuloAndDynamicLane() { + Program modulo; + modulo.wave_size = 32; + modulo.user_data_count = 8; + modulo.blocks.resize(1); + modulo.blocks[0].instructions = {WriteLane(0, 11, 4, 33), ReadLane(4, 0, 11, 65), + BufferUse(8, 0)}; + std::string error; + Check(BuildScalarProvenance(&modulo, &error), error.c_str()); + const auto* modulo_source = + GetDescriptorSource(modulo, modulo.blocks[0].instructions.back().memory.resource_source); + Check(modulo_source != nullptr && + Value(modulo, modulo_source->dwords[0]).op == ScalarValueOp::UserData && + Value(modulo, modulo_source->dwords[0]).imm == 4, + "readlane/writelane did not wrap the lane index to wave size"); + + Program dynamic; + dynamic.wave_size = 64; + dynamic.user_data_count = 8; + dynamic.blocks.resize(1); + auto write = WriteLane(0, 11, 4, 0); + write.src[1] = Sgpr(7); + dynamic.blocks[0].instructions = {write, ReadLane(4, 0, 11, 0), BufferUse(8, 0)}; + Check(BuildScalarProvenance(&dynamic, &error), error.c_str()); + Check(!DescriptorSourceResolved( + dynamic, dynamic.blocks[0].instructions.back().memory.resource_source), + "dynamic writelane selector retained unsafe lane provenance"); +} + +void TestUnresolvedSourceIsMarked() { + Program program; + program.blocks.resize(1); + Instruction unknown; + unknown.op = Opcode::BitwiseNotU64; + unknown.dst = Sgpr(0); + program.blocks[0].instructions = {unknown, BufferUse(4, 0)}; + + std::string error; + Check(BuildScalarProvenance(&program, &error), error.c_str()); + const auto source = program.blocks[0].instructions.back().memory.resource_source; + Check(GetDescriptorSource(program, source) != nullptr && + !DescriptorSourceResolved(program, source), + "unknown descriptor tuple was incorrectly marked resolved"); +} + +void TestUnsupportedWideWriteInvalidatesBothDwords() { + Program program; + program.blocks.resize(1); + Instruction unsupported; + unsupported.op = Opcode::BitwiseNotU64; + unsupported.dst = Sgpr(0); + unsupported.src[0] = Sgpr(20); + unsupported.src_count = 1; + program.blocks[0].instructions = {Move(0, 0, 20), Move(4, 1, 21), unsupported, + BufferUse(12, 0)}; + + std::string error; + Check(BuildScalarProvenance(&program, &error), error.c_str()); + const auto* source = + GetDescriptorSource(program, program.blocks[0].instructions.back().memory.resource_source); + Check(source != nullptr && source->dwords[0] == ScalarProvenance::Unknown && + source->dwords[1] == ScalarProvenance::Unknown, + "unsupported wide write left a stale descriptor half"); +} + +void TestCyclicPhiIsUnresolved() { + Program program; + program.blocks.resize(2); + program.blocks[0].successors = {1}; + program.blocks[1].predecessors = {0, 1}; + program.blocks[1].successors = {1}; + Instruction increment; + increment.op = Opcode::IAddU32; + increment.dst = Sgpr(0); + increment.src[0] = Sgpr(0); + increment.src[1] = Imm(1); + increment.src_count = 2; + program.blocks[1].instructions = {BufferUse(4, 0), increment}; + + std::string error; + Check(BuildScalarProvenance(&program, &error), error.c_str()); + const auto source = program.blocks[1].instructions[0].memory.resource_source; + Check(GetDescriptorSource(program, source) != nullptr && + !DescriptorSourceResolved(program, source), + "loop-carried cyclic descriptor provenance was incorrectly resolved"); + Check(BuildSrtPlan(&program, &error) && program.srt.dynamic_sources.size() == 1 && + program.srt.dynamic_sources[0] == source, + "SRT planning did not isolate a cyclic descriptor as a dynamic source"); +} + +void TestNestedSrtWalk() { + std::array child = {0x11111111u, 0x22334455u, 0, 0}; + std::array root = {}; + const auto child_address = reinterpret_cast(child.data()); + root[0] = static_cast(child_address); + root[1] = static_cast(child_address >> 32u); + + Program program; + program.blocks.resize(1); + for (uint32_t i = 0; i < 2; i++) { + Instruction load; + load.pc = i * 4; + load.op = Opcode::SLoadDword; + load.dst = Sgpr(8 + i); + load.src[0] = Imm(0); + load.src_count = 1; + load.memory.kind = ResourceKind::ScalarBuffer; + load.memory.resource = 4; + load.memory.offset = i * 4; + program.blocks[0].instructions.push_back(load); + } + Instruction child_load; + child_load.pc = 8; + child_load.op = Opcode::SLoadDword; + child_load.dst = Sgpr(16); + child_load.src[0] = Imm(0); + child_load.src_count = 1; + child_load.memory.kind = ResourceKind::ScalarBuffer; + child_load.memory.resource = 8; + child_load.memory.offset = 6; // RDNA2 SMEM ignores the low two address bits. + program.blocks[0].instructions.push_back(child_load); + program.blocks[0].instructions.push_back(BufferUse(12, 16)); + + std::string error; + Check(BuildScalarProvenance(&program, &error) && BuildSrtPlan(&program, &error), error.c_str()); + Check(program.srt.reads.size() == 3 && program.srt.dynamic_reads.empty(), + "nested SRT reads were not compacted and deduplicated"); + std::vector user_data(64); + SetPointer(&user_data, 4, root.data()); + std::vector flat; + const SrtRuntime runtime {user_data, 0, ReadHostMemory, nullptr}; + Check(WalkSrt(program, runtime, &flat, &error), error.c_str()); + Check(flat.size() == 3 && flat.back() == child[1], + "nested SRT walker read the wrong aligned dword"); +} + +void TestDynamicReadIsNotFlattened() { + std::array table = {0xaabbccddu, 0x12345678u, 0, 0}; + Program program; + program.blocks.resize(1); + Instruction load; + load.op = Opcode::SLoadDword; + load.dst = Sgpr(16); + load.src[0] = Sgpr(6); + load.src_count = 1; + load.memory.kind = ResourceKind::ScalarBuffer; + load.memory.resource = 4; + program.blocks[0].instructions = {load, BufferUse(4, 16)}; + + std::string error; + Check(BuildScalarProvenance(&program, &error) && BuildSrtPlan(&program, &error), error.c_str()); + Check(program.srt.reads.empty() && program.srt.dynamic_reads.size() == 1, + "dynamic ReadConst was assigned a flat-buffer offset"); + std::vector user_data(64); + SetPointer(&user_data, 4, table.data()); + user_data[6] = 4; + DescriptorValue descriptor; + const auto source = program.blocks[0].instructions[1].memory.resource_source; + const SrtRuntime runtime {user_data, 0, ReadHostMemory, nullptr}; + Check(EvaluateDescriptorSource(program, source, 4, runtime, &descriptor, &error), + error.c_str()); + Check(descriptor.dwords[0] == table[1], "dynamic ReadConst evaluated the wrong dword"); +} + +void TestReadConstBufferWalk() { + std::array table = {0x76543210u, 0, 0, 0}; + Program program; + program.blocks.resize(1); + Instruction load; + load.op = Opcode::SBufferLoadDword; + load.dst = Sgpr(16); + load.src[0] = Imm(0); + load.src_count = 1; + load.memory.kind = ResourceKind::ScalarBuffer; + load.memory.resource = 1; + program.blocks[0].instructions = {load, BufferUse(4, 16)}; + + std::string error; + Check(BuildScalarProvenance(&program, &error) && BuildSrtPlan(&program, &error), error.c_str()); + std::vector user_data(64); + SetPointer(&user_data, 4, table.data()); + user_data[6] = 4; + std::vector flat; + const SrtRuntime runtime {user_data, 0, ReadHostMemory, nullptr}; + Check(WalkSrt(program, runtime, &flat, &error), error.c_str()); + Check(flat.size() == 1 && flat[0] == table[0], "ReadConstBuffer walker result was wrong"); +} + +void TestM0ScalarOffset() { + std::array table = {0x01234567u, 0x89abcdefu}; + Program program; + program.blocks.resize(1); + Instruction set_m0; + set_m0.op = Opcode::MoveU32; + set_m0.dst = M0(); + set_m0.src[0] = Imm(4); + set_m0.src_count = 1; + Instruction load; + load.pc = 4; + load.op = Opcode::SLoadDword; + load.dst = Sgpr(16); + load.src[0] = M0(); + load.src_count = 1; + load.memory.kind = ResourceKind::ScalarBuffer; + load.memory.resource = 4; + program.blocks[0].instructions = {set_m0, load, BufferUse(8, 16)}; + + std::string error; + Check(BuildScalarProvenance(&program, &error) && BuildSrtPlan(&program, &error), error.c_str()); + Check(program.srt.reads.size() == 1 && program.srt.dynamic_reads.empty(), + "constant M0 offset was not flattened"); + std::vector user_data(64); + SetPointer(&user_data, 4, table.data()); + std::vector flat; + const SrtRuntime runtime {user_data, 0, ReadHostMemory, nullptr}; + Check(WalkSrt(program, runtime, &flat, &error), error.c_str()); + Check(flat.size() == 1 && flat[0] == table[1], "M0 scalar offset evaluated incorrectly"); +} + +void TestM0ArithmeticPreservesScc() { + Program program; + program.blocks.resize(1); + Instruction low; + low.op = Opcode::ScalarAddCarryU32; + low.dst = M0(); + low.src[0] = Sgpr(0); + low.src[1] = Imm(1); + low.src[2] = Imm(0); + low.src_count = 3; + Instruction high = low; + high.pc = 4; + high.dst = Sgpr(16); + high.src[0] = Imm(5); + high.src[1] = Imm(0); + high.src[2].kind = OperandKind::Register; + high.src[2].reg.file = RegisterFile::Scc; + program.blocks[0].instructions = {low, + high, + MoveImmediate(8, 17, 0), + MoveImmediate(12, 18, 0), + MoveImmediate(16, 19, 0), + BufferUse(20, 16)}; + + std::string error; + Check(BuildScalarProvenance(&program, &error) && BuildSrtPlan(&program, &error), error.c_str()); + const std::array user_data = {0xffffffffu}; + const auto source = program.blocks[0].instructions.back().memory.resource_source; + DescriptorValue descriptor; + const SrtRuntime runtime {user_data, 0, nullptr, nullptr}; + Check(EvaluateDescriptorSource(program, source, 20, runtime, &descriptor, &error), + error.c_str()); + Check(descriptor.dwords[0] == 6, "arithmetic M0 destination lost its SCC carry side effect"); +} + +void TestShaderRelativePointer() { + std::array table = {}; + Program program; + program.blocks.resize(1); + Instruction getpc_low = MoveImmediate(0x20, 0, 0x24); + getpc_low.src[0].kind = OperandKind::PcRelativeU32; + Instruction getpc_high = MoveImmediate(0x20, 1, 0); + program.blocks[0].instructions = {getpc_low, getpc_high, BufferUse(0x28, 0)}; + + std::string error; + Check(BuildScalarProvenance(&program, &error) && BuildSrtPlan(&program, &error), error.c_str()); + const auto source = program.blocks[0].instructions.back().memory.resource_source; + DescriptorValue descriptor; + const auto shader_base = reinterpret_cast(table.data()) - 0x24; + const std::array user_data = {}; + const SrtRuntime runtime {user_data, shader_base, nullptr, nullptr}; + Check(EvaluateDescriptorSource(program, source, 0x28, runtime, &descriptor, &error), + error.c_str()); + const auto expected = reinterpret_cast(table.data()); + Check(descriptor.dwords[0] == static_cast(expected) && + descriptor.dwords[1] == static_cast(expected >> 32u), + "shader-relative pointer lost its high half or carry"); +} + +void TestSmemComponentAlignment() { + std::array table = {0x11111111u, 0x22222222u}; + std::vector user_data(20); + SetPointer(&user_data, 4, table.data()); + + Program raw; + raw.blocks.resize(1); + Instruction raw_load; + raw_load.pc = 4; + raw_load.op = Opcode::SLoadDword; + raw_load.dst = Sgpr(16); + raw_load.src[0] = Imm(2); + raw_load.src_count = 1; + raw_load.memory.kind = ResourceKind::ScalarBuffer; + raw_load.memory.resource = 4; + raw_load.memory.offset = 2; + raw.blocks[0].instructions = {raw_load, BufferUse(12, 16)}; + std::string error; + Check(BuildScalarProvenance(&raw, &error) && BuildSrtPlan(&raw, &error), error.c_str()); + std::vector flat; + const SrtRuntime runtime {user_data, 0, ReadHostMemory, nullptr}; + Check(WalkSrt(raw, runtime, &flat, &error), error.c_str()); + Check(flat.size() == 1 && flat[0] == table[0], + "raw S_LOAD did not clear each address component's low bits"); + + Program buffer; + buffer.blocks.resize(1); + Instruction buffer_load = raw_load; + buffer_load.op = Opcode::SBufferLoadDword; + buffer_load.memory.resource = 1; + buffer.blocks[0].instructions = {buffer_load, BufferUse(12, 16)}; + user_data[6] = sizeof(table); + Check(BuildScalarProvenance(&buffer, &error) && BuildSrtPlan(&buffer, &error), error.c_str()); + Check(WalkSrt(buffer, runtime, &flat, &error), error.c_str()); + Check(flat.size() == 1 && flat[0] == table[1], + "S_BUFFER_LOAD did not align the summed final address"); + + buffer.blocks[0].instructions[0].src[0] = Imm(0); + user_data[4] += 2; + Check(BuildScalarProvenance(&buffer, &error) && BuildSrtPlan(&buffer, &error), error.c_str()); + Check(WalkSrt(buffer, runtime, &flat, &error), error.c_str()); + Check(flat.size() == 1 && flat[0] == table[0], + "S_BUFFER_LOAD did not clear the descriptor base low bits before " + "addition"); +} + +void TestReadConstSignedAddress() { + std::array table = {0x11111111u, 0x22222222u}; + std::vector user_data(24); + SetPointer(&user_data, 4, table.data() + 1); + + Program program; + program.blocks.resize(1); + Instruction load; + load.pc = 4; + load.op = Opcode::SLoadDword; + load.dst = Sgpr(16); + load.src[0] = Sgpr(20); + load.src_count = 1; + load.memory.kind = ResourceKind::ScalarBuffer; + load.memory.resource = 4; + load.memory.offset = static_cast(-8); + program.blocks[0].instructions = {MoveImmediate(0, 20, 4), load, BufferUse(8, 16)}; + + std::string error; + Check(BuildScalarProvenance(&program, &error) && BuildSrtPlan(&program, &error), error.c_str()); + std::vector flat; + const SrtRuntime runtime {user_data, 0, ReadHostMemory, nullptr}; + Check(WalkSrt(program, runtime, &flat, &error), error.c_str()); + Check(flat.size() == 1 && flat[0] == table[0], + "signed S_LOAD offset resolved the wrong SBASE-relative address"); + + user_data[4] = 0; + user_data[5] = 0; + Check(!WalkSrt(program, runtime, &flat, &error) && + error.find("outside the 48-bit address space") != std::string::npos, + "signed S_LOAD address underflow did not fail closed"); +} + +void TestReadConstBufferBounds() { + std::array table = {0x11111111u, 0x22222222u}; + Program program; + program.stage = ShaderType::Compute; + program.shader_hash = 0x1234; + program.blocks.resize(1); + Instruction load; + load.pc = 0x40; + load.op = Opcode::SBufferLoadDword; + load.dst = Sgpr(16); + load.src[0] = Imm(0); + load.src_count = 1; + load.memory.kind = ResourceKind::ScalarBuffer; + load.memory.resource = 1; + load.memory.offset = sizeof(table); + program.blocks[0].instructions = {load, BufferUse(0x44, 16)}; + std::string error; + Check(BuildScalarProvenance(&program, &error) && BuildSrtPlan(&program, &error), error.c_str()); + std::vector user_data(20); + SetPointer(&user_data, 4, table.data()); + user_data[5] |= 4u << 16u; + user_data[6] = 2; + ReadCounter counter; + std::vector flat; + const SrtRuntime runtime {user_data, 0, CountedHostMemory, &counter}; + Check(!WalkSrt(program, runtime, &flat, &error), + "out-of-range S_BUFFER_LOAD unexpectedly succeeded"); + Check(counter.reads == 0 && error.find("exceeds size=8") != std::string::npos && + error.find("hash=0x0000000000001234") != std::string::npos && + error.find("stage=compute") != std::string::npos && + error.find("pc=0x00000044") != std::string::npos, + "S_BUFFER_LOAD bounds failure read memory or lost shader context"); +} + +void TestSubBorrowPointerChain() { + Program program; + program.blocks.resize(1); + Instruction low; + low.op = Opcode::ScalarSubBorrowU32; + low.dst = Sgpr(16); + low.src[0] = Sgpr(0); + low.src[1] = Imm(4); + low.src_count = 2; + Instruction high; + high.pc = 4; + high.op = Opcode::ScalarSubBorrowCarryU32; + high.dst = Sgpr(17); + high.src[0] = Sgpr(1); + high.src[1] = Imm(0); + high.src[2].kind = OperandKind::Register; + high.src[2].reg.file = RegisterFile::Scc; + high.src_count = 3; + program.blocks[0].instructions = {low, high, MoveImmediate(8, 18, 0), MoveImmediate(12, 19, 0), + BufferUse(16, 16)}; + std::string error; + Check(BuildScalarProvenance(&program, &error) && BuildSrtPlan(&program, &error), error.c_str()); + const std::array user_data = {2, 5}; + DescriptorValue descriptor; + const auto source = program.blocks[0].instructions.back().memory.resource_source; + const SrtRuntime runtime {user_data, 0, nullptr, nullptr}; + Check(EvaluateDescriptorSource(program, source, 16, runtime, &descriptor, &error), + error.c_str()); + Check(descriptor.dwords[0] == 0xfffffffeu && descriptor.dwords[1] == 4, + "scalar subtraction/borrow pointer chain evaluated incorrectly"); +} + +void TestCommonScalarPointerOps() { + Program program; + program.blocks.resize(1); + const auto binary = [](Opcode op, uint32_t dst, Operand lhs, Operand rhs) { + Instruction inst; + inst.op = op; + inst.dst = Sgpr(dst); + inst.src[0] = lhs; + inst.src[1] = rhs; + inst.src_count = 2; + return inst; + }; + Instruction add3; + add3.op = Opcode::IAdd3U32; + add3.dst = Sgpr(19); + add3.src[0] = Sgpr(3); + add3.src[1] = Imm(1); + add3.src[2] = Imm(2); + add3.src_count = 3; + program.blocks[0].instructions = {binary(Opcode::BitwiseXorU32, 16, Sgpr(0), Sgpr(1)), + binary(Opcode::BitwiseAndNotU32, 17, Sgpr(0), Sgpr(1)), + binary(Opcode::IMulU32, 18, Sgpr(2), Imm(4)), add3, + BufferUse(16, 16)}; + std::string error; + Check(BuildScalarProvenance(&program, &error) && BuildSrtPlan(&program, &error), error.c_str()); + const std::array user_data = {0xf0f0u, 0x0ff0u, 7u, 9u}; + DescriptorValue descriptor; + const auto source = program.blocks[0].instructions.back().memory.resource_source; + const SrtRuntime runtime {user_data, 0, nullptr, nullptr}; + Check(EvaluateDescriptorSource(program, source, 16, runtime, &descriptor, &error), + error.c_str()); + Check(descriptor.dwords[0] == (user_data[0] ^ user_data[1]) && + descriptor.dwords[1] == (user_data[0] & ~user_data[1]) && + descriptor.dwords[2] == 28 && descriptor.dwords[3] == 12, + "common scalar pointer operations evaluated incorrectly"); +} + +void TestBitFieldMaskDescriptor() { + Program program; + program.blocks.resize(1); + auto mask = MoveImmediate(4, 29, 0); + mask.op = Opcode::BitFieldMaskU32; + mask.src[0] = Imm(12); + mask.src[1] = Imm(12); + mask.src_count = 2; + auto high = MoveImmediate(8, 30, 0x05500000u); + high.op = Opcode::MoveU64; + program.blocks[0].instructions = {MoveImmediate(0, 28, 0x92u), mask, high, + BufferUse(12, 28)}; + + std::string error; + Check(BuildScalarProvenance(&program, &error) && BuildSrtPlan(&program, &error), error.c_str()); + DescriptorValue descriptor; + const SrtRuntime runtime {{}, 0, nullptr, nullptr}; + Check(EvaluateDescriptorSource(program, + program.blocks[0].instructions.back().memory.resource_source, + 16, runtime, &descriptor, &error), + error.c_str()); + Check(descriptor.dwords[0] == 0x92u && descriptor.dwords[1] == 0x00fff000u && + descriptor.dwords[2] == 0x05500000u && descriptor.dwords[3] == 0, + "production sampler bit-field mask evaluated incorrectly"); + + mask.src[0] = Imm(0); + program.blocks[0].instructions = {MoveImmediate(0, 28, 0x92u), mask, high, + BufferUse(12, 28)}; + Check(BuildScalarProvenance(&program, &error) && BuildSrtPlan(&program, &error), error.c_str()); + Check(EvaluateDescriptorSource(program, + program.blocks[0].instructions.back().memory.resource_source, + 12, runtime, &descriptor, &error), + error.c_str()); + Check(descriptor.dwords[1] == 0, "zero-width bit-field mask was not zero"); + + mask.src[0] = Imm(31); + mask.src[1] = Imm(31); + program.blocks[0].instructions = {MoveImmediate(0, 28, 0x92u), mask, high, + BufferUse(12, 28)}; + Check(BuildScalarProvenance(&program, &error) && BuildSrtPlan(&program, &error), error.c_str()); + Check(EvaluateDescriptorSource(program, + program.blocks[0].instructions.back().memory.resource_source, + 12, runtime, &descriptor, &error), + error.c_str()); + Check(descriptor.dwords[1] == 0x80000000u, + "maximum bit-field mask count/offset evaluated incorrectly"); +} + +} // namespace + +int main() { + try { + TestPerUseDescriptorDefinitions(); + TestCfgPhi(); + TestDiamondReadPathsAreDynamic(); + TestEquivalentConstantPhiIsStatic(); + TestWideMoveInvalidatesAndCopiesBothDwords(); + TestScalarCarryChain(); + TestReadConstDefinition(); + TestReadConstBufferAndValueNumbering(); + TestReadLaneDescriptorSpill(); + TestReadLaneVectorOverwriteInvalidatesSpill(); + TestReadLanePhi(); + TestReadLaneLoopConvergence(); + TestReadLaneWideAndRelativeWritesInvalidateSpill(); + TestReadLaneModuloAndDynamicLane(); + TestUnresolvedSourceIsMarked(); + TestUnsupportedWideWriteInvalidatesBothDwords(); + TestCyclicPhiIsUnresolved(); + TestNestedSrtWalk(); + TestDynamicReadIsNotFlattened(); + TestReadConstBufferWalk(); + TestM0ScalarOffset(); + TestM0ArithmeticPreservesScc(); + TestShaderRelativePointer(); + TestSmemComponentAlignment(); + TestReadConstSignedAddress(); + TestReadConstBufferBounds(); + TestSubBorrowPointerChain(); + TestCommonScalarPointerOps(); + TestBitFieldMaskDescriptor(); + std::cout << "ScalarProvenanceTests: all cases passed\n"; + return 0; + } catch (const std::exception& e) { + std::cerr << "ScalarProvenanceTests: failed: " << e.what() << '\n'; + return 1; + } +} diff --git a/tests/ShaderRecompilerComputeTests.cpp b/tests/ShaderRecompilerComputeTests.cpp new file mode 100644 index 000000000..0ef9f4a3d --- /dev/null +++ b/tests/ShaderRecompilerComputeTests.cpp @@ -0,0 +1,11095 @@ +#include "common/threads.h" + +#include "common/emulatorConfig.h" +#include "graphics/guest_gpu/gpu_defs.h" +#include "graphics/guest_gpu/hardwareContext.h" +#include "graphics/guest_gpu/tile.h" +#include "graphics/host_gpu/hostMemory.h" +#include "graphics/host_gpu/memoryTracker.h" +#include "graphics/host_gpu/objects/textureCommon.h" +#include "graphics/host_gpu/pageManager.h" +#include "graphics/host_gpu/renderer/bufferCache.h" +#include "graphics/host_gpu/renderer/descriptors.h" +#include "graphics/host_gpu/renderer/image.h" +#include "graphics/host_gpu/renderer/imageView.h" +#include "graphics/host_gpu/renderer/render.h" +#include "graphics/host_gpu/renderer/renderState.h" +#include "graphics/host_gpu/renderer/resourceMutex.h" +#include "graphics/host_gpu/renderer/textureCache.h" +#include "graphics/shader/recompiler/BindingLayout.h" +#include "graphics/shader/recompiler/ShaderDecoder.h" +#include "graphics/shader/recompiler/ShaderRecompiler.h" +#include "graphics/shader/recompiler/SpirvBuilder.h" +#include "graphics/shader/shader.h" +#include "common/assert.h" +#include "common/logging/log.h" + +#include "spirv-tools/libspirv.hpp" +#include "vulkan/vulkan.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS +#ifndef NOMINMAX +#define NOMINMAX +#endif +#include +#undef min +#undef max +#endif + +namespace Libs::Graphics { +namespace { + +using u32 = uint32_t; +using ShaderOpcode = ShaderRecompiler::Decoder::Opcode; + +constexpr u32 InlineU32(u32 value) { return 128u + value; } + +constexpr u32 Vgpr(u32 reg) { return 256u + reg; } + +constexpr u32 EncodeSMovB32(u32 dst, u32 src) { + return 0x80000000u | (0x7du << 23u) | ((dst & 0x7fu) << 16u) | (0x03u << 8u) | + (src & 0xffu); +} + +constexpr u32 EncodeSop1(u32 opcode, u32 dst, u32 src) { + return 0x80000000u | (0x7du << 23u) | ((dst & 0x7fu) << 16u) | + ((opcode & 0xffu) << 8u) | (src & 0xffu); +} + +constexpr u32 EncodeSop2(u32 opcode, u32 dst, u32 src0, u32 src1) { + return 0x80000000u | ((opcode & 0x7fu) << 23u) | ((dst & 0x7fu) << 16u) | + ((src1 & 0xffu) << 8u) | (src0 & 0xffu); +} + +constexpr u32 EncodeSopc(u32 opcode, u32 src0, u32 src1) { + return 0x80000000u | (0x7eu << 23u) | ((opcode & 0x7fu) << 16u) | + ((src1 & 0xffu) << 8u) | (src0 & 0xffu); +} + +constexpr u32 EncodeSopp(u32 opcode, u32 simm = 0) { + return 0x80000000u | (0x7fu << 23u) | ((opcode & 0x7fu) << 16u) | + (simm & 0xffffu); +} + +constexpr u32 EncodeSopk(u32 opcode, u32 dst, u32 imm) { + return 0x80000000u | (((opcode + 0x60u) & 0x7fu) << 23u) | + ((dst & 0x7fu) << 16u) | (imm & 0xffffu); +} + +constexpr u32 EncodeVop1(u32 opcode, u32 dst, u32 src0) { + return (0x3fu << 25u) | ((dst & 0xffu) << 17u) | ((opcode & 0xffu) << 9u) | + (src0 & 0x1ffu); +} + +constexpr u32 EncodeVop1Sdwa(u32 src0, u32 dst_sel = 6, u32 dst_u = 0, + u32 src0_sel = 6, u32 src0_sext = 0, + u32 src0_neg = 0, u32 src0_abs = 0, u32 s0 = 0, + u32 clamp = 0, u32 omod = 0) { + return (src0 & 0xffu) | ((dst_sel & 0x7u) << 8u) | ((dst_u & 0x3u) << 11u) | + ((clamp & 0x1u) << 13u) | ((omod & 0x3u) << 14u) | + ((src0_sel & 0x7u) << 16u) | ((src0_sext & 0x1u) << 19u) | + ((src0_neg & 0x1u) << 20u) | ((src0_abs & 0x1u) << 21u) | + ((s0 & 0x1u) << 23u); +} + +constexpr u32 EncodeVop2(u32 opcode, u32 dst, u32 src0, u32 src1) { + return ((opcode & 0x3fu) << 25u) | ((dst & 0xffu) << 17u) | + ((src1 & 0xffu) << 9u) | (src0 & 0x1ffu); +} + +constexpr u32 EncodeVop2Sdwa(u32 src0, u32 dst_sel = 6, u32 dst_u = 0, + u32 src0_sel = 6, u32 src1_sel = 6, + u32 src0_sext = 0, u32 src1_sext = 0, + u32 src0_neg = 0, u32 src0_abs = 0, + u32 src1_neg = 0, u32 src1_abs = 0, u32 s0 = 0, + u32 s1 = 0, u32 clamp = 0, u32 omod = 0) { + return (src0 & 0xffu) | ((dst_sel & 0x7u) << 8u) | ((dst_u & 0x3u) << 11u) | + ((clamp & 0x1u) << 13u) | ((omod & 0x3u) << 14u) | + ((src0_sel & 0x7u) << 16u) | ((src0_sext & 0x1u) << 19u) | + ((src0_neg & 0x1u) << 20u) | ((src0_abs & 0x1u) << 21u) | + ((s0 & 0x1u) << 23u) | ((src1_sel & 0x7u) << 24u) | + ((src1_sext & 0x1u) << 27u) | ((src1_neg & 0x1u) << 28u) | + ((src1_abs & 0x1u) << 29u) | ((s1 & 0x1u) << 31u); +} + +constexpr u32 EncodeVop2Dpp(u32 src0, u32 dpp_ctrl = 0, u32 row_mask = 0xf, + u32 bank_mask = 0xf, u32 src0_neg = 0, + u32 src0_abs = 0, u32 src1_neg = 0, + u32 src1_abs = 0) { + return (src0 & 0xffu) | ((dpp_ctrl & 0x1ffu) << 8u) | + ((src0_neg & 0x1u) << 20u) | ((src0_abs & 0x1u) << 21u) | + ((src1_neg & 0x1u) << 22u) | ((src1_abs & 0x1u) << 23u) | + ((bank_mask & 0xfu) << 24u) | ((row_mask & 0xfu) << 28u); +} + +constexpr u32 EncodeVop3Word0(u32 opcode, u32 dst, u32 abs = 0, u32 op_sel = 0, + bool clamp = false) { + return (0x35u << 26u) | ((opcode & 0x3ffu) << 16u) | ((abs & 0x7u) << 8u) | + ((op_sel & 0xfu) << 11u) | (clamp ? (1u << 15u) : 0u) | (dst & 0xffu); +} + +constexpr u32 EncodeVop3BWord0(u32 opcode, u32 vdst, u32 sdst) { + return (0x35u << 26u) | ((opcode & 0x3ffu) << 16u) | ((sdst & 0x7fu) << 8u) | + (vdst & 0xffu); +} + +constexpr u32 EncodeVop3Word1(u32 src0, u32 src1, u32 src2 = 0, u32 omod = 0, + u32 neg = 0) { + return (src0 & 0x1ffu) | ((src1 & 0x1ffu) << 9u) | ((src2 & 0x1ffu) << 18u) | + ((omod & 0x3u) << 27u) | ((neg & 0x7u) << 29u); +} + +constexpr u32 EncodeVop3pWord0(u32 opcode, u32 dst, u32 op_sel_hi = 0, + u32 op_sel = 0, u32 neg_hi = 0, + bool clamp = false) { + return (0x33u << 26u) | ((opcode & 0x7fu) << 16u) | ((neg_hi & 0x7u) << 8u) | + ((op_sel & 0x7u) << 11u) | ((op_sel_hi & 0x4u) << 12u) | + (clamp ? (1u << 15u) : 0u) | (dst & 0xffu); +} + +constexpr u32 EncodeVop3pWord1(u32 src0, u32 src1, u32 src2 = 0, + u32 op_sel_hi = 0, u32 neg = 0) { + return (src0 & 0x1ffu) | ((src1 & 0x1ffu) << 9u) | ((src2 & 0x1ffu) << 18u) | + ((op_sel_hi & 0x3u) << 27u) | ((neg & 0x7u) << 29u); +} + +constexpr u32 EncodeVopc(u32 opcode, u32 src0, u32 src1) { + return (0x3eu << 25u) | ((opcode & 0xffu) << 17u) | ((src1 & 0xffu) << 9u) | + (src0 & 0x1ffu); +} + +constexpr u32 EncodeVopcSdwa(u32 src0, u32 sdst = 0, u32 sd = 0, + u32 src0_sel = 6, u32 src1_sel = 6, + u32 src0_sext = 0, u32 src1_sext = 0, + u32 src0_neg = 0, u32 src0_abs = 0, + u32 src1_neg = 0, u32 src1_abs = 0, u32 s0 = 0, + u32 s1 = 0) { + return (src0 & 0xffu) | ((sdst & 0x7fu) << 8u) | ((sd & 0x1u) << 15u) | + ((src0_sel & 0x7u) << 16u) | ((src0_sext & 0x1u) << 19u) | + ((src0_neg & 0x1u) << 20u) | ((src0_abs & 0x1u) << 21u) | + ((s0 & 0x1u) << 23u) | ((src1_sel & 0x7u) << 24u) | + ((src1_sext & 0x1u) << 27u) | ((src1_neg & 0x1u) << 28u) | + ((src1_abs & 0x1u) << 29u) | ((s1 & 0x1u) << 31u); +} + +constexpr u32 EncodeMubuf0(u32 opcode, u32 offset = 0, bool idxen = false, + bool offen = true, bool glc = false) { + return (0x38u << 26u) | ((opcode & 0x7fu) << 18u) | + (offen ? (1u << 12u) : 0u) | (idxen ? (1u << 13u) : 0u) | + (glc ? (1u << 14u) : 0u) | (offset & 0xfffu); +} + +constexpr u32 EncodeMubuf1(u32 vdata, u32 srsrc, u32 vaddr, u32 soffset = 128) { + return ((soffset & 0xffu) << 24u) | ((srsrc & 0x1fu) << 16u) | + ((vdata & 0xffu) << 8u) | (vaddr & 0xffu); +} + +constexpr u32 EncodeMtbuf0(u32 opcode, u32 dfmt, u32 nfmt, u32 offset = 0, + bool idxen = false, bool offen = true) { + return (0x3au << 26u) | (offset & 0xfffu) | (offen ? (1u << 12u) : 0u) | + (idxen ? (1u << 13u) : 0u) | ((opcode & 0x7u) << 16u) | + ((dfmt & 0xfu) << 19u) | ((nfmt & 0x7u) << 23u); +} + +constexpr u32 EncodeMtbuf1(u32 opcode, u32 vdata, u32 srsrc, u32 vaddr, + u32 soffset = 128) { + return (((opcode >> 3u) & 1u) << 21u) | ((soffset & 0xffu) << 24u) | + ((srsrc & 0x1fu) << 16u) | ((vdata & 0xffu) << 8u) | (vaddr & 0xffu); +} + +constexpr u32 EncodeSmem0(u32 opcode, u32 dst, u32 sbase = 0) { + return (0x3du << 26u) | ((opcode & 0xffu) << 18u) | ((dst & 0x7fu) << 6u) | + (sbase & 0x3fu); +} + +constexpr u32 EncodeSmem1(u32 offset, u32 soffset = 0) { + return (offset & 0x1fffffu) | ((soffset & 0x7fu) << 25u); +} + +constexpr u32 EncodeMimg0(u32 opcode, u32 dmask, u32 nsa_dwords = 0, + bool glc = false, u32 dim = 1) { + return (0x3cu << 26u) | ((opcode >> 7u) & 0x1u) | + ((nsa_dwords & 0x3u) << 1u) | ((dim & 0x7u) << 3u) | + ((dmask & 0xfu) << 8u) | (glc ? (1u << 13u) : 0u) | + ((opcode & 0x7fu) << 18u); +} + +constexpr u32 EncodeMimg1(u32 vdata, u32 vaddr, u32 srsrc = 0, u32 ssamp = 0, + bool a16 = false) { + return ((ssamp & 0x1fu) << 21u) | ((srsrc & 0x1fu) << 16u) | + ((vdata & 0xffu) << 8u) | (vaddr & 0xffu) | (a16 ? (1u << 30u) : 0u); +} + +constexpr u32 EncodeVintrp(u32 opcode, u32 dst, u32 attr, u32 chan, u32 src) { + return (0x32u << 26u) | ((dst & 0xffu) << 18u) | ((opcode & 0x3u) << 16u) | + ((attr & 0x3fu) << 10u) | ((chan & 0x3u) << 8u) | (src & 0xffu); +} + +constexpr u32 EncodeExp0(u32 target, u32 en, bool done = true, + bool compr = false, bool vm = false) { + return (0x3eu << 26u) | ((target & 0x3fu) << 4u) | (en & 0xfu) | + (compr ? (1u << 10u) : 0u) | (done ? (1u << 11u) : 0u) | + (vm ? (1u << 12u) : 0u); +} + +constexpr u32 EncodeExp1(u32 src0, u32 src1, u32 src2, u32 src3) { + return (src0 & 0xffu) | ((src1 & 0xffu) << 8u) | ((src2 & 0xffu) << 16u) | + ((src3 & 0xffu) << 24u); +} + +constexpr u32 EncodeFlat0(u32 opcode, u32 segment, u32 offset = 0) { + return (0x37u << 26u) | ((opcode & 0x7fu) << 18u) | + ((segment & 0x3u) << 14u) | (offset & 0xfffu); +} + +constexpr u32 EncodeFlat1(u32 vdst, u32 saddr, u32 data, u32 addr) { + return ((vdst & 0xffu) << 24u) | ((saddr & 0x7fu) << 16u) | + ((data & 0xffu) << 8u) | (addr & 0xffu); +} + +constexpr u32 EncodeDs0(u32 opcode, u32 offset = 0, bool gds = false) { + return (0x36u << 26u) | ((opcode & 0xffu) << 18u) | (gds ? (1u << 17u) : 0u) | + (offset & 0xffffu); +} + +constexpr u32 EncodeDs1Ex(u32 vdst, u32 data1, u32 data0, u32 addr) { + return ((vdst & 0xffu) << 24u) | ((data1 & 0xffu) << 16u) | + ((data0 & 0xffu) << 8u) | (addr & 0xffu); +} + +constexpr u32 EncodeDs1(u32 vdst, u32 data0, u32 addr) { + return EncodeDs1Ex(vdst, 0, data0, addr); +} + +void AppendSMovLiteral(std::vector *code, u32 dst, u32 literal) { + code->push_back(EncodeSMovB32(dst, 255u)); + code->push_back(literal); +} + +void AppendVMovLiteral(std::vector *code, u32 dst, u32 literal) { + code->push_back(EncodeVop1(0x01, dst, 255u)); + code->push_back(literal); +} + +void AppendVMovU32(std::vector *code, u32 dst, u32 value) { + if (value <= 64u) { + code->push_back(EncodeVop1(0x01, dst, InlineU32(value))); + return; + } + AppendVMovLiteral(code, dst, value); +} + +void AppendVop3(std::vector *code, u32 opcode, u32 dst, u32 src0, u32 src1, + u32 src2 = 0, u32 abs = 0, u32 op_sel = 0, bool clamp = false, + u32 omod = 0, u32 neg = 0) { + code->push_back(EncodeVop3Word0(opcode, dst, abs, op_sel, clamp)); + code->push_back(EncodeVop3Word1(src0, src1, src2, omod, neg)); +} + +void AppendVop3B(std::vector *code, u32 opcode, u32 vdst, u32 sdst, + u32 src0, u32 src1, u32 src2 = 0) { + code->push_back(EncodeVop3BWord0(opcode, vdst, sdst)); + code->push_back(EncodeVop3Word1(src0, src1, src2)); +} + +void AppendVop3p(std::vector *code, u32 opcode, u32 dst, u32 src0, + u32 src1, u32 src2 = 0, u32 op_sel_hi = 0, u32 op_sel = 0, + u32 neg_hi = 0, u32 neg = 0) { + code->push_back(EncodeVop3pWord0(opcode, dst, op_sel_hi, op_sel, neg_hi)); + code->push_back(EncodeVop3pWord1(src0, src1, src2, op_sel_hi, neg)); +} + +void AppendBufferLoadDword(std::vector *code, u32 dst_vgpr, + u32 address_vgpr) { + code->push_back(EncodeMubuf0(0x0cu)); + code->push_back(EncodeMubuf1(dst_vgpr, 0, address_vgpr)); +} + +void AppendBufferLoadOpcode(std::vector *code, u32 opcode, u32 dst_vgpr, + u32 address_vgpr) { + code->push_back(EncodeMubuf0(opcode)); + code->push_back(EncodeMubuf1(dst_vgpr, 0, address_vgpr)); +} + +void AppendBufferStoreDword(std::vector *code, u32 value_vgpr, + u32 address_vgpr) { + code->push_back(EncodeMubuf0(0x1cu)); + code->push_back(EncodeMubuf1(value_vgpr, 12, address_vgpr)); +} + +void AppendBufferStoreOpcode(std::vector *code, u32 opcode, u32 value_vgpr, + u32 address_vgpr, bool glc = false) { + code->push_back(EncodeMubuf0(opcode, 0, false, true, glc)); + code->push_back(EncodeMubuf1(value_vgpr, 12, address_vgpr)); +} + +void AppendTBufferLoadOpcode(std::vector *code, u32 opcode, u32 dst_vgpr, + u32 address_vgpr) { + code->push_back(EncodeMtbuf0(opcode, 14, 7)); + code->push_back(EncodeMtbuf1(opcode, dst_vgpr, 0, address_vgpr)); +} + +constexpr u32 BufferFormat(Prospero::BufferFormat format) { + return Prospero::GpuEnumValue(format); +} + +void AppendTBufferLoadFormatOpcode(std::vector *code, u32 opcode, + u32 dst_vgpr, u32 address_vgpr, + Prospero::BufferFormat format) { + const auto value = BufferFormat(format); + code->push_back(EncodeMtbuf0(opcode, value & 0xfu, (value >> 4u) & 0x7u)); + code->push_back(EncodeMtbuf1(opcode, dst_vgpr, 0, address_vgpr)); +} + +void AppendTBufferStoreOpcode(std::vector *code, u32 opcode, + u32 value_vgpr, u32 address_vgpr) { + code->push_back(EncodeMtbuf0(opcode, 14, 7)); + code->push_back(EncodeMtbuf1(opcode, value_vgpr, 0, address_vgpr)); +} + +void AppendSmemLoadOpcode(std::vector *code, u32 opcode, u32 dst_sgpr, + u32 byte_offset) { + code->push_back(EncodeSmem0(opcode, dst_sgpr)); + code->push_back(EncodeSmem1(byte_offset)); +} + +void AppendStoreVgpr(std::vector *code, u32 value_vgpr, u32 dword_index) { + AppendVMovU32(code, 31, dword_index * 4u); + AppendBufferStoreDword(code, value_vgpr, 31); +} + +void AppendStoreVgprAtLaneDwordOffset(std::vector *code, u32 value_vgpr, + u32 lane_vgpr, u32 dword_offset) { + if (dword_offset == 0u) { + code->push_back(EncodeVop2(0x1a, 31, InlineU32(2), lane_vgpr)); + } else { + AppendVMovU32(code, 31, dword_offset); + code->push_back(EncodeVop2(0x25, 31, Vgpr(lane_vgpr), 31)); + code->push_back(EncodeVop2(0x1a, 31, InlineU32(2), 31)); + } + AppendBufferStoreDword(code, value_vgpr, 31); +} + +void AppendStoreSgpr(std::vector *code, u32 value_sgpr, u32 dword_index) { + code->push_back(EncodeVop1(0x01, 30, value_sgpr)); + AppendStoreVgpr(code, 30, dword_index); +} + +void AppendStoreSgprAtLaneDwordOffset(std::vector *code, u32 value_sgpr, + u32 lane_vgpr, u32 dword_offset) { + code->push_back(EncodeVop1(0x01, 30, value_sgpr)); + AppendStoreVgprAtLaneDwordOffset(code, 30, lane_vgpr, dword_offset); +} + +void AppendStoreSgprPair(std::vector *code, u32 value_sgpr, + u32 first_dword_index) { + AppendStoreSgpr(code, value_sgpr, first_dword_index); + AppendStoreSgpr(code, value_sgpr + 1u, first_dword_index + 1u); +} + +void AppendEnd(std::vector *code) { code->push_back(0xbf810000u); } + +std::string Hex(u32 value) { + char buffer[32] = {}; + std::snprintf(buffer, sizeof(buffer), "0x%08" PRIx32, value); + return buffer; +} + +std::string VkResultName(VkResult result) { + switch (result) { + case VK_SUCCESS: + return "VK_SUCCESS"; + case VK_NOT_READY: + return "VK_NOT_READY"; + case VK_TIMEOUT: + return "VK_TIMEOUT"; + case VK_ERROR_OUT_OF_HOST_MEMORY: + return "VK_ERROR_OUT_OF_HOST_MEMORY"; + case VK_ERROR_OUT_OF_DEVICE_MEMORY: + return "VK_ERROR_OUT_OF_DEVICE_MEMORY"; + case VK_ERROR_INITIALIZATION_FAILED: + return "VK_ERROR_INITIALIZATION_FAILED"; + case VK_ERROR_DEVICE_LOST: + return "VK_ERROR_DEVICE_LOST"; + case VK_ERROR_MEMORY_MAP_FAILED: + return "VK_ERROR_MEMORY_MAP_FAILED"; + case VK_ERROR_LAYER_NOT_PRESENT: + return "VK_ERROR_LAYER_NOT_PRESENT"; + case VK_ERROR_EXTENSION_NOT_PRESENT: + return "VK_ERROR_EXTENSION_NOT_PRESENT"; + case VK_ERROR_FEATURE_NOT_PRESENT: + return "VK_ERROR_FEATURE_NOT_PRESENT"; + case VK_ERROR_INCOMPATIBLE_DRIVER: + return "VK_ERROR_INCOMPATIBLE_DRIVER"; + case VK_ERROR_TOO_MANY_OBJECTS: + return "VK_ERROR_TOO_MANY_OBJECTS"; + case VK_ERROR_FORMAT_NOT_SUPPORTED: + return "VK_ERROR_FORMAT_NOT_SUPPORTED"; + default: + return "VkResult(" + std::to_string(static_cast(result)) + ")"; + } +} + +[[noreturn]] void Fail(const char *shader_name, const char *stage, + const std::string &message) { + std::fprintf(stderr, "ShaderRecompilerComputeTests: %s failed at %s: %s\n", + shader_name, stage, message.c_str()); + std::abort(); +} + +void Require(const char *shader_name, const char *stage, bool value, + const std::string &message); + +void EnsureConfigInitialized() { + static bool config_initialized = false; + if (!config_initialized) { + Common::ThreadsSubsystem::Instance()->Init(nullptr); + Config::ConfigSubsystem::Instance()->Init(nullptr); + Config::ConfigOptions options; + options.printf_direction = Config::OutputDirection::Silent; + Config::Load(options); + Log::LogSubsystem::Instance()->Init(nullptr); + config_initialized = true; + } +} + +void RequireVk(const char *shader_name, const char *stage, VkResult result, + const char *action) { + if (result != VK_SUCCESS) { + Fail(shader_name, stage, + std::string(action) + " returned " + VkResultName(result)); + } +} + +void Require(const char *shader_name, const char *stage, bool value, + const std::string &message) { + if (!value) { + Fail(shader_name, stage, message); + } +} + +struct TestCase { + const char *name = ""; + std::vector code; + std::vector initial; + std::vector expected; + std::vector opcodes; + u32 image_width = 4; + u32 image_height = 4; + std::vector sampled_image_rgba; + std::vector> sampled_image_rgba_mips; + VkFormat sampled_image_format = VK_FORMAT_R32G32B32A32_SFLOAT; + u32 sampled_image_dwords_per_pixel = 4; + std::vector storage_image_rgba; + std::vector expected_storage_image_rgba; + VkFormat storage_image_format = VK_FORMAT_R32G32B32A32_SFLOAT; + u32 storage_image_dwords_per_pixel = 4; + std::vector storage_image_r32ui; + std::vector expected_storage_image_r32ui; + std::vector required_spirv; + std::vector forbidden_spirv; + ShaderComputeInputInfo compute_info{}; + bool has_compute_info = false; + u32 dispatch_x = 1; + u32 dispatch_y = 1; + u32 dispatch_z = 1; + std::array user_data{}; + bool has_user_data = false; + u32 image_descriptor_swizzle = DstSel(4, 5, 6, 7); + bool compile_only = false; + size_t storage_buffer_range_dwords = 0; + std::optional flat_memory_base; + std::vector gds_initial; + std::vector expected_gds; +}; + +struct SkippedCase { + const char *name = ""; + const char *reason = ""; +}; + +struct GraphicsCase { + const char *name = ""; + std::vector fragment_code; + std::vector expected_pixel; + std::vector opcodes; + std::array user_data{}; + bool has_user_data = false; + std::vector push_constants; + std::vector pixel_interpolator_settings; + bool pixel_no_perspective = false; + std::vector vertices; +}; + +struct CompiledShader { + std::vector spirv; + ShaderRecompiler::IR::Program program; + std::vector flattened_srt; + std::vector packed_user_data; +}; + +std::array MakeNativeUserData(const std::array *source) { + std::array data{}; + data[50] = 1u << 20u; + if (source != nullptr) { + data = *source; + } + return data; +} + +bool ReadTestMemory(void *userdata, uint64_t address, u32 *value) { + const auto *data = static_cast *>(userdata); + if (data == nullptr || value == nullptr || address % 4u != 0 || + address / 4u >= data->size()) { + return false; + } + *value = (*data)[address / 4u]; + return true; +} + +void ValidateSpirv(const char *shader_name, const std::vector &spirv) { + spvtools::SpirvTools tools(SPV_ENV_VULKAN_1_2); + std::string messages; + tools.SetMessageConsumer([&messages](spv_message_level_t, const char *, + const spv_position_t &position, + const char *message) { + char buffer[1024] = {}; + std::snprintf(buffer, sizeof(buffer), "%zu:%zu: %s\n", position.line, + position.column, message); + messages += buffer; + }); + + if (!tools.Validate(spirv)) { + Fail(shader_name, "SPIR-V validation", messages); + } +} + +void CheckSpirvText(const TestCase &test, const std::vector &spirv) { + if (test.required_spirv.empty() && test.forbidden_spirv.empty()) { + return; + } + + spvtools::SpirvTools tools(SPV_ENV_VULKAN_1_2); + std::string text; + if (!tools.Disassemble(spirv, &text)) { + Fail(test.name, "SPIR-V disassembly", + "failed to disassemble emitted SPIR-V"); + } + for (const auto &required : test.required_spirv) { + if (text.find(required) == std::string::npos) { + Fail(test.name, "SPIR-V disassembly", + std::string("missing required text: ") + required); + } + } + for (const auto &forbidden : test.forbidden_spirv) { + if (text.find(forbidden) != std::string::npos) { + Fail(test.name, "SPIR-V disassembly", + std::string("found forbidden text: ") + forbidden); + } + } +} + +CompiledShader CompileCase(const TestCase &test) { + auto user_data = + MakeNativeUserData(test.has_user_data ? &test.user_data : nullptr); + const auto uses_image = + std::any_of(test.opcodes.begin(), test.opcodes.end(), [](auto op) { + return op >= ShaderOpcode::ImageGetResinfo && + op <= ShaderOpcode::ImageGather4H; + }); + if (uses_image && ((user_data[3] >> 28u) & 0xfu) == 0) { + user_data[3] = Prospero::GpuEnumValue(Prospero::ImageType::kColor2D) << 28u; + } + if (uses_image) { + user_data[3] = (user_data[3] & ~0xfffu) | test.image_descriptor_swizzle; + } + if (!test.has_user_data) { + user_data[2] = static_cast(test.initial.size() * sizeof(u32)); + } + ShaderRecompiler::CompileOptions options; + options.stage = ShaderType::Compute; + options.dump_ir = true; + options.compute_input_info = + test.has_compute_info ? &test.compute_info : nullptr; + options.user_data = user_data.data(); + options.read_memory = ReadTestMemory; + options.read_memory_data = const_cast *>(&test.initial); + options.flat_memory_base = test.flat_memory_base; + if (test.has_compute_info) { + options.wave_size = test.compute_info.wave_size; + } + + ShaderRecompiler::CompileResult result; + std::string error; + if (!ShaderRecompiler::TryRecompile(test.code, options, &result, + &error)) { + Fail(test.name, "decode/IR", error.c_str()); + } + Require(test.name, "SPIR-V emit", !result.spirv.empty(), + "recompiler returned empty SPIR-V"); + ValidateSpirv(test.name, result.spirv); + CheckSpirvText(test, result.spirv); + std::vector packed_user_data; + for (const auto reg : result.program.bindings.user_data_registers) { + packed_user_data.push_back( + result.resources.user_data[reg - result.program.user_data_base]); + } + return {std::move(result.spirv), std::move(result.program), + std::move(result.resources.flattened_srt), + std::move(packed_user_data)}; +} + +std::array MakeStructuredStorageBufferData(u32 stride_bytes, + u32 num_records, + bool add_tid = false, + u32 format = 0) { + std::array data{}; + data[1] = (stride_bytes & 0x3fffu) << 16u; + data[2] = num_records; + data[3] = 1u << 24u; + if (add_tid) { + data[3] |= 1u << 23u; + } + data[3] |= (format & 0x7fu) << 12u; + return data; +} + +std::array MakeStorageTextureData(Prospero::BufferFormat format) { + std::array data{}; + data[0] = 0x1000u; + data[1] = (Prospero::GpuEnumValue(format) & 0x1ffu) << 20u; + data[3] = Prospero::GpuEnumValue(Prospero::ImageType::kColor2D) << 28u; + return data; +} + +CompiledShader CompileFragmentCase(const GraphicsCase &test) { + const auto user_data = + MakeNativeUserData(test.has_user_data ? &test.user_data : nullptr); + ShaderPixelInputInfo pixel_info{}; + pixel_info.input_num = + test.pixel_interpolator_settings.empty() + ? 1u + : static_cast(test.pixel_interpolator_settings.size()); + pixel_info.ps_no_perspective = test.pixel_no_perspective; + for (u32 i = 0; i < std::size(pixel_info.interpolator_settings); i++) { + pixel_info.interpolator_settings[i] = i; + } + for (u32 i = 0; i < test.pixel_interpolator_settings.size() && + i < std::size(pixel_info.interpolator_settings); + i++) { + pixel_info.interpolator_settings[i] = test.pixel_interpolator_settings[i]; + } + + ShaderRecompiler::CompileOptions options; + options.stage = ShaderType::Pixel; + options.dump_ir = false; + options.pixel_input_info = &pixel_info; + options.user_data = user_data.data(); + + ShaderRecompiler::CompileResult result; + std::string error; + if (!ShaderRecompiler::TryRecompile(test.fragment_code, options, + &result, &error)) { + Fail(test.name, "decode/IR", error.c_str()); + } + Require(test.name, "SPIR-V emit", !result.spirv.empty(), + "recompiler returned empty SPIR-V"); + ValidateSpirv(test.name, result.spirv); + std::vector packed_user_data; + for (const auto reg : result.program.bindings.user_data_registers) { + packed_user_data.push_back( + result.resources.user_data[reg - result.program.user_data_base]); + } + return {std::move(result.spirv), std::move(result.program), + std::move(result.resources.flattened_srt), + std::move(packed_user_data)}; +} + +std::array MakeSampledTextureData(Prospero::BufferFormat format) { + return MakeStorageTextureData(format); +} + +namespace TestSpv { + +enum : u32 { + ExecutionModelVertex = 0, + AddressingModelLogical = 0, + MemoryModelGLSL450 = 1, + CapabilityShader = 1, + StorageClassInput = 1, + StorageClassOutput = 3, + FunctionControlNone = 0, + DecorationBlock = 2, + DecorationBuiltIn = 11, + DecorationLocation = 30, + BuiltInPosition = 0, + OpTypeVoid = 19, + OpTypeInt = 21, + OpTypeFloat = 22, + OpTypeVector = 23, + OpTypeStruct = 30, + OpTypePointer = 32, + OpTypeFunction = 33, + OpConstant = 43, + OpFunction = 54, + OpFunctionEnd = 56, + OpVariable = 59, + OpLoad = 61, + OpStore = 62, + OpAccessChain = 65, + OpDecorate = 71, + OpMemberDecorate = 72, + OpCompositeConstruct = 80, + OpCompositeExtract = 81, + OpLabel = 248, + OpReturn = 253, +}; + +std::vector MakePassthroughVertexSpirv() { + using ShaderRecompiler::Spirv::Builder; + + Builder b; + const auto void_type = b.AllocateId(); + const auto uint_type = b.AllocateId(); + const auto float_type = b.AllocateId(); + const auto vec2_type = b.AllocateId(); + const auto vec4_type = b.AllocateId(); + const auto per_vertex_type = b.AllocateId(); + const auto ptr_input_vec2 = b.AllocateId(); + const auto ptr_input_vec4 = b.AllocateId(); + const auto ptr_output_vec4 = b.AllocateId(); + const auto ptr_output_per_vertex = b.AllocateId(); + const auto func_type = b.AllocateId(); + const auto const_u32_0 = b.AllocateId(); + const auto const_f32_0 = b.AllocateId(); + const auto const_f32_1 = b.AllocateId(); + const auto in_pos = b.AllocateId(); + const auto in_color = b.AllocateId(); + const auto out_color = b.AllocateId(); + const auto per_vertex = b.AllocateId(); + const auto main = b.AllocateId(); + const auto label = b.AllocateId(); + const auto pos2 = b.AllocateId(); + const auto color4 = b.AllocateId(); + const auto pos_x = b.AllocateId(); + const auto pos_y = b.AllocateId(); + const auto position = b.AllocateId(); + const auto position_ptr = b.AllocateId(); + + b.AddCapability({CapabilityShader}); + b.AddMemoryModel({AddressingModelLogical, MemoryModelGLSL450}); + b.AddEntryPoint(ExecutionModelVertex, main, "main", + {in_pos, in_color, per_vertex, out_color}); + b.AddAnnotation({OpDecorate, in_pos, DecorationLocation, 0}); + b.AddAnnotation({OpDecorate, in_color, DecorationLocation, 1}); + b.AddAnnotation({OpDecorate, out_color, DecorationLocation, 0}); + b.AddAnnotation({OpDecorate, per_vertex_type, DecorationBlock}); + b.AddAnnotation({OpMemberDecorate, per_vertex_type, 0, DecorationBuiltIn, + BuiltInPosition}); + + b.AddType({OpTypeVoid, void_type}); + b.AddType({OpTypeInt, uint_type, 32, 0}); + b.AddType({OpTypeFloat, float_type, 32}); + b.AddType({OpTypeVector, vec2_type, float_type, 2}); + b.AddType({OpTypeVector, vec4_type, float_type, 4}); + b.AddType({OpTypeStruct, per_vertex_type, vec4_type}); + b.AddType({OpTypePointer, ptr_input_vec2, StorageClassInput, vec2_type}); + b.AddType({OpTypePointer, ptr_input_vec4, StorageClassInput, vec4_type}); + b.AddType({OpTypePointer, ptr_output_vec4, StorageClassOutput, vec4_type}); + b.AddType({OpTypePointer, ptr_output_per_vertex, StorageClassOutput, + per_vertex_type}); + b.AddType({OpTypeFunction, func_type, void_type}); + b.AddType({OpConstant, uint_type, const_u32_0, 0}); + b.AddType({OpConstant, float_type, const_f32_0, 0x00000000u}); + b.AddType({OpConstant, float_type, const_f32_1, 0x3f800000u}); + b.AddType({OpVariable, ptr_input_vec2, in_pos, StorageClassInput}); + b.AddType({OpVariable, ptr_input_vec4, in_color, StorageClassInput}); + b.AddType({OpVariable, ptr_output_vec4, out_color, StorageClassOutput}); + b.AddType( + {OpVariable, ptr_output_per_vertex, per_vertex, StorageClassOutput}); + + b.AddFunction({OpFunction, void_type, main, FunctionControlNone, func_type}); + b.AddFunction({OpLabel, label}); + b.AddFunction({OpLoad, vec2_type, pos2, in_pos}); + b.AddFunction({OpLoad, vec4_type, color4, in_color}); + b.AddFunction({OpCompositeExtract, float_type, pos_x, pos2, 0}); + b.AddFunction({OpCompositeExtract, float_type, pos_y, pos2, 1}); + b.AddFunction({OpCompositeConstruct, vec4_type, position, pos_x, pos_y, + const_f32_0, const_f32_1}); + b.AddFunction( + {OpAccessChain, ptr_output_vec4, position_ptr, per_vertex, const_u32_0}); + b.AddFunction({OpStore, position_ptr, position}); + b.AddFunction({OpStore, out_color, color4}); + b.AddFunction({OpReturn}); + b.AddFunction({OpFunctionEnd}); + return b.Build(); +} + +} // namespace TestSpv + +class VulkanHarness { +public: + VulkanHarness() { Init(); } + ~VulkanHarness() { Destroy(); } + + VulkanHarness(const VulkanHarness &) = delete; + VulkanHarness &operator=(const VulkanHarness &) = delete; + + struct Buffer { + VkBuffer buffer = VK_NULL_HANDLE; + VkDeviceMemory memory = VK_NULL_HANDLE; + VkDeviceSize size = 0; + bool coherent = false; + }; + + struct Image { + VkImage image = VK_NULL_HANDLE; + VkDeviceMemory memory = VK_NULL_HANDLE; + VkImageView view = VK_NULL_HANDLE; + VkFormat format = VK_FORMAT_UNDEFINED; + VkImageLayout layout = VK_IMAGE_LAYOUT_UNDEFINED; + u32 width = 0; + u32 height = 0; + u32 mip_levels = 1; + u32 dwords_per_pixel = 0; + }; + + [[nodiscard]] VkDevice Device() const { return m_device; } + + void CheckMutableStorageSrgbView() { + constexpr const char *name = "StorageTextureMutableSrgbView"; + VkImageCreateInfo image_info{}; + image_info.sType = VK_STRUCTURE_TYPE_IMAGE_CREATE_INFO; + image_info.flags = VK_IMAGE_CREATE_MUTABLE_FORMAT_BIT; + image_info.imageType = VK_IMAGE_TYPE_2D; + image_info.format = VK_FORMAT_R8G8B8A8_SRGB; + image_info.extent = {16, 16, 1}; + image_info.mipLevels = 1; + image_info.arrayLayers = 1; + image_info.samples = VK_SAMPLE_COUNT_1_BIT; + image_info.tiling = VK_IMAGE_TILING_OPTIMAL; + image_info.usage = VK_IMAGE_USAGE_TRANSFER_SRC_BIT | + VK_IMAGE_USAGE_TRANSFER_DST_BIT | + VK_IMAGE_USAGE_SAMPLED_BIT | + VK_IMAGE_USAGE_STORAGE_BIT; + image_info.sharingMode = VK_SHARING_MODE_EXCLUSIVE; + image_info.initialLayout = VK_IMAGE_LAYOUT_UNDEFINED; + + VkImageFormatProperties exact_properties{}; + const bool srgb_storage_supported = + vkGetPhysicalDeviceImageFormatProperties( + m_physical_device, image_info.format, image_info.imageType, + image_info.tiling, image_info.usage, image_info.flags, + &exact_properties) == VK_SUCCESS; + GraphicContext context{}; + context.physical_device = m_physical_device; + Require(name, "format fallback", TextureCheckFormat(&context, &image_info), + "mutable sampled/storage RGBA8 image is unsupported"); + Require(name, "format fallback", + image_info.format == VK_FORMAT_R8G8B8A8_UNORM || + (srgb_storage_supported && + image_info.format == VK_FORMAT_R8G8B8A8_SRGB), + "unsupported sRGB storage format did not fall back to UNORM"); + + VkImage image = VK_NULL_HANDLE; + RequireVk(name, "backing", vkCreateImage(m_device, &image_info, nullptr, &image), + "vkCreateImage"); + VkMemoryRequirements requirements{}; + vkGetImageMemoryRequirements(m_device, image, &requirements); + u32 memory_type = 0; + Require(name, "backing", + FindMemoryType(requirements.memoryTypeBits, + VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT, &memory_type) || + FindMemoryType(requirements.memoryTypeBits, 0, &memory_type), + "no memory type for mutable storage image"); + VkMemoryAllocateInfo allocation{}; + allocation.sType = VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_INFO; + allocation.allocationSize = requirements.size; + allocation.memoryTypeIndex = memory_type; + VkDeviceMemory memory = VK_NULL_HANDLE; + RequireVk(name, "backing", + vkAllocateMemory(m_device, &allocation, nullptr, &memory), + "vkAllocateMemory"); + RequireVk(name, "backing", vkBindImageMemory(m_device, image, memory, 0), + "vkBindImageMemory"); + + VkImageViewUsageCreateInfo view_usage{}; + view_usage.sType = VK_STRUCTURE_TYPE_IMAGE_VIEW_USAGE_CREATE_INFO; + view_usage.usage = VK_IMAGE_USAGE_SAMPLED_BIT | VK_IMAGE_USAGE_STORAGE_BIT; + VkImageViewCreateInfo view_info{}; + view_info.sType = VK_STRUCTURE_TYPE_IMAGE_VIEW_CREATE_INFO; + view_info.pNext = &view_usage; + view_info.image = image; + view_info.viewType = VK_IMAGE_VIEW_TYPE_2D; + view_info.format = image_info.format; + view_info.subresourceRange.aspectMask = VK_IMAGE_ASPECT_COLOR_BIT; + view_info.subresourceRange.levelCount = 1; + view_info.subresourceRange.layerCount = 1; + VkImageView default_view = VK_NULL_HANDLE; + RequireVk(name, "default view", + vkCreateImageView(m_device, &view_info, nullptr, &default_view), + "vkCreateImageView(default sampled/storage)"); + + view_usage.usage = VK_IMAGE_USAGE_SAMPLED_BIT; + view_info.format = VK_FORMAT_R8G8B8A8_SRGB; + VkImageView srgb_view = VK_NULL_HANDLE; + RequireVk(name, "sRGB view", + vkCreateImageView(m_device, &view_info, nullptr, &srgb_view), + "vkCreateImageView(sRGB sampled)"); + + vkDestroyImageView(m_device, srgb_view, nullptr); + vkDestroyImageView(m_device, default_view, nullptr); + vkDestroyImage(m_device, image, nullptr); + vkFreeMemory(m_device, memory, nullptr); + std::printf("[host] %-32s ok (backing=%d)\n", name, + static_cast(image_info.format)); + } + + void CheckMutableRenderTargetBgraStorageView() { + constexpr const char *name = "RenderTargetMutableBgraStorage"; + VkImageCreateInfo image_info{}; + image_info.sType = VK_STRUCTURE_TYPE_IMAGE_CREATE_INFO; + image_info.flags = + VK_IMAGE_CREATE_MUTABLE_FORMAT_BIT | VK_IMAGE_CREATE_EXTENDED_USAGE_BIT; + image_info.imageType = VK_IMAGE_TYPE_2D; + image_info.format = VK_FORMAT_B8G8R8A8_SRGB; + image_info.extent = {8, 8, 1}; + image_info.mipLevels = 1; + image_info.arrayLayers = 1; + image_info.samples = VK_SAMPLE_COUNT_1_BIT; + image_info.tiling = VK_IMAGE_TILING_OPTIMAL; + image_info.usage = VK_IMAGE_USAGE_COLOR_ATTACHMENT_BIT | + VK_IMAGE_USAGE_TRANSFER_SRC_BIT | + VK_IMAGE_USAGE_TRANSFER_DST_BIT | + VK_IMAGE_USAGE_SAMPLED_BIT | VK_IMAGE_USAGE_STORAGE_BIT; + image_info.sharingMode = VK_SHARING_MODE_EXCLUSIVE; + image_info.initialLayout = VK_IMAGE_LAYOUT_UNDEFINED; + + VkImageFormatProperties properties{}; + RequireVk(name, "backing query", + vkGetPhysicalDeviceImageFormatProperties( + m_physical_device, image_info.format, image_info.imageType, + image_info.tiling, image_info.usage, image_info.flags, + &properties), + "mutable extended-usage BGRA sRGB backing"); + + VkImage image = VK_NULL_HANDLE; + RequireVk(name, "backing", + vkCreateImage(m_device, &image_info, nullptr, &image), + "vkCreateImage"); + VkMemoryRequirements requirements{}; + vkGetImageMemoryRequirements(m_device, image, &requirements); + u32 memory_type = 0; + Require(name, "backing", + FindMemoryType(requirements.memoryTypeBits, + VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT, &memory_type) || + FindMemoryType(requirements.memoryTypeBits, 0, &memory_type), + "no memory type for mutable render target"); + VkMemoryAllocateInfo allocation{}; + allocation.sType = VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_INFO; + allocation.allocationSize = requirements.size; + allocation.memoryTypeIndex = memory_type; + VkDeviceMemory memory = VK_NULL_HANDLE; + RequireVk(name, "backing", + vkAllocateMemory(m_device, &allocation, nullptr, &memory), + "vkAllocateMemory"); + RequireVk(name, "backing", vkBindImageMemory(m_device, image, memory, 0), + "vkBindImageMemory"); + + VkImageViewUsageCreateInfo view_usage{}; + view_usage.sType = VK_STRUCTURE_TYPE_IMAGE_VIEW_USAGE_CREATE_INFO; + view_usage.usage = VK_IMAGE_USAGE_STORAGE_BIT; + VkImageViewCreateInfo view_info{}; + view_info.sType = VK_STRUCTURE_TYPE_IMAGE_VIEW_CREATE_INFO; + view_info.pNext = &view_usage; + view_info.image = image; + view_info.viewType = VK_IMAGE_VIEW_TYPE_2D; + view_info.format = VK_FORMAT_R8G8B8A8_UNORM; + view_info.components = { + VK_COMPONENT_SWIZZLE_IDENTITY, VK_COMPONENT_SWIZZLE_IDENTITY, + VK_COMPONENT_SWIZZLE_IDENTITY, VK_COMPONENT_SWIZZLE_IDENTITY}; + view_info.subresourceRange.aspectMask = VK_IMAGE_ASPECT_COLOR_BIT; + view_info.subresourceRange.levelCount = 1; + view_info.subresourceRange.layerCount = 1; + VkImageView storage_view = VK_NULL_HANDLE; + RequireVk(name, "storage view", + vkCreateImageView(m_device, &view_info, nullptr, &storage_view), + "vkCreateImageView(RGBA UNORM storage)"); + + vkDestroyImageView(m_device, storage_view, nullptr); + vkDestroyImage(m_device, image, nullptr); + vkFreeMemory(m_device, memory, nullptr); + std::printf("[host] %-32s ok (backing=%d view=%d)\n", name, + static_cast(image_info.format), + static_cast(view_info.format)); + } + + Buffer CreateStorageBuffer(const char *shader_name, + const std::vector &initial, + size_t dword_count) { + Buffer ret; + ret.size = static_cast(std::max(dword_count, 1u) * + sizeof(u32)); + + VkBufferCreateInfo buffer_info{}; + buffer_info.sType = VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO; + buffer_info.size = ret.size; + buffer_info.usage = VK_BUFFER_USAGE_STORAGE_BUFFER_BIT; + buffer_info.sharingMode = VK_SHARING_MODE_EXCLUSIVE; + RequireVk(shader_name, "dispatch", + vkCreateBuffer(m_device, &buffer_info, nullptr, &ret.buffer), + "vkCreateBuffer"); + + VkMemoryRequirements req{}; + vkGetBufferMemoryRequirements(m_device, ret.buffer, &req); + u32 memory_type = 0; + ret.coherent = FindMemoryType(req.memoryTypeBits, + VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT | + VK_MEMORY_PROPERTY_HOST_COHERENT_BIT, + &memory_type); + if (!ret.coherent) { + Require(shader_name, "dispatch", + FindMemoryType(req.memoryTypeBits, + VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT, &memory_type), + "no host-visible memory type for storage buffer"); + } + + VkMemoryAllocateInfo alloc{}; + alloc.sType = VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_INFO; + alloc.allocationSize = req.size; + alloc.memoryTypeIndex = memory_type; + RequireVk(shader_name, "dispatch", + vkAllocateMemory(m_device, &alloc, nullptr, &ret.memory), + "vkAllocateMemory"); + RequireVk(shader_name, "dispatch", + vkBindBufferMemory(m_device, ret.buffer, ret.memory, 0), + "vkBindBufferMemory"); + + std::vector contents(dword_count, 0); + for (size_t i = 0; i < initial.size() && i < contents.size(); i++) { + contents[i] = initial[i]; + } + WriteBuffer(shader_name, ret, contents); + return ret; + } + + void DestroyBuffer(Buffer *buffer) { + if (buffer == nullptr) { + return; + } + if (buffer->buffer != VK_NULL_HANDLE) { + vkDestroyBuffer(m_device, buffer->buffer, nullptr); + buffer->buffer = VK_NULL_HANDLE; + } + if (buffer->memory != VK_NULL_HANDLE) { + vkFreeMemory(m_device, buffer->memory, nullptr); + buffer->memory = VK_NULL_HANDLE; + } + } + + Image CreateImage2D(const char *shader_name, u32 width, u32 height, + VkFormat format, VkImageUsageFlags usage, + const std::vector &initial, u32 dwords_per_pixel, + VkImageLayout final_layout) { + std::vector> mips; + if (!initial.empty()) { + mips.push_back(initial); + } + return CreateImage2DMips(shader_name, width, height, format, usage, mips, + dwords_per_pixel, final_layout); + } + + static u32 MipExtent(u32 value, u32 level) { + for (u32 i = 0; i < level && value > 1u; i++) { + value >>= 1u; + } + return std::max(value, 1u); + } + + static size_t ImageMipDwordCount(u32 width, u32 height, u32 dwords_per_pixel, + u32 level) { + return static_cast(MipExtent(width, level)) * + static_cast(MipExtent(height, level)) * dwords_per_pixel; + } + + Image CreateImage2DMips(const char *shader_name, u32 width, u32 height, + VkFormat format, VkImageUsageFlags usage, + const std::vector> &initial_mips, + u32 dwords_per_pixel, VkImageLayout final_layout) { + Image ret; + ret.format = format; + ret.width = width; + ret.height = height; + ret.mip_levels = std::max(static_cast(initial_mips.size()), 1u); + ret.dwords_per_pixel = dwords_per_pixel; + + VkImageCreateInfo image_info{}; + image_info.sType = VK_STRUCTURE_TYPE_IMAGE_CREATE_INFO; + image_info.imageType = VK_IMAGE_TYPE_2D; + image_info.format = format; + image_info.extent.width = width; + image_info.extent.height = height; + image_info.extent.depth = 1; + image_info.mipLevels = ret.mip_levels; + image_info.arrayLayers = 1; + image_info.samples = VK_SAMPLE_COUNT_1_BIT; + image_info.tiling = VK_IMAGE_TILING_OPTIMAL; + image_info.usage = usage | VK_IMAGE_USAGE_TRANSFER_DST_BIT | + VK_IMAGE_USAGE_TRANSFER_SRC_BIT; + image_info.sharingMode = VK_SHARING_MODE_EXCLUSIVE; + image_info.initialLayout = VK_IMAGE_LAYOUT_UNDEFINED; + RequireVk(shader_name, "dispatch", + vkCreateImage(m_device, &image_info, nullptr, &ret.image), + "vkCreateImage"); + + VkMemoryRequirements req{}; + vkGetImageMemoryRequirements(m_device, ret.image, &req); + u32 memory_type = 0; + if (!FindMemoryType(req.memoryTypeBits, VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT, + &memory_type)) { + Require(shader_name, "dispatch", + FindMemoryType(req.memoryTypeBits, 0, &memory_type), + "no memory type for image"); + } + + VkMemoryAllocateInfo alloc{}; + alloc.sType = VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_INFO; + alloc.allocationSize = req.size; + alloc.memoryTypeIndex = memory_type; + RequireVk(shader_name, "dispatch", + vkAllocateMemory(m_device, &alloc, nullptr, &ret.memory), + "vkAllocateMemory(image)"); + RequireVk(shader_name, "dispatch", + vkBindImageMemory(m_device, ret.image, ret.memory, 0), + "vkBindImageMemory"); + + VkImageViewCreateInfo view_info{}; + view_info.sType = VK_STRUCTURE_TYPE_IMAGE_VIEW_CREATE_INFO; + view_info.image = ret.image; + view_info.viewType = VK_IMAGE_VIEW_TYPE_2D; + view_info.format = format; + view_info.subresourceRange.aspectMask = VK_IMAGE_ASPECT_COLOR_BIT; + view_info.subresourceRange.baseMipLevel = 0; + view_info.subresourceRange.levelCount = ret.mip_levels; + view_info.subresourceRange.baseArrayLayer = 0; + view_info.subresourceRange.layerCount = 1; + RequireVk(shader_name, "dispatch", + vkCreateImageView(m_device, &view_info, nullptr, &ret.view), + "vkCreateImageView"); + + if (!initial_mips.empty()) { + size_t total_dwords = 0; + for (u32 level = 0; level < ret.mip_levels; level++) { + total_dwords += + ImageMipDwordCount(width, height, dwords_per_pixel, level); + } + std::vector contents(total_dwords, 0); + size_t offset = 0; + for (u32 level = 0; level < ret.mip_levels; level++) { + const auto level_dwords = + ImageMipDwordCount(width, height, dwords_per_pixel, level); + const auto &src = initial_mips[level]; + for (size_t i = 0; i < src.size() && i < level_dwords; i++) { + contents[offset + i] = src[i]; + } + offset += level_dwords; + } + auto staging = + CreateHostBuffer(shader_name, contents.size() * sizeof(u32), + VK_BUFFER_USAGE_TRANSFER_SRC_BIT, contents); + UploadImageMips(shader_name, &ret, staging.buffer, final_layout); + DestroyBuffer(&staging); + } else { + TransitionImage(shader_name, &ret, final_layout, + VK_PIPELINE_STAGE_TOP_OF_PIPE_BIT, + VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT, 0, + AccessForLayout(final_layout)); + } + return ret; + } + + void DestroyImage(Image *image) { + if (image == nullptr) { + return; + } + if (image->view != VK_NULL_HANDLE) { + vkDestroyImageView(m_device, image->view, nullptr); + image->view = VK_NULL_HANDLE; + } + if (image->image != VK_NULL_HANDLE) { + vkDestroyImage(m_device, image->image, nullptr); + image->image = VK_NULL_HANDLE; + } + if (image->memory != VK_NULL_HANDLE) { + vkFreeMemory(m_device, image->memory, nullptr); + image->memory = VK_NULL_HANDLE; + } + } + + std::vector ReadImage(const char *shader_name, Image *image) { + const auto dword_count = static_cast(image->width) * + static_cast(image->height) * + image->dwords_per_pixel; + auto staging = CreateHostBuffer(shader_name, dword_count * sizeof(u32), + VK_BUFFER_USAGE_TRANSFER_DST_BIT, {}); + + VkCommandBuffer cmd = BeginCommands(shader_name, "readback"); + AddImageBarrier( + cmd, image->image, image->layout, VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL, + VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, VK_PIPELINE_STAGE_TRANSFER_BIT, + VK_ACCESS_MEMORY_READ_BIT | VK_ACCESS_MEMORY_WRITE_BIT, + VK_ACCESS_TRANSFER_READ_BIT); + VkBufferImageCopy copy{}; + copy.bufferOffset = 0; + copy.imageSubresource.aspectMask = VK_IMAGE_ASPECT_COLOR_BIT; + copy.imageSubresource.mipLevel = 0; + copy.imageSubresource.baseArrayLayer = 0; + copy.imageSubresource.layerCount = 1; + copy.imageExtent.width = image->width; + copy.imageExtent.height = image->height; + copy.imageExtent.depth = 1; + vkCmdCopyImageToBuffer(cmd, image->image, + VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL, staging.buffer, + 1, ©); + EndSubmitAndFree(shader_name, "readback", cmd); + image->layout = VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL; + + auto ret = ReadBuffer(shader_name, staging, dword_count); + DestroyBuffer(&staging); + return ret; + } + + VkSampler CreateNearestSampler(const char *shader_name) { + VkSamplerCreateInfo sampler_info{}; + sampler_info.sType = VK_STRUCTURE_TYPE_SAMPLER_CREATE_INFO; + sampler_info.magFilter = VK_FILTER_NEAREST; + sampler_info.minFilter = VK_FILTER_NEAREST; + sampler_info.mipmapMode = VK_SAMPLER_MIPMAP_MODE_NEAREST; + sampler_info.addressModeU = VK_SAMPLER_ADDRESS_MODE_CLAMP_TO_EDGE; + sampler_info.addressModeV = VK_SAMPLER_ADDRESS_MODE_CLAMP_TO_EDGE; + sampler_info.addressModeW = VK_SAMPLER_ADDRESS_MODE_CLAMP_TO_EDGE; + sampler_info.minLod = 0.0f; + sampler_info.maxLod = 0.0f; + VkSampler sampler = VK_NULL_HANDLE; + RequireVk(shader_name, "dispatch", + vkCreateSampler(m_device, &sampler_info, nullptr, &sampler), + "vkCreateSampler"); + return sampler; + } + + std::vector ReadBuffer(const char *shader_name, const Buffer &buffer, + size_t dword_count) { + if (!buffer.coherent) { + VkMappedMemoryRange range{}; + range.sType = VK_STRUCTURE_TYPE_MAPPED_MEMORY_RANGE; + range.memory = buffer.memory; + range.offset = 0; + range.size = VK_WHOLE_SIZE; + RequireVk(shader_name, "readback", + vkInvalidateMappedMemoryRanges(m_device, 1, &range), + "vkInvalidateMappedMemoryRanges"); + } + + void *data = nullptr; + RequireVk(shader_name, "readback", + vkMapMemory(m_device, buffer.memory, 0, buffer.size, 0, &data), + "vkMapMemory"); + std::vector ret(dword_count, 0); + std::memcpy(ret.data(), data, dword_count * sizeof(u32)); + vkUnmapMemory(m_device, buffer.memory); + return ret; + } + + void Dispatch(const TestCase &test, const CompiledShader &compiled, + const Buffer &buffer, const Buffer *gds_buffer = nullptr, + const Image *sampled_image = nullptr, + const Image *storage_image = nullptr, + const Image *storage_image_uint = nullptr, + VkSampler sampler = VK_NULL_HANDLE) { + using Kind = ShaderRecompiler::IR::DescriptorBindingKind; + const auto &layout = compiled.program.bindings; + auto Binding = [&](Kind kind) { + return ShaderRecompiler::IR::FindBinding(layout, kind); + }; + auto Count = [&](Kind kind) { + const auto *binding = Binding(kind); + return binding == nullptr ? 0u + : kind == Kind::Gds ? 1u + : static_cast(binding->resources.size()); + }; + Require(test.name, "dispatch", + Count(Kind::Sampled2DArray) == 0 && Count(Kind::Sampled3D) == 0 && + Count(Kind::SampledUint2DArray) == 0 && + Count(Kind::SampledUint3D) == 0 && + Count(Kind::Storage2DArray) == 0 && + Count(Kind::Storage3D) == 0 && + Count(Kind::StorageUint2DArray) == 0 && + Count(Kind::StorageUint3D) == 0, + "array/3D image cases must provide matching Vulkan test views " + "before dispatch"); + + VkShaderModuleCreateInfo module_info{}; + module_info.sType = VK_STRUCTURE_TYPE_SHADER_MODULE_CREATE_INFO; + module_info.codeSize = compiled.spirv.size() * sizeof(u32); + module_info.pCode = compiled.spirv.data(); + VkShaderModule module = VK_NULL_HANDLE; + RequireVk(test.name, "SPIR-V module", + vkCreateShaderModule(m_device, &module_info, nullptr, &module), + "vkCreateShaderModule"); + + std::vector layout_bindings; + auto add_layout_binding = + [&layout_bindings](u32 binding, VkDescriptorType type, u32 count) { + if (count == 0) { + return; + } + VkDescriptorSetLayoutBinding item{}; + item.binding = binding; + item.descriptorType = type; + item.descriptorCount = count; + item.stageFlags = VK_SHADER_STAGE_COMPUTE_BIT; + layout_bindings.push_back(item); + }; + for (const auto &binding : layout.descriptors) { + VkDescriptorType type = VK_DESCRIPTOR_TYPE_STORAGE_BUFFER; + u32 count = static_cast(binding.resources.size()); + switch (binding.kind) { + case Kind::Sampled2D: + case Kind::Sampled2DArray: + case Kind::Sampled3D: + case Kind::SampledUint2D: + case Kind::SampledUint2DArray: + case Kind::SampledUint3D: + type = VK_DESCRIPTOR_TYPE_SAMPLED_IMAGE; + break; + case Kind::Storage2D: + case Kind::Storage2DArray: + case Kind::Storage3D: + case Kind::StorageUint2D: + case Kind::StorageUint2DArray: + case Kind::StorageUint3D: + type = VK_DESCRIPTOR_TYPE_STORAGE_IMAGE; + break; + case Kind::Samplers: + type = VK_DESCRIPTOR_TYPE_SAMPLER; + break; + case Kind::Gds: + case Kind::FlattenedSrt: + case Kind::UserData: + count = 1; + break; + default: + break; + } + add_layout_binding(binding.binding, type, count); + } + + VkDescriptorSetLayoutCreateInfo layout_info{}; + layout_info.sType = VK_STRUCTURE_TYPE_DESCRIPTOR_SET_LAYOUT_CREATE_INFO; + layout_info.bindingCount = static_cast(layout_bindings.size()); + layout_info.pBindings = + layout_bindings.empty() ? nullptr : layout_bindings.data(); + VkDescriptorSetLayout descriptor_layout = VK_NULL_HANDLE; + RequireVk(test.name, "dispatch", + vkCreateDescriptorSetLayout(m_device, &layout_info, nullptr, + &descriptor_layout), + "vkCreateDescriptorSetLayout"); + + VkPipelineLayoutCreateInfo pipeline_layout_info{}; + pipeline_layout_info.sType = VK_STRUCTURE_TYPE_PIPELINE_LAYOUT_CREATE_INFO; + pipeline_layout_info.setLayoutCount = 1; + pipeline_layout_info.pSetLayouts = &descriptor_layout; + VkPushConstantRange push_range{}; + if (layout.push_constant_size != 0) { + push_range.stageFlags = VK_SHADER_STAGE_COMPUTE_BIT; + push_range.offset = layout.push_constant_offset; + push_range.size = layout.push_constant_size; + pipeline_layout_info.pushConstantRangeCount = 1; + pipeline_layout_info.pPushConstantRanges = &push_range; + } + VkPipelineLayout pipeline_layout = VK_NULL_HANDLE; + RequireVk(test.name, "dispatch", + vkCreatePipelineLayout(m_device, &pipeline_layout_info, nullptr, + &pipeline_layout), + "vkCreatePipelineLayout"); + + VkPipelineShaderStageCreateInfo stage{}; + stage.sType = VK_STRUCTURE_TYPE_PIPELINE_SHADER_STAGE_CREATE_INFO; + stage.stage = VK_SHADER_STAGE_COMPUTE_BIT; + stage.module = module; + stage.pName = "main"; + + VkComputePipelineCreateInfo pipeline_info{}; + pipeline_info.sType = VK_STRUCTURE_TYPE_COMPUTE_PIPELINE_CREATE_INFO; + pipeline_info.stage = stage; + pipeline_info.layout = pipeline_layout; + VkPipeline pipeline = VK_NULL_HANDLE; + RequireVk(test.name, "dispatch", + vkCreateComputePipelines(m_device, VK_NULL_HANDLE, 1, + &pipeline_info, nullptr, &pipeline), + "vkCreateComputePipelines"); + + std::vector pool_sizes; + auto add_pool_size = [&pool_sizes](VkDescriptorType type, u32 count) { + if (count == 0) { + return; + } + for (auto &size : pool_sizes) { + if (size.type == type) { + size.descriptorCount += count; + return; + } + } + pool_sizes.push_back({type, count}); + }; + add_pool_size(VK_DESCRIPTOR_TYPE_STORAGE_BUFFER, + Count(Kind::Buffers) + Count(Kind::AddressMemory) + + Count(Kind::Gds) + + (Binding(Kind::FlattenedSrt) != nullptr ? 1u : 0u) + + (Binding(Kind::UserData) != nullptr ? 1u : 0u)); + add_pool_size(VK_DESCRIPTOR_TYPE_SAMPLED_IMAGE, + Count(Kind::Sampled2D) + Count(Kind::Sampled2DArray) + + Count(Kind::Sampled3D) + Count(Kind::SampledUint2D) + + Count(Kind::SampledUint2DArray) + + Count(Kind::SampledUint3D)); + add_pool_size(VK_DESCRIPTOR_TYPE_STORAGE_IMAGE, + Count(Kind::Storage2D) + Count(Kind::Storage2DArray) + + Count(Kind::Storage3D) + Count(Kind::StorageUint2D) + + Count(Kind::StorageUint2DArray) + + Count(Kind::StorageUint3D)); + add_pool_size(VK_DESCRIPTOR_TYPE_SAMPLER, Count(Kind::Samplers)); + VkDescriptorPoolCreateInfo pool_info{}; + pool_info.sType = VK_STRUCTURE_TYPE_DESCRIPTOR_POOL_CREATE_INFO; + pool_info.maxSets = 1; + pool_info.poolSizeCount = static_cast(pool_sizes.size()); + pool_info.pPoolSizes = pool_sizes.empty() ? nullptr : pool_sizes.data(); + VkDescriptorPool descriptor_pool = VK_NULL_HANDLE; + RequireVk( + test.name, "dispatch", + vkCreateDescriptorPool(m_device, &pool_info, nullptr, &descriptor_pool), + "vkCreateDescriptorPool"); + + VkDescriptorSetAllocateInfo set_info{}; + set_info.sType = VK_STRUCTURE_TYPE_DESCRIPTOR_SET_ALLOCATE_INFO; + set_info.descriptorPool = descriptor_pool; + set_info.descriptorSetCount = 1; + set_info.pSetLayouts = &descriptor_layout; + VkDescriptorSet descriptor_set = VK_NULL_HANDLE; + RequireVk(test.name, "dispatch", + vkAllocateDescriptorSets(m_device, &set_info, &descriptor_set), + "vkAllocateDescriptorSets"); + + std::vector writes; + std::vector buffer_infos; + std::vector address_memory_infos; + std::vector sampled_infos; + std::vector storage_infos; + std::vector storage_uint_infos; + std::vector sampler_infos; + Buffer flattened_buffer; + Buffer user_data_buffer; + VkDescriptorBufferInfo flattened_info{}; + VkDescriptorBufferInfo user_data_info{}; + VkDescriptorBufferInfo gds_info{}; + + const auto *buffers = Binding(Kind::Buffers); + if (buffers != nullptr) { + buffer_infos.resize(buffers->resources.size()); + for (auto &info : buffer_infos) { + info.buffer = buffer.buffer; + info.offset = 0; + info.range = buffer.size; + if (test.storage_buffer_range_dwords != 0) { + info.range = static_cast( + test.storage_buffer_range_dwords * sizeof(u32)); + Require(test.name, "dispatch", info.range <= buffer.size, + "storage buffer descriptor range exceeds backing buffer"); + } + } + VkWriteDescriptorSet write{}; + write.sType = VK_STRUCTURE_TYPE_WRITE_DESCRIPTOR_SET; + write.dstSet = descriptor_set; + write.dstBinding = buffers->binding; + write.descriptorCount = static_cast(buffer_infos.size()); + write.descriptorType = VK_DESCRIPTOR_TYPE_STORAGE_BUFFER; + write.pBufferInfo = buffer_infos.data(); + writes.push_back(write); + } + if (const auto *address = Binding(Kind::AddressMemory); + address != nullptr) { + address_memory_infos.resize(address->resources.size()); + for (auto &info : address_memory_infos) { + info = {buffer.buffer, 0, buffer.size}; + } + VkWriteDescriptorSet write{}; + write.sType = VK_STRUCTURE_TYPE_WRITE_DESCRIPTOR_SET; + write.dstSet = descriptor_set; + write.dstBinding = address->binding; + write.descriptorCount = static_cast(address_memory_infos.size()); + write.descriptorType = VK_DESCRIPTOR_TYPE_STORAGE_BUFFER; + write.pBufferInfo = address_memory_infos.data(); + writes.push_back(write); + } + if (const auto *flattened = Binding(Kind::FlattenedSrt); + flattened != nullptr) { + flattened_buffer = CreateStorageBuffer(test.name, compiled.flattened_srt, + compiled.flattened_srt.size()); + flattened_info = {flattened_buffer.buffer, 0, flattened_buffer.size}; + VkWriteDescriptorSet write{}; + write.sType = VK_STRUCTURE_TYPE_WRITE_DESCRIPTOR_SET; + write.dstSet = descriptor_set; + write.dstBinding = flattened->binding; + write.descriptorCount = 1; + write.descriptorType = VK_DESCRIPTOR_TYPE_STORAGE_BUFFER; + write.pBufferInfo = &flattened_info; + writes.push_back(write); + } + if (const auto *user = Binding(Kind::UserData); user != nullptr) { + user_data_buffer = + CreateStorageBuffer(test.name, compiled.packed_user_data, + compiled.packed_user_data.size()); + user_data_info = {user_data_buffer.buffer, 0, user_data_buffer.size}; + VkWriteDescriptorSet write{}; + write.sType = VK_STRUCTURE_TYPE_WRITE_DESCRIPTOR_SET; + write.dstSet = descriptor_set; + write.dstBinding = user->binding; + write.descriptorCount = 1; + write.descriptorType = VK_DESCRIPTOR_TYPE_STORAGE_BUFFER; + write.pBufferInfo = &user_data_info; + writes.push_back(write); + } + if (const auto *gds = Binding(Kind::Gds); gds != nullptr) { + Require(test.name, "dispatch", gds_buffer != nullptr, + "GDS descriptor requested but no GDS buffer was provided"); + gds_info = {gds_buffer->buffer, 0, gds_buffer->size}; + VkWriteDescriptorSet write{}; + write.sType = VK_STRUCTURE_TYPE_WRITE_DESCRIPTOR_SET; + write.dstSet = descriptor_set; + write.dstBinding = gds->binding; + write.descriptorCount = 1; + write.descriptorType = VK_DESCRIPTOR_TYPE_STORAGE_BUFFER; + write.pBufferInfo = &gds_info; + writes.push_back(write); + } + const auto *sampled = Binding(Kind::Sampled2D); + const auto *sampled_uint = Binding(Kind::SampledUint2D); + Require(test.name, "dispatch", + sampled == nullptr || sampled_uint == nullptr, + "Vulkan test harness needs separate float and uint sampled images"); + if (sampled == nullptr) { + sampled = sampled_uint; + } + if (sampled != nullptr) { + Require(test.name, "dispatch", sampled_image != nullptr, + "sampled image descriptor requested but no sampled image was " + "provided"); + sampled_infos.resize(sampled->resources.size()); + for (auto &info : sampled_infos) { + info.imageView = sampled_image->view; + info.imageLayout = sampled_image->layout; + } + VkWriteDescriptorSet write{}; + write.sType = VK_STRUCTURE_TYPE_WRITE_DESCRIPTOR_SET; + write.dstSet = descriptor_set; + write.dstBinding = sampled->binding; + write.descriptorCount = static_cast(sampled_infos.size()); + write.descriptorType = VK_DESCRIPTOR_TYPE_SAMPLED_IMAGE; + write.pImageInfo = sampled_infos.data(); + writes.push_back(write); + } + const auto *storage = Binding(Kind::Storage2D); + const auto *storage_uint = Binding(Kind::StorageUint2D); + if (storage != nullptr || storage_uint != nullptr) { + Require(test.name, "dispatch", storage_image != nullptr, + "storage image descriptor requested but no storage image was " + "provided"); + Require(test.name, "dispatch", storage_image_uint != nullptr, + "uint storage image descriptor requested but no uint storage " + "image was provided"); + storage_infos.resize(storage != nullptr ? storage->resources.size() : 0u); + storage_uint_infos.resize( + storage_uint != nullptr ? storage_uint->resources.size() : 0u); + for (auto &info : storage_infos) { + info.imageView = storage_image->view; + info.imageLayout = storage_image->layout; + } + for (auto &info : storage_uint_infos) { + info.imageView = storage_image_uint->view; + info.imageLayout = storage_image_uint->layout; + } + if (storage != nullptr) { + VkWriteDescriptorSet write{}; + write.sType = VK_STRUCTURE_TYPE_WRITE_DESCRIPTOR_SET; + write.dstSet = descriptor_set; + write.dstBinding = storage->binding; + write.descriptorCount = static_cast(storage_infos.size()); + write.descriptorType = VK_DESCRIPTOR_TYPE_STORAGE_IMAGE; + write.pImageInfo = storage_infos.data(); + writes.push_back(write); + } + if (storage_uint != nullptr) { + VkWriteDescriptorSet write{}; + write.sType = VK_STRUCTURE_TYPE_WRITE_DESCRIPTOR_SET; + write.dstSet = descriptor_set; + write.dstBinding = storage_uint->binding; + write.descriptorCount = static_cast(storage_uint_infos.size()); + write.descriptorType = VK_DESCRIPTOR_TYPE_STORAGE_IMAGE; + write.pImageInfo = storage_uint_infos.data(); + writes.push_back(write); + } + } + const auto *samplers = Binding(Kind::Samplers); + if (samplers != nullptr) { + Require(test.name, "dispatch", sampler != VK_NULL_HANDLE, + "sampler descriptor requested but no sampler was provided"); + sampler_infos.resize(samplers->resources.size()); + for (auto &info : sampler_infos) { + info.sampler = sampler; + } + VkWriteDescriptorSet write{}; + write.sType = VK_STRUCTURE_TYPE_WRITE_DESCRIPTOR_SET; + write.dstSet = descriptor_set; + write.dstBinding = samplers->binding; + write.descriptorCount = static_cast(sampler_infos.size()); + write.descriptorType = VK_DESCRIPTOR_TYPE_SAMPLER; + write.pImageInfo = sampler_infos.data(); + writes.push_back(write); + } + if (!writes.empty()) { + vkUpdateDescriptorSets(m_device, static_cast(writes.size()), + writes.data(), 0, nullptr); + } + + VkCommandBuffer cmd = BeginCommands(test.name, "dispatch"); + vkCmdBindPipeline(cmd, VK_PIPELINE_BIND_POINT_COMPUTE, pipeline); + vkCmdBindDescriptorSets(cmd, VK_PIPELINE_BIND_POINT_COMPUTE, + pipeline_layout, 0, 1, &descriptor_set, 0, nullptr); + if (layout.push_constant_size != 0) { + Require(test.name, "dispatch", + compiled.packed_user_data.size() * sizeof(u32) == + layout.push_constant_size, + "native user-data size does not match push-constant range"); + vkCmdPushConstants(cmd, pipeline_layout, VK_SHADER_STAGE_COMPUTE_BIT, + layout.push_constant_offset, layout.push_constant_size, + compiled.packed_user_data.data()); + } + vkCmdDispatch(cmd, test.dispatch_x, test.dispatch_y, test.dispatch_z); + + if (buffers != nullptr) { + VkBufferMemoryBarrier barrier{}; + barrier.sType = VK_STRUCTURE_TYPE_BUFFER_MEMORY_BARRIER; + barrier.srcAccessMask = + VK_ACCESS_SHADER_READ_BIT | VK_ACCESS_SHADER_WRITE_BIT; + barrier.dstAccessMask = VK_ACCESS_HOST_READ_BIT; + barrier.srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; + barrier.dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; + barrier.buffer = buffer.buffer; + barrier.offset = 0; + barrier.size = buffer.size; + vkCmdPipelineBarrier(cmd, VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT, + VK_PIPELINE_STAGE_HOST_BIT, 0, 0, nullptr, 1, + &barrier, 0, nullptr); + } + if (gds_buffer != nullptr) { + VkBufferMemoryBarrier barrier{}; + barrier.sType = VK_STRUCTURE_TYPE_BUFFER_MEMORY_BARRIER; + barrier.srcAccessMask = + VK_ACCESS_SHADER_READ_BIT | VK_ACCESS_SHADER_WRITE_BIT; + barrier.dstAccessMask = VK_ACCESS_HOST_READ_BIT; + barrier.srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; + barrier.dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; + barrier.buffer = gds_buffer->buffer; + barrier.offset = 0; + barrier.size = gds_buffer->size; + vkCmdPipelineBarrier(cmd, VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT, + VK_PIPELINE_STAGE_HOST_BIT, 0, 0, nullptr, 1, + &barrier, 0, nullptr); + } + EndSubmitAndFree(test.name, "dispatch", cmd); + if (flattened_buffer.buffer != VK_NULL_HANDLE) { + DestroyBuffer(&flattened_buffer); + } + if (user_data_buffer.buffer != VK_NULL_HANDLE) { + DestroyBuffer(&user_data_buffer); + } + vkDestroyDescriptorPool(m_device, descriptor_pool, nullptr); + vkDestroyPipeline(m_device, pipeline, nullptr); + vkDestroyPipelineLayout(m_device, pipeline_layout, nullptr); + vkDestroyDescriptorSetLayout(m_device, descriptor_layout, nullptr); + vkDestroyShaderModule(m_device, module, nullptr); + } + + std::vector RenderFragment(const GraphicsCase &test, + const CompiledShader &fragment) { + const auto vertex_spirv = TestSpv::MakePassthroughVertexSpirv(); + ValidateSpirv(test.name, vertex_spirv); + + Image target = CreateImage2D(test.name, 1, 1, VK_FORMAT_R32G32B32A32_SFLOAT, + VK_IMAGE_USAGE_COLOR_ATTACHMENT_BIT, {}, 4, + VK_IMAGE_LAYOUT_GENERAL); + const std::vector default_vertices = { + 0xbf800000u, 0xbf800000u, 0x3e800000u, 0x3f000000u, 0x3f400000u, + 0x3f800000u, 0x40400000u, 0xbf800000u, 0x3e800000u, 0x3f000000u, + 0x3f400000u, 0x3f800000u, 0xbf800000u, 0x40400000u, 0x3e800000u, + 0x3f000000u, 0x3f400000u, 0x3f800000u, + }; + const auto &vertices = + test.vertices.empty() ? default_vertices : test.vertices; + Require(test.name, "graphics", vertices.size() == 18u, + "graphics vertex buffer must contain three pos2/color4 vertices"); + auto vertex_buffer = + CreateHostBuffer(test.name, vertices.size() * sizeof(u32), + VK_BUFFER_USAGE_VERTEX_BUFFER_BIT, vertices); + + auto make_module = [&](const std::vector &spirv) { + VkShaderModuleCreateInfo module_info{}; + module_info.sType = VK_STRUCTURE_TYPE_SHADER_MODULE_CREATE_INFO; + module_info.codeSize = spirv.size() * sizeof(u32); + module_info.pCode = spirv.data(); + VkShaderModule module = VK_NULL_HANDLE; + RequireVk(test.name, "graphics", + vkCreateShaderModule(m_device, &module_info, nullptr, &module), + "vkCreateShaderModule"); + return module; + }; + VkShaderModule vertex_module = make_module(vertex_spirv); + VkShaderModule fragment_module = make_module(fragment.spirv); + + VkAttachmentDescription attachment{}; + attachment.format = VK_FORMAT_R32G32B32A32_SFLOAT; + attachment.samples = VK_SAMPLE_COUNT_1_BIT; + attachment.loadOp = VK_ATTACHMENT_LOAD_OP_CLEAR; + attachment.storeOp = VK_ATTACHMENT_STORE_OP_STORE; + attachment.stencilLoadOp = VK_ATTACHMENT_LOAD_OP_DONT_CARE; + attachment.stencilStoreOp = VK_ATTACHMENT_STORE_OP_DONT_CARE; + attachment.initialLayout = VK_IMAGE_LAYOUT_GENERAL; + attachment.finalLayout = VK_IMAGE_LAYOUT_GENERAL; + + VkAttachmentReference color_ref{}; + color_ref.attachment = 0; + color_ref.layout = VK_IMAGE_LAYOUT_COLOR_ATTACHMENT_OPTIMAL; + + VkSubpassDescription subpass{}; + subpass.pipelineBindPoint = VK_PIPELINE_BIND_POINT_GRAPHICS; + subpass.colorAttachmentCount = 1; + subpass.pColorAttachments = &color_ref; + + VkRenderPassCreateInfo render_pass_info{}; + render_pass_info.sType = VK_STRUCTURE_TYPE_RENDER_PASS_CREATE_INFO; + render_pass_info.attachmentCount = 1; + render_pass_info.pAttachments = &attachment; + render_pass_info.subpassCount = 1; + render_pass_info.pSubpasses = &subpass; + VkRenderPass render_pass = VK_NULL_HANDLE; + RequireVk( + test.name, "graphics", + vkCreateRenderPass(m_device, &render_pass_info, nullptr, &render_pass), + "vkCreateRenderPass"); + + VkFramebufferCreateInfo framebuffer_info{}; + framebuffer_info.sType = VK_STRUCTURE_TYPE_FRAMEBUFFER_CREATE_INFO; + framebuffer_info.renderPass = render_pass; + framebuffer_info.attachmentCount = 1; + framebuffer_info.pAttachments = &target.view; + framebuffer_info.width = 1; + framebuffer_info.height = 1; + framebuffer_info.layers = 1; + VkFramebuffer framebuffer = VK_NULL_HANDLE; + RequireVk( + test.name, "graphics", + vkCreateFramebuffer(m_device, &framebuffer_info, nullptr, &framebuffer), + "vkCreateFramebuffer"); + + const auto &fragment_bind = fragment.program.bindings; + VkPushConstantRange push_constant_range{}; + if (fragment_bind.push_constant_size > 0) { + Require(test.name, "graphics", + test.push_constants.size() * sizeof(u32) == + fragment_bind.push_constant_size, + "fragment push constant data size does not match reflection"); + push_constant_range.stageFlags = VK_SHADER_STAGE_FRAGMENT_BIT; + push_constant_range.offset = fragment_bind.push_constant_offset; + push_constant_range.size = fragment_bind.push_constant_size; + } + + VkPipelineLayoutCreateInfo pipeline_layout_info{}; + pipeline_layout_info.sType = VK_STRUCTURE_TYPE_PIPELINE_LAYOUT_CREATE_INFO; + pipeline_layout_info.pushConstantRangeCount = + push_constant_range.size != 0 ? 1u : 0u; + pipeline_layout_info.pPushConstantRanges = + push_constant_range.size != 0 ? &push_constant_range : nullptr; + VkPipelineLayout pipeline_layout = VK_NULL_HANDLE; + RequireVk(test.name, "graphics", + vkCreatePipelineLayout(m_device, &pipeline_layout_info, nullptr, + &pipeline_layout), + "vkCreatePipelineLayout"); + + VkPipelineShaderStageCreateInfo stages[2] = {}; + stages[0].sType = VK_STRUCTURE_TYPE_PIPELINE_SHADER_STAGE_CREATE_INFO; + stages[0].stage = VK_SHADER_STAGE_VERTEX_BIT; + stages[0].module = vertex_module; + stages[0].pName = "main"; + stages[1].sType = VK_STRUCTURE_TYPE_PIPELINE_SHADER_STAGE_CREATE_INFO; + stages[1].stage = VK_SHADER_STAGE_FRAGMENT_BIT; + stages[1].module = fragment_module; + stages[1].pName = "main"; + + VkVertexInputBindingDescription vertex_binding{}; + vertex_binding.binding = 0; + vertex_binding.stride = 6u * sizeof(float); + vertex_binding.inputRate = VK_VERTEX_INPUT_RATE_VERTEX; + VkVertexInputAttributeDescription attributes[2] = {}; + attributes[0].location = 0; + attributes[0].binding = 0; + attributes[0].format = VK_FORMAT_R32G32_SFLOAT; + attributes[0].offset = 0; + attributes[1].location = 1; + attributes[1].binding = 0; + attributes[1].format = VK_FORMAT_R32G32B32A32_SFLOAT; + attributes[1].offset = 2u * sizeof(float); + VkPipelineVertexInputStateCreateInfo vertex_input{}; + vertex_input.sType = + VK_STRUCTURE_TYPE_PIPELINE_VERTEX_INPUT_STATE_CREATE_INFO; + vertex_input.vertexBindingDescriptionCount = 1; + vertex_input.pVertexBindingDescriptions = &vertex_binding; + vertex_input.vertexAttributeDescriptionCount = 2; + vertex_input.pVertexAttributeDescriptions = attributes; + + VkPipelineInputAssemblyStateCreateInfo input_assembly{}; + input_assembly.sType = + VK_STRUCTURE_TYPE_PIPELINE_INPUT_ASSEMBLY_STATE_CREATE_INFO; + input_assembly.topology = VK_PRIMITIVE_TOPOLOGY_TRIANGLE_LIST; + + VkViewport viewport{}; + viewport.x = 0.0f; + viewport.y = 0.0f; + viewport.width = 1.0f; + viewport.height = 1.0f; + viewport.minDepth = 0.0f; + viewport.maxDepth = 1.0f; + VkRect2D scissor{}; + scissor.extent.width = 1; + scissor.extent.height = 1; + VkPipelineViewportStateCreateInfo viewport_state{}; + viewport_state.sType = + VK_STRUCTURE_TYPE_PIPELINE_VIEWPORT_STATE_CREATE_INFO; + viewport_state.viewportCount = 1; + viewport_state.pViewports = &viewport; + viewport_state.scissorCount = 1; + viewport_state.pScissors = &scissor; + + VkPipelineRasterizationStateCreateInfo raster{}; + raster.sType = VK_STRUCTURE_TYPE_PIPELINE_RASTERIZATION_STATE_CREATE_INFO; + raster.polygonMode = VK_POLYGON_MODE_FILL; + raster.cullMode = VK_CULL_MODE_NONE; + raster.frontFace = VK_FRONT_FACE_COUNTER_CLOCKWISE; + raster.lineWidth = 1.0f; + + VkPipelineMultisampleStateCreateInfo multisample{}; + multisample.sType = + VK_STRUCTURE_TYPE_PIPELINE_MULTISAMPLE_STATE_CREATE_INFO; + multisample.rasterizationSamples = VK_SAMPLE_COUNT_1_BIT; + + VkPipelineColorBlendAttachmentState color_attachment{}; + color_attachment.colorWriteMask = + VK_COLOR_COMPONENT_R_BIT | VK_COLOR_COMPONENT_G_BIT | + VK_COLOR_COMPONENT_B_BIT | VK_COLOR_COMPONENT_A_BIT; + VkPipelineColorBlendStateCreateInfo color_blend{}; + color_blend.sType = + VK_STRUCTURE_TYPE_PIPELINE_COLOR_BLEND_STATE_CREATE_INFO; + color_blend.attachmentCount = 1; + color_blend.pAttachments = &color_attachment; + + VkGraphicsPipelineCreateInfo pipeline_info{}; + pipeline_info.sType = VK_STRUCTURE_TYPE_GRAPHICS_PIPELINE_CREATE_INFO; + pipeline_info.stageCount = 2; + pipeline_info.pStages = stages; + pipeline_info.pVertexInputState = &vertex_input; + pipeline_info.pInputAssemblyState = &input_assembly; + pipeline_info.pViewportState = &viewport_state; + pipeline_info.pRasterizationState = &raster; + pipeline_info.pMultisampleState = &multisample; + pipeline_info.pColorBlendState = &color_blend; + pipeline_info.layout = pipeline_layout; + pipeline_info.renderPass = render_pass; + VkPipeline pipeline = VK_NULL_HANDLE; + RequireVk(test.name, "graphics", + vkCreateGraphicsPipelines(m_device, VK_NULL_HANDLE, 1, + &pipeline_info, nullptr, &pipeline), + "vkCreateGraphicsPipelines"); + + VkCommandBuffer cmd = BeginCommands(test.name, "graphics"); + VkClearValue clear{}; + VkRenderPassBeginInfo begin{}; + begin.sType = VK_STRUCTURE_TYPE_RENDER_PASS_BEGIN_INFO; + begin.renderPass = render_pass; + begin.framebuffer = framebuffer; + begin.renderArea.extent.width = 1; + begin.renderArea.extent.height = 1; + begin.clearValueCount = 1; + begin.pClearValues = &clear; + vkCmdBeginRenderPass(cmd, &begin, VK_SUBPASS_CONTENTS_INLINE); + vkCmdBindPipeline(cmd, VK_PIPELINE_BIND_POINT_GRAPHICS, pipeline); + if (push_constant_range.size != 0) { + vkCmdPushConstants(cmd, pipeline_layout, VK_SHADER_STAGE_FRAGMENT_BIT, + fragment_bind.push_constant_offset, + fragment_bind.push_constant_size, + test.push_constants.data()); + } + VkDeviceSize offset = 0; + vkCmdBindVertexBuffers(cmd, 0, 1, &vertex_buffer.buffer, &offset); + vkCmdDraw(cmd, 3, 1, 0, 0); + vkCmdEndRenderPass(cmd); + EndSubmitAndFree(test.name, "graphics", cmd); + target.layout = VK_IMAGE_LAYOUT_GENERAL; + + auto pixel = ReadImage(test.name, &target); + pixel.resize(4); + + vkDestroyPipeline(m_device, pipeline, nullptr); + vkDestroyPipelineLayout(m_device, pipeline_layout, nullptr); + vkDestroyFramebuffer(m_device, framebuffer, nullptr); + vkDestroyRenderPass(m_device, render_pass, nullptr); + vkDestroyShaderModule(m_device, fragment_module, nullptr); + vkDestroyShaderModule(m_device, vertex_module, nullptr); + DestroyBuffer(&vertex_buffer); + DestroyImage(&target); + return pixel; + } + +private: + void Init() { + VkApplicationInfo app{}; + app.sType = VK_STRUCTURE_TYPE_APPLICATION_INFO; + app.pApplicationName = "ShaderRecompilerComputeTests"; + app.apiVersion = VK_API_VERSION_1_2; + + VkInstanceCreateInfo instance_info{}; + instance_info.sType = VK_STRUCTURE_TYPE_INSTANCE_CREATE_INFO; + instance_info.pApplicationInfo = &app; + RequireVk("VulkanHarness", "dispatch", + vkCreateInstance(&instance_info, nullptr, &m_instance), + "vkCreateInstance"); + + u32 physical_count = 0; + RequireVk("VulkanHarness", "dispatch", + vkEnumeratePhysicalDevices(m_instance, &physical_count, nullptr), + "vkEnumeratePhysicalDevices"); + Require("VulkanHarness", "dispatch", physical_count != 0, + "no Vulkan physical devices"); + std::vector physical_devices(physical_count); + RequireVk("VulkanHarness", "dispatch", + vkEnumeratePhysicalDevices(m_instance, &physical_count, + physical_devices.data()), + "vkEnumeratePhysicalDevices"); + + for (auto physical : physical_devices) { + u32 queue_count = 0; + vkGetPhysicalDeviceQueueFamilyProperties(physical, &queue_count, nullptr); + std::vector queues(queue_count); + vkGetPhysicalDeviceQueueFamilyProperties(physical, &queue_count, + queues.data()); + for (u32 i = 0; i < queue_count; i++) { + if ((queues[i].queueFlags & + (VK_QUEUE_COMPUTE_BIT | VK_QUEUE_GRAPHICS_BIT)) == + (VK_QUEUE_COMPUTE_BIT | VK_QUEUE_GRAPHICS_BIT)) { + m_physical_device = physical; + m_queue_family = i; + break; + } + } + if (m_physical_device != VK_NULL_HANDLE) { + break; + } + } + Require("VulkanHarness", "dispatch", m_physical_device != VK_NULL_HANDLE, + "no Vulkan graphics+compute queue family"); + vkGetPhysicalDeviceMemoryProperties(m_physical_device, + &m_memory_properties); + + VkPhysicalDeviceFeatures available_features{}; + vkGetPhysicalDeviceFeatures(m_physical_device, &available_features); + Require("VulkanHarness", "dispatch", + available_features.shaderStorageImageWriteWithoutFormat == VK_TRUE, + "shaderStorageImageWriteWithoutFormat is not supported"); + Require("VulkanHarness", "dispatch", + available_features.shaderStorageImageReadWithoutFormat == VK_TRUE, + "shaderStorageImageReadWithoutFormat is not supported"); + + float priority = 1.0f; + VkDeviceQueueCreateInfo queue_info{}; + queue_info.sType = VK_STRUCTURE_TYPE_DEVICE_QUEUE_CREATE_INFO; + queue_info.queueFamilyIndex = m_queue_family; + queue_info.queueCount = 1; + queue_info.pQueuePriorities = &priority; + + VkDeviceCreateInfo device_info{}; + device_info.sType = VK_STRUCTURE_TYPE_DEVICE_CREATE_INFO; + device_info.queueCreateInfoCount = 1; + device_info.pQueueCreateInfos = &queue_info; + VkPhysicalDeviceFeatures device_features{}; + device_features.shaderStorageImageWriteWithoutFormat = VK_TRUE; + device_features.shaderStorageImageReadWithoutFormat = VK_TRUE; + device_info.pEnabledFeatures = &device_features; + RequireVk( + "VulkanHarness", "dispatch", + vkCreateDevice(m_physical_device, &device_info, nullptr, &m_device), + "vkCreateDevice"); + vkGetDeviceQueue(m_device, m_queue_family, 0, &m_queue); + + VkCommandPoolCreateInfo pool_info{}; + pool_info.sType = VK_STRUCTURE_TYPE_COMMAND_POOL_CREATE_INFO; + pool_info.queueFamilyIndex = m_queue_family; + pool_info.flags = VK_COMMAND_POOL_CREATE_RESET_COMMAND_BUFFER_BIT; + RequireVk( + "VulkanHarness", "dispatch", + vkCreateCommandPool(m_device, &pool_info, nullptr, &m_command_pool), + "vkCreateCommandPool"); + } + + void Destroy() { + if (m_device != VK_NULL_HANDLE) { + vkDeviceWaitIdle(m_device); + if (m_command_pool != VK_NULL_HANDLE) { + vkDestroyCommandPool(m_device, m_command_pool, nullptr); + } + vkDestroyDevice(m_device, nullptr); + } + if (m_instance != VK_NULL_HANDLE) { + vkDestroyInstance(m_instance, nullptr); + } + } + + bool FindMemoryType(u32 type_bits, VkMemoryPropertyFlags required, + u32 *index) const { + for (u32 i = 0; i < m_memory_properties.memoryTypeCount; i++) { + if ((type_bits & (1u << i)) == 0) { + continue; + } + if ((m_memory_properties.memoryTypes[i].propertyFlags & required) == + required) { + *index = i; + return true; + } + } + return false; + } + + static VkAccessFlags AccessForLayout(VkImageLayout layout) { + switch (layout) { + case VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL: + return VK_ACCESS_TRANSFER_WRITE_BIT; + case VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL: + return VK_ACCESS_TRANSFER_READ_BIT; + case VK_IMAGE_LAYOUT_SHADER_READ_ONLY_OPTIMAL: + return VK_ACCESS_SHADER_READ_BIT; + case VK_IMAGE_LAYOUT_GENERAL: + return VK_ACCESS_SHADER_READ_BIT | VK_ACCESS_SHADER_WRITE_BIT; + default: + return 0; + } + } + + VkCommandBuffer BeginCommands(const char *shader_name, const char *stage) { + VkCommandBufferAllocateInfo cmd_alloc{}; + cmd_alloc.sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_ALLOCATE_INFO; + cmd_alloc.commandPool = m_command_pool; + cmd_alloc.level = VK_COMMAND_BUFFER_LEVEL_PRIMARY; + cmd_alloc.commandBufferCount = 1; + VkCommandBuffer cmd = VK_NULL_HANDLE; + RequireVk(shader_name, stage, + vkAllocateCommandBuffers(m_device, &cmd_alloc, &cmd), + "vkAllocateCommandBuffers"); + + VkCommandBufferBeginInfo begin{}; + begin.sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_BEGIN_INFO; + begin.flags = VK_COMMAND_BUFFER_USAGE_ONE_TIME_SUBMIT_BIT; + RequireVk(shader_name, stage, vkBeginCommandBuffer(cmd, &begin), + "vkBeginCommandBuffer"); + return cmd; + } + + void EndSubmitAndFree(const char *shader_name, const char *stage, + VkCommandBuffer cmd) { + RequireVk(shader_name, stage, vkEndCommandBuffer(cmd), + "vkEndCommandBuffer"); + + VkFenceCreateInfo fence_info{}; + fence_info.sType = VK_STRUCTURE_TYPE_FENCE_CREATE_INFO; + VkFence fence = VK_NULL_HANDLE; + RequireVk(shader_name, stage, + vkCreateFence(m_device, &fence_info, nullptr, &fence), + "vkCreateFence"); + + VkSubmitInfo submit{}; + submit.sType = VK_STRUCTURE_TYPE_SUBMIT_INFO; + submit.commandBufferCount = 1; + submit.pCommandBuffers = &cmd; + RequireVk(shader_name, stage, vkQueueSubmit(m_queue, 1, &submit, fence), + "vkQueueSubmit"); + RequireVk(shader_name, stage, + vkWaitForFences(m_device, 1, &fence, VK_TRUE, UINT64_MAX), + "vkWaitForFences"); + + vkDestroyFence(m_device, fence, nullptr); + vkFreeCommandBuffers(m_device, m_command_pool, 1, &cmd); + } + + void AddImageBarrier(VkCommandBuffer cmd, VkImage image, + VkImageLayout old_layout, VkImageLayout new_layout, + VkPipelineStageFlags src_stage, + VkPipelineStageFlags dst_stage, VkAccessFlags src_access, + VkAccessFlags dst_access, u32 mip_levels = 1) { + VkImageMemoryBarrier barrier{}; + barrier.sType = VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER; + barrier.srcAccessMask = src_access; + barrier.dstAccessMask = dst_access; + barrier.oldLayout = old_layout; + barrier.newLayout = new_layout; + barrier.srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; + barrier.dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; + barrier.image = image; + barrier.subresourceRange.aspectMask = VK_IMAGE_ASPECT_COLOR_BIT; + barrier.subresourceRange.baseMipLevel = 0; + barrier.subresourceRange.levelCount = mip_levels; + barrier.subresourceRange.baseArrayLayer = 0; + barrier.subresourceRange.layerCount = 1; + vkCmdPipelineBarrier(cmd, src_stage, dst_stage, 0, 0, nullptr, 0, nullptr, + 1, &barrier); + } + + void TransitionImage(const char *shader_name, Image *image, + VkImageLayout new_layout, VkPipelineStageFlags src_stage, + VkPipelineStageFlags dst_stage, VkAccessFlags src_access, + VkAccessFlags dst_access) { + VkCommandBuffer cmd = BeginCommands(shader_name, "dispatch"); + AddImageBarrier(cmd, image->image, image->layout, new_layout, src_stage, + dst_stage, src_access, dst_access, image->mip_levels); + EndSubmitAndFree(shader_name, "dispatch", cmd); + image->layout = new_layout; + } + + void UploadImage(const char *shader_name, Image *image, VkBuffer staging, + VkImageLayout final_layout) { + UploadImageMips(shader_name, image, staging, final_layout); + } + + void UploadImageMips(const char *shader_name, Image *image, VkBuffer staging, + VkImageLayout final_layout) { + VkCommandBuffer cmd = BeginCommands(shader_name, "dispatch"); + AddImageBarrier(cmd, image->image, VK_IMAGE_LAYOUT_UNDEFINED, + VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL, + VK_PIPELINE_STAGE_TOP_OF_PIPE_BIT, + VK_PIPELINE_STAGE_TRANSFER_BIT, 0, + VK_ACCESS_TRANSFER_WRITE_BIT, image->mip_levels); + std::vector copies; + copies.reserve(image->mip_levels); + VkDeviceSize offset = 0; + for (u32 level = 0; level < image->mip_levels; level++) { + VkBufferImageCopy copy{}; + copy.bufferOffset = offset; + copy.imageSubresource.aspectMask = VK_IMAGE_ASPECT_COLOR_BIT; + copy.imageSubresource.mipLevel = level; + copy.imageSubresource.baseArrayLayer = 0; + copy.imageSubresource.layerCount = 1; + copy.imageExtent.width = MipExtent(image->width, level); + copy.imageExtent.height = MipExtent(image->height, level); + copy.imageExtent.depth = 1; + copies.push_back(copy); + offset += static_cast( + ImageMipDwordCount(image->width, image->height, + image->dwords_per_pixel, level) * + sizeof(u32)); + } + vkCmdCopyBufferToImage(cmd, staging, image->image, + VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL, + static_cast(copies.size()), copies.data()); + AddImageBarrier(cmd, image->image, VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL, + final_layout, VK_PIPELINE_STAGE_TRANSFER_BIT, + VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT, + VK_ACCESS_TRANSFER_WRITE_BIT, AccessForLayout(final_layout), + image->mip_levels); + EndSubmitAndFree(shader_name, "dispatch", cmd); + image->layout = final_layout; + } + + Buffer CreateHostBuffer(const char *shader_name, VkDeviceSize size, + VkBufferUsageFlags usage, + const std::vector &contents) { + Buffer ret; + ret.size = std::max(size, sizeof(u32)); + + VkBufferCreateInfo buffer_info{}; + buffer_info.sType = VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO; + buffer_info.size = ret.size; + buffer_info.usage = usage; + buffer_info.sharingMode = VK_SHARING_MODE_EXCLUSIVE; + RequireVk(shader_name, "dispatch", + vkCreateBuffer(m_device, &buffer_info, nullptr, &ret.buffer), + "vkCreateBuffer"); + + VkMemoryRequirements req{}; + vkGetBufferMemoryRequirements(m_device, ret.buffer, &req); + u32 memory_type = 0; + ret.coherent = FindMemoryType(req.memoryTypeBits, + VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT | + VK_MEMORY_PROPERTY_HOST_COHERENT_BIT, + &memory_type); + if (!ret.coherent) { + Require(shader_name, "dispatch", + FindMemoryType(req.memoryTypeBits, + VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT, &memory_type), + "no host-visible memory type for staging buffer"); + } + + VkMemoryAllocateInfo alloc{}; + alloc.sType = VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_INFO; + alloc.allocationSize = req.size; + alloc.memoryTypeIndex = memory_type; + RequireVk(shader_name, "dispatch", + vkAllocateMemory(m_device, &alloc, nullptr, &ret.memory), + "vkAllocateMemory"); + RequireVk(shader_name, "dispatch", + vkBindBufferMemory(m_device, ret.buffer, ret.memory, 0), + "vkBindBufferMemory"); + if (!contents.empty()) { + WriteBuffer(shader_name, ret, contents); + } + return ret; + } + + void WriteBuffer(const char *shader_name, const Buffer &buffer, + const std::vector &contents) { + void *data = nullptr; + RequireVk(shader_name, "dispatch", + vkMapMemory(m_device, buffer.memory, 0, buffer.size, 0, &data), + "vkMapMemory"); + std::memcpy(data, contents.data(), contents.size() * sizeof(u32)); + if (!buffer.coherent) { + VkMappedMemoryRange range{}; + range.sType = VK_STRUCTURE_TYPE_MAPPED_MEMORY_RANGE; + range.memory = buffer.memory; + range.offset = 0; + range.size = VK_WHOLE_SIZE; + RequireVk(shader_name, "dispatch", + vkFlushMappedMemoryRanges(m_device, 1, &range), + "vkFlushMappedMemoryRanges"); + } + vkUnmapMemory(m_device, buffer.memory); + } + + VkInstance m_instance = VK_NULL_HANDLE; + VkPhysicalDevice m_physical_device = VK_NULL_HANDLE; + VkDevice m_device = VK_NULL_HANDLE; + VkQueue m_queue = VK_NULL_HANDLE; + VkCommandPool m_command_pool = VK_NULL_HANDLE; + u32 m_queue_family = 0; + VkPhysicalDeviceMemoryProperties m_memory_properties{}; +}; + +void CompareWords(const TestCase &test, const char *stage, + const std::vector &expected, + const std::vector &actual) { + if (actual == expected) { + return; + } + std::ostringstream out; + out << "expected ["; + for (size_t i = 0; i < expected.size(); i++) { + out << (i == 0 ? "" : ", ") << Hex(expected[i]); + } + out << "] actual ["; + for (size_t i = 0; i < actual.size(); i++) { + out << (i == 0 ? "" : ", ") << Hex(actual[i]); + } + out << "]"; + Fail(test.name, stage, out.str()); +} + +void CompareGraphicsWords(const GraphicsCase &test, + const std::vector &actual) { + if (actual == test.expected_pixel) { + return; + } + std::ostringstream out; + out << "expected ["; + for (size_t i = 0; i < test.expected_pixel.size(); i++) { + out << (i == 0 ? "" : ", ") << Hex(test.expected_pixel[i]); + } + out << "] actual ["; + for (size_t i = 0; i < actual.size(); i++) { + out << (i == 0 ? "" : ", ") << Hex(actual[i]); + } + out << "]"; + Fail(test.name, "graphics readback", out.str()); +} + +void RunCase(VulkanHarness *vulkan, const TestCase &test) { + auto compiled = CompileCase(test); + if (test.image_descriptor_swizzle != DstSel(4, 5, 6, 7)) { + Require(test.name, "resource specialization", + !compiled.program.info.images.empty() && + compiled.program.info.images[0].storage_swizzle == + test.image_descriptor_swizzle, + "storage image descriptor swizzle did not reach the specialized program"); + } + if (test.compile_only) { + std::printf("[compute] %-32s ok\n", test.name); + return; + } + const auto dwords = std::max( + {test.initial.size(), test.expected.size(), static_cast(1)}); + auto buffer = vulkan->CreateStorageBuffer(test.name, test.initial, dwords); + + using Kind = ShaderRecompiler::IR::DescriptorBindingKind; + auto Has = [&](Kind kind) { + return ShaderRecompiler::IR::FindBinding(compiled.program.bindings, kind) != + nullptr; + }; + VulkanHarness::Image sampled_image; + VulkanHarness::Image storage_image; + VulkanHarness::Image storage_image_uint; + VulkanHarness::Buffer gds_buffer; + VkSampler sampler = VK_NULL_HANDLE; + const bool needs_sampled_image = + Has(Kind::Sampled2D) || Has(Kind::Sampled2DArray) || + Has(Kind::Sampled3D) || Has(Kind::SampledUint2D) || + Has(Kind::SampledUint2DArray) || Has(Kind::SampledUint3D); + const bool needs_storage_image = + Has(Kind::Storage2D) || Has(Kind::Storage2DArray) || + Has(Kind::Storage3D) || Has(Kind::StorageUint2D) || + Has(Kind::StorageUint2DArray) || Has(Kind::StorageUint3D); + const bool needs_sampler = Has(Kind::Samplers); + const bool needs_gds = Has(Kind::Gds); + if (needs_gds) { + const auto gds_dwords = std::max( + {test.gds_initial.size(), test.expected_gds.size(), 1u}); + gds_buffer = + vulkan->CreateStorageBuffer(test.name, test.gds_initial, gds_dwords); + } + if (needs_sampled_image) { + if (!test.sampled_image_rgba_mips.empty()) { + sampled_image = vulkan->CreateImage2DMips( + test.name, test.image_width, test.image_height, + VK_FORMAT_R32G32B32A32_SFLOAT, VK_IMAGE_USAGE_SAMPLED_BIT, + test.sampled_image_rgba_mips, 4, + VK_IMAGE_LAYOUT_SHADER_READ_ONLY_OPTIMAL); + } else { + sampled_image = vulkan->CreateImage2D( + test.name, test.image_width, test.image_height, + test.sampled_image_format, VK_IMAGE_USAGE_SAMPLED_BIT, + test.sampled_image_rgba, test.sampled_image_dwords_per_pixel, + VK_IMAGE_LAYOUT_SHADER_READ_ONLY_OPTIMAL); + } + } + if (needs_storage_image) { + storage_image = vulkan->CreateImage2D( + test.name, test.image_width, test.image_height, + test.storage_image_format, VK_IMAGE_USAGE_STORAGE_BIT, + test.storage_image_rgba, test.storage_image_dwords_per_pixel, + VK_IMAGE_LAYOUT_GENERAL); + storage_image_uint = vulkan->CreateImage2D( + test.name, test.image_width, test.image_height, VK_FORMAT_R32_UINT, + VK_IMAGE_USAGE_STORAGE_BIT, test.storage_image_r32ui, 1, + VK_IMAGE_LAYOUT_GENERAL); + } + if (needs_sampler) { + sampler = vulkan->CreateNearestSampler(test.name); + } + + vulkan->Dispatch(test, compiled, buffer, needs_gds ? &gds_buffer : nullptr, + needs_sampled_image ? &sampled_image : nullptr, + needs_storage_image ? &storage_image : nullptr, + needs_storage_image ? &storage_image_uint : nullptr, + sampler); + auto actual = vulkan->ReadBuffer(test.name, buffer, test.expected.size()); + if (!test.expected_gds.empty()) { + const auto gds_actual = + vulkan->ReadBuffer(test.name, gds_buffer, test.expected_gds.size()); + CompareWords(test, "GDS readback", test.expected_gds, gds_actual); + } + if (!test.expected_storage_image_rgba.empty()) { + auto image_actual = vulkan->ReadImage(test.name, &storage_image); + image_actual.resize(test.expected_storage_image_rgba.size()); + CompareWords(test, "storage image readback", + test.expected_storage_image_rgba, image_actual); + } + if (!test.expected_storage_image_r32ui.empty()) { + auto image_actual = vulkan->ReadImage(test.name, &storage_image_uint); + image_actual.resize(test.expected_storage_image_r32ui.size()); + CompareWords(test, "uint storage image readback", + test.expected_storage_image_r32ui, image_actual); + } + if (sampler != VK_NULL_HANDLE) { + vkDestroySampler(vulkan->Device(), sampler, nullptr); + } + vulkan->DestroyImage(&sampled_image); + vulkan->DestroyImage(&storage_image); + vulkan->DestroyImage(&storage_image_uint); + vulkan->DestroyBuffer(&gds_buffer); + vulkan->DestroyBuffer(&buffer); + CompareWords(test, "readback", test.expected, actual); + std::printf("[compute] %-32s ok\n", test.name); +} + +void RunGraphicsCase(VulkanHarness *vulkan, const GraphicsCase &test) { + auto compiled = CompileFragmentCase(test); + auto actual = vulkan->RenderFragment(test, compiled); + CompareGraphicsWords(test, actual); + std::printf("[graphics] %-31s ok\n", test.name); +} + +enum class CoverageClass { + Covered, + ControlOrMarker, + NeedsAluCase, + NeedsFloatCase, + NeedsMemoryCase, + NeedsImageCase, + NeedsGraphicsStageCase, +}; + +bool IsCovered(const std::set &covered, ShaderOpcode opcode) { + return covered.find(opcode) != covered.end(); +} + +CoverageClass ClassifyOpcode(ShaderOpcode opcode, + const std::set &covered) { + using ShaderRecompiler::Decoder::Opcode; + + if (IsCovered(covered, opcode)) { + return CoverageClass::Covered; + } + + switch (opcode) { + case Opcode::SGetpcB64: + case Opcode::SSetpcB64: + case Opcode::SNop: + case Opcode::SWaitcnt: + case Opcode::SBarrier: + case Opcode::SBranch: + case Opcode::SCbranchScc0: + case Opcode::SCbranchScc1: + case Opcode::SCbranchVccz: + case Opcode::SCbranchVccnz: + case Opcode::SCbranchExecz: + case Opcode::SCbranchExecnz: + case Opcode::SSendmsg: + case Opcode::SSetregB32: + case Opcode::SSleep: + case Opcode::STtraceData: + case Opcode::SInstPrefetch: + case Opcode::SEndpgm: + return CoverageClass::ControlOrMarker; + + case Opcode::VAddF32: + case Opcode::VSubF32: + case Opcode::VSubrevF32: + case Opcode::VMulF32: + case Opcode::VMacF32: + case Opcode::VMadmkF32: + case Opcode::VMadakF32: + case Opcode::VMinF32: + case Opcode::VMaxF32: + case Opcode::VMadF32: + case Opcode::VFmaF32: + case Opcode::VMin3F32: + case Opcode::VMax3F32: + case Opcode::VMed3F32: + case Opcode::VDot2cF32F16: + case Opcode::VCvtF32I32: + case Opcode::VCvtF32U32: + case Opcode::VCvtU32F32: + case Opcode::VCvtI32F32: + case Opcode::VCvtF16F32: + case Opcode::VCvtF32F16: + case Opcode::VCvtU16F16: + case Opcode::VCvtRpiI32F32: + case Opcode::VCvtFlrI32F32: + case Opcode::VCvtOffF32I4: + case Opcode::VCvtF32Ubyte0: + case Opcode::VCvtF32Ubyte1: + case Opcode::VCvtF32Ubyte2: + case Opcode::VCvtF32Ubyte3: + case Opcode::VRcpF32: + case Opcode::VFractF32: + case Opcode::VTruncF32: + case Opcode::VCeilF32: + case Opcode::VRndneF32: + case Opcode::VFloorF32: + case Opcode::VExpF32: + case Opcode::VLogF32: + case Opcode::VRsqF32: + case Opcode::VSqrtF32: + case Opcode::VSinF32: + case Opcode::VCosF32: + case Opcode::VCubeidF32: + case Opcode::VCubescF32: + case Opcode::VCubetcF32: + case Opcode::VCubemaF32: + case Opcode::VLdexpF32: + case Opcode::VCvtPkU8F32: + case Opcode::VCvtPknormI16F32: + case Opcode::VCvtPknormU16F32: + case Opcode::VCvtPkrtzF16F32: + case Opcode::VPkAddF16: + case Opcode::VPkMulF16: + case Opcode::VPkMinF16: + case Opcode::VPkMaxF16: + case Opcode::VPkFmaF16: + case Opcode::VAddF16: + case Opcode::VSubF16: + case Opcode::VSubrevF16: + case Opcode::VMulF16: + case Opcode::VMaxF16: + case Opcode::VMinF16: + case Opcode::VMin3F16: + case Opcode::VMax3F16: + case Opcode::VMed3F16: + case Opcode::VRcpF16: + case Opcode::VRsqF16: + case Opcode::VLogF16: + case Opcode::VExpF16: + case Opcode::VMadMixloF16: + case Opcode::VMadMixhiF16: + case Opcode::DsMinF32: + case Opcode::DsMaxF32: + case Opcode::VCmpFF32: + case Opcode::VCmpLtF32: + case Opcode::VCmpEqF32: + case Opcode::VCmpLeF32: + case Opcode::VCmpGtF32: + case Opcode::VCmpLgF32: + case Opcode::VCmpGeF32: + case Opcode::VCmpOF32: + case Opcode::VCmpUF32: + case Opcode::VCmpNgeF32: + case Opcode::VCmpNlgF32: + case Opcode::VCmpNgtF32: + case Opcode::VCmpNleF32: + case Opcode::VCmpNeqF32: + case Opcode::VCmpNltF32: + case Opcode::VCmpTruF32: + case Opcode::VCmpxLtF32: + case Opcode::VCmpxEqF32: + case Opcode::VCmpxLeF32: + case Opcode::VCmpxGtF32: + case Opcode::VCmpxLgF32: + case Opcode::VCmpxGeF32: + case Opcode::VCmpxNgeF32: + case Opcode::VCmpxNlgF32: + case Opcode::VCmpxNgtF32: + case Opcode::VCmpxNleF32: + case Opcode::VCmpxNeqF32: + case Opcode::VCmpxNltF32: + case Opcode::VCmpClassF32: + case Opcode::VCmpLtF16: + case Opcode::VCmpEqF16: + case Opcode::VCmpLeF16: + case Opcode::VCmpGtF16: + case Opcode::VCmpLgF16: + case Opcode::VCmpGeF16: + case Opcode::VCmpNeqF16: + case Opcode::VCmpxLtF16: + case Opcode::VCmpxEqF16: + case Opcode::VCmpxLeF16: + case Opcode::VCmpxGtF16: + case Opcode::VCmpxGeF16: + case Opcode::VCmpxNeqF16: + case Opcode::VCmpxNltF16: + return CoverageClass::NeedsFloatCase; + + case Opcode::SLoadDword: + case Opcode::SLoadDwordx2: + case Opcode::SLoadDwordx4: + case Opcode::SLoadDwordx8: + case Opcode::SLoadDwordx16: + case Opcode::SBufferLoadDword: + case Opcode::SBufferLoadDwordx2: + case Opcode::SBufferLoadDwordx4: + case Opcode::SBufferLoadDwordx8: + case Opcode::SBufferLoadDwordx16: + case Opcode::BufferLoadFormatX: + case Opcode::BufferLoadFormatXy: + case Opcode::BufferLoadFormatXyz: + case Opcode::BufferLoadFormatXyzw: + case Opcode::BufferStoreFormatX: + case Opcode::BufferStoreFormatXy: + case Opcode::BufferStoreFormatXyz: + case Opcode::BufferStoreFormatXyzw: + case Opcode::BufferLoadUbyte: + case Opcode::BufferLoadSbyte: + case Opcode::BufferLoadUshort: + case Opcode::BufferLoadSshort: + case Opcode::BufferLoadDwordx2: + case Opcode::BufferLoadDwordx3: + case Opcode::BufferLoadDwordx4: + case Opcode::BufferStoreByte: + case Opcode::BufferStoreShort: + case Opcode::BufferStoreDwordx2: + case Opcode::BufferStoreDwordx3: + case Opcode::BufferStoreDwordx4: + case Opcode::TBufferLoadFormatX: + case Opcode::TBufferLoadFormatXy: + case Opcode::TBufferLoadFormatXyz: + case Opcode::TBufferLoadFormatXyzw: + case Opcode::TBufferStoreFormatX: + case Opcode::TBufferStoreFormatXy: + case Opcode::TBufferStoreFormatXyz: + case Opcode::TBufferStoreFormatXyzw: + case Opcode::BufferAtomicSwap: + case Opcode::BufferAtomicAdd: + case Opcode::BufferAtomicSub: + case Opcode::BufferAtomicSMin: + case Opcode::BufferAtomicUMin: + case Opcode::BufferAtomicSMax: + case Opcode::BufferAtomicUMax: + case Opcode::BufferAtomicAnd: + case Opcode::BufferAtomicOr: + case Opcode::BufferAtomicXor: + case Opcode::FlatLoadUbyte: + case Opcode::FlatLoadSbyte: + case Opcode::FlatLoadUshort: + case Opcode::FlatLoadSshort: + case Opcode::FlatLoadDword: + case Opcode::FlatLoadDwordx2: + case Opcode::FlatLoadDwordx3: + case Opcode::FlatLoadDwordx4: + case Opcode::FlatStoreByte: + case Opcode::FlatStoreShort: + case Opcode::FlatStoreDword: + case Opcode::FlatStoreDwordx2: + case Opcode::FlatStoreDwordx3: + case Opcode::FlatStoreDwordx4: + case Opcode::DsAddU32: + case Opcode::DsAddRtnU32: + case Opcode::DsSubU32: + case Opcode::DsSubRtnU32: + case Opcode::DsMinI32: + case Opcode::DsMinRtnI32: + case Opcode::DsMaxI32: + case Opcode::DsMaxRtnI32: + case Opcode::DsMinU32: + case Opcode::DsMinRtnU32: + case Opcode::DsMaxU32: + case Opcode::DsMaxRtnU32: + case Opcode::DsAndB32: + case Opcode::DsAndRtnB32: + case Opcode::DsOrB32: + case Opcode::DsOrRtnB32: + case Opcode::DsXorB32: + case Opcode::DsXorRtnB32: + case Opcode::DsWrxchgRtnB32: + case Opcode::DsSwizzleB32: + case Opcode::DsReadSbyte: + case Opcode::DsReadUbyte: + case Opcode::DsReadSshort: + case Opcode::DsReadUshort: + case Opcode::DsRead2B32: + case Opcode::DsReadB32: + case Opcode::DsReadB64: + case Opcode::DsRead2B64: + case Opcode::DsReadB96: + case Opcode::DsReadB128: + case Opcode::DsWriteByte: + case Opcode::DsWriteShort: + case Opcode::DsWrite2B32: + case Opcode::DsWrite2St64B32: + case Opcode::DsWriteB32: + case Opcode::DsWriteB64: + case Opcode::DsWriteB96: + case Opcode::DsWriteB128: + case Opcode::DsWriteAddtidB32: + case Opcode::DsReadAddtidB32: + return CoverageClass::NeedsMemoryCase; + + case Opcode::ImageGetResinfo: + case Opcode::ImageGetLod: + case Opcode::ImageLoad: + case Opcode::ImageLoadMip: + case Opcode::ImageStore: + case Opcode::ImageStoreMip: + case Opcode::ImageAtomicAdd: + case Opcode::ImageAtomicUMin: + case Opcode::ImageAtomicAnd: + case Opcode::ImageAtomicOr: + case Opcode::ImageAtomicXor: + case Opcode::ImageSample: + case Opcode::ImageGather4Lz: + case Opcode::ImageGather4C: + case Opcode::ImageGather4CLz: + case Opcode::ImageGather4LzO: + case Opcode::ImageGather4CO: + case Opcode::ImageGather4CLzO: + return CoverageClass::NeedsImageCase; + + case Opcode::VInterpP1F32: + case Opcode::VInterpP2F32: + case Opcode::VInterpMovF32: + case Opcode::Exp: + return CoverageClass::NeedsGraphicsStageCase; + + default: + return CoverageClass::NeedsAluCase; + } +} + +const char *CoverageClassName(CoverageClass status) { + switch (status) { + case CoverageClass::Covered: + return "covered"; + case CoverageClass::ControlOrMarker: + return "control"; + case CoverageClass::NeedsAluCase: + return "alu"; + case CoverageClass::NeedsFloatCase: + return "float"; + case CoverageClass::NeedsMemoryCase: + return "memory"; + case CoverageClass::NeedsImageCase: + return "image"; + case CoverageClass::NeedsGraphicsStageCase: + return "graphics"; + default: + return "unknown"; + } +} + +void PrintPendingOpcodes(CoverageClass status, + const std::vector &opcodes) { + if (opcodes.empty()) { + return; + } + + std::printf("[coverage] pending_%s:", CoverageClassName(status)); + for (auto opcode : opcodes) { + const auto name = ShaderRecompiler::Decoder::OpcodeToString(opcode); + std::printf(" %s", name.c_str()); + } + std::printf("\n"); +} + +void CheckOpcodeCoverage(const std::vector &tests, + const std::vector &graphics_tests) { + using ShaderRecompiler::Decoder::Opcode; + + std::set covered; + for (const auto &test : tests) { + for (auto opcode : test.opcodes) { + covered.insert(opcode); + } + } + for (const auto &test : graphics_tests) { + for (auto opcode : test.opcodes) { + covered.insert(opcode); + } + } + + uint32_t counts[7] = {}; + std::vector pending[7]; + for (auto value = static_cast(Opcode::SMovB32); + value <= static_cast(Opcode::Exp); value++) { + const auto opcode = static_cast(value); + const auto status = ClassifyOpcode(opcode, covered); + counts[static_cast(status)]++; + if (status != CoverageClass::Covered && + status != CoverageClass::ControlOrMarker) { + pending[static_cast(status)].push_back(opcode); + } + } + + std::printf( + "[coverage] decoder opcodes: covered=%u control=%u alu_pending=%u " + "float_pending=%u memory_pending=%u image_pending=%u " + "graphics_pending=%u\n", + counts[static_cast(CoverageClass::Covered)], + counts[static_cast(CoverageClass::ControlOrMarker)], + counts[static_cast(CoverageClass::NeedsAluCase)], + counts[static_cast(CoverageClass::NeedsFloatCase)], + counts[static_cast(CoverageClass::NeedsMemoryCase)], + counts[static_cast(CoverageClass::NeedsImageCase)], + counts[static_cast(CoverageClass::NeedsGraphicsStageCase)]); + PrintPendingOpcodes( + CoverageClass::NeedsAluCase, + pending[static_cast(CoverageClass::NeedsAluCase)]); + PrintPendingOpcodes( + CoverageClass::NeedsFloatCase, + pending[static_cast(CoverageClass::NeedsFloatCase)]); + PrintPendingOpcodes( + CoverageClass::NeedsMemoryCase, + pending[static_cast(CoverageClass::NeedsMemoryCase)]); + PrintPendingOpcodes( + CoverageClass::NeedsImageCase, + pending[static_cast(CoverageClass::NeedsImageCase)]); + PrintPendingOpcodes( + CoverageClass::NeedsGraphicsStageCase, + pending[static_cast(CoverageClass::NeedsGraphicsStageCase)]); +} + +TestCase IntegerAddSubMul() { + using O = ShaderOpcode; + + std::vector code = { + EncodeSMovB32(0, InlineU32(7)), + EncodeSMovB32(1, InlineU32(3)), + EncodeSop2(0x00, 2, 0, 1), + EncodeSop2(0x01, 3, 2, InlineU32(1)), + EncodeSop2(0x26, 4, 3, InlineU32(2)), + EncodeVop1(0x01, 0, 4), + }; + AppendBufferStoreDword(&code, 0, 30); + AppendEnd(&code); + return {"IntegerAddSubMul", + code, + {}, + {18}, + {O::SMovB32, O::SAddU32, O::SSubU32, O::SMulI32, O::VMovB32, + O::BufferStoreDword, O::SEndpgm}}; +} + +TestCase BitwiseOps() { + using O = ShaderOpcode; + + std::vector code = { + EncodeSMovB32(0, InlineU32(60)), EncodeSMovB32(1, InlineU32(15)), + EncodeSop2(0x0e, 2, 0, 1), EncodeSop2(0x10, 3, 0, 1), + EncodeSop2(0x12, 4, 3, 2), EncodeVop1(0x37, 0, 4), + }; + AppendBufferStoreDword(&code, 0, 30); + AppendEnd(&code); + return {"BitwiseAndOrXorNot", + code, + {}, + {~0x33u}, + {O::SMovB32, O::SAndB32, O::SOrB32, O::SXorB32, O::VNotB32, + O::BufferStoreDword, O::SEndpgm}}; +} + +TestCase Shifts() { + using O = ShaderOpcode; + + std::vector code = { + EncodeSMovB32(0, InlineU32(3)), + EncodeSop2(0x1e, 1, 0, InlineU32(2)), + EncodeSop2(0x20, 2, 1, InlineU32(1)), + EncodeVop1(0x01, 0, 2), + }; + AppendBufferStoreDword(&code, 0, 30); + AppendEnd(&code); + return {"Shifts", + code, + {}, + {6}, + {O::SMovB32, O::SLshlB32, O::SLshrB32, O::VMovB32, + O::BufferStoreDword, O::SEndpgm}}; +} + +TestCase ScalarShiftCountsMaskLowBits() { + using O = ShaderOpcode; + + std::vector code; + code.push_back(EncodeSMovB32(0, InlineU32(1))); + AppendSMovLiteral(&code, 1, 0x80000000u); + code.push_back(EncodeSMovB32(2, InlineU32(32))); + code.push_back(EncodeSMovB32(3, InlineU32(33))); + code.push_back(EncodeSop2(0x1e, 10, 0, 2)); + code.push_back(EncodeSop2(0x1e, 11, 0, 3)); + code.push_back(EncodeSop2(0x20, 12, 1, 2)); + code.push_back(EncodeSop2(0x20, 13, 1, 3)); + code.push_back(EncodeSop2(0x22, 14, 1, 2)); + code.push_back(EncodeSop2(0x22, 15, 1, 3)); + + for (u32 i = 0; i < 6u; i++) { + AppendStoreSgpr(&code, 10u + i, i); + } + AppendEnd(&code); + + return {"ScalarShiftCountsMaskLowBits", + code, + {}, + {1, 2, 0x80000000u, 0x40000000u, 0x80000000u, 0xc0000000u}, + {O::SMovB32, O::SLshlB32, O::SLshrB32, O::SAshrI32, O::VMovB32, + O::BufferStoreDword, O::SEndpgm}}; +} + +TestCase Rdna2ScalarOpcodes() { + using O = ShaderOpcode; + + std::vector code; + code.push_back(EncodeSMovB32(2, InlineU32(7))); + code.push_back(EncodeSMovB32(106, InlineU32(16))); + code.push_back(EncodeSop1(0x1d, 106, InlineU32(0))); + AppendStoreSgpr(&code, 106, 0); + code.push_back(EncodeSopk(0x13, 106, 0x1019u)); + AppendStoreSgpr(&code, 106, 1); + code.push_back(EncodeSop2(0x02, 106, 2, 239u)); + AppendStoreSgpr(&code, 106, 2); + code.push_back(EncodeSopp(0x0e, 0)); + code.push_back(EncodeSop2(0x02, 106, 239u, 2)); + AppendStoreSgpr(&code, 106, 3); + AppendEnd(&code); + + return {"Rdna2ScalarOpcodes", + code, + {}, + {0x11u, 0x11u, 7u, 7u}, + {O::SMovB32, O::SBitset1B32, O::SSetregB32, O::SAddI32, O::SSleep, + O::VMovB32, O::BufferStoreDword, O::SEndpgm}}; +} + +TestCase ScalarExtendedArithmetic() { + using O = ShaderOpcode; + + std::vector code; + AppendSMovLiteral(&code, 0, 0xffffffffu); + code.push_back(EncodeSMovB32(1, InlineU32(1))); + code.push_back(EncodeSop2(0x00, 2, 0, 1)); + code.push_back(EncodeSMovB32(3, InlineU32(5))); + code.push_back(EncodeSMovB32(4, InlineU32(6))); + code.push_back(EncodeSop2(0x04, 5, 3, 4)); + code.push_back(EncodeSop2(0x02, 6, InlineU32(7), InlineU32(8))); + code.push_back(EncodeSop2(0x03, 7, InlineU32(7), InlineU32(9))); + AppendSMovLiteral(&code, 8, 0xfffffffbu); + code.push_back(EncodeSMovB32(9, InlineU32(3))); + code.push_back(EncodeSop2(0x06, 10, 8, 9)); + code.push_back(EncodeSop2(0x08, 11, 8, 9)); + AppendSMovLiteral(&code, 12, 0xfffffffeu); + code.push_back(EncodeSMovB32(13, InlineU32(3))); + code.push_back(EncodeSop2(0x07, 14, 12, 13)); + code.push_back(EncodeSop2(0x09, 15, 12, 13)); + code.push_back(EncodeSMovB32(16, 8)); + code.push_back(EncodeSop1(0x34, 16, 16)); + code.push_back(EncodeSopk(0x00, 17, 0xfff5u)); + code.push_back(EncodeSopk(0x10, 17, 0xfffdu)); + + const u32 results[] = {2, 5, 6, 7, 10, 11, 14, 15, 16, 17}; + for (u32 i = 0; i < static_cast(std::size(results)); i++) { + AppendStoreSgpr(&code, results[i], i); + } + AppendEnd(&code); + + return {"ScalarExtendedArithmetic", + code, + {}, + {0, 12, 15, 0xfffffffeu, 0xfffffffbu, 3, 3, 0xfffffffeu, 5, 33}, + {O::SMovB32, O::SAddU32, O::SAddcU32, O::SAddI32, O::SSubI32, + O::SMinI32, O::SMaxI32, O::SMinU32, O::SMaxU32, O::SAbsI32, + O::SMovkI32, O::SMulkI32, O::VMovB32, O::BufferStoreDword, + O::SEndpgm}}; +} + +TestCase ScalarArithmeticSccCarryBorrowOverflow() { + using O = ShaderOpcode; + + std::vector code; + auto set_scc = [&](bool value) { + code.push_back(EncodeSopc(0x06, InlineU32(1), InlineU32(value ? 1u : 0u))); + }; + auto capture_scc = [&](u32 dst_sgpr) { + code.push_back(EncodeSop2(0x0a, dst_sgpr, InlineU32(1), InlineU32(0))); + }; + auto append_case = [&](bool prior_scc, u32 instruction, u32 out_sgpr) { + set_scc(prior_scc); + code.push_back(instruction); + capture_scc(out_sgpr); + }; + + code.push_back(EncodeSMovB32(0, InlineU32(0))); + code.push_back(EncodeSMovB32(1, InlineU32(1))); + append_case(false, EncodeSop2(0x01, 10, 0, 1), 20); + + code.push_back(EncodeSMovB32(2, InlineU32(5))); + code.push_back(EncodeSMovB32(3, InlineU32(3))); + append_case(true, EncodeSop2(0x01, 11, 2, 3), 21); + + AppendSMovLiteral(&code, 4, 0x7fffffffu); + code.push_back(EncodeSMovB32(5, InlineU32(1))); + append_case(false, EncodeSop2(0x02, 12, 4, 5), 22); + + code.push_back(EncodeSMovB32(6, InlineU32(1))); + code.push_back(EncodeSMovB32(7, InlineU32(2))); + append_case(true, EncodeSop2(0x02, 13, 6, 7), 23); + + AppendSMovLiteral(&code, 8, 0x80000000u); + code.push_back(EncodeSMovB32(9, InlineU32(1))); + append_case(false, EncodeSop2(0x03, 14, 8, 9), 24); + + code.push_back(EncodeSMovB32(15, InlineU32(5))); + code.push_back(EncodeSMovB32(16, InlineU32(3))); + append_case(true, EncodeSop2(0x03, 17, 15, 16), 25); + + AppendSMovLiteral(&code, 18, 0x7fffffffu); + set_scc(false); + code.push_back(EncodeSopk(0x0f, 18, 1)); + capture_scc(26); + + code.push_back(EncodeSMovB32(19, InlineU32(4))); + set_scc(true); + code.push_back(EncodeSopk(0x0f, 19, 0xfffeu)); + capture_scc(27); + + for (u32 i = 0; i < 8u; i++) { + AppendStoreSgpr(&code, 20u + i, i); + } + AppendEnd(&code); + + return {"ScalarArithmeticSccCarryBorrowOverflow", + code, + {}, + {1, 0, 1, 0, 1, 0, 1, 0}, + {O::SMovB32, O::SSubU32, O::SAddI32, O::SSubI32, O::SCmpEqU32, + O::SCselectB32, O::VMovB32, O::BufferStoreDword, O::SEndpgm}}; +} + +TestCase ScalarMinMaxSccComparisonEdges() { + using O = ShaderOpcode; + + std::vector code; + auto set_scc = [&](bool value) { + code.push_back(EncodeSopc(0x06, InlineU32(1), InlineU32(value ? 1u : 0u))); + }; + auto capture_scc = [&](u32 dst_sgpr) { + code.push_back(EncodeSop2(0x0a, dst_sgpr, InlineU32(1), InlineU32(0))); + }; + auto append_case = [&](bool prior_scc, u32 instruction, u32 out_sgpr) { + set_scc(prior_scc); + code.push_back(instruction); + capture_scc(out_sgpr); + }; + + AppendSMovLiteral(&code, 0, 0xfffffffeu); + code.push_back(EncodeSMovB32(1, InlineU32(3))); + append_case(false, EncodeSop2(0x06, 10, 0, 1), 20); + append_case(true, EncodeSop2(0x06, 11, 1, 0), 21); + append_case(false, EncodeSop2(0x08, 12, 1, 0), 22); + append_case(true, EncodeSop2(0x08, 13, 0, 1), 23); + + code.push_back(EncodeSMovB32(2, InlineU32(2))); + code.push_back(EncodeSMovB32(3, InlineU32(3))); + append_case(false, EncodeSop2(0x07, 14, 2, 3), 24); + append_case(true, EncodeSop2(0x07, 15, 3, 2), 25); + append_case(false, EncodeSop2(0x09, 16, 3, 2), 26); + append_case(true, EncodeSop2(0x09, 17, 2, 3), 27); + + for (u32 i = 0; i < 8u; i++) { + AppendStoreSgpr(&code, 20u + i, i); + } + AppendEnd(&code); + + return {"ScalarMinMaxSccComparisonEdges", + code, + {}, + {1, 0, 1, 0, 1, 0, 1, 0}, + {O::SMovB32, O::SMinI32, O::SMaxI32, O::SMinU32, O::SMaxU32, + O::SCmpEqU32, O::SCselectB32, O::VMovB32, O::BufferStoreDword, + O::SEndpgm}}; +} + +TestCase ScalarAbsI32UpdatesScc() { + using O = ShaderOpcode; + + std::vector code; + auto set_scc = [&](bool value) { + code.push_back(EncodeSopc(0x06, InlineU32(1), InlineU32(value ? 1u : 0u))); + }; + auto capture_scc = [&](u32 dst_sgpr) { + code.push_back(EncodeSop2(0x0a, dst_sgpr, InlineU32(1), InlineU32(0))); + }; + + code.push_back(EncodeSMovB32(0, InlineU32(0))); + set_scc(true); + code.push_back(EncodeSop1(0x34, 1, 0)); + capture_scc(2); + + AppendSMovLiteral(&code, 3, 0xfffffffbu); + set_scc(false); + code.push_back(EncodeSop1(0x34, 4, 3)); + capture_scc(5); + + AppendStoreSgpr(&code, 1, 0); + AppendStoreSgpr(&code, 2, 1); + AppendStoreSgpr(&code, 4, 2); + AppendStoreSgpr(&code, 5, 3); + AppendEnd(&code); + + return {"ScalarAbsI32UpdatesScc", + code, + {}, + {0, 0, 5, 1}, + {O::SMovB32, O::SCmpEqU32, O::SAbsI32, O::SCselectB32, O::VMovB32, + O::BufferStoreDword, O::SEndpgm}}; +} + +TestCase ScalarShiftLeftAddSccCarryEdges() { + using O = ShaderOpcode; + + std::vector code; + auto set_scc = [&](bool value) { + code.push_back(EncodeSopc(0x06, InlineU32(1), InlineU32(value ? 1u : 0u))); + }; + auto capture_scc = [&](u32 dst_sgpr) { + code.push_back(EncodeSop2(0x0a, dst_sgpr, InlineU32(1), InlineU32(0))); + }; + auto append_case = [&](bool prior_scc, u32 instruction, u32 out_sgpr) { + set_scc(prior_scc); + code.push_back(instruction); + capture_scc(out_sgpr); + }; + + AppendSMovLiteral(&code, 0, 0x80000000u); + AppendSMovLiteral(&code, 1, 0x40000000u); + AppendSMovLiteral(&code, 2, 0x20000000u); + AppendSMovLiteral(&code, 3, 0x10000000u); + code.push_back(EncodeSMovB32(4, InlineU32(1))); + code.push_back(EncodeSMovB32(5, InlineU32(0))); + append_case(false, EncodeSop2(0x2e, 10, 0, 5), 20); + append_case(true, EncodeSop2(0x2e, 11, 4, 4), 21); + append_case(false, EncodeSop2(0x2f, 12, 1, 5), 22); + append_case(true, EncodeSop2(0x2f, 13, 4, 4), 23); + append_case(false, EncodeSop2(0x30, 14, 2, 5), 24); + append_case(true, EncodeSop2(0x30, 15, 4, 4), 25); + append_case(false, EncodeSop2(0x31, 16, 3, 5), 26); + append_case(true, EncodeSop2(0x31, 17, 4, 4), 27); + + for (u32 i = 0; i < 8u; i++) { + AppendStoreSgpr(&code, 20u + i, i); + } + AppendEnd(&code); + + return {"ScalarShiftLeftAddSccCarryEdges", + code, + {}, + {1, 0, 1, 0, 1, 0, 1, 0}, + {O::SMovB32, O::SLshl1AddU32, O::SLshl2AddU32, O::SLshl3AddU32, + O::SLshl4AddU32, O::SCmpEqU32, O::SCselectB32, O::VMovB32, + O::BufferStoreDword, O::SEndpgm}}; +} + +TestCase ScalarCompareOps() { + using O = ShaderOpcode; + + std::vector code; + AppendSMovLiteral(&code, 0, 0xfffffffeu); + code.push_back(EncodeSMovB32(1, InlineU32(1))); + code.push_back(EncodeSMovB32(2, InlineU32(5))); + code.push_back(EncodeSMovB32(3, InlineU32(2))); + code.push_back(EncodeSMovB32(4, InlineU32(3))); + AppendSMovLiteral(&code, 10, 0x00000001u); + code.push_back(EncodeSMovB32(11, InlineU32(0))); + code.push_back(EncodeSMovB32(12, InlineU32(2))); + code.push_back(EncodeSMovB32(13, InlineU32(0))); + + u32 dst = 20; + auto append_compare = [&](u32 opcode, u32 src0, u32 src1) { + code.push_back(EncodeSopc(opcode, src0, src1)); + code.push_back(EncodeSop2(0x0a, dst++, InlineU32(1), InlineU32(0))); + }; + + append_compare(0x00, 2, 2); + append_compare(0x01, 0, 1); + append_compare(0x02, 1, 0); + append_compare(0x03, 1, 1); + append_compare(0x04, 0, 1); + append_compare(0x05, 0, 0); + append_compare(0x07, 3, 4); + append_compare(0x08, 4, 3); + append_compare(0x09, 4, 4); + append_compare(0x0b, 3, 4); + append_compare(0x13, 10, 12); + + for (u32 i = 0; i < 11u; i++) { + AppendStoreSgpr(&code, 20u + i, i); + } + AppendEnd(&code); + + return {"ScalarCompareOps", + code, + {}, + std::vector(11, 1), + {O::SMovB32, O::SCmpEqI32, O::SCmpLgI32, O::SCmpGtI32, O::SCmpGeI32, + O::SCmpLtI32, O::SCmpLeI32, O::SCmpLgU32, O::SCmpGtU32, O::SCmpGeU32, + O::SCmpLeU32, O::SCmpLgU64, O::SCselectB32, O::VMovB32, + O::BufferStoreDword, O::SEndpgm}}; +} + +TestCase ScalarShiftAddAndMaskOps() { + using O = ShaderOpcode; + + std::vector code; + code.push_back(EncodeSMovB32(0, InlineU32(2))); + code.push_back(EncodeSMovB32(1, InlineU32(3))); + code.push_back(EncodeSop2(0x2e, 2, 0, 1)); + code.push_back(EncodeSop2(0x2f, 3, 0, 1)); + code.push_back(EncodeSop2(0x30, 4, 0, 1)); + code.push_back(EncodeSop2(0x31, 5, 0, 1)); + AppendSMovLiteral(&code, 6, 0xfffffff8u); + code.push_back(EncodeSop2(0x22, 7, 6, InlineU32(2))); + AppendSMovLiteral(&code, 8, 0xffffffffu); + code.push_back(EncodeSMovB32(9, InlineU32(2))); + code.push_back(EncodeSop2(0x35, 10, 8, 9)); + AppendSMovLiteral(&code, 12, 0x0f0f0f0fu); + AppendSMovLiteral(&code, 13, 0x00ff00ffu); + code.push_back(EncodeSop1(0x08, 14, 12)); + code.push_back(EncodeSop1(0x0a, 16, 12)); + + AppendStoreSgpr(&code, 2, 0); + AppendStoreSgpr(&code, 3, 1); + AppendStoreSgpr(&code, 4, 2); + AppendStoreSgpr(&code, 5, 3); + AppendStoreSgpr(&code, 7, 4); + AppendStoreSgpr(&code, 10, 5); + AppendStoreSgprPair(&code, 14, 6); + AppendStoreSgprPair(&code, 16, 8); + AppendEnd(&code); + + return {"ScalarShiftAddAndMaskOps", + code, + {}, + {7, 11, 19, 35, 0xfffffffeu, 1, 0xf0f0f0f0u, 0xff00ff00u, 0x0f0f0f0fu, + 0x00ff00ffu}, + {O::SMovB32, O::SLshl1AddU32, O::SLshl2AddU32, O::SLshl3AddU32, + O::SLshl4AddU32, O::SAshrI32, O::SMulHiU32, O::SNotB64, O::SWqmB64, + O::VMovB32, O::BufferStoreDword, O::SEndpgm}}; +} + +TestCase ScalarNotB64UpdatesScc() { + using O = ShaderOpcode; + + std::vector code; + auto set_scc = [&](bool value) { + code.push_back(EncodeSopc(0x06, InlineU32(1), InlineU32(value ? 1u : 0u))); + }; + auto capture_scc = [&](u32 dst_sgpr) { + code.push_back(EncodeSop2(0x0a, dst_sgpr, InlineU32(1), InlineU32(0))); + }; + + AppendSMovLiteral(&code, 0, 0xffffffffu); + AppendSMovLiteral(&code, 1, 0xffffffffu); + set_scc(true); + code.push_back(EncodeSop1(0x08, 2, 0)); + capture_scc(4); + + AppendSMovLiteral(&code, 6, 0xfffffffeu); + AppendSMovLiteral(&code, 7, 0xffffffffu); + set_scc(false); + code.push_back(EncodeSop1(0x08, 8, 6)); + capture_scc(10); + + AppendStoreSgpr(&code, 4, 0); + AppendStoreSgpr(&code, 10, 1); + AppendStoreSgprPair(&code, 2, 2); + AppendStoreSgprPair(&code, 8, 4); + AppendEnd(&code); + + return {"ScalarNotB64UpdatesScc", + code, + {}, + {0, 1, 0, 0, 1, 0}, + {O::SMovB32, O::SCmpEqU32, O::SNotB64, O::SCselectB32, O::VMovB32, + O::BufferStoreDword, O::SEndpgm}}; +} + +TestCase ScalarFlbitI32B64Gpu() { + using O = ShaderOpcode; + + std::vector code; + AppendSMovLiteral(&code, 0, 0x00000000u); + AppendSMovLiteral(&code, 1, 0x80000000u); + code.push_back(EncodeSop1(0x16, 20, 0)); + + AppendSMovLiteral(&code, 2, 0x00008000u); + AppendSMovLiteral(&code, 3, 0x00000000u); + code.push_back(EncodeSop1(0x16, 21, 2)); + + AppendSMovLiteral(&code, 4, 0x00000000u); + AppendSMovLiteral(&code, 5, 0x00000000u); + code.push_back(EncodeSop1(0x16, 22, 4)); + + AppendSMovLiteral(&code, 14, 0x00000008u); + AppendSMovLiteral(&code, 15, 0x00000000u); + code.push_back(EncodeSop1(0x16, 106, 14)); + + AppendStoreSgpr(&code, 20, 0); + AppendStoreSgpr(&code, 21, 1); + AppendStoreSgpr(&code, 22, 2); + AppendStoreSgpr(&code, 106, 3); + AppendEnd(&code); + + return {"ScalarFlbitI32B64Gpu", + code, + {}, + {0, 48, 0xffffffffu, 60}, + {O::SMovB32, O::SFlbitI32B64, O::VMovB32, O::BufferStoreDword, + O::SEndpgm}}; +} + +TestCase ScalarSaveExecOps() { + using O = ShaderOpcode; + + std::vector code; + AppendSMovLiteral(&code, 106, 0xffffffffu); + AppendSMovLiteral(&code, 107, 0xffffffffu); + code.push_back(0xbe80246au); + code.push_back(EncodeSop1(0x28, 2, 106)); + code.push_back(EncodeSMovB32(106, InlineU32(0))); + code.push_back(EncodeSMovB32(107, InlineU32(0))); + code.push_back(EncodeSop1(0x37, 4, 106)); + AppendSMovLiteral(&code, 106, 0x00000003u); + code.push_back(EncodeSop1(0x3c, 8, 106)); + code.push_back(EncodeSop1(0x04, 10, 126)); + code.push_back(EncodeSMovB32(106, InlineU32(1))); + code.push_back(EncodeSop1(0x44, 12, 106)); + code.push_back(EncodeSop1(0x04, 14, 126)); + AppendSMovLiteral(&code, 126, 0xffffffffu); + AppendSMovLiteral(&code, 127, 0xffffffffu); + + AppendStoreSgprPair(&code, 0, 0); + AppendStoreSgprPair(&code, 2, 2); + AppendStoreSgprPair(&code, 4, 4); + AppendStoreSgpr(&code, 8, 6); + AppendStoreSgprPair(&code, 10, 7); + AppendStoreSgpr(&code, 12, 9); + AppendStoreSgprPair(&code, 14, 10); + AppendStoreSgpr(&code, 253, 12); + AppendEnd(&code); + + return {"ScalarSaveExecOps", + code, + {}, + {0xffffffffu, 0xffffffffu, 0xffffffffu, 0xffffffffu, 0xffffffffu, + 0xffffffffu, 0xffffffffu, 0x00000003u, 0xffffffffu, 0x00000003u, + 0x00000002u, 0xffffffffu, 1}, + {O::SMovB32, O::SAndSaveexecB64, O::SOrn2SaveexecB64, + O::SAndn1SaveexecB64, O::SAndSaveexecB32, O::SAndn1SaveexecB32, + O::SMovB64, O::VMovB32, O::BufferStoreDword, O::SEndpgm}}; +} + +TestCase ScalarOrn2SaveexecUsesSourceOrNotExec() { + using O = ShaderOpcode; + + std::vector code; + AppendSMovLiteral(&code, 126, 0x0000000cu); + AppendSMovLiteral(&code, 127, 0x80000000u); + AppendSMovLiteral(&code, 0, 0x00000001u); + AppendSMovLiteral(&code, 1, 0x00000001u); + code.push_back(EncodeSop1(0x28, 2, 0)); + code.push_back(EncodeSop1(0x04, 4, 126)); + code.push_back(EncodeSMovB32(126, InlineU32(1))); + code.push_back(EncodeSMovB32(127, InlineU32(0))); + AppendStoreSgprPair(&code, 2, 0); + AppendStoreSgprPair(&code, 4, 2); + AppendStoreSgpr(&code, 253, 4); + AppendEnd(&code); + + return {"ScalarOrn2SaveexecUsesSourceOrNotExec", + code, + {}, + {0x0000000cu, 0x80000000u, 0xfffffff3u, 0x7fffffffu, 1}, + {O::SMovB32, O::SOrn2SaveexecB64, O::SMovB64, O::VMovB32, + O::BufferStoreDword, O::SEndpgm}}; +} + +TestCase ScalarGetpcWritesNextInstructionPc() { + using O = ShaderOpcode; + + std::vector code; + code.push_back(EncodeSop1(0x1f, 0, 0)); + AppendStoreSgprPair(&code, 0, 0); + AppendEnd(&code); + + return {"ScalarGetpcWritesNextInstructionPc", + code, + {}, + {4, 0}, + {O::SGetpcB64, O::VMovB32, O::BufferStoreDword, O::SEndpgm}}; +} + +TestCase ScalarBitfieldPack() { + using O = ShaderOpcode; + + std::vector code; + code.push_back(EncodeSMovB32(0, InlineU32(15))); + code.push_back(EncodeSop1(0x0b, 1, 0)); + AppendSMovLiteral(&code, 2, 0xf0000000u); + code.push_back(EncodeSop1(0x10, 3, 0)); + AppendSMovLiteral(&code, 19, 0xf0f00001u); + code.push_back(EncodeSop1(0x0f, 20, 19)); + code.push_back(EncodeSop2(0x0a, 21, InlineU32(1), InlineU32(0))); + code.push_back(EncodeSMovB32(22, InlineU32(0))); + code.push_back(EncodeSop1(0x0f, 23, 22)); + code.push_back(EncodeSop2(0x0a, 24, InlineU32(1), InlineU32(0))); + code.push_back(EncodeSMovB32(4, InlineU32(3))); + code.push_back(EncodeSop1(0x3b, 5, 4)); + code.push_back(EncodeSop2(0x24, 7, InlineU32(4), InlineU32(8))); + AppendSMovLiteral(&code, 8, 0x00f00000u); + AppendSMovLiteral(&code, 9, 0x00040014u); + code.push_back(EncodeSop2(0x27, 10, 8, 9)); + AppendSMovLiteral(&code, 11, 0xaaaabbbbu); + AppendSMovLiteral(&code, 12, 0xccccddddu); + code.push_back(EncodeSop2(0x32, 13, 11, 12)); + code.push_back(EncodeSop2(0x33, 14, 11, 12)); + code.push_back(EncodeSop2(0x34, 15, 11, 12)); + code.push_back(EncodeSopc(0x0d, InlineU32(4), InlineU32(2))); + code.push_back(EncodeSop2(0x0a, 17, InlineU32(1), InlineU32(0))); + code.push_back(EncodeSopc(0x0c, InlineU32(4), InlineU32(1))); + code.push_back(EncodeSop2(0x0a, 18, InlineU32(1), InlineU32(0))); + + AppendStoreSgpr(&code, 1, 0); + AppendStoreSgpr(&code, 3, 1); + AppendStoreSgpr(&code, 20, 2); + AppendStoreSgpr(&code, 21, 3); + AppendStoreSgpr(&code, 23, 4); + AppendStoreSgpr(&code, 24, 5); + AppendStoreSgprPair(&code, 5, 6); + AppendStoreSgpr(&code, 7, 8); + AppendStoreSgpr(&code, 10, 9); + AppendStoreSgpr(&code, 13, 10); + AppendStoreSgpr(&code, 14, 11); + AppendStoreSgpr(&code, 15, 12); + AppendStoreSgpr(&code, 17, 13); + AppendStoreSgpr(&code, 18, 14); + AppendEnd(&code); + + return {"ScalarBitfieldPack", + code, + {}, + {0xf0000000u, 8, 9, 1, 0, 0, 0x0000000fu, 0, 0x00000f00u, 0x0000000fu, + 0xddddbbbbu, 0xccccbbbbu, 0xccccaaaau, 1, 1}, + {O::SMovB32, O::SBrevB32, O::SBcnt1I32B32, O::SBcnt1I32B64, + O::SBitreplicateB64B32, O::SBfmB32, O::SBfeU32, O::SPackLlB32B16, + O::SPackLhB32B16, O::SPackHhB32B16, O::SBitcmp0B32, O::SBitcmp1B32, + O::SCselectB32, O::VMovB32, O::BufferStoreDword, O::SEndpgm}}; +} + +TestCase ScalarBrevB32PreservesScc() { + using O = ShaderOpcode; + + std::vector code; + auto set_scc = [&](bool value) { + code.push_back(EncodeSopc(0x06, InlineU32(1), InlineU32(value ? 1u : 0u))); + }; + auto capture_scc = [&](u32 dst_sgpr) { + code.push_back(EncodeSop2(0x0a, dst_sgpr, InlineU32(1), InlineU32(0))); + }; + + code.push_back(EncodeSMovB32(0, InlineU32(1))); + set_scc(false); + code.push_back(EncodeSop1(0x0b, 1, 0)); + capture_scc(2); + + code.push_back(EncodeSMovB32(3, InlineU32(0))); + set_scc(true); + code.push_back(EncodeSop1(0x0b, 4, 3)); + capture_scc(5); + + AppendStoreSgpr(&code, 1, 0); + AppendStoreSgpr(&code, 2, 1); + AppendStoreSgpr(&code, 4, 2); + AppendStoreSgpr(&code, 5, 3); + AppendEnd(&code); + + return {"ScalarBrevB32PreservesScc", + code, + {}, + {0x80000000u, 0, 0, 1}, + {O::SMovB32, O::SCmpEqU32, O::SBrevB32, O::SCselectB32, O::VMovB32, + O::BufferStoreDword, O::SEndpgm}}; +} + +TestCase BitfieldExtractWidthPastEndEdges() { + using O = ShaderOpcode; + + std::vector code; + AppendSMovLiteral(&code, 0, 0xf0000000u); + AppendSMovLiteral(&code, 1, (20u << 16u) | 20u); + code.push_back(EncodeSop2(0x27, 2, 0, 1)); + AppendSMovLiteral(&code, 3, 0x80000000u); + AppendSMovLiteral(&code, 4, (7u << 16u) | 31u); + code.push_back(EncodeSop2(0x27, 5, 3, 4)); + AppendVMovLiteral(&code, 6, 0xf0000000u); + AppendVMovU32(&code, 7, 28); + AppendVMovU32(&code, 8, 8); + AppendVop3(&code, 0x148, 9, Vgpr(6), Vgpr(7), Vgpr(8)); + AppendStoreSgpr(&code, 2, 0); + AppendStoreSgpr(&code, 5, 1); + AppendStoreVgpr(&code, 9, 2); + AppendEnd(&code); + + return {"BitfieldExtractWidthPastEndEdges", + code, + {}, + {0x00000f00u, 1, 0x0000000fu}, + {O::SMovB32, O::SBfeU32, O::VMovB32, O::VBfeU32, O::BufferStoreDword, + O::SEndpgm}}; +} + +TestCase Scalar64BitOps() { + using O = ShaderOpcode; + + std::vector code; + AppendSMovLiteral(&code, 0, 0x0f0f0f0fu); + AppendSMovLiteral(&code, 1, 0x00ff00ffu); + AppendSMovLiteral(&code, 2, 0x33333333u); + AppendSMovLiteral(&code, 3, 0x0f0f0f0fu); + code.push_back(EncodeSop2(0x0f, 4, 0, 2)); + code.push_back(EncodeSop2(0x15, 6, 0, 2)); + code.push_back(EncodeSop2(0x11, 8, 0, 2)); + code.push_back(EncodeSop2(0x17, 10, 0, 2)); + code.push_back(EncodeSop2(0x13, 12, 0, 2)); + code.push_back(EncodeSop2(0x19, 14, 0, 2)); + code.push_back(EncodeSop2(0x1b, 16, 0, 2)); + code.push_back(EncodeSop2(0x1d, 18, 0, 2)); + code.push_back(EncodeSop1(0x04, 20, 0)); + code.push_back(EncodeSop2(0x1f, 22, 0, InlineU32(4))); + code.push_back(EncodeSop2(0x21, 24, 0, InlineU32(8))); + code.push_back(EncodeSop2(0x25, 26, InlineU32(36), InlineU32(4))); + code.push_back(EncodeSop2(0x21, 34, 193u, InlineU32(1))); + AppendSMovLiteral(&code, 28, 0x000c0004u); + code.push_back(EncodeSop2(0x29, 30, 0, 28)); + code.push_back(EncodeSopc(0x06, 0, 0)); + code.push_back(EncodeSop2(0x0b, 32, 0, 2)); + + const u32 result_pairs[] = {4, 6, 8, 10, 12, 14, 16, 18, + 20, 22, 24, 26, 34, 30, 32}; + u32 out = 0; + for (auto sgpr : result_pairs) { + AppendStoreSgprPair(&code, sgpr, out); + out += 2; + } + AppendEnd(&code); + + return {"Scalar64BitOps", + code, + {}, + {0x03030303u, 0x000f000fu, 0x0c0c0c0cu, 0x00f000f0u, 0x3f3f3f3fu, + 0x0fff0fffu, 0xcfcfcfcfu, 0xf0fff0ffu, 0x3c3c3c3cu, 0x0ff00ff0u, + 0xfcfcfcfcu, 0xfff0fff0u, 0xc0c0c0c0u, 0xf000f000u, 0xc3c3c3c3u, + 0xf00ff00fu, 0x0f0f0f0fu, 0x00ff00ffu, 0xf0f0f0f0u, 0x0ff00ff0u, + 0xff0f0f0fu, 0x0000ff00u, 0xfffffff0u, 0x000000ffu, 0xffffffffu, + 0x7fffffffu, 0x000000f0u, 0, 0x0f0f0f0fu, 0x00ff00ffu}, + {O::SMovB32, O::SMovB64, O::SAndB64, O::SAndn2B64, O::SOrB64, + O::SOrn2B64, O::SXorB64, O::SNandB64, O::SNorB64, O::SXnorB64, + O::SLshlB64, O::SLshrB64, O::SBfmB64, O::SBfeU64, O::SCmpEqU32, + O::SCselectB64, O::VMovB32, O::BufferStoreDword, O::SEndpgm}}; +} + +TestCase ScalarAndn2B64SccBranch() { + using O = ShaderOpcode; + + std::vector code = { + EncodeSMovB32(0, InlineU32(3)), + EncodeSMovB32(1, InlineU32(0)), + EncodeSMovB32(2, InlineU32(1)), + EncodeSMovB32(3, InlineU32(0)), + EncodeSop2(0x15, 4, 0, 2), + EncodeVop1(0x01, 0, InlineU32(1)), + EncodeSopp(0x04, 1), + EncodeVop1(0x01, 0, InlineU32(7)), + }; + AppendStoreVgpr(&code, 0, 0); + AppendEnd(&code); + + return {"ScalarAndn2B64SccBranch", + code, + {}, + {7}, + {O::SMovB32, O::SAndn2B64, O::VMovB32, O::SCbranchScc0, + O::BufferStoreDword, O::SEndpgm}}; +} + +TestCase ScalarLiteral() { + using O = ShaderOpcode; + + std::vector code; + AppendSMovLiteral(&code, 0, 0x12345678u); + code.push_back(EncodeVop1(0x01, 0, 0)); + AppendBufferStoreDword(&code, 0, 30); + AppendEnd(&code); + return {"ScalarLiteral", + code, + {}, + {0x12345678u}, + {O::SMovB32, O::VMovB32, O::BufferStoreDword, O::SEndpgm}}; +} + +TestCase VectorMoves() { + using O = ShaderOpcode; + + std::vector code = { + EncodeVop1(0x00, 0, 0), + EncodeVop1(0x01, 0, InlineU32(5)), + EncodeVop1(0x01, 2, Vgpr(0)), + }; + AppendBufferStoreDword(&code, 2, 30); + AppendEnd(&code); + return {"VectorRegisterMoves", + code, + {}, + {5}, + {O::VNop, O::VMovB32, O::BufferStoreDword, O::SEndpgm}}; +} + +TestCase VectorVop3MoveAppliesFloatSourceModifiers() { + using O = ShaderOpcode; + + std::vector code; + AppendSMovLiteral(&code, 24, 0x40000000u); + AppendVop3(&code, 0x181, 1, 24, 0, 0, 0, 0, false, 0, 0x1); + AppendStoreVgpr(&code, 1, 0); + AppendEnd(&code); + + TestCase test; + test.name = "VectorVop3MoveAppliesFloatSourceModifiers"; + test.code = code; + test.expected = {0xc0000000u}; + test.opcodes = {O::SMovB32, O::VMovB32, O::BufferStoreDword, O::SEndpgm}; + test.required_spirv = {"OpFNegate"}; + return test; +} + +TestCase VectorIntegerOps() { + using O = ShaderOpcode; + + std::vector code; + AppendVMovLiteral(&code, 0, 0xfffffff8u); + code.push_back(EncodeVop1(0x01, 1, InlineU32(5))); + code.push_back(EncodeVop1(0x01, 4, InlineU32(7))); + AppendVMovLiteral(&code, 10, 0x01000001u); + code.push_back(EncodeVop1(0x01, 11, InlineU32(2))); + AppendVMovLiteral(&code, 12, 0x0f0f0f0fu); + AppendVMovLiteral(&code, 13, 0x33333333u); + AppendVMovLiteral(&code, 14, 0xfffffff0u); + code.push_back(EncodeVop1(0x01, 15, InlineU32(4))); + code.push_back(EncodeVop1(0x01, 29, InlineU32(15))); + + code.push_back(EncodeVop2(0x11, 2, Vgpr(0), 1)); + code.push_back(EncodeVop2(0x12, 3, Vgpr(0), 1)); + code.push_back(EncodeVop2(0x13, 5, Vgpr(0), 1)); + code.push_back(EncodeVop2(0x14, 6, Vgpr(0), 1)); + code.push_back(EncodeVop2(0x25, 7, Vgpr(4), 1)); + code.push_back(EncodeVop2(0x26, 8, Vgpr(4), 1)); + code.push_back(EncodeVop2(0x27, 9, InlineU32(3), 4)); + code.push_back(EncodeVop2(0x26, 35, 249, 11)); + code.push_back(EncodeVop2Sdwa(12, 6, 0, 0, 6)); + code.push_back(EncodeVop2(0x27, 36, 249, 11)); + code.push_back(EncodeVop2Sdwa(12, 6, 0, 0, 6)); + code.push_back(EncodeVop2(0x0b, 16, Vgpr(10), 11)); + code.push_back(EncodeVop2(0x09, 37, Vgpr(0), 1)); + code.push_back(EncodeVop2(0x1b, 17, Vgpr(12), 13)); + code.push_back(EncodeVop2(0x1c, 18, Vgpr(12), 13)); + code.push_back(EncodeVop2(0x1d, 19, Vgpr(12), 13)); + code.push_back(EncodeVop2(0x1e, 20, Vgpr(12), 13)); + code.push_back(EncodeVop2(0x19, 21, Vgpr(11), 15)); + code.push_back(EncodeVop2(0x1a, 22, InlineU32(4), 11)); + code.push_back(EncodeVop2(0x15, 23, Vgpr(14), 15)); + code.push_back(EncodeVop2(0x16, 24, InlineU32(4), 14)); + code.push_back(EncodeVop2(0x17, 25, Vgpr(14), 15)); + code.push_back(EncodeVop2(0x18, 26, InlineU32(4), 14)); + code.push_back(EncodeVop1(0x37, 27, Vgpr(12))); + code.push_back(EncodeVop1(0x38, 28, Vgpr(29))); + code.push_back(EncodeVop1(0x3a, 30, Vgpr(11))); + code.push_back(EncodeVop1(0x39, 32, Vgpr(11))); + code.push_back(EncodeVopc(0xc2, Vgpr(4), 4)); + code.push_back(EncodeVop2(0x01, 33, InlineU32(3), 4)); + code.push_back(EncodeVop2(0x11, 34, 249, 4)); + code.push_back(EncodeVop2Sdwa(0)); + + const u32 results[] = {2, 3, 5, 6, 7, 8, 9, 35, 36, 16, 37, 17, 18, 19, + 20, 21, 22, 23, 24, 25, 26, 27, 28, 30, 32, 33, 34}; + for (u32 i = 0; i < static_cast(std::size(results)); i++) { + AppendStoreVgpr(&code, results[i], i); + } + AppendEnd(&code); + + return {"VectorIntegerOps", + code, + {}, + {0xfffffff8u, 5, 5, 0xfffffff8u, 12, + 2, 4, 13, 0xfffffff3u, 2, + 0xffffffd8u, 0x03030303u, 0x3f3f3f3fu, 0x3c3c3c3cu, 0xc3c3c3c3u, + 32, 32, 0x0fffffffu, 0x0fffffffu, 0xffffffffu, + 0xffffffffu, 0xf0f0f0f0u, 0xf0000000u, 1, 30, + 7, 0xfffffff8u}, + {O::VMovB32, O::VMinI32, O::VMaxI32, O::VMinU32, + O::VMaxU32, O::VAddNcU32, O::VSubNcU32, O::VSubrevNcU32, + O::VMulU32U24, O::VMulI32I24, O::VAndB32, O::VOrB32, + O::VXorB32, O::VXnorB32, O::VLshlB32, O::VLshlrevB32, + O::VLshrB32, O::VLshrrevB32, O::VAshrI32, O::VAshrrevI32, + O::VNotB32, O::VBfrevB32, O::VFfblB32, O::VFfbhU32, + O::VCmpEqU32, O::VCndmaskB32, O::BufferStoreDword, O::SEndpgm}}; +} + +TestCase VectorShiftCountsMaskLowBits() { + using O = ShaderOpcode; + + std::vector code; + AppendVMovU32(&code, 0, 1); + AppendVMovU32(&code, 1, 32); + AppendVMovU32(&code, 2, 33); + AppendVMovLiteral(&code, 3, 0x80000000u); + code.push_back(EncodeVop2(0x19, 10, Vgpr(0), 1)); + code.push_back(EncodeVop2(0x1a, 11, Vgpr(2), 0)); + code.push_back(EncodeVop2(0x15, 12, Vgpr(3), 1)); + code.push_back(EncodeVop2(0x16, 13, Vgpr(2), 3)); + code.push_back(EncodeVop2(0x17, 14, Vgpr(3), 1)); + code.push_back(EncodeVop2(0x18, 15, Vgpr(2), 3)); + + for (u32 i = 0; i < 6u; i++) { + AppendStoreVgpr(&code, 10u + i, i); + } + AppendEnd(&code); + + return {"VectorShiftCountsMaskLowBits", + code, + {}, + {1, 2, 0x80000000u, 0x40000000u, 0x80000000u, 0xc0000000u}, + {O::VMovB32, O::VLshlB32, O::VLshlrevB32, O::VLshrB32, O::VLshrrevB32, + O::VAshrI32, O::VAshrrevI32, O::BufferStoreDword, O::SEndpgm}}; +} + +TestCase VectorVop3IntegerOps() { + using O = ShaderOpcode; + + std::vector code; + code.push_back(EncodeVop1(0x01, 0, InlineU32(2))); + code.push_back(EncodeVop1(0x01, 1, InlineU32(3))); + code.push_back(EncodeVop1(0x01, 2, InlineU32(4))); + AppendVMovLiteral(&code, 3, 0xfffffff8u); + code.push_back(EncodeVop1(0x01, 4, InlineU32(5))); + AppendVMovLiteral(&code, 5, 0x11223344u); + AppendVMovLiteral(&code, 6, 0x55667788u); + AppendVMovLiteral(&code, 7, 0x0f0f0f0fu); + AppendVMovLiteral(&code, 8, 0x33333333u); + AppendVMovLiteral(&code, 9, 0xaaaaaaaau); + AppendVMovLiteral(&code, 14, 0x00f00000u); + code.push_back(EncodeVop1(0x01, 15, InlineU32(8))); + AppendVMovLiteral(&code, 30, 0x00010000u); + AppendVMovLiteral(&code, 31, 0x00010000u); + + AppendVop3(&code, 0x36d, 10, Vgpr(0), Vgpr(1), Vgpr(2)); + AppendVop3(&code, 0x152, 11, Vgpr(3), Vgpr(4), Vgpr(0)); + AppendVop3(&code, 0x155, 12, Vgpr(3), Vgpr(4), Vgpr(0)); + AppendVop3(&code, 0x158, 13, Vgpr(3), Vgpr(4), Vgpr(0)); + AppendVop3(&code, 0x153, 16, Vgpr(3), Vgpr(4), Vgpr(0)); + AppendVop3(&code, 0x156, 17, Vgpr(3), Vgpr(4), Vgpr(0)); + AppendVop3(&code, 0x159, 18, Vgpr(3), Vgpr(4), Vgpr(0)); + AppendVop3(&code, 0x15d, 19, Vgpr(0), Vgpr(4), Vgpr(2)); + AppendVop3(&code, 0x346, 20, Vgpr(1), Vgpr(0), Vgpr(2)); + AppendVop3(&code, 0x347, 21, Vgpr(0), Vgpr(1), Vgpr(0)); + AppendVop3(&code, 0x345, 22, Vgpr(7), Vgpr(8), Vgpr(0)); + AppendVop3(&code, 0x36f, 23, Vgpr(1), Vgpr(2), Vgpr(0)); + AppendVop3(&code, 0x371, 24, Vgpr(7), Vgpr(8), Vgpr(2)); + AppendVop3(&code, 0x372, 25, Vgpr(7), Vgpr(8), Vgpr(2)); + AppendVop3(&code, 0x178, 26, Vgpr(7), Vgpr(8), Vgpr(2)); + AppendVop3(&code, 0x148, 27, Vgpr(14), InlineU32(20), Vgpr(2)); + AppendVop3(&code, 0x149, 28, Vgpr(3), Vgpr(2), Vgpr(2)); + AppendVop3(&code, 0x14a, 29, Vgpr(8), Vgpr(7), Vgpr(9)); + AppendVop3(&code, 0x14e, 32, Vgpr(5), Vgpr(6), Vgpr(15)); + AppendVop3(&code, 0x363, 33, Vgpr(2), Vgpr(15)); + AppendVop3(&code, 0x169, 34, Vgpr(30), Vgpr(31)); + AppendVop3(&code, 0x16a, 35, Vgpr(30), Vgpr(31)); + AppendVop3(&code, 0x16b, 36, Vgpr(30), Vgpr(31)); + AppendVop3(&code, 0x16c, 44, Vgpr(3), Vgpr(30)); + AppendVop3(&code, 0x142, 37, Vgpr(0), Vgpr(1), Vgpr(2)); + AppendVop3(&code, 0x143, 38, Vgpr(0), Vgpr(1), Vgpr(2)); + AppendVop3(&code, 0x30f, 39, Vgpr(0), Vgpr(1)); + AppendVop3(&code, 0x310, 40, Vgpr(0), Vgpr(1)); + AppendVop3(&code, 0x319, 41, Vgpr(0), Vgpr(1)); + + const u32 results[] = {10, 11, 12, 13, 16, 17, 18, 19, 20, 21, + 22, 23, 24, 25, 26, 27, 28, 29, 32, 33, + 34, 35, 36, 44, 37, 38, 39, 40, 41}; + for (u32 i = 0; i < static_cast(std::size(results)); i++) { + AppendStoreVgpr(&code, results[i], i); + } + AppendEnd(&code); + + return {"VectorVop3IntegerOps", + code, + {}, + {9, + 0xfffffff8u, + 5, + 2, + 2, + 0xfffffff8u, + 5, + 7, + 16, + 20, + 0x3c3c3c3eu, + 0x32, + 0x03030307u, + 0x3f3f3f3fu, + 0x3c3c3c38u, + 0x0fu, + 0x0fu, + 0x8b8b8b8bu, + 0x44556677u, + 0x00000f00u, + 0, + 1, + 0, + 0xffffffffu, + 10, + 10, + 5, + 0xffffffffu, + 1}, + {O::VMovB32, O::VAdd3U32, O::VMin3I32, O::VMax3I32, + O::VMed3I32, O::VMin3U32, O::VMax3U32, O::VMed3U32, + O::VSadU32, O::VLshlAddU32, O::VAddLshlU32, O::VXadU32, + O::VLshlOrB32, O::VAndOrB32, O::VOr3B32, O::VXor3B32, + O::VBfeU32, O::VBfeI32, O::VBfiB32, O::VAlignbitB32, + O::VBfmB32, O::VMulLoU32, O::VMulHiU32, O::VMulLoI32, + O::VMulHiI32, O::VMadI32I24, O::VMadU32U24, O::VAddI32, + O::VSubI32, O::VSubrevI32, O::BufferStoreDword, O::SEndpgm}}; +} + +TestCase VectorBfeI32ArithmeticShiftMasksField() { + using O = ShaderOpcode; + + std::vector code; + AppendVMovLiteral(&code, 0, 0x80000000u); + AppendVMovLiteral(&code, 1, 0xfffffff8u); + AppendVMovU32(&code, 2, 31); + AppendVMovU32(&code, 3, 1); + AppendVMovU32(&code, 4, 3); + AppendVMovU32(&code, 5, 4); + AppendVop3(&code, 0x149, 10, Vgpr(0), Vgpr(2), Vgpr(3)); + AppendVop3(&code, 0x149, 11, Vgpr(1), Vgpr(4), Vgpr(5)); + AppendStoreVgpr(&code, 10, 0); + AppendStoreVgpr(&code, 11, 1); + AppendEnd(&code); + + return {"VectorBfeI32ArithmeticShiftMasksField", + code, + {}, + {1, 0x0fu}, + {O::VMovB32, O::VBfeI32, O::BufferStoreDword, O::SEndpgm}}; +} + +TestCase VectorCarryAndBitCountOps() { + using O = ShaderOpcode; + + std::vector code; + AppendVMovLiteral(&code, 0, 0xffffffffu); + code.push_back(EncodeVop1(0x01, 1, InlineU32(0))); + AppendVMovLiteral(&code, 3, 0x0000f0f0u); + code.push_back(EncodeVop1(0x01, 4, InlineU32(5))); + + code.push_back(EncodeVopc(0xc7, Vgpr(1), 1)); + code.push_back(EncodeVop2(0x28, 2, Vgpr(0), 1)); + code.push_back(EncodeVop2(0x22, 5, Vgpr(3), 4)); + AppendVMovLiteral(&code, 6, 0xffffffffu); + code.push_back(EncodeVop2(0x23, 7, Vgpr(6), 4)); + code.push_back(EncodeVop2(0x24, 8, Vgpr(6), 4)); + + AppendStoreVgpr(&code, 2, 0); + AppendStoreSgpr(&code, 106, 1); + AppendStoreVgpr(&code, 5, 2); + AppendStoreVgpr(&code, 7, 3); + AppendStoreVgpr(&code, 8, 4); + AppendEnd(&code); + + return {"VectorCarryAndBitCountOps", + code, + {}, + {0, 1, 13, 5, 5}, + {O::VMovB32, O::VCmpTU32, O::VAddcU32, O::VBcntU32B32, + O::VMbcntLoU32B32, O::VMbcntHiU32B32, O::BufferStoreDword, + O::SEndpgm}}; +} + +TestCase VectorMbcntUsesThreadMask() { + using O = ShaderOpcode; + + std::vector code = { + EncodeVop1(0x01, 1, 0), + EncodeVop2(0x1a, 1, InlineU32(2), 1), + EncodeVop2(0x25, 1, Vgpr(0), 1), + EncodeVop2(0x1a, 3, InlineU32(2), 1), + }; + AppendVMovLiteral(&code, 2, 0xffffffffu); + code.push_back(EncodeVop1(0x01, 4, InlineU32(0))); + code.push_back(EncodeVop2(0x23, 5, Vgpr(2), 4)); + code.push_back(EncodeVop2(0x24, 6, Vgpr(2), 5)); + AppendBufferStoreDword(&code, 6, 3); + AppendEnd(&code); + + TestCase test; + test.name = "VectorMbcntUsesThreadMask"; + test.code = code; + test.expected = {0, 1, 2, 3, 0, 1, 2, 3}; + test.opcodes = {O::VMovB32, O::VLshlrevB32, O::VAddNcU32, + O::VMbcntLoU32B32, O::VMbcntHiU32B32, O::BufferStoreDword, + O::SEndpgm}; + test.compute_info.threads_num[0] = 4; + test.compute_info.threads_num[1] = 1; + test.compute_info.threads_num[2] = 1; + test.compute_info.group_id[0] = true; + test.compute_info.thread_ids_num = 1; + test.compute_info.workgroup_register = 0; + test.has_compute_info = true; + test.dispatch_x = 2; + return test; +} + +TestCase VectorAddcUsesPerLaneCarryIn() { + using O = ShaderOpcode; + + std::vector code = { + EncodeVop1(0x01, 1, 0), + EncodeVop2(0x1a, 1, InlineU32(2), 1), + EncodeVop2(0x25, 1, Vgpr(0), 1), + EncodeVopc(0xc7, Vgpr(0), 0), + EncodeVop1(0x01, 2, InlineU32(0)), + EncodeVop2(0x28, 3, Vgpr(2), 2), + EncodeVop2(0x1a, 4, InlineU32(2), 1), + }; + AppendBufferStoreDword(&code, 3, 4); + AppendEnd(&code); + + TestCase test; + test.name = "VectorAddcUsesPerLaneCarryIn"; + test.code = code; + test.expected = std::vector(8, 1); + test.opcodes = {O::VMovB32, O::VLshlrevB32, O::VAddNcU32, O::VCmpTU32, + O::VAddcU32, O::BufferStoreDword, O::SEndpgm}; + test.compute_info.threads_num[0] = 4; + test.compute_info.threads_num[1] = 1; + test.compute_info.threads_num[2] = 1; + test.compute_info.group_id[0] = true; + test.compute_info.thread_ids_num = 1; + test.compute_info.workgroup_register = 0; + test.has_compute_info = true; + test.dispatch_x = 2; + return test; +} + +TestCase VectorAddcWritesPerLaneCarryOut() { + using O = ShaderOpcode; + + std::vector code = { + EncodeVop1(0x01, 1, 0), + EncodeVop2(0x1a, 1, InlineU32(2), 1), + EncodeVop2(0x25, 1, Vgpr(0), 1), + EncodeVopc(0xc0, Vgpr(0), 0), + }; + AppendVMovLiteral(&code, 2, 0xffffffffu); + code.push_back(EncodeVop1(0x01, 3, InlineU32(1))); + code.push_back(EncodeVop2(0x28, 5, Vgpr(2), 3)); + code.push_back(EncodeVop1(0x01, 6, 106)); + code.push_back(EncodeVop2(0x1a, 4, InlineU32(2), 1)); + AppendBufferStoreDword(&code, 6, 4); + AppendEnd(&code); + + TestCase test; + test.name = "VectorAddcWritesPerLaneCarryOut"; + test.code = code; + test.expected = std::vector(8, 0x0fu); + test.opcodes = {O::VMovB32, O::VLshlrevB32, O::VAddNcU32, O::VCmpFU32, + O::VAddcU32, O::BufferStoreDword, O::SEndpgm}; + test.compute_info.threads_num[0] = 4; + test.compute_info.threads_num[1] = 1; + test.compute_info.threads_num[2] = 1; + test.compute_info.group_id[0] = true; + test.compute_info.thread_ids_num = 1; + test.compute_info.workgroup_register = 0; + test.has_compute_info = true; + test.dispatch_x = 2; + return test; +} + +TestCase VectorVop3BCarryOutWritesSgprMask() { + using O = ShaderOpcode; + + std::vector code; + AppendVMovLiteral(&code, 1, 0xffffffffu); + AppendVMovU32(&code, 2, 1); + AppendVMovU32(&code, 3, 0); + + AppendVop3B(&code, 0x30fu, 10, 0, Vgpr(1), Vgpr(2)); + AppendStoreSgprAtLaneDwordOffset(&code, 0, 0, 0); + AppendVop3B(&code, 0x310u, 11, 0, Vgpr(3), Vgpr(2)); + AppendStoreSgprAtLaneDwordOffset(&code, 0, 0, 4); + AppendVop3B(&code, 0x319u, 12, 0, Vgpr(2), Vgpr(3)); + AppendStoreSgprAtLaneDwordOffset(&code, 0, 0, 8); + AppendEnd(&code); + + TestCase test; + test.name = "VectorVop3BCarryOutWritesSgprMask"; + test.code = code; + test.expected = std::vector(12, 0x0fu); + test.opcodes = {O::VMovB32, O::VAddI32, O::VSubI32, + O::VSubrevI32, O::BufferStoreDword, O::SEndpgm}; + test.compute_info.threads_num[0] = 4; + test.compute_info.threads_num[1] = 1; + test.compute_info.threads_num[2] = 1; + test.compute_info.group_id[0] = true; + test.compute_info.thread_ids_num = 1; + test.compute_info.workgroup_register = 0; + test.has_compute_info = true; + return test; +} + +TestCase VectorVop3BCarryOutUsesEncodedSdst() { + using O = ShaderOpcode; + + std::vector code; + AppendVMovLiteral(&code, 1, 0xffffffffu); + AppendVMovU32(&code, 2, 1); + AppendVMovU32(&code, 3, 0); + + AppendVop3B(&code, 0x30fu, 10, 20, Vgpr(1), Vgpr(2)); + AppendStoreSgprAtLaneDwordOffset(&code, 20, 0, 0); + AppendVop3B(&code, 0x310u, 11, 22, Vgpr(3), Vgpr(2)); + AppendStoreSgprAtLaneDwordOffset(&code, 22, 0, 4); + AppendVop3B(&code, 0x319u, 12, 24, Vgpr(2), Vgpr(3)); + AppendStoreSgprAtLaneDwordOffset(&code, 24, 0, 8); + AppendEnd(&code); + + TestCase test; + test.name = "VectorVop3BCarryOutUsesEncodedSdst"; + test.code = code; + test.expected = std::vector(12, 0x0fu); + test.opcodes = {O::VMovB32, O::VAddI32, O::VSubI32, + O::VSubrevI32, O::BufferStoreDword, O::SEndpgm}; + test.compute_info.threads_num[0] = 4; + test.compute_info.threads_num[1] = 1; + test.compute_info.threads_num[2] = 1; + test.compute_info.group_id[0] = true; + test.compute_info.thread_ids_num = 1; + test.compute_info.workgroup_register = 0; + test.has_compute_info = true; + return test; +} + +TestCase VectorVop3BSubCoU32UsesRdna2Opcode310() { + using O = ShaderOpcode; + + std::vector code; + AppendVMovU32(&code, 1, 0); + AppendVMovU32(&code, 2, 1); + AppendVMovU32(&code, 3, 1); + AppendVMovLiteral(&code, 4, 0xffffffffu); + AppendVMovLiteral(&code, 5, 0x80000000u); + + code.push_back(EncodeVopc(0xc2, InlineU32(1), 0)); + code.push_back(EncodeVop2(0x01, 1, Vgpr(1), Vgpr(3))); + code.push_back(EncodeVopc(0xc2, InlineU32(2), 0)); + code.push_back(EncodeVop2(0x01, 1, Vgpr(1), Vgpr(4))); + code.push_back(EncodeVopc(0xc2, InlineU32(3), 0)); + code.push_back(EncodeVop2(0x01, 1, Vgpr(1), Vgpr(5))); + code.push_back(EncodeVop2(0x01, 2, Vgpr(2), Vgpr(4))); + + AppendVop3B(&code, 0x310u, 10, 20, Vgpr(1), Vgpr(2)); + AppendStoreVgprAtLaneDwordOffset(&code, 10, 0, 0); + AppendStoreSgprAtLaneDwordOffset(&code, 20, 0, 4); + AppendEnd(&code); + + TestCase test; + test.name = "VectorVop3BSubCoU32UsesRdna2Opcode310"; + test.code = code; + test.expected = {0xffffffffu, 0, 0xfffffffeu, 0x80000001u, 9, 9, 9, 9}; + test.opcodes = {O::VMovB32, O::VCmpEqU32, O::VCndmaskB32, + O::VSubI32, O::BufferStoreDword, O::SEndpgm}; + test.compute_info.threads_num[0] = 4; + test.compute_info.threads_num[1] = 1; + test.compute_info.threads_num[2] = 1; + test.compute_info.group_id[0] = true; + test.compute_info.thread_ids_num = 1; + test.compute_info.workgroup_register = 0; + test.has_compute_info = true; + return test; +} + +TestCase VectorMadU64U32UnsignedCarryOut() { + using O = ShaderOpcode; + + std::vector code; + AppendVMovLiteral(&code, 1, 0xffffffffu); + AppendVMovU32(&code, 2, 2); + AppendVMovU32(&code, 4, 2); + AppendVMovLiteral(&code, 5, 0xffffffffu); + AppendVop3B(&code, 0x176u, 10, 20, Vgpr(1), Vgpr(2), Vgpr(4)); + AppendStoreVgpr(&code, 10, 0); + AppendStoreVgpr(&code, 11, 1); + AppendStoreSgprPair(&code, 20, 2); + AppendEnd(&code); + + return {"VectorMadU64U32UnsignedCarryOut", + code, + {}, + {0x00000000u, 0x00000001u, 0x00000001u, 0x00000000u}, + {O::VMovB32, O::VMadU64U32, O::BufferStoreDword, O::SEndpgm}}; +} + +TestCase VectorLaneAndPackedOps() { + using O = ShaderOpcode; + + std::vector code; + AppendVMovLiteral(&code, 0, 0x12345678u); + code.push_back(EncodeVop1(0x01, 1, InlineU32(0))); + AppendVMovLiteral(&code, 4, 0x40003c00u); + AppendVMovLiteral(&code, 5, 0x44004200u); + AppendVMovLiteral(&code, 6, 0x3c003c00u); + AppendVMovLiteral(&code, 7, 0x3f800000u); + AppendVMovLiteral(&code, 8, 0x40000000u); + AppendVMovLiteral(&code, 9, 0x12345678u); + AppendVMovLiteral(&code, 10, 0xabcdef01u); + + code.push_back(EncodeVop1(0x02, 0, Vgpr(0))); + AppendVop3(&code, 0x360, 1, Vgpr(0), InlineU32(0)); + AppendVop3(&code, 0x361, 1, 0, InlineU32(0)); + AppendVop3(&code, 0x377, 2, Vgpr(0), InlineU32(0), InlineU32(0)); + code.push_back(EncodeVop2(0x2f, 3, Vgpr(7), 8)); + AppendVop3p(&code, 0x0f, 11, Vgpr(4), Vgpr(5), 0, 0x3); + AppendVop3p(&code, 0x10, 12, Vgpr(4), Vgpr(5), 0, 0x3); + AppendVop3p(&code, 0x11, 13, Vgpr(4), Vgpr(5), 0, 0x3); + AppendVop3p(&code, 0x12, 14, Vgpr(4), Vgpr(5), 0, 0x3); + AppendVop3p(&code, 0x0e, 15, Vgpr(4), Vgpr(5), Vgpr(6), 0x7); + AppendVop3(&code, 0x362, 16, Vgpr(7), InlineU32(1)); + AppendVop3(&code, 0x36a, 17, Vgpr(9), Vgpr(10)); + + AppendStoreSgpr(&code, 0, 0); + AppendStoreSgpr(&code, 1, 1); + const u32 results[] = {1, 2, 3, 11, 12, 13, 14, 15, 16, 17}; + for (u32 i = 0; i < static_cast(std::size(results)); i++) { + AppendStoreVgpr(&code, results[i], i + 2u); + } + AppendEnd(&code); + + return {"VectorLaneAndPackedOps", + code, + {}, + {0x12345678u, 0x12345678u, 0x12345678u, 0x12345678u, 0x40003c00u, + 0x46004400u, 0x48004200u, 0x40003c00u, 0x44004200u, 0x48804400u, + 0x40000000u, 0xef015678u}, + {O::VMovB32, O::VReadfirstlaneB32, O::VReadlaneB32, O::VWritelaneB32, + O::VPermlane16B32, O::VCvtPkrtzF16F32, O::VPkAddF16, O::VPkMulF16, + O::VPkMinF16, O::VPkMaxF16, O::VPkFmaF16, O::VLdexpF32, + O::VCvtPkU16U32, O::BufferStoreDword, O::SEndpgm}}; +} + +TestCase CvtPkU8F32PacksSelectedByte() { + using O = ShaderOpcode; + + std::vector code; + // RDNA2 V_CVT_PK_U8_F32: uint8(S0) replaces byte S1[1:0] in S2. + AppendVMovLiteral(&code, 0, 0x414c0000u); // 12.75f truncates to 12. + AppendVMovLiteral(&code, 1, 0x43960000u); // 300.0f saturates to 255. + AppendVMovLiteral(&code, 2, 0xbf800000u); // Negative values saturate to zero. + AppendVMovLiteral(&code, 3, 0x7fc00000u); // NaN saturates to zero. + AppendVMovLiteral(&code, 4, 0x11223344u); + + AppendVop3(&code, 0x15eu, 10, Vgpr(0), InlineU32(0), Vgpr(4)); + AppendVop3(&code, 0x15eu, 11, Vgpr(1), InlineU32(5), Vgpr(4)); + AppendVop3(&code, 0x15eu, 12, Vgpr(2), InlineU32(2), Vgpr(4)); + AppendVop3(&code, 0x15eu, 13, Vgpr(3), InlineU32(3), Vgpr(4)); + for (u32 i = 0; i < 4; i++) { + AppendStoreVgpr(&code, 10 + i, i); + } + AppendEnd(&code); + + return {"CvtPkU8F32PacksSelectedByte", + code, + {}, + {0x1122330cu, 0x1122ff44u, 0x11003344u, 0x00223344u}, + {O::VMovB32, O::VCvtPkU8F32, O::BufferStoreDword, O::SEndpgm}}; +} + +TestCase CvtPkrtzF16F32SubnormalRoundsTowardZero() { + using O = ShaderOpcode; + + std::vector code; + AppendVMovLiteral(&code, 0, 0x33c00000u); + AppendVMovLiteral(&code, 1, 0xb3c00000u); + code.push_back(EncodeVop2(0x2f, 2, Vgpr(0), 1)); + AppendStoreVgpr(&code, 2, 0); + AppendEnd(&code); + + return {"CvtPkrtzF16F32SubnormalRoundsTowardZero", + code, + {}, + {0x80010001u}, + {O::VMovB32, O::VCvtPkrtzF16F32, O::BufferStoreDword, O::SEndpgm}}; +} + +TestCase PackedMinMaxF16NanAndSignedZeroEdges() { + using O = ShaderOpcode; + + std::vector code; + AppendVMovLiteral(&code, 0, 0x80004000u); + AppendVMovLiteral(&code, 1, 0x00007e00u); + AppendVMovLiteral(&code, 2, 0x00004000u); + AppendVMovLiteral(&code, 3, 0x80007e00u); + AppendVMovLiteral(&code, 4, 0x40007d01u); + AppendVMovLiteral(&code, 5, 0x7d014000u); + + AppendVop3p(&code, 0x11, 10, Vgpr(0), Vgpr(1), 0, 0x3); + AppendVop3p(&code, 0x12, 11, Vgpr(2), Vgpr(3), 0, 0x3); + AppendVop3p(&code, 0x11, 12, Vgpr(4), Vgpr(5), 0, 0x3); + AppendVop3p(&code, 0x12, 13, Vgpr(4), Vgpr(5), 0, 0x3); + + for (u32 i = 0; i < 4; i++) { + AppendStoreVgpr(&code, 10 + i, i); + } + AppendEnd(&code); + + return {"PackedMinMaxF16NanAndSignedZeroEdges", + code, + {}, + {0x80004000u, 0x00004000u, 0x7f017f01u, 0x7f017f01u}, + {O::VMovB32, O::VPkMinF16, O::VPkMaxF16, O::BufferStoreDword, + O::SEndpgm}}; +} + +TestCase VectorMinMaxF16Ops() { + using O = ShaderOpcode; + + std::vector code; + AppendVMovLiteral(&code, 0, 0xaaaa4000u); // low=2.0h + AppendVMovLiteral(&code, 1, 0xbbbb3c00u); // low=1.0h + AppendVMovLiteral(&code, 2, 0x12345678u); + AppendVMovLiteral(&code, 3, 0x87654321u); + AppendVMovLiteral(&code, 4, 0x5555aaaau); + AppendVMovLiteral(&code, 5, 0x6666bbbbu); + AppendVMovLiteral(&code, 6, 0x7777bbbbu); + AppendVMovLiteral(&code, 7, 0x8888bbbbu); + AppendVMovLiteral(&code, 8, 0x9999bbbbu); + AppendVMovLiteral(&code, 9, 0x99990000u); + code.push_back(EncodeVop2(0x32, 5, Vgpr(0), 1)); + code.push_back(EncodeVop2(0x33, 6, Vgpr(0), 1)); + code.push_back(EncodeVop2(0x34, 7, Vgpr(0), 1)); + code.push_back(EncodeVop2(0x35, 4, Vgpr(0), 1)); + code.push_back(EncodeVop2(0x39, 2, Vgpr(0), 1)); + code.push_back(EncodeVop2(0x3a, 3, Vgpr(0), 1)); + code.push_back(EncodeVop2(0x35, 9, 249, 1)); + code.push_back(EncodeVop2Sdwa(0, 4, 2, 4, 4)); + code.push_back(0x660000f9u); + code.push_back(EncodeVop2Sdwa(0, 4, 2, 4, 4)); + AppendStoreVgpr(&code, 5, 0); + AppendStoreVgpr(&code, 6, 1); + AppendStoreVgpr(&code, 7, 2); + AppendStoreVgpr(&code, 0, 3); + AppendStoreVgpr(&code, 4, 4); + AppendStoreVgpr(&code, 2, 5); + AppendStoreVgpr(&code, 3, 6); + AppendStoreVgpr(&code, 9, 7); + AppendEnd(&code); + + return {"VectorMinMaxF16Ops", + code, + {}, + {0x66664200u, 0x77773c00u, 0x8888bc00u, 0xaaaa0000u, 0x55554000u, + 0x12344000u, 0x87653c00u, 0x99994000u}, + {O::VMovB32, O::VAddF16, O::VSubF16, O::VSubrevF16, O::VMulF16, + O::VMaxF16, O::VMinF16, O::BufferStoreDword, O::SEndpgm}}; +} + +TestCase VectorCvtU16F16Sdwa() { + using O = ShaderOpcode; + + std::vector code; + AppendVMovLiteral(&code, 0, 0x45004200u); // low=3.0h, high=5.0h + AppendVMovLiteral(&code, 1, 0xbc007c00u); // low=+inf, high=-1.0h + AppendVMovLiteral(&code, 2, 0x00007e00u); // low=qNaN + AppendVMovLiteral(&code, 10, 0x12345678u); + AppendVMovLiteral(&code, 11, 0x87654321u); + AppendVMovLiteral(&code, 12, 0xabcd1111u); + AppendVMovLiteral(&code, 13, 0x77772222u); + AppendVMovLiteral(&code, 14, 0x5555aaaau); + code.push_back(EncodeVop1(0x52, 10, 249)); + code.push_back(EncodeVop1Sdwa(0, 4, 2, 4)); + code.push_back(EncodeVop1(0x52, 11, 249)); + code.push_back(EncodeVop1Sdwa(0, 5, 2, 5)); + code.push_back(EncodeVop1(0x52, 12, 249)); + code.push_back(EncodeVop1Sdwa(1, 4, 2, 4)); + code.push_back(EncodeVop1(0x52, 13, 249)); + code.push_back(EncodeVop1Sdwa(1, 4, 2, 5)); + code.push_back(EncodeVop1(0x52, 14, 249)); + code.push_back(EncodeVop1Sdwa(2, 4, 2, 4)); + for (u32 i = 0; i < 5; i++) { + AppendStoreVgpr(&code, 10 + i, i); + } + AppendEnd(&code); + + return {"VectorCvtU16F16Sdwa", + code, + {}, + {0x12340003u, 0x00054321u, 0xabcdffffu, 0x77770000u, 0x55550000u}, + {O::VMovB32, O::VCvtU16F16, O::BufferStoreDword, O::SEndpgm}}; +} + +TestCase VectorMinMaxMed3F16Ops() { + using O = ShaderOpcode; + + std::vector code; + AppendVMovLiteral(&code, 0, 0x40003c00u); // low=1.0h, high=2.0h + AppendVMovLiteral(&code, 1, 0x44004200u); // low=3.0h, high=4.0h + AppendVMovLiteral(&code, 2, 0xc8004000u); // low=2.0h, high=-8.0h + AppendVMovLiteral(&code, 3, 0x00007e00u); // low=qNaN + AppendVMovLiteral(&code, 4, 0x00004000u); // low=2.0h + AppendVMovLiteral(&code, 5, 0x00003c00u); // low=1.0h + AppendVMovLiteral(&code, 10, 0xaaaa5555u); + AppendVMovLiteral(&code, 11, 0x12345678u); + AppendVMovLiteral(&code, 12, 0x77772222u); + + AppendVop3(&code, 0x351, 10, Vgpr(0), Vgpr(1), Vgpr(2)); + AppendVop3(&code, 0x354, 11, Vgpr(0), Vgpr(1), Vgpr(2), 0, 0xf, false, 0, + 0x4); + AppendVop3(&code, 0x357, 12, Vgpr(3), Vgpr(4), Vgpr(5)); + for (u32 i = 0; i < 3; i++) { + AppendStoreVgpr(&code, 10 + i, i); + } + AppendEnd(&code); + + return {"VectorMinMaxMed3F16Ops", + code, + {}, + {0xaaaa3c00u, 0x48005678u, 0x77773c00u}, + {O::VMovB32, O::VMin3F16, O::VMax3F16, O::VMed3F16, + O::BufferStoreDword, O::SEndpgm}}; +} + +TestCase VectorSpecialF16Ops() { + using O = ShaderOpcode; + + std::vector code; + AppendVMovLiteral(&code, 0, 0xc000fc00u); // low=-inf, high=-2.0h + AppendVMovLiteral(&code, 1, 0x00004400u); // low=+4.0h + AppendVMovLiteral(&code, 2, 0x00003c00u); // low=+1.0h + AppendVMovLiteral(&code, 3, 0x0000bc00u); // low=-1.0h + AppendVMovLiteral(&code, 4, 0x0000fc00u); // low=-inf + AppendVMovLiteral(&code, 11, 0x12345678u); + + code.push_back(EncodeVop1(0x54, 10, Vgpr(0))); + code.push_back(EncodeVop1(0x54, 11, 249)); + code.push_back(EncodeVop1Sdwa(0, 5, 2, 5)); + code.push_back(EncodeVop1(0x56, 12, Vgpr(1))); + code.push_back(EncodeVop1(0x57, 13, Vgpr(2))); + code.push_back(EncodeVop1(0x57, 14, Vgpr(3))); + code.push_back(EncodeVop1(0x58, 15, Vgpr(4))); + for (u32 i = 0; i < 6; i++) { + AppendStoreVgpr(&code, 10 + i, i); + } + AppendEnd(&code); + + return {"VectorSpecialF16Ops", + code, + {}, + {0x00008000u, 0xb8005678u, 0x00003800u, 0x00000000u, 0x0000fe00u, + 0x00000000u}, + {O::VMovB32, O::VRcpF16, O::VRsqF16, O::VLogF16, O::VExpF16, + O::BufferStoreDword, O::SEndpgm}}; +} + +TestCase VectorWritelaneIgnoresExecMask() { + using O = ShaderOpcode; + + std::vector code; + AppendVMovLiteral(&code, 2, 0xaaaaaaaau); + AppendSMovLiteral(&code, 4, 0x12345678u); + code.push_back(EncodeSMovB32(126, InlineU32(1))); + code.push_back(EncodeSMovB32(127, InlineU32(0))); + AppendVop3(&code, 0x361, 2, 4, InlineU32(1)); + code.push_back(EncodeSMovB32(126, InlineU32(0xf))); + code.push_back(EncodeSMovB32(127, InlineU32(0))); + code.push_back(EncodeVop2(0x1a, 3, InlineU32(2), 0)); + AppendBufferStoreDword(&code, 2, 3); + AppendEnd(&code); + + TestCase test; + test.name = "VectorWritelaneIgnoresExecMask"; + test.code = code; + test.expected = {0xaaaaaaaau, 0x12345678u, 0xaaaaaaaau, 0xaaaaaaaau}; + test.opcodes = {O::VMovB32, O::SMovB32, O::VWritelaneB32, + O::VLshlrevB32, O::BufferStoreDword, O::SEndpgm}; + test.compute_info.threads_num[0] = 4; + test.compute_info.threads_num[1] = 1; + test.compute_info.threads_num[2] = 1; + test.compute_info.thread_ids_num = 1; + test.has_compute_info = true; + return test; +} + +TestCase VectorPermlanex16() { + using O = ShaderOpcode; + + std::vector code; + AppendVMovLiteral(&code, 3, 0x76543210u); + AppendVMovLiteral(&code, 4, 0xfedcba98u); + code.push_back(EncodeVop2(0x1a, 2, InlineU32(2), 0)); + AppendVMovLiteral(&code, 5, 0xfeedbabeu); + AppendVop3(&code, 0x378, 1, Vgpr(5), Vgpr(3), Vgpr(4)); + AppendBufferStoreDword(&code, 1, 2); + AppendEnd(&code); + + TestCase test; + test.name = "VectorPermlanex16"; + test.code = code; + test.expected = std::vector(32, 0xfeedbabeu); + test.opcodes = {O::VMovB32, O::VPermlanex16B32, O::VLshlrevB32, + O::BufferStoreDword, O::SEndpgm}; + test.compute_info.threads_num[0] = 32; + test.compute_info.threads_num[1] = 1; + test.compute_info.threads_num[2] = 1; + test.compute_info.thread_ids_num = 1; + test.has_compute_info = true; + return test; +} + +TestCase VectorPermlane16FetchInactiveZero() { + using O = ShaderOpcode; + + std::vector code = { + EncodeVop1(0x01, 1, 0), EncodeVop2(0x25, 1, InlineU32(10), 1), + EncodeSMovB32(126, InlineU32(1)), EncodeSMovB32(127, InlineU32(0)), + EncodeSMovB32(0, InlineU32(1)), EncodeSMovB32(1, InlineU32(0)), + }; + AppendVop3(&code, 0x377, 2, Vgpr(1), 0, 1); + AppendStoreVgpr(&code, 2, 0); + AppendEnd(&code); + + TestCase test; + test.name = "VectorPermlane16FetchInactiveZero"; + test.code = code; + test.expected = {0}; + test.opcodes = {O::VMovB32, O::VAddNcU32, O::SMovB32, + O::VPermlane16B32, O::BufferStoreDword, O::SEndpgm}; + test.compute_info.threads_num[0] = 4; + test.compute_info.threads_num[1] = 1; + test.compute_info.threads_num[2] = 1; + test.compute_info.thread_ids_num = 1; + test.has_compute_info = true; + return test; +} + +TestCase VectorPermlane16FetchInactiveFi() { + using O = ShaderOpcode; + + std::vector code = { + EncodeVop1(0x01, 1, Vgpr(0)), EncodeVop2(0x25, 1, InlineU32(10), 1), + EncodeSMovB32(126, InlineU32(1)), EncodeSMovB32(127, InlineU32(0)), + EncodeSMovB32(0, InlineU32(1)), EncodeSMovB32(1, InlineU32(0)), + }; + AppendVop3(&code, 0x377, 2, Vgpr(1), 0, 1, 0, 1); + AppendStoreVgpr(&code, 2, 0); + AppendEnd(&code); + + TestCase test; + test.name = "VectorPermlane16FetchInactiveFi"; + test.code = code; + test.expected = {11}; + test.opcodes = {O::VMovB32, O::VAddNcU32, O::SMovB32, + O::VPermlane16B32, O::BufferStoreDword, O::SEndpgm}; + test.compute_info.threads_num[0] = 4; + test.compute_info.threads_num[1] = 1; + test.compute_info.threads_num[2] = 1; + test.compute_info.thread_ids_num = 1; + test.has_compute_info = true; + return test; +} + +TestCase VectorDppQuadPermuteReverse() { + using O = ShaderOpcode; + + std::vector code; + AppendVMovU32(&code, 1, 100); + code.push_back(EncodeVop2(0x25, 2, 250, 1)); + code.push_back(EncodeVop2Dpp(0, 0x01b)); + code.push_back(EncodeVop2(0x1a, 3, InlineU32(2), 0)); + AppendBufferStoreDword(&code, 2, 3); + AppendEnd(&code); + + TestCase test; + test.name = "VectorDppQuadPermuteReverse"; + test.code = code; + test.expected = {103, 102, 101, 100, 107, 106, 105, 104}; + test.opcodes = {O::VMovB32, O::VAddNcU32, O::VLshlrevB32, O::BufferStoreDword, + O::SEndpgm}; + test.compute_info.threads_num[0] = 8; + test.compute_info.threads_num[1] = 1; + test.compute_info.threads_num[2] = 1; + test.compute_info.thread_ids_num = 1; + test.has_compute_info = true; + return test; +} + +TestCase VectorDppBankMaskPreservesDestination() { + using O = ShaderOpcode; + + std::vector code; + AppendVMovLiteral(&code, 2, 0xaaaaaaaau); + AppendVMovU32(&code, 1, 100); + code.push_back(EncodeVop2(0x25, 2, 250, 1)); + code.push_back(EncodeVop2Dpp(0, 0x0e4, 0xf, 0xe)); + code.push_back(EncodeVop2(0x1a, 3, InlineU32(2), 0)); + AppendBufferStoreDword(&code, 2, 3); + AppendEnd(&code); + + TestCase test; + test.name = "VectorDppBankMaskPreservesDestination"; + test.code = code; + test.expected = {0xaaaaaaaau, 0xaaaaaaaau, 0xaaaaaaaau, 0xaaaaaaaau, + 104, 105, 106, 107}; + test.opcodes = {O::VMovB32, O::VAddNcU32, O::VLshlrevB32, O::BufferStoreDword, + O::SEndpgm}; + test.compute_info.threads_num[0] = 8; + test.compute_info.threads_num[1] = 1; + test.compute_info.threads_num[2] = 1; + test.compute_info.thread_ids_num = 1; + test.has_compute_info = true; + return test; +} + +TestCase VectorDppBoundsControlZeroPreservesDestination() { + using O = ShaderOpcode; + + std::vector code; + AppendVMovLiteral(&code, 2, 0xaaaaaaaau); + AppendVMovU32(&code, 1, 100); + code.push_back(EncodeVop2(0x25, 2, 250, 1)); + code.push_back(EncodeVop2Dpp(0, 0x111)); + code.push_back(EncodeVop2(0x1a, 3, InlineU32(2), 0)); + AppendBufferStoreDword(&code, 2, 3); + AppendEnd(&code); + + TestCase test; + test.name = "VectorDppBoundsControlZeroPreservesDestination"; + test.code = code; + test.expected = {0xaaaaaaaau, 100, 101, 102, 103, 104, 105, 106}; + test.opcodes = {O::VMovB32, O::VAddNcU32, O::VLshlrevB32, O::BufferStoreDword, + O::SEndpgm}; + test.compute_info.threads_num[0] = 8; + test.compute_info.threads_num[1] = 1; + test.compute_info.threads_num[2] = 1; + test.compute_info.thread_ids_num = 1; + test.has_compute_info = true; + return test; +} + +TestCase Vop3LdexpSourceModifier() { + using O = ShaderOpcode; + + std::vector code; + AppendVMovLiteral(&code, 9, 0xbfc00000u); + code.push_back(0xd7620107u); + code.push_back(0x00018509u); + AppendStoreVgpr(&code, 7, 0); + AppendEnd(&code); + + return {"Vop3LdexpSourceModifier", + code, + {}, + {0x3ec00000u}, + {O::VMovB32, O::VLdexpF32, O::BufferStoreDword, O::SEndpgm}}; +} + +TestCase Vop1MoveRelSource() { + using O = ShaderOpcode; + + std::vector code; + AppendVMovLiteral(&code, 12, 0x11111111u); + AppendVMovLiteral(&code, 13, 0x22222222u); + AppendVMovLiteral(&code, 14, 0x33333333u); + code.push_back(EncodeSMovB32(124, InlineU32(2))); + code.push_back(0x7e6e870cu); + AppendStoreVgpr(&code, 55, 0); + AppendEnd(&code); + + return {"Vop1MoveRelSource", + code, + {}, + {0x33333333u}, + {O::VMovB32, O::SMovB32, O::VMovrelsB32, O::BufferStoreDword, + O::SEndpgm}}; +} + +TestCase Vop1MoveRelDestination() { + using O = ShaderOpcode; + + std::vector code; + AppendVMovLiteral(&code, 1, 0x12345678u); + AppendVMovLiteral(&code, 5, 0xaaaaaaaau); + AppendVMovLiteral(&code, 6, 0xbbbbbbbbu); + AppendVMovLiteral(&code, 7, 0xccccccccu); + code.push_back(EncodeSMovB32(124, InlineU32(2))); + code.push_back(EncodeSMovB32(126, InlineU32(1))); + code.push_back(EncodeSMovB32(127, InlineU32(0))); + code.push_back(EncodeVop1(0x42, 5, Vgpr(1))); + code.push_back(EncodeSMovB32(126, InlineU32(0xf))); + code.push_back(EncodeSMovB32(127, InlineU32(0))); + code.push_back(EncodeVop2(0x1a, 4, InlineU32(2), 0)); + AppendBufferStoreDword(&code, 7, 4); + AppendEnd(&code); + + TestCase test; + test.name = "Vop1MoveRelDestination"; + test.code = code; + test.expected = {0x12345678u, 0xccccccccu, 0xccccccccu, 0xccccccccu}; + test.opcodes = {O::VMovB32, O::SMovB32, O::VMovreldB32, + O::VLshlrevB32, O::BufferStoreDword, O::SEndpgm}; + test.compute_info.threads_num[0] = 4; + test.compute_info.threads_num[1] = 1; + test.compute_info.threads_num[2] = 1; + test.compute_info.thread_ids_num = 1; + test.has_compute_info = true; + return test; +} + +TestCase VectorFloatSpecialOps() { + using O = ShaderOpcode; + + std::vector code; + AppendVMovLiteral(&code, 0, 0x3f800000u); + AppendVMovLiteral(&code, 1, 0x40000000u); + AppendVMovLiteral(&code, 2, 0x40400000u); + AppendVMovLiteral(&code, 3, 0x40800000u); + AppendVMovLiteral(&code, 4, 0x40003c00u); + AppendVMovLiteral(&code, 5, 0x44004200u); + AppendVMovLiteral(&code, 6, 0x3f800000u); + AppendVMovLiteral(&code, 7, 0x3f800000u); + AppendVMovLiteral(&code, 8, 0xbf800000u); + AppendVMovLiteral(&code, 10, 0x3f000000u); + AppendVMovLiteral(&code, 12, 0xaaaabbbbu); + AppendVMovLiteral(&code, 13, 0xaaaabbbbu); + + code.push_back(EncodeVop2(0x02, 6, Vgpr(4), 5)); + AppendVop3(&code, 0x368, 9, Vgpr(7), Vgpr(8)); + AppendVop3(&code, 0x369, 11, Vgpr(7), Vgpr(10)); + AppendVop3p(&code, 0x21, 12, Vgpr(1), Vgpr(2), Vgpr(0)); + AppendVop3p(&code, 0x22, 13, Vgpr(1), Vgpr(2), Vgpr(0)); + AppendVop3(&code, 0x144, 14, Vgpr(0), Vgpr(1), Vgpr(2)); + AppendVop3(&code, 0x145, 15, Vgpr(0), Vgpr(1), Vgpr(2)); + AppendVop3(&code, 0x146, 16, Vgpr(0), Vgpr(1), Vgpr(2)); + AppendVop3(&code, 0x147, 17, Vgpr(0), Vgpr(1), Vgpr(2)); + + const u32 results[] = {6, 9, 11, 12, 13, 14, 15, 16, 17}; + for (u32 i = 0; i < static_cast(std::size(results)); i++) { + AppendStoreVgpr(&code, results[i], i); + } + AppendEnd(&code); + + return {"VectorFloatSpecialOps", + code, + {}, + {0x41400000u, 0x80017fffu, 0x8000ffffu, 0xaaaa4700u, 0x4700bbbbu, + 0x40800000u, 0x3f800000u, 0xc0000000u, 0x40c00000u}, + {O::VMovB32, O::VDot2cF32F16, O::VCvtPknormI16F32, + O::VCvtPknormU16F32, O::VMadMixloF16, O::VMadMixhiF16, O::VCubeidF32, + O::VCubescF32, O::VCubetcF32, O::VCubemaF32, O::BufferStoreDword, + O::SEndpgm}}; +} + +TestCase MadMixF16LiteralHalfSourceUsesOpsel() { + using O = ShaderOpcode; + + std::vector code; + AppendVMovLiteral(&code, 0, 0x3f800000u); + AppendVMovLiteral(&code, 12, 0xaaaabbbbu); + AppendVMovLiteral(&code, 13, 0x11112222u); + + AppendVop3p(&code, 0x21, 12, 255u, Vgpr(0), Vgpr(0), 0x1, 0x1); + code.push_back(0x40003c00u); + AppendVop3p(&code, 0x22, 13, 255u, Vgpr(0), Vgpr(0), 0x1, 0x0); + code.push_back(0x40003c00u); + + AppendStoreVgpr(&code, 12, 0); + AppendStoreVgpr(&code, 13, 1); + AppendEnd(&code); + + return {"MadMixF16LiteralHalfSourceUsesOpsel", + code, + {}, + {0xaaaa4200u, 0x40002222u}, + {O::VMovB32, O::VMadMixloF16, O::VMadMixhiF16, O::BufferStoreDword, + O::SEndpgm}}; +} + +TestCase MadMixF16NegHiIsAbsAndNegIsIndependent() { + using O = ShaderOpcode; + + std::vector code; + AppendVMovLiteral(&code, 0, 0x3f800000u); + AppendVMovLiteral(&code, 1, 0xc0000000u); + AppendVMovLiteral(&code, 2, 0x40000000u); + AppendVMovLiteral(&code, 12, 0xaaaabbbbu); + AppendVMovLiteral(&code, 13, 0x11112222u); + + AppendVop3p(&code, 0x21, 12, Vgpr(1), Vgpr(0), Vgpr(0), 0, 0, 0x1); + AppendVop3p(&code, 0x22, 13, Vgpr(2), Vgpr(0), Vgpr(0), 0, 0, 0, 0x1); + + AppendStoreVgpr(&code, 12, 0); + AppendStoreVgpr(&code, 13, 1); + AppendEnd(&code); + + return {"MadMixF16NegHiIsAbsAndNegIsIndependent", + code, + {}, + {0xaaaa4200u, 0xbc002222u}, + {O::VMovB32, O::VMadMixloF16, O::VMadMixhiF16, O::BufferStoreDword, + O::SEndpgm}}; +} + +TestCase VectorVop3FmaF16UsesRdna2Opcode34b() { + using O = ShaderOpcode; + + std::vector code; + AppendVMovLiteral(&code, 1, 0x40003c00u); + AppendVMovLiteral(&code, 2, 0x44004200u); + AppendVMovLiteral(&code, 3, 0x3c003c00u); + AppendVMovLiteral(&code, 10, 0xaaaa5555u); + AppendVMovLiteral(&code, 11, 0xbbbb5555u); + + AppendVop3(&code, 0x34bu, 10, Vgpr(1), Vgpr(2), Vgpr(3)); + AppendVop3(&code, 0x34bu, 11, Vgpr(1), Vgpr(2), Vgpr(3), 0, 0xfu); + + AppendStoreVgpr(&code, 10, 0); + AppendStoreVgpr(&code, 11, 1); + AppendEnd(&code); + + return {"VectorVop3FmaF16UsesRdna2Opcode34b", + code, + {}, + {0xaaaa4400u, 0x48805555u}, + {O::VMovB32, O::VFmaF16, O::BufferStoreDword, O::SEndpgm}}; +} + +TestCase VectorFloatArithmeticOps() { + using O = ShaderOpcode; + + std::vector code; + AppendVMovLiteral(&code, 0, 0x3f800000u); + AppendVMovLiteral(&code, 1, 0x40000000u); + AppendVMovLiteral(&code, 2, 0x40800000u); + AppendVMovLiteral(&code, 3, 0x40a00000u); + AppendVMovLiteral(&code, 4, 0x40400000u); + AppendVMovLiteral(&code, 11, 0x40800000u); + AppendVMovLiteral(&code, 17, 0x40800000u); + + code.push_back(EncodeVop2(0x03, 5, Vgpr(0), 1)); + code.push_back(EncodeVop2(0x04, 6, Vgpr(3), 1)); + code.push_back(EncodeVop2(0x05, 7, Vgpr(1), 3)); + code.push_back(EncodeVop2(0x08, 8, Vgpr(1), 2)); + code.push_back(EncodeVop1(0x01, 21, Vgpr(8))); + code.push_back(EncodeVop2(0x0f, 9, Vgpr(3), 1)); + code.push_back(EncodeVop2(0x10, 10, Vgpr(3), 1)); + code.push_back(EncodeVop2(0x1f, 11, Vgpr(1), 4)); + AppendVop3(&code, 0x141, 12, Vgpr(1), Vgpr(4), Vgpr(2)); + AppendVop3(&code, 0x14b, 13, Vgpr(1), Vgpr(4), Vgpr(2)); + AppendVop3(&code, 0x151, 14, Vgpr(3), Vgpr(1), Vgpr(2)); + AppendVop3(&code, 0x154, 15, Vgpr(3), Vgpr(1), Vgpr(2)); + AppendVop3(&code, 0x157, 16, Vgpr(3), Vgpr(1), Vgpr(2)); + code.push_back(EncodeVop2(0x20, 18, Vgpr(1), 17)); + code.push_back(0x3f800000u); + code.push_back(EncodeVop2(0x21, 19, Vgpr(1), 4)); + code.push_back(0x40000000u); + code.push_back(EncodeVop2(0x03, 20, 249, 1)); + code.push_back(EncodeVop2Sdwa(0)); + AppendSMovLiteral(&code, 70, 0x40000000u); + code.push_back(0x06088cf9u); + code.push_back(0x868606f2u); + AppendSMovLiteral(&code, 26, 0x40000000u); + code.push_back(0x081034f9u); + code.push_back(0x868606f2u); + code.push_back(0xd51f8011u); + code.push_back(0x00020d0bu); + AppendVMovLiteral(&code, 22, 0xbf800000u); + code.push_back(EncodeVop2(0x05, 23, 249, 26)); + code.push_back(EncodeVop2Sdwa(22, 6, 0, 6, 6, 0, 0, 0, 1, 0, 0, 0, 1)); + AppendVMovLiteral(&code, 24, 0x3f800000u); + AppendVMovLiteral(&code, 25, 0x3f800000u); + code.push_back(EncodeVop2(0x03, 26, 249, 25)); + code.push_back(EncodeVop2Sdwa(24, 6, 0, 6, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1)); + AppendVMovLiteral(&code, 27, 0xbf800000u); + AppendVMovLiteral(&code, 28, 0x40000000u); + AppendVop3(&code, 0x104, 29, Vgpr(27), Vgpr(28), 0, 1); + AppendVop3(&code, 0x103, 30, Vgpr(0), Vgpr(1), 0, 0, 0, false, 1); + AppendVop3(&code, 0x151, 32, Vgpr(0), Vgpr(1), Vgpr(2), 0, 0, false, 1); + AppendVMovLiteral(&code, 33, 0xbf800000u); + AppendVMovLiteral(&code, 34, 0x40000000u); + code.push_back(EncodeVop2(0x03, 35, 250, 34)); + code.push_back(EncodeVop2Dpp(33, 0, 0xf, 0xf, 0, 1)); + + const u32 results[] = {5, 6, 7, 21, 9, 10, 11, 12, 13, 14, 15, 16, + 18, 19, 20, 4, 8, 17, 23, 26, 29, 30, 32, 35}; + for (u32 i = 0; i < static_cast(std::size(results)); i++) { + AppendStoreVgpr(&code, results[i], i); + } + AppendEnd(&code); + + return {"VectorFloatArithmeticOps", + code, + {}, + {0x40400000u, 0x40400000u, 0x40400000u, 0x41000000u, 0x40000000u, + 0x40a00000u, 0x41200000u, 0x41200000u, 0x41200000u, 0x40000000u, + 0x40a00000u, 0x40800000u, 0x40c00000u, 0x41000000u, 0x40400000u, + 0x40400000u, 0xbf800000u, 0x3f800000u, 0x3f800000u, 0x40800000u, + 0xbf800000u, 0x40c00000u, 0x40000000u, 0x40400000u}, + {O::SMovB32, O::VMovB32, O::VAddF32, O::VSubF32, O::VSubrevF32, + O::VMulF32, O::VMinF32, O::VMaxF32, O::VMacF32, O::VMadF32, + O::VFmaF32, O::VMin3F32, O::VMax3F32, O::VMed3F32, O::VMadmkF32, + O::VMadakF32, O::BufferStoreDword, O::SEndpgm}}; +} + +TestCase VectorMinMaxF32NanAndSignedZeroEdges() { + using O = ShaderOpcode; + + std::vector code; + AppendVMovLiteral(&code, 0, 0x40000000u); + AppendVMovLiteral(&code, 1, 0x7fc00000u); + AppendVMovLiteral(&code, 2, 0x00000000u); + AppendVMovLiteral(&code, 3, 0x80000000u); + AppendVMovLiteral(&code, 4, 0x40400000u); + AppendVMovLiteral(&code, 5, 0x3f800000u); + AppendVMovLiteral(&code, 6, 0x7fa00001u); + + code.push_back(EncodeVop2(0x0f, 10, Vgpr(0), 1)); + code.push_back(EncodeVop2(0x10, 11, Vgpr(0), 1)); + code.push_back(EncodeVop2(0x0f, 12, Vgpr(3), 2)); + code.push_back(EncodeVop2(0x10, 13, Vgpr(2), 3)); + code.push_back(EncodeVop2(0x0f, 14, Vgpr(6), 0)); + code.push_back(EncodeVop2(0x10, 15, Vgpr(0), 6)); + AppendVop3(&code, 0x151, 16, Vgpr(0), Vgpr(1), Vgpr(4)); + AppendVop3(&code, 0x154, 17, Vgpr(0), Vgpr(1), Vgpr(5)); + + for (u32 i = 0; i < 8; i++) { + AppendStoreVgpr(&code, 10 + i, i); + } + AppendEnd(&code); + + return {"VectorMinMaxF32NanAndSignedZeroEdges", + code, + {}, + {0x40000000u, 0x40000000u, 0x80000000u, 0x00000000u, 0x7fe00001u, + 0x7fe00001u, 0x40000000u, 0x40000000u}, + {O::VMovB32, O::VMinF32, O::VMaxF32, O::VMin3F32, O::VMax3F32, + O::BufferStoreDword, O::SEndpgm}}; +} + +TestCase VectorMed3F32NanUsesMin3Path() { + using O = ShaderOpcode; + + std::vector code; + AppendVMovLiteral(&code, 0, 0x40000000u); + AppendVMovLiteral(&code, 1, 0x40400000u); + AppendVMovLiteral(&code, 2, 0x7fa00001u); + AppendVMovLiteral(&code, 3, 0x7fc00000u); + + AppendVop3(&code, 0x157, 10, Vgpr(0), Vgpr(1), Vgpr(2)); + AppendVop3(&code, 0x157, 11, Vgpr(0), Vgpr(1), Vgpr(3)); + AppendVop3(&code, 0x157, 12, Vgpr(2), Vgpr(0), Vgpr(1)); + + AppendStoreVgpr(&code, 10, 0); + AppendStoreVgpr(&code, 11, 1); + AppendStoreVgpr(&code, 12, 2); + AppendEnd(&code); + + return {"VectorMed3F32NanUsesMin3Path", + code, + {}, + {0x7fe00001u, 0x40000000u, 0x40400000u}, + {O::VMovB32, O::VMed3F32, O::BufferStoreDword, O::SEndpgm}}; +} + +TestCase VectorFloatConversionOps() { + using O = ShaderOpcode; + + std::vector code; + AppendVMovLiteral(&code, 0, 0xfffffffdu); + code.push_back(EncodeVop1(0x01, 1, InlineU32(7))); + AppendVMovLiteral(&code, 2, 0x40e00000u); + AppendVMovLiteral(&code, 3, 0xc0400000u); + AppendVMovLiteral(&code, 4, 0x3f800000u); + AppendVMovLiteral(&code, 5, 0x00003c00u); + AppendVMovLiteral(&code, 6, 0x40700000u); + AppendVMovLiteral(&code, 7, 0x44332211u); + AppendVMovLiteral(&code, 8, 0x40000000u); + AppendVMovLiteral(&code, 9, 0x40800000u); + AppendVMovLiteral(&code, 10, 0x40100000u); + AppendVMovLiteral(&code, 11, 0x40300000u); + code.push_back(EncodeVop1(0x01, 12, InlineU32(0))); + code.push_back(EncodeVop1(0x01, 37, InlineU32(15))); + + code.push_back(EncodeVop1(0x05, 13, Vgpr(0))); + code.push_back(EncodeVop1(0x06, 14, Vgpr(1))); + code.push_back(EncodeVop1(0x07, 15, Vgpr(2))); + code.push_back(EncodeVop1(0x08, 16, Vgpr(3))); + code.push_back(EncodeVop1(0x0a, 17, Vgpr(4))); + code.push_back(EncodeVop1(0x0b, 18, Vgpr(5))); + code.push_back(EncodeVop1(0x0c, 42, Vgpr(11))); + code.push_back(EncodeVop1(0x0d, 19, Vgpr(6))); + code.push_back(EncodeVop1(0x11, 20, Vgpr(7))); + code.push_back(EncodeVop1(0x12, 21, Vgpr(7))); + code.push_back(EncodeVop1(0x13, 22, Vgpr(7))); + code.push_back(EncodeVop1(0x14, 23, Vgpr(7))); + code.push_back(EncodeVop1(0x2a, 24, Vgpr(8))); + code.push_back(EncodeVop1(0x20, 25, Vgpr(10))); + code.push_back(EncodeVop1(0x21, 26, Vgpr(11))); + code.push_back(EncodeVop1(0x22, 27, Vgpr(10))); + code.push_back(EncodeVop1(0x23, 28, Vgpr(10))); + code.push_back(EncodeVop1(0x24, 29, Vgpr(11))); + code.push_back(EncodeVop1(0x25, 30, Vgpr(4))); + code.push_back(EncodeVop1(0x27, 36, Vgpr(9))); + code.push_back(EncodeVop1(0x2e, 32, Vgpr(9))); + code.push_back(EncodeVop1(0x33, 33, Vgpr(9))); + code.push_back(EncodeVop1(0x35, 34, Vgpr(12))); + code.push_back(EncodeVop1(0x36, 35, Vgpr(12))); + code.push_back(EncodeVop1(0x0e, 38, Vgpr(37))); + code.push_back(EncodeVop1(0x2a, 39, 249)); + code.push_back(EncodeVop1Sdwa(8, 6, 0, 6, 0, 0, 0, 0, 0, 1)); + AppendVMovLiteral(&code, 40, 0xc0000000u); + AppendVop3(&code, 0x1aa, 41, Vgpr(40), 0, 0, 1, 0, false, 1); + AppendVMovLiteral(&code, 2, 0xc0003c00u); // low=1.0h, high=-2.0h + code.push_back(0x7e1016f9u); + code.push_back(0x00250602u); + + const u32 results[] = {13, 14, 15, 16, 17, 18, 42, 19, 20, 21, 22, 23, 24, 25, + 26, 27, 28, 29, 30, 36, 32, 33, 34, 35, 38, 39, 41, 8}; + for (u32 i = 0; i < static_cast(std::size(results)); i++) { + AppendStoreVgpr(&code, results[i], i); + } + AppendEnd(&code); + + return {"VectorFloatConversionOps", + code, + {}, + {0xc0400000u, 0x40e00000u, 7, 0xfffffffdu, 0x00003c00u, + 0x3f800000u, 3, 3, 0x41880000u, 0x42080000u, + 0x424c0000u, 0x42880000u, 0x3f000000u, 0x3e800000u, 0x40000000u, + 0x40400000u, 0x40000000u, 0x40000000u, 0x40000000u, 0x40000000u, + 0x3f000000u, 0x40000000u, 0, 0x3f800000u, 0xbd800000u, + 0x3f800000u, 0x3f800000u, 0x40000000u}, + {O::VMovB32, O::VCvtF32I32, O::VCvtF32U32, + O::VCvtU32F32, O::VCvtI32F32, O::VCvtF16F32, + O::VCvtF32F16, O::VCvtRpiI32F32, O::VCvtFlrI32F32, + O::VCvtOffF32I4, O::VCvtF32Ubyte0, O::VCvtF32Ubyte1, + O::VCvtF32Ubyte2, O::VCvtF32Ubyte3, O::VRcpF32, + O::VFractF32, O::VTruncF32, O::VCeilF32, + O::VRndneF32, O::VFloorF32, O::VExpF32, + O::VLogF32, O::VRsqF32, O::VSqrtF32, + O::VSinF32, O::VCosF32, O::BufferStoreDword, + O::SEndpgm}}; +} + +TestCase CvtF32ToIntSaturatesNaNAndOutOfRange() { + using O = ShaderOpcode; + + std::vector code; + AppendVMovLiteral(&code, 0, 0x7fc00000u); + AppendVMovLiteral(&code, 1, 0xbf800000u); + AppendVMovLiteral(&code, 2, 0x7f800000u); + AppendVMovLiteral(&code, 3, 0x4f800000u); + AppendVMovLiteral(&code, 4, 0xff800000u); + AppendVMovLiteral(&code, 5, 0x4f000000u); + + code.push_back(EncodeVop1(0x07, 10, Vgpr(0))); + code.push_back(EncodeVop1(0x07, 11, Vgpr(1))); + code.push_back(EncodeVop1(0x07, 12, Vgpr(2))); + code.push_back(EncodeVop1(0x07, 13, Vgpr(3))); + code.push_back(EncodeVop1(0x08, 14, Vgpr(0))); + code.push_back(EncodeVop1(0x08, 15, Vgpr(2))); + code.push_back(EncodeVop1(0x08, 16, Vgpr(4))); + code.push_back(EncodeVop1(0x08, 17, Vgpr(5))); + + for (u32 i = 0; i < 8; i++) { + AppendStoreVgpr(&code, 10 + i, i); + } + AppendEnd(&code); + + return {"CvtF32ToIntSaturatesNaNAndOutOfRange", + code, + {}, + {0, 0, 0xffffffffu, 0xffffffffu, 0, 0x7fffffffu, 0x80000000u, + 0x7fffffffu}, + {O::VMovB32, O::VCvtU32F32, O::VCvtI32F32, O::BufferStoreDword, + O::SEndpgm}}; +} + +TestCase VectorSpecialF32FlushesDenormalInputs() { + using O = ShaderOpcode; + + std::vector code; + AppendVMovLiteral(&code, 0, 0x00000001u); + code.push_back(EncodeVop1(0x27, 1, Vgpr(0))); + code.push_back(EncodeVop1(0x2a, 2, Vgpr(0))); + code.push_back(EncodeVop1(0x2e, 3, Vgpr(0))); + code.push_back(EncodeVop1(0x33, 4, Vgpr(0))); + AppendStoreVgpr(&code, 1, 0); + AppendStoreVgpr(&code, 2, 1); + AppendStoreVgpr(&code, 3, 2); + AppendStoreVgpr(&code, 4, 3); + AppendEnd(&code); + + return {"VectorSpecialF32FlushesDenormalInputs", + code, + {}, + {0xff800000u, 0x7f800000u, 0x7f800000u, 0x00000000u}, + {O::VMovB32, O::VLogF32, O::VRcpF32, O::VRsqF32, O::VSqrtF32, + O::BufferStoreDword, O::SEndpgm}}; +} + +TestCase VectorSinCosMaxFiniteSpecialCases() { + using O = ShaderOpcode; + + std::vector code; + AppendVMovLiteral(&code, 0, 0xff7fffffu); + code.push_back(EncodeVop1(0x35, 1, Vgpr(0))); + code.push_back(EncodeVop1(0x36, 2, Vgpr(0))); + AppendStoreVgpr(&code, 1, 0); + AppendStoreVgpr(&code, 2, 1); + AppendEnd(&code); + + return { + "VectorSinCosMaxFiniteSpecialCases", + code, + {}, + {0x00000000u, 0x3f800000u}, + {O::VMovB32, O::VSinF32, O::VCosF32, O::BufferStoreDword, O::SEndpgm}}; +} + +TestCase VectorCompareOps() { + using O = ShaderOpcode; + + std::vector code; + std::vector results; + std::vector expected; + u32 dst = 10; + + code.push_back(EncodeVop1(0x01, 1, InlineU32(1))); + AppendVMovLiteral(&code, 2, 0x40000000u); + AppendVMovLiteral(&code, 3, 0x40400000u); + code.push_back(EncodeVop1(0x01, 4, InlineU32(1))); + AppendVMovLiteral(&code, 5, 0xffffffffu); + + auto append_compare = [&](u32 opcode, u32 src0, u32 src1, bool value) { + if (dst == 31u) { + dst++; + } + code.push_back(EncodeVopc(opcode, src0, src1)); + code.push_back(EncodeVop2(0x01, dst, InlineU32(0), 1)); + results.push_back(dst); + expected.push_back(value ? 1u : 0u); + dst++; + }; + + append_compare(0x00, Vgpr(2), 3, false); + append_compare(0x01, Vgpr(2), 3, true); + append_compare(0x02, Vgpr(2), 3, false); + append_compare(0x03, Vgpr(2), 3, true); + append_compare(0x04, Vgpr(2), 3, false); + append_compare(0x05, Vgpr(2), 3, true); + append_compare(0x06, Vgpr(2), 3, false); + append_compare(0x07, Vgpr(2), 3, true); + append_compare(0x08, Vgpr(2), 3, false); + append_compare(0x09, Vgpr(2), 3, true); + append_compare(0x0a, Vgpr(2), 3, false); + append_compare(0x0b, Vgpr(2), 3, true); + append_compare(0x0c, Vgpr(2), 3, false); + append_compare(0x0d, Vgpr(2), 3, true); + append_compare(0x0e, Vgpr(2), 3, false); + append_compare(0x0f, Vgpr(2), 3, true); + + append_compare(0x80, Vgpr(5), 4, false); + append_compare(0x81, Vgpr(5), 4, true); + append_compare(0x82, Vgpr(5), 4, false); + append_compare(0x83, Vgpr(5), 4, true); + append_compare(0x84, Vgpr(5), 4, false); + append_compare(0x85, Vgpr(5), 4, true); + append_compare(0x86, Vgpr(5), 4, false); + append_compare(0x87, Vgpr(5), 4, true); + + append_compare(0xc0, Vgpr(4), 5, false); + append_compare(0xc1, Vgpr(4), 5, true); + append_compare(0xc2, Vgpr(4), 5, false); + append_compare(0xc3, Vgpr(4), 5, true); + append_compare(0xc4, Vgpr(4), 5, false); + append_compare(0xc5, Vgpr(4), 5, true); + append_compare(0xc6, Vgpr(4), 5, false); + append_compare(0xc7, Vgpr(4), 5, true); + + code.push_back(0x7c1d00f9u); + code.push_back(0x86069201u); + code.push_back(EncodeVopc(0x84, 249, 4)); + code.push_back(EncodeVopcSdwa(5, 19, 1)); + code.push_back(EncodeVopc(0x01, 250, 3)); + code.push_back(EncodeVop2Dpp(2)); + AppendVMovLiteral(&code, 6, 0xc0000000u); + AppendVMovLiteral(&code, 7, 0x3f800000u); + AppendVop3(&code, 0x04, 20, Vgpr(6), Vgpr(7), 0, 1); + for (u32 i = 0; i < static_cast(results.size()); i++) { + AppendStoreVgpr(&code, results[i], i); + } + AppendStoreSgpr(&code, 18, static_cast(results.size())); + expected.push_back(1u); + AppendStoreSgpr(&code, 19, static_cast(results.size() + 1u)); + expected.push_back(0u); + AppendStoreSgpr(&code, 106, static_cast(results.size() + 2u)); + expected.push_back(1u); + AppendStoreSgpr(&code, 20, static_cast(results.size() + 3u)); + expected.push_back(1u); + AppendEnd(&code); + + return {"VectorCompareOps", + code, + {}, + expected, + {O::VMovB32, O::VCndmaskB32, O::VCmpFF32, O::VCmpLtF32, + O::VCmpEqF32, O::VCmpLeF32, O::VCmpGtF32, O::VCmpLgF32, + O::VCmpGeF32, O::VCmpOF32, O::VCmpUF32, O::VCmpNgeF32, + O::VCmpNlgF32, O::VCmpNgtF32, O::VCmpNleF32, O::VCmpNeqF32, + O::VCmpNltF32, O::VCmpTruF32, O::VCmpFI32, O::VCmpLtI32, + O::VCmpEqI32, O::VCmpLeI32, O::VCmpGtI32, O::VCmpNeI32, + O::VCmpGeI32, O::VCmpTI32, O::VCmpFU32, O::VCmpLtU32, + O::VCmpEqU32, O::VCmpLeU32, O::VCmpGtU32, O::VCmpNeU32, + O::VCmpGeU32, O::VCmpTU32, O::BufferStoreDword, O::SEndpgm}}; +} + +TestCase VectorVop3CompareNeU64OnGpu() { + using O = ShaderOpcode; + + std::vector code; + code.push_back(EncodeVop1(0x01, 1, InlineU32(1))); + + code.push_back(EncodeSop1(0x04, 106, 126)); // s_mov_b64 vcc, exec + code.push_back(0xd4e5006au); + code.push_back(0x0000d47eu); // v_cmp_ne_u64 vcc, exec, vcc + code.push_back(EncodeVop2(0x01, 2, InlineU32(0), 1)); + AppendStoreVgpr(&code, 2, 0); + + AppendSMovLiteral(&code, 106, 0); + AppendSMovLiteral(&code, 107, 0); + code.push_back(0xd4e5006au); + code.push_back(0x0000d47eu); // v_cmp_ne_u64 vcc, exec, vcc + code.push_back(EncodeVop2(0x01, 3, InlineU32(0), 1)); + AppendStoreVgpr(&code, 3, 1); + AppendEnd(&code); + + return {"VectorVop3CompareNeU64OnGpu", + code, + {}, + {0, 1}, + {O::VMovB32, O::SMovB64, O::VCmpNeU64, O::VCndmaskB32, O::SMovB32, + O::BufferStoreDword, O::SEndpgm}}; +} + +TestCase VectorCompareClassF32() { + using O = ShaderOpcode; + + std::vector code; + std::vector expected; + AppendVMovU32(&code, 1, 1); + + u32 out = 0; + auto append_class = [&](u32 bits, u32 class_mask, bool match) { + AppendVMovLiteral(&code, 0, bits); + AppendVMovU32(&code, 2, class_mask); + code.push_back(EncodeVopc(0x88, Vgpr(0), 2)); + code.push_back(EncodeVop2(0x01, 3, InlineU32(0), 1)); + AppendStoreVgpr(&code, 3, out++); + expected.push_back(match ? 1u : 0u); + }; + + append_class(0x7fc00000u, 1u << 1u, true); + append_class(0xff800000u, 1u << 2u, true); + append_class(0xbf800000u, 1u << 3u, true); + append_class(0x80000000u, 1u << 5u, true); + append_class(0x00000000u, 1u << 6u, true); + append_class(0x40000000u, 1u << 8u, true); + append_class(0x7f800000u, 1u << 9u, true); + append_class(0x40000000u, 1u << 3u, false); + + AppendVMovLiteral(&code, 0, 0x3f800000u); + AppendVMovU32(&code, 2, 1u << 3u); + AppendVop3(&code, 0x88, 20, Vgpr(0), Vgpr(2), 0, 0, 0, false, 0, 1); + AppendStoreSgprPair(&code, 20, out); + expected.push_back(1u); + expected.push_back(0u); + AppendEnd(&code); + + return {"VectorCompareClassF32", + code, + {}, + expected, + {O::VMovB32, O::VCmpClassF32, O::VCndmaskB32, O::BufferStoreDword, + O::SEndpgm}}; +} + +TestCase VectorCompareF16Ops() { + using O = ShaderOpcode; + + std::vector code; + std::vector expected; + AppendVMovU32(&code, 1, 1); + AppendVMovLiteral(&code, 2, 0x40003c00u); // low=1.0h, high=2.0h + AppendVMovLiteral(&code, 3, 0x44004200u); // low=3.0h, high=4.0h + AppendVMovLiteral(&code, 4, 0x00007e00u); // low=qNaN + + u32 dst = 10; + u32 out = 0; + auto append_compare = [&](u32 opcode, u32 src0, u32 src1, bool value) { + code.push_back(EncodeVopc(opcode, src0, src1)); + code.push_back(EncodeVop2(0x01, dst, InlineU32(0), 1)); + AppendStoreVgpr(&code, dst, out++); + expected.push_back(value ? 1u : 0u); + dst++; + }; + + append_compare(0xc9, Vgpr(2), 3, true); + append_compare(0xca, Vgpr(2), 2, true); + append_compare(0xcb, Vgpr(2), 3, true); + append_compare(0xcc, Vgpr(3), 2, true); + append_compare(0xcd, Vgpr(2), 3, true); + append_compare(0xce, Vgpr(3), 2, true); + append_compare(0xed, Vgpr(4), 2, true); + + auto append_cmpx = [&](u32 opcode, u32 src0, u32 src1) { + code.push_back(EncodeSMovB32(126, InlineU32(1))); + code.push_back(EncodeVopc(opcode, src0, src1)); + AppendStoreVgpr(&code, 1, out++); + expected.push_back(1u); + }; + + append_cmpx(0xd9, Vgpr(2), 3); + append_cmpx(0xda, Vgpr(2), 2); + append_cmpx(0xdb, Vgpr(2), 3); + append_cmpx(0xdc, Vgpr(3), 2); + append_cmpx(0xde, Vgpr(3), 2); + append_cmpx(0xfd, Vgpr(4), 2); + append_cmpx(0xfe, Vgpr(2), 2); + AppendEnd(&code); + + return {"VectorCompareF16Ops", + code, + {}, + expected, + {O::VMovB32, O::SMovB32, O::VCmpLtF16, O::VCmpEqF16, O::VCmpLeF16, + O::VCmpGtF16, O::VCmpLgF16, O::VCmpGeF16, O::VCmpNeqF16, + O::VCmpxLtF16, O::VCmpxEqF16, O::VCmpxLeF16, O::VCmpxGtF16, + O::VCmpxGeF16, O::VCmpxNeqF16, O::VCmpxNltF16, O::VCndmaskB32, + O::BufferStoreDword, O::SEndpgm}}; +} + +TestCase Vop2SdwaCndmaskSourceModifier() { + using O = ShaderOpcode; + + std::vector code; + AppendVMovLiteral(&code, 6, 0x00000000u); + AppendVMovLiteral(&code, 47, 0x12345678u); + AppendVMovLiteral(&code, 53, 0x3f800000u); + code.push_back(EncodeVopc(0xc7, Vgpr(6), 6)); // v_cmp_t_u32 + code.push_back(0x025e6af9u); + code.push_back(0x16060635u); + AppendStoreVgpr(&code, 47, 0); + AppendEnd(&code); + + return {"Vop2SdwaCndmaskSourceModifier", + code, + {}, + {0xbf800000u}, + {O::VMovB32, O::VCmpTU32, O::VCndmaskB32, O::BufferStoreDword, + O::SEndpgm}}; +} + +TestCase Vop3CndmaskUsesSgprMaskLaneBits() { + using O = ShaderOpcode; + + std::vector code = { + EncodeVop1(0x01, 1, 0), + EncodeVop2(0x1a, 1, InlineU32(2), 1), + EncodeVop2(0x25, 1, Vgpr(0), 1), + EncodeVopc(0xc0, Vgpr(0), 0), + }; + AppendSMovLiteral(&code, 4, 1); + AppendSMovLiteral(&code, 5, 0); + AppendVop3(&code, 0x101, 2, InlineU32(10), InlineU32(20), 4); + code.push_back(EncodeVop2(0x1a, 3, InlineU32(2), 1)); + AppendBufferStoreDword(&code, 2, 3); + AppendEnd(&code); + + TestCase test; + test.name = "Vop3CndmaskUsesSgprMaskLaneBits"; + test.code = code; + test.expected = {20, 10, 10, 10, 20, 10, 10, 10}; + test.opcodes = {O::VMovB32, O::VLshlrevB32, O::VAddNcU32, O::VCmpFU32, + O::SMovB32, O::VCndmaskB32, O::BufferStoreDword, O::SEndpgm}; + test.compute_info.threads_num[0] = 4; + test.compute_info.threads_num[1] = 1; + test.compute_info.threads_num[2] = 1; + test.compute_info.group_id[0] = true; + test.compute_info.thread_ids_num = 1; + test.compute_info.workgroup_register = 0; + test.has_compute_info = true; + test.dispatch_x = 2; + return test; +} + +TestCase Vop3CndmaskAllowsDataSourceModifier() { + using O = ShaderOpcode; + + std::vector code; + AppendVMovLiteral(&code, 7, 0x3f800000u); + AppendSMovLiteral(&code, 34, 1); + AppendSMovLiteral(&code, 35, 0); + code.push_back(0xd5010004u); + code.push_back(0x408a0f07u); + AppendStoreVgpr(&code, 4, 0); + AppendEnd(&code); + + return {"Vop3CndmaskAllowsDataSourceModifier", + code, + {}, + {0xbf800000u}, + {O::VMovB32, O::SMovB32, O::VCndmaskB32, O::BufferStoreDword, + O::SEndpgm}}; +} + +TestCase VectorCompareExecOps() { + using O = ShaderOpcode; + + std::vector code; + code.push_back(EncodeVop1(0x01, 1, InlineU32(1))); + AppendVMovLiteral(&code, 2, 0x40000000u); + AppendVMovLiteral(&code, 3, 0x40400000u); + code.push_back(EncodeVop1(0x01, 4, InlineU32(1))); + AppendVMovLiteral(&code, 5, 0xffffffffu); + + u32 out = 0; + auto append_cmpx = [&](u32 opcode, u32 src0, u32 src1) { + code.push_back(EncodeSMovB32(126, InlineU32(1))); + code.push_back(EncodeVopc(opcode, src0, src1)); + AppendStoreVgpr(&code, 1, out++); + }; + + append_cmpx(0x11, Vgpr(2), 3); + append_cmpx(0x12, Vgpr(3), 3); + append_cmpx(0x13, Vgpr(2), 3); + append_cmpx(0x14, Vgpr(3), 2); + append_cmpx(0x15, Vgpr(2), 3); + append_cmpx(0x16, Vgpr(3), 3); + append_cmpx(0x19, Vgpr(2), 3); + append_cmpx(0x1a, Vgpr(3), 3); + append_cmpx(0x1b, Vgpr(2), 3); + append_cmpx(0x1c, Vgpr(3), 2); + append_cmpx(0x1d, Vgpr(2), 3); + append_cmpx(0x1e, Vgpr(3), 3); + append_cmpx(0x91, Vgpr(5), 4); + append_cmpx(0x92, Vgpr(4), 4); + append_cmpx(0x93, Vgpr(5), 4); + append_cmpx(0x94, Vgpr(4), 5); + append_cmpx(0x95, Vgpr(5), 4); + append_cmpx(0x96, Vgpr(4), 4); + append_cmpx(0xd1, Vgpr(4), 5); + append_cmpx(0xd2, Vgpr(4), 4); + append_cmpx(0xd3, Vgpr(4), 5); + append_cmpx(0xd4, Vgpr(5), 4); + append_cmpx(0xd5, Vgpr(4), 5); + append_cmpx(0xd6, Vgpr(4), 4); + AppendEnd(&code); + + return {"VectorCompareExecOps", + code, + {}, + std::vector(out, 1), + {O::SMovB32, O::VMovB32, O::VCmpxLtF32, O::VCmpxEqF32, + O::VCmpxLeF32, O::VCmpxGtF32, O::VCmpxLgF32, O::VCmpxGeF32, + O::VCmpxNgeF32, O::VCmpxNlgF32, O::VCmpxNgtF32, O::VCmpxNleF32, + O::VCmpxNeqF32, O::VCmpxNltF32, O::VCmpxLtI32, O::VCmpxEqI32, + O::VCmpxLeI32, O::VCmpxGtI32, O::VCmpxNeI32, O::VCmpxGeI32, + O::VCmpxLtU32, O::VCmpxEqU32, O::VCmpxLeU32, O::VCmpxGtU32, + O::VCmpxNeU32, O::VCmpxGeU32, O::BufferStoreDword, O::SEndpgm}}; +} + +TestCase VectorVop3FloatCompareNegSourceModifier() { + using O = ShaderOpcode; + + std::vector code; + AppendVMovLiteral(&code, 0, 0x3f800000u); + AppendVMovU32(&code, 1, 0); + AppendVop3(&code, 0x01, 20, Vgpr(0), Vgpr(1), 0, 0, 0, false, 0, + 0x1); // -1.0 < 0.0 + AppendStoreSgprPair(&code, 20, 0); + AppendEnd(&code); + + return {"VectorVop3FloatCompareNegSourceModifier", + code, + {}, + {1, 0}, + {O::VMovB32, O::VCmpLtF32, O::BufferStoreDword, O::SEndpgm}}; +} + +TestCase VectorVop3CmpxWritesExecMask() { + using O = ShaderOpcode; + + std::vector code; + AppendVMovU32(&code, 0, 2); + AppendVMovU32(&code, 1, 1); + AppendVMovU32(&code, 2, 0); + AppendVMovU32(&code, 30, 0); + AppendVop3(&code, 0xd1, 5, Vgpr(0), Vgpr(1)); // v_cmpx_lt_u32, false + AppendVMovU32(&code, 2, 7); + AppendBufferStoreDword(&code, 2, 30); + AppendEnd(&code); + + return {"VectorVop3CmpxWritesExecMask", + code, + {0}, + {0}, + {O::VMovB32, O::VCmpxLtU32, O::BufferStoreDword, O::SEndpgm}}; +} + +TestCase VectorVopcSdwaCmpxWritesExecMask() { + using O = ShaderOpcode; + + std::vector code; + AppendVMovU32(&code, 0, 2); + AppendVMovU32(&code, 1, 1); + AppendVMovU32(&code, 2, 0); + AppendVMovU32(&code, 30, 0); + code.push_back(EncodeVopc(0xd1, 249, 1)); // v_cmpx_lt_u32.sdwa, false + code.push_back(EncodeVopcSdwa(0)); + AppendVMovU32(&code, 2, 7); + AppendBufferStoreDword(&code, 2, 30); + AppendEnd(&code); + + return {"VectorVopcSdwaCmpxWritesExecMask", + code, + {0}, + {0}, + {O::VMovB32, O::VCmpxLtU32, O::BufferStoreDword, O::SEndpgm}}; +} + +TestCase VectorCompareInvertedMaskSelect() { + using O = ShaderOpcode; + + std::vector code; + AppendVMovLiteral(&code, 0, 0x12345678u); + code.push_back(EncodeVopc(0xc7, Vgpr(0), 0)); // v_cmp_t_u32 v0, v0 + code.push_back(EncodeSop1(0x08, 106, 106)); // s_not_b64 vcc, vcc + code.push_back(EncodeVop2(0x01, 1, Vgpr(0), 128)); // v_cndmask_b32 v1, v0, 0 + AppendBufferStoreDword(&code, 1, 30); + AppendEnd(&code); + + return {"VectorCompareInvertedMaskSelect", + code, + {}, + {0x12345678u}, + {O::VMovB32, O::VCmpTU32, O::SNotB64, O::VCndmaskB32, + O::BufferStoreDword, O::SEndpgm}}; +} + +TestCase BranchSelect() { + using O = ShaderOpcode; + + std::vector code = { + EncodeSMovB32(0, InlineU32(0)), + EncodeSopc(0x06, 0, InlineU32(0)), + EncodeSopp(0x05, 2), + EncodeSMovB32(1, InlineU32(1)), + EncodeSopp(0x02, 1), + EncodeSMovB32(1, InlineU32(7)), + EncodeSop2(0x0a, 2, 1, InlineU32(2)), + EncodeVop1(0x01, 0, 2), + }; + AppendBufferStoreDword(&code, 0, 30); + AppendEnd(&code); + return {"BranchSelect", + code, + {}, + {7}, + {O::SMovB32, O::SCmpEqU32, O::SCbranchScc1, O::SBranch, + O::SCselectB32, O::VMovB32, O::BufferStoreDword, O::SEndpgm}}; +} + +TestCase SimpleLoop() { + using O = ShaderOpcode; + + std::vector code = { + EncodeSMovB32(0, InlineU32(0)), + EncodeSopc(0x0a, 0, InlineU32(4)), + EncodeSopp(0x04, 2), + EncodeSop2(0x00, 0, 0, InlineU32(1)), + EncodeSopp(0x02, 0xfffcu), + EncodeVop1(0x01, 0, 0), + }; + AppendBufferStoreDword(&code, 0, 30); + AppendEnd(&code); + return {"SimpleLoop", + code, + {}, + {4}, + {O::SMovB32, O::SCmpLtU32, O::SCbranchScc0, O::SAddU32, O::SBranch, + O::VMovB32, O::BufferStoreDword, O::SEndpgm}}; +} + +TestCase BranchVccnzUsesWholeMask() { + using O = ShaderOpcode; + + std::vector code = { + EncodeVop1(0x01, 1, 0), + EncodeVop2(0x1a, 1, InlineU32(2), 1), + EncodeVop2(0x25, 1, Vgpr(0), 1), + EncodeVop2(0x1a, 3, InlineU32(2), 1), + EncodeVopc(0xc2, InlineU32(0), 0), + EncodeSopp(0x07, 2), + EncodeVop1(0x01, 2, InlineU32(11)), + EncodeSopp(0x02, 1), + }; + AppendVMovU32(&code, 2, 42); + AppendBufferStoreDword(&code, 2, 3); + AppendEnd(&code); + + TestCase test; + test.name = "BranchVccnzUsesWholeMask"; + test.code = code; + test.expected = std::vector(8, 42); + test.opcodes = {O::VMovB32, O::VLshlrevB32, O::VAddNcU32, + O::VCmpEqU32, O::SCbranchVccnz, O::SBranch, + O::BufferStoreDword, O::SEndpgm}; + test.compute_info.threads_num[0] = 4; + test.compute_info.threads_num[1] = 1; + test.compute_info.threads_num[2] = 1; + test.compute_info.group_id[0] = true; + test.compute_info.thread_ids_num = 1; + test.compute_info.workgroup_register = 0; + test.has_compute_info = true; + test.dispatch_x = 2; + return test; +} + +TestCase ScalarMemoryLoadVariants() { + using O = ShaderOpcode; + + struct Load { + u32 opcode; + u32 dst; + u32 byte_offset; + u32 dwords; + }; + + const Load s_loads[] = { + {0x00, 20, 0, 1}, {0x01, 21, 4, 2}, {0x02, 23, 12, 4}, + {0x03, 27, 28, 8}, {0x04, 40, 60, 16}, + }; + const Load s_buffer_loads[] = { + {0x08, 56, 0, 1}, {0x09, 57, 4, 2}, {0x0a, 59, 12, 4}, + {0x0b, 63, 28, 8}, {0x0c, 71, 60, 16}, + }; + + std::vector code; + code.push_back(EncodeSMovB32(0, InlineU32(0))); + for (const auto &load : s_loads) { + AppendSmemLoadOpcode(&code, load.opcode, load.dst, load.byte_offset); + } + for (const auto &load : s_buffer_loads) { + AppendSmemLoadOpcode(&code, load.opcode, load.dst, load.byte_offset); + } + + u32 out = 0; + for (const auto &load : s_loads) { + for (u32 i = 0; i < load.dwords; i++) { + AppendStoreSgpr(&code, load.dst + i, out++); + } + } + for (const auto &load : s_buffer_loads) { + for (u32 i = 0; i < load.dwords; i++) { + AppendStoreSgpr(&code, load.dst + i, out++); + } + } + AppendEnd(&code); + + std::vector initial; + for (u32 i = 0; i < 31u; i++) { + initial.push_back(0x10000000u + i); + } + std::vector expected = initial; + expected.insert(expected.end(), initial.begin(), initial.end()); + + return {"ScalarMemoryLoadVariants", + code, + initial, + expected, + {O::SMovB32, O::SLoadDword, O::SLoadDwordx2, O::SLoadDwordx4, + O::SLoadDwordx8, O::SLoadDwordx16, O::SBufferLoadDword, + O::SBufferLoadDwordx2, O::SBufferLoadDwordx4, O::SBufferLoadDwordx8, + O::SBufferLoadDwordx16, O::VMovB32, O::BufferStoreDword, + O::SEndpgm}}; +} + +TestCase ScalarLoadSignedImmediateOffsetAddsSoffset() { + using O = ShaderOpcode; + + std::vector code; + code.push_back(EncodeSMovB32(0, InlineU32(8))); + code.push_back(EncodeSMovB32(2, InlineU32(0))); + code.push_back(EncodeSMovB32(3, InlineU32(0))); + code.push_back(EncodeSmem0(0x00, 1, 1)); + code.push_back(EncodeSmem1(0x1ffffcu, 0)); + AppendStoreSgpr(&code, 1, 0); + AppendEnd(&code); + + return {"ScalarLoadSignedImmediateOffsetAddsSoffset", + code, + {0x11111111u, 0x22222222u}, + {0x22222222u, 0x22222222u}, + {O::SMovB32, O::SLoadDword, O::BufferStoreDword, O::SEndpgm}}; +} + +TestCase BufferLoadStore() { + using O = ShaderOpcode; + + std::vector code; + AppendBufferLoadDword(&code, 0, 30); + code.push_back(EncodeVop1(0x01, 31, InlineU32(4))); + AppendBufferStoreDword(&code, 0, 31); + AppendEnd(&code); + return {"BufferLoadStore", + code, + {0x11223344u, 0}, + {0x11223344u, 0x11223344u}, + {O::BufferLoadDword, O::VMovB32, O::BufferStoreDword, O::SEndpgm}}; +} + +TestCase BufferLoadDwordOffenIdxenUsesVaddrPlusOneOffset() { + using O = ShaderOpcode; + + std::vector code; + AppendVMovU32(&code, 20, 0); + AppendVMovU32(&code, 21, 8); + code.push_back(EncodeMubuf0(0x0cu, 0, true, true)); + code.push_back(EncodeMubuf1(0, 0, 20)); + AppendStoreVgpr(&code, 0, 0); + AppendEnd(&code); + + return {"BufferLoadDwordOffenIdxenUsesVaddrPlusOneOffset", + code, + {0x11111111u, 0x22222222u, 0x33333333u}, + {0x33333333u, 0x22222222u, 0x33333333u}, + {O::VMovB32, O::BufferLoadDword, O::BufferStoreDword, O::SEndpgm}}; +} + +TestCase BufferStoreDwordOffenIdxenUsesVaddrPlusOneOffset() { + using O = ShaderOpcode; + + std::vector code; + AppendVMovU32(&code, 20, 0); + AppendVMovU32(&code, 21, 12); + AppendVMovLiteral(&code, 0, 0xabcdef01u); + code.push_back(EncodeMubuf0(0x1cu, 0, true, true)); + code.push_back(EncodeMubuf1(0, 0, 20)); + AppendEnd(&code); + + return {"BufferStoreDwordOffenIdxenUsesVaddrPlusOneOffset", + code, + {0x11111111u, 0x22222222u, 0x33333333u, 0x44444444u}, + {0x11111111u, 0x22222222u, 0x33333333u, 0xabcdef01u}, + {O::VMovB32, O::BufferStoreDword, O::SEndpgm}}; +} + +TestCase BufferLoadDwordNoAddressFlagsIgnoresVaddr() { + using O = ShaderOpcode; + + std::vector code; + AppendVMovU32(&code, 20, 8); + code.push_back(EncodeMubuf0(0x0cu, 0, false, false)); + code.push_back(EncodeMubuf1(0, 0, 20)); + AppendStoreVgpr(&code, 0, 3); + AppendEnd(&code); + + return {"BufferLoadDwordNoAddressFlagsIgnoresVaddr", + code, + {0x11111111u, 0x22222222u, 0x33333333u, 0}, + {0x11111111u, 0x22222222u, 0x33333333u, 0x11111111u}, + {O::VMovB32, O::BufferLoadDword, O::BufferStoreDword, O::SEndpgm}}; +} + +TestCase BufferLoadDwordIdxenUsesDescriptorStride() { + using O = ShaderOpcode; + + std::vector code; + AppendVMovU32(&code, 20, 2); + code.push_back(EncodeMubuf0(0x0cu, 0, true, false)); + code.push_back(EncodeMubuf1(0, 0, 20)); + AppendStoreVgpr(&code, 0, 0); + AppendEnd(&code); + + TestCase test; + test.name = "BufferLoadDwordIdxenUsesDescriptorStride"; + test.code = code; + test.initial = {0x11111111u, 0x22222222u, 0x33333333u, + 0x44444444u, 0x55555555u, 0x66666666u, + 0x77777777u, 0x88888888u, 0x99aabbccu}; + test.expected = {0x99aabbccu, 0x22222222u, 0x33333333u, + 0x44444444u, 0x55555555u, 0x66666666u, + 0x77777777u, 0x88888888u, 0x99aabbccu}; + test.opcodes = {O::VMovB32, O::BufferLoadDword, O::BufferStoreDword, + O::SEndpgm}; + test.user_data = MakeStructuredStorageBufferData(16, 3); + test.has_user_data = true; + return test; +} + +TestCase BufferStoreDwordIdxenUsesDescriptorStride() { + using O = ShaderOpcode; + + std::vector code; + AppendVMovU32(&code, 20, 2); + AppendVMovLiteral(&code, 0, 0xabcdef01u); + code.push_back(EncodeMubuf0(0x1cu, 0, true, false)); + code.push_back(EncodeMubuf1(0, 0, 20)); + AppendEnd(&code); + + TestCase test; + test.name = "BufferStoreDwordIdxenUsesDescriptorStride"; + test.code = code; + test.initial = {0x11111111u, 0x22222222u, 0x33333333u, + 0x44444444u, 0x55555555u, 0x66666666u, + 0x77777777u, 0x88888888u, 0x99aabbccu}; + test.expected = {0x11111111u, 0x22222222u, 0x33333333u, + 0x44444444u, 0x55555555u, 0x66666666u, + 0x77777777u, 0x88888888u, 0xabcdef01u}; + test.opcodes = {O::VMovB32, O::BufferStoreDword, O::SEndpgm}; + test.user_data = MakeStructuredStorageBufferData(16, 3); + test.has_user_data = true; + return test; +} + +TestCase BufferLoadVariants() { + using O = ShaderOpcode; + + std::vector code; + AppendVMovU32(&code, 20, 0); + AppendBufferLoadOpcode(&code, 0x08, 0, 20); + AppendVMovU32(&code, 20, 2); + AppendBufferLoadOpcode(&code, 0x09, 1, 20); + AppendVMovU32(&code, 20, 0); + AppendBufferLoadOpcode(&code, 0x0a, 2, 20); + AppendVMovU32(&code, 20, 2); + AppendBufferLoadOpcode(&code, 0x0b, 3, 20); + AppendVMovU32(&code, 20, 8); + AppendBufferLoadOpcode(&code, 0x0d, 4, 20); + AppendBufferLoadOpcode(&code, 0x0f, 6, 20); + AppendBufferLoadOpcode(&code, 0x0e, 9, 20); + + for (u32 i = 0; i < 13u; i++) { + AppendStoreVgpr(&code, i, i); + } + AppendEnd(&code); + + return {"BufferLoadVariants", + code, + {0x80ff7f01u, 0x00008001u, 0x11223344u, 0x55667788u, 0x99aabbccu, + 0xddeeff00u}, + {0x01u, 0xffffffffu, 0x7f01u, 0xffff80ffu, 0x11223344u, 0x55667788u, + 0x11223344u, 0x55667788u, 0x99aabbccu, 0x11223344u, 0x55667788u, + 0x99aabbccu, 0xddeeff00u}, + {O::BufferLoadUbyte, O::BufferLoadSbyte, O::BufferLoadUshort, + O::BufferLoadSshort, O::BufferLoadDwordx2, O::BufferLoadDwordx3, + O::BufferLoadDwordx4, O::VMovB32, O::BufferStoreDword, O::SEndpgm}}; +} + +TestCase BufferStoreVariants() { + using O = ShaderOpcode; + + std::vector code; + AppendVMovU32(&code, 20, 1); + AppendVMovU32(&code, 0, 0xaa); + AppendBufferStoreOpcode(&code, 0x18, 0, 20); + AppendVMovU32(&code, 20, 2); + AppendVMovLiteral(&code, 1, 0x0000bbccu); + AppendBufferStoreOpcode(&code, 0x1a, 1, 20); + AppendVMovU32(&code, 20, 4); + AppendVMovLiteral(&code, 2, 0x11111111u); + AppendVMovLiteral(&code, 3, 0x22222222u); + AppendBufferStoreOpcode(&code, 0x1d, 2, 20); + AppendVMovU32(&code, 20, 12); + AppendVMovLiteral(&code, 4, 0x33333333u); + AppendVMovLiteral(&code, 5, 0x44444444u); + AppendVMovLiteral(&code, 6, 0x55555555u); + AppendBufferStoreOpcode(&code, 0x1f, 4, 20); + AppendVMovU32(&code, 20, 24); + AppendVMovLiteral(&code, 7, 0x66666666u); + AppendVMovLiteral(&code, 8, 0x77777777u); + AppendVMovLiteral(&code, 9, 0x88888888u); + AppendVMovLiteral(&code, 10, 0x99999999u); + AppendBufferStoreOpcode(&code, 0x1e, 7, 20); + AppendEnd(&code); + + return {"BufferStoreVariants", + code, + std::vector(10, 0), + {0xbbccaa00u, 0x11111111u, 0x22222222u, 0x33333333u, 0x44444444u, + 0x55555555u, 0x66666666u, 0x77777777u, 0x88888888u, 0x99999999u}, + {O::VMovB32, O::BufferStoreByte, O::BufferStoreShort, + O::BufferStoreDwordx2, O::BufferStoreDwordx3, O::BufferStoreDwordx4, + O::SEndpgm}}; +} + +TestCase BufferFormatVariants() { + using O = ShaderOpcode; + + std::vector code; + AppendVMovU32(&code, 20, 0); + AppendBufferLoadOpcode(&code, 0x00, 0, 20); + AppendBufferLoadOpcode(&code, 0x01, 1, 20); + AppendBufferLoadOpcode(&code, 0x02, 3, 20); + AppendBufferLoadOpcode(&code, 0x03, 6, 20); + for (u32 i = 0; i < 10u; i++) { + AppendStoreVgpr(&code, i, i); + } + AppendEnd(&code); + + TestCase load; + load.name = "BufferFormatLoadVariants"; + load.code = code; + load.initial = {0x01020304u, 0x11121314u, 0x21222324u, 0x31323334u}; + load.expected = {0x01020304u, 0x01020304u, 0x11121314u, 0x01020304u, + 0x11121314u, 0x21222324u, 0x01020304u, 0x11121314u, + 0x21222324u, 0x31323334u}; + load.opcodes = {O::BufferLoadFormatX, + O::BufferLoadFormatXy, + O::BufferLoadFormatXyz, + O::BufferLoadFormatXyzw, + O::VMovB32, + O::BufferStoreDword, + O::SEndpgm}; + return load; +} + +TestCase BufferFormatStoreVariants() { + using O = ShaderOpcode; + + std::vector code; + AppendVMovU32(&code, 20, 0); + AppendVMovLiteral(&code, 0, 0x01020304u); + AppendBufferStoreOpcode(&code, 0x04, 0, 20); + AppendVMovU32(&code, 20, 4); + AppendVMovLiteral(&code, 1, 0x11121314u); + AppendVMovLiteral(&code, 2, 0x21222324u); + AppendBufferStoreOpcode(&code, 0x05, 1, 20); + AppendVMovU32(&code, 20, 12); + AppendVMovLiteral(&code, 3, 0x31323334u); + AppendVMovLiteral(&code, 4, 0x41424344u); + AppendVMovLiteral(&code, 5, 0x51525354u); + AppendBufferStoreOpcode(&code, 0x06, 3, 20); + AppendVMovU32(&code, 20, 24); + AppendVMovLiteral(&code, 6, 0x61626364u); + AppendVMovLiteral(&code, 7, 0x71727374u); + AppendVMovLiteral(&code, 8, 0x81828384u); + AppendVMovLiteral(&code, 9, 0x91929394u); + AppendBufferStoreOpcode(&code, 0x07, 6, 20); + AppendEnd(&code); + + return {"BufferFormatStoreVariants", + code, + std::vector(10, 0), + {0x01020304u, 0x11121314u, 0x21222324u, 0x31323334u, 0x41424344u, + 0x51525354u, 0x61626364u, 0x71727374u, 0x81828384u, 0x91929394u}, + {O::VMovB32, O::BufferStoreFormatX, O::BufferStoreFormatXy, + O::BufferStoreFormatXyz, O::BufferStoreFormatXyzw, O::SEndpgm}}; +} + +TestCase BufferLoadFormatXResource8UintZeroExtendsByte() { + using O = ShaderOpcode; + + std::vector code; + AppendVMovU32(&code, 20, 0); + code.push_back(EncodeMubuf0(0x00u)); + code.push_back(EncodeMubuf1(0, 0, 20)); + AppendStoreVgpr(&code, 0, 1); + AppendEnd(&code); + + TestCase test; + test.name = "BufferLoadFormatXResource8UintZeroExtendsByte"; + test.code = code; + test.initial = {0x11223344u, 0}; + test.expected = {0x11223344u, 0x00000044u}; + test.opcodes = {O::VMovB32, O::BufferLoadFormatX, O::BufferStoreDword, + O::SEndpgm}; + test.user_data = MakeStructuredStorageBufferData(0, 8, false, 5); + test.has_user_data = true; + return test; +} + +TestCase BufferStoreFormatXResource16UintWritesHalfword() { + using O = ShaderOpcode; + + std::vector code; + AppendVMovU32(&code, 20, 0); + AppendVMovLiteral(&code, 0, 0x0000aaaau); + code.push_back(EncodeMubuf0(0x04u)); + code.push_back(EncodeMubuf1(0, 0, 20)); + AppendEnd(&code); + + TestCase test; + test.name = "BufferStoreFormatXResource16UintWritesHalfword"; + test.code = code; + test.initial = {0x11223344u}; + test.expected = {0x1122aaaau}; + test.opcodes = {O::VMovB32, O::BufferStoreFormatX, O::SEndpgm}; + test.user_data = MakeStructuredStorageBufferData(0, 4, false, 11); + test.has_user_data = true; + return test; +} + +TestCase BufferLoadFormatXyResource88UintExtractsBytes() { + using O = ShaderOpcode; + + std::vector code; + AppendVMovU32(&code, 20, 0); + code.push_back(EncodeMubuf0(0x01u)); + code.push_back(EncodeMubuf1(0, 0, 20)); + AppendStoreVgpr(&code, 0, 2); + AppendStoreVgpr(&code, 1, 3); + AppendEnd(&code); + + TestCase test; + test.name = "BufferLoadFormatXyResource8_8UintExtractsBytes"; + test.code = code; + test.initial = {0x0000807fu, 0x55667788u, 0, 0}; + test.expected = {0x0000807fu, 0x55667788u, 0x0000007fu, 0x00000080u}; + test.opcodes = {O::VMovB32, O::BufferLoadFormatXy, O::BufferStoreDword, + O::SEndpgm}; + test.user_data = MakeStructuredStorageBufferData(0, 8, false, 18); + test.has_user_data = true; + return test; +} + +TestCase BufferLoadFormatXyResource8888UnormConvertsFirstTwoComponents() { + using O = ShaderOpcode; + + std::vector code; + AppendVMovU32(&code, 20, 0); + code.push_back(EncodeMubuf0(0x01u)); + code.push_back(EncodeMubuf1(0, 0, 20)); + AppendStoreVgpr(&code, 0, 0); + AppendStoreVgpr(&code, 1, 1); + AppendEnd(&code); + + TestCase test; + test.name = + "BufferLoadFormatXyResource8_8_8_8UnormConvertsFirstTwoComponents"; + test.code = code; + test.initial = {0x44332211u, 0xdeadbeefu}; + test.expected = {0x3d888889u, 0x3e088889u}; + test.opcodes = {O::VMovB32, O::BufferLoadFormatXy, O::BufferStoreDword, + O::SEndpgm}; + test.user_data = MakeStructuredStorageBufferData(0, 8, false, 56); + test.has_user_data = true; + return test; +} + +TestCase BufferStoreFormatXyResource88UintWritesBytes() { + using O = ShaderOpcode; + + std::vector code; + AppendVMovU32(&code, 20, 0); + AppendVMovLiteral(&code, 0, 0x000000aau); + AppendVMovLiteral(&code, 1, 0x000000bbu); + code.push_back(EncodeMubuf0(0x05u)); + code.push_back(EncodeMubuf1(0, 0, 20)); + AppendEnd(&code); + + TestCase test; + test.name = "BufferStoreFormatXyResource8_8UintWritesBytes"; + test.code = code; + test.initial = {0x11223344u, 0x55667788u}; + test.expected = {0x1122bbaau, 0x55667788u}; + test.opcodes = {O::VMovB32, O::BufferStoreFormatXy, O::SEndpgm}; + test.user_data = MakeStructuredStorageBufferData(0, 8, false, 18); + test.has_user_data = true; + return test; +} + +TestCase BufferStoreFormatXyResource32UintWritesOneDword() { + using O = ShaderOpcode; + + std::vector code; + AppendVMovU32(&code, 20, 0); + AppendVMovLiteral(&code, 0, 0xabcdef01u); + AppendVMovLiteral(&code, 1, 0x12345678u); + code.push_back(EncodeMubuf0(0x05u)); + code.push_back(EncodeMubuf1(0, 0, 20)); + AppendEnd(&code); + + TestCase test; + test.name = "BufferStoreFormatXyResource32UintWritesOneDword"; + test.code = code; + test.initial = {0x11111111u, 0x22222222u}; + test.expected = {0xabcdef01u, 0x22222222u}; + test.opcodes = {O::VMovB32, O::BufferStoreFormatXy, O::SEndpgm}; + test.user_data = MakeStructuredStorageBufferData(0, 8, false, 20); + test.has_user_data = true; + return test; +} + +TestCase BufferStoreFormatXyzResource3232UintWritesTwoDwords() { + using O = ShaderOpcode; + + std::vector code; + AppendVMovU32(&code, 20, 0); + AppendVMovLiteral(&code, 0, 0xabcdef01u); + AppendVMovLiteral(&code, 1, 0x12345678u); + AppendVMovLiteral(&code, 2, 0x0badc0deu); + code.push_back(EncodeMubuf0(0x06u)); + code.push_back(EncodeMubuf1(0, 0, 20)); + AppendEnd(&code); + + TestCase test; + test.name = "BufferStoreFormatXyzResource32_32UintWritesTwoDwords"; + test.code = code; + test.initial = {0x11111111u, 0x22222222u, 0x33333333u}; + test.expected = {0xabcdef01u, 0x12345678u, 0x33333333u}; + test.opcodes = {O::VMovB32, O::BufferStoreFormatXyz, O::SEndpgm}; + test.user_data = MakeStructuredStorageBufferData(0, 8, false, 62); + test.has_user_data = true; + return test; +} + +TestCase BufferStoreFormatXyzwResource323232UintWritesThreeDwords() { + using O = ShaderOpcode; + + std::vector code; + AppendVMovU32(&code, 20, 0); + AppendVMovLiteral(&code, 0, 0xabcdef01u); + AppendVMovLiteral(&code, 1, 0x12345678u); + AppendVMovLiteral(&code, 2, 0x0badc0deu); + AppendVMovLiteral(&code, 3, 0xfeedfaceu); + code.push_back(EncodeMubuf0(0x07u)); + code.push_back(EncodeMubuf1(0, 0, 20)); + AppendEnd(&code); + + TestCase test; + test.name = "BufferStoreFormatXyzwResource32_32_32UintWritesThreeDwords"; + test.code = code; + test.initial = {0x11111111u, 0x22222222u, 0x33333333u, 0x44444444u}; + test.expected = {0xabcdef01u, 0x12345678u, 0x0badc0deu, 0x44444444u}; + test.opcodes = {O::VMovB32, O::BufferStoreFormatXyzw, O::SEndpgm}; + test.user_data = MakeStructuredStorageBufferData(0, 8, false, 72); + test.has_user_data = true; + return test; +} + +TestCase BufferStoreFormatXyzResource8UintWritesOneByte() { + using O = ShaderOpcode; + + std::vector code; + AppendVMovU32(&code, 20, 0); + AppendVMovLiteral(&code, 0, 0x000000aau); + AppendVMovLiteral(&code, 1, 0x000000bbu); + AppendVMovLiteral(&code, 2, 0x000000ccu); + code.push_back(EncodeMubuf0(0x06u)); + code.push_back(EncodeMubuf1(0, 0, 20)); + AppendEnd(&code); + + TestCase test; + test.name = "BufferStoreFormatXyzResource8UintWritesOneByte"; + test.code = code; + test.initial = {0x11223344u, 0x55667788u, 0x99aabbccu}; + test.expected = {0x112233aau, 0x55667788u, 0x99aabbccu}; + test.opcodes = {O::VMovB32, O::BufferStoreFormatXyz, O::SEndpgm}; + test.user_data = MakeStructuredStorageBufferData(0, 8, false, 5); + test.has_user_data = true; + return test; +} + +TestCase BufferStoreFormatXAddTidUsesLaneIndex() { + using O = ShaderOpcode; + + std::vector code; + AppendVMovLiteral(&code, 0, 0x12345678u); + code.push_back(EncodeMubuf0(0x04u, 0, false, false)); + code.push_back(EncodeMubuf1(0, 0, 20)); + AppendEnd(&code); + + TestCase test; + test.name = "BufferStoreFormatXAddTidUsesLaneIndex"; + test.code = code; + test.initial = std::vector(4, 0); + test.expected = std::vector(4, 0x12345678u); + test.opcodes = {O::VMovB32, O::BufferStoreFormatX, O::SEndpgm}; + test.compute_info.threads_num[0] = 4; + test.compute_info.threads_num[1] = 1; + test.compute_info.threads_num[2] = 1; + test.compute_info.thread_ids_num = 1; + test.has_compute_info = true; + test.user_data = MakeStructuredStorageBufferData(4, 4, true); + test.has_user_data = true; + return test; +} + +TestCase BufferStoreFormatXDropsOutOfRangeRecord() { + using O = ShaderOpcode; + + std::vector code; + AppendVMovU32(&code, 20, 1); + AppendVMovLiteral(&code, 0, 0xabcdef01u); + code.push_back(EncodeMubuf0(0x04u, 0, true, false)); + code.push_back(EncodeMubuf1(0, 0, 20)); + AppendEnd(&code); + + TestCase test; + test.name = "BufferStoreFormatXDropsOutOfRangeRecord"; + test.code = code; + test.initial = {0x11111111u, 0x22222222u}; + test.expected = {0x11111111u, 0x22222222u}; + test.storage_buffer_range_dwords = 1; + test.opcodes = {O::VMovB32, O::BufferStoreFormatX, O::SEndpgm}; + test.user_data = MakeStructuredStorageBufferData(4, 1, false, 20); + test.has_user_data = true; + return test; +} + +TestCase TBufferLoadVariants() { + using O = ShaderOpcode; + + std::vector code; + AppendVMovU32(&code, 20, 0); + AppendTBufferLoadOpcode(&code, 0x00, 0, 20); + AppendTBufferLoadOpcode(&code, 0x01, 1, 20); + AppendTBufferLoadOpcode(&code, 0x02, 3, 20); + AppendTBufferLoadOpcode(&code, 0x03, 6, 20); + for (u32 i = 0; i < 10u; i++) { + AppendStoreVgpr(&code, i, i); + } + AppendEnd(&code); + + return {"TBufferLoadVariants", + code, + {0x01020304u, 0x11121314u, 0x21222324u, 0x31323334u}, + {0x01020304u, 0x01020304u, 0x11121314u, 0x01020304u, 0x11121314u, + 0x21222324u, 0x01020304u, 0x11121314u, 0x21222324u, 0x31323334u}, + {O::TBufferLoadFormatX, O::TBufferLoadFormatXy, + O::TBufferLoadFormatXyz, O::TBufferLoadFormatXyzw, O::VMovB32, + O::BufferStoreDword, O::SEndpgm}}; +} + +TestCase TBufferStoreFormatX8UintWritesOneByte() { + using O = ShaderOpcode; + + std::vector code; + AppendVMovU32(&code, 20, 0); + AppendVMovU32(&code, 0, 0xaa); + code.push_back(EncodeMtbuf0(0x04, 5, 0)); + code.push_back(EncodeMtbuf1(0x04, 0, 0, 20)); + AppendEnd(&code); + + return {"TBufferStoreFormatX8UintWritesOneByte", + code, + {0x11223344u}, + {0x112233aau}, + {O::VMovB32, O::TBufferStoreFormatX, O::SEndpgm}}; +} + +TestCase TBufferLoadFormatX8UintZeroExtendsByte() { + using O = ShaderOpcode; + + std::vector code; + AppendVMovU32(&code, 20, 0); + code.push_back(EncodeMtbuf0(0x00, 5, 0)); + code.push_back(EncodeMtbuf1(0x00, 0, 0, 20)); + AppendStoreVgpr(&code, 0, 0); + AppendEnd(&code); + + return {"TBufferLoadFormatX8UintZeroExtendsByte", + code, + {0x11223344u, 0}, + {0x00000044u, 0}, + {O::VMovB32, O::TBufferLoadFormatX, O::BufferStoreDword, O::SEndpgm}}; +} + +TestCase TBufferLoadFormatX8888UintExtractsFirstByte() { + using O = ShaderOpcode; + + std::vector code; + AppendVMovU32(&code, 20, 0); + AppendTBufferLoadFormatOpcode(&code, 0x00, 0, 20, + Prospero::BufferFormat::k8_8_8_8UInt); + AppendStoreVgpr(&code, 0, 0); + AppendEnd(&code); + + return {"TBufferLoadFormatX8_8_8_8UintExtractsFirstByte", + code, + {0x44332211u, 0}, + {0x00000011u, 0}, + {O::VMovB32, O::TBufferLoadFormatX, O::BufferStoreDword, O::SEndpgm}}; +} + +TestCase TBufferLoadFormatXIdxenUsesDescriptorStride() { + using O = ShaderOpcode; + + std::vector code; + AppendVMovU32(&code, 20, 2); + code.push_back(EncodeMtbuf0(0x00, 5, 0, 0, true, false)); + code.push_back(EncodeMtbuf1(0x00, 0, 0, 20)); + AppendStoreVgpr(&code, 0, 0); + AppendEnd(&code); + + TestCase test; + test.name = "TBufferLoadFormatXIdxenUsesDescriptorStride"; + test.code = code; + test.initial = {0x01020304u, 0, 0, 0, 0, 0, 0, 0, 0x0000007eu}; + test.expected = {0x0000007eu, 0, 0, 0, 0, 0, 0, 0, 0x0000007eu}; + test.opcodes = {O::VMovB32, O::TBufferLoadFormatX, O::BufferStoreDword, + O::SEndpgm}; + test.user_data = MakeStructuredStorageBufferData(16, 3); + test.has_user_data = true; + return test; +} + +TestCase TBufferLoadFormatX16FloatConvertsToFloat() { + using O = ShaderOpcode; + + std::vector code; + AppendVMovU32(&code, 20, 0); + AppendTBufferLoadFormatOpcode(&code, 0x00, 0, 20, Prospero::BufferFormat::k16Float); + AppendStoreVgpr(&code, 0, 0); + AppendEnd(&code); + + return {"TBufferLoadFormatX16FloatConvertsToFloat", + code, + {0x00003c00u, 0}, + {0x3f800000u, 0}, + {O::VMovB32, O::TBufferLoadFormatX, O::BufferStoreDword, O::SEndpgm}}; +} + +TestCase TBufferLoadFormatXSintSignExtendsSubDword() { + using O = ShaderOpcode; + + std::vector code; + AppendVMovU32(&code, 20, 0); + code.push_back(EncodeMtbuf0(0x00, 6, 0)); + code.push_back(EncodeMtbuf1(0x00, 0, 0, 20)); + AppendVMovU32(&code, 20, 4); + code.push_back(EncodeMtbuf0(0x00, 12, 0)); + code.push_back(EncodeMtbuf1(0x00, 1, 0, 20)); + AppendStoreVgpr(&code, 0, 0); + AppendStoreVgpr(&code, 1, 1); + AppendEnd(&code); + + return {"TBufferLoadFormatXSintSignExtendsSubDword", + code, + {0x00000080u, 0x00008001u}, + {0xffffff80u, 0xffff8001u}, + {O::VMovB32, O::TBufferLoadFormatX, O::BufferStoreDword, O::SEndpgm}}; +} + +TestCase TBufferStoreFormatXSintWritesSubDword() { + using O = ShaderOpcode; + + std::vector code; + AppendVMovU32(&code, 20, 0); + AppendVMovLiteral(&code, 0, 0xffffff80u); + code.push_back(EncodeMtbuf0(0x04, 6, 0)); + code.push_back(EncodeMtbuf1(0x04, 0, 0, 20)); + AppendVMovU32(&code, 20, 4); + AppendVMovLiteral(&code, 1, 0xffff8001u); + code.push_back(EncodeMtbuf0(0x04, 12, 0)); + code.push_back(EncodeMtbuf1(0x04, 1, 0, 20)); + AppendEnd(&code); + + return {"TBufferStoreFormatXSintWritesSubDword", + code, + {0x11223344u, 0x55667788u}, + {0x11223380u, 0x55668001u}, + {O::VMovB32, O::TBufferStoreFormatX, O::SEndpgm}}; +} + +TestCase TBufferLoadFormatXy88IntegerComponents() { + using O = ShaderOpcode; + + std::vector code; + AppendVMovU32(&code, 20, 0); + code.push_back(EncodeMtbuf0(0x01, 2, 1)); + code.push_back(EncodeMtbuf1(0x01, 0, 0, 20)); + AppendVMovU32(&code, 20, 4); + code.push_back(EncodeMtbuf0(0x01, 3, 1)); + code.push_back(EncodeMtbuf1(0x01, 2, 0, 20)); + for (u32 i = 0; i < 4; i++) { + AppendStoreVgpr(&code, i, i); + } + AppendEnd(&code); + + return { + "TBufferLoadFormatXy8_8IntegerComponents", + code, + {0x0000807fu, 0x00007f80u, 0, 0}, + {0x0000007fu, 0x00000080u, 0xffffff80u, 0x0000007fu}, + {O::VMovB32, O::TBufferLoadFormatXy, O::BufferStoreDword, O::SEndpgm}}; +} + +TestCase TBufferStoreFormatXy88IntegerComponents() { + using O = ShaderOpcode; + + std::vector code; + AppendVMovU32(&code, 20, 0); + AppendVMovU32(&code, 0, 0xaa); + AppendVMovU32(&code, 1, 0xbb); + code.push_back(EncodeMtbuf0(0x05, 2, 1)); + code.push_back(EncodeMtbuf1(0x05, 0, 0, 20)); + AppendVMovU32(&code, 20, 4); + AppendVMovLiteral(&code, 2, 0xffffff80u); + AppendVMovU32(&code, 3, 0x7f); + code.push_back(EncodeMtbuf0(0x05, 3, 1)); + code.push_back(EncodeMtbuf1(0x05, 2, 0, 20)); + AppendEnd(&code); + + return {"TBufferStoreFormatXy8_8IntegerComponents", + code, + {0x11223344u, 0x55667788u}, + {0x1122bbaau, 0x55667f80u}, + {O::VMovB32, O::TBufferStoreFormatXy, O::SEndpgm}}; +} + +TestCase TBufferLoadFormatXy1616IntegerComponents() { + using O = ShaderOpcode; + + std::vector code; + AppendVMovU32(&code, 20, 0); + code.push_back(EncodeMtbuf0(0x01, 11, 1)); + code.push_back(EncodeMtbuf1(0x01, 0, 0, 20)); + AppendVMovU32(&code, 20, 4); + code.push_back(EncodeMtbuf0(0x01, 12, 1)); + code.push_back(EncodeMtbuf1(0x01, 2, 0, 20)); + for (u32 i = 0; i < 4; i++) { + AppendStoreVgpr(&code, i, i); + } + AppendEnd(&code); + + return { + "TBufferLoadFormatXy16_16IntegerComponents", + code, + {0x80017fffu, 0x7fff8000u, 0, 0}, + {0x00007fffu, 0x00008001u, 0xffff8000u, 0x00007fffu}, + {O::VMovB32, O::TBufferLoadFormatXy, O::BufferStoreDword, O::SEndpgm}}; +} + +TestCase TBufferStoreFormatXy1616IntegerComponents() { + using O = ShaderOpcode; + + std::vector code; + AppendVMovU32(&code, 20, 0); + AppendVMovU32(&code, 0, 0xaaaa); + AppendVMovU32(&code, 1, 0xbbbb); + code.push_back(EncodeMtbuf0(0x05, 11, 1)); + code.push_back(EncodeMtbuf1(0x05, 0, 0, 20)); + AppendVMovU32(&code, 20, 4); + AppendVMovLiteral(&code, 2, 0xffff8000u); + AppendVMovU32(&code, 3, 0x7fff); + code.push_back(EncodeMtbuf0(0x05, 12, 1)); + code.push_back(EncodeMtbuf1(0x05, 2, 0, 20)); + AppendEnd(&code); + + return {"TBufferStoreFormatXy16_16IntegerComponents", + code, + {0x11223344u, 0x55667788u}, + {0xbbbbaaaau, 0x7fff8000u}, + {O::VMovB32, O::TBufferStoreFormatXy, O::SEndpgm}}; +} + +TestCase TBufferLoadFormatXyz16161616UintLoadsHalfwords() { + using O = ShaderOpcode; + + std::vector code; + AppendVMovU32(&code, 20, 0); + AppendTBufferLoadFormatOpcode(&code, 0x02, 0, 20, + Prospero::BufferFormat::k16_16_16_16UInt); + for (u32 i = 0; i < 3; i++) { + AppendStoreVgpr(&code, i, i); + } + AppendEnd(&code); + + return { + "TBufferLoadFormatXyz16_16_16_16UintLoadsHalfwords", + code, + {0x22221111u, 0x44443333u, 0}, + {0x00001111u, 0x00002222u, 0x00003333u}, + {O::VMovB32, O::TBufferLoadFormatXyz, O::BufferStoreDword, O::SEndpgm}}; +} + +TestCase TBufferLoadFormatXy88UnormConvertsToFloat() { + using O = ShaderOpcode; + + std::vector code; + AppendVMovU32(&code, 20, 0); + code.push_back(EncodeMtbuf0(0x01, 14, 0)); + code.push_back(EncodeMtbuf1(0x01, 0, 0, 20)); + AppendStoreVgpr(&code, 0, 0); + AppendStoreVgpr(&code, 1, 1); + AppendEnd(&code); + + return { + "TBufferLoadFormatXy8_8UnormConvertsToFloat", + code, + {0x0000ff80u, 0}, + {0x3f008081u, 0x3f800000u}, + {O::VMovB32, O::TBufferLoadFormatXy, O::BufferStoreDword, O::SEndpgm}}; +} + +TestCase TBufferLoadFormatXy88SnormConvertsToFloat() { + using O = ShaderOpcode; + + std::vector code; + AppendVMovU32(&code, 20, 0); + AppendTBufferLoadFormatOpcode(&code, 0x01, 0, 20, Prospero::BufferFormat::k8_8SNorm); + AppendStoreVgpr(&code, 0, 0); + AppendStoreVgpr(&code, 1, 1); + AppendEnd(&code); + + return { + "TBufferLoadFormatXy8_8SnormConvertsToFloat", + code, + {0x00007f80u, 0}, + {0xbf800000u, 0x3f800000u}, + {O::VMovB32, O::TBufferLoadFormatXy, O::BufferStoreDword, O::SEndpgm}}; +} + +TestCase TBufferLoadFormatXy1616UnormConvertsToFloat() { + using O = ShaderOpcode; + + std::vector code; + AppendVMovU32(&code, 20, 0); + code.push_back(EncodeMtbuf0(0x01, 7, 1)); + code.push_back(EncodeMtbuf1(0x01, 0, 0, 20)); + AppendStoreVgpr(&code, 0, 0); + AppendStoreVgpr(&code, 1, 1); + AppendEnd(&code); + + return { + "TBufferLoadFormatXy16_16UnormConvertsToFloat", + code, + {0xffff8000u, 0}, + {0x3f000080u, 0x3f800000u}, + {O::VMovB32, O::TBufferLoadFormatXy, O::BufferStoreDword, O::SEndpgm}}; +} + +TestCase TBufferLoadFormatXy8888UnormConvertsFirstTwoComponents() { + using O = ShaderOpcode; + + std::vector code; + AppendVMovU32(&code, 20, 0); + code.push_back(EncodeMtbuf0(0x01, 8, 3)); + code.push_back(EncodeMtbuf1(0x01, 0, 0, 20)); + AppendStoreVgpr(&code, 0, 0); + AppendStoreVgpr(&code, 1, 1); + AppendEnd(&code); + + return { + "TBufferLoadFormatXy8_8_8_8UnormConvertsFirstTwoComponents", + code, + {0x44332211u, 0xdeadbeefu}, + {0x3d888889u, 0x3e088889u}, + {O::VMovB32, O::TBufferLoadFormatXy, O::BufferStoreDword, O::SEndpgm}}; +} + +TestCase TBufferLoadFormatXyzw8888UintExtractsBytes() { + using O = ShaderOpcode; + + std::vector code; + AppendVMovU32(&code, 20, 0); + AppendTBufferLoadFormatOpcode(&code, 0x03, 0, 20, + Prospero::BufferFormat::k8_8_8_8UInt); + for (u32 i = 0; i < 4; i++) { + AppendStoreVgpr(&code, i, i); + } + AppendEnd(&code); + + return { + "TBufferLoadFormatXyzw8_8_8_8UintExtractsBytes", + code, + {0x44332211u, 0xaaaaaaaau, 0xbbbbbbbbu, 0xccccccccu}, + {0x00000011u, 0x00000022u, 0x00000033u, 0x00000044u}, + {O::VMovB32, O::TBufferLoadFormatXyzw, O::BufferStoreDword, O::SEndpgm}}; +} + +TestCase TBufferLoadFormatXyzw1010102SnormConvertsToFloat() { + using O = ShaderOpcode; + + std::vector code; + AppendVMovU32(&code, 20, 0); + AppendTBufferLoadFormatOpcode(&code, 0x03, 0, 20, + Prospero::BufferFormat::k10_10_10_2SNorm); + for (u32 i = 0; i < 4; i++) { + AppendStoreVgpr(&code, i, i); + } + AppendEnd(&code); + + return { + "TBufferLoadFormatXyzw10_10_10_2SnormConvertsToFloat", + code, + {0x800801ffu, 0, 0, 0}, + {0x3f800000u, 0xbf800000u, 0, 0xbf800000u}, + {O::VMovB32, O::TBufferLoadFormatXyzw, O::BufferStoreDword, O::SEndpgm}}; +} + +TestCase TBufferLoadFormatXyz111110FloatUnpacks() { + using O = ShaderOpcode; + + std::vector code; + AppendVMovU32(&code, 20, 0); + AppendTBufferLoadFormatOpcode(&code, 0x02, 0, 20, + Prospero::BufferFormat::k11_11_10Float); + for (u32 i = 0; i < 3; i++) { + AppendStoreVgpr(&code, i, i); + } + AppendEnd(&code); + + return { + "TBufferLoadFormatXyz11_11_10FloatUnpacks", + code, + {0x781e03c0u, 0, 0}, + {0x3f800000u, 0x3f800000u, 0x3f800000u}, + {O::VMovB32, O::TBufferLoadFormatXyz, O::BufferStoreDword, O::SEndpgm}}; +} + +TestCase TBufferLoadFormatXyzw3232FloatZerosMissingComponents() { + using O = ShaderOpcode; + + std::vector code; + AppendVMovU32(&code, 20, 0); + code.push_back(EncodeMtbuf0(0x03, 0, 4)); + code.push_back(EncodeMtbuf1(0x03, 0, 0, 20)); + for (u32 i = 0; i < 4; i++) { + AppendStoreVgpr(&code, i, i); + } + AppendEnd(&code); + + return { + "TBufferLoadFormatXyzw32_32FloatZerosMissingComponents", + code, + {0x11111111u, 0x22222222u, 0x33333333u, 0x44444444u}, + {0x11111111u, 0x22222222u, 0, 0}, + {O::VMovB32, O::TBufferLoadFormatXyzw, O::BufferStoreDword, O::SEndpgm}}; +} + +TestCase TBufferStoreFormatXyzw3232FloatWritesOnlyPresentComponents() { + using O = ShaderOpcode; + + std::vector code; + AppendVMovU32(&code, 20, 0); + AppendVMovLiteral(&code, 0, 0x3f800000u); + AppendVMovLiteral(&code, 1, 0x40000000u); + AppendVMovLiteral(&code, 2, 0x40400000u); + AppendVMovLiteral(&code, 3, 0x40800000u); + code.push_back(EncodeMtbuf0(0x07, 0, 4)); + code.push_back(EncodeMtbuf1(0x07, 0, 0, 20)); + AppendEnd(&code); + + return {"TBufferStoreFormatXyzw32_32FloatWritesOnlyPresentComponents", + code, + {0xaaaaaaaau, 0xbbbbbbbbu, 0xccccccccu, 0xddddddddu}, + {0x3f800000u, 0x40000000u, 0xccccccccu, 0xddddddddu}, + {O::VMovB32, O::TBufferStoreFormatXyzw, O::SEndpgm}}; +} + +TestCase TBufferStoreVariants() { + using O = ShaderOpcode; + + std::vector code; + AppendVMovU32(&code, 20, 0); + AppendVMovLiteral(&code, 0, 0x01020304u); + AppendTBufferStoreOpcode(&code, 0x04, 0, 20); + AppendVMovU32(&code, 20, 4); + AppendVMovLiteral(&code, 1, 0x11121314u); + AppendVMovLiteral(&code, 2, 0x21222324u); + AppendTBufferStoreOpcode(&code, 0x05, 1, 20); + AppendVMovU32(&code, 20, 12); + AppendVMovLiteral(&code, 3, 0x31323334u); + AppendVMovLiteral(&code, 4, 0x41424344u); + AppendVMovLiteral(&code, 5, 0x51525354u); + AppendTBufferStoreOpcode(&code, 0x06, 3, 20); + AppendVMovU32(&code, 20, 24); + AppendVMovLiteral(&code, 6, 0x61626364u); + AppendVMovLiteral(&code, 7, 0x71727374u); + AppendVMovLiteral(&code, 8, 0x81828384u); + AppendVMovLiteral(&code, 9, 0x91929394u); + AppendTBufferStoreOpcode(&code, 0x07, 6, 20); + AppendEnd(&code); + + return {"TBufferStoreVariants", + code, + std::vector(10, 0), + {0x01020304u, 0x11121314u, 0x21222324u, 0x31323334u, 0x41424344u, + 0x51525354u, 0x61626364u, 0x71727374u, 0x81828384u, 0x91929394u}, + {O::VMovB32, O::TBufferStoreFormatX, O::TBufferStoreFormatXy, + O::TBufferStoreFormatXyz, O::TBufferStoreFormatXyzw, O::SEndpgm}}; +} + +TestCase FlatLoadVariants() { + using O = ShaderOpcode; + + std::vector code; + AppendVMovU32(&code, 20, 0); + code.push_back(EncodeFlat0(0x08, 0, 0)); + code.push_back(EncodeFlat1(0, 0x7d, 0, 20)); + code.push_back(EncodeFlat0(0x09, 0, 2)); + code.push_back(EncodeFlat1(1, 0x7d, 0, 20)); + code.push_back(EncodeFlat0(0x0a, 0, 0)); + code.push_back(EncodeFlat1(2, 0x7d, 0, 20)); + code.push_back(EncodeFlat0(0x0b, 0, 2)); + code.push_back(EncodeFlat1(3, 0x7d, 0, 20)); + code.push_back(EncodeFlat0(0x0c, 0, 8)); + code.push_back(EncodeFlat1(13, 0x7d, 0, 20)); + code.push_back(EncodeFlat0(0x0d, 0, 8)); + code.push_back(EncodeFlat1(4, 0x7d, 0, 20)); + code.push_back(EncodeFlat0(0x0f, 0, 8)); + code.push_back(EncodeFlat1(6, 0x7d, 0, 20)); + code.push_back(EncodeFlat0(0x0e, 0, 8)); + code.push_back(EncodeFlat1(9, 0x7d, 0, 20)); + + for (u32 i = 0; i < 14u; i++) { + AppendStoreVgpr(&code, i, i); + } + AppendEnd(&code); + + TestCase test{"FlatLoadVariants", + code, + {0x80ff7f01u, 0x00008001u, 0x11223344u, 0x55667788u, + 0x99aabbccu, 0xddeeff00u}, + {0x01u, 0xffffffffu, 0x7f01u, 0xffff80ffu, 0x11223344u, + 0x55667788u, 0x11223344u, 0x55667788u, 0x99aabbccu, + 0x11223344u, 0x55667788u, 0x99aabbccu, 0xddeeff00u, + 0x11223344u}, + {O::VMovB32, O::FlatLoadUbyte, O::FlatLoadSbyte, + O::FlatLoadUshort, O::FlatLoadSshort, O::FlatLoadDword, + O::FlatLoadDwordx2, O::FlatLoadDwordx3, O::FlatLoadDwordx4, + O::BufferStoreDword, O::SEndpgm}}; + test.flat_memory_base = 0; + return test; +} + +TestCase BranchVccnzUsesCarryProducedWholeMask() { + using O = ShaderOpcode; + + std::vector code = { + EncodeVop1(0x01, 1, 0), + EncodeVop2(0x1a, 1, InlineU32(2), 1), + EncodeVop2(0x25, 1, Vgpr(0), 1), + EncodeVop2(0x1a, 3, InlineU32(2), 1), + EncodeVopc(0xc0, Vgpr(0), 0), + }; + AppendVMovLiteral(&code, 2, 0xffffffffu); + code.push_back(EncodeVop2(0x28, 5, Vgpr(2), 1)); + code.push_back(EncodeSopp(0x07, 2)); + code.push_back(EncodeVop1(0x01, 2, InlineU32(11))); + code.push_back(EncodeSopp(0x02, 1)); + AppendVMovU32(&code, 2, 42); + AppendBufferStoreDword(&code, 2, 3); + AppendEnd(&code); + + TestCase test; + test.name = "BranchVccnzUsesCarryProducedWholeMask"; + test.code = code; + test.expected = std::vector(8, 42); + test.opcodes = {O::VMovB32, O::VLshlrevB32, O::VAddNcU32, + O::VCmpFU32, O::VAddcU32, O::SCbranchVccnz, + O::SBranch, O::BufferStoreDword, O::SEndpgm}; + test.compute_info.threads_num[0] = 4; + test.compute_info.threads_num[1] = 1; + test.compute_info.threads_num[2] = 1; + test.compute_info.group_id[0] = true; + test.compute_info.thread_ids_num = 1; + test.compute_info.workgroup_register = 0; + test.has_compute_info = true; + test.dispatch_x = 2; + return test; +} + +TestCase ScalarLoadAlignsComponentsAndMasksAddress() { + using O = ShaderOpcode; + + std::vector code; + AppendSMovLiteral(&code, 0, 1u); + AppendSMovLiteral(&code, 2, 3u); + AppendSMovLiteral(&code, 3, 0xffff0000u); + code.push_back(EncodeSmem0(0x00, 1, 1)); + code.push_back(EncodeSmem1(3, 0)); + AppendStoreSgpr(&code, 1, 0); + AppendEnd(&code); + + return {"ScalarLoadAlignsComponentsAndMasksAddress", + code, + {0x11111111u, 0x22222222u}, + {0x11111111u, 0x22222222u}, + {O::SMovB32, O::SLoadDword, O::BufferStoreDword, O::SEndpgm}}; +} + +TestCase FlatVirtualAddressRebasesGuestAllocation() { + using O = ShaderOpcode; + + constexpr uint64_t GuestBase = 0x0000000110000000ull; + std::vector code; + AppendVMovLiteral(&code, 20, static_cast(GuestBase + 4u)); + AppendVMovLiteral(&code, 21, static_cast(GuestBase >> 32u)); + code.push_back(EncodeFlat0(0x0c, 0, 0)); + code.push_back(EncodeFlat1(0, 0x7d, 0, 20)); + code.push_back(EncodeFlat0(0x0c, 2, 0)); + code.push_back(EncodeFlat1(1, 0x7d, 0, 20)); + AppendVMovLiteral(&code, 21, static_cast((GuestBase >> 32u) + 1u)); + code.push_back(EncodeFlat0(0x0c, 0, 0)); + code.push_back(EncodeFlat1(2, 0x7d, 0, 20)); + AppendStoreVgpr(&code, 0, 0); + AppendStoreVgpr(&code, 1, 1); + AppendStoreVgpr(&code, 2, 2); + AppendEnd(&code); + + TestCase test; + test.name = "FlatVirtualAddressRebasesGuestAllocation"; + test.code = std::move(code); + test.initial = {0, 0x12345678u}; + test.expected = {0x12345678u, 0x12345678u, 0}; + test.flat_memory_base = GuestBase; + test.opcodes = {O::VMovB32, O::FlatLoadDword, O::BufferStoreDword, + O::SEndpgm}; + return test; +} + +TestCase GlobalSignedImmediateRebasesBeforeSaddr() { + using O = ShaderOpcode; + + constexpr uint64_t GuestBase = 0x0000000110000000ull; + std::vector code; + AppendSMovLiteral(&code, 0, static_cast(GuestBase)); + AppendSMovLiteral(&code, 1, static_cast(GuestBase >> 32u)); + AppendVMovU32(&code, 20, 8); + code.push_back(EncodeFlat0(0x0c, 2, 0xffcu)); + code.push_back(EncodeFlat1(0, 0, 0, 20)); + AppendStoreVgpr(&code, 0, 0); + AppendEnd(&code); + + return {"GlobalSignedImmediateRebasesBeforeSaddr", + code, + {0x11111111u, 0x22222222u, 0x12345678u}, + {0x12345678u}, + {O::SMovB32, O::VMovB32, O::FlatLoadDword, O::BufferStoreDword, + O::SEndpgm}}; +} + +TestCase FlatSegmentIgnoresSaddrAndMasksOffsetMsb() { + using O = ShaderOpcode; + + std::vector code; + code.push_back(EncodeSMovB32(0, InlineU32(4))); + AppendVMovU32(&code, 20, 0); + code.push_back(EncodeFlat0(0x0c, 0, 4)); + code.push_back(EncodeFlat1(0, 0, 0, 20)); + AppendStoreVgpr(&code, 0, 0); + AppendVMovU32(&code, 20, 0); + code.push_back(EncodeFlat0(0x0c, 0, 0x804)); + code.push_back(EncodeFlat1(1, 0x7d, 0, 20)); + AppendStoreVgpr(&code, 1, 1); + AppendEnd(&code); + + std::vector initial(514, 0); + initial[1] = 0x11111111u; + initial[2] = 0x22222222u; + initial[513] = 0x33333333u; + + TestCase test; + test.name = "FlatSegmentIgnoresSaddrAndMasksOffsetMsb"; + test.code = code; + test.initial = initial; + test.expected = {0x11111111u, 0x11111111u}; + test.opcodes = {O::SMovB32, O::VMovB32, O::FlatLoadDword, O::BufferStoreDword, + O::SEndpgm}; + test.flat_memory_base = 0; + return test; +} + +TestCase FlatStoreVariants() { + using O = ShaderOpcode; + + std::vector code; + AppendVMovU32(&code, 20, 1); + AppendVMovU32(&code, 0, 0xaa); + code.push_back(EncodeFlat0(0x18, 0, 0)); + code.push_back(EncodeFlat1(0, 0x7d, 0, 20)); + AppendVMovU32(&code, 20, 2); + AppendVMovLiteral(&code, 1, 0x0000bbccu); + code.push_back(EncodeFlat0(0x1a, 0, 0)); + code.push_back(EncodeFlat1(0, 0x7d, 1, 20)); + AppendVMovU32(&code, 20, 4); + AppendVMovLiteral(&code, 2, 0x11111111u); + AppendVMovLiteral(&code, 3, 0x22222222u); + code.push_back(EncodeFlat0(0x1d, 0, 0)); + code.push_back(EncodeFlat1(0, 0x7d, 2, 20)); + AppendVMovU32(&code, 20, 12); + AppendVMovLiteral(&code, 4, 0x33333333u); + AppendVMovLiteral(&code, 5, 0x44444444u); + AppendVMovLiteral(&code, 6, 0x55555555u); + code.push_back(EncodeFlat0(0x1f, 0, 0)); + code.push_back(EncodeFlat1(0, 0x7d, 4, 20)); + AppendVMovU32(&code, 20, 24); + AppendVMovLiteral(&code, 7, 0x66666666u); + AppendVMovLiteral(&code, 8, 0x77777777u); + AppendVMovLiteral(&code, 9, 0x88888888u); + AppendVMovLiteral(&code, 10, 0x99999999u); + code.push_back(EncodeFlat0(0x1e, 0, 0)); + code.push_back(EncodeFlat1(0, 0x7d, 7, 20)); + AppendVMovU32(&code, 20, 0); + code.push_back(EncodeFlat0(0x1c, 0, 40)); + code.push_back(EncodeFlat1(0, 0x7d, 2, 20)); + AppendEnd(&code); + + TestCase test{"FlatStoreVariants", + code, + std::vector(12, 0), + {0xbbccaa00u, 0x11111111u, 0x22222222u, 0x33333333u, + 0x44444444u, 0x55555555u, 0x66666666u, 0x77777777u, + 0x88888888u, 0x99999999u, 0x11111111u, 0}, + {O::VMovB32, O::FlatStoreByte, O::FlatStoreShort, + O::FlatStoreDword, O::FlatStoreDwordx2, O::FlatStoreDwordx3, + O::FlatStoreDwordx4, O::SEndpgm}}; + test.flat_memory_base = 0; + return test; +} + +TestCase DsReadWriteVariants() { + using O = ShaderOpcode; + + std::vector code; + AppendVMovU32(&code, 1, 0); + AppendVMovLiteral(&code, 2, 0x11223344u); + code.push_back(EncodeDs0(0x0d, 0)); + code.push_back(EncodeDs1(0, 2, 1)); + code.push_back(EncodeDs0(0x36, 0)); + code.push_back(EncodeDs1(3, 0, 1)); + AppendVMovU32(&code, 4, 0xaa); + code.push_back(EncodeDs0(0x1e, 4)); + code.push_back(EncodeDs1(0, 4, 1)); + code.push_back(EncodeDs0(0x3a, 4)); + code.push_back(EncodeDs1(5, 0, 1)); + code.push_back(EncodeDs0(0x39, 4)); + code.push_back(EncodeDs1(6, 0, 1)); + AppendVMovLiteral(&code, 7, 0x000080ffu); + code.push_back(EncodeDs0(0x1f, 8)); + code.push_back(EncodeDs1(0, 7, 1)); + code.push_back(EncodeDs0(0x3c, 8)); + code.push_back(EncodeDs1(8, 0, 1)); + code.push_back(EncodeDs0(0x3b, 8)); + code.push_back(EncodeDs1(9, 0, 1)); + AppendVMovLiteral(&code, 10, 0x10101010u); + AppendVMovLiteral(&code, 11, 0x11111111u); + code.push_back(EncodeDs0(0x4d, 12)); + code.push_back(EncodeDs1(0, 10, 1)); + code.push_back(EncodeDs0(0x76, 12)); + code.push_back(EncodeDs1(14, 0, 1)); + AppendVMovLiteral(&code, 16, 0x20202020u); + AppendVMovLiteral(&code, 17, 0x21212121u); + AppendVMovLiteral(&code, 18, 0x22222222u); + code.push_back(EncodeDs0(0xde, 20)); + code.push_back(EncodeDs1(0, 16, 1)); + code.push_back(EncodeDs0(0xfe, 20)); + code.push_back(EncodeDs1(19, 0, 1)); + AppendVMovLiteral(&code, 22, 0x30303030u); + AppendVMovLiteral(&code, 23, 0x31313131u); + AppendVMovLiteral(&code, 24, 0x32323232u); + AppendVMovLiteral(&code, 25, 0x33333333u); + code.push_back(EncodeDs0(0xdf, 32)); + code.push_back(EncodeDs1(0, 22, 1)); + code.push_back(EncodeDs0(0xff, 32)); + code.push_back(EncodeDs1(26, 0, 1)); + + const u32 results[] = {3, 5, 6, 8, 9, 14, 15, 19, 20, 21, 26, 27, 28, 29}; + for (u32 i = 0; i < static_cast(std::size(results)); i++) { + AppendStoreVgpr(&code, results[i], i); + } + AppendEnd(&code); + + return {"DsReadWriteVariants", + code, + std::vector(14, 0), + {0x11223344u, 0xaau, 0xffffffaau, 0x80ffu, 0xffff80ffu, 0x10101010u, + 0x11111111u, 0x20202020u, 0x21212121u, 0x22222222u, 0x30303030u, + 0x31313131u, 0x32323232u, 0x33333333u}, + {O::VMovB32, O::DsWriteB32, O::DsReadB32, O::DsWriteByte, + O::DsReadUbyte, O::DsReadSbyte, O::DsWriteShort, O::DsReadUshort, + O::DsReadSshort, O::DsWriteB64, O::DsReadB64, O::DsWriteB96, + O::DsReadB96, O::DsWriteB128, O::DsReadB128, O::BufferStoreDword, + O::SEndpgm}}; +} + +TestCase DsReadWrite2Variants() { + using O = ShaderOpcode; + + std::vector code; + AppendVMovU32(&code, 1, 0); + AppendVMovLiteral(&code, 2, 0x11111111u); + AppendVMovLiteral(&code, 3, 0x22222222u); + code.push_back(EncodeDs0(0x0e, (3u << 8u) | 1u)); + code.push_back(EncodeDs1Ex(0, 3, 2, 1)); + code.push_back(EncodeDs0(0x37, (3u << 8u) | 1u)); + code.push_back(EncodeDs1Ex(4, 0, 0, 1)); + AppendVMovLiteral(&code, 6, 0x33333333u); + AppendVMovLiteral(&code, 7, 0x44444444u); + AppendVMovLiteral(&code, 8, 0x55555555u); + AppendVMovLiteral(&code, 9, 0x66666666u); + code.push_back(EncodeDs0(0x4d, 32)); + code.push_back(EncodeDs1(0, 6, 1)); + code.push_back(EncodeDs0(0x4d, 48)); + code.push_back(EncodeDs1(0, 8, 1)); + code.push_back(EncodeDs0(0x77, (6u << 8u) | 4u)); + code.push_back(EncodeDs1Ex(10, 0, 0, 1)); + AppendVMovLiteral(&code, 14, 0x77777777u); + AppendVMovLiteral(&code, 15, 0x88888888u); + code.push_back(EncodeDs0(0x0f, (2u << 8u) | 1u)); + code.push_back(EncodeDs1Ex(0, 15, 14, 1)); + code.push_back(EncodeDs0(0x38, (2u << 8u) | 1u)); + code.push_back(EncodeDs1Ex(16, 0, 0, 1)); + AppendVMovLiteral(&code, 18, 0x99999999u); + AppendVMovLiteral(&code, 19, 0xaaaaaaaau); + AppendVMovLiteral(&code, 20, 0xbbbbbbbbu); + AppendVMovLiteral(&code, 21, 0xccccccccu); + code.push_back(EncodeDs0(0x4e, (10u << 8u) | 8u)); + code.push_back(EncodeDs1Ex(0, 20, 18, 1)); + code.push_back(EncodeDs0(0x77, (10u << 8u) | 8u)); + code.push_back(EncodeDs1Ex(22, 0, 0, 1)); + AppendVMovLiteral(&code, 26, 0xddddddddu); + AppendVMovLiteral(&code, 27, 0xeeeeeeeeu); + AppendVMovLiteral(&code, 28, 0xf0f0f0f0u); + AppendVMovLiteral(&code, 29, 0x12345678u); + code.push_back(EncodeDs0(0x4f, (2u << 8u) | 1u)); + code.push_back(EncodeDs1Ex(0, 28, 26, 1)); + code.push_back(EncodeDs0(0x78, (2u << 8u) | 1u)); + code.push_back(EncodeDs1Ex(34, 0, 0, 1)); + + const u32 results[] = {4, 5, 10, 11, 12, 13, 22, 23, + 24, 25, 16, 17, 34, 35, 36, 37}; + for (u32 i = 0; i < static_cast(std::size(results)); i++) { + AppendStoreVgpr(&code, results[i], i); + } + AppendEnd(&code); + + return {"DsReadWrite2Variants", + code, + std::vector(16, 0), + {0x11111111u, 0x22222222u, 0x33333333u, 0x44444444u, 0x55555555u, + 0x66666666u, 0x99999999u, 0xaaaaaaaau, 0xbbbbbbbbu, 0xccccccccu, + 0x77777777u, 0x88888888u, 0xddddddddu, 0xeeeeeeeeu, 0xf0f0f0f0u, + 0x12345678u}, + {O::VMovB32, O::DsWrite2B32, O::DsRead2B32, O::DsWriteB64, + O::DsRead2B64, O::DsWrite2B64, O::DsWrite2St64B32, + O::DsWrite2St64B64, O::DsRead2St64B64, O::BufferStoreDword, + O::SEndpgm}}; +} + +TestCase DsAtomicNoReturnVariants() { + using O = ShaderOpcode; + + std::vector code; + AppendVMovU32(&code, 1, 0); + const u32 initial[] = {10, 10, 0xfffffff0u, 0xfffffff0u, 10, + 10, 0xf0f0u, 0xf000u, 0xf00fu}; + const u32 values[] = {5, 3, 5, 5, 5, 20, 0x0ff0u, 0x0f00u, 0x00ffu}; + const u32 ops[] = {0x00, 0x01, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0a, 0x0b}; + for (u32 i = 0; i < static_cast(std::size(values)); i++) { + AppendVMovLiteral(&code, 2, initial[i]); + code.push_back(EncodeDs0(0x0d, i * 4u)); + code.push_back(EncodeDs1(0, 2, 1)); + AppendVMovLiteral(&code, 3, values[i]); + code.push_back(EncodeDs0(ops[i], i * 4u)); + code.push_back(EncodeDs1(0, 3, 1)); + code.push_back(EncodeDs0(0x36, i * 4u)); + code.push_back(EncodeDs1(10u + i, 0, 1)); + } + for (u32 i = 0; i < static_cast(std::size(values)); i++) { + AppendStoreVgpr(&code, 10u + i, i); + } + AppendEnd(&code); + + return {"DsAtomicNoReturnVariants", + code, + std::vector(9, 0), + {15, 7, 0xfffffff0u, 5, 5, 20, 0x00f0u, 0xff00u, 0xf0f0u}, + {O::VMovB32, O::DsWriteB32, O::DsAddU32, O::DsSubU32, O::DsMinI32, + O::DsMaxI32, O::DsMinU32, O::DsMaxU32, O::DsAndB32, O::DsOrB32, + O::DsXorB32, O::DsReadB32, O::BufferStoreDword, O::SEndpgm}}; +} + +TestCase DsAtomicReturnVariants() { + using O = ShaderOpcode; + + std::vector code; + AppendVMovU32(&code, 1, 0); + const u32 initial[] = {10, 10, 0xfffffff0u, 0xfffffff0u, 10, + 10, 0xf0f0u, 0xf000u, 0xf00fu, 10}; + const u32 values[] = {5, 3, 5, 5, 5, 20, 0x0ff0u, 0x0f00u, 0x00ffu, 99}; + const u32 ops[] = {0x20, 0x21, 0x25, 0x26, 0x27, + 0x28, 0x29, 0x2a, 0x2b, 0x2d}; + for (u32 i = 0; i < static_cast(std::size(values)); i++) { + AppendVMovLiteral(&code, 2, initial[i]); + code.push_back(EncodeDs0(0x0d, i * 4u)); + code.push_back(EncodeDs1(0, 2, 1)); + AppendVMovLiteral(&code, 3, values[i]); + code.push_back(EncodeDs0(ops[i], i * 4u)); + code.push_back(EncodeDs1(10u + i, 3, 1)); + code.push_back(EncodeDs0(0x36, i * 4u)); + code.push_back(EncodeDs1(20u + i, 0, 1)); + } + for (u32 i = 0; i < static_cast(std::size(values)); i++) { + AppendStoreVgpr(&code, 10u + i, i); + } + for (u32 i = 0; i < static_cast(std::size(values)); i++) { + AppendStoreVgpr(&code, 20u + i, i + 10u); + } + AppendEnd(&code); + + return { + "DsAtomicReturnVariants", + code, + std::vector(20, 0), + {10, 10, 0xfffffff0u, 0xfffffff0u, 10, 10, 0xf0f0u, 0xf000u, 0xf00fu, 10, + 15, 7, 0xfffffff0u, 5, 5, 20, 0x00f0u, 0xff00u, 0xf0f0u, 99}, + {O::VMovB32, O::DsWriteB32, O::DsAddRtnU32, O::DsSubRtnU32, + O::DsMinRtnI32, O::DsMaxRtnI32, O::DsMinRtnU32, O::DsMaxRtnU32, + O::DsAndRtnB32, O::DsOrRtnB32, O::DsXorRtnB32, O::DsWrxchgRtnB32, + O::DsReadB32, O::BufferStoreDword, O::SEndpgm}}; +} + +TestCase DsMiscVariants() { + using O = ShaderOpcode; + + std::vector code; + AppendVMovU32(&code, 1, 0); + AppendVMovLiteral(&code, 2, 0x40800000u); + code.push_back(EncodeDs0(0x0d, 0)); + code.push_back(EncodeDs1(0, 2, 1)); + AppendVMovLiteral(&code, 3, 0x3f800000u); + code.push_back(EncodeDs0(0x0d, 4)); + code.push_back(EncodeDs1(0, 3, 1)); + AppendVMovLiteral(&code, 4, 0x40000000u); + code.push_back(EncodeDs0(0x12, 0)); + code.push_back(EncodeDs1(0, 4, 1)); + AppendVMovLiteral(&code, 5, 0x40400000u); + code.push_back(EncodeDs0(0x13, 4)); + code.push_back(EncodeDs1(0, 5, 1)); + code.push_back(EncodeDs0(0x36, 0)); + code.push_back(EncodeDs1(6, 0, 1)); + code.push_back(EncodeDs0(0x36, 4)); + code.push_back(EncodeDs1(7, 0, 1)); + AppendVMovLiteral(&code, 8, 0x12345678u); + code.push_back(EncodeDs0(0x35, 0x001f)); + code.push_back(EncodeDs1(9, 0, 8)); + code.push_back(EncodeSMovB32(124, InlineU32(0))); + AppendVMovLiteral(&code, 10, 0xabcdef01u); + code.push_back(EncodeDs0(0xb0, 8)); + code.push_back(EncodeDs1(0, 10, 0)); + code.push_back(EncodeDs0(0xb1, 8)); + code.push_back(EncodeDs1(11, 0, 0)); + + const u32 results[] = {6, 7, 9, 11}; + for (u32 i = 0; i < static_cast(std::size(results)); i++) { + AppendStoreVgpr(&code, results[i], i); + } + AppendEnd(&code); + + TestCase test; + test.name = "DsMiscVariants"; + test.code = code; + test.initial = std::vector(4, 0); + test.expected = {0x40000000u, 0x40400000u, 0x12345678u, 0xabcdef01u}; + test.opcodes = {O::VMovB32, O::SMovB32, O::DsWriteB32, + O::DsMinF32, O::DsMaxF32, O::DsReadB32, + O::DsSwizzleB32, O::DsWriteAddtidB32, O::DsReadAddtidB32, + O::BufferStoreDword, O::SEndpgm}; + test.compute_info.threads_num[0] = 1; + test.compute_info.threads_num[1] = 1; + test.compute_info.threads_num[2] = 1; + test.has_compute_info = true; + return test; +} + +TestCase DsFloatMinMaxUsesSeparateCompareOperand() { + using O = ShaderOpcode; + + std::vector code; + AppendVMovU32(&code, 1, 0); + AppendVMovLiteral(&code, 2, 0x40800000u); + code.push_back(EncodeDs0(0x0d, 0)); + code.push_back(EncodeDs1(0, 2, 1)); + AppendVMovLiteral(&code, 3, 0x40800000u); + code.push_back(EncodeDs0(0x0d, 4)); + code.push_back(EncodeDs1(0, 3, 1)); + AppendVMovLiteral(&code, 4, 0x41100000u); + AppendVMovLiteral(&code, 5, 0x40000000u); + code.push_back(EncodeDs0(0x12, 0)); + code.push_back(EncodeDs1Ex(0, 5, 4, 1)); + AppendVMovLiteral(&code, 6, 0x3f800000u); + AppendVMovLiteral(&code, 7, 0x40400000u); + code.push_back(EncodeDs0(0x13, 4)); + code.push_back(EncodeDs1Ex(0, 7, 6, 1)); + code.push_back(EncodeDs0(0x36, 0)); + code.push_back(EncodeDs1(8, 0, 1)); + code.push_back(EncodeDs0(0x36, 4)); + code.push_back(EncodeDs1(9, 0, 1)); + AppendStoreVgpr(&code, 8, 0); + AppendStoreVgpr(&code, 9, 1); + AppendEnd(&code); + + TestCase test; + test.name = "DsFloatMinMaxUsesSeparateCompareOperand"; + test.code = code; + test.initial = std::vector(2, 0); + test.expected = {0x41100000u, 0x3f800000u}; + test.opcodes = {O::VMovB32, O::DsWriteB32, O::DsMinF32, O::DsMaxF32, + O::DsReadB32, O::BufferStoreDword, O::SEndpgm}; + test.compute_info.threads_num[0] = 1; + test.compute_info.threads_num[1] = 1; + test.compute_info.threads_num[2] = 1; + test.has_compute_info = true; + return test; +} + +TestCase DsSwizzleInvalidSourceLaneZero() { + using O = ShaderOpcode; + + std::vector code; + AppendVMovU32(&code, 1, 100); + code.push_back(EncodeDs0(0x35, 0x00e0)); + code.push_back(EncodeDs1(2, 0, 1)); + code.push_back(EncodeVop2(0x1a, 3, InlineU32(2), 0)); + AppendBufferStoreDword(&code, 2, 3); + AppendEnd(&code); + + TestCase test; + test.name = "DsSwizzleInvalidSourceLaneZero"; + test.code = code; + test.expected = {0, 0, 0, 0}; + test.opcodes = {O::VMovB32, O::DsSwizzleB32, O::VLshlrevB32, + O::BufferStoreDword, O::SEndpgm}; + test.compute_info.threads_num[0] = 4; + test.compute_info.threads_num[1] = 1; + test.compute_info.threads_num[2] = 1; + test.compute_info.thread_ids_num = 1; + test.has_compute_info = true; + return test; +} + +TestCase BufferAtomicVariants() { + using O = ShaderOpcode; + + std::vector code; + const u32 values[] = {100, 5, 3, 5, 5, 5, 20, 0x0ff0u, 0x0f00u, 0x00ffu}; + const u32 ops[] = {0x30, 0x32, 0x33, 0x35, 0x36, + 0x37, 0x38, 0x39, 0x3a, 0x3b}; + for (u32 i = 0; i < static_cast(std::size(values)); i++) { + AppendVMovU32(&code, 20, i * 4u); + AppendVMovU32(&code, i, values[i]); + AppendBufferStoreOpcode(&code, ops[i], i, 20, true); + } + for (u32 i = 0; i < static_cast(std::size(values)); i++) { + AppendStoreVgpr(&code, i, i + 10u); + } + AppendEnd(&code); + + return { + "BufferAtomicVariants", + code, + {10, 10, 10, 0xfffffff0u, 10, 0xfffffff0u, 10, 0xf0f0u, 0xf000u, 0xf00fu, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, + {100, 15, 7, 0xfffffff0u, 5, 5, 20, + 0x00f0u, 0xff00u, 0xf0f0u, 10, 10, 10, 0xfffffff0u, + 10, 0xfffffff0u, 10, 0xf0f0u, 0xf000u, 0xf00fu}, + {O::VMovB32, O::BufferAtomicSwap, O::BufferAtomicAdd, O::BufferAtomicSub, + O::BufferAtomicSMin, O::BufferAtomicUMin, O::BufferAtomicSMax, + O::BufferAtomicUMax, O::BufferAtomicAnd, O::BufferAtomicOr, + O::BufferAtomicXor, O::BufferStoreDword, O::SEndpgm}}; +} + +TestCase BufferAtomicGlc0DoesNotReturnOldValue() { + using O = ShaderOpcode; + + std::vector code; + AppendVMovU32(&code, 20, 0); + AppendVMovU32(&code, 0, 5); + AppendBufferStoreOpcode(&code, 0x32, 0, 20); + AppendStoreVgpr(&code, 0, 1); + AppendEnd(&code); + + return {"BufferAtomicGlc0DoesNotReturnOldValue", + code, + {10, 0}, + {15, 5}, + {O::VMovB32, O::BufferAtomicAdd, O::BufferStoreDword, O::SEndpgm}}; +} + +std::vector MakeRgbaImage(u32 width, u32 height, u32 value = 0) { + return std::vector(static_cast(width) * height * 4u, value); +} + +void SetRgbaPixel(std::vector *image, u32 width, u32 x, u32 y, u32 r, + u32 g, u32 b, u32 a) { + const auto base = static_cast((y * width + x) * 4u); + (*image)[base + 0] = r; + (*image)[base + 1] = g; + (*image)[base + 2] = b; + (*image)[base + 3] = a; +} + +TestCase ImageLoadVariants() { + using O = ShaderOpcode; + + std::vector code; + AppendVMovU32(&code, 20, 2); + AppendVMovU32(&code, 21, 1); + AppendVMovU32(&code, 22, 0); + AppendVMovU32(&code, 23, 0); + code.push_back(EncodeMimg0(0x00, 0xf)); + code.push_back(EncodeMimg1(0, 20)); + code.push_back(EncodeMimg0(0x01, 0xf)); + code.push_back(EncodeMimg1(4, 20)); + AppendVMovU32(&code, 24, 0); + code.push_back(EncodeMimg0(0x0e, 0x1)); + code.push_back(EncodeMimg1(8, 24)); + for (u32 i = 0; i < 9u; i++) { + AppendStoreVgpr(&code, i, i); + } + AppendEnd(&code); + + auto image = MakeRgbaImage(4, 4); + SetRgbaPixel(&image, 4, 2, 1, 0x3f800000u, 0x40000000u, 0x40400000u, + 0x40800000u); + + TestCase test; + test.name = "ImageLoadVariants"; + test.code = code; + test.expected = {0x3f800000u, 0x40000000u, 0x40400000u, + 0x40800000u, 0x3f800000u, 0x40000000u, + 0x40400000u, 0x40800000u, 4}; + test.opcodes = {O::VMovB32, O::ImageLoad, O::ImageLoadMip, + O::ImageGetResinfo, O::BufferStoreDword, O::SEndpgm}; + test.sampled_image_rgba = image; + return test; +} + +TestCase DsAppendConsumeUsesEncodedLdsSelector() { + using O = ShaderOpcode; + + std::vector code; + AppendSMovLiteral(&code, 124, 0x0000ffffu); + AppendVMovU32(&code, 1, 0); + AppendVMovU32(&code, 2, 10); + code.push_back(EncodeDs0(0x0d, 0)); + code.push_back(EncodeDs1(0, 2, 1)); + code.push_back(EncodeDs0(0x3e, 0)); + code.push_back(EncodeDs1(3, 0, 0)); + code.push_back(EncodeDs0(0x3d, 0)); + code.push_back(EncodeDs1(4, 0, 0)); + AppendSMovLiteral(&code, 124, 0); + code.push_back(EncodeDs0(0x3e, 0)); + code.push_back(EncodeDs1(5, 0, 0)); + code.push_back(EncodeDs0(0x36, 0)); + code.push_back(EncodeDs1(6, 0, 1)); + AppendStoreVgpr(&code, 3, 0); + AppendStoreVgpr(&code, 4, 1); + AppendStoreVgpr(&code, 5, 2); + AppendStoreVgpr(&code, 6, 3); + AppendEnd(&code); + + return {"DsAppendConsumeLdsSelector", + code, + {}, + {10, 74, 0, 10}, + {O::SMovB32, O::VMovB32, O::DsWriteB32, O::DsReadB32, O::DsAppend, + O::DsConsume, O::BufferStoreDword, O::SEndpgm}}; +} + +TestCase DsAppendUsesEncodedGdsSelector() { + using O = ShaderOpcode; + + std::vector code; + AppendSMovLiteral(&code, 124, 0x00000001u); + code.push_back(EncodeDs0(0x3e, 0, true)); + code.push_back(EncodeDs1(0, 0, 0)); + code.push_back(EncodeDs0(0x3d, 0, true)); + code.push_back(EncodeDs1(1, 0, 0)); + AppendStoreVgpr(&code, 0, 0); + AppendStoreVgpr(&code, 1, 1); + AppendEnd(&code); + + TestCase test{ + "DsAppendGdsSelector", + code, + {}, + {10, 74}, + {O::SMovB32, O::DsAppend, O::DsConsume, O::BufferStoreDword, O::SEndpgm}}; + test.gds_initial = {10}; + test.expected_gds = {10}; + return test; +} + +TestCase DsGdsSubdwordAndAtomicWrites() { + using O = ShaderOpcode; + + std::vector code; + code.push_back(EncodeVop1(0x01, 1, Vgpr(0))); + code.push_back(EncodeVop2(0x25, 2, InlineU32(1), 0)); + code.push_back(EncodeDs0(0x1e, 0, true)); + code.push_back(EncodeDs1(0, 2, 1)); + code.push_back(EncodeVop2(0x1a, 8, InlineU32(1), 0)); + code.push_back(EncodeVop2(0x25, 8, InlineU32(12), 8)); + code.push_back(EncodeDs0(0x1f, 0, true)); + code.push_back(EncodeDs1(0, 2, 8)); + AppendVMovU32(&code, 3, 4); + AppendVMovU32(&code, 4, 1); + code.push_back(EncodeDs0(0x00, 0, true)); + code.push_back(EncodeDs1(0, 4, 3)); + AppendVMovU32(&code, 5, 8); + AppendVMovLiteral(&code, 6, 0x40a00000u); + AppendVMovLiteral(&code, 7, 0x40a00000u); + code.push_back(EncodeDs0(0x12, 0, true)); + code.push_back(EncodeDs1Ex(0, 7, 6, 5)); + AppendEnd(&code); + + TestCase test; + test.name = "DsGdsSubdwordAndAtomics"; + test.code = code; + test.opcodes = {O::VMovB32, O::VAddNcU32, O::VLshlrevB32, O::DsWriteByte, + O::DsWriteShort, O::DsAddU32, O::DsMinF32, O::SEndpgm}; + test.compute_info.threads_num[0] = 4; + test.compute_info.threads_num[1] = 1; + test.compute_info.threads_num[2] = 1; + test.compute_info.thread_ids_num = 1; + test.has_compute_info = true; + test.gds_initial = {0, 0, 0x42c80000u, 0, 0}; + test.expected_gds = {0x04030201u, 4, 0x40a00000u, 0x00020001u, 0x00040003u}; + return test; +} + +TestCase ImageLoadR32UintUsesIntegerSampledImage() { + using O = ShaderOpcode; + + std::vector code; + AppendVMovU32(&code, 20, 2); + AppendVMovU32(&code, 21, 1); + code.push_back(EncodeMimg0(0x00, 0x1)); + code.push_back(EncodeMimg1(0, 20)); + AppendStoreVgpr(&code, 0, 0); + AppendEnd(&code); + + TestCase test; + test.name = "ImageLoadR32UintUsesIntegerSampledImage"; + test.code = code; + test.expected = {0xdeadbeefu}; + test.opcodes = {O::VMovB32, O::ImageLoad, O::BufferStoreDword, O::SEndpgm}; + test.sampled_image_rgba.resize(16); + test.sampled_image_rgba[6] = 0xdeadbeefu; + test.sampled_image_format = VK_FORMAT_R32_UINT; + test.sampled_image_dwords_per_pixel = 1; + test.user_data = MakeSampledTextureData(Prospero::BufferFormat::k32UInt); + test.has_user_data = true; + test.required_spirv = {"sampled_uint_2d"}; + return test; +} + +TestCase ImageLoadMipUsesVaddr2Lod2D() { + using O = ShaderOpcode; + + std::vector code; + AppendVMovU32(&code, 20, 1); + AppendVMovU32(&code, 21, 1); + AppendVMovU32(&code, 22, 1); + AppendVMovU32(&code, 23, 0); + code.push_back(EncodeMimg0(0x01, 0x1)); + code.push_back(EncodeMimg1(0, 20)); + AppendStoreVgpr(&code, 0, 0); + AppendEnd(&code); + + auto base = MakeRgbaImage(4, 4); + SetRgbaPixel(&base, 4, 1, 1, 0x3f800000u, 0, 0, 0); + auto mip1 = MakeRgbaImage(2, 2); + SetRgbaPixel(&mip1, 2, 1, 1, 0x40000000u, 0, 0, 0); + + TestCase test; + test.name = "ImageLoadMipUsesVaddr2Lod2D"; + test.code = code; + test.expected = {0x40000000u}; + test.opcodes = {O::VMovB32, O::ImageLoadMip, O::BufferStoreDword, O::SEndpgm}; + test.sampled_image_rgba_mips = {base, mip1}; + return test; +} + +TestCase ImageLoadMipNsaUsesSelectedAddressVgprs() { + using O = ShaderOpcode; + + std::vector code; + AppendVMovU32(&code, 20, 1); + AppendVMovU32(&code, 21, 0); + AppendVMovU32(&code, 22, 0); + AppendVMovU32(&code, 30, 1); + AppendVMovU32(&code, 31, 1); + code.push_back(EncodeMimg0(0x01, 0x1, 1)); + code.push_back(EncodeMimg1(0, 20)); + code.push_back((30u << 0u) | (31u << 8u)); + AppendStoreVgpr(&code, 0, 0); + AppendEnd(&code); + + auto base = MakeRgbaImage(4, 4); + SetRgbaPixel(&base, 4, 1, 0, 0x3f800000u, 0, 0, 0); + auto mip1 = MakeRgbaImage(2, 2); + SetRgbaPixel(&mip1, 2, 1, 1, 0x40a00000u, 0, 0, 0); + + TestCase test; + test.name = "ImageLoadMipNsaUsesSelectedAddressVgprs"; + test.code = code; + test.expected = {0x40a00000u}; + test.opcodes = {O::VMovB32, O::ImageLoadMip, O::BufferStoreDword, O::SEndpgm}; + test.sampled_image_rgba_mips = {base, mip1}; + return test; +} + +TestCase ImageLoadA16UintCoordsOnGpu() { + using O = ShaderOpcode; + + std::vector code; + AppendVMovLiteral(&code, 20, 0x00010002u); // x=2, y=1 packed as u16. + AppendVMovU32(&code, 21, 0); + code.push_back(EncodeMimg0(0x00, 0xf)); + code.push_back(EncodeMimg1(0, 20, 0, 0, true)); + for (u32 i = 0; i < 4u; i++) { + AppendStoreVgpr(&code, i, i); + } + AppendEnd(&code); + + auto image = MakeRgbaImage(4, 4); + SetRgbaPixel(&image, 4, 2, 1, 0x3f800000u, 0x40000000u, 0x40400000u, + 0x40800000u); + + TestCase test; + test.name = "ImageLoadA16UintCoordsOnGpu"; + test.code = code; + test.expected = {0x3f800000u, 0x40000000u, 0x40400000u, 0x40800000u}; + test.opcodes = {O::VMovB32, O::ImageLoad, O::BufferStoreDword, O::SEndpgm}; + test.sampled_image_rgba = image; + test.required_spirv = {"OpShiftRightLogical", "OpBitwiseAnd"}; + test.forbidden_spirv = {"UnpackHalf2x16"}; + return test; +} + +TestCase ImageGetResinfoDmaskWidthHeight() { + using O = ShaderOpcode; + + std::vector code; + AppendVMovU32(&code, 24, 0); + code.push_back(EncodeMimg0(0x0e, 0x3)); + code.push_back(EncodeMimg1(0, 24)); + AppendStoreVgpr(&code, 0, 0); + AppendStoreVgpr(&code, 1, 1); + AppendEnd(&code); + + TestCase test; + test.name = "ImageGetResinfoDmaskWidthHeight"; + test.code = code; + test.expected = {4, 2}; + test.opcodes = {O::VMovB32, O::ImageGetResinfo, O::BufferStoreDword, + O::SEndpgm}; + test.image_width = 4; + test.image_height = 2; + test.sampled_image_rgba = MakeRgbaImage(test.image_width, test.image_height); + return test; +} + +TestCase ImageGetResinfoDmaskMipLevels() { + using O = ShaderOpcode; + + std::vector code; + AppendVMovU32(&code, 24, 0); + code.push_back(EncodeMimg0(0x0e, 0x8)); + code.push_back(EncodeMimg1(0, 24)); + AppendStoreVgpr(&code, 0, 0); + AppendEnd(&code); + + TestCase test; + test.name = "ImageGetResinfoDmaskMipLevels"; + test.code = code; + test.expected = {2}; + test.opcodes = {O::VMovB32, O::ImageGetResinfo, O::BufferStoreDword, + O::SEndpgm}; + auto base = MakeRgbaImage(4, 4); + auto mip1 = MakeRgbaImage(2, 2); + test.sampled_image_rgba_mips = {base, mip1}; + return test; +} + +SkippedCase ImageStoreMipWritesExplicitMip2D() { + return {"ImageStoreMipWritesExplicitMip2D", + "requires per-mip storage-image view descriptors; Vulkan " + "OpImageWrite cannot take Lod"}; +} + +TestCase ImageSampleAndGather() { + using O = ShaderOpcode; + + std::vector code; + AppendVMovLiteral(&code, 20, 0x3f200000u); + AppendVMovLiteral(&code, 21, 0x3ec00000u); + AppendVMovU32(&code, 22, 0); + code.push_back(EncodeMimg0(0x20, 0xf)); + code.push_back(EncodeMimg1(0, 20)); + code.push_back(EncodeMimg0(0x47, 0x1)); + code.push_back(EncodeMimg1(4, 20)); + AppendVMovU32(&code, 24, 0); + AppendVMovLiteral(&code, 25, 0x3f200000u); + AppendVMovLiteral(&code, 26, 0x3ec00000u); + AppendVMovU32(&code, 27, 0); + code.push_back(EncodeMimg0(0x57, 0x1)); + code.push_back(EncodeMimg1(8, 24)); + code.push_back(EncodeMimg0(0x60, 0x1)); + code.push_back(EncodeMimg1(12, 20)); + for (u32 i = 0; i < 13u; i++) { + AppendStoreVgpr(&code, i, i); + } + AppendEnd(&code); + + auto image = MakeRgbaImage(4, 4); + for (u32 y = 0; y < 4u; y++) { + for (u32 x = 0; x < 4u; x++) { + SetRgbaPixel(&image, 4, x, y, 0x3f800000u, 0, 0, 0); + } + } + SetRgbaPixel(&image, 4, 2, 1, 0x3f800000u, 0x40000000u, 0x40400000u, + 0x40800000u); + + TestCase test; + test.name = "ImageSampleAndGather"; + test.code = code; + test.expected = {0x3f800000u, + 0x40000000u, + 0x40400000u, + 0x40800000u, + 0x3f800000u, + 0x3f800000u, + 0x3f800000u, + 0x3f800000u, + 0x3f800000u, + 0x3f800000u, + 0x3f800000u, + 0x3f800000u, + 0}; + test.opcodes = {O::VMovB32, O::ImageSample, O::ImageGetLod, + O::ImageGather4Lz, O::ImageGather4LzO, O::BufferStoreDword, + O::SEndpgm}; + test.sampled_image_rgba = image; + return test; +} + +TestCase ImageSampleA16SamplerCoordsOnGpu() { + using O = ShaderOpcode; + + std::vector code; + AppendVMovLiteral(&code, 20, 0x36003900u); // x=0.625, y=0.375 packed as f16. + AppendVMovU32(&code, 21, 0); + code.push_back(EncodeMimg0(0x20, 0xf)); + code.push_back(EncodeMimg1(0, 20, 0, 0, true)); + for (u32 i = 0; i < 4u; i++) { + AppendStoreVgpr(&code, i, i); + } + AppendEnd(&code); + + auto image = MakeRgbaImage(4, 4); + SetRgbaPixel(&image, 4, 2, 1, 0x3f800000u, 0x40000000u, 0x40400000u, + 0x40800000u); + + TestCase test; + test.name = "ImageSampleA16SamplerCoordsOnGpu"; + test.code = code; + test.expected = {0x3f800000u, 0x40000000u, 0x40400000u, 0x40800000u}; + test.opcodes = {O::VMovB32, O::ImageSample, O::BufferStoreDword, O::SEndpgm}; + test.sampled_image_rgba = image; + test.required_spirv = {"UnpackHalf2x16"}; + return test; +} + +TestCase ImageSampleOpcodeAliasUsesNormalCoords() { + using O = ShaderOpcode; + + std::vector code; + AppendVMovLiteral(&code, 2, 0x3f000000u); + AppendVMovLiteral(&code, 3, 0x3f000000u); + code.push_back(0xf0800109u); // observed image_sample_a v6, v2, s0, s24 + code.push_back(0x00c00602u); + AppendEnd(&code); + + TestCase test; + test.name = "ImageSampleOpcodeAliasUsesNormalCoords"; + test.code = code; + test.opcodes = {O::VMovB32, O::ImageSample, O::SEndpgm}; + test.required_spirv = {"OpImageSampleExplicitLod"}; + test.forbidden_spirv = {"UnpackHalf2x16"}; + test.compile_only = true; + return test; +} + +TestCase ImageSampleA16OffsetKeepsTexelOffset32BitOnGpu() { + using O = ShaderOpcode; + + std::vector code; + AppendVMovU32(&code, 20, 0); // Texel offset stays one packed 32-bit VGPR. + AppendVMovLiteral(&code, 21, 0x36003900u); // x=0.625, y=0.375 packed as f16. + AppendVMovU32(&code, 22, 0); + code.push_back(EncodeMimg0(0x30, 0xf)); + code.push_back(EncodeMimg1(0, 20, 0, 0, true)); + for (u32 i = 0; i < 4u; i++) { + AppendStoreVgpr(&code, i, i); + } + AppendEnd(&code); + + auto image = MakeRgbaImage(4, 4); + SetRgbaPixel(&image, 4, 2, 1, 0x3f800000u, 0x40000000u, 0x40400000u, + 0x40800000u); + + TestCase test; + test.name = "ImageSampleA16OffsetKeepsTexelOffset32BitOnGpu"; + test.code = code; + test.expected = {0x3f800000u, 0x40000000u, 0x40400000u, 0x40800000u}; + test.opcodes = {O::VMovB32, O::ImageSample, O::BufferStoreDword, O::SEndpgm}; + test.sampled_image_rgba = image; + test.required_spirv = {"UnpackHalf2x16", "OpBitFieldSExtract"}; + return test; +} + +TestCase ImageSampleA16CompareBiasRdna2AddressOrder() { + using O = ShaderOpcode; + + std::vector code; + AppendVMovLiteral(&code, 20, 0x00003800u); // bias=0.5 as low f16. + AppendVMovLiteral(&code, 21, 0x3f000000u); // PCF reference stays 32-bit. + AppendVMovLiteral(&code, 22, 0x36003900u); // x=0.625, y=0.375 packed as f16. + code.push_back(EncodeMimg0(0x2d, 0x1)); + code.push_back(EncodeMimg1(0, 20, 0, 0, true)); + AppendEnd(&code); + + TestCase test; + test.name = "ImageSampleA16CompareBiasRdna2AddressOrder"; + test.code = code; + test.opcodes = {O::VMovB32, O::ImageSample, O::SEndpgm}; + test.required_spirv = {"OpImageSampleDrefExplicitLod", "UnpackHalf2x16"}; + test.compile_only = true; + return test; +} + +TestCase ImageGatherCompareOpcodes() { + using O = ShaderOpcode; + + std::vector code; + AppendVMovU32(&code, 20, 0); + AppendVMovLiteral(&code, 21, 0x3f000000u); + AppendVMovLiteral(&code, 22, 0x3f200000u); + AppendVMovLiteral(&code, 23, 0x3ec00000u); + AppendVMovLiteral(&code, 24, 0x3f000000u); + AppendVMovLiteral(&code, 25, 0x3f200000u); + AppendVMovLiteral(&code, 26, 0x3ec00000u); + AppendVMovLiteral(&code, 28, 0x3f000000u); + AppendVMovLiteral(&code, 29, 0x3f200000u); + AppendVMovLiteral(&code, 30, 0x3ec00000u); + AppendVMovU32(&code, 32, 0); + AppendVMovLiteral(&code, 33, 0x3f000000u); + AppendVMovLiteral(&code, 34, 0x3f200000u); + AppendVMovLiteral(&code, 35, 0x3ec00000u); + code.push_back(EncodeMimg0(0x48, 0x1)); + code.push_back(EncodeMimg1(4, 24)); + code.push_back(EncodeMimg0(0x4f, 0x1)); + code.push_back(EncodeMimg1(8, 28)); + code.push_back(EncodeMimg0(0x58, 0x1)); + code.push_back(EncodeMimg1(0, 20)); + code.push_back(EncodeMimg0(0x5f, 0x1)); + code.push_back(EncodeMimg1(12, 32)); + AppendEnd(&code); + + TestCase test; + test.name = "ImageGatherCompareOpcodes"; + test.code = code; + test.opcodes = {O::VMovB32, O::ImageGather4C, O::ImageGather4CLz, + O::ImageGather4CO, O::ImageGather4CLzO, O::SEndpgm}; + test.required_spirv = {"OpImageDrefGather", "OpBitFieldSExtract"}; + test.compile_only = true; + return test; +} + +TestCase ImageStoreVariants() { + using O = ShaderOpcode; + + std::vector code; + AppendVMovU32(&code, 20, 1); + AppendVMovU32(&code, 21, 2); + AppendVMovU32(&code, 22, 0); + AppendVMovLiteral(&code, 0, 0x3f800000u); + AppendVMovLiteral(&code, 1, 0x40000000u); + AppendVMovLiteral(&code, 2, 0x40400000u); + AppendVMovLiteral(&code, 3, 0x40800000u); + code.push_back(EncodeMimg0(0x08, 0xf)); + code.push_back(EncodeMimg1(0, 20)); + AppendVMovU32(&code, 24, 3); + AppendVMovU32(&code, 25, 2); + AppendVMovU32(&code, 26, 0); + AppendVMovU32(&code, 27, 0); + AppendVMovLiteral(&code, 4, 0x40a00000u); + AppendVMovLiteral(&code, 5, 0x40c00000u); + AppendVMovLiteral(&code, 6, 0x40e00000u); + AppendVMovLiteral(&code, 7, 0x41000000u); + code.push_back(EncodeMimg0(0x09, 0xf)); + code.push_back(EncodeMimg1(4, 24)); + AppendVMovLiteral(&code, 8, 0x12345678u); + AppendStoreVgpr(&code, 8, 0); + AppendEnd(&code); + + auto expected_image = MakeRgbaImage(4, 4); + SetRgbaPixel(&expected_image, 4, 1, 2, 0x3f800000u, 0x40000000u, 0x40400000u, + 0x40800000u); + SetRgbaPixel(&expected_image, 4, 3, 2, 0x40a00000u, 0x40c00000u, 0x40e00000u, + 0x41000000u); + + TestCase test; + test.name = "ImageStoreVariants"; + test.code = code; + test.expected = {0x12345678u}; + test.opcodes = {O::VMovB32, O::ImageStore, O::ImageStoreMip, + O::BufferStoreDword, O::SEndpgm}; + test.storage_image_rgba = MakeRgbaImage(4, 4); + test.storage_image_r32ui = std::vector(16, 0); + test.expected_storage_image_rgba = expected_image; + return test; +} + +TestCase ImageStoreRgbOneUsesInverseSwizzle() { + using O = ShaderOpcode; + + std::vector code; + AppendVMovU32(&code, 20, 1); + AppendVMovU32(&code, 21, 2); + AppendVMovU32(&code, 22, 0); + AppendVMovLiteral(&code, 0, 0x3f800000u); + AppendVMovLiteral(&code, 1, 0x3f000000u); + AppendVMovLiteral(&code, 2, 0x3e800000u); + AppendVMovLiteral(&code, 3, 0x3f400000u); + code.push_back(EncodeMimg0(0x08, 0xf)); + code.push_back(EncodeMimg1(0, 20)); + AppendEnd(&code); + + TestCase test; + test.name = "ImageStoreRgbOneUsesInverseSwizzle"; + test.code = code; + test.opcodes = {O::VMovB32, O::ImageStore, O::SEndpgm}; + test.has_user_data = true; + test.user_data[0] = 0x1000u; + test.user_data[1] = + Prospero::GpuEnumValue(Prospero::BufferFormat::k8_8_8_8UNorm) << 20u; + test.image_descriptor_swizzle = DstSel(4, 5, 6, 1); + test.storage_image_format = VK_FORMAT_R8G8B8A8_UNORM; + test.storage_image_dwords_per_pixel = 1; + test.storage_image_rgba = std::vector(16, 0); + test.expected_storage_image_rgba = std::vector(16, 0); + test.expected_storage_image_rgba[2 * 4 + 1] = 0x004080ffu; + return test; +} + +TestCase ImageStoreBgraUsesInverseSwizzle() { + using O = ShaderOpcode; + + std::vector code; + AppendVMovU32(&code, 20, 1); + AppendVMovU32(&code, 21, 2); + AppendVMovU32(&code, 22, 0); + AppendVMovLiteral(&code, 0, 0x3f800000u); + AppendVMovLiteral(&code, 1, 0x3f000000u); + AppendVMovLiteral(&code, 2, 0x3e800000u); + AppendVMovLiteral(&code, 3, 0x3f400000u); + code.push_back(EncodeMimg0(0x08, 0xf)); + code.push_back(EncodeMimg1(0, 20)); + AppendEnd(&code); + + TestCase test; + test.name = "ImageStoreBgraUsesInverseSwizzle"; + test.code = code; + test.opcodes = {O::VMovB32, O::ImageStore, O::SEndpgm}; + test.has_user_data = true; + test.user_data[0] = 0x1000u; + test.user_data[1] = + Prospero::GpuEnumValue(Prospero::BufferFormat::k8_8_8_8UNorm) << 20u; + test.image_descriptor_swizzle = DstSel(6, 5, 4, 7); + test.storage_image_format = VK_FORMAT_R8G8B8A8_UNORM; + test.storage_image_dwords_per_pixel = 1; + test.storage_image_rgba = std::vector(16, 0); + test.expected_storage_image_rgba = std::vector(16, 0); + test.expected_storage_image_rgba[2 * 4 + 1] = 0xbfff8040u; + return test; +} + +TestCase ImageStoreYzwxUsesInverseSwizzle() { + using O = ShaderOpcode; + + std::vector code; + AppendVMovU32(&code, 20, 1); + AppendVMovU32(&code, 21, 2); + AppendVMovU32(&code, 22, 0); + AppendVMovLiteral(&code, 0, 0x3f800000u); + AppendVMovLiteral(&code, 1, 0x3f000000u); + AppendVMovLiteral(&code, 2, 0x3e800000u); + AppendVMovLiteral(&code, 3, 0x3f400000u); + code.push_back(EncodeMimg0(0x08, 0xf)); + code.push_back(EncodeMimg1(0, 20)); + AppendEnd(&code); + + TestCase test; + test.name = "ImageStoreYzwxUsesInverseSwizzle"; + test.code = code; + test.opcodes = {O::VMovB32, O::ImageStore, O::SEndpgm}; + test.user_data = MakeStorageTextureData(Prospero::BufferFormat::k32_32_32_32Float); + test.has_user_data = true; + test.image_descriptor_swizzle = DstSel(5, 6, 7, 4); + test.storage_image_rgba = MakeRgbaImage(4, 4); + test.expected_storage_image_rgba = MakeRgbaImage(4, 4); + SetRgbaPixel(&test.expected_storage_image_rgba, 4, 1, 2, 0x3f400000u, + 0x3f800000u, 0x3f000000u, 0x3e800000u); + return test; +} + +TestCase ImageStoreR32FloatUsesFormatlessStorageImage() { + using O = ShaderOpcode; + + std::vector code; + AppendVMovU32(&code, 20, 2); + AppendVMovU32(&code, 21, 1); + AppendVMovU32(&code, 22, 0); + AppendVMovLiteral(&code, 0, 0x3f800000u); + code.push_back(EncodeMimg0(0x08, 0x1)); + code.push_back(EncodeMimg1(0, 20)); + AppendEnd(&code); + + std::vector expected_image(16, 0); + expected_image[1 * 4 + 2] = 0x3f800000u; + + TestCase test; + test.name = "ImageStoreR32FloatUsesFormatlessStorageImage"; + test.code = code; + test.opcodes = {O::VMovB32, O::ImageStore, O::SEndpgm}; + test.storage_image_format = VK_FORMAT_R32_SFLOAT; + test.storage_image_dwords_per_pixel = 1; + test.storage_image_rgba = std::vector(16, 0); + test.expected_storage_image_rgba = expected_image; + test.required_spirv = {"OpCapability StorageImageReadWithoutFormat", + "OpCapability StorageImageWriteWithoutFormat"}; + test.forbidden_spirv = {"Rgba32f"}; + return test; +} + +TestCase ImageStoreR32UintUsesUintStorageImage() { + using O = ShaderOpcode; + + std::vector code; + AppendVMovU32(&code, 20, 2); + AppendVMovU32(&code, 21, 1); + AppendVMovU32(&code, 22, 0); + AppendVMovLiteral(&code, 0, 0x12345678u); + code.push_back(EncodeMimg0(0x08, 0x1)); + code.push_back(EncodeMimg1(0, 20)); + AppendEnd(&code); + + std::vector expected_image(16, 0); + expected_image[1 * 4 + 2] = 0x12345678u; + + TestCase test; + test.name = "ImageStoreR32UintUsesUintStorageImage"; + test.code = code; + test.opcodes = {O::VMovB32, O::ImageStore, O::SEndpgm}; + test.user_data = MakeStorageTextureData(Prospero::BufferFormat::k32UInt); + test.has_user_data = true; + test.storage_image_rgba = MakeRgbaImage(4, 4); + test.storage_image_r32ui = std::vector(16, 0); + test.expected_storage_image_r32ui = expected_image; + test.required_spirv = {"R32ui", "textures2D_L_U"}; + return test; +} + +TestCase ComputeTgSizeSgprUsesWaveMetadata() { + using O = ShaderOpcode; + + std::vector code; + AppendStoreSgpr(&code, 2, 0); + AppendEnd(&code); + + TestCase test; + test.name = "ComputeTgSizeSgprUsesWaveMetadata"; + test.code = code; + test.opcodes = {O::VMovB32, O::BufferStoreDword, O::SEndpgm}; + test.compute_info.threads_num[0] = 4; + test.compute_info.threads_num[1] = 4; + test.compute_info.threads_num[2] = 16; + test.compute_info.group_id[0] = true; + test.compute_info.group_id[1] = true; + test.compute_info.wave_size = 32; + test.compute_info.thread_ids_num = 3; + test.compute_info.workgroup_register = 0; + test.compute_info.tg_size_en = true; + test.has_compute_info = true; + test.compile_only = true; + test.required_spirv = {"OpUDiv", "OpShiftLeftLogical", "2147483648"}; + return test; +} + +TestCase ImageAtomicVariants() { + using O = ShaderOpcode; + + const u32 initial[] = {10, 10, 0xf0f0u, 0xf000u, 0xf00fu}; + const u32 values[] = {5, 5, 0x0ff0u, 0x0f00u, 0x00ffu}; + const u32 ops[] = {0x11, 0x15, 0x18, 0x19, 0x1a}; + + std::vector code; + for (u32 i = 0; i < static_cast(std::size(values)); i++) { + AppendVMovU32(&code, 20, i & 3u); + AppendVMovU32(&code, 21, i >> 2u); + AppendVMovU32(&code, 22, 0); + AppendVMovLiteral(&code, 0, values[i]); + code.push_back(EncodeMimg0(ops[i], 0x1, 0, true)); + code.push_back(EncodeMimg1(0, 20)); + AppendStoreVgpr(&code, 0, i); + } + AppendEnd(&code); + + TestCase test; + test.name = "ImageAtomicVariants"; + test.code = code; + test.expected = {10, 10, 0xf0f0u, 0xf000u, 0xf00fu}; + test.opcodes = {O::VMovB32, O::ImageAtomicAdd, O::ImageAtomicUMin, + O::ImageAtomicAnd, O::ImageAtomicOr, O::ImageAtomicXor, + O::BufferStoreDword, O::SEndpgm}; + test.storage_image_rgba = MakeRgbaImage(4, 4); + test.storage_image_r32ui = std::vector(16, 0); + for (u32 i = 0; i < static_cast(std::size(initial)); i++) { + test.storage_image_r32ui[i] = initial[i]; + } + test.expected_storage_image_r32ui = std::vector(16, 0); + test.expected_storage_image_r32ui[0] = 15; + test.expected_storage_image_r32ui[1] = 5; + test.expected_storage_image_r32ui[2] = 0x00f0u; + test.expected_storage_image_r32ui[3] = 0xff00u; + test.expected_storage_image_r32ui[4] = 0xf0f0u; + return test; +} + +TestCase ImageAtomicGlc0DoesNotReturnOldValue() { + using O = ShaderOpcode; + + std::vector code; + AppendVMovU32(&code, 20, 0); + AppendVMovU32(&code, 21, 0); + AppendVMovU32(&code, 22, 0); + AppendVMovU32(&code, 0, 5); + code.push_back(EncodeMimg0(0x11, 0x1)); + code.push_back(EncodeMimg1(0, 20)); + AppendStoreVgpr(&code, 0, 0); + AppendEnd(&code); + + TestCase test; + test.name = "ImageAtomicGlc0DoesNotReturnOldValue"; + test.code = code; + test.expected = {5}; + test.opcodes = {O::VMovB32, O::ImageAtomicAdd, O::BufferStoreDword, + O::SEndpgm}; + test.storage_image_rgba = MakeRgbaImage(4, 4); + test.storage_image_r32ui = std::vector(16, 0); + test.storage_image_r32ui[0] = 10; + test.expected_storage_image_r32ui = std::vector(16, 0); + test.expected_storage_image_r32ui[0] = 15; + return test; +} + +GraphicsCase GraphicsInterpolationExport() { + using O = ShaderOpcode; + + std::vector code; + code.push_back(EncodeVintrp(0x00, 0, 0, 0, 0)); + code.push_back(EncodeVintrp(0x01, 0, 0, 0, 0)); + code.push_back(EncodeVintrp(0x02, 1, 0, 1, 2)); + AppendVMovLiteral(&code, 2, 0x3f400000u); + AppendVMovLiteral(&code, 3, 0x3f800000u); + code.push_back(EncodeExp0(0x00, 0xf)); + code.push_back(EncodeExp1(0, 1, 2, 3)); + AppendEnd(&code); + + return {"GraphicsInterpolationExport", + code, + {0x3e800000u, 0x3f000000u, 0x3f400000u, 0x3f800000u}, + {O::VInterpP1F32, O::VInterpP2F32, O::VInterpMovF32, O::VMovB32, + O::Exp, O::SEndpgm}}; +} + +GraphicsCase GraphicsFlatInterpolatorExport() { + using O = ShaderOpcode; + + std::vector code; + code.push_back(EncodeVintrp(0x02, 0, 0, 0, 2)); + AppendVMovLiteral(&code, 1, 0x00000000u); + AppendVMovLiteral(&code, 2, 0x00000000u); + AppendVMovLiteral(&code, 3, 0x3f800000u); + code.push_back(EncodeExp0(0x00, 0xf)); + code.push_back(EncodeExp1(0, 1, 2, 3)); + AppendEnd(&code); + + GraphicsCase test; + test.name = "GraphicsFlatInterpolatorExport"; + test.fragment_code = code; + test.expected_pixel = {0x3e800000u, 0x00000000u, 0x00000000u, 0x3f800000u}; + test.opcodes = {O::VInterpMovF32, O::VMovB32, O::Exp, O::SEndpgm}; + test.pixel_interpolator_settings = {0x00000400u}; + test.vertices = { + 0xbf800000u, 0xbf800000u, 0x3e800000u, 0x00000000u, 0x00000000u, + 0x3f800000u, 0x40400000u, 0xbf800000u, 0x3f400000u, 0x00000000u, + 0x00000000u, 0x3f800000u, 0xbf800000u, 0x40400000u, 0x3e800000u, + 0x00000000u, 0x00000000u, 0x3f800000u, + }; + return test; +} + +GraphicsCase GraphicsDsAddtidScratchExport() { + using O = ShaderOpcode; + + std::vector code; + code.push_back(EncodeSMovB32(124, InlineU32(0))); + AppendVMovLiteral(&code, 0, 0x3f000000u); + code.push_back(EncodeDs0(0xb0, 0)); + code.push_back(EncodeDs1(0, 0, 0)); + code.push_back(EncodeDs0(0xb1, 0)); + code.push_back(EncodeDs1(4, 0, 0)); + code.push_back(EncodeExp0(0x00, 0xf)); + code.push_back(EncodeExp1(4, 4, 4, 4)); + AppendEnd(&code); + + return {"GraphicsDsAddtidScratchExport", + code, + {0x3f000000u, 0x3f000000u, 0x3f000000u, 0x3f000000u}, + {O::SMovB32, O::VMovB32, O::DsWriteAddtidB32, O::DsReadAddtidB32, + O::Exp, O::SEndpgm}}; +} + +GraphicsCase GraphicsDirectSgprPushConstantExport() { + using O = ShaderOpcode; + + const std::vector values = {0x3e800000u, 0x3f000000u, 0x3f400000u, + 0x3f800000u}; + + std::vector code; + code.push_back(EncodeVop1(0x01, 0, 0)); + code.push_back(EncodeVop1(0x01, 1, 1)); + code.push_back(EncodeVop1(0x01, 2, 2)); + code.push_back(EncodeVop1(0x01, 3, 3)); + code.push_back(EncodeExp0(0x00, 0xf)); + code.push_back(EncodeExp1(0, 1, 2, 3)); + AppendEnd(&code); + + GraphicsCase test; + test.name = "GraphicsDirectSgprPushConstantExport"; + test.fragment_code = code; + test.expected_pixel = values; + test.opcodes = {O::VMovB32, O::Exp, O::SEndpgm}; + + for (size_t i = 0; i < values.size(); i++) { + test.user_data[i] = values[i]; + } + test.has_user_data = true; + test.push_constants = values; + return test; +} + +GraphicsCase GraphicsInlineSrtScalarPromotionExport() { + using O = ShaderOpcode; + + const std::vector values = {0x00000000u, 0x00000000u, 0x00000000u, + 0x3f800000u}; + + std::vector code; + AppendVop3(&code, 0x12fu, 0, 0, 1); + AppendVop3(&code, 0x12fu, 1, 2, 3); + code.push_back(EncodeExp0(0x00, 0xf, true, true, true)); + code.push_back(EncodeExp1(0, 1, 0, 0)); + AppendEnd(&code); + + GraphicsCase test; + test.name = "GraphicsInlineSrtScalarPromotionExport"; + test.fragment_code = code; + test.expected_pixel = values; + test.opcodes = {O::VCvtPkrtzF16F32, O::Exp, O::SEndpgm}; + + for (size_t i = 0; i < values.size(); i++) { + test.user_data[i] = values[i]; + } + test.has_user_data = true; + test.push_constants = values; + return test; +} + +GraphicsCase GraphicsNullVmExportDiscardsInactiveExec() { + using O = ShaderOpcode; + + std::vector code; + AppendVMovLiteral(&code, 0, 0x3f000000u); + AppendVMovLiteral(&code, 1, 0x3f400000u); + AppendVMovLiteral(&code, 2, 0x3f800000u); + AppendVMovLiteral(&code, 3, 0x40000000u); + code.push_back(EncodeExp0(0x00, 0xf, false)); + code.push_back(EncodeExp1(0, 1, 2, 3)); + code.push_back(EncodeSop1(0x04, 126, InlineU32(0))); + code.push_back(EncodeExp0(0x09, 0x0, true, false, true)); + code.push_back(EncodeExp1(0, 0, 0, 0)); + AppendEnd(&code); + + return {"GraphicsNullVmExportDiscardsInactiveExec", + code, + {0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u}, + {O::VMovB32, O::Exp, O::SMovB64, O::SEndpgm}}; +} + +GraphicsCase GraphicsMrt0OffVmExportDiscardsInactiveExec() { + using O = ShaderOpcode; + + std::vector code; + AppendVMovLiteral(&code, 0, 0x3f000000u); + AppendVMovLiteral(&code, 1, 0x3f400000u); + AppendVMovLiteral(&code, 2, 0x3f800000u); + AppendVMovLiteral(&code, 3, 0x40000000u); + code.push_back(EncodeExp0(0x00, 0xf, false)); + code.push_back(EncodeExp1(0, 1, 2, 3)); + code.push_back(EncodeSop1(0x04, 126, InlineU32(0))); + code.push_back(EncodeExp0(0x00, 0x0, true, true, true)); + code.push_back(EncodeExp1(0, 0, 0, 0)); + AppendEnd(&code); + + return {"GraphicsMrt0OffVmExportDiscardsInactiveExec", + code, + {0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u}, + {O::VMovB32, O::Exp, O::SMovB64, O::SEndpgm}}; +} + +GraphicsCase GraphicsFinalVmExportSupersedesEarlierVmMask() { + using O = ShaderOpcode; + + std::vector code; + AppendVMovLiteral(&code, 0, 0x3f000000u); + AppendVMovLiteral(&code, 1, 0x3f400000u); + AppendVMovLiteral(&code, 2, 0x3f800000u); + AppendVMovLiteral(&code, 3, 0x40000000u); + code.push_back(EncodeSop1(0x04, 126, InlineU32(0))); + code.push_back(EncodeExp0(0x00, 0xf, false, false, true)); + code.push_back(EncodeExp1(0, 1, 2, 3)); + code.push_back(EncodeSMovB32(126, 193u)); + code.push_back(EncodeSMovB32(127, 193u)); + AppendVMovLiteral(&code, 4, 0x40400000u); + AppendVMovLiteral(&code, 5, 0x40800000u); + AppendVMovLiteral(&code, 6, 0x40a00000u); + AppendVMovLiteral(&code, 7, 0x40c00000u); + code.push_back(EncodeExp0(0x00, 0xf, true, false, true)); + code.push_back(EncodeExp1(4, 5, 6, 7)); + AppendEnd(&code); + + return {"GraphicsFinalVmExportSupersedesEarlierVmMask", + code, + {0x40400000u, 0x40800000u, 0x40a00000u, 0x40c00000u}, + {O::VMovB32, O::SMovB64, O::Exp, O::SMovB32, O::SEndpgm}}; +} + +GraphicsCase GraphicsBranchPathFinalVmExportDiscardsInactiveExec() { + using O = ShaderOpcode; + + std::vector code; + AppendVMovLiteral(&code, 0, 0x3f000000u); + AppendVMovLiteral(&code, 1, 0x3f400000u); + AppendVMovLiteral(&code, 2, 0x3f800000u); + AppendVMovLiteral(&code, 3, 0x40000000u); + code.push_back(EncodeExp0(0x00, 0xf, false)); + code.push_back(EncodeExp1(0, 1, 2, 3)); + code.push_back(EncodeSop1(0x04, 126, InlineU32(0))); + code.push_back(EncodeExp0(0x09, 0x0, true, false, true)); + code.push_back(EncodeExp1(0, 0, 0, 0)); + code.push_back(EncodeSopc(0x06, InlineU32(0), InlineU32(1))); + const auto branch_index = code.size(); + code.push_back(0); + AppendEnd(&code); + const auto later_export_index = code.size(); + code[branch_index] = EncodeSopp( + 0x05, static_cast(later_export_index - branch_index - 1u)); + code.push_back(EncodeSMovB32(126, 193u)); + code.push_back(EncodeSMovB32(127, 193u)); + AppendVMovLiteral(&code, 4, 0x40400000u); + AppendVMovLiteral(&code, 5, 0x40800000u); + AppendVMovLiteral(&code, 6, 0x40a00000u); + AppendVMovLiteral(&code, 7, 0x40c00000u); + code.push_back(EncodeExp0(0x00, 0xf, true, false, true)); + code.push_back(EncodeExp1(4, 5, 6, 7)); + AppendEnd(&code); + + return {"GraphicsBranchPathFinalVmExportDiscardsInactiveExec", + code, + {0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u}, + {O::VMovB32, O::Exp, O::SMovB64, O::SCmpEqU32, O::SCbranchScc1, + O::SMovB32, O::SEndpgm}}; +} + +TestCase MultipleWorkitemsGlobalId() { + using O = ShaderOpcode; + + std::vector code = { + EncodeVop1(0x01, 1, 0), + EncodeVop2(0x1a, 1, InlineU32(2), 1), + EncodeVop2(0x25, 1, Vgpr(0), 1), + EncodeVop2(0x25, 2, InlineU32(64), 1), + EncodeVop2(0x1a, 3, InlineU32(2), 1), + }; + AppendBufferStoreDword(&code, 2, 3); + AppendEnd(&code); + + TestCase test; + test.name = "MultipleWorkitemsGlobalId"; + test.code = code; + test.expected = {64, 65, 66, 67, 68, 69, 70, 71}; + test.opcodes = {O::VMovB32, O::VLshlrevB32, O::VAddNcU32, O::BufferStoreDword, + O::SEndpgm}; + test.compute_info.threads_num[0] = 4; + test.compute_info.threads_num[1] = 1; + test.compute_info.threads_num[2] = 1; + test.compute_info.group_id[0] = true; + test.compute_info.thread_ids_num = 1; + test.compute_info.workgroup_register = 0; + test.has_compute_info = true; + test.dispatch_x = 2; + return test; +} + +TestCase DispatcherIrreducibleControlFlow() { + using O = ShaderOpcode; + + std::vector code; + code.push_back(EncodeSopp(0x05, 5)); // entry -> B, fallthrough A + AppendVMovU32(&code, 0, 7); + AppendStoreVgpr(&code, 0, 0); + code.push_back(EncodeSopp(0x04, 1)); // A exits while SCC is initially zero + code.push_back(EncodeSopp(0x02, 0xfffau)); // B -> A + AppendEnd(&code); + + TestCase test; + test.name = "DispatcherIrreducibleControlFlow"; + test.code = code; + test.expected = {7}; + test.opcodes = {O::SCbranchScc1, O::VMovB32, O::BufferStoreDword, + O::SCbranchScc0, O::SBranch, O::SEndpgm}; + return test; +} + +std::vector MakeCases() { + std::vector cases; + cases.reserve(128); + auto AddCase = [&cases](TestCase (*factory)()) { + cases.push_back(factory()); + }; + + AddCase(IntegerAddSubMul); + AddCase(BitwiseOps); + AddCase(Shifts); + AddCase(ScalarShiftCountsMaskLowBits); + AddCase(Rdna2ScalarOpcodes); + AddCase(ScalarExtendedArithmetic); + AddCase(ScalarArithmeticSccCarryBorrowOverflow); + AddCase(ScalarMinMaxSccComparisonEdges); + AddCase(ScalarAbsI32UpdatesScc); + AddCase(ScalarShiftLeftAddSccCarryEdges); + AddCase(ScalarCompareOps); + AddCase(ScalarShiftAddAndMaskOps); + AddCase(ScalarNotB64UpdatesScc); + AddCase(ScalarFlbitI32B64Gpu); + AddCase(ScalarSaveExecOps); + AddCase(ScalarOrn2SaveexecUsesSourceOrNotExec); + AddCase(ScalarGetpcWritesNextInstructionPc); + AddCase(ScalarBitfieldPack); + AddCase(ScalarBrevB32PreservesScc); + AddCase(BitfieldExtractWidthPastEndEdges); + AddCase(Scalar64BitOps); + AddCase(ScalarAndn2B64SccBranch); + AddCase(ScalarLiteral); + AddCase(VectorMoves); + AddCase(VectorVop3MoveAppliesFloatSourceModifiers); + AddCase(VectorIntegerOps); + AddCase(VectorShiftCountsMaskLowBits); + AddCase(VectorVop3IntegerOps); + AddCase(VectorBfeI32ArithmeticShiftMasksField); + AddCase(VectorCarryAndBitCountOps); + AddCase(VectorMbcntUsesThreadMask); + AddCase(VectorAddcWritesPerLaneCarryOut); + AddCase(VectorAddcUsesPerLaneCarryIn); + AddCase(VectorVop3BCarryOutWritesSgprMask); + AddCase(VectorVop3BCarryOutUsesEncodedSdst); + AddCase(VectorVop3BSubCoU32UsesRdna2Opcode310); + AddCase(VectorMadU64U32UnsignedCarryOut); + AddCase(VectorLaneAndPackedOps); + AddCase(CvtPkU8F32PacksSelectedByte); + AddCase(CvtPkrtzF16F32SubnormalRoundsTowardZero); + AddCase(PackedMinMaxF16NanAndSignedZeroEdges); + AddCase(VectorMinMaxF16Ops); + AddCase(VectorCvtU16F16Sdwa); + AddCase(VectorMinMaxMed3F16Ops); + AddCase(VectorSpecialF16Ops); + AddCase(VectorWritelaneIgnoresExecMask); + AddCase(VectorPermlanex16); + AddCase(VectorPermlane16FetchInactiveZero); + AddCase(VectorPermlane16FetchInactiveFi); + AddCase(VectorDppQuadPermuteReverse); + AddCase(VectorDppBankMaskPreservesDestination); + AddCase(VectorDppBoundsControlZeroPreservesDestination); + AddCase(Vop3LdexpSourceModifier); + AddCase(Vop1MoveRelSource); + AddCase(Vop1MoveRelDestination); + AddCase(VectorFloatSpecialOps); + AddCase(MadMixF16LiteralHalfSourceUsesOpsel); + AddCase(MadMixF16NegHiIsAbsAndNegIsIndependent); + AddCase(VectorVop3FmaF16UsesRdna2Opcode34b); + AddCase(VectorFloatArithmeticOps); + AddCase(VectorMinMaxF32NanAndSignedZeroEdges); + AddCase(VectorMed3F32NanUsesMin3Path); + AddCase(VectorFloatConversionOps); + AddCase(CvtF32ToIntSaturatesNaNAndOutOfRange); + AddCase(VectorSpecialF32FlushesDenormalInputs); + AddCase(VectorSinCosMaxFiniteSpecialCases); + AddCase(VectorCompareOps); + AddCase(VectorVop3CompareNeU64OnGpu); + AddCase(VectorCompareClassF32); + AddCase(VectorCompareF16Ops); + AddCase(Vop2SdwaCndmaskSourceModifier); + AddCase(Vop3CndmaskUsesSgprMaskLaneBits); + AddCase(Vop3CndmaskAllowsDataSourceModifier); + AddCase(VectorCompareExecOps); + AddCase(VectorVop3FloatCompareNegSourceModifier); + AddCase(VectorVop3CmpxWritesExecMask); + AddCase(VectorVopcSdwaCmpxWritesExecMask); + AddCase(VectorCompareInvertedMaskSelect); + AddCase(BranchSelect); + AddCase(SimpleLoop); + AddCase(BranchVccnzUsesWholeMask); + AddCase(BranchVccnzUsesCarryProducedWholeMask); + AddCase(ScalarMemoryLoadVariants); + AddCase(ScalarLoadSignedImmediateOffsetAddsSoffset); + AddCase(ScalarLoadAlignsComponentsAndMasksAddress); + AddCase(BufferLoadStore); + AddCase(BufferLoadDwordOffenIdxenUsesVaddrPlusOneOffset); + AddCase(BufferStoreDwordOffenIdxenUsesVaddrPlusOneOffset); + AddCase(BufferLoadDwordNoAddressFlagsIgnoresVaddr); + AddCase(BufferLoadDwordIdxenUsesDescriptorStride); + AddCase(BufferStoreDwordIdxenUsesDescriptorStride); + AddCase(BufferLoadVariants); + AddCase(BufferStoreVariants); + AddCase(BufferFormatVariants); + AddCase(BufferFormatStoreVariants); + AddCase(BufferStoreFormatXResource16UintWritesHalfword); + AddCase(BufferLoadFormatXResource8UintZeroExtendsByte); + AddCase(BufferLoadFormatXyResource88UintExtractsBytes); + AddCase(BufferLoadFormatXyResource8888UnormConvertsFirstTwoComponents); + AddCase(BufferStoreFormatXyResource88UintWritesBytes); + AddCase(BufferStoreFormatXyzResource3232UintWritesTwoDwords); + AddCase(BufferStoreFormatXyzwResource323232UintWritesThreeDwords); + AddCase(BufferStoreFormatXyResource32UintWritesOneDword); + AddCase(BufferStoreFormatXyzResource8UintWritesOneByte); + AddCase(BufferStoreFormatXAddTidUsesLaneIndex); + AddCase(BufferStoreFormatXDropsOutOfRangeRecord); + AddCase(TBufferLoadVariants); + AddCase(TBufferLoadFormatX8UintZeroExtendsByte); + AddCase(TBufferLoadFormatX8888UintExtractsFirstByte); + AddCase(TBufferLoadFormatXIdxenUsesDescriptorStride); + AddCase(TBufferLoadFormatX16FloatConvertsToFloat); + AddCase(TBufferStoreFormatX8UintWritesOneByte); + AddCase(TBufferStoreFormatXSintWritesSubDword); + AddCase(TBufferLoadFormatXSintSignExtendsSubDword); + AddCase(TBufferLoadFormatXy1616IntegerComponents); + AddCase(TBufferStoreFormatXy1616IntegerComponents); + AddCase(TBufferLoadFormatXyz16161616UintLoadsHalfwords); + AddCase(TBufferLoadFormatXy1616UnormConvertsToFloat); + AddCase(TBufferLoadFormatXy88UnormConvertsToFloat); + AddCase(TBufferLoadFormatXy88SnormConvertsToFloat); + AddCase(TBufferLoadFormatXy8888UnormConvertsFirstTwoComponents); + AddCase(TBufferLoadFormatXyzw8888UintExtractsBytes); + AddCase(TBufferLoadFormatXyzw1010102SnormConvertsToFloat); + AddCase(TBufferLoadFormatXyz111110FloatUnpacks); + AddCase(TBufferLoadFormatXyzw3232FloatZerosMissingComponents); + AddCase(TBufferStoreFormatXyzw3232FloatWritesOnlyPresentComponents); + AddCase(TBufferStoreFormatXy88IntegerComponents); + AddCase(TBufferLoadFormatXy88IntegerComponents); + AddCase(TBufferStoreVariants); + AddCase(FlatLoadVariants); + AddCase(FlatVirtualAddressRebasesGuestAllocation); + AddCase(GlobalSignedImmediateRebasesBeforeSaddr); + AddCase(FlatSegmentIgnoresSaddrAndMasksOffsetMsb); + AddCase(FlatStoreVariants); + AddCase(DsReadWriteVariants); + AddCase(DsAppendConsumeUsesEncodedLdsSelector); + AddCase(DsAppendUsesEncodedGdsSelector); + AddCase(DsGdsSubdwordAndAtomicWrites); + AddCase(DsReadWrite2Variants); + AddCase(DsAtomicNoReturnVariants); + AddCase(DsAtomicReturnVariants); + AddCase(DsMiscVariants); + AddCase(DsFloatMinMaxUsesSeparateCompareOperand); + AddCase(DsSwizzleInvalidSourceLaneZero); + AddCase(BufferAtomicVariants); + AddCase(BufferAtomicGlc0DoesNotReturnOldValue); + AddCase(ImageLoadVariants); + AddCase(ImageLoadR32UintUsesIntegerSampledImage); + AddCase(ImageLoadMipUsesVaddr2Lod2D); + AddCase(ImageLoadMipNsaUsesSelectedAddressVgprs); + AddCase(ImageLoadA16UintCoordsOnGpu); + AddCase(ImageGetResinfoDmaskWidthHeight); + AddCase(ImageGetResinfoDmaskMipLevels); + AddCase(ImageSampleAndGather); + AddCase(ImageSampleA16SamplerCoordsOnGpu); + AddCase(ImageSampleOpcodeAliasUsesNormalCoords); + AddCase(ImageSampleA16OffsetKeepsTexelOffset32BitOnGpu); + AddCase(ImageSampleA16CompareBiasRdna2AddressOrder); + AddCase(ImageGatherCompareOpcodes); + AddCase(ImageStoreVariants); + AddCase(ImageStoreRgbOneUsesInverseSwizzle); + AddCase(ImageStoreBgraUsesInverseSwizzle); + AddCase(ImageStoreYzwxUsesInverseSwizzle); + AddCase(ImageStoreR32FloatUsesFormatlessStorageImage); + AddCase(ImageStoreR32UintUsesUintStorageImage); + AddCase(ComputeTgSizeSgprUsesWaveMetadata); + AddCase(ImageAtomicVariants); + AddCase(ImageAtomicGlc0DoesNotReturnOldValue); + AddCase(MultipleWorkitemsGlobalId); + AddCase(DispatcherIrreducibleControlFlow); + + return cases; +} + +std::vector MakeGraphicsCases() { + return { + GraphicsInterpolationExport(), + GraphicsFlatInterpolatorExport(), + GraphicsDsAddtidScratchExport(), + GraphicsDirectSgprPushConstantExport(), + GraphicsInlineSrtScalarPromotionExport(), + GraphicsNullVmExportDiscardsInactiveExec(), + GraphicsMrt0OffVmExportDiscardsInactiveExec(), + GraphicsFinalVmExportSupersedesEarlierVmMask(), + GraphicsBranchPathFinalVmExportDiscardsInactiveExec(), + }; +} + +std::vector MakeSkippedCases() { + return {ImageStoreMipWritesExplicitMip2D()}; +} + +void CheckPs5GameExampleImageClearRuntimeShape() { + const auto MakeCode = [] { + std::vector code; + AppendVop3(&code, 0x347u, 4, 8, InlineU32(6), Vgpr(0)); + for (u32 i = 0; i < 4; i++) { + code.push_back(EncodeVop1(0x01u, i, i + 4u)); + } + code.push_back(EncodeMubuf0(0x07u, 0, true, false)); + code.push_back(EncodeMubuf1(0, 0, 4)); + AppendEnd(&code); + return code; + }; + std::array user_data{}; + user_data[0] = 0x00010000u; + user_data[1] = 16u << 16u; + user_data[2] = 64; + user_data[3] = (Prospero::GpuEnumValue(Prospero::BufferFormat::k32_32_32_32UInt) << 12u) | + 4u | (5u << 3u) | (6u << 6u) | (7u << 9u); + std::fill(user_data.begin() + 4, user_data.end(), 0xff000000u); + ShaderComputeInputInfo compute{}; + compute.threads_num[0] = 64; + compute.threads_num[1] = 1; + compute.threads_num[2] = 1; + compute.dispatch_threads_num[0] = 64; + compute.dispatch_threads_num[1] = 1; + compute.dispatch_threads_num[2] = 1; + compute.group_id[0] = true; + compute.dispatch_thread_dimensions = true; + compute.wave_size = 32; + compute.thread_ids_num = 1; + compute.workgroup_register = 8; + + const auto Compile = [&](const char *stage, const std::vector &code) { + ShaderRecompiler::CompileOptions options; + options.stage = ShaderType::Compute; + options.wave_size = 32; + options.user_data_base = 0; + options.user_data_count = static_cast(user_data.size()); + options.user_data = user_data.data(); + options.compute_input_info = &compute; + options.dump_ir = false; + ShaderRecompiler::CompileResult result; + std::string error; + Require("Ps5GameExampleImageClear", stage, + ShaderRecompiler::TryRecompile(code, options, &result, &error), + error); + ValidateSpirv("Ps5GameExampleImageClear", result.spirv); + return result; + }; + + const auto code = MakeCode(); + auto positive = Compile("exact Prospero kernel", code); + + compute.stage.program = + std::make_shared(positive.program); + compute.stage.resources = + std::make_shared(positive.resources); + ShaderBufferResource descriptor{}; + u32 packed_clear = 0; + uint64_t size = 0; + Require("Ps5GameExampleImageClear", "runtime shape", + ResolveComputeImageClear(compute, 64, 1, 1, 0x61u, &descriptor, + &packed_clear, &size) && + descriptor.Base48() == 0x10000u && size == 64u * 16u && + packed_clear == 0xff000000u, + "exact Prospero runtime binding did not resolve to a complete clear"); + + auto non_repeated = positive.resources; + non_repeated.user_data[7] ^= 1u; + compute.stage.resources = + std::make_shared(non_repeated); + Require("Ps5GameExampleImageClear", "non-repeated clear", + !ResolveComputeImageClear(compute, 64, 1, 1, 0x61u, &descriptor, + &packed_clear, &size), + "non-uniform uint4 data was replaced with a color clear"); + compute.stage.resources = + std::make_shared(positive.resources); + compute.dispatch_threads_num[0] = 32; + Require("Ps5GameExampleImageClear", "partial dispatch", + !ResolveComputeImageClear(compute, 32, 1, 1, 0x61u, &descriptor, + &packed_clear, &size), + "partial buffer coverage was classified as a complete clear"); + std::printf("[host] %-32s ok\n", "Ps5GameExampleImageClear"); +} + +void CheckEmbeddedFetchVertexOffset() { + const auto MakeFetch = [](std::initializer_list> adds, + std::optional> late_add = {}) { + std::vector code; + code.push_back(EncodeSMovB32(0, InlineU32(0))); + code.push_back(EncodeSmem0(0x02u, 20, 4)); + code.push_back(EncodeSmem1(0)); + code.push_back(EncodeVop2(0x01u, 0, Vgpr(8), 5)); + for (const auto [sgpr, index_vgpr] : adds) { + AppendVop3B(&code, 0x30fu, 0, 0, sgpr, Vgpr(index_vgpr)); + } + code.push_back(EncodeMubuf0(0x03u, 0, true)); + code.push_back(EncodeMubuf1(9, 5, 0)); + if (late_add.has_value()) { + AppendVop3B(&code, 0x30fu, 0, 0, late_add->first, + Vgpr(late_add->second)); + } + AppendEnd(&code); + return code; + }; + + const auto Compile = [&](const char *name, const std::vector &code, + u32 slot10) { + std::array user_data{}; + user_data[10] = slot10; + ShaderVertexInputInfo vertex; + vertex.fetch_embedded = true; + vertex.fetch_buffer_reg = 0; + vertex.fetch_attrib_reg = 2; + vertex.resources_num = 1; + vertex.resources_dst[0].attr_id = 0; + vertex.resources_dst[0].registers_num = 4; + + ShaderRecompiler::CompileOptions options; + options.stage = ShaderType::Vertex; + options.user_data_base = 8; + options.user_data_count = static_cast(user_data.size()); + options.user_data = user_data.data(); + options.vertex_input_info = &vertex; + + ShaderRecompiler::CompileResult result; + std::string error; + Require(name, "compile", + ShaderRecompiler::TryRecompile(code, options, &result, &error), + error); + Require(name, "fetch rewrite", vertex.resource_fetch_components[0] == 4, + "encoded fetch sequence was not recognized and rewritten"); + return result; + }; + + const auto Resolve = [](const ShaderRecompiler::CompileResult &result, + u32 index_offset) { + ShaderVertexInputInfo vertex; + vertex.fetch_embedded = true; + vertex.stage.program = + std::make_shared(result.program); + vertex.stage.resources = + std::make_shared(result.resources); + return ResolveVertexOffset(index_offset, vertex); + }; + + const auto valid = Compile("EmbeddedFetchVertexOffset", + MakeFetch({{18, 0}}), 7); + Require("EmbeddedFetchVertexOffset", "parse", + valid.program.info.vertex_offset_sgpr == 18 && + Resolve(valid, 0) == 7 && Resolve(valid, 5) == 5, + "canonical fetch offset or register index-offset precedence is wrong"); + + const auto pointer = 0x5b7c5100u; + const auto late = Compile("EmbeddedFetchLateOffset", + MakeFetch({}, std::pair{18, 0}), pointer); + const auto conflict = Compile("EmbeddedFetchConflictingOffset", + MakeFetch({{17, 0}, {18, 0}}), pointer); + const auto malformed = Compile("EmbeddedFetchMalformedOffset", + MakeFetch({{18, 1}}), pointer); + const auto outside = Compile("EmbeddedFetchOutsideOffset", + MakeFetch({{19, 0}}), pointer); + for (const auto *result : {&late, &conflict, &malformed, &outside}) { + Require("EmbeddedFetchVertexOffset", "fail closed", + result->program.info.vertex_offset_sgpr == -1 && + Resolve(*result, 0) == 0, + "non-prolog, conflicting, malformed, or out-of-window add was classified"); + } + std::printf("[host] %-32s ok\n", "EmbeddedFetchVertexOffset"); +} + +#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS +struct CacheFaultContext { + TextureCache *texture = nullptr; +}; + +bool CacheFault(void *opaque, PageFaultAccess access, uint64_t vaddr, + uint64_t size, PageFaultPhase phase) noexcept { + auto *context = static_cast(opaque); + return context != nullptr && context->texture != nullptr && + context->texture->InvalidateMemory(access, vaddr, size, phase); +} + +[[noreturn]] void RunImageViewDeathCase(const char *kind) { + if (std::strcmp(kind, "sampled") == 0) { + (void)SelectSampledColorView(VK_FORMAT_R8G8B8A8_UNORM, + VK_FORMAT_R8G8B8A8_UNORM, + DstSel(4, 5, 6, 0)); + } else if (std::strcmp(kind, "sampled-compatible-swizzle") == 0) { + (void)SelectSampledColorView(VK_FORMAT_B8G8R8A8_UNORM, + VK_FORMAT_R8G8B8A8_UNORM, + DstSel(4, 5, 6, 7)); + } else if (std::strcmp(kind, "sampled-compatible-reverse") == 0) { + (void)SelectSampledColorView(VK_FORMAT_R8G8B8A8_UNORM, + VK_FORMAT_B8G8R8A8_UNORM, + DstSel(6, 5, 4, 7)); + } else if (std::strcmp(kind, "sampled-compatible-class") == 0) { + (void)SelectSampledColorView(VK_FORMAT_R8G8B8A8_UINT, + VK_FORMAT_R8G8B8A8_UNORM, + DstSel(4, 5, 6, 7)); + } else if (std::strcmp(kind, "sampled-compatible-colorspace") == 0) { + (void)SelectSampledColorView(VK_FORMAT_B8G8R8A8_SRGB, + VK_FORMAT_R8G8B8A8_UNORM, + DstSel(4, 5, 6, 7)); + } else if (std::strcmp(kind, "sampled-compatible-snorm") == 0) { + (void)SelectSampledColorView(VK_FORMAT_R8G8B8A8_SNORM, + VK_FORMAT_R8G8B8A8_UNORM, + DstSel(4, 5, 6, 7)); + } else if (std::strcmp(kind, "sampled-depth-format") == 0) { + (void)SelectSampledDepthView(VK_FORMAT_D32_SFLOAT_S8_UINT, + VK_FORMAT_R16_UNORM, DstSel(4, 4, 4, 4)); + } else if (std::strcmp(kind, "sampled-depth-swizzle") == 0) { + (void)SelectSampledDepthView(VK_FORMAT_D32_SFLOAT_S8_UINT, + VK_FORMAT_R32_SFLOAT, DstSel(4, 5, 6, 7)); + } else if (std::strcmp(kind, "storage") == 0) { + (void)SelectStorageColorView(VK_FORMAT_R8G8B8A8_UNORM, + VK_FORMAT_R8G8B8A8_UNORM, DstSel(4, 5, 6, 0)); + } else if (std::strcmp(kind, "storage-format") == 0) { + (void)SelectStorageColorView(VK_FORMAT_B8G8R8A8_UNORM, + VK_FORMAT_R8G8B8A8_UNORM, DstSel(4, 5, 6, 7)); + } else if (std::strcmp(kind, "storage-compatible-swizzle") == 0) { + (void)SelectStorageColorView(VK_FORMAT_B8G8R8A8_SRGB, + VK_FORMAT_R8G8B8A8_UNORM, DstSel(4, 5, 6, 7)); + } else { + ShaderRecompiler::IR::ImageResource resource{}; + resource.kind = ShaderRecompiler::IR::ResourceKind::StorageImage; + resource.dimension = ShaderRecompiler::Decoder::ImageDimension::Dim2D; + resource.written = true; + if (std::strcmp(kind, "storage-kind") == 0) { + resource.kind = ShaderRecompiler::IR::ResourceKind::Image; + } else if (std::strcmp(kind, "storage-no-write") == 0) { + resource.written = false; + } else if (std::strcmp(kind, "storage-atomic") == 0) { + resource.atomic = true; + } else if (std::strcmp(kind, "storage-compare") == 0) { + resource.depth_compare = true; + } else if (std::strcmp(kind, "storage-mip") == 0) { + resource.mip_mode = ShaderRecompiler::IR::ImageMipMode::DynamicStorage; + } else if (std::strcmp(kind, "storage-dimension") == 0) { + resource.dimension = ShaderRecompiler::Decoder::ImageDimension::Unknown; + } else { + std::_Exit(0x7e); + } + ValidateStorageImageResource(resource); + } + std::_Exit(0x7f); +} + +void CheckSampledColorViews() { + Require("SampledColorViews", "identity", + SelectSampledColorView(VK_FORMAT_R8G8B8A8_UNORM, + VK_FORMAT_R8G8B8A8_UNORM, + DstSel(4, 5, 6, 7)) == VulkanImage::VIEW_DEFAULT, + "RGBA did not select the identity view"); + Require("SampledColorViews", "R8 R001", + SelectSampledColorView(VK_FORMAT_R8_UNORM, VK_FORMAT_R8_UNORM, + DstSel(4, 0, 0, 1)) == VulkanImage::VIEW_R001, + "R8 did not select its R001 view"); + Require("SampledColorViews", "R16G16 RG01", + SelectSampledColorView(VK_FORMAT_R16G16_SFLOAT, + VK_FORMAT_R16G16_SFLOAT, + DstSel(4, 5, 0, 1)) == VulkanImage::VIEW_RG01, + "R16G16 did not select its RG01 view"); + Require("SampledColorViews", "alpha one", + SelectSampledColorView(VK_FORMAT_R8G8B8A8_UNORM, + VK_FORMAT_R8G8B8A8_UNORM, + DstSel(4, 5, 6, 1)) == VulkanImage::VIEW_RGB1, + "RGB1 did not select the alpha-one view"); + Require("SampledColorViews", "mutable BGRA target", + SelectSampledColorView(VK_FORMAT_B8G8R8A8_UNORM, + VK_FORMAT_R8G8B8A8_UNORM, + DstSel(6, 5, 4, 7)) == VulkanImage::VIEW_RGBA8_BGRA, + "BGRA target did not select the exact RGBA/BGRA mutable view"); + Require("SampledColorViews", "mutable sRGB view of UNORM BGRA target", + SelectSampledColorView(VK_FORMAT_B8G8R8A8_UNORM, + VK_FORMAT_R8G8B8A8_SRGB, + DstSel(6, 5, 4, 7)) == VulkanImage::VIEW_RGBA8_BGRA, + "UNORM BGRA target did not select its compatible sRGB RGBA sampled view"); + Require("SampledColorViews", "mutable sRGB BGRA target", + BgraToRgbaSampledViewFormat(VK_FORMAT_B8G8R8A8_SRGB) == + VK_FORMAT_R8G8B8A8_SRGB && + SelectSampledColorView(VK_FORMAT_B8G8R8A8_SRGB, + VK_FORMAT_R8G8B8A8_SRGB, + DstSel(6, 5, 4, 7)) == + VulkanImage::VIEW_RGBA8_BGRA, + "sRGB BGRA target did not select its matching mutable RGBA view"); + Require("SampledColorViews", "D32 depth target", + SelectSampledDepthView(VK_FORMAT_D32_SFLOAT_S8_UINT, + VK_FORMAT_R32_SFLOAT, + DstSel(4, 4, 4, 4)) == + VulkanImage::VIEW_DEPTH_TEXTURE, + "D32 depth target did not select its depth-aspect view"); + Require("SampledColorViews", "D16 R000 depth target", + SelectSampledDepthView(VK_FORMAT_D16_UNORM, VK_FORMAT_R16_UNORM, + DstSel(4, 0, 0, 0)) == + VulkanImage::VIEW_R000, + "D16 depth target did not select its R000 depth-aspect view"); + Require("SampledColorViews", "storage identity", + SelectStorageColorView(VK_FORMAT_R8G8B8A8_UNORM, + VK_FORMAT_R8G8B8A8_UNORM, + DstSel(4, 5, 6, 7)) == VulkanImage::VIEW_STORAGE, + "RGBA storage did not select the identity storage view"); + Require( + "SampledColorViews", "storage RGB1 write mapping", + SelectStorageColorView(VK_FORMAT_R8G8B8A8_UNORM, VK_FORMAT_R8G8B8A8_UNORM, + DstSel(4, 5, 6, 1)) == VulkanImage::VIEW_STORAGE, + "RGBA8 RGB1 storage did not select the identity storage view"); + Require( + "SampledColorViews", "storage BGRA write mapping", + SelectStorageColorView(VK_FORMAT_R8G8B8A8_UNORM, VK_FORMAT_R8G8B8A8_UNORM, + DstSel(6, 5, 4, 7)) == VulkanImage::VIEW_STORAGE, + "RGBA8 BGRA storage did not select the identity storage view"); + Require( + "SampledColorViews", "mutable BGRA sRGB storage view", + SelectStorageColorView(VK_FORMAT_B8G8R8A8_SRGB, VK_FORMAT_R8G8B8A8_UNORM, + DstSel(6, 5, 4, 7)) == VulkanImage::VIEW_STORAGE, + "BGRA sRGB target did not select its RGBA UNORM identity storage view"); + Require("SampledColorViews", "storage YZWX write mapping", + SelectStorageColorView( + VK_FORMAT_R32G32B32A32_SFLOAT, VK_FORMAT_R32G32B32A32_SFLOAT, + DstSel(5, 6, 7, 4)) == VulkanImage::VIEW_STORAGE, + "RGBA32F YZWX storage did not select the identity storage view"); + Require("SampledColorViews", "storage R32 uint R001 write mapping", + SelectStorageColorView(VK_FORMAT_R32_UINT, VK_FORMAT_R32_UINT, + DstSel(4, 0, 0, 1)) == + VulkanImage::VIEW_STORAGE, + "R32_UINT R001 storage did not select the identity storage view"); + Require("SampledColorViews", "storage R16 uint R000 write mapping", + SelectStorageColorView(VK_FORMAT_R16_UINT, VK_FORMAT_R16_UINT, + DstSel(4, 0, 0, 0)) == VulkanImage::VIEW_STORAGE, + "R16_UINT R000 storage did not select the identity storage view"); + ShaderRecompiler::IR::ImageResource storage_resource{}; + storage_resource.kind = ShaderRecompiler::IR::ResourceKind::StorageImage; + storage_resource.dimension = ShaderRecompiler::Decoder::ImageDimension::Dim2D; + storage_resource.written = true; + Require("SampledColorViews", "storage resource", + IsSupportedStorageImageResource(storage_resource), + "exact storage resource contract was rejected"); + storage_resource.dimension = ShaderRecompiler::Decoder::ImageDimension::Dim3D; + storage_resource.read = true; + Require("SampledColorViews", "read-write 3D storage resource", + IsSupportedStorageImageResource(storage_resource), + "basic read-write 3D storage resource was rejected"); + storage_resource.dimension = ShaderRecompiler::Decoder::ImageDimension::Dim2DArray; + storage_resource.read = false; + Require("SampledColorViews", "write-only 2D-array storage resource", + IsSupportedStorageImageResource(storage_resource), + "basic write-only 2D-array storage resource was rejected"); + storage_resource.kind = ShaderRecompiler::IR::ResourceKind::StorageImageUint; + Require("SampledColorViews", "write-only uint 2D-array storage resource", + IsSupportedStorageImageResource(storage_resource), + "basic write-only uint 2D-array storage resource was rejected"); + + char path[MAX_PATH]{}; + Require("SampledColorViews", "host", + GetModuleFileNameA(nullptr, path, MAX_PATH) != 0, + "GetModuleFileName failed"); + for (const char *kind : + {"sampled", "sampled-compatible-swizzle", "sampled-compatible-reverse", + "sampled-compatible-class", "sampled-compatible-colorspace", + "sampled-compatible-snorm", "sampled-depth-format", + "sampled-depth-swizzle", "storage", "storage-format", + "storage-compatible-swizzle", "storage-kind", "storage-no-write", + "storage-atomic", "storage-compare", "storage-mip", + "storage-dimension"}) { + std::string command = + std::string("\"") + path + "\" --image-view-death " + kind; + std::vector mutable_command(command.begin(), command.end()); + mutable_command.push_back('\0'); + STARTUPINFOA startup{sizeof(startup)}; + PROCESS_INFORMATION process{}; + Require("SampledColorViews", "host", + CreateProcessA(nullptr, mutable_command.data(), nullptr, nullptr, + FALSE, CREATE_NO_WINDOW, nullptr, nullptr, &startup, + &process) != 0, + "CreateProcess failed"); + Require("SampledColorViews", "host", + WaitForSingleObject(process.hProcess, 10000) == WAIT_OBJECT_0, + "unsupported view death case timed out"); + DWORD exit_code = 0; + const bool exited = GetExitCodeProcess(process.hProcess, &exit_code) != 0; + CloseHandle(process.hThread); + CloseHandle(process.hProcess); + Require("SampledColorViews", "host", exited && exit_code == 321, + std::string(kind) + " component mapping did not report a fatal error"); + } + std::printf("[host] %-32s ok\n", "SampledColorRenderTargetViews"); +} + +void CheckSampledDepthResource() { + ShaderRecompiler::IR::ImageResource resource{}; + resource.kind = ShaderRecompiler::IR::ResourceKind::Image; + resource.dimension = ShaderRecompiler::Decoder::ImageDimension::Dim2D; + resource.read = true; + resource.depth_compare = true; + Require("SampledDepthResource", "comparison read", + IsSupportedSampledDepthResource(resource), + "basic comparison-sampled depth resource was rejected"); + + resource.depth_compare = false; + Require("SampledDepthResource", "ordinary read", + IsSupportedSampledDepthResource(resource), + "basic non-comparison depth read was rejected"); + + const auto basic = resource; + resource.read = false; + Require("SampledDepthResource", "read required", + !IsSupportedSampledDepthResource(resource), + "non-reading depth resource was accepted"); + resource = basic; + resource.written = true; + Require("SampledDepthResource", "write rejected", + !IsSupportedSampledDepthResource(resource), + "writable depth resource was accepted"); + resource = basic; + resource.atomic = true; + Require("SampledDepthResource", "atomic rejected", + !IsSupportedSampledDepthResource(resource), + "atomic depth resource was accepted"); + resource = basic; + resource.mip_mode = ShaderRecompiler::IR::ImageMipMode::DynamicStorage; + Require("SampledDepthResource", "dynamic mip rejected", + !IsSupportedSampledDepthResource(resource), + "dynamic-storage mip depth resource was accepted"); + resource = basic; + resource.dimension = ShaderRecompiler::Decoder::ImageDimension::Dim2DArray; + Require("SampledDepthResource", "array rejected", + !IsSupportedSampledDepthResource(resource), + "array depth resource was accepted"); + resource = basic; + resource.kind = ShaderRecompiler::IR::ResourceKind::ImageUint; + Require("SampledDepthResource", "integer rejected", + !IsSupportedSampledDepthResource(resource), + "integer depth resource was accepted"); + std::printf("[host] %-32s ok\n", "SampledDepthResource"); +} + +void CheckSampledDepthDescriptor() { + ShaderTextureResource descriptor{{0x00eb0900u, 0xc1600000u, 0x00bcc14fu, + 0x91800924u, 0x00000000u, 0x00700000u, + 0x00000000u, 0x00000000u}}; + DepthStencilVulkanImage image; + image.extent = {1344, 756}; + image.guest_pitch = 1408; + image.layers = 1; + image.format = VK_FORMAT_D32_SFLOAT_S8_UINT; + Require("SampledDepthDescriptor", "PPSA04319 padded pitch", + descriptor.Width5() + 1u == 1344 && descriptor.Height5() + 1u == 756 && + TileGetTexturePitch(descriptor.Format(), 1344, 1, + descriptor.TileMode()) == 1408 && + IsSupportedDepthTargetDescriptor(descriptor, image), + "valid padded-pitch depth texture was rejected"); + + image.guest_pitch = 1344; + Require("SampledDepthDescriptor", "target pitch mismatch", + !IsSupportedDepthTargetDescriptor(descriptor, image), + "mismatched unpadded target pitch was accepted"); + image.guest_pitch = 1472; + Require("SampledDepthDescriptor", "different padded pitch", + !IsSupportedDepthTargetDescriptor(descriptor, image), + "different padded target pitch was accepted"); + image.guest_pitch = 1408; + descriptor.fields[3] = + (descriptor.fields[3] & ~(0x1fu << 20u)) | + (Prospero::GpuEnumValue(Prospero::TileMode::kStandard4KB) << 20u); + Require("SampledDepthDescriptor", "non-depth tile", + !IsSupportedDepthTargetDescriptor(descriptor, image), + "non-depth texture descriptor was accepted"); + std::printf("[host] %-32s ok\n", "SampledDepthDescriptor"); +} + +void CheckBufferCacheRangeMerge() { + BufferCacheRange merged{0x10000, 0x4000}; + Require("BufferCacheRangeMerge", "disjoint", + !MergeOverlappingBufferCacheRange(&merged, {0x14000, 0x4000}) && + merged.address == 0x10000 && merged.size == 0x4000, + "adjacent range was merged"); + Require("BufferCacheRangeMerge", "right growth", + MergeOverlappingBufferCacheRange(&merged, {0x12000, 0x8000}) && + merged.address == 0x10000 && merged.size == 0xa000, + "right overlap did not grow the cache range"); + Require("BufferCacheRangeMerge", "left growth", + MergeOverlappingBufferCacheRange(&merged, {0xc000, 0x8000}) && + merged.address == 0xc000 && merged.size == 0xe000, + "left overlap did not grow the cache range"); + Require("BufferCacheRangeMerge", "contained", + MergeOverlappingBufferCacheRange(&merged, {0x10000, 0x1000}) && + merged.address == 0xc000 && merged.size == 0xe000, + "contained range changed the cache union"); + Require("BufferCacheRangeMerge", "queue ownership", + CanMergeBufferCacheQueueMask(0, 3) && + CanMergeBufferCacheQueueMask(uint64_t{1} << 3u, 3) && + !CanMergeBufferCacheQueueMask((uint64_t{1} << 2u) | + (uint64_t{1} << 3u), + 3) && + !CanMergeBufferCacheQueueMask(0, 64), + "cross-queue or invalid queue ownership was accepted"); + Require("BufferCacheRangeMerge", "readback queue ownership", + !CanReadbackBufferCacheQueueMask(0, 3) && + CanReadbackBufferCacheQueueMask(uint64_t{1} << 3u, 3) && + !CanReadbackBufferCacheQueueMask((uint64_t{1} << 2u) | + (uint64_t{1} << 3u), + 3) && + !CanReadbackBufferCacheQueueMask(uint64_t{1} << 3u, 64), + "zero, cross-queue, or invalid readback ownership was accepted"); + std::printf("[host] %-32s ok\n", "BufferCacheRangeMerge"); +} + +ShaderRecompiler::IR::ImageResource BasicStorageTextureResource() { + ShaderRecompiler::IR::ImageResource resource{}; + resource.kind = ShaderRecompiler::IR::ResourceKind::StorageImage; + resource.dimension = ShaderRecompiler::Decoder::ImageDimension::Dim3D; + resource.read = true; + resource.written = true; + return resource; +} + +ShaderTextureResource BasicStorageTextureDescriptor() { + return {{0x00785d00u, 0x04700000u, 0x00080008u, 0xa1b00facu, + 0x00000020u, 0x00700000u, 0x00000000u, 0x00000000u}}; +} + +ShaderRecompiler::IR::ImageResource BasicLinearStorageTextureResource() { + auto resource = BasicStorageTextureResource(); + resource.dimension = ShaderRecompiler::Decoder::ImageDimension::Dim2D; + resource.read = false; + return resource; +} + +ShaderTextureResource BasicLinearStorageTextureDescriptor() { + return {{0x04bcc401u, 0xc3800000u, 0x021bc3bfu, 0x900003acu, + 0x00000000u, 0x00700000u, 0x00000000u, 0x00000000u}}; +} + +ShaderRecompiler::IR::ImageResource BasicBgraStorageTextureResource() { + auto resource = BasicStorageTextureResource(); + resource.dimension = ShaderRecompiler::Decoder::ImageDimension::Dim2D; + resource.read = false; + return resource; +} + +ShaderTextureResource BasicBgraStorageTextureDescriptor() { + return {{0x007c6500u, 0xc3800000u, 0x010dc1dfu, 0x91b00f2eu, + 0x00000000u, 0x00700000u, 0x00000000u, 0x00000000u}}; +} + +ShaderTextureResource BasicYzwxStorageTextureDescriptor() { + return {{0x00627801u, 0xc4d00000u, 0x0001c001u, 0x900009f5u, + 0x00000000u, 0x00700000u, 0x00000000u, 0x00000000u}}; +} + +ShaderRecompiler::IR::ImageResource BasicArrayStorageTextureResource() { + auto resource = BasicLinearStorageTextureResource(); + resource.dimension = ShaderRecompiler::Decoder::ImageDimension::Dim2DArray; + return resource; +} + +ShaderTextureResource BasicArrayStorageTextureDescriptor() { + return {{0x20179000u, 0x03800000u, 0x00000000u, 0xd1b00f2eu, + 0x00000000u, 0x00700000u, 0x00000000u, 0x00000000u}}; +} + +ShaderRecompiler::IR::ImageResource BasicUintArrayStorageTextureResource() { + auto resource = BasicArrayStorageTextureResource(); + resource.kind = ShaderRecompiler::IR::ResourceKind::StorageImageUint; + return resource; +} + +ShaderTextureResource BasicUintArrayStorageTextureDescriptor() { + return {{0x20179200u, 0x01400000u, 0x00000000u, 0xd1b00204u, + 0x00000000u, 0x00700000u, 0x00000000u, 0x00000000u}}; +} + +ShaderRecompiler::IR::ImageResource BasicUintVolumeStorageTextureResource() { + auto resource = BasicStorageTextureResource(); + resource.kind = ShaderRecompiler::IR::ResourceKind::StorageImageUint; + resource.read = false; + return resource; +} + +ShaderTextureResource BasicUintVolumeStorageTextureDescriptor() { + return {{0x20180600u, 0xc0b00000u, 0x0003c003u, 0xa0000004u, + 0x0000000fu, 0x00700000u, 0x00000000u, 0x00000000u}}; +} + +[[noreturn]] void RunStorageTextureDescriptorDeathCase(const char *kind) { + auto resource = BasicStorageTextureResource(); + auto descriptor = BasicStorageTextureDescriptor(); + if (std::strcmp(kind, "linear-rgb1-read") == 0) { + resource = BasicLinearStorageTextureResource(); + descriptor = BasicLinearStorageTextureDescriptor(); + resource.read = true; + } else if (std::strcmp(kind, "bgra-read") == 0) { + resource = BasicBgraStorageTextureResource(); + descriptor = BasicBgraStorageTextureDescriptor(); + resource.read = true; + } else if (std::strcmp(kind, "yzwx-read") == 0) { + resource = BasicLinearStorageTextureResource(); + descriptor = BasicYzwxStorageTextureDescriptor(); + resource.read = true; + } else if (std::strcmp(kind, "yzwx-format") == 0) { + resource = BasicLinearStorageTextureResource(); + descriptor = BasicYzwxStorageTextureDescriptor(); + descriptor.fields[1] = + (descriptor.fields[1] & ~0x1ff00000u) | + (Prospero::GpuEnumValue(Prospero::BufferFormat::k16_16_16_16Float) << 20u); + } else if (std::strcmp(kind, "resource") == 0) { + resource.written = false; + } else if (std::strcmp(kind, "type") == 0) { + descriptor.fields[3] = + (descriptor.fields[3] & 0x0fffffffu) | + (Prospero::GpuEnumValue(Prospero::ImageType::kColor2D) << 28u); + } else if (std::strcmp(kind, "tile") == 0) { + descriptor.fields[3] = + (descriptor.fields[3] & ~(0x1fu << 20u)) | + (Prospero::GpuEnumValue(Prospero::TileMode::kStandard4KB) << 20u); + } else if (std::strcmp(kind, "mip") == 0) { + descriptor.fields[3] |= 1u << 16u; + } else if (std::strcmp(kind, "swizzle") == 0) { + descriptor.fields[3] = + (descriptor.fields[3] & ~0xfffu) | DstSel(4, 5, 6, 1); + } else if (std::strcmp(kind, "array-base-view") == 0) { + resource = BasicArrayStorageTextureResource(); + descriptor = BasicArrayStorageTextureDescriptor(); + descriptor.fields[4] |= 1u << 16u; + } else if (std::strcmp(kind, "reserved") == 0) { + descriptor.fields[1] |= 1u << 29u; + } else if (std::strcmp(kind, "metadata") == 0) { + descriptor.fields[6] |= 1u << 21u; + } else if (std::strcmp(kind, "uint-format") == 0) { + descriptor.fields[1] = + (descriptor.fields[1] & ~0x1ff00000u) | + (Prospero::GpuEnumValue(Prospero::BufferFormat::k8_8_8_8UInt) << 20u); + } else if (std::strcmp(kind, "uint-resource-float-format") == 0) { + resource = BasicUintArrayStorageTextureResource(); + descriptor = BasicArrayStorageTextureDescriptor(); + } else { + std::_Exit(0x7e); + } + ValidateStorageTexture(resource, descriptor, 0x10000); + std::_Exit(0x7f); +} + +void CheckBasicStorageTextureDescriptor() { + const auto descriptor = BasicStorageTextureDescriptor(); + Require("BasicStorageTexture", "descriptor", + descriptor.Base40() == 0x785d0000ull && descriptor.Width5() + 1u == 33 && + descriptor.Height5() + 1u == 33 && descriptor.Depth() + 1u == 33, + "basic 3D storage descriptor fixture is malformed"); + ValidateStorageTexture(BasicStorageTextureResource(), descriptor, 0x10000); + + const auto linear = BasicLinearStorageTextureDescriptor(); + Require("BasicStorageTexture", "linear descriptor", + linear.Base40() == 0x4bcc40100ull && linear.Width5() + 1u == 3840 && + linear.Height5() + 1u == 2160 && linear.Depth() + 1u == 1 && + linear.Format() == Prospero::GpuEnumValue(Prospero::BufferFormat::k8_8_8_8UNorm) && + linear.TileMode() == Prospero::GpuEnumValue(Prospero::TileMode::kLinear), + "PPSA07429 linear 2D storage descriptor fixture is malformed"); + ValidateStorageTexture(BasicLinearStorageTextureResource(), linear, 0x1fa4000); + + const auto bgra = BasicBgraStorageTextureDescriptor(); + Require("BasicStorageTexture", "BGRA descriptor", + bgra.Base40() == 0x7c650000ull && bgra.Width5() + 1u == 1920 && + bgra.Height5() + 1u == 1080 && bgra.Depth() + 1u == 1 && + bgra.Format() == Prospero::GpuEnumValue(Prospero::BufferFormat::k8_8_8_8UNorm) && + bgra.TileMode() == Prospero::GpuEnumValue(Prospero::TileMode::kRenderTarget) && + bgra.DstSelXYZW() == DstSel(6, 5, 4, 7), + "PPSA02604 BGRA 2D storage descriptor fixture is malformed"); + ValidateStorageTexture(BasicBgraStorageTextureResource(), bgra, 0x870000); + + const auto yzwx = BasicYzwxStorageTextureDescriptor(); + Require("BasicStorageTexture", "YZWX descriptor", + yzwx.Base40() == 0x62780100ull && yzwx.Width5() + 1u == 8 && + yzwx.Height5() + 1u == 8 && yzwx.Depth() + 1u == 1 && + yzwx.Format() == Prospero::GpuEnumValue(Prospero::BufferFormat::k32_32_32_32Float) && + yzwx.TileMode() == Prospero::GpuEnumValue(Prospero::TileMode::kLinear) && + yzwx.DstSelXYZW() == DstSel(5, 6, 7, 4), + "PPSA04181 linear YZWX storage descriptor fixture is malformed"); + ValidateStorageTexture(BasicLinearStorageTextureResource(), yzwx, 0x800); + + const auto array = BasicArrayStorageTextureDescriptor(); + Require("BasicStorageTexture", "2D-array descriptor", + array.Base40() == 0x2017900000ull && array.Width5() + 1u == 1 && + array.Height5() + 1u == 1 && array.Depth() + 1u == 1 && + array.BaseArray5() == 0 && + array.Type() == Prospero::GpuEnumValue(Prospero::ImageType::kColor2DArray) && + array.Format() == Prospero::GpuEnumValue(Prospero::BufferFormat::k8_8_8_8UNorm) && + array.TileMode() == Prospero::GpuEnumValue(Prospero::TileMode::kRenderTarget) && + array.DstSelXYZW() == DstSel(6, 5, 4, 7), + "PPSA21268 2D-array storage descriptor fixture is malformed"); + ValidateStorageTexture(BasicArrayStorageTextureResource(), array, 0x10000); + + const auto uint_array = BasicUintArrayStorageTextureDescriptor(); + Require("BasicStorageTexture", "uint 2D-array descriptor", + uint_array.Base40() == 0x2017920000ull && uint_array.Width5() + 1u == 1 && + uint_array.Height5() + 1u == 1 && uint_array.Depth() + 1u == 1 && + uint_array.BaseArray5() == 0 && + uint_array.Type() == Prospero::GpuEnumValue(Prospero::ImageType::kColor2DArray) && + uint_array.Format() == Prospero::GpuEnumValue(Prospero::BufferFormat::k32UInt) && + uint_array.TileMode() == Prospero::GpuEnumValue(Prospero::TileMode::kRenderTarget) && + uint_array.DstSelXYZW() == DstSel(4, 0, 0, 1), + "PPSA21268 uint 2D-array storage descriptor fixture is malformed"); + ValidateStorageTexture(BasicUintArrayStorageTextureResource(), uint_array, 0x10000); + + const auto uint_volume = BasicUintVolumeStorageTextureDescriptor(); + Require("BasicStorageTexture", "uint 3D descriptor", + uint_volume.Base40() == 0x2018060000ull && uint_volume.Width5() + 1u == 16 && + uint_volume.Height5() + 1u == 16 && uint_volume.Depth() + 1u == 16 && + uint_volume.BaseArray5() == 0 && + uint_volume.Type() == Prospero::GpuEnumValue(Prospero::ImageType::kColor3D) && + uint_volume.Format() == Prospero::GpuEnumValue(Prospero::BufferFormat::k16UInt) && + uint_volume.TileMode() == Prospero::GpuEnumValue(Prospero::TileMode::kLinear) && + uint_volume.DstSelXYZW() == DstSel(4, 0, 0, 0), + "PPSA21268 uint 3D storage descriptor fixture is malformed"); + ValidateStorageTexture(BasicUintVolumeStorageTextureResource(), uint_volume, 0x10000); + + char path[MAX_PATH]{}; + Require("BasicStorageTexture", "host", + GetModuleFileNameA(nullptr, path, MAX_PATH) != 0, + "GetModuleFileName failed"); + for (const char *kind : {"resource", "type", "tile", "mip", "swizzle", + "linear-rgb1-read", "bgra-read", "yzwx-read", "yzwx-format", + "array-base-view", "reserved", "metadata", "uint-format", + "uint-resource-float-format"}) { + std::string command = std::string("\"") + path + + "\" --storage-texture-descriptor-death " + kind; + std::vector mutable_command(command.begin(), command.end()); + mutable_command.push_back('\0'); + STARTUPINFOA startup{sizeof(startup)}; + PROCESS_INFORMATION process{}; + Require("BasicStorageTexture", "host", + CreateProcessA(nullptr, mutable_command.data(), nullptr, nullptr, FALSE, + CREATE_NO_WINDOW, nullptr, nullptr, &startup, &process) != 0, + "CreateProcess failed"); + Require("BasicStorageTexture", "host", + WaitForSingleObject(process.hProcess, 10000) == WAIT_OBJECT_0, + "descriptor death case timed out"); + DWORD exit_code = 0; + const bool exited = GetExitCodeProcess(process.hProcess, &exit_code) != 0; + CloseHandle(process.hThread); + CloseHandle(process.hProcess); + Require("BasicStorageTexture", "host", exited && exit_code == 321, + std::string(kind) + " storage descriptor did not report a fatal error"); + } + std::printf("[host] %-32s ok\n", "BasicStorageTextureDescriptor"); +} + +void CheckStorageTextureLinearUploadLayout() { + constexpr uint32_t format = Prospero::GpuEnumValue(Prospero::BufferFormat::k8_8_8_8UNorm); + constexpr uint32_t width = 3840; + constexpr uint32_t height = 2160; + constexpr uint32_t depth = 1; + constexpr uint32_t tile = Prospero::GpuEnumValue(Prospero::TileMode::kLinear); + const auto pitch = TileGetTexturePitch(format, width, 1, tile); + TileSizeAlign total{}; + TileGetTextureTotalSize(format, width, height, depth, pitch, 1, tile, false, + &total); + const auto layout = TextureCalcUploadLayout( + format, width, height, 1, depth, pitch, tile, total.size, true, false, + false, "StorageTextureLinearTest"); + const auto regions = TextureBuildUploadRegions( + layout, VK_FORMAT_R8G8B8A8_UNORM, width, height, depth, 1, false, + false, TextureUploadDestination::MipLevels, + TextureUploadSliceLayout::MipChainPerSlice); + Require("StorageTextureLinearUpload", "layout", + pitch == width && total.size == 0x1fa4000 && total.align == 256 && + layout.tile == tile && layout.pitch == width && + layout.slice_stride == total.size && regions.size() == 1 && + regions[0].offset == 0 && regions[0].width == width && + regions[0].height == height && regions[0].pitch == width && + TextureCalcUploadSize(layout, regions, 1, depth, + TextureUploadSliceLayout::MipChainPerSlice) == + total.size, + "linear RGBA8 storage upload lost Prospero pitch or allocation size"); + std::printf("[host] %-32s ok\n", "StorageTextureLinearUpload"); +} + +void CheckStorageTextureLinearReadbackLayout() { + ImageInfo info{}; + info.address = 0x4bcc40100ull; + info.size = 0x1fa4000; + info.format = Prospero::GpuEnumValue(Prospero::BufferFormat::k8_8_8_8UNorm); + info.width = 3840; + info.height = 2160; + info.pitch = 3840; + info.levels = 1; + info.tile = Prospero::GpuEnumValue(Prospero::TileMode::kLinear); + info.swizzle = DstSel(4, 5, 6, 1); + info.depth = 1; + info.type = Prospero::GpuEnumValue(Prospero::ImageType::kColor2D); + const auto transfer = + MakeColorImageTransferInfo(info, VK_FORMAT_R8G8B8A8_UNORM, 4); + const auto linear_size = + ((static_cast(transfer.height) - 1) * transfer.pitch + + transfer.width) * + transfer.bytes_per_element; + Require("StorageTextureLinearReadback", "layout", + transfer.address == info.address && transfer.size == info.size && + transfer.format == VK_FORMAT_R8G8B8A8_UNORM && + transfer.width == info.width && transfer.height == info.height && + transfer.pitch == info.pitch && transfer.bytes_per_element == 4 && + transfer.tile_mode == info.tile && linear_size == info.size, + "normalized storage readback lost the exact PS5 linear image layout"); + std::printf("[host] %-32s ok\n", "StorageTextureLinearReadback"); +} + +void CheckStorageImageSwizzleSpecializationId() { + std::array code{}; + HW::ComputeShaderInfo regs{}; + regs.cs_regs.data_addr = reinterpret_cast(code.data()); + + ShaderRecompiler::IR::Program identity_program; + identity_program.binding_layout_complete = true; + ShaderRecompiler::IR::ImageResource image; + image.kind = ShaderRecompiler::IR::ResourceKind::StorageImage; + image.dimension = ShaderRecompiler::Decoder::ImageDimension::Dim2D; + identity_program.info.images.push_back(image); + auto rgb1_program = identity_program; + rgb1_program.info.images[0].storage_swizzle = DstSel(4, 5, 6, 1); + + const auto resources = + std::make_shared(); + ShaderComputeInputInfo identity_info; + identity_info.stage.program = + std::make_shared(identity_program); + identity_info.stage.resources = resources; + auto rgb1_info = identity_info; + rgb1_info.stage.program = + std::make_shared(rgb1_program); + + const auto identity_id = ShaderGetIdCS(®s, &identity_info, true); + const auto rgb1_id = ShaderGetIdCS(®s, &rgb1_info, true); + Require("StorageImageSwizzleSpecializationId", "pipeline cache key", + identity_id != rgb1_id && identity_id.ids.size() == rgb1_id.ids.size(), + "storage swizzle-specialized SPIR-V variants share a pipeline ID"); + std::printf("[host] %-32s ok\n", "StorageImageSwizzlePipelineId"); +} + +void CheckStorageTextureVolumeUploadLayout() { + constexpr uint32_t format = Prospero::GpuEnumValue(Prospero::BufferFormat::k16_16_16_16Float); + constexpr uint32_t width = 33; + constexpr uint32_t height = 33; + constexpr uint32_t depth = 33; + constexpr uint32_t bytes_per_element = 8; + const auto pitch = TileGetTexturePitch( + format, width, 1, Prospero::GpuEnumValue(Prospero::TileMode::kRenderTarget)); + TileSizeAlign total{}; + TileGetTextureTotalSize(format, width, height, depth, pitch, 1, + Prospero::GpuEnumValue(Prospero::TileMode::kRenderTarget), true, &total); + const auto layout = TextureCalcUploadLayout( + format, width, height, 1, depth, pitch, + Prospero::GpuEnumValue(Prospero::TileMode::kRenderTarget), total.size, true, false, true, + "StorageTextureVolumeTest"); + const auto regions = TextureBuildUploadRegions( + layout, VK_FORMAT_R16G16B16A16_SFLOAT, width, height, depth, 1, false, true, + TextureUploadDestination::MipLevels, + TextureUploadSliceLayout::MipChainPerSlice); + Require("StorageTextureVolumeUpload", "layout", + pitch == 128 && total.size == 0x210000 && + layout.slice_stride == 0x8400 && + layout.source_slice_stride == 0x10000 && + layout.level_sizes[0].size == 0x8400 && + layout.level_sizes[0].src_size == 0x10000 && + TextureCalcUploadSize(layout, regions, 1, depth, + TextureUploadSliceLayout::MipChainPerSlice) == + total.size, + "3D render-target upload did not preserve distinct linear and guest strides"); + + std::vector source(total.size); + for (uint32_t z = 0; z < depth; z++) { + std::fill_n(source.data() + z * layout.source_slice_stride, + layout.source_slice_stride, static_cast(z + 1)); + } + std::vector linear(layout.slice_stride * depth, 0); + for (const uint32_t z : {0u, 1u, depth - 1u}) { + const auto src_offset = TextureUploadSliceSourceOffset( + layout, 0, z, TextureUploadSliceLayout::MipChainPerSlice); + Require("StorageTextureVolumeUpload", "source offset", + src_offset == static_cast(z) * 0x10000, + "volume slice selected a compact linear source offset"); + TileConvertTiledToLinearRenderTarget( + linear.data() + regions[z].offset, source.data() + src_offset, width, height, + regions[z].pitch, bytes_per_element, layout.level_sizes[0].size, + layout.level_sizes[0].src_size, layout.level_sizes[0].x, + layout.level_sizes[0].y); + const auto value = static_cast(z + 1); + for (uint32_t y = 0; y < height; y++) { + const auto row = regions[z].offset + + static_cast(y) * regions[z].pitch * bytes_per_element; + Require("StorageTextureVolumeUpload", "contents", + std::all_of(linear.begin() + row, + linear.begin() + row + width * bytes_per_element, + [value](uint8_t byte) { return byte == value; }), + "volume slice detiled from another slice's padded allocation"); + } + } + std::printf("[host] %-32s ok\n", "StorageTextureVolumeUpload"); +} + +void CheckRenderTargetTileRoundTrip() { + struct Case { + uint32_t width; + uint32_t height; + uint32_t bytes_per_element; + }; + constexpr std::array cases{ + Case{129, 65, 1}, Case{257, 131, 2}, Case{3840, 2160, 4}, Case{65, 67, 8}}; + for (const auto test : cases) { + const auto pitch = + TileGetRenderTargetPitch(test.width, test.bytes_per_element); + TileSizeAlign storage{}; + Require("RenderTargetTileRoundTrip", "layout", + pitch >= test.width && + TileGetRenderTargetSize(test.width, test.height, pitch, + test.bytes_per_element, &storage) && + storage.size != 0, + "render-target test layout was rejected"); + std::vector linear(storage.size, 0); + for (uint32_t y = 0; y < test.height; y++) { + for (uint32_t x = 0; x < test.width * test.bytes_per_element; x++) { + const auto offset = static_cast(y) * pitch * + test.bytes_per_element + + x; + linear[offset] = static_cast( + (x * 37u + y * 101u + test.bytes_per_element * 13u) & 0xffu); + } + } + std::vector tiled(storage.size, 0xcd); + std::vector restored(storage.size, 0xa5); + TileConvertLinearToTiledRenderTarget( + tiled.data(), linear.data(), test.width, test.height, pitch, + test.bytes_per_element, storage.size); + TileConvertTiledToLinearRenderTarget( + restored.data(), tiled.data(), test.width, test.height, pitch, + test.bytes_per_element, storage.size); + for (uint32_t y = 0; y < test.height; y++) { + const auto row = static_cast(y) * pitch * test.bytes_per_element; + Require("RenderTargetTileRoundTrip", "contents", + std::memcmp(linear.data() + row, restored.data() + row, + test.width * test.bytes_per_element) == 0, + "linear-to-tiled render-target conversion did not round-trip"); + } + } + std::printf("[host] %-32s ok\n", "RenderTargetTileRoundTrip"); +} + +void CheckDepthTargetTileRoundTrip() { + struct Case { + uint32_t width; + uint32_t height; + uint32_t guest_format; + uint32_t depth_format; + uint32_t bytes_per_element; + }; + constexpr std::array cases{ + Case{8, 8, Prospero::GpuEnumValue(Prospero::BufferFormat::k16UNorm), + Prospero::GpuEnumValue(Prospero::DepthFormat::kZ16), 2}, + Case{129, 130, Prospero::GpuEnumValue(Prospero::BufferFormat::k16UNorm), + Prospero::GpuEnumValue(Prospero::DepthFormat::kZ16), 2}, + Case{8, 8, Prospero::GpuEnumValue(Prospero::BufferFormat::k32Float), + Prospero::GpuEnumValue(Prospero::DepthFormat::kZ32F), 4}, + Case{640, 360, Prospero::GpuEnumValue(Prospero::BufferFormat::k32Float), + Prospero::GpuEnumValue(Prospero::DepthFormat::kZ32F), 4}}; + for (const auto test : cases) { + const auto pitch = TileGetTexturePitch( + test.guest_format, test.width, 1, Prospero::GpuEnumValue(Prospero::TileMode::kDepth)); + TileSizeAlign stencil{}; + TileSizeAlign htile{}; + TileSizeAlign depth{}; + Require("DepthTargetTileRoundTrip", "layout", + pitch >= test.width && + TileGetDepthSize(test.width, test.height, 0, test.depth_format, + Prospero::GpuEnumValue(Prospero::StencilFormat::kInvalid), false, + &stencil, &htile, &depth) && + depth.size != 0 && depth.align == 0x10000, + "Prospero depth layout was rejected"); + std::vector linear(depth.size, 0); + for (uint32_t y = 0; y < test.height; y++) { + for (uint32_t x = 0; x < test.width * test.bytes_per_element; x++) { + const auto offset = static_cast(y) * pitch * + test.bytes_per_element + + x; + linear[offset] = static_cast( + (x * 29u + y * 83u + test.bytes_per_element * 17u) & 0xffu); + } + } + std::vector tiled(depth.size, 0xcd); + std::vector restored(depth.size, 0xa5); + TileConvertLinearToTiledDepth(tiled.data(), linear.data(), test.guest_format, + test.width, test.height, pitch, depth.size); + TileConvertTiledToLinearDepth(restored.data(), tiled.data(), test.guest_format, + test.width, test.height, pitch, depth.size); + for (uint32_t y = 0; y < test.height; y++) { + const auto row = static_cast(y) * pitch * test.bytes_per_element; + Require("DepthTargetTileRoundTrip", "contents", + std::memcmp(linear.data() + row, restored.data() + row, + test.width * test.bytes_per_element) == 0, + "linear-to-tiled PS5 depth conversion did not round-trip"); + } + if (test.width == 8 && test.height == 8 && test.bytes_per_element == 2) { + const auto *linear16 = reinterpret_cast(linear.data()); + const auto *tiled16 = reinterpret_cast(tiled.data()); + Require("DepthTargetTileRoundTrip", "Prospero D16 SW_64KB_Z_X anchors", + tiled16[0] == linear16[0] && tiled16[1] == linear16[1] && + tiled16[4] == linear16[2] && tiled16[2] == linear16[pitch] && + tiled16[0x27] == linear16[5 * pitch + 3], + "Prospero D16 depth address anchors changed"); + } + if (test.width == 8 && test.height == 8 && test.bytes_per_element == 4) { + const auto *linear32 = reinterpret_cast(linear.data()); + const auto *tiled32 = reinterpret_cast(tiled.data()); + Require("DepthTargetTileRoundTrip", "Prospero D32 SW_64KB_Z_X anchors", + tiled32[0] == linear32[0] && tiled32[1] == linear32[1] && + tiled32[4] == linear32[2] && tiled32[2] == linear32[pitch] && + tiled32[0x27] == linear32[5 * pitch + 3], + "Prospero D32 depth address anchors changed"); + } + if (test.width == 129 && test.height == 130 && + test.bytes_per_element == 2) { + const auto *linear16 = reinterpret_cast(linear.data()); + const auto *tiled16 = reinterpret_cast(tiled.data()); + Require("DepthTargetTileRoundTrip", "Prospero D16 block row", + tiled16[0x10000 / sizeof(uint16_t)] == linear16[128 * pitch], + "Prospero D16 depth block-row order changed"); + } + if (test.width == 640 && test.height == 360 && + test.bytes_per_element == 4) { + const auto *linear32 = reinterpret_cast(linear.data()); + const auto *tiled32 = reinterpret_cast(tiled.data()); + Require("DepthTargetTileRoundTrip", "Prospero D32 block order", + tiled32[0x10000 / sizeof(uint32_t)] == linear32[128] && + tiled32[0x50000 / sizeof(uint32_t)] == linear32[128 * pitch], + "Prospero D32 depth block order changed"); + } + } + std::printf("[host] %-32s ok\n", "DepthTargetTileRoundTrip"); +} + +void CheckStorageTextureGpuOwnedRebindState() { + constexpr uintptr_t base = 0x0000000200200000ull; + constexpr uint64_t size = 0x10000; + auto *memory = static_cast(VirtualAlloc( + reinterpret_cast(base), size, MEM_RESERVE | MEM_COMMIT, PAGE_READWRITE)); + Require("StorageTextureGpuOwnedRebind", "allocation", + memory == reinterpret_cast(base), "fixed VirtualAlloc failed"); + PageManager page_manager(CacheFault, nullptr); + MemoryTracker tracker(page_manager); + page_manager.OnGpuMap(base, size); + tracker.ForEachUploadRange(base, size, true, + [](uint64_t, uint64_t) noexcept {}, []() noexcept {}); + uint64_t readable = 0; + MEMORY_BASIC_INFORMATION protection{}; + Require("StorageTextureGpuOwnedRebind", "owned", + tracker.IsRegionGpuModified(base, size) && page_manager.IsMapped(base, size) && + (!HostMemoryQueryReadable(base, size, &readable) || readable < size) && + VirtualQuery(memory, &protection, sizeof(protection)) != 0 && + protection.Protect == PAGE_NOACCESS, + "GPU-owned storage pages remained host-readable or lost tracker identity"); + + tracker.UnmarkRegionAsGpuModified(base, size); + readable = 0; + Require("StorageTextureGpuOwnedRebind", "clean readback", + !tracker.IsRegionGpuModified(base, size) && + !tracker.IsRegionCpuModified(base, size) && + HostMemoryQueryReadable(base, size, &readable) && readable == size, + "clean storage readback did not publish readable coherent backing"); + tracker.MarkRegionAsGpuModified(base, size); + readable = 0; + Require("StorageTextureGpuOwnedRebind", "clean reclaim", + tracker.IsRegionGpuModified(base, size) && + !tracker.IsRegionCpuModified(base, size) && + (!HostMemoryQueryReadable(base, size, &readable) || readable < size), + "clean storage rebind did not reclaim GPU ownership without an upload"); + + tracker.UnmarkRegionAsGpuModified(base, size); + tracker.MarkRegionAsCpuModified(base, size); + uint32_t dirty_ranges = 0; + bool upload_called = false; + tracker.ForEachUploadRange( + base, size, true, + [&](uint64_t, uint64_t) noexcept { dirty_ranges++; }, + [&]() noexcept { upload_called = true; }); + readable = 0; + Require("StorageTextureGpuOwnedRebind", "dirty refresh", + dirty_ranges == 1 && upload_called && tracker.IsRegionGpuModified(base, size) && + !tracker.IsRegionCpuModified(base, size) && + (!HostMemoryQueryReadable(base, size, &readable) || readable < size), + "CPU-dirty storage rebind did not refresh once and reclaim GPU ownership"); + + tracker.UnmarkRegionAsGpuModified(base, size); + tracker.UntrackMemory(base, size); + page_manager.OnGpuUnmap(base, size); + Require("StorageTextureGpuOwnedRebind", "free", VirtualFree(memory, 0, MEM_RELEASE) != 0, + "VirtualFree failed"); + std::printf("[host] %-32s ok\n", "StorageTextureGpuOwnedRebind"); +} + +void CheckStorageTextureSampledReuse() { + ImageInfo storage{}; + storage.address = 0x78650000; + storage.size = 0x210000; + storage.format = Prospero::GpuEnumValue(Prospero::BufferFormat::k16_16_16_16Float); + storage.width = 33; + storage.height = 33; + storage.pitch = 128; + storage.levels = 1; + storage.tile = Prospero::GpuEnumValue(Prospero::TileMode::kRenderTarget); + storage.swizzle = DstSel(4, 5, 6, 7); + storage.depth = 33; + storage.type = Prospero::GpuEnumValue(Prospero::ImageType::kColor3D); + + Require("StorageTextureSampledReuse", "exact", + ClassifyStorageSampledOverlap(storage, storage, + VK_FORMAT_R16G16B16A16_SFLOAT, + VK_FORMAT_R16G16B16A16_SFLOAT, + true, false, true) == + StorageSampledOverlap::ExactImage, + "exact GPU-owned storage image was not reusable for sampling"); + auto incompatible = storage; + incompatible.format = Prospero::GpuEnumValue(Prospero::BufferFormat::k16_16_16_16UNorm); + Require("StorageTextureSampledReuse", "incompatible", + ClassifyStorageSampledOverlap(incompatible, storage, + VK_FORMAT_R16G16B16A16_UNORM, + VK_FORMAT_R16G16B16A16_SFLOAT, + true, false, true) == + StorageSampledOverlap::Unsupported && + ClassifyStorageSampledOverlap(storage, storage, + VK_FORMAT_R16G16B16A16_SFLOAT, + VK_FORMAT_R16G16B16A16_SFLOAT, + false, false, true) == + StorageSampledOverlap::Unsupported && + ClassifyStorageSampledOverlap(storage, storage, + VK_FORMAT_R16G16B16A16_SFLOAT, + VK_FORMAT_R16G16B16A16_SFLOAT, + true, true, true) == + StorageSampledOverlap::Unsupported && + ClassifyStorageSampledOverlap(storage, storage, + VK_FORMAT_R16G16B16A16_SFLOAT, + VK_FORMAT_R16G16B16A16_SFLOAT, + true, false, false) == + StorageSampledOverlap::Unsupported, + "incompatible storage-image state was accepted for sampling"); + auto separate = storage; + separate.address += 0x400000; + Require("StorageTextureSampledReuse", "separate", + ClassifyStorageSampledOverlap(separate, storage, + VK_FORMAT_R16G16B16A16_SFLOAT, + VK_FORMAT_R16G16B16A16_SFLOAT, + true, false, true) == + StorageSampledOverlap::None, + "disjoint storage image was classified as an alias"); + + ImageInfo ppsa02604_storage{}; + ppsa02604_storage.address = 0x7c690000; + ppsa02604_storage.size = 0x870000; + ppsa02604_storage.format = Prospero::GpuEnumValue(Prospero::BufferFormat::k8_8_8_8UNorm); + ppsa02604_storage.width = 1920; + ppsa02604_storage.height = 1080; + ppsa02604_storage.pitch = 1920; + ppsa02604_storage.tile = Prospero::GpuEnumValue(Prospero::TileMode::kRenderTarget); + ppsa02604_storage.swizzle = DstSel(6, 5, 4, 7); + ppsa02604_storage.type = Prospero::GpuEnumValue(Prospero::ImageType::kColor2D); + auto ppsa02604_sampled = ppsa02604_storage; + ppsa02604_sampled.format = Prospero::GpuEnumValue(Prospero::BufferFormat::k8_8_8_8Srgb); + Require("StorageTextureSampledReuse", "PPSA02604 sRGB view", + ClassifyStorageSampledOverlap( + ppsa02604_sampled, ppsa02604_storage, VK_FORMAT_R8G8B8A8_SRGB, + VK_FORMAT_R8G8B8A8_UNORM, true, false, true) == + StorageSampledOverlap::ExactImage, + "PPSA02604 GPU-owned UNORM storage image did not accept its exact sRGB view"); + auto alternate_swizzle = ppsa02604_storage; + alternate_swizzle.swizzle = DstSel(4, 5, 6, 7); + Require("StorageTextureSampledReuse", "descriptor swizzle view", + ClassifyStorageSampledOverlap( + alternate_swizzle, ppsa02604_storage, VK_FORMAT_R8G8B8A8_UNORM, + VK_FORMAT_R8G8B8A8_UNORM, true, false, true) == + StorageSampledOverlap::ExactImage, + "exact storage backing rejected a distinct sampled descriptor swizzle"); + const auto usage = TextureFormatUsage::Sampled | TextureFormatUsage::Storage; + Require("StorageTextureSampledReuse", "usage", + (TextureGetUsage(usage) & + (VK_IMAGE_USAGE_SAMPLED_BIT | VK_IMAGE_USAGE_STORAGE_BIT)) == + (VK_IMAGE_USAGE_SAMPLED_BIT | VK_IMAGE_USAGE_STORAGE_BIT) && + (TextureGetViewUsage(usage) & + (VK_IMAGE_USAGE_SAMPLED_BIT | VK_IMAGE_USAGE_STORAGE_BIT)) == + (VK_IMAGE_USAGE_SAMPLED_BIT | VK_IMAGE_USAGE_STORAGE_BIT), + "shared storage image or view is missing sampled/storage usage"); + std::printf("[host] %-32s ok\n", "StorageTextureSampledReuse"); +} + +[[noreturn]] void RunStorageTextureAccessDeathCase(const char *kind) { + constexpr uintptr_t base = 0x0000000200300000ull; + constexpr uint64_t size = 0x10000; + auto *memory = VirtualAlloc(reinterpret_cast(base), size, + MEM_RESERVE | MEM_COMMIT, PAGE_READWRITE); + if (memory != reinterpret_cast(base)) { + EXIT("storage access child could not reserve its fixed test address\n"); + } + ResourceMutex resource_mutex; + PageManager page_manager(CacheFault, nullptr); + BufferCache buffer_cache(page_manager, resource_mutex); + if (std::strcmp(kind, "read") == 0) { + page_manager.OnGpuMap(base, size, GpuAccess::Write); + buffer_cache.ValidateGpuAccess(base, size, true, true); + } else if (std::strcmp(kind, "write") == 0) { + page_manager.OnGpuMap(base, size, GpuAccess::Read); + buffer_cache.ValidateGpuAccess(base, size, true, true); + } else { + std::_Exit(0x7e); + } + std::_Exit(0x7f); +} + +void CheckStorageTextureAccessPermissions() { + constexpr uintptr_t base = 0x0000000200300000ull; + constexpr uint64_t size = 0x10000; + auto *memory = VirtualAlloc(reinterpret_cast(base), size, + MEM_RESERVE | MEM_COMMIT, PAGE_READWRITE); + Require("StorageTextureAccess", "allocation", memory == reinterpret_cast(base), + "fixed VirtualAlloc failed"); + ResourceMutex resource_mutex; + PageManager page_manager(CacheFault, nullptr); + BufferCache buffer_cache(page_manager, resource_mutex); + page_manager.OnGpuMap(base, size, GpuAccess::ReadWrite); + buffer_cache.ValidateGpuAccess(base, size, true, true); + page_manager.OnGpuUnmap(base, size, GpuAccess::ReadWrite); + Require("StorageTextureAccess", "free", VirtualFree(memory, 0, MEM_RELEASE) != 0, + "VirtualFree failed"); + + char path[MAX_PATH]{}; + Require("StorageTextureAccess", "host", + GetModuleFileNameA(nullptr, path, MAX_PATH) != 0, + "GetModuleFileName failed"); + for (const char *kind : {"read", "write"}) { + std::string command = std::string("\"") + path + + "\" --storage-texture-access-death " + kind; + std::vector mutable_command(command.begin(), command.end()); + mutable_command.push_back('\0'); + STARTUPINFOA startup{sizeof(startup)}; + PROCESS_INFORMATION process{}; + Require("StorageTextureAccess", "host", + CreateProcessA(nullptr, mutable_command.data(), nullptr, nullptr, FALSE, + CREATE_NO_WINDOW, nullptr, nullptr, &startup, &process) != 0, + "CreateProcess failed"); + Require("StorageTextureAccess", "host", + WaitForSingleObject(process.hProcess, 10000) == WAIT_OBJECT_0, + "access death case timed out"); + DWORD exit_code = 0; + const bool exited = GetExitCodeProcess(process.hProcess, &exit_code) != 0; + CloseHandle(process.hThread); + CloseHandle(process.hProcess); + Require("StorageTextureAccess", "host", exited && exit_code == 321, + std::string(kind) + " access denial did not report a fatal error"); + } + std::printf("[host] %-32s ok\n", "StorageTextureAccessPermissions"); +} + +[[noreturn]] void RunMetaOverlapDeathCase(const char *kind) { + constexpr uint64_t address = 0x0000000200010000ull; + constexpr uint64_t allocation_size = 0x10000; + auto *memory = static_cast(VirtualAlloc( + reinterpret_cast(address), allocation_size, + MEM_RESERVE | MEM_COMMIT, PAGE_READWRITE)); + if (memory != reinterpret_cast(address)) { + EXIT("metadata-overlap child could not reserve its fixed test address\n"); + } + ResourceMutex resource_mutex; + PageManager page_manager(CacheFault, nullptr); + BufferCache buffer_cache(page_manager, resource_mutex); + TextureCache texture_cache(page_manager, buffer_cache, resource_mutex); + buffer_cache.SetTextureCache(texture_cache); + page_manager.OnGpuMap(address, allocation_size); + texture_cache.RegisterMeta(address, 0x8000); + auto *ctx = reinterpret_cast(1); + + if (std::strcmp(kind, "metadata-size") == 0) { + texture_cache.RegisterMeta(address, 0x10000); + } else if (std::strcmp(kind, "texture") == 0) { + ImageInfo info{}; + info.address = address; + info.size = 0x1000; + info.width = 1; + info.height = 1; + (void)texture_cache.FindTexture(reinterpret_cast(1), ctx, info, false); + } else if (std::strcmp(kind, "render-target") == 0) { + RenderTargetInfo info{}; + info.address = address; + info.size = 0x1000; + info.format = VK_FORMAT_R8G8B8A8_UNORM; + info.width = 1; + info.height = 1; + info.pitch = 1; + info.bytes_per_element = 4; + info.tile_mode = Prospero::GpuEnumValue(Prospero::TileMode::kLinear); + (void)texture_cache.FindRenderTarget(reinterpret_cast(1), ctx, info); + } else if (std::strcmp(kind, "depth-target") == 0) { + DepthTargetInfo info{}; + info.address = address; + info.size = 0x1000; + info.format = VK_FORMAT_D16_UNORM; + info.guest_format = Prospero::GpuEnumValue(Prospero::BufferFormat::k16UNorm); + info.width = 1; + info.height = 1; + info.pitch = 1; + info.bytes_per_element = 2; + info.tile_mode = Prospero::GpuEnumValue(Prospero::TileMode::kDepth); + (void)texture_cache.FindDepthTarget(reinterpret_cast(1), ctx, + info); + } else if (std::strcmp(kind, "copy-stale") == 0) { + if (!texture_cache.ClearMeta(address)) { + std::_Exit(0x7f); + } + buffer_cache.CopyBuffer(nullptr, ctx, address + 0x9000, address, 0x1000); + } else { + EXIT("metadata-reuse child received an unknown mode\n"); + } + std::_Exit(0x7f); +} + +ShaderRecompiler::IR::ImageResource BasicMetadataReuseResource() { + ShaderRecompiler::IR::ImageResource resource{}; + resource.kind = ShaderRecompiler::IR::ResourceKind::Image; + resource.dimension = ShaderRecompiler::Decoder::ImageDimension::Dim2D; + resource.read = true; + return resource; +} + +ShaderTextureResource BasicMetadataReuseDescriptor() { + return {{0x0062b100u, 0xcad00000u, 0x003fc03fu, 0x90500facu, + 0x00000000u, 0x00700000u, 0x00000000u, 0x00000000u}}; +} + +[[noreturn]] void RunMetadataDescriptorDeathCase(const char *kind) { + auto resource = BasicMetadataReuseResource(); + auto descriptor = BasicMetadataReuseDescriptor(); + if (std::strcmp(kind, "field1-reserved") == 0) { + descriptor.fields[1] |= 0x20000000u; + } else if (std::strcmp(kind, "field2-low-reserved") == 0) { + descriptor.fields[2] |= 0x00001000u; + } else if (std::strcmp(kind, "field2-high-reserved") == 0) { + descriptor.fields[2] |= 0x10000000u; + } else if (std::strcmp(kind, "unsupported-format") == 0) { + descriptor.fields[1] = + (descriptor.fields[1] & ~0x1ff00000u) | + (Prospero::GpuEnumValue(Prospero::BufferFormat::k8_8_8_8SInt) << 20u); + } else if (std::strcmp(kind, "uint-format") == 0) { + descriptor.fields[1] = + (descriptor.fields[1] & ~0x1ff00000u) | + (Prospero::GpuEnumValue(Prospero::BufferFormat::k8_8_8_8UInt) << 20u); + } else { + std::_Exit(0x7e); + } + ValidateMetadataReuseTexture(resource, descriptor, 0x10000); + std::_Exit(0x7f); +} + +void CheckMetaOverlapDeaths() { + char path[MAX_PATH]{}; + Require("MetaOverlap", "host", GetModuleFileNameA(nullptr, path, MAX_PATH) != 0, + "GetModuleFileName failed"); + for (const char *kind : {"metadata-size", "texture", "render-target", "depth-target", + "copy-stale"}) { + std::string command = std::string("\"") + path + + "\" --meta-overlap-death " + kind; + std::vector mutable_command(command.begin(), command.end()); + mutable_command.push_back('\0'); + STARTUPINFOA startup{sizeof(startup)}; + PROCESS_INFORMATION process{}; + Require("MetaOverlap", "host", + CreateProcessA(nullptr, mutable_command.data(), nullptr, nullptr, FALSE, + CREATE_NO_WINDOW, nullptr, nullptr, &startup, &process) != 0, + "CreateProcess failed"); + Require("MetaOverlap", "host", + WaitForSingleObject(process.hProcess, 10000) == WAIT_OBJECT_0, + "death case timed out"); + DWORD exit_code = 0; + const bool exited = GetExitCodeProcess(process.hProcess, &exit_code) != 0; + CloseHandle(process.hThread); + CloseHandle(process.hProcess); + Require("MetaOverlap", "host", exited && exit_code == 321, + std::string(kind) + " did not reject registered metadata overlap"); + } + std::printf("[host] %-32s ok\n", "MetaOverlapRegistration"); +} + +void CheckOverlappingMetadataViews() { + constexpr uintptr_t base = 0x0000000200010000ull; + constexpr uint64_t allocation_size = 0x60000; + constexpr uint64_t metadata_size = 0x30000; + constexpr uint64_t second = base + 0x18000; + auto *memory = static_cast(VirtualAlloc( + reinterpret_cast(base), allocation_size, + MEM_RESERVE | MEM_COMMIT, PAGE_READWRITE)); + Require("OverlappingMetadataViews", "allocation", + memory == reinterpret_cast(base), "fixed VirtualAlloc failed"); + + ResourceMutex resource_mutex; + CacheFaultContext fault_context; + PageManager page_manager(CacheFault, &fault_context); + BufferCache buffer_cache(page_manager, resource_mutex); + TextureCache texture_cache(page_manager, buffer_cache, resource_mutex); + fault_context.texture = &texture_cache; + buffer_cache.SetTextureCache(texture_cache); + page_manager.OnGpuMap(base, allocation_size); + + texture_cache.RegisterMeta(base, metadata_size); + texture_cache.RegisterMeta(second, metadata_size); + Require("OverlappingMetadataViews", "registration", + texture_cache.IsMeta(base) && texture_cache.IsMeta(second) && + texture_cache.IsMetaRange(base, metadata_size) && + texture_cache.IsMetaRange(second, metadata_size) && + !texture_cache.IsMetaRange(base, metadata_size - 0x8000) && + !texture_cache.IsMetaRange(base + 0x8000, metadata_size), + "overlapping metadata identities were not retained"); + Require("OverlappingMetadataViews", "clear", + texture_cache.ClearMeta(base) && texture_cache.ClearMeta(second) && + texture_cache.IsMetaCleared(base, 0) && + texture_cache.IsMetaCleared(second, 0), + "overlapping metadata clears were not retained independently"); + Require("OverlappingMetadataViews", "fault", + page_manager.HandleFault(PageFaultAccess::Write, second), + "shared metadata page did not transfer to CPU ownership"); + Require("OverlappingMetadataViews", "ownership", + !texture_cache.IsMetaGpuModified(second, 0x1000), + "shared metadata page retained GPU ownership after a CPU fault"); + + texture_cache.UnmapMemory(base, allocation_size); + page_manager.OnGpuUnmap(base, allocation_size); + Require("OverlappingMetadataViews", "free", + VirtualFree(memory, 0, MEM_RELEASE) != 0, "VirtualFree failed"); + std::printf("[host] %-32s ok\n", "OverlappingMetadataViews"); +} + +void CheckMetadataReuseDescriptors() { + ValidateMetadataReuseTexture(BasicMetadataReuseResource(), + BasicMetadataReuseDescriptor(), 0x10000); + char path[MAX_PATH]{}; + Require("MetadataReuseDescriptor", "host", + GetModuleFileNameA(nullptr, path, MAX_PATH) != 0, + "GetModuleFileName failed"); + for (const char *kind : {"field1-reserved", "field2-low-reserved", + "field2-high-reserved", "unsupported-format", "uint-format"}) { + std::string command = std::string("\"") + path + + "\" --metadata-descriptor-death " + kind; + std::vector mutable_command(command.begin(), command.end()); + mutable_command.push_back('\0'); + STARTUPINFOA startup{sizeof(startup)}; + PROCESS_INFORMATION process{}; + Require("MetadataReuseDescriptor", "host", + CreateProcessA(nullptr, mutable_command.data(), nullptr, nullptr, FALSE, + CREATE_NO_WINDOW, nullptr, nullptr, &startup, &process) != 0, + "CreateProcess failed"); + Require("MetadataReuseDescriptor", "host", + WaitForSingleObject(process.hProcess, 10000) == WAIT_OBJECT_0, + "death case timed out"); + DWORD exit_code = 0; + const bool exited = GetExitCodeProcess(process.hProcess, &exit_code) != 0; + CloseHandle(process.hThread); + CloseHandle(process.hProcess); + Require("MetadataReuseDescriptor", "host", exited && exit_code == 321, + std::string(kind) + " descriptor was admitted"); + } + std::printf("[host] %-32s ok\n", "MetadataReuseDescriptors"); +} + +void CheckImageOverlapResolution() { + VideoOutPixelFormatInfo video_format {}; + Require("ImageOverlapResolution", "existing video-out formats", + DecodeVideoOutPixelFormat(0x8000000022000000ull, &video_format) && + video_format.format == VK_FORMAT_R8G8B8A8_SRGB && + video_format.guest_format == Prospero::GpuEnumValue(Prospero::BufferFormat::k8_8_8_8Srgb) && + DecodeVideoOutPixelFormat(0x8000000000000000ull, &video_format) && + video_format.format == VK_FORMAT_B8G8R8A8_SRGB && + video_format.guest_format == Prospero::GpuEnumValue(Prospero::BufferFormat::k8_8_8_8Srgb) && + DecodeVideoOutPixelFormat(0x8100000022000000ull, &video_format) && + video_format.format == VK_FORMAT_A2B10G10R10_UNORM_PACK32 && + video_format.guest_format == Prospero::GpuEnumValue(Prospero::BufferFormat::k10_10_10_2UNorm), + "pre-existing PS5 video-out format mappings changed during decoder centralization"); + Require("ImageOverlapResolution", "B10G10R10A2 video-out format", + DecodeVideoOutPixelFormat(0x8100000000000000ull, &video_format) && + video_format.format == VK_FORMAT_A2R10G10B10_UNORM_PACK32 && + video_format.guest_format == + Prospero::GpuEnumValue(Prospero::BufferFormat::k10_10_10_2UNorm) && + IsSupportedVideoOutFormat(video_format.guest_format, video_format.format), + "PS5 B10G10R10A2 sRGB video-out format did not preserve alternate channel order"); + Require("ImageOverlapResolution", "video-out format guards", + !DecodeVideoOutPixelFormat(0x8100070400000000ull, &video_format) && + !DecodeVideoOutPixelFormat(0x8100000000000000ull, nullptr) && + !IsSupportedVideoOutFormat(Prospero::GpuEnumValue(Prospero::BufferFormat::k8_8_8_8Srgb), + VK_FORMAT_A2R10G10B10_UNORM_PACK32) && + !IsSupportedVideoOutFormat(Prospero::GpuEnumValue(Prospero::BufferFormat::k10_10_10_2UNorm), + VK_FORMAT_R8G8B8A8_SRGB), + "unsupported PS5 video-out formats or mismatched guest/host pairs were admitted"); + TileSizeAlign render_target_layout{}; + Require("ImageOverlapResolution", "R8 render-target pitch", + TileGetRenderTargetPitch(1920, 1) == 2048, + "R8 target did not use native 256-pixel block width"); + Require("ImageOverlapResolution", "R8 render-target size", + TileGetRenderTargetSize(1920, 1080, 2048, 1, + &render_target_layout) && + render_target_layout.size == 0x280000 && + render_target_layout.align == 0x10000, + "R8 target did not use native 64-KiB block geometry"); + Require("ImageOverlapResolution", "RGBA8 render-target size", + TileGetRenderTargetPitch(1920, 4) == 1920 && + TileGetRenderTargetSize(1920, 1080, 1920, 4, + &render_target_layout) && + render_target_layout.size == 0x870000, + "RGBA8 target layout regressed"); + Require("ImageOverlapResolution", "PPSA19268 render-target edge", + TileGetRenderTargetPitch(120, 8) == 128 && + TileGetRenderTargetSize(120, 67, 128, 8, + &render_target_layout) && + render_target_layout.size == 0x20000 && + ImageRangeOverlaps(0x11029fff8ull, 1, 0x110280000ull, + render_target_layout.size), + "title write did not land in the final native render-target block"); + Require("ImageOverlapResolution", "render-target layout rejection", + TileGetRenderTargetPitch(1920, 3) == 0 && + !TileGetRenderTargetSize(1920, 1080, 1920, 1, + &render_target_layout), + "unsupported element size or mismatched pitch was admitted"); + + TileSizeOffset mip_levels[16]{}; + TilePaddedSize mip_padded[16]{}; + Require("ImageOverlapResolution", "PPSA09076 mip-chain layout", + TileGetRenderTargetMipLayout(960, 540, 1024, 8, 10, + &render_target_layout, mip_levels, + mip_padded) && + render_target_layout.size == 0x650000 && + render_target_layout.align == 0x10000 && + mip_levels[0].offset == 0x1d0000 && + mip_levels[0].size == 0x480000 && + mip_levels[1].offset == 0x90000 && + mip_levels[1].size == 0x140000 && + mip_padded[1].width == 512 && mip_padded[1].height == 320, + "native render-target mip allocation or subresource geometry regressed"); + Require("ImageOverlapResolution", "render-target mip-chain rejection", + !TileGetRenderTargetMipLayout(960, 540, 960, 8, 10, + &render_target_layout, nullptr, nullptr) && + !TileGetRenderTargetMipLayout(960, 540, 1024, 3, 10, + &render_target_layout, nullptr, + nullptr) && + !TileGetRenderTargetMipLayout(960, 540, 1024, 8, 0, + &render_target_layout, nullptr, + nullptr) && + !TileGetRenderTargetMipLayout(960, 540, 1024, 8, 17, + &render_target_layout, nullptr, + nullptr) && + !TileGetRenderTargetMipLayout( + 4, 4, TileGetRenderTargetPitch(4, 4), 4, 4, + &render_target_layout, nullptr, + nullptr), + "invalid render-target mip topology was admitted"); + + ImageInfo sampled{}; + sampled.address = 0x200000; + sampled.size = 0x8000; + ImageInfo sampled_alias = sampled; + sampled_alias.address += 0x1000; + Require("ImageOverlapResolution", "sampled alias", + ClassifySampledOverlap(sampled_alias, sampled, false, true) == + SampledOverlap::ReadOnlyAlias, + "read-only sampled pool alias was rejected"); + Require("ImageOverlapResolution", "sampled GPU owner", + ClassifySampledOverlap(sampled_alias, sampled, true, true) == + SampledOverlap::Unsupported, + "GPU-modified sampled alias was admitted"); + Require("ImageOverlapResolution", "sampled context", + ClassifySampledOverlap(sampled_alias, sampled, false, false) == + SampledOverlap::Unsupported, + "cross-context sampled alias was admitted"); + ImageInfo page_left = sampled; + page_left.size = TRACKER_PAGE_SIZE / 2; + ImageInfo same_page = page_left; + same_page.address += page_left.size; + Require("ImageOverlapResolution", "sampled shared page", + !ImageRangeOverlaps(same_page.address, same_page.size, + page_left.address, page_left.size) && + ClassifySampledOverlap(same_page, page_left, false, true) == + SampledOverlap::ReadOnlyAlias, + "byte-disjoint sampled images sharing a tracker page were not aliased"); + ImageInfo separate_page = same_page; + separate_page.address = page_left.address + TRACKER_PAGE_SIZE; + Require("ImageOverlapResolution", "sampled separate page", + ClassifySampledOverlap(separate_page, page_left, false, true) == + SampledOverlap::None, + "sampled images on separate pages were aliased"); + constexpr uint64_t sampled_buffer_address = 0xe80e0000ull; + constexpr uint64_t sampled_buffer_size = 0x870000ull; + Require("ImageOverlapResolution", "sampled buffer write transition", + IsSampledBufferWriteTransition( + sampled_buffer_address, sampled_buffer_size, + sampled_buffer_address, sampled_buffer_size, false), + "exact page-aligned clean sampled image did not accept a buffer writer"); + Require("ImageOverlapResolution", "sampled buffer write guards", + !IsSampledBufferWriteTransition( + sampled_buffer_address, sampled_buffer_size - TRACKER_PAGE_SIZE, + sampled_buffer_address, sampled_buffer_size, false) && + !IsSampledBufferWriteTransition( + sampled_buffer_address + 1, sampled_buffer_size, + sampled_buffer_address + 1, sampled_buffer_size, false) && + !IsSampledBufferWriteTransition( + sampled_buffer_address, sampled_buffer_size, + sampled_buffer_address, sampled_buffer_size, true), + "partial, unaligned, or GPU-owned sampled image accepted a buffer writer"); + constexpr uint64_t edge_target_address = 0x108ad00100ull; + constexpr uint64_t edge_target_size = 0x1fa400; + constexpr uint64_t edge_fault_address = edge_target_address + edge_target_size + 0x1b8; + Require("ImageOverlapResolution", "GPU target shared edge page", + !ImageRangeOverlaps(edge_fault_address, 1, edge_target_address, + edge_target_size) && + ImagePageRangesOverlap(edge_fault_address, 1, + edge_target_address, edge_target_size), + "byte-disjoint CPU access lost the GPU owner of its shared tracker page"); + Image sampled_image{}; + sampled_image = page_left; + Image same_page_image{}; + same_page_image = same_page; + sampled_image.InvalidateCpuWrite(page_left.address + 1, 1); + same_page_image.InvalidateCpuWrite(page_left.address + 1, 1); + Require("ImageOverlapResolution", "sampled dirty fanout", + sampled_image.IsCpuDirty() && same_page_image.IsCpuDirty(), + "one guest write did not dirty every sampled image sharing its page"); + sampled_image.RefreshComplete(); + Require("ImageOverlapResolution", "sampled dirty independence", + !sampled_image.IsCpuDirty() && same_page_image.IsCpuDirty(), + "refreshing one sampled alias cleaned another alias"); + + constexpr uint64_t fill_address = 0x73771080; + constexpr uint64_t fill_size = sizeof(uint32_t); + constexpr uint64_t texture_address = 0x73771000; + constexpr uint64_t texture_size = 0x4000; + Require("ImageOverlapResolution", "host fill sampled image", + ClassifyHostWriteOverlap(fill_address, fill_size, texture_address, + texture_size, true, false, false) == + HostWriteOverlap::InvalidateImage, + "host-backed fill did not invalidate its CPU-current sampled image"); + Require("ImageOverlapResolution", "host fill GPU owner", + ClassifyHostWriteOverlap(fill_address, fill_size, texture_address, + texture_size, true, true, false) == + HostWriteOverlap::Unsupported, + "host-backed fill was admitted over a GPU-owned sampled image"); + Require("ImageOverlapResolution", "host fill clean render target", + ClassifyHostWriteOverlap(fill_address, fill_size, texture_address, + texture_size, true, false, false) == + HostWriteOverlap::InvalidateImage, + "host-backed fill did not invalidate its clean render target"); + Require("ImageOverlapResolution", "host fill non-refreshable image", + ClassifyHostWriteOverlap(fill_address, fill_size, texture_address, + texture_size, false, false, false) == + HostWriteOverlap::Unsupported, + "host-backed fill was admitted over a non-refreshable image"); + constexpr uint64_t dma_address = 0x10f562000ull; + constexpr uint64_t dma_size = 0x40000; + constexpr uint64_t target_address = 0x10f570000ull; + constexpr uint64_t target_size = 0x20000; + Require("ImageOverlapResolution", "host copy clean render target", + ClassifyHostWriteOverlap(dma_address, dma_size, target_address, + target_size, true, false, false) == + HostWriteOverlap::InvalidateImage, + "host-backed copy did not invalidate the covered clean render target"); + Require("ImageOverlapResolution", "host copy GPU render target", + ClassifyHostWriteOverlap(dma_address, dma_size, target_address, + target_size, true, true, false) == + HostWriteOverlap::Unsupported, + "host-backed copy was admitted over a GPU-owned render target"); + Require("ImageOverlapResolution", "host copy partial clean render target", + ClassifyHostWriteOverlap(target_address + 0x1000, 0x1000, + target_address, target_size, true, false, + false) == HostWriteOverlap::InvalidateImage, + "partial host copy did not invalidate its clean render target"); + Require("ImageOverlapResolution", "host copy shared target page", + ClassifyHostWriteOverlap(target_address + target_size - 1, 1, + target_address, target_size - 1, true, false, + false) == HostWriteOverlap::InvalidateImage, + "shared-page host copy did not conservatively invalidate the clean target"); + Require("ImageOverlapResolution", "host fill metadata", + ClassifyHostWriteOverlap(fill_address, fill_size, texture_address, + texture_size, true, false, true) == + HostWriteOverlap::Unsupported, + "host-backed sampled-image fill was admitted over metadata pages"); + Require("ImageOverlapResolution", "host fill disjoint page", + ClassifyHostWriteOverlap(texture_address + texture_size, fill_size, + texture_address, texture_size, true, false, + true) == + HostWriteOverlap::None, + "host-backed fill aliased a sampled image on another page"); + + constexpr uint64_t video_address = 0x1000c10000ull; + constexpr uint64_t video_size = 0x1fe0000ull; + Require("ImageOverlapResolution", "formatted video-out write", + ClassifyBufferImageAlias(video_address, video_size, video_address, + video_size, true, false, false, true, true) == + BufferImageAlias::VideoOutWrite, + "exact formatted video-out buffer write was rejected"); + Require("ImageOverlapResolution", "video-out alias guards", + ClassifyBufferImageAlias(video_address, video_size, + video_address + video_size, video_size, true, + false, false, true, true) == BufferImageAlias::None && + ClassifyBufferImageAlias(video_address, video_size - 4, + video_address, video_size, true, false, false, + true, true) == + BufferImageAlias::Unsupported && + ClassifyBufferImageAlias(video_address, video_size, video_address, + video_size, false, false, false, true, true) == + BufferImageAlias::Unsupported && + ClassifyBufferImageAlias(video_address, video_size, video_address, + video_size, true, false, true, true, true) == + BufferImageAlias::Unsupported && + ClassifyBufferImageAlias(video_address, video_size, video_address, + video_size, true, false, false, false, true) == + BufferImageAlias::Unsupported && + ClassifyBufferImageAlias(video_address, video_size, video_address, + video_size, true, false, false, true, false) == + BufferImageAlias::Unsupported, + "unsupported video-out buffer alias was admitted"); + + constexpr uint64_t video_metadata = 0x1020c10000ull; + Require("ImageOverlapResolution", "video-out DCC 256/64/64", + ClassifyVideoOutCompression(true, video_metadata, 0x208u, 0) == + VideoOutCompression::Dcc256_64_64, + "Prospero video-out DCC 256/64/64 mode was rejected"); + Require("ImageOverlapResolution", "video-out compression guards", + ClassifyVideoOutCompression(false, 0, 0, 0) == + VideoOutCompression::Uncompressed && + ClassifyVideoOutCompression(true, 0, 0x208u, 0) == + VideoOutCompression::Unsupported && + ClassifyVideoOutCompression(true, video_metadata + 1, 0x208u, 0) == + VideoOutCompression::Unsupported && + ClassifyVideoOutCompression(true, video_metadata, 0x204u, 0) == + VideoOutCompression::Unsupported && + ClassifyVideoOutCompression(true, video_metadata, 0x208u, 1) == + VideoOutCompression::Unsupported && + ClassifyVideoOutCompression(false, video_metadata, 0, 0) == + VideoOutCompression::Unsupported, + "unsupported video-out compression state was admitted"); + Require("ImageOverlapResolution", "compressed video-out native ownership", + CanUseVideoOutNativeWithoutUpload( + VideoOutCompression::Dcc256_64_64, true, false, false) && + CanUseVideoOutNativeWithoutUpload( + VideoOutCompression::Dcc256_64_64, false, true, false) && + !CanUseVideoOutNativeWithoutUpload( + VideoOutCompression::Dcc256_64_64, false, false, false) && + !CanUseVideoOutNativeWithoutUpload( + VideoOutCompression::Dcc256_64_64, true, false, true) && + !CanUseVideoOutNativeWithoutUpload( + VideoOutCompression::Uncompressed, true, true, false), + "compressed video-out upload/read ownership boundary regressed"); + + VkClearColorValue clear{}; + Require("ImageOverlapResolution", "RGBA8 compute clear decode", + DecodePackedColorClear(VK_FORMAT_R8G8B8A8_UNORM, 0x80402010u, + &clear) && + std::abs(clear.float32[0] - 16.0f / 255.0f) < 0.000001f && + std::abs(clear.float32[1] - 32.0f / 255.0f) < 0.000001f && + std::abs(clear.float32[2] - 64.0f / 255.0f) < 0.000001f && + std::abs(clear.float32[3] - 128.0f / 255.0f) < 0.000001f, + "packed RGBA8 clear did not preserve component values"); + Require("ImageOverlapResolution", "BGRA8 compute clear decode", + DecodePackedColorClear(VK_FORMAT_B8G8R8A8_UNORM, 0x80402010u, + &clear) && + std::abs(clear.float32[0] - 64.0f / 255.0f) < 0.000001f && + std::abs(clear.float32[1] - 32.0f / 255.0f) < 0.000001f && + std::abs(clear.float32[2] - 16.0f / 255.0f) < 0.000001f, + "packed BGRA8 clear did not account for memory channel order"); + const uint32_t rgb10_clear = 1023u | (512u << 10u) | (1u << 20u) | (3u << 30u); + Require("ImageOverlapResolution", "RGB10 compute clear decode", + DecodePackedColorClear(VK_FORMAT_A2B10G10R10_UNORM_PACK32, + rgb10_clear, &clear) && + clear.float32[0] == 1.0f && + std::abs(clear.float32[1] - 512.0f / 1023.0f) < 0.000001f && + std::abs(clear.float32[2] - 1.0f / 1023.0f) < 0.000001f && + clear.float32[3] == 1.0f, + "packed RGB10 clear did not preserve component values"); + Require("ImageOverlapResolution", "sRGB compute clear decode", + DecodePackedColorClear(VK_FORMAT_R8G8B8A8_SRGB, 0xff808080u, + &clear) && + std::abs(clear.float32[0] - 0.215861f) < 0.00001f && + std::abs(clear.float32[1] - 0.215861f) < 0.00001f && + std::abs(clear.float32[2] - 0.215861f) < 0.00001f && + clear.float32[3] == 1.0f && + !DecodePackedColorClear(VK_FORMAT_R16G16_UNORM, 0, &clear), + "sRGB clear decode or unsupported-format guard regressed"); + uint8_t stencil_clear = 0; + Require("ImageOverlapResolution", "stencil compute clear decode", + DecodePackedStencilClear(0x7f7f7f7fu, &stencil_clear) && + stencil_clear == 0x7fu && + DecodePackedStencilClear(0, &stencil_clear) && + stencil_clear == 0 && + !DecodePackedStencilClear(0x0000007fu, &stencil_clear) && + !DecodePackedStencilClear(0, nullptr), + "packed stencil clear admitted non-uniform byte values"); + float depth_clear = -1.0f; + Require("ImageOverlapResolution", "D32 compute clear decode", + DecodePackedDepthClear(VK_FORMAT_D32_SFLOAT, 0u, &depth_clear) && + depth_clear == 0.0f && + DecodePackedDepthClear(VK_FORMAT_D32_SFLOAT_S8_UINT, + 0x3f800000u, &depth_clear) && + depth_clear == 1.0f && + !DecodePackedDepthClear(VK_FORMAT_D16_UNORM, 0u, &depth_clear) && + !DecodePackedDepthClear(VK_FORMAT_D32_SFLOAT, 0xbf800000u, + &depth_clear) && + !DecodePackedDepthClear(VK_FORMAT_D32_SFLOAT, 0x40000000u, + &depth_clear) && + !DecodePackedDepthClear(VK_FORMAT_D32_SFLOAT, 0x7f800000u, + &depth_clear) && + !DecodePackedDepthClear(VK_FORMAT_D32_SFLOAT, 0u, nullptr), + "D32 compute clear decode admitted an unsupported format or value"); + DepthTargetInfo native_clear_target{}; + native_clear_target.address = 0x1158d80000ull; + native_clear_target.size = 0x870000ull; + native_clear_target.format = VK_FORMAT_D32_SFLOAT; + native_clear_target.guest_format = Prospero::GpuEnumValue(Prospero::BufferFormat::k32Float); + native_clear_target.bytes_per_element = 4; + native_clear_target.tile_mode = Prospero::GpuEnumValue(Prospero::TileMode::kDepth); + Require("ImageOverlapResolution", "native D32 buffer clear target", + CanNativeClearDepthFromBuffer(native_clear_target, + native_clear_target.address, + native_clear_target.size), + "exact uncompressed PS5 D32 target was rejected"); + auto compressed_depth = native_clear_target; + compressed_depth.htile_address = 0x1161480000ull; + compressed_depth.htile_size = 0x10000ull; + auto d16_depth = native_clear_target; + d16_depth.format = VK_FORMAT_D16_UNORM; + d16_depth.guest_format = Prospero::GpuEnumValue(Prospero::BufferFormat::k16UNorm); + d16_depth.bytes_per_element = 2; + Require("ImageOverlapResolution", "native depth buffer clear guards", + !CanNativeClearDepthFromBuffer(native_clear_target, + native_clear_target.address, + native_clear_target.size - 4) && + !CanNativeClearDepthFromBuffer(native_clear_target, + native_clear_target.address + 4, + native_clear_target.size) && + !CanNativeClearDepthFromBuffer(compressed_depth, + compressed_depth.address, + compressed_depth.size) && + !CanNativeClearDepthFromBuffer(d16_depth, d16_depth.address, + d16_depth.size), + "partial, offset, HTile-backed, or D16 depth clear was admitted"); + + constexpr uint64_t target_buffer_address = 0x1158d80000ull; + constexpr uint64_t target_buffer_size = 0x870000ull; + TileSizeAlign target_storage{}; + Require("ImageOverlapResolution", "formatted render-target write", + TileGetRenderTargetPitch(1920, 4) == 1920 && + TileGetRenderTargetSize(1920, 1080, 1920, 4, &target_storage) && + target_storage.align == 0x10000 && + target_storage.size == target_buffer_size && + ClassifyBufferImageAlias(target_buffer_address, target_buffer_size, + target_buffer_address, target_buffer_size, + false, true, true, true, true) == + BufferImageAlias::RenderTargetWrite, + "exact GPU-owned render target was not admitted for formatted buffer synchronization"); + Require("ImageOverlapResolution", "render-target buffer alias guards", + ClassifyBufferImageAlias(target_buffer_address, + target_buffer_size - 4, + target_buffer_address, target_buffer_size, + false, true, true, true, true) == + BufferImageAlias::Unsupported && + ClassifyBufferImageAlias(target_buffer_address, target_buffer_size, + target_buffer_address, target_buffer_size, + false, true, false, true, true) == + BufferImageAlias::Unsupported && + ClassifyBufferImageAlias(target_buffer_address, target_buffer_size, + target_buffer_address, target_buffer_size, + false, true, true, false, true) == + BufferImageAlias::Unsupported && + ClassifyBufferImageAlias(target_buffer_address, target_buffer_size, + target_buffer_address, target_buffer_size, + false, true, true, true, false) == + BufferImageAlias::Unsupported && + ClassifyBufferImageAlias(target_buffer_address + 0x100, + target_buffer_size, + target_buffer_address + 0x100, + target_buffer_size, + false, true, true, true, true) == + BufferImageAlias::Unsupported, + "unsupported render-target buffer alias was admitted"); + + RenderTargetInfo target{}; + target.address = sampled.address; + target.size = sampled.size; + Require("ImageOverlapResolution", "render target reuse", + ClassifyRenderTargetOverlap(sampled, false, true, target) == + RenderTargetOverlap::RetireSampled, + "exact CPU-owned sampled allocation was not retired for a render target"); + Require("ImageOverlapResolution", "render target GPU owner", + ClassifyRenderTargetOverlap(sampled, true, true, target) == + RenderTargetOverlap::Unsupported, + "GPU-owned sampled allocation was retired for a render target"); + Require("ImageOverlapResolution", "render target context", + ClassifyRenderTargetOverlap(sampled, false, false, target) == + RenderTargetOverlap::Unsupported, + "cross-context sampled allocation was retired for a render target"); + target.address += 0x2000; + target.size = 0x2000; + Require("ImageOverlapResolution", "render target pool reuse", + ClassifyRenderTargetOverlap(sampled, false, true, target) == + RenderTargetOverlap::RetireSampled, + "page-contained CPU-owned sampled pool was not retired for a render target"); + target.address = sampled.address + 0x6000; + target.size = 0x4000; + Require("ImageOverlapResolution", "render target chance overlap", + ClassifyRenderTargetOverlap(sampled, false, true, target) == + RenderTargetOverlap::RetireSampled, + "page-isolated sampled pool overlap was not retired for a render target"); + target.address++; + Require("ImageOverlapResolution", "render target partial page", + ClassifyRenderTargetOverlap(sampled, false, true, target) == + RenderTargetOverlap::Unsupported, + "partially shared tracker page was retired for a render target"); + target.address = page_left.address + page_left.size; + target.size = page_left.size; + Require("ImageOverlapResolution", "render target shared page", + ClassifyRenderTargetOverlap(page_left, false, true, target) == + RenderTargetOverlap::Unsupported, + "byte-disjoint allocation sharing a tracker page was silently admitted"); + target.address = sampled.address + sampled.size; + target.size = sampled.size; + Require("ImageOverlapResolution", "render target adjacent", + ClassifyRenderTargetOverlap(sampled, false, true, target) == + RenderTargetOverlap::None, + "adjacent sampled and render-target allocations were treated as aliases"); + + RenderTargetInfo old_target{}; + old_target.address = 0x108d80000ull; + old_target.size = 0x10000; + old_target.format = VK_FORMAT_R8G8B8A8_UNORM; + old_target.width = 48; + old_target.height = 48; + old_target.pitch = 128; + old_target.bytes_per_element = 4; + old_target.tile_mode = Prospero::GpuEnumValue(Prospero::TileMode::kRenderTarget); + RenderTargetInfo replacement_target = old_target; + replacement_target.size = 0x20000; + replacement_target.format = VK_FORMAT_R16G16B16A16_SFLOAT; + replacement_target.width = 128; + replacement_target.height = 128; + replacement_target.bytes_per_element = 8; + Require("ImageOverlapResolution", "render target allocation-pool replacement", + ClassifyRenderTargetOverlap(old_target, false, false, true, + replacement_target) == + RenderTargetOverlap::RetireTarget, + "clean same-base render-target pool allocation was not retired"); + Require("ImageOverlapResolution", "render target allocation-pool GPU owner", + ClassifyRenderTargetOverlap(old_target, true, false, true, + replacement_target) == + RenderTargetOverlap::Unsupported, + "GPU-owned render-target pool allocation was retired"); + Require("ImageOverlapResolution", "render target allocation-pool buffer owner", + ClassifyRenderTargetOverlap(old_target, false, true, true, + replacement_target) == + RenderTargetOverlap::Unsupported, + "buffer-dirty render-target pool allocation was retired"); + Require("ImageOverlapResolution", "render target allocation-pool context", + ClassifyRenderTargetOverlap(old_target, false, false, false, + replacement_target) == + RenderTargetOverlap::Unsupported, + "cross-context render-target pool allocation was retired"); + auto offset_target = replacement_target; + offset_target.address += TRACKER_PAGE_SIZE; + Require("ImageOverlapResolution", "render target allocation-pool offset", + ClassifyRenderTargetOverlap(old_target, false, false, true, offset_target) == + RenderTargetOverlap::Unsupported, + "offset render-target overlap was treated as allocation-pool replacement"); + auto partial_page_target = replacement_target; + partial_page_target.address++; + Require("ImageOverlapResolution", "render target allocation-pool partial page", + ClassifyRenderTargetOverlap(old_target, false, false, true, + partial_page_target) == + RenderTargetOverlap::Unsupported, + "partially shared tracker page was treated as allocation-pool replacement"); + auto same_shape_target = old_target; + same_shape_target.format = VK_FORMAT_R8G8B8A8_SRGB; + Require("ImageOverlapResolution", "render target RGBA8 sRGB reinterpretation", + IsCompatibleRenderTargetView(old_target, same_shape_target), + "exact RGBA8 UNORM-to-sRGB target was not recognized as a compatible view"); + Require("ImageOverlapResolution", "render target sRGB no retirement", + ClassifyRenderTargetOverlap(old_target, false, false, true, same_shape_target) == + RenderTargetOverlap::Unsupported, + "compatible render-target view fell through to target retirement"); + auto incompatible_same_shape_target = same_shape_target; + incompatible_same_shape_target.format = VK_FORMAT_R8G8B8A8_UINT; + Require("ImageOverlapResolution", "render target incompatible same shape", + !IsCompatibleRenderTargetView(old_target, incompatible_same_shape_target) && + ClassifyRenderTargetOverlap(old_target, false, false, true, + incompatible_same_shape_target) == + RenderTargetOverlap::Unsupported, + "incompatible same-storage render-target format was admitted"); + + RenderTargetInfo ppsa02604_unorm_target{}; + ppsa02604_unorm_target.address = 0x79c50000ull; + ppsa02604_unorm_target.size = 0x870000; + ppsa02604_unorm_target.format = VK_FORMAT_B8G8R8A8_UNORM; + ppsa02604_unorm_target.width = 1920; + ppsa02604_unorm_target.height = 1080; + ppsa02604_unorm_target.pitch = 1920; + ppsa02604_unorm_target.bytes_per_element = 4; + ppsa02604_unorm_target.tile_mode = Prospero::GpuEnumValue(Prospero::TileMode::kRenderTarget); + auto ppsa02604_srgb_target = ppsa02604_unorm_target; + ppsa02604_srgb_target.format = VK_FORMAT_B8G8R8A8_SRGB; + Require("ImageOverlapResolution", "PPSA02604 BGRA8 sRGB reinterpretation", + IsCompatibleRenderTargetView(ppsa02604_unorm_target, ppsa02604_srgb_target), + "PPSA02604 exact BGRA8 UNORM-to-sRGB target was not recognized as a view"); + auto adjacent_target = replacement_target; + adjacent_target.address = old_target.address + old_target.size; + Require("ImageOverlapResolution", "render target allocation-pool adjacent", + ClassifyRenderTargetOverlap(old_target, false, false, true, adjacent_target) == + RenderTargetOverlap::None, + "adjacent render targets were treated as allocation-pool replacement"); + + Require("ImageOverlapResolution", "metadata retains sampled image", + ClassifyMetaImageOverlap(true, false, false, false) == + MetaImageOverlap::RetainSampled, + "CPU-current sampled metadata alias was rejected"); + Require("ImageOverlapResolution", "metadata retires render target", + ClassifyMetaImageOverlap(false, true, false, false) == + MetaImageOverlap::RetireTarget, + "CPU-current render target was not retired for metadata reuse"); + Require("ImageOverlapResolution", "metadata rejects GPU target", + ClassifyMetaImageOverlap(false, true, true, false) == + MetaImageOverlap::Unsupported && + ClassifyMetaImageOverlap(false, true, false, true) == + MetaImageOverlap::Unsupported, + "unordered or buffer-dirty render target was admitted for metadata reuse"); + Require("ImageOverlapResolution", "metadata rejects unsupported image", + ClassifyMetaImageOverlap(false, false, false, false) == + MetaImageOverlap::Unsupported, + "unsupported cached image kind was admitted for metadata reuse"); + + const std::array isolated_retirement{ + ImageRetirementRange{sampled.address, TRACKER_PAGE_SIZE, true}, + ImageRetirementRange{sampled.address + 2 * TRACKER_PAGE_SIZE, + TRACKER_PAGE_SIZE, false}}; + Require("ImageOverlapResolution", "isolated retirement", + !FindImageRetirementConflict(isolated_retirement).Exists(), + "page-isolated sampled retirement was rejected"); + const std::array tail_alias_retirement{ + ImageRetirementRange{sampled.address, 3 * TRACKER_PAGE_SIZE, true}, + ImageRetirementRange{sampled.address + 2 * TRACKER_PAGE_SIZE, + TRACKER_PAGE_SIZE, false}}; + const auto retirement_conflict = + FindImageRetirementConflict(tail_alias_retirement); + Require("ImageOverlapResolution", "retained tail alias", + retirement_conflict.Exists() && retirement_conflict.retired == 0 && + retirement_conflict.retained == 1, + "retiring a wide image silently untracked a retained tail alias"); + DepthTargetInfo depth{}; + depth.address = sampled.address; + depth.size = 0x6000; + depth.depth_load_clear = true; + Require("ImageOverlapResolution", "clear depth reuse", + ClassifyDepthOverlap(sampled, false, depth) == + DepthOverlap::RetireSampled, + "equal-base CPU-owned sampled image was not classified as pool reuse"); + Require("ImageOverlapResolution", "gpu owner", + ClassifyDepthOverlap(sampled, true, depth) == + DepthOverlap::Unsupported, + "GPU-owned sampled image was admitted for retirement"); + depth.depth_load_clear = false; + Require("ImageOverlapResolution", "depth load", + ClassifyDepthOverlap(sampled, false, depth) == + DepthOverlap::Unsupported, + "load-preserving depth use was admitted for retirement"); + depth.depth_load_clear = true; + depth.stencil_address = sampled.address + 0x6000; + depth.stencil_size = 0x2000; + Require("ImageOverlapResolution", "stencil load", + ClassifyDepthOverlap(sampled, false, depth) == + DepthOverlap::Unsupported, + "load-preserving stencil use was admitted for retirement"); + depth.stencil_load_clear = true; + Require("ImageOverlapResolution", "clear stencil reuse", + ClassifyDepthOverlap(sampled, false, depth) == + DepthOverlap::RetireSampled, + "fully cleared depth/stencil reuse was rejected"); + depth.stencil_load_clear = false; + depth.stencil_access = false; + Require("ImageOverlapResolution", "unused stencil first load", + CanLoadStencilAttachment(depth, false), + "unused stencil unnecessarily required initialized contents"); + depth.stencil_access = true; + Require("ImageOverlapResolution", "uninitialized stencil access", + !CanLoadStencilAttachment(depth, false), + "uninitialized stencil access was silently admitted"); + Require("ImageOverlapResolution", "initialized stencil load", + CanLoadStencilAttachment(depth, true), + "initialized stencil contents could not be loaded"); + depth.stencil_load_clear = true; + Require("ImageOverlapResolution", "stencil clear initializes", + CanLoadStencilAttachment(depth, false), + "stencil clear did not initialize the attachment"); + depth.address = sampled.address; + depth.size = 0xff0000; + depth.stencil_address = sampled.address + 0x1000000; + depth.stencil_size = 0x200000; + Require("ImageOverlapResolution", "logical depth view", + IsDepthTargetRangeCompatible(depth, depth.address, 0xfd2000), + "same-base logical depth range did not reuse its padded target"); + Require("ImageOverlapResolution", "complete depth view", + IsDepthTargetRangeCompatible(depth, depth.address, depth.size), + "complete depth range was rejected"); + Require("ImageOverlapResolution", "oversized depth view", + !IsDepthTargetRangeCompatible(depth, depth.address, depth.size + 1), + "oversized depth range was silently admitted"); + Require("ImageOverlapResolution", "offset depth view", + !IsDepthTargetRangeCompatible(depth, depth.address + 1, depth.size - 1), + "offset depth range was silently admitted"); + Require("ImageOverlapResolution", "complete stencil view", + IsDepthTargetRangeCompatible(depth, depth.stencil_address, depth.stencil_size), + "complete stencil range was rejected"); + Require("ImageOverlapResolution", "partial stencil view", + !IsDepthTargetRangeCompatible(depth, depth.stencil_address, depth.stencil_size - 1), + "partial stencil range was silently admitted"); + depth.address = sampled.address + 0x1000; + Require("ImageOverlapResolution", "offset alias", + ClassifyDepthOverlap(sampled, false, depth) == + DepthOverlap::Unsupported, + "offset image overlap was admitted for retirement"); + depth.address = sampled.address + sampled.size; + depth.stencil_address = 0; + depth.stencil_size = 0; + depth.stencil_load_clear = false; + Require("ImageOverlapResolution", "adjacent", + ClassifyDepthOverlap(sampled, false, depth) == DepthOverlap::None, + "byte-disjoint adjacent image ranges were treated as aliases"); + std::printf("[host] %-32s ok\n", "ImageOverlapResolution"); +} + +void CheckMsaaCompatibility() { + Require("MsaaCompatibility", "single sample", + !depth_msaa_single_sample_compatible(0), + "native single-sample state was classified as compatibility"); + Require("MsaaCompatibility", "two fragments", + depth_msaa_single_sample_compatible(1), + "two-fragment compatibility state was rejected"); + Require("MsaaCompatibility", "four fragments", + depth_msaa_single_sample_compatible(2), + "existing four-fragment compatibility state regressed"); + Require("MsaaCompatibility", "eight fragments", + !depth_msaa_single_sample_compatible(3), + "unsupported eight-fragment state was silently admitted"); + Require("MsaaCompatibility", "color two fragments", + color_msaa_single_sample_compatible(1, 1), + "matching two-sample color compatibility state was rejected"); + Require("MsaaCompatibility", "color four fragments", + color_msaa_single_sample_compatible(2, 2), + "existing matching four-sample color compatibility state regressed"); + Require("MsaaCompatibility", "color mismatch", + !color_msaa_single_sample_compatible(2, 1), + "mismatched color sample/fragment state was silently admitted"); + Require("MsaaCompatibility", "color eight fragments", + !color_msaa_single_sample_compatible(3, 3), + "unsupported eight-sample color state was silently admitted"); + std::printf("[host] %-32s ok\n", "MsaaCompatibility"); +} + +void CheckDepthHtileStencilCompatibility() { + Require("DepthHtileStencilCompatibility", "disabled acceleration", + depth_htile_stencil_acceleration_compatible(false, false, true), + "disabled Hi-Stencil state was rejected"); + Require("DepthHtileStencilCompatibility", "PS5 stencil plus HTile", + depth_htile_stencil_acceleration_compatible(true, true, false), + "valid PS5 Hi-Stencil attachment was rejected"); + Require("DepthHtileStencilCompatibility", "missing stencil plane", + !depth_htile_stencil_acceleration_compatible(false, true, false), + "Hi-Stencil without a stencil plane was silently admitted"); + Require("DepthHtileStencilCompatibility", "missing HTile metadata", + !depth_htile_stencil_acceleration_compatible(true, false, false), + "Hi-Stencil without HTile metadata was silently admitted"); + std::printf("[host] %-32s ok\n", "DepthHtileStencilCompatibility"); +} + +void CheckStencilAttachmentAccess() { + PipelineStencilStaticState state{VK_STENCIL_OP_KEEP, VK_STENCIL_OP_KEEP, + VK_STENCIL_OP_KEEP, VK_COMPARE_OP_ALWAYS}; + PipelineStencilDynamicState dynamic{0xff, 0xff, 0}; + Require("StencilAttachmentAccess", "always keep", + !stencil_face_accesses_attachment(state, dynamic), + "ALWAYS/KEEP state was classified as stencil access"); + state.compareOp = VK_COMPARE_OP_EQUAL; + Require("StencilAttachmentAccess", "compare reads", + stencil_face_accesses_attachment(state, dynamic), + "real stencil comparison was classified as no access"); + state.compareOp = VK_COMPARE_OP_ALWAYS; + state.passOp = VK_STENCIL_OP_ZERO; + Require("StencilAttachmentAccess", "write operation", + stencil_face_accesses_attachment(state, dynamic), + "write-capable stencil operation was classified as no access"); + dynamic.writeMask = 0; + Require("StencilAttachmentAccess", "masked write", + !stencil_face_accesses_attachment(state, dynamic), + "fully masked stencil write was classified as access"); + std::printf("[host] %-32s ok\n", "StencilAttachmentAccess"); +} + +void CheckDepthTargetFootprints() { + TileSizeAlign stencil{}; + TileSizeAlign htile{}; + TileSizeAlign depth{}; + Require("DepthTargetFootprints", "640x360 Z32S8 without HTile", + TileGetDepthSize(640, 360, 0, Prospero::GpuEnumValue(Prospero::DepthFormat::kZ32F), + Prospero::GpuEnumValue(Prospero::StencilFormat::k8UInt), false, + &stencil, &htile, &depth), + "valid non-HTile depth/stencil footprint was rejected"); + Require("DepthTargetFootprints", "640x360 Prospero block sizes", + depth.size == 0xf0000 && depth.align == 0x10000 && + stencil.size == 0x60000 && stencil.align == 0x10000 && + htile.size == 0 && htile.align == 0, + "non-HTile depth/stencil footprint disagrees with Prospero block rules"); + + const auto depth_pitch = TileGetTexturePitch( + Prospero::GpuEnumValue(Prospero::BufferFormat::k32Float), 640, 1, + Prospero::GpuEnumValue(Prospero::TileMode::kDepth)); + TileSizeAlign texture_depth{}; + TileGetTextureTotalSize(Prospero::GpuEnumValue(Prospero::BufferFormat::k32Float), 640, 360, + 1, depth_pitch, 1, + Prospero::GpuEnumValue(Prospero::TileMode::kDepth), false, + &texture_depth); + Require("DepthTargetFootprints", "640x360 generic depth tile", + depth_pitch == 640 && texture_depth.size == 0xf0000 && + texture_depth.align == 0x10000, + "generic depth texture sizing bypassed 64 KiB block padding"); + + Require("DepthTargetFootprints", "960x540 Z32S8 with HTile", + TileGetDepthSize(960, 540, 0, + Prospero::GpuEnumValue(Prospero::DepthFormat::kZ32F), + Prospero::GpuEnumValue(Prospero::StencilFormat::k8UInt), true, + &stencil, &htile, &depth) && + depth.size == 0x280000 && depth.align == 0x10000 && + stencil.size == 0xc0000 && stencil.align == 0x10000 && + htile.size == 0x10000 && htile.align == 0x8000, + "generic Prospero HTile block calculation rejected the title footprint"); + Require("DepthTargetFootprints", "known HTile extent", + TileGetDepthSize(1280, 720, 0, + Prospero::GpuEnumValue(Prospero::DepthFormat::kZ32F), + Prospero::GpuEnumValue(Prospero::StencilFormat::k8UInt), true, + &stencil, &htile, &depth) && + depth.size == 0x3c0000 && stencil.size == 0xf0000 && + htile.size == 0x20000, + "validated 1280x720 HTile footprint regressed"); + Require("DepthTargetFootprints", "PPSA06228 3840x2160 Z32S8 with HTile", + TileGetDepthSize(3840, 2160, 0, + Prospero::GpuEnumValue(Prospero::DepthFormat::kZ32F), + Prospero::GpuEnumValue(Prospero::StencilFormat::k8UInt), true, + &stencil, &htile, &depth) && + depth.size == 0x1fe0000 && depth.align == 0x10000 && + stencil.size == 0x870000 && stencil.align == 0x10000 && + htile.size == 0xa0000 && htile.align == 0x8000, + "captured 3840x2160 depth/stencil/HTile footprint disagrees with Prospero rules"); + Require("DepthTargetFootprints", "invalid depth format", + !TileGetDepthSize(960, 540, 0, 2, + Prospero::GpuEnumValue(Prospero::StencilFormat::k8UInt), true, + &stencil, &htile, &depth), + "unsupported depth format was silently admitted"); + Require("DepthTargetFootprints", "invalid stencil format", + !TileGetDepthSize(960, 540, 0, + Prospero::GpuEnumValue(Prospero::DepthFormat::kZ32F), 2, true, + &stencil, &htile, &depth), + "unsupported stencil format was silently admitted"); + Require("DepthTargetFootprints", "invalid extent", + !TileGetDepthSize(0, 540, 0, + Prospero::GpuEnumValue(Prospero::DepthFormat::kZ32F), + Prospero::GpuEnumValue(Prospero::StencilFormat::k8UInt), true, + &stencil, &htile, &depth) && + !TileGetDepthSize(16385, 540, 0, + Prospero::GpuEnumValue(Prospero::DepthFormat::kZ32F), + Prospero::GpuEnumValue(Prospero::StencilFormat::k8UInt), true, + &stencil, &htile, &depth), + "invalid HTile extent was silently admitted"); + std::printf("[host] %-32s ok\n", "DepthTargetFootprints"); +} + +void CheckHtileClearTargetResolution() { + HW::DepthRenderTarget descriptor_backed{}; + descriptor_backed.z_info.format = Prospero::GpuEnumValue(Prospero::DepthFormat::kZ16); + descriptor_backed.z_info.tile_surface_enable = true; + descriptor_backed.z_info.zrange_precision = 1; + descriptor_backed.stencil_info.tile_stencil_disable = true; + descriptor_backed.z_read_base_addr = 0xc1a80000; + descriptor_backed.z_write_base_addr = descriptor_backed.z_read_base_addr; + descriptor_backed.htile_data_base_addr = 0xc1a98000; + + HtileClearTarget resolved{}; + Require("HtileClearTargetResolution", "PPSA09076 descriptor-backed Z16 target", + ResolveHtileClearTarget(descriptor_backed, 0x8000, &resolved) && + resolved.address == 0xc1a98000 && resolved.size == 0x8000, + "valid descriptor-proven Z16 HTile clear target was rejected"); + + auto partial_extent = descriptor_backed; + partial_extent.size.x_max = 1; + auto missing_stencil = descriptor_backed; + missing_stencil.stencil_info.format = Prospero::GpuEnumValue(Prospero::StencilFormat::k8UInt); + auto mismatched_write = descriptor_backed; + mismatched_write.z_write_base_addr += 0x10000; + Require("HtileClearTargetResolution", "descriptor-backed rejection boundaries", + !ResolveHtileClearTarget(descriptor_backed, 0x4000, &resolved) && + !ResolveHtileClearTarget(partial_extent, 0x8000, &resolved) && + !ResolveHtileClearTarget(missing_stencil, 0x8000, &resolved) && + !ResolveHtileClearTarget(mismatched_write, 0x8000, &resolved), + "ambiguous or inconsistent descriptor-backed HTile target was admitted"); + + HW::DepthRenderTarget derived{}; + derived.z_info.format = Prospero::GpuEnumValue(Prospero::DepthFormat::kZ32F); + derived.z_info.tile_surface_enable = true; + derived.z_info.zrange_precision = 1; + derived.stencil_info.format = Prospero::GpuEnumValue(Prospero::StencilFormat::k8UInt); + derived.stencil_info.tile_stencil_disable = true; + derived.z_read_base_addr = 0x100000; + derived.z_write_base_addr = derived.z_read_base_addr; + derived.stencil_read_base_addr = 0x500000; + derived.stencil_write_base_addr = derived.stencil_read_base_addr; + derived.htile_data_base_addr = 0x700000; + derived.size.valid = true; + derived.size.x_max = 959; + derived.size.y_max = 539; + TileSizeAlign depth_size{}; + TileSizeAlign stencil_size{}; + TileSizeAlign htile_size{}; + Require("HtileClearTargetResolution", "derived Z32S8 target", + TileGetDepthSize(960, 540, 0, derived.z_info.format, + derived.stencil_info.format, true, &stencil_size, + &htile_size, &depth_size) && + ResolveHtileClearTarget(derived, htile_size.size, &resolved) && + resolved.address == derived.htile_data_base_addr && + resolved.size == htile_size.size && + !ResolveHtileClearTarget(derived, htile_size.size + 0x8000, &resolved), + "extent-derived HTile validation or descriptor cross-check regressed"); + + auto derived_z16 = derived; + derived_z16.z_info.format = Prospero::GpuEnumValue(Prospero::DepthFormat::kZ16); + derived_z16.stencil_info.format = Prospero::GpuEnumValue(Prospero::StencilFormat::kInvalid); + derived_z16.stencil_read_base_addr = 0; + derived_z16.stencil_write_base_addr = 0; + Require("HtileClearTargetResolution", "derived Z16 depth-only target", + TileGetDepthSize(960, 540, 0, derived_z16.z_info.format, + derived_z16.stencil_info.format, true, &stencil_size, + &htile_size, &depth_size) && + stencil_size.align == 0 && stencil_size.size == 0 && + ResolveHtileClearTarget(derived_z16, htile_size.size, &resolved) && + resolved.address == derived_z16.htile_data_base_addr && + resolved.size == htile_size.size, + "extent-derived Z16 depth-only HTile target was rejected"); + std::printf("[host] %-32s ok\n", "HtileClearTargetResolution"); +} + +struct FenceLifetimeProbe { + explicit FenceLifetimeProbe(bool *destroyed) : destroyed(destroyed) { + if (destroyed == nullptr || *destroyed) { + EXIT("fence-lifetime probe has invalid construction state\n"); + } + } + ~FenceLifetimeProbe() { *destroyed = true; } + + bool *destroyed = nullptr; +}; + +void CheckCrossQueueImageLifetime() { + bool destroyed = false; + auto image = std::make_shared(&destroyed); + FenceResourceRetainer graphics; + FenceResourceRetainer compute; + graphics.Retain(image); + compute.Retain(image); + graphics.Retain(image); + image.reset(); + Require("CrossQueueImageLifetime", "retained", !destroyed && !graphics.Empty() && + !compute.Empty(), + "cache removal destroyed an image retained by command buffers"); + graphics.ReleaseAfterFence(); + Require("CrossQueueImageLifetime", "first fence", !destroyed && graphics.Empty() && + !compute.Empty(), + "first command-buffer fence destroyed another queue's image"); + compute.ReleaseAfterFence(); + Require("CrossQueueImageLifetime", "last fence", destroyed && compute.Empty(), + "last referencing command-buffer fence did not destroy the image"); + std::printf("[host] %-32s ok\n", "CrossQueueImageLifetime"); +} + +void CheckHostDmaMetadataReuse() { + constexpr uintptr_t base = 0x0000000200010000ull; + constexpr uint64_t allocation_size = 0x10000; + constexpr uint64_t metadata_size = 0x8000; + auto *memory = static_cast(VirtualAlloc( + reinterpret_cast(base), allocation_size, + MEM_RESERVE | MEM_COMMIT, PAGE_READWRITE)); + Require("HostDmaMetadataReuse", "allocation", + memory == reinterpret_cast(base), "fixed VirtualAlloc failed"); + std::memset(memory + metadata_size, 0x5a, 0x1000); + + ResourceMutex resource_mutex; + CacheFaultContext fault_context; + PageManager page_manager(CacheFault, &fault_context); + BufferCache buffer_cache(page_manager, resource_mutex); + TextureCache texture_cache(page_manager, buffer_cache, resource_mutex); + fault_context.texture = &texture_cache; + buffer_cache.SetTextureCache(texture_cache); + page_manager.OnGpuMap(base, allocation_size); + texture_cache.RegisterMeta(base, metadata_size); + auto *ctx = reinterpret_cast(1); + + Require("HostDmaMetadataReuse", "clear", texture_cache.ClearMeta(base), + "metadata clear setup failed"); + MEMORY_BASIC_INFORMATION protection{}; + Require("HostDmaMetadataReuse", "protected", + VirtualQuery(memory, &protection, sizeof(protection)) != 0 && + protection.Protect == PAGE_READONLY, + "virtual metadata was not write-watched"); + Require("HostDmaMetadataReuse", "copy fault", + page_manager.HandleFault(PageFaultAccess::Write, base), + "copy destination did not transfer to CPU ownership"); + buffer_cache.CopyBuffer(nullptr, ctx, base, base + metadata_size, 0x1000); + Require("HostDmaMetadataReuse", "copy", + std::memcmp(memory, memory + metadata_size, 0x1000) == 0, + "post-clear CPU DMA copy did not publish backing"); + Require("HostDmaMetadataReuse", "fill fault", + page_manager.HandleFault(PageFaultAccess::Write, base + 0x1000), + "fill destination did not transfer to CPU ownership"); + buffer_cache.FillBuffer(nullptr, ctx, base + 0x1000, 0x1000, 0x11223344); + Require("HostDmaMetadataReuse", "fill", + reinterpret_cast(memory + 0x1000)[0] == 0x11223344 && + reinterpret_cast(memory + 0x1ffc)[0] == 0x11223344, + "post-clear CPU DMA fill did not publish backing"); + Require("HostDmaMetadataReuse", "identity", + texture_cache.HasMetaOverlap(base, metadata_size) && + !texture_cache.IsMetaGpuModified(base, 0x2000) && + VirtualQuery(memory, &protection, sizeof(protection)) != 0 && + protection.Protect == PAGE_READWRITE, + "CPU writes erased metadata identity or retained virtual ownership"); + + texture_cache.UnmapMemory(base, allocation_size); + page_manager.OnGpuUnmap(base, allocation_size); + Require("HostDmaMetadataReuse", "free", + VirtualFree(memory, 0, MEM_RELEASE) != 0, "VirtualFree failed"); + std::printf("[host] %-32s ok\n", "HostDmaMetadataReuse"); +} +#endif + +void CheckEmbeddedFetchLaneSpill() { + std::vector code; + code.push_back(EncodeSMovB32(0, InlineU32(0))); + code.push_back(EncodeSmem0(0x02u, 20, 4)); + code.push_back(EncodeSmem1(0)); // s_load_dwordx4 s[20:23], s[8:9] + AppendVop3(&code, 0x361u, 6, 20, InlineU32(0)); // v_writelane_b32 v6, s20, 0 + code.push_back(EncodeSMovB32(20, InlineU32(0))); + AppendVop3(&code, 0x360u, 20, Vgpr(6), InlineU32(0)); // v_readlane_b32 s20, v6, 0 + code.push_back(EncodeVop2(0x01u, 0, Vgpr(8), 5)); + code.push_back(EncodeMubuf0(0x03u, 0, true)); + code.push_back(EncodeMubuf1(9, 5, 0)); + AppendEnd(&code); + + std::array user_data{}; + ShaderVertexInputInfo vertex; + vertex.fetch_embedded = true; + vertex.fetch_buffer_reg = 0; + vertex.resources_num = 1; + vertex.resources_dst[0].attr_id = 0; + vertex.resources_dst[0].registers_num = 4; + + ShaderRecompiler::CompileOptions options; + options.stage = ShaderType::Vertex; + options.user_data_base = 8; + options.user_data_count = static_cast(user_data.size()); + options.user_data = user_data.data(); + options.vertex_input_info = &vertex; + + ShaderRecompiler::CompileResult result; + std::string error; + Require("EmbeddedFetchLaneSpill", "compile", + ShaderRecompiler::TryRecompile(code, options, &result, &error), error); + Require("EmbeddedFetchLaneSpill", "fetch rewrite", + vertex.resource_fetch_components[0] == 4, + "lane-spilled fetch descriptor was not recognized and rewritten"); + std::printf("[host] %-32s ok\n", "EmbeddedFetchLaneSpill"); +} + +} // namespace +} // namespace Libs::Graphics + +int main(int argc, char **argv) { + using namespace Libs::Graphics; + + EnsureConfigInitialized(); + TileInit(); +#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS + if (argc == 2 && std::strcmp(argv[1], "--image-overlap-only") == 0) { + CheckImageOverlapResolution(); + return 0; + } + if (argc == 2 && std::strcmp(argv[1], "--htile-clear-only") == 0) { + CheckHtileClearTargetResolution(); + CheckOverlappingMetadataViews(); + return 0; + } + if (argc == 2 && std::strcmp(argv[1], "--image-view-only") == 0) { + CheckSampledColorViews(); + return 0; + } + if (argc == 2 && std::strcmp(argv[1], "--sampled-depth-resource-only") == 0) { + CheckSampledDepthResource(); + CheckSampledDepthDescriptor(); + return 0; + } + if (argc == 2 && std::strcmp(argv[1], "--buffer-cache-range-only") == 0) { + CheckBufferCacheRangeMerge(); + return 0; + } + if (argc == 2 && std::strcmp(argv[1], "--storage-bgra-only") == 0) { + CheckSampledColorViews(); + CheckBasicStorageTextureDescriptor(); + VulkanHarness vulkan; + vulkan.CheckMutableRenderTargetBgraStorageView(); + RunCase(&vulkan, ImageStoreBgraUsesInverseSwizzle()); + return 0; + } + if (argc == 2 && std::strcmp(argv[1], "--storage-yzwx-only") == 0) { + CheckSampledColorViews(); + CheckBasicStorageTextureDescriptor(); + CheckStorageTextureGpuOwnedRebindState(); + VulkanHarness vulkan; + RunCase(&vulkan, ImageStoreYzwxUsesInverseSwizzle()); + return 0; + } + if (argc == 2 && std::strcmp(argv[1], "--storage-sampled-only") == 0) { + CheckStorageTextureSampledReuse(); + VulkanHarness vulkan; + vulkan.CheckMutableStorageSrgbView(); + return 0; + } + if (argc == 2 && std::strcmp(argv[1], "--depth-readback-only") == 0) { + CheckDepthTargetTileRoundTrip(); + CheckDepthTargetFootprints(); + return 0; + } + if (argc == 3 && std::strcmp(argv[1], "--image-view-death") == 0) { + RunImageViewDeathCase(argv[2]); + } + if (argc == 3 && + std::strcmp(argv[1], "--storage-texture-descriptor-death") == 0) { + RunStorageTextureDescriptorDeathCase(argv[2]); + } + if (argc == 3 && std::strcmp(argv[1], "--storage-texture-access-death") == 0) { + RunStorageTextureAccessDeathCase(argv[2]); + } + if (argc == 3 && std::strcmp(argv[1], "--meta-overlap-death") == 0) { + RunMetaOverlapDeathCase(argv[2]); + } + if (argc == 3 && std::strcmp(argv[1], "--metadata-descriptor-death") == 0) { + RunMetadataDescriptorDeathCase(argv[2]); + } + CheckSampledColorViews(); + CheckSampledDepthResource(); + CheckSampledDepthDescriptor(); + CheckBufferCacheRangeMerge(); + CheckBasicStorageTextureDescriptor(); + CheckStorageTextureLinearUploadLayout(); + CheckStorageTextureLinearReadbackLayout(); + CheckStorageImageSwizzleSpecializationId(); + CheckRenderTargetTileRoundTrip(); + CheckDepthTargetTileRoundTrip(); + CheckStorageTextureVolumeUploadLayout(); + CheckStorageTextureGpuOwnedRebindState(); + CheckStorageTextureSampledReuse(); + CheckStorageTextureAccessPermissions(); + CheckMetaOverlapDeaths(); + CheckOverlappingMetadataViews(); + CheckMetadataReuseDescriptors(); + CheckImageOverlapResolution(); + CheckMsaaCompatibility(); + CheckDepthHtileStencilCompatibility(); + CheckStencilAttachmentAccess(); + CheckDepthTargetFootprints(); + CheckHtileClearTargetResolution(); + CheckCrossQueueImageLifetime(); + CheckHostDmaMetadataReuse(); +#else + (void)argc; + (void)argv; +#endif + CheckEmbeddedFetchVertexOffset(); + CheckEmbeddedFetchLaneSpill(); + CheckPs5GameExampleImageClearRuntimeShape(); + VulkanHarness vulkan; + vulkan.CheckMutableStorageSrgbView(); + vulkan.CheckMutableRenderTargetBgraStorageView(); + const auto tests = MakeCases(); + const auto graphics_tests = MakeGraphicsCases(); + CheckOpcodeCoverage(tests, graphics_tests); + for (const auto &test : tests) { + RunCase(&vulkan, test); + } + for (const auto &test : MakeSkippedCases()) { + std::printf("[skip] %-32s %s\n", test.name, test.reason); + } + for (const auto &test : graphics_tests) { + RunGraphicsCase(&vulkan, test); + } + std::printf("ShaderRecompilerComputeTests: all cases passed\n"); + return 0; +} diff --git a/tests/ShaderStageRuntimeTests.cpp b/tests/ShaderStageRuntimeTests.cpp new file mode 100644 index 000000000..b120240a4 --- /dev/null +++ b/tests/ShaderStageRuntimeTests.cpp @@ -0,0 +1,107 @@ +#include "graphics/shader/shader.h" + +#include "graphics/shader/recompiler/ResourceMaterialization.h" +#include "graphics/shader/recompiler/ShaderIR.h" + +#include +#include +#include +#include + +namespace { + +void Check(bool value, const char* text) { + if (!value) { + std::fprintf(stderr, "ShaderStageRuntimeTests: failed: %s\n", text); + std::abort(); + } +} + +std::shared_ptr SrtProgram(uint64_t address) { + using namespace Libs::Graphics::ShaderRecompiler::IR; + Program program; + program.stage = Libs::Graphics::ShaderType::Compute; + program.srt_plan_complete = true; + program.srt_patching_complete = true; + program.resource_tracking_complete = true; + program.provenance.values.resize(6); + program.provenance.values[0].op = ScalarValueOp::Undefined; + program.provenance.values[1].op = ScalarValueOp::Unknown; + program.provenance.values[2].op = ScalarValueOp::Constant; + program.provenance.values[2].imm = static_cast(address); + program.provenance.values[3].op = ScalarValueOp::Constant; + program.provenance.values[3].imm = static_cast(address >> 32u); + program.provenance.values[4].op = ScalarValueOp::Constant; + program.provenance.values[5].op = ScalarValueOp::ReadConst; + program.provenance.values[5].args = {2, 3, 4}; + program.srt.reads.push_back({5, 0, 0x40}); + return std::make_shared(std::move(program)); +} + +std::shared_ptr UnbasedFlatProgram() { + using namespace Libs::Graphics::ShaderRecompiler::IR; + Program program; + program.stage = Libs::Graphics::ShaderType::Compute; + program.srt_plan_complete = true; + program.srt_patching_complete = true; + program.resource_tracking_complete = true; + AddressResource address; + address.source = ScalarProvenance::Unknown; + address.first_use_pc = 0x80; + address.kind = ResourceKind::Flat; + program.info.addresses.push_back(address); + return std::make_shared(std::move(program)); +} + +void TestFailurePreservesPriorStage() { + using namespace Libs::Graphics; + auto cached_program = SrtProgram(0x1001); + auto prior_program = std::make_shared(); + auto prior_resources = std::make_shared(); + ShaderStageRuntime stage {prior_program, prior_resources}; + const std::array user_data = {}; + std::string error; + Check(!ShaderMaterializeStageRuntime(cached_program, user_data, 0, &stage, &error) && + error.find("failed at 0x0000000000001000") != std::string::npos, + "unmapped cache ReadConst did not fail through the checked production reader"); + Check(stage.program == prior_program && stage.resources == prior_resources, + "failed cache rematerialization replaced the prior stage snapshot"); +} + +void TestMappedSrtUsesCheckedReaderByDefault() { + using namespace Libs::Graphics; + const uint32_t dword = 0x12345678; + auto cached_program = SrtProgram(reinterpret_cast(&dword)); + ShaderStageRuntime stage; + std::string error; + Check(ShaderMaterializeStageRuntime(cached_program, {}, 0, &stage, &error), error.c_str()); + Check(stage.program == cached_program && stage.resources != nullptr, + "cache rematerialization did not publish the mapped stage"); + Check(stage.resources->flattened_srt.size() == 1 && + stage.resources->flattened_srt[0] == dword, + "cache rematerialization did not use the checked reader by default"); +} + +void TestUnbasedFlatCacheHitFailsClosed() { + using namespace Libs::Graphics; + auto cached_program = UnbasedFlatProgram(); + auto prior_program = std::make_shared(); + auto prior_resources = std::make_shared(); + ShaderStageRuntime stage {prior_program, prior_resources}; + std::string error; + Check(!ShaderMaterializeStageRuntime(cached_program, {}, 0, &stage, &error) && + error.find("requires runtime guest-address translation") != std::string::npos, + "unbased FLAT cache hit did not fail without a runtime translator"); + Check(stage.program == prior_program && stage.resources == prior_resources, + "unbased FLAT cache rejection replaced the prior stage snapshot"); +} + +} // namespace + +int main() { + TestFailurePreservesPriorStage(); + TestMappedSrtUsesCheckedReaderByDefault(); + TestUnbasedFlatCacheHitFailsClosed(); + std::puts("ShaderStageRuntimeTests: all cases passed"); + return 0; +} diff --git a/tests/ShaderVertexMetadataTests.cpp b/tests/ShaderVertexMetadataTests.cpp new file mode 100644 index 000000000..94af86ab6 --- /dev/null +++ b/tests/ShaderVertexMetadataTests.cpp @@ -0,0 +1,152 @@ +#include "graphics/shader/shaderVertexMetadata.h" + +#include +#include +#include +#include +#include + +#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS +#ifndef NOMINMAX +#define NOMINMAX +#endif +#include +#undef min +#undef max +#elif KYTY_PLATFORM == KYTY_PLATFORM_LINUX +#include +#include +#endif + +namespace { + +using namespace Libs::Graphics; + +void Check(bool value, const char* text) { + if (!value) { + std::fprintf(stderr, "ShaderVertexMetadataTests: failed: %s\n", text); + std::abort(); + } +} + +struct Fixture { + std::array(AgcDirectResourceType::Last) + 1> offsets {}; + ShaderUserData user_data {}; + ShaderSemantic semantic {}; + ShaderMappedData mapped {}; + + Fixture() { + offsets.fill(AGC_ILLEGAL_DIRECT_OFFSET); + offsets[static_cast(AgcDirectResourceType::PtrVertexBufferTable)] = 2; + offsets[static_cast(AgcDirectResourceType::PtrVertexAttribDescTable)] = 4; + user_data.direct_resource_offset = offsets.data(); + user_data.direct_resource_count = static_cast(offsets.size()); + mapped.user_data = &user_data; + mapped.input_semantics = &semantic; + mapped.num_input_semantics = 1; + } +}; + +void CheckRejected(const ShaderMappedData& data, const char* text) { + ShaderVertexMetadata output; + output.vertex_buffer_reg = 37; + output.vertex_attrib_reg = 41; + output.input_semantics_count = 7; + std::string error; + Check(!ShaderReadVertexMetadata(data, 64, &output, &error), text); + Check(!error.empty(), "metadata rejection omitted its diagnostic"); + Check(output.vertex_buffer_reg == 37 && output.vertex_attrib_reg == 41 && + output.input_semantics_count == 7, + "metadata rejection changed the prior output"); +} + +void TestValidAndHostileMetadata() { + Fixture fixture; + ShaderVertexMetadata output; + std::string error; + Check(ShaderReadVertexMetadata(fixture.mapped, 64, &output, &error), + "valid AGC vertex metadata was rejected"); + Check(output.vertex_buffer_reg == 2 && output.vertex_attrib_reg == 4 && + output.input_semantics_count == 1, + "valid AGC vertex metadata was decoded incorrectly"); + + auto unreadable = fixture.mapped; + unreadable.user_data = reinterpret_cast(uintptr_t {1}); + CheckRejected(unreadable, "unreadable AGC user-data header was accepted"); + + Fixture excessive_resources; + excessive_resources.user_data.direct_resource_count = + static_cast(excessive_resources.offsets.size() + 1); + CheckRejected(excessive_resources.mapped, "excessive direct-resource count was accepted"); + + Fixture excessive_semantics; + excessive_semantics.mapped.num_input_semantics = ShaderVertexInputInfo::RES_MAX + 1; + CheckRejected(excessive_semantics.mapped, "excessive vertex semantic count was accepted"); + + Fixture excessive_register; + excessive_register.offsets[ + static_cast(AgcDirectResourceType::PtrVertexBufferTable)] = 63; + CheckRejected(excessive_register.mapped, "out-of-domain vertex table SGPR was accepted"); + + Fixture bad_semantics; + bad_semantics.mapped.input_semantics = + reinterpret_cast(uintptr_t {1}); + CheckRejected(bad_semantics.mapped, "unreadable vertex semantic array was accepted"); +} + +void TestCrossPageMetadata() { +#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS + SYSTEM_INFO system_info {}; + GetSystemInfo(&system_info); + const auto page_size = static_cast(system_info.dwPageSize); + auto* pages = static_cast( + VirtualAlloc(nullptr, page_size * 2, MEM_RESERVE | MEM_COMMIT, PAGE_READWRITE)); + Check(pages != nullptr, "failed to allocate metadata test pages"); + DWORD old_protect = 0; + Check(VirtualProtect(pages + page_size, page_size, PAGE_NOACCESS, &old_protect) != 0, + "failed to protect metadata test page"); +#elif KYTY_PLATFORM == KYTY_PLATFORM_LINUX + const auto native_page_size = sysconf(_SC_PAGESIZE); + Check(native_page_size > 0, "failed to query page size"); + const auto page_size = static_cast(native_page_size); + auto* pages = static_cast( + mmap(nullptr, page_size * 2, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0)); + Check(pages != MAP_FAILED, "failed to allocate metadata test pages"); + Check(mprotect(pages + page_size, page_size, PROT_NONE) == 0, + "failed to protect metadata test page"); +#else + return; +#endif + + Fixture fixture; + auto header_crossing = fixture.mapped; + header_crossing.user_data = reinterpret_cast( + pages + page_size - sizeof(ShaderUserData) + 1); + CheckRejected(header_crossing, "cross-page AGC user-data header was accepted"); + + auto offsets_crossing = fixture.mapped; + fixture.user_data.direct_resource_offset = + reinterpret_cast(pages + page_size - sizeof(uint16_t)); + CheckRejected(offsets_crossing, "cross-page direct-resource offsets were accepted"); + + fixture.user_data.direct_resource_offset = fixture.offsets.data(); + auto semantics_crossing = fixture.mapped; + semantics_crossing.input_semantics = + reinterpret_cast(pages + page_size - sizeof(uint16_t)); + CheckRejected(semantics_crossing, "cross-page vertex semantics were accepted"); + +#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS + Check(VirtualFree(pages, 0, MEM_RELEASE) != 0, "failed to free metadata test pages"); +#elif KYTY_PLATFORM == KYTY_PLATFORM_LINUX + Check(munmap(pages, page_size * 2) == 0, "failed to free metadata test pages"); +#endif +} + +} // namespace + +int main() { + TestValidAndHostileMetadata(); + TestCrossPageMetadata(); + std::puts("ShaderVertexMetadataTests: all cases passed"); + return 0; +} diff --git a/tests/VirtualMemoryAllocationTests.cpp b/tests/VirtualMemoryAllocationTests.cpp new file mode 100644 index 000000000..c46cf3a63 --- /dev/null +++ b/tests/VirtualMemoryAllocationTests.cpp @@ -0,0 +1,1226 @@ +#include "common/commonSubsystem.h" +#include "common/subsystems.h" +#include "common/threads.h" + +#include "common/emulatorConfig.h" +#include "kernel/memory.h" +#include "libs/errno.h" +#include "common/logging/log.h" + +#include +#include +#include +#include +#include + +namespace { + +using Libs::LibKernel::Memory::VirtualQueryInfo; + +// Prospero ABI? +constexpr uint64_t SceKernelPageSize = 0x4000; +constexpr int SceKernelProtCpuRead = 0x01; +constexpr int SceKernelProtCpuRw = 0x02; +constexpr int SceKernelMapFixed = 0x10; +constexpr int SceKernelMapNoOverwrite = 0x80; +constexpr int SceKernelMapNoCoalesce = 0x400000; +constexpr int SceKernelVqFindNext = 1; +constexpr int SceKernelMtypeC = 11; +constexpr uint64_t SceKernelDirectMemoryStart = 0; +constexpr uint64_t SceKernelMemoryPoolReserveLen = 0x200000; +constexpr uint64_t SceKernelMemoryPoolCommitLen = 0x10000; +constexpr uint64_t SceKernelMemoryPoolExpandLen = 0x400000; +constexpr uint64_t SceKernelMemoryPoolAlignment = 0x10000; +constexpr int ErrorAccess = Libs::LibKernel::KERNEL_ERROR_EACCES; + +struct TestFailure {}; + +int g_failed_tests = 0; + +[[noreturn]] void Fail(const char* test, const std::string& message) { + std::fflush(stdout); + std::fprintf(stderr, "VirtualMemoryAllocationTests: %s failed: %s\n", test, message.c_str()); + g_failed_tests++; + throw TestFailure {}; +} + +void Check(const char* test, bool value, const std::string& message) { + if (!value) { + Fail(test, message); + } +} + +void CheckOk(const char* test, int result, const char* action) { + if (result != OK) { + char buffer[256] = {}; + std::snprintf(buffer, sizeof(buffer), "%s returned 0x%08" PRIx32, action, + static_cast(result)); + Fail(test, buffer); + } +} + +void CheckFailed(const char* test, int result, const char* action) { + if (result >= OK) { + char buffer[256] = {}; + std::snprintf(buffer, sizeof(buffer), "%s returned success, expected negative error", + action); + Fail(test, buffer); + } +} + +void InitSubsystems() { + static bool initialized = false; + if (initialized) { + return; + } + + static char arg0[] = "virtual_memory_allocation_tests"; + static char* argv[] = {arg0}; + + auto* slist = Common::SubsystemsList::Instance(); + auto* core = Common::CommonSubsystem::Instance(); + auto* config = Config::ConfigSubsystem::Instance(); + auto* log = Log::LogSubsystem::Instance(); + auto* memory = Libs::LibKernel::Memory::MemorySubsystem::Instance(); + auto* thread = Common::ThreadsSubsystem::Instance(); + + slist->SetArgs(1, argv); + slist->Add(thread, {}); + slist->Add(core, {}); + slist->Add(config, {core}); + Check("InitSubsystems", slist->InitAll(false), "failed to initialize base subsystems"); + + Config::ConfigOptions options; + options.printf_direction = Config::OutputDirection::Silent; + Config::Load(options); + + slist->Add(log, {core, config}); + slist->Add(memory, {core, log, thread}); + Check("InitSubsystems", slist->InitAll(false), "failed to initialize memory subsystem"); + + initialized = true; +} + +void RunTest(void (*test_func)()) { + if (g_failed_tests != 0) { + return; + } + try { + test_func(); + } catch (const TestFailure&) { + } +} + +VirtualQueryInfo Query(const char* test, uint64_t addr, int flags = 0) { + VirtualQueryInfo info {}; + const int ret = Libs::LibKernel::Memory::KernelVirtualQuery(reinterpret_cast(addr), + flags, &info, sizeof(info)); + CheckOk(test, ret, "KernelVirtualQuery"); + return info; +} + +int QueryResult(uint64_t addr, int flags = 0) { + VirtualQueryInfo info {}; + return Libs::LibKernel::Memory::KernelVirtualQuery(reinterpret_cast(addr), flags, + &info, sizeof(info)); +} + +size_t AvailableFlexibleMemory(const char* test) { + size_t size = 0; + const int ret = Libs::LibKernel::Memory::KernelAvailableFlexibleMemorySize(&size); + CheckOk(test, ret, "KernelAvailableFlexibleMemorySize"); + return size; +} + +uint64_t MapNamedFlexible(const char* test, uint64_t size, int prot, const char* name) { + void* addr = nullptr; + const int ret = + Libs::LibKernel::Memory::KernelMapNamedFlexibleMemory(&addr, size, prot, 0, name); + CheckOk(test, ret, "KernelMapNamedFlexibleMemory"); + Check(test, addr != nullptr, "flexible mapping returned null"); + return reinterpret_cast(addr); +} + +void ExpectRange(const char* test, const VirtualQueryInfo& info, uint64_t start, uint64_t end, + int prot, uint32_t flexible, uint32_t direct, uint32_t pooled, uint32_t committed, + const char* name = nullptr, uint64_t offset = 0) { + Check(test, info.start == start, "unexpected range start"); + Check(test, info.end == end, "unexpected range end"); + Check(test, info.protection == prot, "unexpected range protection"); + Check(test, info.is_flexible == flexible, "unexpected flexible flag"); + Check(test, info.is_direct == direct, "unexpected direct flag"); + Check(test, info.is_pooled == pooled, "unexpected pooled flag"); + Check(test, info.is_committed == committed, "unexpected committed flag"); + Check(test, info.offset == offset, "unexpected range offset"); + if (name != nullptr) { + Check(test, + std::strncmp(info.name, name, Libs::LibKernel::Memory::KERNEL_MAXIMUM_NAME_LENGTH) == + 0, + "unexpected range name"); + } +} + +void ExpectUnmapped(const char* test, uint64_t addr) { + const int ret = QueryResult(addr); + if (ret != ErrorAccess) { + char buffer[256] = {}; + std::snprintf(buffer, sizeof(buffer), "KernelVirtualQuery(unmapped) returned 0x%08" PRIx32, + static_cast(ret)); + Fail(test, buffer); + } +} + +void TestProsperoArgumentAndInfoSizeContracts() { + const char* test = "ProsperoArgumentAndInfoSizeContracts"; + void* addr = nullptr; + + Check(test, sizeof(VirtualQueryInfo) == 72, "SceKernelVirtualQueryInfo layout drifted"); + CheckFailed(test, + Libs::LibKernel::Memory::KernelMapNamedFlexibleMemory(&addr, 0, SceKernelProtCpuRw, + 0, "zero_len"), + "KernelMapNamedFlexibleMemory(len=0)"); + CheckFailed(test, QueryResult(0), "KernelVirtualQuery(null)"); + + VirtualQueryInfo info {}; + CheckFailed(test, + Libs::LibKernel::Memory::KernelVirtualQuery(nullptr, 0, &info, sizeof(info) - 1), + "KernelVirtualQuery(short info)"); + CheckFailed(test, Libs::LibKernel::Memory::KernelVirtualQuery(nullptr, 2, &info, sizeof(info)), + "KernelVirtualQuery(unknown flags)"); + + std::printf("[host] %-48s ok\n", test); +} + +void TestFlexibleMapQueryAndWholeMunmap() { + const char* test = "FlexibleMapQueryAndWholeMunmap"; + const auto baseline = AvailableFlexibleMemory(test); + const auto size = SceKernelPageSize * 2; + const auto base = MapNamedFlexible(test, size, SceKernelProtCpuRw, "prospero_flex"); + + ExpectRange(test, Query(test, base), base, base + size, SceKernelProtCpuRw, 1, 0, 0, 1, + "prospero_flex"); + Check(test, AvailableFlexibleMemory(test) + size == baseline, + "flexible allocation should consume Prospero-reported flexible budget"); + + CheckOk(test, Libs::LibKernel::Memory::KernelMunmap(base, size), "KernelMunmap"); + ExpectUnmapped(test, base); + Check(test, AvailableFlexibleMemory(test) == baseline, + "whole munmap should return flexible memory to Prospero-reported budget"); + + std::printf("[host] %-48s ok\n", test); +} + +void TestPartialFlexibleMunmapAndFindNext() { + const char* test = "PartialFlexibleMunmapAndFindNext"; + const auto baseline = AvailableFlexibleMemory(test); + const auto base = MapNamedFlexible(test, SceKernelPageSize * 3, SceKernelProtCpuRw, "prospero_part"); + + CheckOk(test, + Libs::LibKernel::Memory::KernelMunmap(base + SceKernelPageSize, SceKernelPageSize), + "KernelMunmap(middle page)"); + + ExpectRange(test, Query(test, base), base, base + SceKernelPageSize, SceKernelProtCpuRw, 1, 0, + 0, 1, "prospero_part"); + ExpectUnmapped(test, base + SceKernelPageSize); + ExpectRange(test, Query(test, base + SceKernelPageSize, SceKernelVqFindNext), + base + SceKernelPageSize * 2, base + SceKernelPageSize * 3, SceKernelProtCpuRw, 1, + 0, 0, 1, "prospero_part"); + Check(test, AvailableFlexibleMemory(test) + SceKernelPageSize * 2 == baseline, + "partial munmap should return only the unmapped flexible page"); + + CheckOk(test, Libs::LibKernel::Memory::KernelMunmap(base, SceKernelPageSize), + "KernelMunmap(left cleanup)"); + CheckOk(test, + Libs::LibKernel::Memory::KernelMunmap(base + SceKernelPageSize * 2, SceKernelPageSize), + "KernelMunmap(right cleanup)"); + Check(test, AvailableFlexibleMemory(test) == baseline, + "cleanup should return all flexible memory to Prospero-reported budget"); + + std::printf("[host] %-48s ok\n", test); +} + +void TestReserveMapFixedAndNoOverwrite() { + const char* test = "ReserveMapFixedAndNoOverwrite"; + void* addr = nullptr; + + CheckOk(test, + Libs::LibKernel::Memory::KernelReserveVirtualRange(&addr, SceKernelPageSize * 3, 0, + SceKernelPageSize), + "KernelReserveVirtualRange"); + const auto base = reinterpret_cast(addr); + + ExpectRange(test, Query(test, base), base, base + SceKernelPageSize * 3, 0, 0, 0, 0, 0); + + void* fixed = reinterpret_cast(base + SceKernelPageSize); + CheckOk(test, + Libs::LibKernel::Memory::KernelMapNamedFlexibleMemory( + &fixed, SceKernelPageSize, SceKernelProtCpuRead, SceKernelMapFixed, "fixed_mid"), + "KernelMapNamedFlexibleMemory(fixed)"); + Check(test, reinterpret_cast(fixed) == base + SceKernelPageSize, + "MAP_FIXED mapping moved"); + + ExpectRange(test, Query(test, base), base, base + SceKernelPageSize, 0, 0, 0, 0, 0); + ExpectRange(test, Query(test, base + SceKernelPageSize), base + SceKernelPageSize, + base + SceKernelPageSize * 2, SceKernelProtCpuRead, 1, 0, 0, 1, "fixed_mid"); + ExpectRange(test, Query(test, base + SceKernelPageSize * 2), base + SceKernelPageSize * 2, + base + SceKernelPageSize * 3, 0, 0, 0, 0, 0); + + void* blocked = reinterpret_cast(base + SceKernelPageSize); + CheckFailed(test, + Libs::LibKernel::Memory::KernelMapNamedFlexibleMemory( + &blocked, SceKernelPageSize, SceKernelProtCpuRw, + SceKernelMapFixed | SceKernelMapNoOverwrite, "blocked"), + "KernelMapNamedFlexibleMemory(MAP_FIXED|MAP_NO_OVERWRITE)"); + ExpectRange(test, Query(test, base + SceKernelPageSize), base + SceKernelPageSize, + base + SceKernelPageSize * 2, SceKernelProtCpuRead, 1, 0, 0, 1, "fixed_mid"); + + CheckOk(test, Libs::LibKernel::Memory::KernelMunmap(base, SceKernelPageSize), + "KernelMunmap(left reserve cleanup)"); + CheckOk(test, + Libs::LibKernel::Memory::KernelMunmap(base + SceKernelPageSize, SceKernelPageSize), + "KernelMunmap(fixed cleanup)"); + CheckOk(test, + Libs::LibKernel::Memory::KernelMunmap(base + SceKernelPageSize * 2, SceKernelPageSize), + "KernelMunmap(right reserve cleanup)"); + + std::printf("[host] %-48s ok\n", test); +} + +void TestFixedNoOverwriteRejectsReservedRange() { + const char* test = "FixedNoOverwriteRejectsReservedRange"; + void* addr = nullptr; + + CheckOk(test, + Libs::LibKernel::Memory::KernelReserveVirtualRange(&addr, SceKernelPageSize, 0, + SceKernelPageSize), + "KernelReserveVirtualRange"); + const auto base = reinterpret_cast(addr); + + ExpectRange(test, Query(test, base), base, base + SceKernelPageSize, 0, 0, 0, 0, 0); + + void* fixed = reinterpret_cast(base); + const int ret = Libs::LibKernel::Memory::KernelMapNamedFlexibleMemory( + &fixed, SceKernelPageSize, SceKernelProtCpuRw, SceKernelMapFixed | SceKernelMapNoOverwrite, + "reserved_blocked"); + const bool rejected = ret < OK; + + if (ret == OK) { + CheckOk(test, + Libs::LibKernel::Memory::KernelMunmap(reinterpret_cast(fixed), + SceKernelPageSize), + "KernelMunmap(unexpected fixed map cleanup)"); + if (reinterpret_cast(fixed) != base) { + CheckOk(test, Libs::LibKernel::Memory::KernelMunmap(base, SceKernelPageSize), + "KernelMunmap(reserve cleanup)"); + } + } else { + CheckOk(test, Libs::LibKernel::Memory::KernelMunmap(base, SceKernelPageSize), + "KernelMunmap(reserve cleanup)"); + } + + Check(test, rejected, + "MAP_FIXED|MAP_NO_OVERWRITE should reject an already reserved virtual range"); + + std::printf("[host] %-48s ok\n", test); +} + +void TestDirectMapQueryOffsetAndPartialMunmap() { + const char* test = "DirectMapQueryOffsetAndPartialMunmap"; + + int64_t phys_addr = 0; + CheckOk(test, + Libs::LibKernel::Memory::KernelAllocateDirectMemory( + SceKernelDirectMemoryStart, Libs::LibKernel::Memory::KernelGetDirectMemorySize(), + SceKernelPageSize * 4, SceKernelPageSize, SceKernelMtypeC, &phys_addr), + "KernelAllocateDirectMemory"); + + void* addr = nullptr; + CheckOk(test, + Libs::LibKernel::Memory::KernelMapNamedDirectMemory(&addr, SceKernelPageSize * 4, + SceKernelProtCpuRw, 0, phys_addr, + SceKernelPageSize, "prospero_direct"), + "KernelMapNamedDirectMemory"); + const auto base = reinterpret_cast(addr); + const auto phys = static_cast(phys_addr); + void* alias = nullptr; + CheckOk(test, + Libs::LibKernel::Memory::KernelMapNamedDirectMemory( + &alias, SceKernelPageSize * 4, SceKernelProtCpuRw, 0, phys_addr, + SceKernelPageSize, "prospero_direct_alias"), + "KernelMapNamedDirectMemory(alias)"); + const auto alias_base = reinterpret_cast(alias); + + constexpr uint64_t alias_test_value = 0x4b595459444d454dull; // "KYTYDMEM" + *reinterpret_cast(base) = alias_test_value; + Check(test, *reinterpret_cast(alias_base) == alias_test_value, + "direct mappings of the same physical offset must share backing storage"); + uint64_t backing_read = 0; + Check(test, Libs::LibKernel::Memory::TryReadBacking(base, &backing_read, sizeof(backing_read)), + "TryReadBacking should resolve a direct mapping"); + Check(test, backing_read == alias_test_value, + "TryReadBacking should observe the physical backing bytes"); + constexpr uint64_t backing_write = 0x524541444241434bull; // "READBACK" + Check(test, + Libs::LibKernel::Memory::TryWriteBacking(alias_base + sizeof(uint64_t), &backing_write, + sizeof(backing_write)), + "TryWriteBacking should resolve a direct alias"); + backing_read = 0; + Check(test, + Libs::LibKernel::Memory::TryReadBacking(base + sizeof(uint64_t), &backing_read, + sizeof(backing_read)), + "TryReadBacking should resolve an aliased physical offset"); + Check(test, backing_read == backing_write, + "backing reads and writes should preserve direct-memory aliasing"); + + auto info = Query(test, base); + ExpectRange(test, info, base, base + SceKernelPageSize * 4, SceKernelProtCpuRw, 0, 1, 0, 1, + "prospero_direct", phys); + Check(test, info.memory_type == SceKernelMtypeC, "unexpected direct memory type"); + + CheckOk(test, + Libs::LibKernel::Memory::KernelMunmap(base + SceKernelPageSize, SceKernelPageSize), + "KernelMunmap(direct middle page)"); + ExpectUnmapped(test, base + SceKernelPageSize); + + constexpr uint64_t transaction_sentinel = 0x5452414e53414354ull; // "TRANSACT" + constexpr uint64_t rejected_write = 0x4e4f504152544941ull; // "NOPARTIA" + const auto crossing_address = base + SceKernelPageSize - sizeof(uint32_t); + std::memcpy(reinterpret_cast(alias_base + SceKernelPageSize - sizeof(uint32_t)), + &transaction_sentinel, sizeof(transaction_sentinel)); + Check(test, + !Libs::LibKernel::Memory::TryWriteBacking(crossing_address, &rejected_write, + sizeof(rejected_write)), + "TryWriteBacking should reject a range crossing an unmapped span"); + uint64_t backing_after_rejected_write = 0; + std::memcpy(&backing_after_rejected_write, + reinterpret_cast(alias_base + SceKernelPageSize - sizeof(uint32_t)), + sizeof(backing_after_rejected_write)); + Check(test, backing_after_rejected_write == transaction_sentinel, + "failed backing writes must not modify a validated prefix"); + uint64_t rejected_read = transaction_sentinel; + Check(test, + !Libs::LibKernel::Memory::TryReadBacking(crossing_address, &rejected_read, + sizeof(rejected_read)), + "TryReadBacking should reject a range crossing an unmapped span"); + Check(test, rejected_read == transaction_sentinel, + "failed backing reads must not modify a destination prefix"); + + info = Query(test, base + SceKernelPageSize, SceKernelVqFindNext); + ExpectRange(test, info, base + SceKernelPageSize * 2, base + SceKernelPageSize * 4, + SceKernelProtCpuRw, 0, 1, 0, 1, "prospero_direct", phys + SceKernelPageSize * 2); + Check(test, info.memory_type == SceKernelMtypeC, "unexpected right direct memory type"); + + CheckOk(test, Libs::LibKernel::Memory::KernelMunmap(base, SceKernelPageSize), + "KernelMunmap(direct left cleanup)"); + CheckOk( + test, + Libs::LibKernel::Memory::KernelMunmap(base + SceKernelPageSize * 2, SceKernelPageSize * 2), + "KernelMunmap(direct right cleanup)"); + CheckOk(test, + Libs::LibKernel::Memory::KernelMunmap(alias_base, SceKernelPageSize * 4), + "KernelMunmap(direct alias cleanup)"); + CheckOk(test, + Libs::LibKernel::Memory::KernelReleaseDirectMemory(phys_addr, SceKernelPageSize * 4), + "KernelReleaseDirectMemory"); + + std::printf("[host] %-48s ok\n", test); +} + +void TestReleasedReserveCanBeReused() { + const char* test = "ReleasedReserveCanBeReused"; + void* addr = nullptr; + + CheckOk(test, + Libs::LibKernel::Memory::KernelReserveVirtualRange(&addr, SceKernelPageSize, 0, + SceKernelPageSize), + "KernelReserveVirtualRange"); + const auto base = reinterpret_cast(addr); + CheckOk(test, Libs::LibKernel::Memory::KernelMunmap(base, SceKernelPageSize), + "KernelMunmap"); + + void* reused = reinterpret_cast(base); + CheckOk(test, + Libs::LibKernel::Memory::KernelReserveVirtualRange( + &reused, SceKernelPageSize, SceKernelMapFixed | SceKernelMapNoOverwrite, + SceKernelPageSize), + "KernelReserveVirtualRange(reuse)"); + Check(test, reinterpret_cast(reused) == base, + "released host reservation was not reusable at the same address"); + CheckOk(test, Libs::LibKernel::Memory::KernelMunmap(base, SceKernelPageSize), + "KernelMunmap(reuse cleanup)"); + + std::printf("[host] %-48s ok\n", test); +} + +void TestMunmapAcrossAdjacentFlexibleMappings() { + const char* test = "MunmapAcrossAdjacentFlexibleMappings"; + const auto baseline = AvailableFlexibleMemory(test); + void* reserve = nullptr; + + CheckOk(test, + Libs::LibKernel::Memory::KernelReserveVirtualRange( + &reserve, SceKernelPageSize * 2, 0, SceKernelPageSize), + "KernelReserveVirtualRange"); + const auto base = reinterpret_cast(reserve); + + void* left = reinterpret_cast(base); + void* right = reinterpret_cast(base + SceKernelPageSize); + CheckOk(test, + Libs::LibKernel::Memory::KernelMapNamedFlexibleMemory( + &left, SceKernelPageSize, SceKernelProtCpuRw, SceKernelMapFixed, "adjacent_left"), + "KernelMapNamedFlexibleMemory(left)"); + CheckOk(test, + Libs::LibKernel::Memory::KernelMapNamedFlexibleMemory( + &right, SceKernelPageSize, SceKernelProtCpuRw, SceKernelMapFixed, + "adjacent_right"), + "KernelMapNamedFlexibleMemory(right)"); + + CheckOk(test, Libs::LibKernel::Memory::KernelMunmap(base, SceKernelPageSize * 2), + "KernelMunmap(adjacent mappings)"); + Check(test, AvailableFlexibleMemory(test) == baseline, + "multi-range unmap leaked flexible-memory budget"); + ExpectRange(test, Query(test, base), base, base + SceKernelPageSize, 0, 0, 0, 0, 0, + "adjacent_left"); + ExpectRange(test, Query(test, base + SceKernelPageSize), base + SceKernelPageSize, + base + SceKernelPageSize * 2, 0, 0, 0, 0, 0, "adjacent_right"); + + CheckOk(test, Libs::LibKernel::Memory::KernelMunmap(base, SceKernelPageSize * 2), + "KernelMunmap(restored reserve)"); + + std::printf("[host] %-48s ok\n", test); +} + +void TestNonzeroDirectOffsetAliasesSharedBacking() { + const char* test = "NonzeroDirectOffsetAliasesSharedBacking"; + + int64_t first = 0; + int64_t second = 0; + CheckOk(test, + Libs::LibKernel::Memory::KernelAllocateDirectMemory( + 0, Libs::LibKernel::Memory::KernelGetDirectMemorySize(), SceKernelPageSize, + SceKernelPageSize, SceKernelMtypeC, &first), + "KernelAllocateDirectMemory(first)"); + CheckOk(test, + Libs::LibKernel::Memory::KernelAllocateDirectMemory( + 0, Libs::LibKernel::Memory::KernelGetDirectMemorySize(), SceKernelPageSize, + SceKernelPageSize, SceKernelMtypeC, &second), + "KernelAllocateDirectMemory(second)"); + Check(test, second == first + static_cast(SceKernelPageSize), + "second allocation should use a nonzero 16 KiB offset"); + + void* first_alias = nullptr; + void* second_alias = nullptr; + CheckOk(test, + Libs::LibKernel::Memory::KernelMapNamedDirectMemory( + &first_alias, SceKernelPageSize, SceKernelProtCpuRw, 0, second, + SceKernelPageSize, "prospero_nonzero_a"), + "KernelMapNamedDirectMemory(first alias)"); + CheckOk(test, + Libs::LibKernel::Memory::KernelMapNamedDirectMemory( + &second_alias, SceKernelPageSize, SceKernelProtCpuRw, 0, second, + SceKernelPageSize, "prospero_nonzero_b"), + "KernelMapNamedDirectMemory(second alias)"); + + *reinterpret_cast(first_alias) = 0x4b59545931364b42ull; // "KYTY16KB" + Check(test, *reinterpret_cast(second_alias) == 0x4b59545931364b42ull, + "nonzero-offset mappings must share backing storage"); + + CheckOk(test, Libs::LibKernel::Memory::KernelMunmap( + reinterpret_cast(first_alias), SceKernelPageSize), + "KernelMunmap(first alias)"); + CheckOk(test, Libs::LibKernel::Memory::KernelMunmap( + reinterpret_cast(second_alias), SceKernelPageSize), + "KernelMunmap(second alias)"); + CheckOk(test, Libs::LibKernel::Memory::KernelReleaseDirectMemory(second, SceKernelPageSize), + "KernelReleaseDirectMemory(second)"); + CheckOk(test, Libs::LibKernel::Memory::KernelReleaseDirectMemory(first, SceKernelPageSize), + "KernelReleaseDirectMemory(first)"); + + std::printf("[host] %-48s ok\n", test); +} + +void TestDirectPhysicalFreeRangeReuseAndCoalescing() { + const char* test = "DirectPhysicalFreeRangeReuseAndCoalescing"; + const auto end = Libs::LibKernel::Memory::KernelGetDirectMemorySize(); + + int64_t first = 0; + CheckOk(test, + Libs::LibKernel::Memory::KernelAllocateDirectMemory( + 0, end, SceKernelPageSize * 3, SceKernelPageSize, SceKernelMtypeC, &first), + "KernelAllocateDirectMemory(first)"); + const auto middle = first + static_cast(SceKernelPageSize); + const auto last = middle + static_cast(SceKernelPageSize); + + CheckOk(test, Libs::LibKernel::Memory::KernelReleaseDirectMemory(middle, SceKernelPageSize), + "KernelReleaseDirectMemory(middle split)"); + int64_t reused = 0; + CheckOk(test, + Libs::LibKernel::Memory::KernelAllocateDirectMemory( + 0, end, SceKernelPageSize, SceKernelPageSize, SceKernelMtypeC, &reused), + "KernelAllocateDirectMemory(reused)"); + Check(test, reused == middle, "released physical gap was not reused"); + + CheckOk(test, Libs::LibKernel::Memory::KernelReleaseDirectMemory(first, SceKernelPageSize), + "KernelReleaseDirectMemory(left split)"); + CheckOk(test, Libs::LibKernel::Memory::KernelReleaseDirectMemory(reused, SceKernelPageSize), + "KernelReleaseDirectMemory(reused)"); + CheckOk(test, Libs::LibKernel::Memory::KernelReleaseDirectMemory(last, SceKernelPageSize), + "KernelReleaseDirectMemory(right split)"); + + int64_t coalesced = 0; + CheckOk(test, + Libs::LibKernel::Memory::KernelAllocateDirectMemory( + 0, end, SceKernelPageSize * 3, SceKernelPageSize, SceKernelMtypeC, &coalesced), + "KernelAllocateDirectMemory(coalesced)"); + Check(test, coalesced == first, "adjacent released physical ranges were not coalesced"); + CheckOk(test, + Libs::LibKernel::Memory::KernelReleaseDirectMemory(coalesced, SceKernelPageSize * 3), + "KernelReleaseDirectMemory(coalesced)"); + + std::printf("[host] %-48s ok\n", test); +} + +void TestDirectAlignmentStaysWithinSearchRange() { + const char* test = "DirectAlignmentStaysWithinSearchRange"; + constexpr int64_t search_start = SceKernelPageSize * 2; + constexpr uint64_t alignment = SceKernelPageSize * 3; + const auto search_end = Libs::LibKernel::Memory::KernelGetDirectMemorySize(); + + int64_t phys_addr = -1; + CheckOk(test, + Libs::LibKernel::Memory::KernelAllocateDirectMemory( + search_start, search_end, SceKernelPageSize, alignment, SceKernelMtypeC, + &phys_addr), + "KernelAllocateDirectMemory(non-power-of-two alignment)"); + Check(test, phys_addr >= search_start, "aligned allocation escaped below search_start"); + Check(test, static_cast(phys_addr) % alignment == 0, + "allocation did not honor the requested alignment"); + CheckOk(test, Libs::LibKernel::Memory::KernelReleaseDirectMemory(phys_addr, SceKernelPageSize), + "KernelReleaseDirectMemory"); + + constexpr size_t out_of_range_alignment = UINT64_MAX - (SceKernelPageSize - 1); + phys_addr = -1; + const int result = Libs::LibKernel::Memory::KernelAllocateDirectMemory( + search_start, search_end, SceKernelPageSize, out_of_range_alignment, SceKernelMtypeC, + &phys_addr); + CheckFailed(test, result, "KernelAllocateDirectMemory(out-of-range alignment)"); + Check(test, phys_addr == -1, "failed allocation modified physAddrOut"); + + std::printf("[host] %-48s ok\n", test); +} + +void TestDefaultDirectMapUsesSystemAddressRange() { + const char* test = "DefaultDirectMapUsesSystemAddressRange"; + + int64_t phys_addr = 0; + CheckOk(test, + Libs::LibKernel::Memory::KernelAllocateDirectMemory( + 0, Libs::LibKernel::Memory::KernelGetDirectMemorySize(), SceKernelPageSize, + SceKernelPageSize, SceKernelMtypeC, &phys_addr), + "KernelAllocateDirectMemory"); + + void* address = nullptr; + CheckOk(test, + Libs::LibKernel::Memory::KernelMapNamedDirectMemory( + &address, SceKernelPageSize, SceKernelProtCpuRw, 0, phys_addr, + SceKernelPageSize, "system_direct"), + "KernelMapNamedDirectMemory"); + Check(test, address != nullptr, "direct mapping returned null"); +#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS + constexpr uint64_t SystemManagedMin = 0x0000040000ull; + constexpr uint64_t SystemManagedMax = 0x07fffeffffull; + const auto mapped = reinterpret_cast(address); + Check(test, mapped >= SystemManagedMin && + mapped + SceKernelPageSize - 1 <= SystemManagedMax, + "default direct mapping fell outside the system-managed host range"); +#endif + + CheckOk(test, + Libs::LibKernel::Memory::KernelMunmap(reinterpret_cast(address), + SceKernelPageSize), + "KernelMunmap"); + CheckOk(test, + Libs::LibKernel::Memory::KernelReleaseDirectMemory(phys_addr, SceKernelPageSize), + "KernelReleaseDirectMemory"); + + std::printf("[host] %-48s ok\n", test); +} + +void TestLargeDirectMapAliasesAcrossChunks() { + const char* test = "LargeDirectMapAliasesAcrossChunks"; + constexpr uint64_t size = 0x400000; + constexpr uint64_t boundary = 0x200000; + + int64_t phys_addr = 0; + CheckOk(test, + Libs::LibKernel::Memory::KernelAllocateDirectMemory( + 0, Libs::LibKernel::Memory::KernelGetDirectMemorySize(), size, 0x10000, + SceKernelMtypeC, &phys_addr), + "KernelAllocateDirectMemory"); + + void* first_alias = nullptr; + void* second_alias = nullptr; + CheckOk(test, + Libs::LibKernel::Memory::KernelMapNamedDirectMemory( + &first_alias, size, SceKernelProtCpuRw, 0, phys_addr, 0x10000, "large_direct_a"), + "KernelMapNamedDirectMemory(first alias)"); + CheckOk(test, + Libs::LibKernel::Memory::KernelMapNamedDirectMemory( + &second_alias, size, SceKernelProtCpuRw, 0, phys_addr, 0x10000, "large_direct_b"), + "KernelMapNamedDirectMemory(second alias)"); + + auto* first = static_cast(first_alias); + auto* second = static_cast(second_alias); + *reinterpret_cast(first) = 0x1111222233334444ull; + *reinterpret_cast(first + boundary - 8) = 0x5555666677778888ull; + *reinterpret_cast(first + boundary) = 0x9999aaaabbbbccccull; + *reinterpret_cast(first + size - 8) = 0xddddeeeeffff0001ull; + Check(test, *reinterpret_cast(second) == 0x1111222233334444ull && + *reinterpret_cast(second + boundary - 8) == + 0x5555666677778888ull && + *reinterpret_cast(second + boundary) == + 0x9999aaaabbbbccccull && + *reinterpret_cast(second + size - 8) == + 0xddddeeeeffff0001ull, + "large direct aliases diverged at a mapping chunk boundary"); + + CheckOk(test, + Libs::LibKernel::Memory::KernelMunmap(reinterpret_cast(first_alias), size), + "KernelMunmap(first alias)"); + CheckOk(test, + Libs::LibKernel::Memory::KernelMunmap(reinterpret_cast(second_alias), size), + "KernelMunmap(second alias)"); + CheckOk(test, Libs::LibKernel::Memory::KernelReleaseDirectMemory(phys_addr, size), + "KernelReleaseDirectMemory"); + + std::printf("[host] %-48s ok\n", test); +} + +void TestDirectMapUnmapReusesHostAddress() { + const char* test = "DirectMapUnmapReusesHostAddress"; + + int64_t phys_addr = 0; + CheckOk(test, + Libs::LibKernel::Memory::KernelAllocateDirectMemory( + SceKernelDirectMemoryStart, Libs::LibKernel::Memory::KernelGetDirectMemorySize(), + SceKernelPageSize, SceKernelPageSize, SceKernelMtypeC, &phys_addr), + "KernelAllocateDirectMemory"); + + uint64_t first_address = 0; + for (int iteration = 0; iteration < 64; iteration++) { + void* address = nullptr; + CheckOk(test, + Libs::LibKernel::Memory::KernelMapNamedDirectMemory( + &address, SceKernelPageSize, SceKernelProtCpuRw, 0, phys_addr, + SceKernelPageSize, "reuse_direct"), + "KernelMapNamedDirectMemory"); + const auto current_address = reinterpret_cast(address); + if (iteration == 0) { + first_address = current_address; + } else { + char message[160] = {}; + std::snprintf(message, sizeof(message), + "direct map address changed from 0x%016" PRIx64 " to 0x%016" PRIx64, + first_address, current_address); + Check(test, current_address == first_address, message); + } + CheckOk(test, Libs::LibKernel::Memory::KernelMunmap(current_address, SceKernelPageSize), + "KernelMunmap"); + } + + CheckOk(test, + Libs::LibKernel::Memory::KernelReleaseDirectMemory(phys_addr, SceKernelPageSize), + "KernelReleaseDirectMemory"); + + std::printf("[host] %-48s ok\n", test); +} + +void TestLargeHintedReserveHostsSmallDirectMap() { + const char* test = "LargeHintedReserveHostsSmallDirectMap"; + + constexpr uint64_t arena_base = 0x1000000000ull; + constexpr uint64_t arena_size = 0x04000000ull; + constexpr uint64_t window_size = 0x00200000ull; + + void* arena = reinterpret_cast(arena_base); + CheckOk(test, Libs::LibKernel::Memory::KernelReserveVirtualRange( + &arena, arena_size, 0, 0x200000), + "KernelReserveVirtualRange(arena)"); + const auto actual_arena = reinterpret_cast(arena); + Check(test, actual_arena >= arena_base && (actual_arena & (0x200000 - 1u)) == 0, + "large hinted reserve violated its search start or alignment"); + + void* window = reinterpret_cast(arena_base); + CheckOk(test, Libs::LibKernel::Memory::KernelReserveVirtualRange( + &window, window_size, 0, SceKernelPageSize), + "KernelReserveVirtualRange(window)"); + Check(test, reinterpret_cast(window) >= actual_arena + arena_size, + "second hinted reserve overlaps the large arena"); + + int64_t phys = 0; + CheckOk(test, + Libs::LibKernel::Memory::KernelAllocateDirectMemory( + 0, Libs::LibKernel::Memory::KernelGetDirectMemorySize(), SceKernelPageSize * 2, + SceKernelPageSize, SceKernelMtypeC, &phys), + "KernelAllocateDirectMemory"); + + void* mapped = window; + CheckOk(test, + Libs::LibKernel::Memory::KernelMapNamedDirectMemory( + &mapped, SceKernelPageSize * 2, SceKernelProtCpuRw, + SceKernelMapFixed | SceKernelMapNoCoalesce, phys, 0, "prospero_large_reserve"), + "KernelMapNamedDirectMemory"); + Check(test, mapped == window, "fixed direct mapping moved away from the reserved window"); + *reinterpret_cast(mapped) = 0x4b59545952455356ull; // "KYTYRESV" + + CheckOk(test, Libs::LibKernel::Memory::KernelMunmap( + reinterpret_cast(mapped), SceKernelPageSize * 2), + "KernelMunmap(direct)"); + CheckOk(test, Libs::LibKernel::Memory::KernelReleaseDirectMemory( + phys, SceKernelPageSize * 2), + "KernelReleaseDirectMemory"); + CheckOk(test, Libs::LibKernel::Memory::KernelMunmap( + reinterpret_cast(window), window_size), + "KernelMunmap(window reserve)"); + CheckOk(test, Libs::LibKernel::Memory::KernelMunmap( + reinterpret_cast(arena), arena_size), + "KernelMunmap(arena reserve)"); + + std::printf("[host] %-48s ok\n", test); +} + +void TestMemoryPoolAlignmentContracts() { + const char* test = "MemoryPoolAlignmentContracts"; + void* addr = nullptr; + + CheckFailed( + test, + Libs::LibKernel::Memory::KernelMemoryPoolReserve(nullptr, SceKernelPageSize, 0, 0, &addr), + "KernelMemoryPoolReserve(16KiB len)"); + + CheckOk(test, + Libs::LibKernel::Memory::KernelMemoryPoolReserve(nullptr, SceKernelMemoryPoolReserveLen, + 0, 0, &addr), + "KernelMemoryPoolReserve"); + const auto base = reinterpret_cast(addr); + + CheckFailed(test, + Libs::LibKernel::Memory::KernelMemoryPoolCommit(reinterpret_cast(base), + SceKernelPageSize, SceKernelMtypeC, + SceKernelProtCpuRw, 0), + "KernelMemoryPoolCommit(16KiB len)"); + CheckFailed(test, + Libs::LibKernel::Memory::KernelMemoryPoolDecommit(reinterpret_cast(base), + SceKernelPageSize, 0), + "KernelMemoryPoolDecommit(16KiB len)"); + + CheckOk(test, Libs::LibKernel::Memory::KernelMunmap(base, SceKernelMemoryPoolReserveLen), + "KernelMunmap(pool reserve cleanup)"); + + std::printf("[host] %-48s ok\n", test); +} + +void TestProsperoSampleMemoryPoolExpandCommit() { + const char* test = "ProsperoSampleMemoryPoolExpandCommit"; + + int64_t pool_offset = -1; + CheckFailed(test, + Libs::LibKernel::Memory::KernelMemoryPoolExpand( + 0, Libs::LibKernel::Memory::KernelGetDirectMemorySize(), SceKernelPageSize, + SceKernelMemoryPoolAlignment, &pool_offset), + "KernelMemoryPoolExpand(16KiB len)"); + CheckFailed(test, + Libs::LibKernel::Memory::KernelMemoryPoolExpand( + 0, Libs::LibKernel::Memory::KernelGetDirectMemorySize(), + SceKernelMemoryPoolExpandLen, SceKernelPageSize, &pool_offset), + "KernelMemoryPoolExpand(16KiB alignment)"); + CheckFailed(test, + Libs::LibKernel::Memory::KernelMemoryPoolExpand( + 0, Libs::LibKernel::Memory::KernelGetDirectMemorySize(), + SceKernelMemoryPoolExpandLen, SceKernelMemoryPoolAlignment * 3, &pool_offset), + "KernelMemoryPoolExpand(non-power-of-two alignment)"); + + CheckOk(test, + Libs::LibKernel::Memory::KernelMemoryPoolExpand( + 0, Libs::LibKernel::Memory::KernelGetDirectMemorySize(), + SceKernelMemoryPoolExpandLen, SceKernelMemoryPoolAlignment, &pool_offset), + "KernelMemoryPoolExpand"); + Check(test, + pool_offset >= 0 && + (static_cast(pool_offset) & (SceKernelMemoryPoolAlignment - 1u)) == 0, + "expanded physical range is not 64 KiB aligned"); + void* direct_alias = nullptr; + CheckFailed(test, + Libs::LibKernel::Memory::KernelMapDirectMemory( + &direct_alias, SceKernelMemoryPoolCommitLen, SceKernelProtCpuRw, 0, pool_offset, + SceKernelMemoryPoolAlignment), + "KernelMapDirectMemory(pool expansion)"); + + Libs::LibKernel::Memory::KernelMemoryPoolBlockStats stats {}; + CheckOk(test, Libs::LibKernel::Memory::KernelMemoryPoolGetBlockStats(&stats, sizeof(stats)), + "KernelMemoryPoolGetBlockStats(expanded)"); + Check(test, + stats.available_flushed_blocks == + static_cast(SceKernelMemoryPoolExpandLen / SceKernelMemoryPoolAlignment), + "expanded pages were not added to the pool budget"); + + void* arena = reinterpret_cast(0x1000000000ull); + CheckOk(test, + Libs::LibKernel::Memory::KernelMemoryPoolReserve(arena, SceKernelMemoryPoolReserveLen, + 0, 0, &arena), + "KernelMemoryPoolReserve"); + const auto base = reinterpret_cast(arena); + const auto flexible_baseline = AvailableFlexibleMemory(test); + const auto commit_len = SceKernelMemoryPoolCommitLen * 2; + + CheckOk(test, + Libs::LibKernel::Memory::KernelMemoryPoolCommit(arena, commit_len, SceKernelMtypeC, + SceKernelProtCpuRw, 0), + "KernelMemoryPoolCommit"); + ExpectRange(test, Query(test, base), base, base + commit_len, SceKernelProtCpuRw, 0, 0, 1, 1); + Check(test, AvailableFlexibleMemory(test) == flexible_baseline, + "pooled commit consumed flexible memory instead of expanded direct " + "backing"); + CheckFailed(test, + Libs::LibKernel::Memory::KernelReleaseDirectMemory(pool_offset, + SceKernelMemoryPoolExpandLen), + "KernelReleaseDirectMemory(committed pool expansion)"); + + constexpr uint64_t first_value = 0x504f4f4c4241434bull; // "POOLBACK" + constexpr uint64_t second_value = 0x5348415245444d45ull; // "SHAREDME" + *reinterpret_cast(base) = first_value; + *reinterpret_cast(base + SceKernelMemoryPoolCommitLen) = second_value; + + CheckOk( + test, + Libs::LibKernel::Memory::KernelMemoryPoolDecommit(arena, SceKernelMemoryPoolCommitLen, 0), + "KernelMemoryPoolDecommit(first page)"); + ExpectRange(test, Query(test, base), base, base + SceKernelMemoryPoolCommitLen, 0, 0, 0, 1, 0); + ExpectRange(test, Query(test, base + SceKernelMemoryPoolCommitLen), + base + SceKernelMemoryPoolCommitLen, base + commit_len, SceKernelProtCpuRw, 0, 0, 1, + 1); + CheckOk(test, Libs::LibKernel::Memory::KernelMemoryPoolGetBlockStats(&stats, sizeof(stats)), + "KernelMemoryPoolGetBlockStats(partially decommitted)"); + Check(test, + stats.available_flushed_blocks == + static_cast(SceKernelMemoryPoolExpandLen / SceKernelMemoryPoolAlignment - + 1) && + stats.allocated_flushed_blocks == 1, + "partial decommit returned the wrong number of pages to the expanded " + "pool"); + + CheckOk(test, + Libs::LibKernel::Memory::KernelMemoryPoolCommit(arena, SceKernelMemoryPoolCommitLen, + SceKernelMtypeC, SceKernelProtCpuRw, 0), + "KernelMemoryPoolCommit(first-page recommit)"); + Check(test, + *reinterpret_cast(base) == first_value && + *reinterpret_cast(base + SceKernelMemoryPoolCommitLen) == + second_value, + "partially recommitted pooled pages did not retain shared-backing " + "contents"); + + CheckOk(test, Libs::LibKernel::Memory::KernelMemoryPoolDecommit(arena, commit_len, 0), + "KernelMemoryPoolDecommit(cleanup)"); + CheckOk(test, Libs::LibKernel::Memory::KernelMunmap(base, SceKernelMemoryPoolReserveLen), + "KernelMunmap(pool reserve cleanup)"); + CheckOk(test, + Libs::LibKernel::Memory::KernelReleaseDirectMemory(pool_offset, + SceKernelMemoryPoolExpandLen), + "KernelReleaseDirectMemory(pool expansion)"); + CheckOk(test, Libs::LibKernel::Memory::KernelMemoryPoolGetBlockStats(&stats, sizeof(stats)), + "KernelMemoryPoolGetBlockStats(released)"); + Check(test, stats.available_flushed_blocks == 0 && stats.allocated_flushed_blocks == 0, + "released expansion remained in the pool budget"); + + std::printf("[host] %-48s ok\n", test); +} + +void TestFragmentedMemoryPoolBacking() { + const char* test = "FragmentedMemoryPoolBacking"; + + int64_t first_pool = -1; + int64_t direct_gap = -1; + int64_t second_pool = -1; + const auto direct_end = + static_cast(Libs::LibKernel::Memory::KernelGetDirectMemorySize()); + CheckOk( + test, + Libs::LibKernel::Memory::KernelMemoryPoolExpand(0, direct_end, SceKernelMemoryPoolCommitLen, + SceKernelMemoryPoolAlignment, &first_pool), + "KernelMemoryPoolExpand(first)"); + CheckOk(test, + Libs::LibKernel::Memory::KernelAllocateDirectMemory( + 0, direct_end, SceKernelMemoryPoolCommitLen, SceKernelMemoryPoolAlignment, + SceKernelMtypeC, &direct_gap), + "KernelAllocateDirectMemory(gap)"); + CheckOk( + test, + Libs::LibKernel::Memory::KernelMemoryPoolExpand(0, direct_end, SceKernelMemoryPoolCommitLen, + SceKernelMemoryPoolAlignment, &second_pool), + "KernelMemoryPoolExpand(second)"); + Check(test, + first_pool + static_cast(SceKernelMemoryPoolCommitLen) == direct_gap && + direct_gap + static_cast(SceKernelMemoryPoolCommitLen) == second_pool, + "test setup did not create nonadjacent pool expansions"); + + void* arena = nullptr; + CheckOk(test, + Libs::LibKernel::Memory::KernelMemoryPoolReserve(nullptr, SceKernelMemoryPoolReserveLen, + 0, 0, &arena), + "KernelMemoryPoolReserve"); + const auto base = reinterpret_cast(arena); + const auto commit_len = SceKernelMemoryPoolCommitLen * 2; + CheckOk(test, + Libs::LibKernel::Memory::KernelMemoryPoolCommit(arena, commit_len, SceKernelMtypeC, + SceKernelProtCpuRw, 0), + "KernelMemoryPoolCommit(fragmented)"); + + CheckFailed(test, + Libs::LibKernel::Memory::KernelMemoryPoolCommit( + reinterpret_cast(base + commit_len), SceKernelMemoryPoolCommitLen, + SceKernelMtypeC, SceKernelProtCpuRw, 0), + "KernelMemoryPoolCommit(exhausted)"); + ExpectRange(test, Query(test, base + commit_len), base + commit_len, + base + SceKernelMemoryPoolReserveLen, 0, 0, 0, 1, 0); + + *reinterpret_cast(base) = 0x465241474d454e54ull; // "FRAGMENT" + *reinterpret_cast(base + SceKernelMemoryPoolCommitLen) = 0x504f4f4c50414745ull; + CheckOk(test, Libs::LibKernel::Memory::KernelMemoryPoolDecommit(arena, commit_len, 0), + "KernelMemoryPoolDecommit(fragmented)"); + + Libs::LibKernel::Memory::KernelMemoryPoolBlockStats stats {}; + CheckOk(test, Libs::LibKernel::Memory::KernelMemoryPoolGetBlockStats(&stats, sizeof(stats)), + "KernelMemoryPoolGetBlockStats(fragmented decommit)"); + Check(test, stats.available_flushed_blocks == 2 && stats.allocated_flushed_blocks == 0, + "fragmented decommit did not restore both pool pages"); + + CheckOk(test, + Libs::LibKernel::Memory::KernelMemoryPoolCommit(arena, commit_len, SceKernelMtypeC, + SceKernelProtCpuRw, 0), + "KernelMemoryPoolCommit(fragmented recommit)"); + CheckOk(test, Libs::LibKernel::Memory::KernelMunmap(base, commit_len), + "KernelMunmap(fragmented commit)"); + CheckOk(test, Libs::LibKernel::Memory::KernelMunmap(base, SceKernelMemoryPoolReserveLen), + "KernelMunmap(fragmented reserve cleanup)"); + + CheckOk(test, + Libs::LibKernel::Memory::KernelReleaseDirectMemory(first_pool, + SceKernelMemoryPoolCommitLen), + "KernelReleaseDirectMemory(first pool)"); + CheckOk(test, + Libs::LibKernel::Memory::KernelReleaseDirectMemory(second_pool, + SceKernelMemoryPoolCommitLen), + "KernelReleaseDirectMemory(second pool)"); + CheckOk(test, + Libs::LibKernel::Memory::KernelReleaseDirectMemory(direct_gap, + SceKernelMemoryPoolCommitLen), + "KernelReleaseDirectMemory(gap)"); + + std::printf("[host] %-48s ok\n", test); +} + +void TestMemoryPoolMultiRangeDecommit() { + const char* test = "MemoryPoolMultiRangeDecommit"; + const auto expand_len = SceKernelMemoryPoolCommitLen * 2; + int64_t pool_offset = -1; + CheckOk(test, + Libs::LibKernel::Memory::KernelMemoryPoolExpand( + 0, Libs::LibKernel::Memory::KernelGetDirectMemorySize(), expand_len, + SceKernelMemoryPoolAlignment, &pool_offset), + "KernelMemoryPoolExpand"); + + void* arena = nullptr; + CheckOk(test, + Libs::LibKernel::Memory::KernelMemoryPoolReserve(nullptr, SceKernelMemoryPoolReserveLen, + 0, 0, &arena), + "KernelMemoryPoolReserve"); + const auto base = reinterpret_cast(arena); + CheckOk(test, + Libs::LibKernel::Memory::KernelMemoryPoolCommit(arena, SceKernelMemoryPoolCommitLen, + SceKernelMtypeC, SceKernelProtCpuRw, 0), + "KernelMemoryPoolCommit(read-write)"); + CheckOk(test, + Libs::LibKernel::Memory::KernelMemoryPoolCommit( + reinterpret_cast(base + SceKernelMemoryPoolCommitLen), + SceKernelMemoryPoolCommitLen, SceKernelMtypeC, SceKernelProtCpuRead, 0), + "KernelMemoryPoolCommit(read-only)"); + CheckOk(test, Libs::LibKernel::Memory::KernelMemoryPoolDecommit(arena, expand_len, 0), + "KernelMemoryPoolDecommit(different protections)"); + const auto decommitted = Query(test, base); + Check(test, + decommitted.start <= base && decommitted.end >= base + expand_len && + decommitted.is_pooled == 1 && decommitted.is_committed == 0, + "multi-range decommit did not restore the reserved pool span"); + + CheckOk(test, + Libs::LibKernel::Memory::KernelMemoryPoolCommit( + reinterpret_cast(base + SceKernelMemoryPoolCommitLen), + SceKernelMemoryPoolCommitLen, SceKernelMtypeC, SceKernelProtCpuRead, 0), + "KernelMemoryPoolCommit(mixed span)"); + CheckOk(test, Libs::LibKernel::Memory::KernelMemoryPoolDecommit(arena, expand_len, 0), + "KernelMemoryPoolDecommit(reserved and committed span)"); + const auto mixed_decommitted = Query(test, base); + Check(test, + mixed_decommitted.start <= base && mixed_decommitted.end >= base + expand_len && + mixed_decommitted.is_pooled == 1 && mixed_decommitted.is_committed == 0, + "mixed reserved/committed decommit left committed pages behind"); + + CheckOk(test, + Libs::LibKernel::Memory::KernelMemoryPoolCommit(arena, SceKernelMemoryPoolCommitLen, + SceKernelMtypeC, SceKernelProtCpuRw, 0), + "KernelMemoryPoolCommit(preflight prefix)"); + CheckOk(test, + Libs::LibKernel::Memory::KernelMunmap(base + SceKernelMemoryPoolCommitLen, + SceKernelMemoryPoolCommitLen), + "KernelMunmap(preflight tail reserve)"); + void* flexible_tail = reinterpret_cast(base + SceKernelMemoryPoolCommitLen); + CheckOk(test, + Libs::LibKernel::Memory::KernelMapNamedFlexibleMemory( + &flexible_tail, SceKernelMemoryPoolCommitLen, SceKernelProtCpuRead, + SceKernelMapFixed, "pool_invalid_tail"), + "KernelMapNamedFlexibleMemory(preflight tail)"); + CheckFailed(test, Libs::LibKernel::Memory::KernelMemoryPoolDecommit(arena, expand_len, 0), + "KernelMemoryPoolDecommit(invalid tail)"); + ExpectRange(test, Query(test, base), base, base + SceKernelMemoryPoolCommitLen, + SceKernelProtCpuRw, 0, 0, 1, 1); + ExpectRange(test, Query(test, base + SceKernelMemoryPoolCommitLen), + base + SceKernelMemoryPoolCommitLen, base + expand_len, SceKernelProtCpuRead, 1, 0, + 0, 1, "pool_invalid_tail"); + + CheckOk( + test, + Libs::LibKernel::Memory::KernelMemoryPoolDecommit(arena, SceKernelMemoryPoolCommitLen, 0), + "KernelMemoryPoolDecommit(preflight prefix cleanup)"); + CheckOk(test, Libs::LibKernel::Memory::KernelMunmap(base, SceKernelMemoryPoolReserveLen), + "KernelMunmap(pool reserve cleanup)"); + CheckOk(test, Libs::LibKernel::Memory::KernelReleaseDirectMemory(pool_offset, expand_len), + "KernelReleaseDirectMemory(pool expansion)"); + + std::printf("[host] %-48s ok\n", test); +} + +void TestMemoryPoolCommitDecommitQueryFlags() { + const char* test = "MemoryPoolCommitDecommitQueryFlags"; + void* addr = nullptr; + int64_t pool_offset = -1; + CheckOk(test, + Libs::LibKernel::Memory::KernelMemoryPoolExpand( + 0, Libs::LibKernel::Memory::KernelGetDirectMemorySize(), + SceKernelMemoryPoolCommitLen, SceKernelMemoryPoolAlignment, &pool_offset), + "KernelMemoryPoolExpand"); + + CheckOk(test, + Libs::LibKernel::Memory::KernelMemoryPoolReserve(nullptr, SceKernelMemoryPoolReserveLen, + 0, 0, &addr), + "KernelMemoryPoolReserve"); + const auto base = reinterpret_cast(addr); + + const auto reserved = Query(test, base); + const bool reserved_ok = reserved.start <= base && base < reserved.end && + reserved.is_pooled == 1 && reserved.is_committed == 0; + + CheckOk(test, + Libs::LibKernel::Memory::KernelMemoryPoolCommit(reinterpret_cast(base), + SceKernelMemoryPoolCommitLen, + SceKernelMtypeC, SceKernelProtCpuRw, 0), + "KernelMemoryPoolCommit"); + const auto committed = Query(test, base); + const bool committed_ok = committed.start == base && + committed.end == base + SceKernelMemoryPoolCommitLen && + committed.protection == SceKernelProtCpuRw && + committed.is_pooled == 1 && committed.is_committed == 1; + + CheckOk(test, + Libs::LibKernel::Memory::KernelMemoryPoolDecommit(reinterpret_cast(base), + SceKernelMemoryPoolCommitLen, 0), + "KernelMemoryPoolDecommit"); + const auto decommitted = Query(test, base); + const bool decommitted_ok = decommitted.start <= base && base < decommitted.end && + decommitted.is_pooled == 1 && decommitted.is_committed == 0; + + CheckOk(test, Libs::LibKernel::Memory::KernelMunmap(base, SceKernelMemoryPoolReserveLen), + "KernelMunmap(pool reserve cleanup)"); + Check(test, reserved_ok, "pool reserve should query as pooled/uncommitted"); + Check(test, committed_ok, "pool commit should query as pooled/committed"); + Check(test, decommitted_ok, "pool decommit should return to pooled/uncommitted"); + CheckOk(test, + Libs::LibKernel::Memory::KernelReleaseDirectMemory(pool_offset, + SceKernelMemoryPoolCommitLen), + "KernelReleaseDirectMemory(pool expansion)"); + + std::printf("[host] %-48s ok\n", test); +} + +void TestProgramMemoryRegistrationAndProtection() { + const char* test = "ProgramMemoryRegistrationAndProtection"; + const auto size = SceKernelPageSize * 3; + const auto base = Common::VirtualMemory::Alloc(0, size, Common::VirtualMemory::Mode::ReadWrite); + Check(test, base != 0, "program host allocation failed"); + + Libs::LibKernel::Memory::RegisterProgramMemory(base, size, + Common::VirtualMemory::Mode::ReadWrite, + "program_test"); + ExpectRange(test, Query(test, base), base, base + size, SceKernelProtCpuRead | SceKernelProtCpuRw, + 0, 0, 0, 1, "program_test"); + + Libs::LibKernel::Memory::UpdateProgramMemoryProtection( + base, SceKernelPageSize, Common::VirtualMemory::Mode::Read); + ExpectRange(test, Query(test, base), base, base + SceKernelPageSize, SceKernelProtCpuRead, 0, 0, + 0, 1, "program_test"); + + CheckOk(test, + Libs::LibKernel::Memory::KernelMprotect( + reinterpret_cast(base + SceKernelPageSize - 0x10), 0x20, + SceKernelProtCpuRead | SceKernelProtCpuRw), + "KernelMprotect(program split span)"); + ExpectRange(test, Query(test, base), base, base + size, + SceKernelProtCpuRead | SceKernelProtCpuRw, 0, 0, 0, 1, "program_test"); + + Libs::LibKernel::Memory::UpdateProgramMemoryProtection( + base + SceKernelPageSize * 2, SceKernelPageSize, Common::VirtualMemory::Mode::Read); + ExpectRange(test, Query(test, base + SceKernelPageSize * 2), base + SceKernelPageSize * 2, + base + size, SceKernelProtCpuRead, 0, 0, 0, 1, "program_test"); + + Libs::LibKernel::Memory::UnregisterProgramMemory(base, size); + ExpectUnmapped(test, base); + Check(test, Common::VirtualMemory::Free(base), "program host free failed"); + + std::printf("[host] %-48s ok\n", test); +} + +} // namespace + +int main() { + InitSubsystems(); + + RunTest(TestProsperoArgumentAndInfoSizeContracts); + RunTest(TestFlexibleMapQueryAndWholeMunmap); + RunTest(TestPartialFlexibleMunmapAndFindNext); + RunTest(TestReserveMapFixedAndNoOverwrite); + RunTest(TestFixedNoOverwriteRejectsReservedRange); + RunTest(TestReleasedReserveCanBeReused); + RunTest(TestMunmapAcrossAdjacentFlexibleMappings); + RunTest(TestDirectMapQueryOffsetAndPartialMunmap); + RunTest(TestNonzeroDirectOffsetAliasesSharedBacking); + RunTest(TestDirectPhysicalFreeRangeReuseAndCoalescing); + RunTest(TestDirectAlignmentStaysWithinSearchRange); + RunTest(TestDefaultDirectMapUsesSystemAddressRange); + RunTest(TestLargeDirectMapAliasesAcrossChunks); + RunTest(TestDirectMapUnmapReusesHostAddress); + RunTest(TestLargeHintedReserveHostsSmallDirectMap); + RunTest(TestMemoryPoolAlignmentContracts); + RunTest(TestProsperoSampleMemoryPoolExpandCommit); + RunTest(TestFragmentedMemoryPoolBacking); + RunTest(TestMemoryPoolMultiRangeDecommit); + RunTest(TestMemoryPoolCommitDecommitQueryFlags); + RunTest(TestProgramMemoryRegistrationAndProtection); + + if (g_failed_tests != 0) { + std::printf("VirtualMemoryAllocationTests: %d case(s) failed\n", g_failed_tests); + return 1; + } + + std::printf("VirtualMemoryAllocationTests: all cases passed\n"); + return 0; +} diff --git a/tests/shaderCfgTests.cpp b/tests/shaderCfgTests.cpp new file mode 100644 index 000000000..3ad83bd3b --- /dev/null +++ b/tests/shaderCfgTests.cpp @@ -0,0 +1,6735 @@ +#include "common/emulatorConfig.h" +#include "common/logging/log.h" +#include "common/stringUtils.h" +#include "common/threads.h" +#include "graphics/guest_gpu/hardwareContext.h" +#include "graphics/guest_gpu/pm4.h" +#include "graphics/host_gpu/renderer/shaderResourceBarrier.h" +#include "graphics/host_gpu/renderer/shaderSubgroup.h" +#include "graphics/shader/recompiler/ExecMask.h" +#include "graphics/shader/recompiler/ResourceTracking.h" +#include "graphics/shader/recompiler/ScalarProvenance.h" +#include "graphics/shader/recompiler/ShaderCFG.h" +#include "graphics/shader/recompiler/ShaderDecoder.h" +#include "graphics/shader/recompiler/ShaderIR.h" +#include "graphics/shader/recompiler/ShaderInfoCollection.h" +#include "graphics/shader/recompiler/ShaderRecompiler.h" +#include "graphics/shader/recompiler/SpirvEmitter.h" +#include "graphics/shader/recompiler/SrtPatcher.h" +#include "graphics/shader/recompiler/SrtWalker.h" +#include "graphics/shader/recompiler/spirvEmitter/spirvEmitterInternal.h" +#include "graphics/shader/shader.h" +#include "libs/agc.h" +#include "spirv-tools/libspirv.hpp" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace Libs::Graphics { +namespace { + +void Check(bool value, const char* text) { + if (!value) { + std::fprintf(stderr, "ShaderCfgTests: failed: %s\n", text); + std::abort(); + } +} + +void CheckSpirvBinaryValidates(const std::vector& binary) { + spvtools::SpirvTools tools(SPV_ENV_VULKAN_1_2); + std::string messages; + + tools.SetMessageConsumer([&messages](spv_message_level_t, const char*, + const spv_position_t& position, const char* message) { + char buffer[1024] = {}; + std::snprintf(buffer, sizeof(buffer), "%zu:%zu: %s\n", position.line, position.column, + message); + messages += buffer; + }); + + if (!tools.Validate(binary)) { + std::fprintf(stderr, "SPIR-V binary validation failed:\n%s\n", messages.c_str()); + std::abort(); + } +} + +std::string DisassembleSpirvBinary(const std::vector& binary) { + spvtools::SpirvTools tools(SPV_ENV_VULKAN_1_2); + std::string messages; + std::string source; + + tools.SetMessageConsumer([&messages](spv_message_level_t, const char*, + const spv_position_t& position, const char* message) { + char buffer[1024] = {}; + std::snprintf(buffer, sizeof(buffer), "%zu:%zu: %s\n", position.line, position.column, + message); + messages += buffer; + }); + + if (!tools.Disassemble(binary.data(), binary.size(), &source, + SPV_BINARY_TO_TEXT_OPTION_FRIENDLY_NAMES | + SPV_BINARY_TO_TEXT_OPTION_INDENT)) { + std::fprintf(stderr, "SPIR-V binary disassembly failed:\n%s\n", messages.c_str()); + std::abort(); + } + return std::string(source.c_str()); +} + +uint32_t CountSourceOccurrences(const std::string& source, const char* needle) { + uint32_t count = 0; + uint32_t from = 0; + for (;;) { + const auto found = Common::FindIndex(source, std::string(needle), from); + if (found == Common::FIND_INVALID_INDEX) { + return count; + } + count++; + from = found + static_cast(std::strlen(needle)); + } +} + +bool SpirvSourceHasInstructionUsing(const std::string& source, const char* opcode, + const char* name) { + std::istringstream stream(source); + std::string line; + while (std::getline(stream, line)) { + if (Common::ContainsStr(line, opcode) && Common::ContainsStr(line, name)) { + return true; + } + } + return false; +} + +bool SpirvContainsOpcode(const std::vector& binary, uint32_t opcode) { + for (uint32_t word: binary) { + if ((word & 0xffffu) == opcode) { + return true; + } + } + return false; +} + +uint32_t SpirvInstructionOpcodeCount(const std::vector& binary, uint32_t opcode) { + uint32_t count = 0; + for (size_t i = 5; i < binary.size();) { + const uint32_t word = binary[i]; + const uint32_t op = word & 0xffffu; + const uint32_t word_count = word >> 16u; + if (word_count == 0 || i + word_count > binary.size()) { + return count; + } + if (op == opcode) { + count++; + } + i += word_count; + } + return count; +} + +bool SpirvContainsCapability(const std::vector& binary, uint32_t capability) { + for (size_t i = 5; i < binary.size();) { + const uint32_t word = binary[i]; + const uint32_t opcode = word & 0xffffu; + const uint32_t word_count = word >> 16u; + if (word_count == 0 || i + word_count > binary.size()) { + return false; + } + if (opcode == 17u && word_count >= 2u && binary[i + 1] == capability) { + return true; + } + i += word_count; + } + return false; +} + +bool SpirvContainsExecutionMode(const std::vector& binary, uint32_t mode) { + for (size_t i = 5; i < binary.size();) { + const uint32_t word = binary[i]; + const uint32_t opcode = word & 0xffffu; + const uint32_t word_count = word >> 16u; + if (word_count == 0 || i + word_count > binary.size()) { + return false; + } + if (opcode == 16u && word_count >= 3u && binary[i + 2] == mode) { + return true; + } + i += word_count; + } + return false; +} + +bool SpirvContainsExtInst(const std::vector& binary, uint32_t ext_inst) { + for (size_t i = 5; i < binary.size();) { + const uint32_t word = binary[i]; + const uint32_t opcode = word & 0xffffu; + const uint32_t word_count = word >> 16u; + if (word_count == 0 || i + word_count > binary.size()) { + return false; + } + if (opcode == 12u && word_count >= 5u && binary[i + 4] == ext_inst) { + return true; + } + i += word_count; + } + return false; +} + +uint32_t SpirvExtInstCount(const std::vector& binary, uint32_t ext_inst) { + uint32_t count = 0; + for (size_t i = 5; i < binary.size();) { + const uint32_t word = binary[i]; + const uint32_t opcode = word & 0xffffu; + const uint32_t word_count = word >> 16u; + if (word_count == 0 || i + word_count > binary.size()) { + return count; + } + if (opcode == 12u && word_count >= 5u && binary[i + 4] == ext_inst) { + count++; + } + i += word_count; + } + return count; +} + +bool SpirvContainsTypeImage(const std::vector& binary, uint32_t dim, uint32_t arrayed, + uint32_t sampled) { + for (size_t i = 5; i < binary.size();) { + const uint32_t word = binary[i]; + const uint32_t opcode = word & 0xffffu; + const uint32_t word_count = word >> 16u; + if (word_count == 0 || i + word_count > binary.size()) { + return false; + } + if (opcode == 25u && word_count >= 9u && binary[i + 3] == dim && binary[i + 5] == arrayed && + binary[i + 7] == sampled) { + return true; + } + i += word_count; + } + return false; +} + +uint32_t SpirvDecorationValueCount(const std::vector& binary, uint32_t decoration, + uint32_t value) { + uint32_t count = 0; + for (size_t i = 5; i < binary.size();) { + const uint32_t word = binary[i]; + const uint32_t opcode = word & 0xffffu; + const uint32_t word_count = word >> 16u; + if (word_count == 0 || i + word_count > binary.size()) { + return count; + } + if (opcode == 71u && word_count >= 4u && binary[i + 2] == decoration && + binary[i + 3] == value) { + count++; + } + i += word_count; + } + return count; +} + +bool SpirvHasDecorationValue(const std::vector& binary, uint32_t decoration, + uint32_t value) { + return SpirvDecorationValueCount(binary, decoration, value) != 0; +} + +std::vector SpirvDecorationValueTargets(const std::vector& binary, + uint32_t decoration, uint32_t value) { + std::vector targets; + for (size_t i = 5; i < binary.size();) { + const uint32_t word = binary[i]; + const uint32_t opcode = word & 0xffffu; + const uint32_t word_count = word >> 16u; + if (word_count == 0 || i + word_count > binary.size()) { + return targets; + } + if (opcode == 71u && word_count >= 4u && binary[i + 2] == decoration && + binary[i + 3] == value) { + targets.push_back(binary[i + 1]); + } + i += word_count; + } + return targets; +} + +bool SpirvTargetHasDecoration(const std::vector& binary, uint32_t target, + uint32_t decoration) { + for (size_t i = 5; i < binary.size();) { + const uint32_t word = binary[i]; + const uint32_t opcode = word & 0xffffu; + const uint32_t word_count = word >> 16u; + if (word_count == 0 || i + word_count > binary.size()) { + return false; + } + if (opcode == 71u && word_count >= 3u && binary[i + 1] == target && + binary[i + 2] == decoration) { + return true; + } + i += word_count; + } + return false; +} + +bool SpirvHasDecorationValueWithDecoration(const std::vector& binary, + uint32_t value_decoration, uint32_t value, + uint32_t decoration) { + const auto targets = SpirvDecorationValueTargets(binary, value_decoration, value); + return std::any_of(targets.begin(), targets.end(), [&](uint32_t target) { + return SpirvTargetHasDecoration(binary, target, decoration); + }); +} + +bool ProgramHasInput(const ShaderRecompiler::IR::Program& program, + ShaderRecompiler::IR::StageInputKind kind) { + return std::any_of(program.info.inputs.begin(), program.info.inputs.end(), + [kind](const auto& input) { return input.kind == kind; }); +} + +uint32_t ProgramInputCount(const ShaderRecompiler::IR::Program& program, + ShaderRecompiler::IR::StageInputKind kind) { + return static_cast( + std::count_if(program.info.inputs.begin(), program.info.inputs.end(), + [kind](const auto& input) { return input.kind == kind; })); +} + +ShaderComputeInputInfo RegressionComputeInputInfo() { + ShaderComputeInputInfo input_info; + input_info.threads_num[0] = 1; + input_info.threads_num[1] = 1; + input_info.threads_num[2] = 1; + input_info.workgroup_register = 40; + return input_info; +} + +ShaderPixelInputInfo RegressionPixelInputInfo() { + return {}; +} + +void SetIdentityInterpolatorSettings(ShaderPixelInputInfo* input_info) { + Check(input_info != nullptr, "invalid pixel input info"); + for (uint32_t i = 0; i < std::size(input_info->interpolator_settings); i++) { + input_info->interpolator_settings[i] = i; + } +} + +void EnsureConfigInitialized() { + static bool config_initialized = false; + if (!config_initialized) { + Common::ThreadsSubsystem::Instance()->Init(nullptr); + Config::ConfigSubsystem::Instance()->Init(nullptr); + Config::ConfigOptions options; + options.printf_direction = Config::OutputDirection::Silent; + Config::Load(options); + Log::LogSubsystem::Instance()->Init(nullptr); + ShaderInit(); + config_initialized = true; + } +} + +void TestResourceDescriptorClassification() { + uint32_t raw_texture[8] = {}; + raw_texture[3] = 9u << 28u; + raw_texture[5] = 2u << 27u; + Check(ShaderClassifyResourceDescriptor(raw_texture) == ResourceDescriptorType::Texture, + "raw texture descriptor must not be classified by ResourceDescriptor byte 23 fields"); + + uint32_t tagged_buffer[8] = {}; + tagged_buffer[5] = 1u << 27u; + Check(ShaderClassifyResourceDescriptor(tagged_buffer) == ResourceDescriptorType::Buffer, + "tagged ResourceDescriptor buffer was not recognized"); + + uint32_t tagged_sampler[8] = {}; + tagged_sampler[5] = 2u << 27u; + Check(ShaderClassifyResourceDescriptor(tagged_sampler) == ResourceDescriptorType::Sampler, + "tagged ResourceDescriptor sampler was not recognized"); + + uint32_t tagged_unused[8] = {}; + tagged_unused[5] = 3u << 27u; + Check(ShaderClassifyResourceDescriptor(tagged_unused) == ResourceDescriptorType::Unused, + "tagged ResourceDescriptor unused slot was not recognized"); + + uint32_t raw_buffer[8] = {}; + Check(ShaderClassifyResourceDescriptor(raw_buffer) == ResourceDescriptorType::Buffer, + "raw buffer descriptor fallback was not preserved"); +} + +constexpr uint32_t EncodeSMovB32(uint32_t dst, uint32_t src) { + return 0x80000000u | (0x7du << 23u) | ((dst & 0x7fu) << 16u) | (0x03u << 8u) | (src & 0xffu); +} + +constexpr uint32_t EncodeSop1(uint32_t opcode, uint32_t dst, uint32_t src) { + return 0x80000000u | (0x7du << 23u) | ((dst & 0x7fu) << 16u) | ((opcode & 0xffu) << 8u) | + (src & 0xffu); +} + +constexpr uint32_t EncodeSop2(uint32_t opcode, uint32_t dst, uint32_t src0, uint32_t src1) { + return 0x80000000u | ((opcode & 0x7fu) << 23u) | ((dst & 0x7fu) << 16u) | + ((src1 & 0xffu) << 8u) | (src0 & 0xffu); +} + +constexpr uint32_t EncodeSopk(uint32_t opcode, uint32_t dst, int16_t imm) { + return 0x80000000u | (((opcode + 0x60u) & 0x7fu) << 23u) | ((dst & 0x7fu) << 16u) | + static_cast(imm); +} + +constexpr uint32_t EncodeSopc(uint32_t opcode, uint32_t src0, uint32_t src1) { + return 0x80000000u | (0x7eu << 23u) | ((opcode & 0x7fu) << 16u) | ((src1 & 0xffu) << 8u) | + (src0 & 0xffu); +} + +constexpr uint32_t EncodeSopp(uint32_t opcode, uint32_t simm = 0) { + return 0x80000000u | (0x7fu << 23u) | ((opcode & 0x7fu) << 16u) | (simm & 0xffffu); +} + +void TestNativeShaderResourceDependencies() { + const auto stages = ShaderPipelineStages( + VK_SHADER_STAGE_VERTEX_BIT | VK_SHADER_STAGE_FRAGMENT_BIT | VK_SHADER_STAGE_COMPUTE_BIT); + Check(stages == (VK_PIPELINE_STAGE_VERTEX_SHADER_BIT | VK_PIPELINE_STAGE_FRAGMENT_SHADER_BIT | + VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT), + "shader-stage dependency mapping was not exact"); + const auto shader_barrier = MakeShaderWriteDependency(); + Check(shader_barrier.srcAccessMask == VK_ACCESS_SHADER_WRITE_BIT && + (shader_barrier.dstAccessMask & VK_ACCESS_SHADER_READ_BIT) != 0 && + (shader_barrier.dstAccessMask & VK_ACCESS_VERTEX_ATTRIBUTE_READ_BIT) != 0, + "shader-write dependency does not expose draw/dispatch buffer writes"); + + ShaderRecompiler::IR::Program program; + program.info.buffers.resize(3); + program.info.buffers[0].written = true; + program.info.buffers[1].read = true; + program.info.buffers[2].written = true; + ShaderRecompiler::IR::ResourceSnapshot resources; + resources.buffers.resize(3); + auto set_buffer = [&](uint32_t index, uint64_t address, uint16_t stride, uint32_t records) { + ShaderBufferResource descriptor; + descriptor.UpdateAddress48(address); + descriptor.fields[1] |= static_cast(stride) << 16u; + descriptor.fields[2] = records; + std::memcpy(resources.buffers[index].dwords.data(), descriptor.fields, + sizeof(descriptor.fields)); + resources.buffers[index].dword_count = 4; + }; + set_buffer(0, 0x1000, 16, 3); + set_buffer(1, 0x1800, 4, 7); + set_buffer(2, 0x2000, 0, 64); + const auto writes = CollectShaderBufferWrites(program, resources); + Check(writes == std::vector({{0x1000, 48}, {0x2000, 64}}), + "graphics/compute write collector lost or invented a storage-buffer range"); + + VulkanImage image {VulkanImageType::StorageTexture}; + image.image = reinterpret_cast(uintptr_t {1}); + image.layout = VK_IMAGE_LAYOUT_GENERAL; + image.layers = 3; + const auto image_barrier = MakeStorageImageDependency(image, true, true); + Check(image_barrier.oldLayout == VK_IMAGE_LAYOUT_GENERAL && + image_barrier.newLayout == VK_IMAGE_LAYOUT_GENERAL && + (image_barrier.srcAccessMask & VK_ACCESS_MEMORY_WRITE_BIT) != 0 && + (image_barrier.dstAccessMask & VK_ACCESS_SHADER_READ_BIT) != 0 && + (image_barrier.dstAccessMask & VK_ACCESS_SHADER_WRITE_BIT) != 0 && + image_barrier.subresourceRange.levelCount == VK_REMAINING_MIP_LEVELS && + image_barrier.subresourceRange.layerCount == image.layers, + "GENERAL storage-image dependency lacks a complete memory barrier"); + + VulkanBuffer buffer; + buffer.buffer = reinterpret_cast(uintptr_t {1}); + const auto gds_barrier = MakeGdsDependency(buffer); + Check((gds_barrier.srcAccessMask & VK_ACCESS_HOST_WRITE_BIT) != 0 && + (gds_barrier.srcAccessMask & VK_ACCESS_TRANSFER_WRITE_BIT) != 0 && + (gds_barrier.srcAccessMask & VK_ACCESS_SHADER_WRITE_BIT) != 0 && + gds_barrier.dstAccessMask == + (VK_ACCESS_SHADER_READ_BIT | VK_ACCESS_SHADER_WRITE_BIT) && + gds_barrier.size == VK_WHOLE_SIZE, + "GDS dependency does not order host/transfer/shader writes"); +} + +void TestNativeSubgroupPolicy() { + GraphicContext context; + context.subgroup_size = 32; + context.min_subgroup_size = 32; + context.max_subgroup_size = 32; + context.subgroup_size_control_enabled = true; + context.required_subgroup_size_stages = VK_SHADER_STAGE_ALL; + ShaderRecompiler::IR::Program safe; + safe.wave_size = 32; + safe.lane_mask_mode = ShaderLaneMaskMode::NativeWave; + Check(ConfigureShaderSubgroup(context, VK_SHADER_STAGE_VERTEX_BIT, safe).mode == + ShaderSubgroupMode::Natural, + "native wave32 policy changed"); + safe.wave_size = 64; + Check(SelectGraphicsLaneMaskMode(context, safe.wave_size) == + ShaderLaneMaskMode::PerInvocation && + ConfigureShaderSubgroup(context, VK_SHADER_STAGE_VERTEX_BIT, safe).mode == + ShaderSubgroupMode::Unsupported, + "wave64 graphics mismatch accepted native-wave mask lowering"); + safe.lane_mask_mode = ShaderLaneMaskMode::PerInvocation; + Check(ConfigureShaderSubgroup(context, VK_SHADER_STAGE_VERTEX_BIT, safe).mode == + ShaderSubgroupMode::PerInvocationGraphics, + "wave64 graphics mismatch did not select per-invocation masks"); + + ShaderRecompiler::IR::Program cross_lane = safe; + cross_lane.blocks.emplace_back().instructions.emplace_back().op = + ShaderRecompiler::IR::Opcode::ReadLaneU32; + Check(ShaderRecompiler::Spirv::ProgramRequiresExactSubgroupSize(cross_lane) && + ConfigureShaderSubgroup(context, VK_SHADER_STAGE_VERTEX_BIT, cross_lane).mode == + ShaderSubgroupMode::PerInvocationGraphics, + "graphics mismatch did not select per-invocation masks"); + auto cross_lane_compute = cross_lane; + cross_lane_compute.lane_mask_mode = ShaderLaneMaskMode::NativeWave; + Check(ConfigureShaderSubgroup(context, VK_SHADER_STAGE_COMPUTE_BIT, cross_lane_compute).mode == + ShaderSubgroupMode::Unsupported, + "cross-lane compute mismatch bypassed the exact subgroup requirement"); + + ShaderRecompiler::IR::Program zero_exec = safe; + zero_exec.lane_mask_mode = ShaderLaneMaskMode::NativeWave; + zero_exec.provenance.values.resize(3); + zero_exec.provenance.values[2].op = ShaderRecompiler::IR::ScalarValueOp::Constant; + zero_exec.provenance.values[2].imm = 0; + auto& zero_bfm = zero_exec.blocks.emplace_back().instructions.emplace_back(); + zero_bfm.op = ShaderRecompiler::IR::Opcode::BitFieldMaskU64; + zero_bfm.dst.kind = ShaderRecompiler::IR::OperandKind::Register; + zero_bfm.dst.reg = {ShaderRecompiler::IR::RegisterFile::Exec, 0}; + zero_bfm.src_count = 2; + zero_bfm.scalar_sources[0] = 2; + Check(!ShaderRecompiler::Spirv::ProgramRequiresExactSubgroupSize(zero_exec) && + ConfigureShaderSubgroup(context, VK_SHADER_STAGE_COMPUTE_BIT, zero_exec).mode == + ShaderSubgroupMode::FlattenedMasks, + "compile-time uniform-zero EXEC write did not stay on the mask-free path"); + + ShaderRecompiler::IR::Program selective_exec = safe; + auto& move_exec = selective_exec.blocks.emplace_back().instructions.emplace_back(); + move_exec.op = ShaderRecompiler::IR::Opcode::MoveU32; + move_exec.dst.kind = ShaderRecompiler::IR::OperandKind::Register; + move_exec.dst.reg = {ShaderRecompiler::IR::RegisterFile::Exec, 0}; + Check(ShaderRecompiler::Spirv::ProgramRequiresExactSubgroupSize(selective_exec), + "selective EXEC write was classified mask-free"); + + ShaderRecompiler::IR::Program carry = safe; + auto& add_carry = carry.blocks.emplace_back().instructions.emplace_back(); + add_carry.op = ShaderRecompiler::IR::Opcode::IAddCarryU32; + add_carry.dst2.kind = ShaderRecompiler::IR::OperandKind::Register; + add_carry.dst2.reg = {ShaderRecompiler::IR::RegisterFile::Vcc, 0}; + Check(ShaderRecompiler::Spirv::ProgramRequiresExactSubgroupSize(carry), + "lane-varying VCC carry producer was classified mask-free"); + + ShaderRecompiler::IR::Program vcc_branch = safe; + auto& branch_block = vcc_branch.blocks.emplace_back(); + branch_block.terminator.kind = ShaderRecompiler::CFG::TerminatorKind::ConditionalBranch; + branch_block.terminator.condition = ShaderRecompiler::CFG::BranchCondition::VccNonZero; + Check(ShaderRecompiler::Spirv::ProgramRequiresExactSubgroupSize(vcc_branch), + "whole-wave VCC branch was classified mask-free"); + + ShaderRecompiler::IR::Program ds_partial = safe; + ds_partial.lane_mask_mode = ShaderLaneMaskMode::NativeWave; + ds_partial.blocks.emplace_back().instructions.emplace_back().op = + ShaderRecompiler::IR::Opcode::DsAppend; + Check(ConfigureShaderSubgroup(context, VK_SHADER_STAGE_COMPUTE_BIT, ds_partial).mode == + ShaderSubgroupMode::Unsupported, + "partial wave64 DS append bypassed the exact subgroup requirement"); + context.max_subgroup_size = 64; + const auto controlled = + ConfigureShaderSubgroup(context, VK_SHADER_STAGE_COMPUTE_BIT, cross_lane_compute); + Check(controlled.mode == ShaderSubgroupMode::Controlled && controlled.required_size == 64, + "supported controlled wave64 was not preferred over splitting"); + context.subgroup_size = 64; + context.subgroup_size_control_enabled = false; + cross_lane.wave_size = 32; + cross_lane.lane_mask_mode = ShaderLaneMaskMode::PerInvocation; + Check(ConfigureShaderSubgroup(context, VK_SHADER_STAGE_FRAGMENT_BIT, cross_lane).mode == + ShaderSubgroupMode::Unsupported, + "inverse graphics mismatch was accepted as one guest wave"); + cross_lane_compute.wave_size = 32; + Check(ConfigureShaderSubgroup(context, VK_SHADER_STAGE_COMPUTE_BIT, cross_lane_compute).mode == + ShaderSubgroupMode::Unsupported, + "inverse cross-lane compute mismatch was accepted"); +} + +std::array ImageTestUserData(Prospero::ImageType type = Prospero::ImageType::kColor2D) { + std::array data {}; + for (uint32_t start = 0; start + 3u < data.size(); start += 4u) { + data[start] = 0x1000u + start * 0x100u; + data[start + 2u] = UINT32_MAX; + data[start + 3u] = Prospero::GpuEnumValue(type) << 28u; + } + return data; +} + +void SetImageTestType(std::array* data, uint32_t srsrc, Prospero::ImageType type) { + const auto type_dword = srsrc * 4u + 3u; + Check(data != nullptr && type_dword < data->size(), "invalid image test descriptor source"); + (*data)[type_dword] = Prospero::GpuEnumValue(type) << 28u; +} + +bool ReadZeroTestMemory(void*, uint64_t, uint32_t* value) { + if (value == nullptr) { + return false; + } + *value = 0; + return true; +} + +constexpr uint32_t EncodeVop2(uint32_t opcode, uint32_t dst, uint32_t src0, uint32_t src1) { + return ((opcode & 0x3fu) << 25u) | ((dst & 0xffu) << 17u) | ((src1 & 0xffu) << 9u) | + (src0 & 0x1ffu); +} + +constexpr uint32_t EncodeVop1(uint32_t opcode, uint32_t dst, uint32_t src0) { + return (0x3fu << 25u) | ((dst & 0xffu) << 17u) | ((opcode & 0xffu) << 9u) | (src0 & 0x1ffu); +} + +constexpr uint32_t EncodeVop1Sdwa(uint32_t src0, uint32_t dst_sel = 6, uint32_t dst_u = 0, + uint32_t src0_sel = 6, uint32_t src0_sext = 0, + uint32_t src0_neg = 0, uint32_t src0_abs = 0, uint32_t s0 = 0) { + return (src0 & 0xffu) | ((dst_sel & 0x7u) << 8u) | ((dst_u & 0x3u) << 11u) | + ((src0_sel & 0x7u) << 16u) | ((src0_sext & 0x1u) << 19u) | ((src0_neg & 0x1u) << 20u) | + ((src0_abs & 0x1u) << 21u) | ((s0 & 0x1u) << 23u); +} + +constexpr uint32_t EncodeVop1Dpp(uint32_t src0, uint32_t dpp_ctrl = 0, uint32_t row_mask = 0xf, + uint32_t bank_mask = 0xf) { + return (src0 & 0xffu) | ((dpp_ctrl & 0x1ffu) << 8u) | ((bank_mask & 0xfu) << 24u) | + ((row_mask & 0xfu) << 28u); +} + +constexpr uint32_t EncodeVop2Sdwa(uint32_t src0, uint32_t dst_sel = 6, uint32_t dst_u = 0, + uint32_t src0_sel = 6, uint32_t src1_sel = 6, + uint32_t src0_sext = 0, uint32_t src1_sext = 0, + uint32_t src0_neg = 0, uint32_t src0_abs = 0, + uint32_t src1_neg = 0, uint32_t src1_abs = 0, uint32_t s0 = 0, + uint32_t s1 = 0) { + return (src0 & 0xffu) | ((dst_sel & 0x7u) << 8u) | ((dst_u & 0x3u) << 11u) | + ((src0_sel & 0x7u) << 16u) | ((src0_sext & 0x1u) << 19u) | ((src0_neg & 0x1u) << 20u) | + ((src0_abs & 0x1u) << 21u) | ((s0 & 0x1u) << 23u) | ((src1_sel & 0x7u) << 24u) | + ((src1_sext & 0x1u) << 27u) | ((src1_neg & 0x1u) << 28u) | ((src1_abs & 0x1u) << 29u) | + ((s1 & 0x1u) << 31u); +} + +constexpr uint32_t EncodeVop2Dpp(uint32_t src0, uint32_t dpp_ctrl = 0, uint32_t row_mask = 0xf, + uint32_t bank_mask = 0xf, uint32_t src0_neg = 0, + uint32_t src0_abs = 0, uint32_t src1_neg = 0, + uint32_t src1_abs = 0) { + return (src0 & 0xffu) | ((dpp_ctrl & 0x1ffu) << 8u) | ((src0_neg & 0x1u) << 20u) | + ((src0_abs & 0x1u) << 21u) | ((src1_neg & 0x1u) << 22u) | ((src1_abs & 0x1u) << 23u) | + ((bank_mask & 0xfu) << 24u) | ((row_mask & 0xfu) << 28u); +} + +constexpr uint32_t EncodeVopc(uint32_t opcode, uint32_t src0, uint32_t src1) { + return (0x3eu << 25u) | ((opcode & 0xffu) << 17u) | ((src1 & 0xffu) << 9u) | (src0 & 0x1ffu); +} + +constexpr uint32_t EncodeVopcSdwa(uint32_t src0, uint32_t sdst = 0, uint32_t sd = 0, + uint32_t src0_sel = 6, uint32_t src1_sel = 6, + uint32_t src0_sext = 0, uint32_t src1_sext = 0, + uint32_t src0_neg = 0, uint32_t src0_abs = 0, + uint32_t src1_neg = 0, uint32_t src1_abs = 0, uint32_t s0 = 0, + uint32_t s1 = 0) { + return (src0 & 0xffu) | ((sdst & 0x7fu) << 8u) | ((sd & 0x1u) << 15u) | + ((src0_sel & 0x7u) << 16u) | ((src0_sext & 0x1u) << 19u) | ((src0_neg & 0x1u) << 20u) | + ((src0_abs & 0x1u) << 21u) | ((s0 & 0x1u) << 23u) | ((src1_sel & 0x7u) << 24u) | + ((src1_sext & 0x1u) << 27u) | ((src1_neg & 0x1u) << 28u) | ((src1_abs & 0x1u) << 29u) | + ((s1 & 0x1u) << 31u); +} + +constexpr uint32_t EncodeVop3Word0(uint32_t opcode, uint32_t dst, uint32_t op_sel = 0, + uint32_t abs = 0, bool clamp = false) { + return (0x35u << 26u) | ((opcode & 0x3ffu) << 16u) | (dst & 0xffu) | ((abs & 0x7u) << 8u) | + ((op_sel & 0xfu) << 11u) | (clamp ? (1u << 15u) : 0u); +} + +constexpr uint32_t EncodeVop3Word0Sdst(uint32_t opcode, uint32_t dst, uint32_t sdst) { + return (0x35u << 26u) | ((opcode & 0x3ffu) << 16u) | ((sdst & 0x7fu) << 8u) | (dst & 0xffu); +} + +constexpr uint32_t EncodeVop3Word1(uint32_t src0, uint32_t src1, uint32_t src2) { + return (src0 & 0x1ffu) | ((src1 & 0x1ffu) << 9u) | ((src2 & 0x1ffu) << 18u); +} + +constexpr uint32_t EncodeVop3pWord0(uint32_t opcode, uint32_t dst, uint32_t op_sel = 0, + uint32_t op_sel_hi = 0, uint32_t neg_hi = 0, + bool clamp = false) { + return (0x33u << 26u) | ((opcode & 0x7fu) << 16u) | (dst & 0xffu) | ((neg_hi & 0x7u) << 8u) | + ((op_sel & 0x7u) << 11u) | (((op_sel_hi >> 2u) & 0x1u) << 14u) | + (clamp ? (1u << 15u) : 0u); +} + +constexpr uint32_t EncodeVop3pWord1(uint32_t src0, uint32_t src1, uint32_t src2, + uint32_t op_sel_hi = 0, uint32_t neg = 0) { + return (src0 & 0x1ffu) | ((src1 & 0x1ffu) << 9u) | ((src2 & 0x1ffu) << 18u) | + ((op_sel_hi & 0x3u) << 27u) | ((neg & 0x7u) << 29u); +} + +constexpr uint32_t EncodeSmem0(uint32_t opcode, uint32_t dst, uint32_t sbase) { + return (0x3du << 26u) | ((opcode & 0xffu) << 18u) | ((dst & 0x7fu) << 6u) | (sbase & 0x3fu); +} + +constexpr uint32_t EncodeMubuf0(uint32_t opcode, uint32_t offset = 0, bool idxen = true, + bool glc = false) { + return (0x38u << 26u) | ((opcode & 0x7fu) << 18u) | (idxen ? (1u << 13u) : 0u) | + (glc ? (1u << 14u) : 0u) | (offset & 0xfffu); +} + +constexpr uint32_t EncodeMubuf1(uint32_t vdata, uint32_t srsrc, uint32_t vaddr, + uint32_t soffset = 128) { + return ((soffset & 0xffu) << 24u) | ((srsrc & 0x1fu) << 16u) | ((vdata & 0xffu) << 8u) | + (vaddr & 0xffu); +} + +constexpr uint32_t EncodeMtbuf0(uint32_t opcode, uint32_t dfmt, uint32_t nfmt, uint32_t offset = 0, + bool idxen = true, bool offen = false) { + return (0x3au << 26u) | (offset & 0xfffu) | (offen ? (1u << 12u) : 0u) | + (idxen ? (1u << 13u) : 0u) | ((opcode & 0x7u) << 16u) | ((dfmt & 0xfu) << 19u) | + ((nfmt & 0x7u) << 23u); +} + +constexpr uint32_t EncodeMtbuf1(uint32_t opcode, uint32_t vdata, uint32_t srsrc, uint32_t vaddr, + uint32_t soffset = 128) { + return ((opcode >> 3u) & 1u) << 21u | ((soffset & 0xffu) << 24u) | ((srsrc & 0x1fu) << 16u) | + ((vdata & 0xffu) << 8u) | (vaddr & 0xffu); +} + +constexpr uint32_t EncodeFlat0(uint32_t opcode, uint32_t seg, uint32_t offset = 0) { + return (0x37u << 26u) | ((opcode & 0x7fu) << 18u) | ((seg & 0x3u) << 14u) | (offset & 0xfffu); +} + +constexpr uint32_t EncodeFlat1(uint32_t vdst, uint32_t saddr, uint32_t data, uint32_t addr) { + return ((vdst & 0xffu) << 24u) | ((saddr & 0x7fu) << 16u) | ((data & 0xffu) << 8u) | + (addr & 0xffu); +} + +constexpr uint32_t EncodeDs0(uint32_t opcode, uint32_t offset = 0) { + return (0x36u << 26u) | ((opcode & 0xffu) << 18u) | (offset & 0xffffu); +} + +constexpr uint32_t EncodeDs1Ex(uint32_t vdst, uint32_t data1, uint32_t data0, uint32_t addr) { + return ((vdst & 0xffu) << 24u) | ((data1 & 0xffu) << 16u) | ((data0 & 0xffu) << 8u) | + (addr & 0xffu); +} + +constexpr uint32_t EncodeDs1(uint32_t vdst, uint32_t data0, uint32_t addr) { + return EncodeDs1Ex(vdst, 0, data0, addr); +} + +constexpr uint32_t EncodeMimg0(uint32_t opcode, uint32_t dmask, bool glc = false, + uint32_t dim = 1) { + return (0x3cu << 26u) | ((opcode & 0x7fu) << 18u) | ((dmask & 0xfu) << 8u) | + ((dim & 0x7u) << 3u) | (glc ? (1u << 13u) : 0u) | ((opcode >> 7u) & 1u); +} + +constexpr uint32_t EncodeMimg1(uint32_t vdata, uint32_t srsrc, uint32_t ssamp, uint32_t vaddr, + bool a16 = false) { + return ((ssamp & 0x1fu) << 21u) | ((srsrc & 0x1fu) << 16u) | ((vdata & 0xffu) << 8u) | + (vaddr & 0xffu) | (a16 ? (1u << 30u) : 0u); +} + +constexpr uint32_t EncodeVintrp(uint32_t opcode, uint32_t vdst, uint32_t attr, uint32_t chan, + uint32_t vsrc) { + return (0x32u << 26u) | ((opcode & 0x3u) << 16u) | ((vdst & 0xffu) << 18u) | + ((attr & 0x3fu) << 10u) | ((chan & 0x3u) << 8u) | (vsrc & 0xffu); +} + +constexpr uint32_t EncodeExp0(uint32_t target, uint32_t en, bool done = true, bool compr = false, + bool vm = false) { + return (0x3eu << 26u) | ((target & 0x3fu) << 4u) | (en & 0xfu) | (compr ? (1u << 10u) : 0u) | + (done ? (1u << 11u) : 0u) | (vm ? (1u << 12u) : 0u); +} + +constexpr uint32_t EncodeExp1(uint32_t src0, uint32_t src1, uint32_t src2, uint32_t src3) { + return (src0 & 0xffu) | ((src1 & 0xffu) << 8u) | ((src2 & 0xffu) << 16u) | + ((src3 & 0xffu) << 24u); +} + +void TestNewShaderRecompilerSMovB32() { + const uint32_t shader[] = { + EncodeSMovB32(0, 129), // s_mov_b32 s0, 1 + EncodeSMovB32(1, 255), // s_mov_b32 s1, 0x12345678 + 0x12345678u, EncodeSMovB32(2, 1), // s_mov_b32 s2, s1 + 0xbf810000u, + }; + + ShaderRecompiler::CompileOptions options; + options.stage = ShaderType::Compute; + options.dump_ir = true; + + ShaderRecompiler::CompileResult result; + std::string error; + Check(ShaderRecompiler::TryRecompile(shader, options, &result, &error), error.c_str()); + Check(!result.spirv.empty(), "new shader recompiler produced no SPIR-V"); + Check(result.spirv.front() == 0x07230203u, "new shader recompiler did not emit SPIR-V binary"); + Check(Common::ContainsStr(result.decoded_dump, "s_mov_b32 s0, 1"), + "new decoder did not decode inline S_MOV_B32 operand"); + Check(Common::ContainsStr(result.decoded_dump, "s_mov_b32 s1, 0x12345678"), + "new decoder did not decode literal S_MOV_B32 operand"); + Check(Common::ContainsStr(result.decoded_dump, "s_mov_b32 s2, s1"), + "new decoder did not decode register S_MOV_B32 operand"); + Check(Common::ContainsStr(result.ir_dump, "MoveU32 s0, 0x00000001"), + "new IR did not lower inline S_MOV_B32"); + Check(Common::ContainsStr(result.ir_dump, "MoveU32 s1, 0x12345678"), + "new IR did not lower literal S_MOV_B32"); + Check(Common::ContainsStr(result.ir_dump, "MoveU32 s2, s1"), + "new IR did not lower register S_MOV_B32"); + Check(std::find(result.spirv.begin(), result.spirv.end(), 0x12345678u) != result.spirv.end(), + "new SPIR-V emitter did not encode the literal as a binary word"); + CheckSpirvBinaryValidates(result.spirv); +} + +void TestNewShaderRecompilerSoppMarkers() { + const uint32_t shader[] = { + EncodeSopp(0x00, 3), // s_nop 3 + EncodeSopp(0x0c, 0), // s_waitcnt 0 + EncodeSopp(0x10, 0x0f), // s_sendmsg 15 + EncodeSopp(0x16, 0x2a), // s_ttracedata 42 + EncodeSopp(0x20, 1), // s_inst_prefetch 1 + EncodeSopp(0x0a, 0), // s_barrier + EncodeSopp(0x01, 0), // s_endpgm + }; + + ShaderRecompiler::CompileOptions options; + options.stage = ShaderType::Compute; + options.dump_ir = true; + + ShaderRecompiler::CompileResult result; + std::string error; + Check(ShaderRecompiler::TryRecompile(shader, options, &result, &error), error.c_str()); + Check(Common::ContainsStr(result.decoded_dump, "s_nop 0x00000003"), + "new decoder did not decode SOPP s_nop"); + Check(Common::ContainsStr(result.decoded_dump, "s_waitcnt 0x00000000"), + "new decoder did not decode SOPP s_waitcnt"); + Check(Common::ContainsStr(result.decoded_dump, "s_sendmsg 0x0000000f"), + "new decoder did not decode SOPP s_sendmsg"); + Check(Common::ContainsStr(result.decoded_dump, "s_ttracedata 0x0000002a"), + "new decoder did not decode SOPP s_ttracedata"); + Check(Common::ContainsStr(result.decoded_dump, "s_inst_prefetch 0x00000001"), + "new decoder did not decode SOPP s_inst_prefetch"); + Check(Common::ContainsStr(result.decoded_dump, "s_barrier"), + "new decoder did not decode SOPP s_barrier"); + Check(Common::ContainsStr(result.ir_dump, "ControlNop null, 0x00000003"), + "SOPP s_nop did not lower to an IR marker"); + Check(Common::ContainsStr(result.ir_dump, "Waitcnt null, 0x00000000"), + "SOPP s_waitcnt did not lower to an IR marker"); + Check(Common::ContainsStr(result.ir_dump, "Sendmsg null, 0x0000000f"), + "SOPP s_sendmsg did not lower to an IR marker"); + Check(Common::ContainsStr(result.ir_dump, "TtraceData null, 0x0000002a"), + "SOPP s_ttracedata did not lower to an IR marker"); + Check(Common::ContainsStr(result.ir_dump, "InstPrefetch null, 0x00000001"), + "SOPP s_inst_prefetch did not lower to an IR marker"); + Check(Common::ContainsStr(result.ir_dump, "Barrier null"), + "SOPP s_barrier did not lower to an IR marker"); + Check(SpirvContainsOpcode(result.spirv, 224), + "SPIR-V binary does not contain OpControlBarrier"); + Check(std::find(result.spirv.begin(), result.spirv.end(), 264u) != result.spirv.end(), + "SPIR-V barrier does not use workgroup acquire-release memory semantics"); + CheckSpirvBinaryValidates(result.spirv); +} + +void TestNewShaderRecompilerSopkWaitcntMarkers() { + const uint32_t shader[] = { + EncodeSopk(0x17, 125, 0xffff), // s_waitcnt_vscnt null, 0xffff + EncodeSopk(0x18, 125, 0), // s_waitcnt_vmcnt null, 0 + EncodeSopk(0x19, 125, 0), // s_waitcnt_expcnt null, 0 + EncodeSopk(0x1a, 125, 0), // s_waitcnt_lgkmcnt null, 0 + EncodeSopp(0x01, 0), // s_endpgm + }; + + ShaderRecompiler::CompileOptions options; + options.stage = ShaderType::Compute; + options.dump_ir = true; + + ShaderRecompiler::CompileResult result; + std::string error; + Check(ShaderRecompiler::TryRecompile(shader, options, &result, &error), error.c_str()); + Check(Common::ContainsStr(result.decoded_dump, "s_waitcnt 0"), + "new decoder did not decode SOPK waitcnt marker"); + Check(Common::ContainsStr(result.decoded_dump, "s_waitcnt 65535"), + "SOPK waitcnt marker immediate was not kept unsigned"); + Check(Common::ContainsStr(result.ir_dump, "Waitcnt null, 0x00000000"), + "SOPK waitcnt did not lower to an IR marker"); + Check(Common::ContainsStr(result.ir_dump, "Waitcnt null, 0x0000ffff"), + "SOPK waitcnt marker immediate was not lowered as 16-bit unsigned"); + CheckSpirvBinaryValidates(result.spirv); +} + +void TestNewShaderRecompilerRdna2ScalarOpcodes() { + const uint32_t shader[] = { + EncodeSMovB32(2, 135), // s2 = 7 + EncodeSMovB32(106, 144), // vcc_lo = 16 + EncodeSop1(0x1d, 106, 128), // s_bitset1_b32 vcc_lo, 0 + EncodeSopk(0x13, 106, 0x1019), // s_setreg_b32 vcc_lo, 0x1019 + EncodeSop2(0x02, 106, 2, 239), // s_add_i32 vcc_lo, s2, pops_exiting_wave_id + EncodeSopp(0x0e, 0), // s_sleep 0 + EncodeSop2(0x02, 106, 239, 2), // s_add_i32 vcc_lo, pops_exiting_wave_id, s2 + EncodeSopp(0x01, 0), // s_endpgm + }; + + ShaderRecompiler::CompileOptions options; + options.stage = ShaderType::Compute; + options.dump_ir = true; + + ShaderRecompiler::CompileResult result; + std::string error; + Check(ShaderRecompiler::TryRecompile(shader, options, &result, &error), error.c_str()); + Check(Common::ContainsStr(result.decoded_dump, "s_bitset1_b32 vcc_lo, 0"), + "new decoder did not decode RDNA2 S_BITSET1_B32"); + Check(Common::ContainsStr(result.decoded_dump, "s_setreg_b32 vcc_lo, 0x00001019"), + "new decoder did not decode S_SETREG_B32"); + Check(Common::ContainsStr(result.decoded_dump, "s_add_i32 vcc_lo, s2, pops_exiting_wave_id"), + "new decoder did not decode pops_exiting_wave_id as RHS scalar source"); + Check(Common::ContainsStr(result.decoded_dump, "s_add_i32 vcc_lo, pops_exiting_wave_id, s2"), + "new decoder did not decode pops_exiting_wave_id as LHS scalar source"); + Check(Common::ContainsStr(result.decoded_dump, "s_sleep 0x00000000"), + "new decoder did not decode S_SLEEP"); + Check(Common::ContainsStr(result.ir_dump, "BitSetU32 vcc_lo, vcc_lo, 0x00000000"), + "S_BITSET1_B32 did not lower to bit-set IR using the destination as input"); + Check(Common::ContainsStr(result.ir_dump, "ScalarSignedAddOverflowI32 vcc_lo, s2, 0x00000000"), + "pops_exiting_wave_id RHS did not lower to a deterministic zero value"); + Check(Common::ContainsStr(result.ir_dump, "ScalarSignedAddOverflowI32 vcc_lo, 0x00000000, s2"), + "pops_exiting_wave_id LHS did not lower to a deterministic zero value"); + Check(Common::ContainsStr(result.ir_dump, "ControlNop null, vcc_lo"), + "S_SETREG_B32 did not lower to an explicit control marker"); + Check(Common::ContainsStr(result.ir_dump, "ControlNop null, 0x00000000"), + "S_SLEEP did not lower to an explicit control marker"); + Check(SpirvContainsOpcode(result.spirv, 196), + "SPIR-V binary does not contain OpShiftLeftLogical for S_BITSET1_B32"); + Check(SpirvContainsOpcode(result.spirv, 197), + "SPIR-V binary does not contain OpBitwiseOr for S_BITSET1_B32"); + CheckSpirvBinaryValidates(result.spirv); +} + +void TestNewShaderRecompilerScalarVectorAlu() { + const uint32_t shader[] = { + EncodeSMovB32(0, 129), // s0 = 1 + EncodeSMovB32(1, 130), // s1 = 2 + EncodeSop2(0x00, 2, 0, 1), // s_add_u32 s2, s0, s1 + EncodeSop2(0x01, 3, 2, 129), // s_sub_u32 s3, s2, 1 + EncodeSop2(0x0e, 4, 2, 3), // s_and_b32 s4, s2, s3 + EncodeSop2(0x10, 5, 2, 3), // s_or_b32 s5, s2, s3 + EncodeSop2(0x12, 6, 2, 3), // s_xor_b32 s6, s2, s3 + EncodeSop2(0x1e, 7, 2, 129), // s_lshl_b32 s7, s2, 1 + EncodeSop2(0x20, 8, 7, 129), // s_lshr_b32 s8, s7, 1 + EncodeSop2(0x2e, 9, 2, 1), // s_lshl1_add_u32 s9, s2, s1 + EncodeSop2(0x2f, 10, 9, 1), // s_lshl2_add_u32 s10, s9, s1 + EncodeSop2(0x30, 11, 10, 1), // s_lshl3_add_u32 s11, s10, s1 + EncodeSop2(0x31, 12, 11, 1), // s_lshl4_add_u32 s12, s11, s1 + EncodeSop2(0x35, 13, 12, 1), // s_mul_hi_u32 s13, s12, s1 + EncodeSopc(0x08, 8, 1), // s_cmp_gt_u32 s8, s1 + EncodeSop2(0x04, 14, 13, 129), // s_addc_u32 s14, s13, 1 + EncodeVop2(0x03, 1, 242, 0), // v_add_f32 v1, 1.0, v0 + EncodeVop2(0x08, 2, 1 + 256, 1), // v_mul_f32 v2, v1, v1 + EncodeVop2(0x25, 3, 1 + 256, 2), // v_add_nc_u32 v3, v1, v2 + EncodeVop2(0x1b, 4, 3 + 256, 2), // v_and_b32 v4, v3, v2 + EncodeVop2(0x0b, 6, 3 + 256, 4), // v_mul_u32_u24 v6, v3, v4 + EncodeVop2(0x09, 8, 4 + 256, 6), // v_mul_i32_i24 v8, v4, v6 + EncodeVop2(0x22, 7, 4 + 256, 6), // v_bcnt_u32_b32 v7, v4, v6 + EncodeVopc(0xc4, 3 + 256, 2), // v_cmp_gt_u32 v3, v2 + EncodeVop2(0x01, 5, 1 + 256, 2), // v_cndmask_b32 v5, v1, v2 + 0xbf810000u, + }; + + ShaderRecompiler::CompileOptions options; + options.stage = ShaderType::Compute; + options.dump_ir = true; + + ShaderRecompiler::CompileResult result; + std::string error; + Check(ShaderRecompiler::TryRecompile(shader, options, &result, &error), error.c_str()); + Check(Common::ContainsStr(result.decoded_dump, "s_add_u32 s2, s0, s1"), + "new decoder did not decode SOP2 add"); + Check(Common::ContainsStr(result.decoded_dump, "s_addc_u32 s14, s13, 1"), + "new decoder did not decode old-backed S_ADD_C_U32"); + Check(Common::ContainsStr(result.decoded_dump, "s_cmp_gt_u32 s8, s1"), + "new decoder did not decode SOPC compare"); + Check(Common::ContainsStr(result.decoded_dump, "s_lshl1_add_u32 s9, s2, s1"), + "new decoder did not decode old-backed S_LSHL1_ADD_U32"); + Check(Common::ContainsStr(result.decoded_dump, "s_lshl2_add_u32 s10, s9, s1"), + "new decoder did not decode old-backed S_LSHL2_ADD_U32"); + Check(Common::ContainsStr(result.decoded_dump, "s_lshl3_add_u32 s11, s10, s1"), + "new decoder did not decode old-backed S_LSHL3_ADD_U32"); + Check(Common::ContainsStr(result.decoded_dump, "s_lshl4_add_u32 s12, s11, s1"), + "new decoder did not decode old-backed S_LSHL4_ADD_U32"); + Check(Common::ContainsStr(result.decoded_dump, "s_mul_hi_u32 s13, s12, s1"), + "new decoder did not decode old-backed S_MUL_HI_U32"); + Check(Common::ContainsStr(result.decoded_dump, "v_add_f32 v1"), + "new decoder did not decode VOP2 float add"); + Check(Common::ContainsStr(result.decoded_dump, "v_cndmask_b32 v5"), + "new decoder did not decode VOP2 conditional mask select"); + Check(Common::ContainsStr(result.decoded_dump, "v_mul_u32_u24 v6"), + "new decoder did not decode VOP2 24-bit multiply"); + Check(Common::ContainsStr(result.decoded_dump, "v_mul_i32_i24 v8"), + "new decoder did not decode VOP2 signed 24-bit multiply"); + Check(Common::ContainsStr(result.decoded_dump, "v_bcnt_u32_b32 v7"), + "new decoder did not decode old-backed V_BCNT_U32_B32"); + Check(Common::ContainsStr(result.ir_dump, "ScalarAddCarryU32 s2, s0, s1, 0x00000000"), + "SOP2 add did not lower to scalar carry-writing IR"); + Check(Common::ContainsStr(result.ir_dump, "ScalarAddCarryU32 s14, s13, 0x00000001, scc"), + "S_ADD_C_U32 did not lower to scalar carry IR"); + Check(Common::ContainsStr(result.ir_dump, "ScalarShiftLeftAddCarryU32 s9, s2, 0x00000001, s1"), + "S_LSHL1_ADD_U32 did not lower through carry-writing shift-left-add IR"); + Check(Common::ContainsStr(result.ir_dump, "ScalarShiftLeftAddCarryU32 s10, s9, 0x00000002, s1"), + "S_LSHL2_ADD_U32 did not lower through carry-writing shift-left-add IR"); + Check( + Common::ContainsStr(result.ir_dump, "ScalarShiftLeftAddCarryU32 s11, s10, 0x00000003, s1"), + "S_LSHL3_ADD_U32 did not lower through carry-writing shift-left-add IR"); + Check( + Common::ContainsStr(result.ir_dump, "ScalarShiftLeftAddCarryU32 s12, s11, 0x00000004, s1"), + "S_LSHL4_ADD_U32 did not lower through carry-writing shift-left-add IR"); + Check(Common::ContainsStr(result.ir_dump, "UMulHighU32 s13, s12, s1"), + "S_MUL_HI_U32 did not lower to unsigned high-multiply IR"); + Check(Common::ContainsStr(result.ir_dump, "CompareGtU32"), "SOPC compare did not lower to IR"); + Check(Common::ContainsStr(result.ir_dump, "FAddF32 v1"), "VOP2 float add did not lower to IR"); + Check(Common::ContainsStr(result.ir_dump, "BitwiseAndU32 v4"), + "VOP2 bitwise op did not lower to IR"); + Check(Common::ContainsStr(result.ir_dump, "SelectMaskU32 v5, vcc_lo, v2, v1"), + "VOP2 conditional mask did not lower through lane-mask select IR"); + Check(Common::ContainsStr(result.ir_dump, "UMulU24U32 v6"), + "VOP2 24-bit multiply did not lower to IR"); + Check(Common::ContainsStr(result.ir_dump, "IMulI24U32 v8"), + "VOP2 signed 24-bit multiply did not lower to IR"); + Check(Common::ContainsStr(result.ir_dump, "BitCountAddU32 v7, v4, v6"), + "V_BCNT_U32_B32 did not lower to bit-count-add IR"); + Check(SpirvContainsOpcode(result.spirv, 128), "SPIR-V binary does not contain OpIAdd"); + Check(SpirvContainsOpcode(result.spirv, 149), "SPIR-V binary does not contain OpIAddCarry"); + Check(SpirvContainsOpcode(result.spirv, 129), "SPIR-V binary does not contain OpFAdd"); + Check(SpirvContainsOpcode(result.spirv, 132), "SPIR-V binary does not contain OpIMul"); + Check(SpirvContainsOpcode(result.spirv, 151), "SPIR-V binary does not contain OpUMulExtended"); + Check(SpirvContainsOpcode(result.spirv, 205), "SPIR-V binary does not contain OpBitCount"); + Check(SpirvContainsOpcode(result.spirv, 169), "SPIR-V binary does not contain OpSelect"); + Check(SpirvContainsOpcode(result.spirv, 199), "SPIR-V binary does not contain OpBitwiseAnd"); + CheckSpirvBinaryValidates(result.spirv); +} + +void TestNewShaderRecompilerVop3LaneReadDestinationEncoding() { + // Lane-read instructions have scalar results, but their VOP3A destination is encoded in + // VDST [7:0], not the VOP3B SDST field [14:8]. + const uint32_t shader[] = { + EncodeVop3Word0(0x182, 25), EncodeVop3Word1(5 + 256, 0, 0), // v_readfirstlane_b32 s25, v5 + EncodeVop3Word0(0x360, 26), EncodeVop3Word1(5 + 256, 130, 0), // v_readlane_b32 s26, v5, 2 + EncodeSopp(0x01, 0), + }; + + ShaderRecompiler::CompileOptions options; + options.stage = ShaderType::Compute; + options.dump_ir = true; + + ShaderRecompiler::CompileResult result; + std::string error; + Check(ShaderRecompiler::TryRecompile(shader, options, &result, &error), error.c_str()); + Check(Common::ContainsStr(result.decoded_dump, "v_readfirstlane_b32 s25, v5"), + "VOP3 V_READFIRSTLANE_B32 destination was not decoded from VDST"); + Check(Common::ContainsStr(result.decoded_dump, "v_readlane_b32 s26, v5, 2"), + "VOP3 V_READLANE_B32 destination was not decoded from VDST"); + CheckSpirvBinaryValidates(result.spirv); +} + +void TestNewShaderRecompilerMoreAluFamilies() { + const uint32_t shader[] = { + EncodeSopk(0x00, 9, 7), // s_movk_i32 s9, 7 + EncodeSopk(0x0b, 9, 3), // s_cmp_gt_u32 s9, 3 + EncodeVop1(0x00, 0, 0), // v_nop + EncodeVop1(0x01, 5, 9), // v_mov_b32 v5, s9 + EncodeVop1(0x01, 103, 249), + EncodeVop1Sdwa(5, 6, 0, 4), // v_mov_b32 v103, v5 word0 + EncodeVop1(0x06, 104, 249), + EncodeVop1Sdwa(103, 6, 0, 4), // v_cvt_f32_u32 v104, v103 word0 + EncodeVop1(0x0a, 105, 249), + EncodeVop1Sdwa(6, 5, 2, 6), // v_cvt_f16_f32 v105.hi, v6 + EncodeVop1(0x01, 106, 250), + EncodeVop1Dpp(5), // v_mov_b32 v106, v5 dpp + EncodeVop1(0x02, 24, 5 + 256), // v_readfirstlane_b32 s24, v5 + EncodeVop1(0x06, 6, 5 + 256), // v_cvt_f32_u32 v6, v5 + EncodeVop1(0x07, 7, 6 + 256), // v_cvt_u32_f32 v7, v6 + EncodeVop1(0x05, 9, 5 + 256), // v_cvt_f32_i32 v9, v5 + EncodeVop1(0x08, 10, 6 + 256), // v_cvt_i32_f32 v10, v6 + EncodeVop1(0x0a, 99, 6 + 256), // v_cvt_f16_f32 v99, v6 + EncodeVop1(0x0b, 100, 99 + 256), // v_cvt_f32_f16 v100, v99 + 0x7e1016f9u, + 0x00250602u, // v_cvt_f32_f16 v8, abs(v2.hi) + EncodeVop1(0x0d, 64, 6 + 256), // v_cvt_flr_i32_f32 v64, v6 + EncodeVop1(0x0e, 81, 5 + 256), // v_cvt_off_f32_i4 v81, v5 + EncodeVop1(0x11, 65, 5 + 256), // v_cvt_f32_ubyte0 v65, v5 + EncodeVop1(0x12, 66, 5 + 256), // v_cvt_f32_ubyte1 v66, v5 + EncodeVop1(0x13, 67, 5 + 256), // v_cvt_f32_ubyte2 v67, v5 + EncodeVop1(0x14, 68, 5 + 256), // v_cvt_f32_ubyte3 v68, v5 + EncodeVop1(0x2a, 11, 6 + 256), // v_rcp_f32 v11, v6 + EncodeVop1(0x20, 12, 6 + 256), // v_fract_f32 v12, v6 + EncodeVop1(0x21, 13, 6 + 256), // v_trunc_f32 v13, v6 + EncodeVop1(0x22, 14, 6 + 256), // v_ceil_f32 v14, v6 + EncodeVop1(0x23, 15, 6 + 256), // v_rndne_f32 v15, v6 + EncodeVop1(0x24, 16, 6 + 256), // v_floor_f32 v16, v6 + EncodeVop1(0x25, 17, 6 + 256), // v_exp_f32 v17, v6 + EncodeVop1(0x27, 18, 6 + 256), // v_log_f32 v18, v6 + EncodeVop1(0x2e, 19, 6 + 256), // v_rsq_f32 v19, v6 + EncodeVop1(0x33, 20, 6 + 256), // v_sqrt_f32 v20, v6 + EncodeVop1(0x35, 21, 6 + 256), // v_sin_f32 v21, v6 + EncodeVop1(0x36, 22, 6 + 256), // v_cos_f32 v22, v6 + EncodeVop1(0x37, 27, 5 + 256), // v_not_b32 v27, v5 + EncodeVop1(0x38, 28, 5 + 256), // v_bfrev_b32 v28, v5 + EncodeVop1(0x39, 31, 5 + 256), // v_ffbh_u32 v31, v5 + EncodeVop1(0x3a, 32, 5 + 256), // v_ffbl_b32 v32, v5 + 0x7e6e870cu, // v_movrels_b32 v55, v12 + EncodeVop2(0x1e, 83, 5 + 256, 6), // v_xnor_b32 v83, v5, v6 + EncodeVop2(0x23, 84, 5 + 256, 6), // v_mbcnt_lo_u32_b32 v84, v5, v6 + EncodeVop2(0x24, 85, 5 + 256, 6), // v_mbcnt_hi_u32_b32 v85, v5, v6 + EncodeVop2(0x1f, 89, 6 + 256, 6), // v_mac_f32 v89, v6, v6 + EncodeVop2(0x20, 90, 6 + 256, 5), + 0x3f800000u, // v_madmk_f32 v90, v6, 1.0, v5 + EncodeVop2(0x21, 91, 6 + 256, 5), + 0x40000000u, // v_madak_f32 v91, v6, v5, 2.0 + EncodeVop2(0x2b, 92, 6 + 256, 6), // v_mac_f32 v92, v6, v6 + EncodeVop2(0x2c, 93, 6 + 256, 5), + 0x3f000000u, // v_madmk_f32 v93, v6, 0.5, v5 + EncodeVop2(0x2d, 94, 6 + 256, 5), + 0x40400000u, // v_madak_f32 v94, v6, v5, 3.0 + EncodeVop2(0x2f, 95, 6 + 256, 6), // v_cvt_pkrtz_f16_f32 v95, v6, v6 + EncodeVop2(0x02, 102, 95 + 256, 95), // v_dot2c_f32_f16 v102, v95, v95 + EncodeVop2(0x28, 97, 5 + 256, 6), // v_addc_u32 v97, vcc, v5, v6, vcc + EncodeVop2(0x25, 123, 249, 6), + EncodeVop2Sdwa(5, 6, 0, 4, 6), + EncodeVop2(0x1b, 124, 249, 6), + EncodeVop2Sdwa(5, 6, 0, 4, 5), + 0x025e6af9u, + 0x16060635u, // v_cndmask_b32 v47, v53, -v53 + 0x100490f9u, + 0x86860600u, // v_mul_f32 v2, s0, s72 (SDWA full) + EncodeVop2(0x35, 9, 249, 1), + EncodeVop2Sdwa(0, 4, 2, 4, 4), + 0x660000f9u, + EncodeVop2Sdwa(0, 4, 2, 4, 4), + EncodeVop2(0x03, 125, 250, 6), + EncodeVop2Dpp(5), + EncodeVop3Word0(0x103, 35), + EncodeVop3Word1(6 + 256, 6 + 256, 0), + EncodeVop3Word0(0x10b, 36), + EncodeVop3Word1(5 + 256, 5 + 256, 0), + EncodeVop3Word0(0x11e, 86), + EncodeVop3Word1(5 + 256, 6 + 256, 0), + EncodeVop3Word0(0x11f, 96), + EncodeVop3Word1(6 + 256, 6 + 256, 0), + EncodeVop3Word0Sdst(0x128, 98, 30), + EncodeVop3Word1(5 + 256, 6 + 256, 106), + EncodeVop3Word0(0x123, 87), + EncodeVop3Word1(5 + 256, 6 + 256, 0), + EncodeVop3Word0(0x124, 88), + EncodeVop3Word1(5 + 256, 6 + 256, 0), + EncodeVop3Word0(0x180, 0), + EncodeVop3Word1(0, 0, 0), + EncodeVop3Word0(0x181, 23), + EncodeVop3Word1(5 + 256, 0, 0), + EncodeVop3Word0(0x182, 25), + EncodeVop3Word1(5 + 256, 0, 0), + EncodeVop3Word0(0x185, 24), + EncodeVop3Word1(5 + 256, 0, 0), + EncodeVop3Word0(0x18a, 101), + EncodeVop3Word1(6 + 256, 0, 0), + EncodeVop3Word0(0x18d, 74), + EncodeVop3Word1(6 + 256, 0, 0), + EncodeVop3Word0(0x18e, 82), + EncodeVop3Word1(5 + 256, 0, 0), + EncodeVop3Word0(0x1a0, 25), + EncodeVop3Word1(6 + 256, 0, 0), + EncodeVop3Word0(0x1aa, 26), + EncodeVop3Word1(6 + 256, 0, 0), + EncodeVop3Word0(0x1b7, 29), + EncodeVop3Word1(5 + 256, 0, 0), + EncodeVop3Word0(0x1b8, 30), + EncodeVop3Word1(5 + 256, 0, 0), + EncodeVop3Word0(0x1b9, 33), + EncodeVop3Word1(5 + 256, 0, 0), + EncodeVop3Word0(0x1ba, 34), + EncodeVop3Word1(5 + 256, 0, 0), + EncodeVopc(0x04, 6 + 256, 6), // v_cmp_gt_f32 v6, v6 + EncodeVopc(0xc4, 7 + 256, 5), // v_cmp_gt_u32 v7, v5 + EncodeVopc(0x14, 6 + 256, 6), // v_cmpx_gt_f32 v6, v6 + EncodeVopc(0xd4, 7 + 256, 5), // v_cmpx_gt_u32 v7, v5 + EncodeVopc(0x00, 6 + 256, 6), // v_cmp_f_f32 v6, v6 + EncodeVopc(0x0f, 6 + 256, 6), // v_cmp_tru_f32 v6, v6 + EncodeVopc(0x07, 6 + 256, 6), // v_cmp_o_f32 v6, v6 + EncodeVopc(0x08, 6 + 256, 6), // v_cmp_u_f32 v6, v6 + EncodeVopc(0x09, 6 + 256, 6), // v_cmp_nge_f32 v6, v6 + EncodeVopc(0x0a, 6 + 256, 6), // v_cmp_nlg_f32 v6, v6 + EncodeVopc(0x0b, 6 + 256, 6), // v_cmp_ngt_f32 v6, v6 + EncodeVopc(0x0c, 6 + 256, 6), // v_cmp_nle_f32 v6, v6 + EncodeVopc(0x0d, 6 + 256, 6), // v_cmp_neq_f32 v6, v6 + 0x7c1a02f9u, + 0x068680f0u, // v_cmp_neq_f32 s0, 0.5, v1 (SDWA) + EncodeVopc(0xd1, 249, 6), + EncodeVopcSdwa(5, 0, 0, 4), // v_cmpx_lt_u32 exec, v5, v6 (SDWA) + EncodeVopc(0x0e, 6 + 256, 6), // v_cmp_nlt_f32 v6, v6 + EncodeVopc(0x19, 6 + 256, 6), // v_cmpx_nge_f32 v6, v6 + EncodeVopc(0x1a, 6 + 256, 6), // v_cmpx_nlg_f32 v6, v6 + EncodeVopc(0x1b, 6 + 256, 6), // v_cmpx_ngt_f32 v6, v6 + EncodeVopc(0x1c, 6 + 256, 6), // v_cmpx_nle_f32 v6, v6 + EncodeVopc(0x1d, 6 + 256, 6), // v_cmpx_neq_f32 v6, v6 + EncodeVopc(0x1e, 6 + 256, 6), // v_cmpx_nlt_f32 v6, v6 + EncodeVopc(0x80, 5 + 256, 5), // v_cmp_f_i32 v5, v5 + EncodeVopc(0x87, 5 + 256, 5), // v_cmp_t_i32 v5, v5 + EncodeVopc(0xc0, 5 + 256, 5), // v_cmp_f_u32 v5, v5 + EncodeVopc(0xc7, 5 + 256, 5), // v_cmp_t_u32 v5, v5 + 0xd4e5006au, + 0x0000d47eu, // v_cmp_ne_u64 vcc, exec, vcc + EncodeVop3Word0(0x141, 8), + EncodeVop3Word1(6 + 256, 6 + 256, 6 + 256), + 0xd5410004u, + 0x20121301u, // v_mad_f32 v4, -v1, v9, s4 + EncodeVop3Word0(0x144, 110), + EncodeVop3Word1(5 + 256, 6 + 256, 7 + 256), + EncodeVop3Word0(0x145, 111), + EncodeVop3Word1(5 + 256, 6 + 256, 7 + 256), + EncodeVop3Word0(0x146, 112), + EncodeVop3Word1(5 + 256, 6 + 256, 7 + 256), + EncodeVop3Word0(0x147, 113), + EncodeVop3Word1(5 + 256, 6 + 256, 7 + 256), + 0xd5470005u, + 0x841a0102u, // v_cubema_f32 v5, v2, v0, -v6 + EncodeVop3Word0(0x14b, 37), + EncodeVop3Word1(6 + 256, 6 + 256, 6 + 256), + EncodeVop3Word0(0x151, 38), + EncodeVop3Word1(6 + 256, 6 + 256, 6 + 256), + EncodeVop3Word0(0x154, 39), + EncodeVop3Word1(6 + 256, 6 + 256, 6 + 256), + EncodeVop3Word0(0x157, 40), + EncodeVop3Word1(6 + 256, 6 + 256, 6 + 256), + EncodeVop3Word0(0x152, 41), + EncodeVop3Word1(5 + 256, 5 + 256, 5 + 256), + EncodeVop3Word0(0x153, 42), + EncodeVop3Word1(5 + 256, 5 + 256, 5 + 256), + EncodeVop3Word0(0x155, 43), + EncodeVop3Word1(5 + 256, 5 + 256, 5 + 256), + EncodeVop3Word0(0x156, 44), + EncodeVop3Word1(5 + 256, 5 + 256, 5 + 256), + EncodeVop3Word0(0x158, 45), + EncodeVop3Word1(5 + 256, 5 + 256, 5 + 256), + EncodeVop3Word0(0x159, 46), + EncodeVop3Word1(5 + 256, 5 + 256, 5 + 256), + EncodeVop3Word0(0x148, 47), + EncodeVop3Word1(5 + 256, 129, 132), + EncodeVop3Word0(0x149, 48), + EncodeVop3Word1(5 + 256, 129, 132), + EncodeVop3Word0(0x14a, 49), + EncodeVop3Word1(5 + 256, 6 + 256, 7 + 256), + EncodeVop3Word0(0x14e, 50), + EncodeVop3Word1(5 + 256, 6 + 256, 129), + EncodeVop3Word0(0x36d, 51), + EncodeVop3Word1(5 + 256, 6 + 256, 7 + 256), + EncodeVop3Word0(0x169, 52), + EncodeVop3Word1(5 + 256, 6 + 256, 0), + EncodeVop3Word0(0x16a, 53), + EncodeVop3Word1(5 + 256, 6 + 256, 0), + EncodeVop3Word0(0x16c, 114), + EncodeVop3Word1(5 + 256, 6 + 256, 0), + EncodeVop3Word0(0x371, 54), + EncodeVop3Word1(5 + 256, 6 + 256, 7 + 256), + EncodeVop3Word0(0x372, 55), + EncodeVop3Word1(5 + 256, 6 + 256, 7 + 256), + EncodeVop3Word0(0x178, 56), + EncodeVop3Word1(5 + 256, 6 + 256, 7 + 256), + EncodeVop3Word0(0x346, 57), + EncodeVop3Word1(5 + 256, 6 + 256, 7 + 256), + EncodeVop3Word0(0x347, 58), + EncodeVop3Word1(5 + 256, 6 + 256, 7 + 256), + EncodeVop3Word0(0x345, 59), + EncodeVop3Word1(5 + 256, 6 + 256, 7 + 256), + EncodeVop3Word0(0x36f, 60), + EncodeVop3Word1(5 + 256, 6 + 256, 7 + 256), + EncodeVop3Word0(0x15d, 61), + EncodeVop3Word1(5 + 256, 6 + 256, 7 + 256), + EncodeVop3Word0(0x142, 62), + EncodeVop3Word1(5 + 256, 6 + 256, 7 + 256), + EncodeVop3Word0(0x143, 63), + EncodeVop3Word1(5 + 256, 6 + 256, 7 + 256), + EncodeVop3Word0(0x16b, 69), + EncodeVop3Word1(5 + 256, 6 + 256, 0), + EncodeVop3Word0(0x30f, 70), + EncodeVop3Word1(5 + 256, 6 + 256, 0), + EncodeVop3Word0(0x310, 71), + EncodeVop3Word1(5 + 256, 6 + 256, 0), + EncodeVop3Word0(0x319, 72), + EncodeVop3Word1(5 + 256, 6 + 256, 0), + 0xd70a1007u, + 0x00020affu, + 0x0000ffffu, // v_max_i16 v7, 0xffff, v5.hi + 0xd70c5007u, + 0x00020882u, // v_min_i16 v7, 2, v4.hi + EncodeVop3Word0(0x363, 73), + EncodeVop3Word1(129, 132, 0), + EncodeVop3Word0(0x360, 26), + EncodeVop3Word1(5 + 256, 130, 0), + EncodeVop3Word0(0x361, 107), + EncodeVop3Word1(5 + 256, 130, 0), + EncodeVop3Word0(0x377, 108), + EncodeVop3Word1(5 + 256, 128, 128), + EncodeVop3Word0(0x378, 109), + EncodeVop3Word1(5 + 256, 128, 128), + EncodeVop3Word0(0x12f, 75), + EncodeVop3Word1(6 + 256, 6 + 256, 0), + 0xd52f0000u, + 0x60020300u, // v_cvt_pkrtz_f16_f32 v0, -v0, -v1 + EncodeVop3Word0(0x362, 76), + EncodeVop3Word1(6 + 256, 129, 0), + 0xd7620107u, + 0x00018509u, // v_ldexp_f32 v7, abs(v9), -2 + 0xd762800du, + 0x00018906u, // v_ldexp_f32 clamp v13, v6, -4 + EncodeVop3Word0(0x364, 77), + EncodeVop3Word1(5 + 256, 6 + 256, 0), + EncodeVop3Word0(0x368, 78), + EncodeVop3Word1(6 + 256, 6 + 256, 0), + EncodeVop3Word0(0x369, 79), + EncodeVop3Word1(6 + 256, 6 + 256, 0), + EncodeVop3Word0(0x36a, 80), + EncodeVop3Word1(5 + 256, 7 + 256, 0), + 0xbf810000u, + }; + + ShaderRecompiler::CompileOptions options; + options.stage = ShaderType::Compute; + options.dump_ir = true; + + ShaderRecompiler::CompileResult result; + std::string error; + Check(ShaderRecompiler::TryRecompile(shader, options, &result, &error), error.c_str()); + Check(Common::ContainsStr(result.decoded_dump, "s_movk_i32 s9"), + "new decoder did not decode SOPK mov"); + Check(Common::ContainsStr(result.decoded_dump, "s_cmp_gt_u32"), + "new decoder did not decode SOPK compare"); + Check(Common::ContainsStr(result.decoded_dump, "v_nop"), + "new decoder did not decode old-backed VOP1 no-op"); + Check(Common::ContainsStr(result.decoded_dump, "v_mov_b32 v5"), + "new decoder did not decode VOP1 mov"); + Check(Common::ContainsStr(result.decoded_dump, "v_movrels_b32 v55, v12"), + "new decoder did not decode VOP1 V_MOVRELS_B32"); + Check(Common::ContainsStr(result.decoded_dump, "v_mov_b32 v103, v5.sdwa(sel=4"), + "new decoder did not decode VOP1 SDWA source selector"); + Check(Common::ContainsStr(result.decoded_dump, "v_cvt_f16_f32 v105.sdwa(sel=5"), + "new decoder did not decode VOP1 SDWA destination selector"); + Check(Common::ContainsStr(result.decoded_dump, "v_mov_b32 v106, v5.dpp"), + "new decoder did not decode VOP1 DPP source metadata"); + Check(!Common::ContainsStr(result.decoded_dump, "VOP1 SDWA/DPP modifiers are not implemented"), + "new decoder still reports blanket VOP1 SDWA/DPP unsupported reason"); + Check(Common::ContainsStr(result.decoded_dump, "v_add_nc_u32 v123, v5.sdwa(sel=4"), + "new decoder did not decode VOP2 SDWA source selector"); + Check(Common::ContainsStr(result.decoded_dump, "v_and_b32 v124, v5.sdwa(sel=4"), + "new decoder did not decode VOP2 SDWA first source selector"); + Check(Common::ContainsStr(result.decoded_dump, "v6.sdwa(sel=5"), + "new decoder did not decode VOP2 SDWA second source selector"); + Check(Common::ContainsStr(result.decoded_dump, "v_cndmask_b32 v47, v53,") && + Common::ContainsStr(result.decoded_dump, "v53.neg"), + "new decoder did not decode V_CNDMASK_B32 SDWA source modifier"); + Check(Common::ContainsStr(result.decoded_dump, "v_add_f32 v125, v5.dpp"), + "new decoder did not decode VOP2 DPP source metadata"); + Check(!Common::ContainsStr(result.decoded_dump, "VOP2 SDWA/DPP modifiers are not implemented"), + "new decoder still reports blanket VOP2 SDWA/DPP unsupported reason"); + Check(Common::ContainsStr(result.decoded_dump, "v_readfirstlane_b32 s24, v5"), + "new decoder did not decode old-backed V_READFIRSTLANE_B32"); + Check(Common::ContainsStr(result.decoded_dump, "v_cvt_f32_u32 v6"), + "new decoder did not decode VOP1 conversion"); + Check(Common::ContainsStr(result.decoded_dump, "v_cvt_f32_i32 v9"), + "new decoder did not decode VOP1 signed int-to-float conversion"); + Check(Common::ContainsStr(result.decoded_dump, "v_cvt_i32_f32 v10"), + "new decoder did not decode VOP1 float-to-signed-int conversion"); + Check(Common::ContainsStr(result.decoded_dump, "v_cvt_f16_f32 v99"), + "new decoder did not decode old-backed V_CVT_F16_F32"); + Check(Common::ContainsStr(result.decoded_dump, "v_cvt_f32_f16 v100"), + "new decoder did not decode old-backed native V_CVT_F32_F16"); + Check(Common::ContainsStr(result.decoded_dump, "v_cvt_f32_f16 v8, v2.sdwa(sel=5") && + Common::ContainsStr(result.decoded_dump, "v2.sdwa(sel=5,sext=0).abs"), + "new decoder did not decode V_CVT_F32_F16 SDWA source selector modifier"); + Check(Common::ContainsStr(result.decoded_dump, "v_cvt_flr_i32_f32 v64"), + "new decoder did not decode old-backed V_CVT_FLR_I32_F32"); + Check(Common::ContainsStr(result.decoded_dump, "v_cvt_off_f32_i4 v81"), + "new decoder did not decode old-backed V_CVT_OFF_F32_I4"); + Check(Common::ContainsStr(result.decoded_dump, "v_cvt_f32_ubyte0 v65"), + "new decoder did not decode old-backed V_CVT_F32_UBYTE0"); + Check(Common::ContainsStr(result.decoded_dump, "v_cvt_f32_ubyte1 v66"), + "new decoder did not decode old-backed V_CVT_F32_UBYTE1"); + Check(Common::ContainsStr(result.decoded_dump, "v_cvt_f32_ubyte2 v67"), + "new decoder did not decode old-backed V_CVT_F32_UBYTE2"); + Check(Common::ContainsStr(result.decoded_dump, "v_cvt_f32_ubyte3 v68"), + "new decoder did not decode old-backed V_CVT_F32_UBYTE3"); + Check(Common::ContainsStr(result.decoded_dump, "v_rcp_f32 v11"), + "new decoder did not decode VOP1 reciprocal"); + Check(Common::ContainsStr(result.decoded_dump, "v_fract_f32 v12"), + "new decoder did not decode VOP1 fract"); + Check(Common::ContainsStr(result.decoded_dump, "v_cos_f32 v22"), + "new decoder did not decode VOP1 cosine"); + Check(Common::ContainsStr(result.decoded_dump, "v_not_b32 v27"), + "new decoder did not decode VOP1 not"); + Check(Common::ContainsStr(result.decoded_dump, "v_bfrev_b32 v28"), + "new decoder did not decode VOP1 bit reverse"); + Check(Common::ContainsStr(result.decoded_dump, "v_ffbh_u32 v31"), + "new decoder did not decode VOP1 find-first-bit-high"); + Check(Common::ContainsStr(result.decoded_dump, "v_ffbl_b32 v32"), + "new decoder did not decode VOP1 find-first-bit-low"); + Check(Common::ContainsStr(result.decoded_dump, "v_xnor_b32 v83, v5, v6"), + "new decoder did not decode old-backed V_XNOR_B32"); + Check(Common::ContainsStr(result.decoded_dump, "v_mbcnt_lo_u32_b32 v84, v5, v6"), + "new decoder did not decode old-backed V_MBCNT_LO_U32_B32"); + Check(Common::ContainsStr(result.decoded_dump, "v_mbcnt_hi_u32_b32 v85, v5, v6"), + "new decoder did not decode old-backed V_MBCNT_HI_U32_B32"); + Check(Common::ContainsStr(result.decoded_dump, "v_mac_f32 v89, v6, v6"), + "new decoder did not decode old-backed V_MAC_F32"); + Check(Common::ContainsStr(result.decoded_dump, "v_madmk_f32 v90, v6, 0x3f800000, v5"), + "new decoder did not decode old-backed V_MADMK_F32 literal form"); + Check(Common::ContainsStr(result.decoded_dump, "v_madak_f32 v91, v6, v5, 0x40000000"), + "new decoder did not decode old-backed V_MADAK_F32 literal form"); + Check(Common::ContainsStr(result.decoded_dump, "v_mac_f32 v92, v6, v6"), + "new decoder did not decode old-backed alternate V_MAC_F32"); + Check(Common::ContainsStr(result.decoded_dump, "v_madmk_f32 v93, v6, 0x3f000000, v5"), + "new decoder did not decode old-backed alternate V_MADMK_F32"); + Check(Common::ContainsStr(result.decoded_dump, "v_madak_f32 v94, v6, v5, 0x40400000"), + "new decoder did not decode old-backed alternate V_MADAK_F32"); + Check(Common::ContainsStr(result.decoded_dump, "v_cvt_pkrtz_f16_f32 v95, v6, v6"), + "new decoder did not decode old-backed native V_CVT_PKRTZ_F16_F32"); + Check(Common::ContainsStr(result.decoded_dump, "v_dot2c_f32_f16 v102, v95, v95"), + "new decoder did not decode old-backed V_DOT2C_F32_F16"); + Check(Common::ContainsStr(result.decoded_dump, "v_addc_u32 v97, vcc_lo, v5, v6, vcc_lo"), + "new decoder did not decode old-backed V_ADD_CO_U32 carry form"); + Check(Common::ContainsStr(result.decoded_dump, "v_add_f32 v35"), + "new decoder did not decode VOP3-encoded VOP2 float add"); + Check(Common::ContainsStr(result.decoded_dump, "v_mul_u32_u24 v36"), + "new decoder did not decode VOP3-encoded VOP2 24-bit multiply"); + Check(Common::ContainsStr(result.decoded_dump, "v_xnor_b32 v86, v5, v6"), + "new decoder did not decode old-backed VOP3-encoded V_XNOR_B32"); + Check(Common::ContainsStr(result.decoded_dump, "v_mac_f32 v96, v6, v6"), + "new decoder did not decode old-backed VOP3-encoded V_MAC_F32"); + Check(Common::ContainsStr(result.decoded_dump, "v_addc_u32 v98, s30, v5, v6, vcc_lo"), + "new decoder did not decode old-backed VOP3 V_ADD_CO_U32 carry form"); + Check(Common::ContainsStr(result.decoded_dump, "v_mbcnt_lo_u32_b32 v87, v5, v6"), + "new decoder did not decode old-backed VOP3-encoded V_MBCNT_LO_U32_B32"); + Check(Common::ContainsStr(result.decoded_dump, "v_mbcnt_hi_u32_b32 v88, v5, v6"), + "new decoder did not decode old-backed VOP3-encoded V_MBCNT_HI_U32_B32"); + Check(Common::ContainsStr(result.decoded_dump, "v_mov_b32 v23"), + "new decoder did not decode VOP3-encoded VOP1 move"); + Check(Common::ContainsStr(result.decoded_dump, "v_readfirstlane_b32 s25, v5"), + "new decoder did not decode old-backed VOP3-encoded V_READFIRSTLANE_B32"); + Check(Common::ContainsStr(result.decoded_dump, "v_cvt_f32_i32 v24"), + "new decoder did not decode VOP3-encoded VOP1 signed conversion"); + Check(Common::ContainsStr(result.decoded_dump, "v_cvt_f16_f32 v101"), + "new decoder did not decode old-backed VOP3-encoded V_CVT_F16_F32"); + Check(Common::ContainsStr(result.decoded_dump, "v_cvt_flr_i32_f32 v74"), + "new decoder did not decode old-backed VOP3-encoded V_CVT_FLR_I32_F32"); + Check(Common::ContainsStr(result.decoded_dump, "v_cvt_off_f32_i4 v82"), + "new decoder did not decode old-backed VOP3-encoded V_CVT_OFF_F32_I4"); + Check(Common::ContainsStr(result.decoded_dump, "v_fract_f32 v25"), + "new decoder did not decode VOP3-encoded VOP1 fract"); + Check(Common::ContainsStr(result.decoded_dump, "v_rcp_f32 v26"), + "new decoder did not decode VOP3-encoded VOP1 reciprocal"); + Check(Common::ContainsStr(result.decoded_dump, "v_not_b32 v29"), + "new decoder did not decode VOP3-encoded VOP1 not"); + Check(Common::ContainsStr(result.decoded_dump, "v_bfrev_b32 v30"), + "new decoder did not decode VOP3-encoded VOP1 bit reverse"); + Check(Common::ContainsStr(result.decoded_dump, "v_ffbh_u32 v33"), + "new decoder did not decode VOP3-encoded VOP1 find-first-bit-high"); + Check(Common::ContainsStr(result.decoded_dump, "v_ffbl_b32 v34"), + "new decoder did not decode VOP3-encoded VOP1 find-first-bit-low"); + Check(Common::ContainsStr(result.decoded_dump, "v_cmp_gt_f32"), + "new decoder did not decode VOPC float compare"); + Check(Common::ContainsStr(result.decoded_dump, "v_cmpx_gt_f32"), + "new decoder did not decode VOPC float compare-and-mask"); + Check(Common::ContainsStr(result.decoded_dump, "v_cmpx_gt_u32"), + "new decoder did not decode VOPC uint compare-and-mask"); + Check(Common::ContainsStr(result.decoded_dump, "v_cmp_f_f32"), + "new decoder did not decode old-backed V_CMP_F_F32"); + Check(Common::ContainsStr(result.decoded_dump, "v_cmp_tru_f32"), + "new decoder did not decode old-backed V_CMP_TRU_F32"); + Check(Common::ContainsStr(result.decoded_dump, "v_cmp_o_f32"), + "new decoder did not decode old-backed V_CMP_O_F32"); + Check(Common::ContainsStr(result.decoded_dump, "v_cmp_u_f32"), + "new decoder did not decode old-backed V_CMP_U_F32"); + Check(Common::ContainsStr(result.decoded_dump, "v_cmp_nge_f32"), + "new decoder did not decode old-backed V_CMP_NGE_F32"); + Check(Common::ContainsStr(result.decoded_dump, "v_cmp_nlg_f32"), + "new decoder did not decode old-backed V_CMP_NLG_F32"); + Check(Common::ContainsStr(result.decoded_dump, "v_cmp_ngt_f32"), + "new decoder did not decode old-backed V_CMP_NGT_F32"); + Check(Common::ContainsStr(result.decoded_dump, "v_cmp_nle_f32"), + "new decoder did not decode old-backed V_CMP_NLE_F32"); + Check(Common::ContainsStr(result.decoded_dump, "v_cmp_neq_f32"), + "new decoder did not decode old-backed V_CMP_NEQ_F32"); + Check(Common::ContainsStr(result.decoded_dump, "v_cmp_neq_f32 s0, 0.500000, v1"), + "new decoder did not decode old-backed V_CMP_NEQ_F32 SDWA scalar destination"); + Check(Common::ContainsStr(result.decoded_dump, "v_cmpx_lt_u32 exec_lo, v5.sdwa(sel=4") && + Common::ContainsStr(result.ir_dump, "CompareMaskLtU32 exec_lo"), + "new decoder did not route V_CMPX SDWA destination to exec"); + Check(Common::ContainsStr(result.decoded_dump, "v_cmp_nlt_f32"), + "new decoder did not decode old-backed V_CMP_NLT_F32"); + Check(Common::ContainsStr(result.decoded_dump, "v_cmpx_nge_f32"), + "new decoder did not decode old-backed V_CMPX_NGE_F32"); + Check(Common::ContainsStr(result.decoded_dump, "v_cmpx_nlg_f32"), + "new decoder did not decode old-backed V_CMPX_NLG_F32"); + Check(Common::ContainsStr(result.decoded_dump, "v_cmpx_ngt_f32"), + "new decoder did not decode old-backed V_CMPX_NGT_F32"); + Check(Common::ContainsStr(result.decoded_dump, "v_cmpx_nle_f32"), + "new decoder did not decode old-backed V_CMPX_NLE_F32"); + Check(Common::ContainsStr(result.decoded_dump, "v_cmpx_neq_f32"), + "new decoder did not decode old-backed V_CMPX_NEQ_F32"); + Check(Common::ContainsStr(result.decoded_dump, "v_cmpx_nlt_f32"), + "new decoder did not decode old-backed V_CMPX_NLT_F32"); + Check(Common::ContainsStr(result.decoded_dump, "v_cmp_f_i32"), + "new decoder did not decode old-backed V_CMP_F_I32"); + Check(Common::ContainsStr(result.decoded_dump, "v_cmp_t_i32"), + "new decoder did not decode old-backed V_CMP_T_I32"); + Check(Common::ContainsStr(result.decoded_dump, "v_cmp_f_u32"), + "new decoder did not decode old-backed V_CMP_F_U32"); + Check(Common::ContainsStr(result.decoded_dump, "v_cmp_t_u32"), + "new decoder did not decode old-backed V_CMP_T_U32"); + Check(Common::ContainsStr(result.decoded_dump, "v_cmp_ne_u64 vcc_lo, exec_lo, vcc_lo"), + "new decoder did not decode VOP3-encoded V_CMP_NE_U64"); + Check(Common::ContainsStr(result.ir_dump, "CompareNeU64 vcc_lo, exec_lo, vcc_lo"), + "V_CMP_NE_U64 did not lower to 64-bit compare IR"); + Check(Common::ContainsStr(result.decoded_dump, "v_mad_f32 v8"), + "new decoder did not decode VOP3 mad"); + Check(Common::ContainsStr(result.decoded_dump, "v_mad_f32 v4, v1.neg, v9, s4"), + "new decoder did not decode VOP3 mad source modifiers"); + Check(Common::ContainsStr(result.decoded_dump, "v_cubeid_f32 v110, v5, v6, v7"), + "new decoder did not decode old-backed V_CUBEID_F32"); + Check(Common::ContainsStr(result.decoded_dump, "v_cubesc_f32 v111, v5, v6, v7"), + "new decoder did not decode old-backed V_CUBESC_F32"); + Check(Common::ContainsStr(result.decoded_dump, "v_cubetc_f32 v112, v5, v6, v7"), + "new decoder did not decode old-backed V_CUBETC_F32"); + Check(Common::ContainsStr(result.decoded_dump, "v_cubema_f32 v113, v5, v6, v7"), + "new decoder did not decode old-backed V_CUBEMA_F32"); + Check(Common::ContainsStr(result.decoded_dump, "v_cubema_f32 v5, v2, v0, v6.neg"), + "new decoder did not decode V_CUBEMA_F32 source modifier"); + Check(Common::ContainsStr(result.decoded_dump, "v_fma_f32 v37"), + "new decoder did not decode VOP3 fma"); + Check(Common::ContainsStr(result.decoded_dump, "v_min3_f32 v38"), + "new decoder did not decode VOP3 min3"); + Check(Common::ContainsStr(result.decoded_dump, "v_max3_f32 v39"), + "new decoder did not decode VOP3 max3"); + Check(Common::ContainsStr(result.decoded_dump, "v_med3_f32 v40"), + "new decoder did not decode VOP3 med3"); + Check(Common::ContainsStr(result.decoded_dump, "v_min3_i32 v41"), + "new decoder did not decode VOP3 signed min3"); + Check(Common::ContainsStr(result.decoded_dump, "v_min3_u32 v42"), + "new decoder did not decode VOP3 unsigned min3"); + Check(Common::ContainsStr(result.decoded_dump, "v_max3_i32 v43"), + "new decoder did not decode VOP3 signed max3"); + Check(Common::ContainsStr(result.decoded_dump, "v_max3_u32 v44"), + "new decoder did not decode VOP3 unsigned max3"); + Check(Common::ContainsStr(result.decoded_dump, "v_med3_i32 v45"), + "new decoder did not decode VOP3 signed med3"); + Check(Common::ContainsStr(result.decoded_dump, "v_med3_u32 v46"), + "new decoder did not decode VOP3 unsigned med3"); + Check(Common::ContainsStr(result.decoded_dump, "v_bfe_u32 v47"), + "new decoder did not decode VOP3 unsigned bitfield extract"); + Check(Common::ContainsStr(result.decoded_dump, "v_bfe_i32 v48"), + "new decoder did not decode VOP3 signed bitfield extract"); + Check(Common::ContainsStr(result.decoded_dump, "v_bfi_b32 v49"), + "new decoder did not decode VOP3 bitfield insert-select"); + Check(Common::ContainsStr(result.decoded_dump, "v_alignbit_b32 v50"), + "new decoder did not decode VOP3 alignbit"); + Check(Common::ContainsStr(result.decoded_dump, "v_add3_u32 v51"), + "new decoder did not decode VOP3 add3"); + Check(Common::ContainsStr(result.decoded_dump, "v_mul_lo_u32 v52, v5, v6"), + "new decoder did not decode old-backed V_MUL_LO_U32"); + Check(Common::ContainsStr(result.decoded_dump, "v_mul_hi_u32 v53, v5, v6"), + "new decoder did not decode old-backed V_MUL_HI_U32"); + Check(Common::ContainsStr(result.decoded_dump, "v_mul_hi_i32 v114, v5, v6"), + "new decoder did not decode RDNA2 V_MUL_HI_I32"); + Check(Common::ContainsStr(result.decoded_dump, "v_and_or_b32 v54, v5, v6, v7"), + "new decoder did not decode old-backed V_AND_OR_B32"); + Check(Common::ContainsStr(result.decoded_dump, "v_or3_b32 v55, v5, v6, v7"), + "new decoder did not decode old-backed V_OR3_B32"); + Check(Common::ContainsStr(result.decoded_dump, "v_xor3_b32 v56, v5, v6, v7"), + "new decoder did not decode old-backed V_XOR3_B32"); + Check(Common::ContainsStr(result.decoded_dump, "v_lshl_add_u32 v57, v5, v6, v7"), + "new decoder did not decode old-backed V_LSHL_ADD_U32"); + Check(Common::ContainsStr(result.decoded_dump, "v_add_lshl_u32 v58, v5, v6, v7"), + "new decoder did not decode old-backed V_ADD_LSHL_U32"); + Check(Common::ContainsStr(result.decoded_dump, "v_xad_u32 v59, v5, v6, v7"), + "new decoder did not decode old-backed V_XAD_U32"); + Check(Common::ContainsStr(result.decoded_dump, "v_lshl_or_b32 v60, v5, v6, v7"), + "new decoder did not decode old-backed V_LSHL_OR_B32"); + Check(Common::ContainsStr(result.decoded_dump, "v_sad_u32 v61, v5, v6, v7"), + "new decoder did not decode old-backed V_SAD_U32"); + Check(Common::ContainsStr(result.decoded_dump, "v_mad_i32_i24 v62, v5, v6, v7"), + "new decoder did not decode old-backed V_MAD_I32_I24"); + Check(Common::ContainsStr(result.decoded_dump, "v_mad_u32_u24 v63, v5, v6, v7"), + "new decoder did not decode old-backed V_MAD_U32_U24"); + Check(Common::ContainsStr(result.decoded_dump, "v_mul_lo_i32 v69, v5, v6"), + "new decoder did not decode old-backed V_MUL_LO_I32"); + Check(Common::ContainsStr(result.decoded_dump, "v_add_i32 v70, s0, v5, v6"), + "new decoder did not decode old-backed V_ADD_I32"); + Check(Common::ContainsStr(result.decoded_dump, "v_sub_i32 v71, s0, v5, v6"), + "new decoder did not decode RDNA2 V_SUB_CO_U32"); + Check(Common::ContainsStr(result.decoded_dump, "v_subrev_i32 v72, s0, v5, v6"), + "new decoder did not decode old-backed V_SUBREV_I32"); + Check(Common::ContainsStr( + result.decoded_dump, + "v_max_i16 v7.sdwa(sel=4,sext=0), 0x0000ffff, v5.opsel(lo=1,hi=0,neghi=0)"), + "new decoder did not decode RDNA2 V_MAX_I16 literal/op_sel form"); + Check(Common::ContainsStr(result.decoded_dump, + "v_min_i16 v7.sdwa(sel=5,sext=0), 2, v4.opsel(lo=1,hi=0,neghi=0)"), + "new decoder did not decode RDNA2 V_MIN_I16 op_sel form"); + Check(Common::ContainsStr(result.decoded_dump, "v_bfm_b32 v73, 1, 4"), + "new decoder did not decode old-backed V_BFM_B32"); + Check(Common::ContainsStr(result.decoded_dump, "v_readlane_b32 s26, v5, 2"), + "new decoder did not decode old-backed V_READLANE_B32"); + Check(Common::ContainsStr(result.decoded_dump, "v_writelane_b32 v107, v5, 2"), + "new decoder did not decode old-backed V_WRITELANE_B32"); + Check(Common::ContainsStr(result.decoded_dump, "v_permlane16_b32 v108, v5, 0, 0"), + "new decoder did not decode old-backed V_PERMLANE16_B32"); + Check(Common::ContainsStr(result.decoded_dump, "v_permlanex16_b32 v109, v5, 0, 0"), + "new decoder did not decode old-backed V_PERMLANEX16_B32"); + Check(Common::ContainsStr(result.decoded_dump, "v_cvt_pkrtz_f16_f32 v75, v6, v6"), + "new decoder did not decode old-backed V_CVT_PKRTZ_F16_F32"); + Check(Common::ContainsStr(result.decoded_dump, "v_cvt_pkrtz_f16_f32 v0, v0.neg, v1.neg"), + "new decoder did not decode V_CVT_PKRTZ_F16_F32 source modifiers"); + Check(Common::ContainsStr(result.decoded_dump, "v_ldexp_f32 v76, v6, 1"), + "new decoder did not decode old-backed V_LDEXP_F32"); + Check(Common::ContainsStr(result.decoded_dump, "v_ldexp_f32 v7, v9.abs, -2"), + "new decoder did not decode V_LDEXP_F32 source modifier"); + Check(Common::ContainsStr(result.decoded_dump, "v_ldexp_f32 v13.clamp, v6, -4"), + "new decoder did not decode V_LDEXP_F32 clamp modifier"); + Check(Common::ContainsStr(result.decoded_dump, "v_bcnt_u32_b32 v77, v5, v6"), + "new decoder did not decode old-backed VOP3 V_BCNT_U32_B32"); + Check(Common::ContainsStr(result.decoded_dump, "v_cvt_pknorm_i16_f32 v78, v6, v6"), + "new decoder did not decode old-backed V_CVT_PKNORM_I16_F32"); + Check(Common::ContainsStr(result.decoded_dump, "v_cvt_pknorm_u16_f32 v79, v6, v6"), + "new decoder did not decode old-backed V_CVT_PKNORM_U16_F32"); + Check(Common::ContainsStr(result.decoded_dump, "v_cvt_pk_u16_u32 v80, v5, v7"), + "new decoder did not decode old-backed V_CVT_PK_U16_U32"); + Check(Common::ContainsStr(result.decoded_dump, "v_mul_f32 v2, s0, s72"), + "new decoder did not decode old-backed V_MUL_F32 SDWA full-width form"); + Check(Common::ContainsStr(result.decoded_dump, "v_mul_f16 v9.sdwa(sel=4") && + Common::ContainsStr(result.decoded_dump, "v0.sdwa(sel=4") && + Common::ContainsStr(result.decoded_dump, "v1.sdwa(sel=4"), + "new decoder did not decode V_MUL_F16 SDWA low-half form"); + Check(Common::ContainsStr(result.decoded_dump, "v_sub_f16 v0.sdwa(sel=4") && + Common::ContainsStr(result.decoded_dump, "v0.sdwa(sel=4"), + "new decoder did not decode V_SUB_F16 SDWA low-half form"); + Check(Common::ContainsStr(result.ir_dump, "ConvertU32ToF32 v6"), + "VOP1 uint-to-float conversion did not lower to IR"); + Check(Common::ContainsStr(result.ir_dump, "MoveU32 v103, v5.sdwa(sel=4"), + "VOP1 SDWA source selector did not lower to IR metadata"); + Check(Common::ContainsStr(result.ir_dump, "ConvertF32ToF16 v105.sdwa(sel=5"), + "VOP1 SDWA destination selector did not lower to IR metadata"); + Check(Common::ContainsStr(result.ir_dump, "MoveU32 v106.dpp") && + Common::ContainsStr(result.ir_dump, "v5.dpp"), + "VOP1 DPP source/destination did not lower to IR metadata"); + Check(Common::ContainsStr(result.ir_dump, "MoveRelSourceU32 v55, v12, m0"), + "V_MOVRELS_B32 did not lower to indexed VGPR-source IR"); + Check(Common::ContainsStr(result.ir_dump, "IAddU32 v123, v5.sdwa(sel=4"), + "VOP2 SDWA did not lower first source metadata to IR"); + Check(Common::ContainsStr(result.ir_dump, "BitwiseAndU32 v124, v5.sdwa(sel=4"), + "VOP2 SDWA bitwise op did not lower first source metadata to IR"); + Check(Common::ContainsStr(result.ir_dump, "v6.sdwa(sel=5"), + "VOP2 SDWA bitwise op did not lower second source metadata to IR"); + Check(Common::ContainsStr(result.ir_dump, "SelectMaskF32Bits v47, vcc_lo, v53.neg, v53") && + Common::ContainsStr(result.ir_dump, "v53.neg"), + "V_CNDMASK_B32 SDWA source modifier did not lower to float-bit select IR"); + Check(Common::ContainsStr(result.ir_dump, "FAddF32 v125.dpp") && + Common::ContainsStr(result.ir_dump, "v5.dpp"), + "VOP2 DPP source/destination did not lower to IR metadata"); + Check(Common::ContainsStr(result.ir_dump, "FMulF32 v2, s0, s72"), + "V_MUL_F32 SDWA full-width form did not lower to IR"); + Check(Common::ContainsStr(result.ir_dump, "MulF16 v9.sdwa(sel=4") && + Common::ContainsStr(result.ir_dump, "v0.sdwa(sel=4") && + Common::ContainsStr(result.ir_dump, "v1.sdwa(sel=4"), + "V_MUL_F16 SDWA low-half form did not lower to IR"); + Check(Common::ContainsStr(result.ir_dump, "SubF16 v0.sdwa(sel=4") && + Common::ContainsStr(result.ir_dump, "v0.sdwa(sel=4"), + "V_SUB_F16 SDWA low-half form did not lower to IR"); + Check(Common::ContainsStr(result.ir_dump, "ConvertF32ToU32 v7"), + "VOP1 float-to-uint conversion did not lower to IR"); + Check(Common::ContainsStr(result.ir_dump, "ConvertI32ToF32 v9"), + "VOP1 signed int-to-float conversion did not lower to IR"); + Check(Common::ContainsStr(result.ir_dump, "ConvertF32ToI32 v10"), + "VOP1 float-to-signed-int conversion did not lower to IR"); + Check(Common::ContainsStr(result.ir_dump, "ConvertF32ToF16 v99, v6"), + "V_CVT_F16_F32 did not lower to shared half-pack IR"); + Check(Common::ContainsStr(result.ir_dump, "ConvertF16ToF32 v100, v99"), + "V_CVT_F32_F16 did not lower to shared half-unpack IR"); + Check(Common::ContainsStr(result.ir_dump, "ConvertF16ToF32 v8, v2.sdwa(sel=5") && + Common::ContainsStr(result.ir_dump, "v2.sdwa(sel=5,sext=0).abs"), + "V_CVT_F32_F16 SDWA source selector modifier did not lower to IR"); + Check(Common::ContainsStr(result.ir_dump, "ReadFirstLaneU32 s24, v5"), + "V_READFIRSTLANE_B32 did not lower to subgroup IR"); + Check(Common::ContainsStr(result.ir_dump, "ConvertFloorF32ToI32 v64, v6"), + "V_CVT_FLR_I32_F32 did not lower to shared IR"); + Check(Common::ContainsStr(result.ir_dump, "ConvertI4ToOffsetF32 v81, v5"), + "V_CVT_OFF_F32_I4 did not lower to shared offset-convert IR"); + Check(Common::ContainsStr(result.ir_dump, "ConvertByteU32ToF32 v65, v5, 0x00000000"), + "V_CVT_F32_UBYTE0 did not lower to shared byte-convert IR"); + Check(Common::ContainsStr(result.ir_dump, "ConvertByteU32ToF32 v66, v5, 0x00000001"), + "V_CVT_F32_UBYTE1 did not lower to shared byte-convert IR"); + Check(Common::ContainsStr(result.ir_dump, "ConvertByteU32ToF32 v67, v5, 0x00000002"), + "V_CVT_F32_UBYTE2 did not lower to shared byte-convert IR"); + Check(Common::ContainsStr(result.ir_dump, "ConvertByteU32ToF32 v68, v5, 0x00000003"), + "V_CVT_F32_UBYTE3 did not lower to shared byte-convert IR"); + Check(Common::ContainsStr(result.ir_dump, "RcpF32 v11"), "VOP1 reciprocal did not lower to IR"); + Check(Common::ContainsStr(result.ir_dump, "FractF32 v12"), "VOP1 fract did not lower to IR"); + Check(Common::ContainsStr(result.ir_dump, "TruncF32 v13"), "VOP1 trunc did not lower to IR"); + Check(Common::ContainsStr(result.ir_dump, "CeilF32 v14"), "VOP1 ceil did not lower to IR"); + Check(Common::ContainsStr(result.ir_dump, "RoundEvenF32 v15"), + "VOP1 round-even did not lower to IR"); + Check(Common::ContainsStr(result.ir_dump, "FloorF32 v16"), "VOP1 floor did not lower to IR"); + Check(Common::ContainsStr(result.ir_dump, "Exp2F32 v17"), "VOP1 exp2 did not lower to IR"); + Check(Common::ContainsStr(result.ir_dump, "Log2F32 v18"), "VOP1 log2 did not lower to IR"); + Check(Common::ContainsStr(result.ir_dump, "InverseSqrtF32 v19"), + "VOP1 inverse-sqrt did not lower to IR"); + Check(Common::ContainsStr(result.ir_dump, "SqrtF32 v20"), "VOP1 sqrt did not lower to IR"); + Check(Common::ContainsStr(result.ir_dump, "SinF32 v21"), "VOP1 sin did not lower to IR"); + Check(Common::ContainsStr(result.ir_dump, "CosF32 v22"), "VOP1 cos did not lower to IR"); + Check(Common::ContainsStr(result.ir_dump, "BitwiseNotU32 v27"), "VOP1 not did not lower to IR"); + Check(Common::ContainsStr(result.ir_dump, "BitReverseU32 v28"), + "VOP1 bit reverse did not lower to IR"); + Check(Common::ContainsStr(result.ir_dump, "FindMsbFromHighU32 v31"), + "VOP1 find-first-bit-high did not lower to IR"); + Check(Common::ContainsStr(result.ir_dump, "FindLsbU32 v32"), + "VOP1 find-first-bit-low did not lower to IR"); + Check(Common::ContainsStr(result.ir_dump, "BitwiseXnorU32 v83, v5, v6"), + "V_XNOR_B32 did not lower to shared xnor IR"); + Check(Common::ContainsStr(result.ir_dump, "MaskedBitCountLowU32 v84, v5, v6"), + "V_MBCNT_LO_U32_B32 did not lower to shared masked bit-count IR"); + Check(Common::ContainsStr(result.ir_dump, "MaskedBitCountHighU32 v85, v5, v6"), + "V_MBCNT_HI_U32_B32 did not lower to shared masked bit-count IR"); + Check(Common::ContainsStr(result.ir_dump, "FMadF32 v89, v6, v6, v89"), + "V_MAC_F32 did not lower with the destination register as addend"); + Check(Common::ContainsStr(result.ir_dump, "FMadF32 v90, v6, 0x3f800000, v5"), + "V_MADMK_F32 did not lower with the literal in source 1"); + Check(Common::ContainsStr(result.ir_dump, "FMadF32 v91, v6, v5, 0x40000000"), + "V_MADAK_F32 did not lower with the literal in source 2"); + Check(Common::ContainsStr(result.ir_dump, "FMadF32 v92, v6, v6, v92"), + "alternate V_MAC_F32 did not lower with the destination register as addend"); + Check(Common::ContainsStr(result.ir_dump, "FMadF32 v93, v6, 0x3f000000, v5"), + "alternate V_MADMK_F32 did not lower with the literal in source 1"); + Check(Common::ContainsStr(result.ir_dump, "FMadF32 v94, v6, v5, 0x40400000"), + "alternate V_MADAK_F32 did not lower with the literal in source 2"); + Check(Common::ContainsStr(result.ir_dump, "PackF32ToF16Rtz v95, v6, v6"), + "native V_CVT_PKRTZ_F16_F32 did not lower to shared pack IR"); + Check(Common::ContainsStr(result.ir_dump, "Dot2AccF32F16 v102, v95, v95, v102"), + "V_DOT2C_F32_F16 did not lower to explicit dot-accumulate IR"); + Check(Common::ContainsStr(result.ir_dump, "IAddCarryU32 v97, vcc_lo, v5, v6, vcc_lo"), + "V_ADD_CO_U32 did not lower to carry-add IR"); + Check(Common::ContainsStr(result.ir_dump, "FAddF32 v35"), + "VOP3-encoded VOP2 float add did not lower through shared IR"); + Check(Common::ContainsStr(result.ir_dump, "UMulU24U32 v36"), + "VOP3-encoded VOP2 24-bit multiply did not lower through shared IR"); + Check(Common::ContainsStr(result.ir_dump, "BitwiseXnorU32 v86, v5, v6"), + "VOP3-encoded V_XNOR_B32 did not lower through shared IR"); + Check(Common::ContainsStr(result.ir_dump, "FMadF32 v96, v6, v6, v96"), + "VOP3-encoded V_MAC_F32 did not lower with the destination register as addend"); + Check(Common::ContainsStr(result.ir_dump, "IAddCarryU32 v98, s30, v5, v6, vcc_lo"), + "VOP3-encoded V_ADD_CO_U32 did not lower to carry-add IR"); + Check(Common::ContainsStr(result.ir_dump, "MaskedBitCountLowU32 v87, v5, v6"), + "VOP3-encoded V_MBCNT_LO_U32_B32 did not lower through shared IR"); + Check(Common::ContainsStr(result.ir_dump, "MaskedBitCountHighU32 v88, v5, v6"), + "VOP3-encoded V_MBCNT_HI_U32_B32 did not lower through shared IR"); + Check(Common::ContainsStr(result.ir_dump, "MoveU32 v23"), + "VOP3-encoded VOP1 move did not lower through shared IR"); + Check(Common::ContainsStr(result.ir_dump, "ReadFirstLaneU32 s25, v5"), + "VOP3-encoded V_READFIRSTLANE_B32 did not lower through shared IR"); + Check(Common::ContainsStr(result.ir_dump, "ReadLaneU32 s26, v5, 0x00000002"), + "V_READLANE_B32 did not lower to subgroup lane IR"); + Check(Common::ContainsStr(result.ir_dump, "WriteLaneU32 v107, v5, 0x00000002"), + "V_WRITELANE_B32 did not lower to subgroup lane IR"); + Check(Common::ContainsStr(result.ir_dump, "Permlane16B32 v108, v5, 0x00000000, 0x00000000"), + "V_PERMLANE16_B32 did not lower to subgroup perm-lane IR"); + Check(Common::ContainsStr(result.ir_dump, "Permlanex16B32 v109, v5, 0x00000000, 0x00000000"), + "V_PERMLANEX16_B32 did not lower to subgroup perm-lane IR"); + Check(Common::ContainsStr(result.ir_dump, "ConvertI32ToF32 v24"), + "VOP3-encoded VOP1 signed conversion did not lower through shared IR"); + Check(Common::ContainsStr(result.ir_dump, "ConvertFloorF32ToI32 v74, v6"), + "VOP3-encoded V_CVT_FLR_I32_F32 did not lower through shared IR"); + Check(Common::ContainsStr(result.ir_dump, "ConvertI4ToOffsetF32 v82, v5"), + "VOP3-encoded V_CVT_OFF_F32_I4 did not lower through shared IR"); + Check(Common::ContainsStr(result.ir_dump, "FractF32 v25"), + "VOP3-encoded VOP1 fract did not lower through shared IR"); + Check(Common::ContainsStr(result.ir_dump, "RcpF32 v26"), + "VOP3-encoded VOP1 reciprocal did not lower through shared IR"); + Check(Common::ContainsStr(result.ir_dump, "BitwiseNotU32 v29"), + "VOP3-encoded VOP1 not did not lower through shared IR"); + Check(Common::ContainsStr(result.ir_dump, "BitReverseU32 v30"), + "VOP3-encoded VOP1 bit reverse did not lower through shared IR"); + Check(Common::ContainsStr(result.ir_dump, "FindMsbFromHighU32 v33"), + "VOP3-encoded VOP1 find-first-bit-high did not lower through shared IR"); + Check(Common::ContainsStr(result.ir_dump, "FindLsbU32 v34"), + "VOP3-encoded VOP1 find-first-bit-low did not lower through shared IR"); + Check(Common::ContainsStr(result.ir_dump, "CompareGtF32"), + "VOPC float compare did not lower to IR"); + Check(Common::ContainsStr(result.ir_dump, "CompareMaskGtF32 exec_lo"), + "VOPC float compare-and-mask did not lower to exec mask IR"); + Check(Common::ContainsStr(result.ir_dump, "CompareMaskGtU32 exec_lo"), + "VOPC uint compare-and-mask did not lower to exec mask IR"); + Check(Common::ContainsStr(result.ir_dump, "CompareFalse vcc_lo, v6, v6"), + "VOPC false compare did not lower to shared IR"); + Check(Common::ContainsStr(result.ir_dump, "CompareTrue vcc_lo, v6, v6"), + "VOPC true compare did not lower to shared IR"); + Check(Common::ContainsStr(result.ir_dump, "CompareOrderedF32 vcc_lo, v6, v6"), + "VOPC ordered compare did not lower to shared IR"); + Check(Common::ContainsStr(result.ir_dump, "CompareUnorderedF32 vcc_lo, v6, v6"), + "VOPC unordered compare did not lower to shared IR"); + Check(Common::ContainsStr(result.ir_dump, "CompareUnordLtF32 vcc_lo, v6, v6"), + "VOPC unordered-less compare did not lower to shared IR"); + Check(Common::ContainsStr(result.ir_dump, "CompareUnordEqF32 vcc_lo, v6, v6"), + "VOPC unordered-equal compare did not lower to shared IR"); + Check(Common::ContainsStr(result.ir_dump, "CompareUnordLeF32 vcc_lo, v6, v6"), + "VOPC unordered-less-equal compare did not lower to shared IR"); + Check(Common::ContainsStr(result.ir_dump, "CompareUnordGtF32 vcc_lo, v6, v6"), + "VOPC unordered-greater compare did not lower to shared IR"); + Check(Common::ContainsStr(result.ir_dump, "CompareUnordNeF32 vcc_lo, v6, v6"), + "VOPC unordered-not-equal compare did not lower to shared IR"); + Check(Common::ContainsStr(result.ir_dump, "CompareUnordNeF32 s0, 0x3f000000, v1"), + "VOPC SDWA unordered-not-equal compare did not lower to scalar-destination IR"); + Check(Common::ContainsStr(result.ir_dump, "CompareUnordGeF32 vcc_lo, v6, v6"), + "VOPC unordered-greater-equal compare did not lower to shared IR"); + Check(Common::ContainsStr(result.ir_dump, "CompareMaskUnordLtF32 exec_lo, v6, v6"), + "VOPC unordered-less compare-and-mask did not lower to shared IR"); + Check(Common::ContainsStr(result.ir_dump, "CompareMaskUnordEqF32 exec_lo, v6, v6"), + "VOPC unordered-equal compare-and-mask did not lower to shared IR"); + Check(Common::ContainsStr(result.ir_dump, "CompareMaskUnordLeF32 exec_lo, v6, v6"), + "VOPC unordered-less-equal compare-and-mask did not lower to shared IR"); + Check(Common::ContainsStr(result.ir_dump, "CompareMaskUnordGtF32 exec_lo, v6, v6"), + "VOPC unordered-greater compare-and-mask did not lower to shared IR"); + Check(Common::ContainsStr(result.ir_dump, "CompareMaskUnordNeF32 exec_lo, v6, v6"), + "VOPC unordered-not-equal compare-and-mask did not lower to shared IR"); + Check(Common::ContainsStr(result.ir_dump, "CompareMaskUnordGeF32 exec_lo, v6, v6"), + "VOPC unordered-greater-equal compare-and-mask did not lower to shared IR"); + Check(Common::ContainsStr(result.ir_dump, "CompareFalse vcc_lo, v5, v5"), + "VOPC integer false compare did not lower to shared IR"); + Check(Common::ContainsStr(result.ir_dump, "CompareTrue vcc_lo, v5, v5"), + "VOPC integer true compare did not lower to shared IR"); + Check(Common::ContainsStr(result.ir_dump, "FMadF32 v8"), "VOP3 mad did not lower to IR"); + Check(Common::ContainsStr(result.ir_dump, "FMadF32 v4, v1.neg, v9, s4"), + "VOP3 mad source modifiers did not lower to IR metadata"); + Check(Common::ContainsStr(result.ir_dump, "CubeIdF32 v110, v5, v6, v7"), + "V_CUBEID_F32 did not lower to cube IR"); + Check(Common::ContainsStr(result.ir_dump, "CubeScF32 v111, v5, v6, v7"), + "V_CUBESC_F32 did not lower to cube IR"); + Check(Common::ContainsStr(result.ir_dump, "CubeTcF32 v112, v5, v6, v7"), + "V_CUBETC_F32 did not lower to cube IR"); + Check(Common::ContainsStr(result.ir_dump, "CubeMaF32 v113, v5, v6, v7"), + "V_CUBEMA_F32 did not lower to cube IR"); + Check(Common::ContainsStr(result.ir_dump, "CubeMaF32 v5, v2, v0, v6.neg"), + "V_CUBEMA_F32 source modifier did not lower to cube IR"); + Check(Common::ContainsStr(result.ir_dump, "FMadF32 v37"), "VOP3 fma did not lower to IR"); + Check(Common::ContainsStr(result.ir_dump, "FMin3F32 v38"), "VOP3 min3 did not lower to IR"); + Check(Common::ContainsStr(result.ir_dump, "FMax3F32 v39"), "VOP3 max3 did not lower to IR"); + Check(Common::ContainsStr(result.ir_dump, "FMed3F32 v40"), "VOP3 med3 did not lower to IR"); + Check(Common::ContainsStr(result.ir_dump, "IMin3I32 v41"), + "VOP3 signed min3 did not lower to IR"); + Check(Common::ContainsStr(result.ir_dump, "UMin3U32 v42"), + "VOP3 unsigned min3 did not lower to IR"); + Check(Common::ContainsStr(result.ir_dump, "IMax3I32 v43"), + "VOP3 signed max3 did not lower to IR"); + Check(Common::ContainsStr(result.ir_dump, "UMax3U32 v44"), + "VOP3 unsigned max3 did not lower to IR"); + Check(Common::ContainsStr(result.ir_dump, "IMed3I32 v45"), + "VOP3 signed med3 did not lower to IR"); + Check(Common::ContainsStr(result.ir_dump, "UMed3U32 v46"), + "VOP3 unsigned med3 did not lower to IR"); + Check(Common::ContainsStr(result.ir_dump, "BitFieldExtract3U32 v47"), + "VOP3 unsigned bitfield extract did not lower to IR"); + Check(Common::ContainsStr(result.ir_dump, "BitFieldExtract3I32 v48"), + "VOP3 signed bitfield extract did not lower to IR"); + Check(Common::ContainsStr(result.ir_dump, "BitFieldInsertSelectU32 v49"), + "VOP3 bitfield insert-select did not lower to IR"); + Check(Common::ContainsStr(result.ir_dump, "AlignBitU32 v50"), + "VOP3 alignbit did not lower to IR"); + Check(Common::ContainsStr(result.ir_dump, "IAdd3U32 v51"), "VOP3 add3 did not lower to IR"); + Check(Common::ContainsStr(result.ir_dump, "IMulU32 v52, v5, v6"), + "V_MUL_LO_U32 did not lower to multiply IR"); + Check(Common::ContainsStr(result.ir_dump, "UMulHighU32 v53, v5, v6"), + "V_MUL_HI_U32 did not lower to high-multiply IR"); + Check(Common::ContainsStr(result.ir_dump, "SMulHighI32 v114, v5, v6"), + "V_MUL_HI_I32 did not lower to signed high-multiply IR"); + Check(Common::ContainsStr(result.ir_dump, "BitwiseAndOrU32 v54, v5, v6, v7"), + "V_AND_OR_B32 did not lower to ternary bitwise IR"); + Check(Common::ContainsStr(result.ir_dump, "BitwiseOr3U32 v55, v5, v6, v7"), + "V_OR3_B32 did not lower to ternary bitwise IR"); + Check(Common::ContainsStr(result.ir_dump, "BitwiseXor3U32 v56, v5, v6, v7"), + "V_XOR3_B32 did not lower to ternary bitwise IR"); + Check(Common::ContainsStr(result.ir_dump, "ShiftLeftAddU32 v57, v5, v6, v7"), + "V_LSHL_ADD_U32 did not lower to shared shift-left-add IR"); + Check(Common::ContainsStr(result.ir_dump, "AddShiftLeftU32 v58, v5, v6, v7"), + "V_ADD_LSHL_U32 did not lower to add-shift-left IR"); + Check(Common::ContainsStr(result.ir_dump, "XorAddU32 v59, v5, v6, v7"), + "V_XAD_U32 did not lower to xor-add IR"); + Check(Common::ContainsStr(result.ir_dump, "ShiftLeftOrU32 v60, v5, v6, v7"), + "V_LSHL_OR_B32 did not lower to shift-left-or IR"); + Check(Common::ContainsStr(result.ir_dump, "SadU32 v61, v5, v6, v7"), + "V_SAD_U32 did not lower to sad IR"); + Check(Common::ContainsStr(result.ir_dump, "IMadI24U32 v62, v5, v6, v7"), + "V_MAD_I32_I24 did not lower to signed 24-bit mad IR"); + Check(Common::ContainsStr(result.ir_dump, "UMadU24U32 v63, v5, v6, v7"), + "V_MAD_U32_U24 did not lower to unsigned 24-bit mad IR"); + Check(Common::ContainsStr(result.ir_dump, "IMulU32 v69, v5, v6"), + "V_MUL_LO_I32 did not lower to multiply IR"); + Check(Common::ContainsStr(result.ir_dump, "IAddCarryU32 v70, s0, v5, v6, 0x00000000"), + "V_ADD_I32 did not lower to carry-out add IR"); + Check(Common::ContainsStr(result.ir_dump, "ISubBorrowU32 v71, s0, v5, v6"), + "V_SUB_I32 did not lower to borrow-out subtract IR"); + Check(Common::ContainsStr(result.ir_dump, "ISubBorrowU32 v72, s0, v6, v5"), + "V_SUBREV_I32 did not reverse source order in borrow-out IR"); + Check(Common::ContainsStr( + result.ir_dump, + "IMaxI16 v7.sdwa(sel=4,sext=0), 0x0000ffff, v5.opsel(lo=1,hi=0,neghi=0)"), + "V_MAX_I16 did not lower to signed halfword max IR"); + Check(Common::ContainsStr( + result.ir_dump, + "IMinI16 v7.sdwa(sel=5,sext=0), 0x00000002, v4.opsel(lo=1,hi=0,neghi=0)"), + "V_MIN_I16 did not lower to signed halfword min IR"); + Check(Common::ContainsStr(result.ir_dump, "BitFieldMaskU32 v73, 0x00000001, 0x00000004"), + "V_BFM_B32 did not lower to shared bitfield-mask IR"); + Check(Common::ContainsStr(result.ir_dump, "PackF32ToF16Rtz v75, v6, v6"), + "V_CVT_PKRTZ_F16_F32 did not lower to shared pack IR"); + Check(Common::ContainsStr(result.ir_dump, "PackF32ToF16Rtz v0, v0.neg, v1.neg"), + "V_CVT_PKRTZ_F16_F32 source modifiers did not lower to shared pack IR"); + Check(Common::ContainsStr(result.ir_dump, "LdexpF32 v76, v6, 0x00000001"), + "V_LDEXP_F32 did not lower to ldexp IR"); + Check(Common::ContainsStr(result.ir_dump, "LdexpF32 v7, v9.abs, 0xfffffffe"), + "V_LDEXP_F32 source modifier did not lower to IR"); + Check(Common::ContainsStr(result.ir_dump, "LdexpF32 v13.clamp, v6, 0xfffffffc"), + "V_LDEXP_F32 clamp modifier did not lower to IR"); + Check(Common::ContainsStr(result.ir_dump, "BitCountAddU32 v77, v5, v6"), + "VOP3 V_BCNT_U32_B32 did not lower to bit-count-add IR"); + Check(Common::ContainsStr(result.ir_dump, "PackSnorm2x16F32 v78, v6, v6"), + "V_CVT_PKNORM_I16_F32 did not lower to shared pack IR"); + Check(Common::ContainsStr(result.ir_dump, "PackUnorm2x16F32 v79, v6, v6"), + "V_CVT_PKNORM_U16_F32 did not lower to shared pack IR"); + Check(Common::ContainsStr(result.ir_dump, "PackU16U32 v80, v5, v7"), + "V_CVT_PK_U16_U32 did not lower to shared pack IR"); + Check(SpirvContainsOpcode(result.spirv, 112), "SPIR-V binary does not contain OpConvertUToF"); + Check(SpirvContainsOpcode(result.spirv, 109), "SPIR-V binary does not contain OpConvertFToU"); + Check(SpirvContainsOpcode(result.spirv, 111), "SPIR-V binary does not contain OpConvertSToF"); + Check(SpirvContainsOpcode(result.spirv, 110), "SPIR-V binary does not contain OpConvertFToS"); + Check(SpirvContainsOpcode(result.spirv, 136), "SPIR-V binary does not contain OpFDiv"); + Check(SpirvContainsOpcode(result.spirv, 127), "SPIR-V binary does not contain OpFNegate"); + Check(SpirvContainsOpcode(result.spirv, 12), "SPIR-V binary does not contain OpExtInst"); + Check(SpirvContainsExtInst(result.spirv, 58), + "SPIR-V binary does not contain GLSL.std.450 PackHalf2x16"); + Check(SpirvContainsExtInst(result.spirv, 62), + "SPIR-V binary does not contain GLSL.std.450 UnpackHalf2x16"); + Check(SpirvContainsExtInst(result.spirv, 50), + "SPIR-V binary does not contain GLSL.std.450 Fma"); + Check(SpirvContainsExtInst(result.spirv, 43), + "SPIR-V binary does not contain GLSL.std.450 FClamp"); + Check(SpirvContainsExtInst(result.spirv, 53), + "SPIR-V binary does not contain GLSL.std.450 Ldexp"); + Check(SpirvContainsOpcode(result.spirv, 128), "SPIR-V binary does not contain OpIAdd"); + Check(SpirvContainsOpcode(result.spirv, 149), "SPIR-V binary does not contain OpIAddCarry"); + Check(SpirvContainsOpcode(result.spirv, 130), "SPIR-V binary does not contain OpISub"); + Check(SpirvContainsOpcode(result.spirv, 132), "SPIR-V binary does not contain OpIMul"); + Check(SpirvContainsOpcode(result.spirv, 171), "SPIR-V binary does not contain OpINotEqual"); + Check(SpirvContainsOpcode(result.spirv, 200), "SPIR-V binary does not contain OpNot"); + Check(SpirvContainsOpcode(result.spirv, 204), "SPIR-V binary does not contain OpBitReverse"); + Check(SpirvContainsOpcode(result.spirv, 205), "SPIR-V binary does not contain OpBitCount"); + Check(SpirvContainsOpcode(result.spirv, 166), "SPIR-V binary does not contain OpLogicalOr"); + Check(SpirvContainsOpcode(result.spirv, 167), "SPIR-V binary does not contain OpLogicalAnd"); + Check(SpirvContainsOpcode(result.spirv, 186), + "SPIR-V binary does not contain OpFOrdGreaterThan"); + Check(SpirvContainsOpcode(result.spirv, 181), "SPIR-V binary does not contain OpFUnordEqual"); + Check(SpirvContainsOpcode(result.spirv, 183), + "SPIR-V binary does not contain OpFUnordNotEqual"); + Check(SpirvContainsOpcode(result.spirv, 185), + "SPIR-V binary does not contain OpFUnordLessThan"); + Check(SpirvContainsOpcode(result.spirv, 187), + "SPIR-V binary does not contain OpFUnordGreaterThan"); + Check(SpirvContainsOpcode(result.spirv, 189), + "SPIR-V binary does not contain OpFUnordLessThanEqual"); + Check(SpirvContainsOpcode(result.spirv, 191), + "SPIR-V binary does not contain OpFUnordGreaterThanEqual"); + Check(SpirvContainsOpcode(result.spirv, 190), + "SPIR-V binary does not contain OpFOrdGreaterThanEqual"); + Check(SpirvContainsOpcode(result.spirv, 184), "SPIR-V binary does not contain OpFOrdLessThan"); + Check(SpirvContainsOpcode(result.spirv, 173), "SPIR-V binary does not contain OpSGreaterThan"); + Check(SpirvContainsOpcode(result.spirv, 174), + "SPIR-V binary does not contain OpUGreaterThanEqual"); + Check(SpirvContainsOpcode(result.spirv, 177), "SPIR-V binary does not contain OpSLessThan"); + Check(SpirvContainsOpcode(result.spirv, 202), + "SPIR-V binary does not contain OpBitFieldSExtract"); + Check(SpirvContainsOpcode(result.spirv, 203), + "SPIR-V binary does not contain OpBitFieldUExtract"); + Check(SpirvContainsOpcode(result.spirv, 194), + "SPIR-V binary does not contain OpShiftRightLogical"); + Check(SpirvContainsOpcode(result.spirv, 61), + "SPIR-V binary does not contain OpLoad for cmpx old exec mask"); + Check(SpirvContainsOpcode(result.spirv, 133), "SPIR-V binary does not contain OpFMul"); + Check(SpirvContainsOpcode(result.spirv, 129), "SPIR-V binary does not contain OpFAdd"); + Check(SpirvContainsOpcode(result.spirv, 151), "SPIR-V binary does not contain OpUMulExtended"); + Check(SpirvContainsOpcode(result.spirv, 152), "SPIR-V binary does not contain OpSMulExtended"); + Check(SpirvContainsOpcode(result.spirv, 196), + "SPIR-V binary does not contain OpShiftLeftLogical"); + Check(SpirvContainsOpcode(result.spirv, 197), "SPIR-V binary does not contain OpBitwiseOr"); + Check(SpirvContainsOpcode(result.spirv, 198), "SPIR-V binary does not contain OpBitwiseXor"); + Check(SpirvContainsOpcode(result.spirv, 199), "SPIR-V binary does not contain OpBitwiseAnd"); + Check(SpirvContainsOpcode(result.spirv, 80), + "SPIR-V binary does not contain OpCompositeConstruct"); + Check(SpirvContainsOpcode(result.spirv, 81), + "SPIR-V binary does not contain OpCompositeExtract"); + Check(SpirvContainsOpcode(result.spirv, 339), + "SPIR-V binary does not contain OpGroupNonUniformBallot"); + Check(SpirvContainsOpcode(result.spirv, 343), + "SPIR-V binary does not contain OpGroupNonUniformBallotFindLSB"); + Check(SpirvContainsOpcode(result.spirv, 345), + "SPIR-V binary does not contain OpGroupNonUniformShuffle"); + CheckSpirvBinaryValidates(result.spirv); +} + +void TestNewShaderRecompilerExpandedAluBatch() { + const uint32_t shader[] = { + EncodeSopk(0x00, 9, 7), // s_movk_i32 s9, 7 + EncodeSopk(0x0f, 9, 2), // s_add_i32 s9, s9, 2 + EncodeSopk(0x10, 9, 3), // s_mulk_i32 s9, s9, 3 + EncodeSop2(0x07, 10, 9, 128), // s_min_u32 s10, s9, 0 + EncodeSop2(0x09, 11, 10, 130), // s_max_u32 s11, s10, 2 + EncodeSop2(0x26, 12, 11, 130), // s_mul_i32 s12, s11, 2 + EncodeVop2(0x05, 1, 242, 0), // v_subrev_f32 v1, 1.0, v0 + EncodeVop2(0x0f, 2, 1 + 256, 0), // v_min_f32 v2, v1, v0 + EncodeVop2(0x10, 3, 1 + 256, 2), // v_max_f32 v3, v1, v2 + EncodeVop2(0x27, 4, 130, 3), // v_subrev_nc_u32 v4, 2, v3 + EncodeVop2(0x16, 5, 129, 4), // v_lshrrev_b32 v5, 1, v4 + EncodeVop2(0x1a, 6, 129, 5), // v_lshlrev_b32 v6, 1, v5 + EncodeVop2(0x13, 7, 6 + 256, 5), // v_min_u32 v7, v6, v5 + EncodeVop2(0x14, 8, 7 + 256, 6), // v_max_u32 v8, v7, v6 + 0xbf810000u, + }; + + ShaderRecompiler::CompileOptions options; + options.stage = ShaderType::Compute; + options.dump_ir = true; + + ShaderRecompiler::CompileResult result; + std::string error; + Check(ShaderRecompiler::TryRecompile(shader, options, &result, &error), error.c_str()); + Check(Common::ContainsStr(result.decoded_dump, "s_min_u32"), + "new decoder did not decode S_MIN_U32"); + Check(Common::ContainsStr(result.decoded_dump, "s_mulk_i32"), + "new decoder did not decode S_MULK_I32"); + Check(Common::ContainsStr(result.decoded_dump, "v_subrev_f32"), + "new decoder did not decode V_SUBREV_F32"); + Check(Common::ContainsStr(result.decoded_dump, "v_lshlrev_b32"), + "new decoder did not decode V_LSHLREV_B32"); + Check(Common::ContainsStr(result.ir_dump, "IMulU32 s9, s9, 0x00000003"), + "SOPK multiply did not lower to self-multiply IR"); + Check(Common::ContainsStr(result.ir_dump, "UMinU32 s10"), "S_MIN_U32 did not lower to IR"); + Check(Common::ContainsStr(result.ir_dump, "UMaxU32 s11"), "S_MAX_U32 did not lower to IR"); + Check(Common::ContainsStr(result.ir_dump, "FSubF32 v1, v0, 0x3f800000"), + "V_SUBREV_F32 did not reverse source order in IR"); + Check(Common::ContainsStr(result.ir_dump, "FMinF32 v2"), "V_MIN_F32 did not lower to IR"); + Check(Common::ContainsStr(result.ir_dump, "FMaxF32 v3"), "V_MAX_F32 did not lower to IR"); + Check(Common::ContainsStr(result.ir_dump, "ISubU32 v4, v3, 0x00000002"), + "V_SUBREV_NC_U32 did not reverse source order in IR"); + Check(Common::ContainsStr(result.ir_dump, "ShiftLeftLogicalU32 v6, v5, 0x00000001"), + "V_LSHLREV_B32 did not reverse source order in IR"); + Check(Common::ContainsStr(result.ir_dump, "UMinU32 v7"), "V_MIN_U32 did not lower to IR"); + Check(Common::ContainsStr(result.ir_dump, "UMaxU32 v8"), "V_MAX_U32 did not lower to IR"); + Check(SpirvContainsOpcode(result.spirv, 132), "SPIR-V binary does not contain OpIMul"); + Check(SpirvContainsOpcode(result.spirv, 169), "SPIR-V binary does not contain OpSelect"); + Check(SpirvContainsOpcode(result.spirv, 172), "SPIR-V binary does not contain OpUGreaterThan"); + Check(SpirvContainsOpcode(result.spirv, 176), "SPIR-V binary does not contain OpULessThan"); + Check(SpirvContainsOpcode(result.spirv, 184), "SPIR-V binary does not contain OpFOrdLessThan"); + Check(SpirvContainsOpcode(result.spirv, 186), + "SPIR-V binary does not contain OpFOrdGreaterThan"); + CheckSpirvBinaryValidates(result.spirv); +} + +void TestNewShaderRecompilerVop3pPackedF16() { + const uint32_t shader[] = { + EncodeVop3pWord0(0x0f, 114, 0, 7), + EncodeVop3pWord1(5 + 256, 6 + 256, 0, 7), // v_pk_add_f16 v114, v5, v6 + EncodeVop3pWord0(0x10, 115, 0, 7), + EncodeVop3pWord1(5 + 256, 6 + 256, 0, 7), // v_pk_mul_f16 v115, v5, v6 + EncodeVop3pWord0(0x11, 116, 0, 7), + EncodeVop3pWord1(5 + 256, 6 + 256, 0, 7), // v_pk_min_f16 v116, v5, v6 + EncodeVop3pWord0(0x12, 117, 0, 7), + EncodeVop3pWord1(5 + 256, 6 + 256, 0, 7), // v_pk_max_f16 v117, v5, v6 + EncodeVop3pWord0(0x0e, 118, 0, 7, 2), + EncodeVop3pWord1(5 + 256, 6 + 256, 7 + 256, 7, 1), + EncodeVop3pWord0(0x20, 119), + EncodeVop3pWord1(6 + 256, 6 + 256, 6 + 256), // v_fma_f32 via VOP3P + 0xcc204044u, + 0x1a022110u, // v_fma_mix_f32 v68, v16.lo, v16.lo, 0.lo + EncodeVop3pWord0(0x21, 120, 0, 7), + EncodeVop3pWord1(5 + 256, 6 + 256, 7 + 256, 7), // v_mad_mixlo_f16 + EncodeVop3pWord0(0x22, 121, 0, 7, 1), + EncodeVop3pWord1(5 + 256, 6 + 256, 7 + 256, 7, 2), // v_mad_mixhi_f16 + EncodeVop3Word0(0x34b, 122, 0x0f), + EncodeVop3Word1(5 + 256, 6 + 256, 7 + 256), // native v_fma_f16 + EncodeVop3pWord0(0x22, 126, 0, 1, 0, true), + EncodeVop3pWord1(242, 243, 3 + 256, 1), // clamped v_mad_mixhi_f16 + EncodeVop3Word0(0x34b, 127, 0x8, 0, true), + EncodeVop3Word1(240, 4 + 256, 241), // clamped native high-half v_fma_f16 + 0xd711001au, + 0x0002170au, // v_pack_b32_f16 v26, v10, v11 + 0xd711182bu, + 0x0002170au, // v_pack_b32_f16 v43, v10.hi, v11.hi + 0x78765714u, // v_pk_fmac_f16 from boot shader 0x0000001c00f5c000 + 0xbf810000u, + }; + + ShaderRecompiler::CompileOptions options; + options.stage = ShaderType::Compute; + options.dump_ir = true; + + ShaderRecompiler::CompileResult result; + std::string error; + Check(ShaderRecompiler::TryRecompile(shader, options, &result, &error), error.c_str()); + Check(Common::ContainsStr(result.decoded_dump, "v_pk_add_f16 v114"), + "new decoder did not decode old-backed V_PK_ADD_F16"); + Check(Common::ContainsStr(result.decoded_dump, "v_pk_mul_f16 v115"), + "new decoder did not decode old-backed V_PK_MUL_F16"); + Check(Common::ContainsStr(result.decoded_dump, "v_pk_min_f16 v116"), + "new decoder did not decode old-backed V_PK_MIN_F16"); + Check(Common::ContainsStr(result.decoded_dump, "v_pk_max_f16 v117"), + "new decoder did not decode old-backed V_PK_MAX_F16"); + Check(Common::ContainsStr(result.decoded_dump, "v_pk_fma_f16 v118"), + "new decoder did not decode old-backed V_PK_FMA_F16"); + Check(Common::ContainsStr(result.decoded_dump, "v_fma_f32 v119"), + "new decoder did not decode old-backed VOP3P V_FMA_F32"); + Check(Common::ContainsStr(result.decoded_dump, "v_fma_f32 v68, v16.opsel(lo=0,hi=1,neghi=0)"), + "new decoder did not decode VOP3P V_FMA_MIX_F32 source selectors"); + Check(Common::ContainsStr(result.decoded_dump, "v_mad_mixlo_f16 v120"), + "new decoder did not decode old-backed V_MAD_MIXLO_F16"); + Check(Common::ContainsStr(result.decoded_dump, "v_mad_mixhi_f16 v121"), + "new decoder did not decode old-backed V_MAD_MIXHI_F16"); + Check(Common::ContainsStr(result.decoded_dump, "v_fma_f16 v122"), + "new decoder did not decode native VOP3 V_FMA_F16"); + Check(Common::ContainsStr(result.decoded_dump, "v_mad_mixhi_f16 v126.sdwa(sel=5"), + "new decoder did not decode clamped V_MAD_MIXHI_F16 high-half destination"); + Check(Common::ContainsStr(result.decoded_dump, "v_fma_f16 v127.sdwa(sel=5"), + "new decoder did not decode clamped native high-half V_FMA_F16"); + Check(Common::ContainsStr(result.decoded_dump, "v_pack_b32_f16 v26"), + "new decoder did not decode native V_PACK_B32_F16"); + Check(Common::ContainsStr(result.decoded_dump, "v_pack_b32_f16 v43") && + Common::ContainsStr(result.decoded_dump, "v10.opsel(lo=1"), + "new decoder did not decode V_PACK_B32_F16 source lane selectors"); + Check(Common::ContainsStr(result.decoded_dump, "v_pk_fmac_f16 v59"), + "new decoder did not decode VOP2 V_PK_FMAC_F16"); + Check(Common::ContainsStr(result.decoded_dump, "v6.opsel(lo=0,hi=1,neghi=1)"), + "VOP3P high-lane source modifier was not dumped"); + Check(Common::ContainsStr(result.decoded_dump, "v120.sdwa(sel=4"), + "V_MAD_MIXLO_F16 did not expose low-half destination merge"); + Check(Common::ContainsStr(result.decoded_dump, "v121.sdwa(sel=5"), + "V_MAD_MIXHI_F16 did not expose high-half destination merge"); + Check(Common::ContainsStr(result.ir_dump, "PackedAddF16 v114"), + "V_PK_ADD_F16 did not lower to packed f16 IR"); + Check(Common::ContainsStr(result.ir_dump, "PackedMulF16 v115"), + "V_PK_MUL_F16 did not lower to packed f16 IR"); + Check(Common::ContainsStr(result.ir_dump, "PackedMinF16 v116"), + "V_PK_MIN_F16 did not lower to packed f16 IR"); + Check(Common::ContainsStr(result.ir_dump, "PackedMaxF16 v117"), + "V_PK_MAX_F16 did not lower to packed f16 IR"); + Check(Common::ContainsStr(result.ir_dump, "PackedFmaF16 v118"), + "V_PK_FMA_F16 did not lower to packed f16 IR"); + Check(Common::ContainsStr(result.ir_dump, "FMadF32 v119"), + "VOP3P V_FMA_F32 did not lower through shared fma IR"); + Check(Common::ContainsStr(result.ir_dump, "FMadF32 v68, v16.opsel(lo=0,hi=1,neghi=0)"), + "VOP3P V_FMA_MIX_F32 source selectors did not lower to IR"); + Check(Common::ContainsStr(result.ir_dump, "MadMixF16 v120.sdwa(sel=4"), + "V_MAD_MIXLO_F16 did not lower to shared mad-mix IR"); + Check(Common::ContainsStr(result.ir_dump, "MadMixF16 v121.sdwa(sel=5"), + "V_MAD_MIXHI_F16 did not lower to shared mad-mix IR"); + Check(Common::ContainsStr(result.ir_dump, "FmaF16 v122"), + "native VOP3 V_FMA_F16 did not lower to native f16 fma IR"); + Check(Common::ContainsStr(result.ir_dump, "MadMixF16 v126.sdwa(sel=5"), + "clamped V_MAD_MIXHI_F16 did not lower to shared mad-mix IR"); + Check(Common::ContainsStr(result.ir_dump, "FmaF16 v127.sdwa(sel=5"), + "clamped native V_FMA_F16 did not lower to native f16 fma IR"); + Check(Common::ContainsStr(result.ir_dump, "PackB32F16 v26"), + "V_PACK_B32_F16 did not lower to packed-bit IR"); + Check(Common::ContainsStr(result.ir_dump, "PackB32F16 v43") && + Common::ContainsStr(result.ir_dump, "v10.opsel(lo=1"), + "V_PACK_B32_F16 source selectors did not lower to IR"); + Check(Common::ContainsStr(result.ir_dump, "PackedFmaF16 v59, v20, v43, v59"), + "V_PK_FMAC_F16 did not lower using destination as packed FMA accumulator"); + Check(SpirvContainsExtInst(result.spirv, 62), + "SPIR-V binary does not contain GLSL.std.450 UnpackHalf2x16 for VOP3P"); + Check(SpirvContainsExtInst(result.spirv, 58), + "SPIR-V binary does not contain GLSL.std.450 PackHalf2x16 for VOP3P"); + Check(SpirvContainsExtInst(result.spirv, 50), + "SPIR-V binary does not contain GLSL.std.450 Fma for VOP3P"); + Check(SpirvContainsExtInst(result.spirv, 43), + "SPIR-V binary does not contain GLSL.std.450 FClamp for clamped f16 ops"); + Check(!SpirvContainsExtInst(result.spirv, 37), + "SPIR-V binary still contains GLSL.std.450 FMin for VOP3P packed min"); + Check(!SpirvContainsExtInst(result.spirv, 40), + "SPIR-V binary still contains GLSL.std.450 FMax for VOP3P packed max"); + Check(SpirvContainsOpcode(result.spirv, 169), + "SPIR-V binary does not contain OpSelect for VOP3P packed min/max"); + Check(SpirvContainsOpcode(result.spirv, 184), + "SPIR-V binary does not contain OpFOrdLessThan for VOP3P packed min"); + Check(SpirvContainsOpcode(result.spirv, 190), + "SPIR-V binary does not contain OpFOrdGreaterThanEqual for VOP3P packed max"); + Check(SpirvContainsOpcode(result.spirv, 197), + "SPIR-V binary does not contain OpBitwiseOr for VOP3P packed min/max"); + Check(SpirvContainsOpcode(result.spirv, 199), + "SPIR-V binary does not contain OpBitwiseAnd for VOP3P packed min/max"); + Check(SpirvContainsOpcode(result.spirv, 129), "SPIR-V binary does not contain OpFAdd"); + Check(SpirvContainsOpcode(result.spirv, 133), "SPIR-V binary does not contain OpFMul"); + Check(SpirvContainsOpcode(result.spirv, 81), + "SPIR-V binary does not contain OpCompositeExtract"); + CheckSpirvBinaryValidates(result.spirv); +} + +void TestNewShaderRecompilerStagedShaderOps() { + const uint32_t shader[] = { + EncodeSMovB32(0, 132), // s0 = 4 + EncodeSMovB32(1, 129), // s1 = 1 + EncodeSop2(0x05, 2, 0, 1), // s_subb_u32 s2, s0, s1 + EncodeSop1(0x1b, 3, 1), // s_bitset0_b32 s3, s1 + EncodeVop2(0x36, 70, 5 + 256, 6), // v_fmac_f16 v70, v5, v6 + EncodeVop2(0x37, 71, 7 + 256, 8), + 0x3c003c00u, // v_fmamk_f16 + EncodeVop2(0x38, 72, 9 + 256, 10), + 0x40004000u, // v_fmaak_f16 + 0xbf810000u, + }; + + ShaderRecompiler::CompileOptions options; + options.stage = ShaderType::Compute; + options.dump_ir = true; + + ShaderRecompiler::CompileResult result; + std::string error; + Check(ShaderRecompiler::TryRecompile(shader, options, &result, &error), error.c_str()); + Check(Common::ContainsStr(result.decoded_dump, "s_subb_u32 s2, s0, s1"), + "new decoder did not decode RDNA2 S_SUBB_U32"); + Check(Common::ContainsStr(result.decoded_dump, "s_bitset0_b32 s3, s1"), + "new decoder did not decode RDNA2 S_BITSET0_B32"); + Check(Common::ContainsStr(result.decoded_dump, "v_fmac_f16 v70.sdwa(sel=4"), + "new decoder did not decode RDNA2 V_FMAC_F16"); + Check(Common::ContainsStr(result.decoded_dump, + "v_fmamk_f16 v71.sdwa(sel=4,sext=0), v7, 0x3c003c00, v8"), + "new decoder did not consume V_FMAMK_F16 literal as the multiply source"); + Check(Common::ContainsStr(result.decoded_dump, + "v_fmaak_f16 v72.sdwa(sel=4,sext=0), v9, v10, 0x40004000"), + "new decoder did not consume V_FMAAK_F16 literal as the add source"); + Check(Common::ContainsStr(result.ir_dump, "ScalarSubBorrowCarryU32 s2, s0, s1, scc"), + "S_SUBB_U32 did not lower to scalar subtract-with-borrow IR"); + Check(Common::ContainsStr(result.ir_dump, "BitClearU32 s3, s3, s1"), + "S_BITSET0_B32 did not lower to bit-clear IR using the destination as input"); + Check(Common::ContainsStr(result.ir_dump, + "FmaF16 v70.sdwa(sel=4,sext=0), v5, v6, v70.sdwa(sel=4,sext=0)"), + "V_FMAC_F16 did not lower using the destination as the FMA accumulator"); + Check(Common::ContainsStr(result.ir_dump, "FmaF16 v71.sdwa(sel=4,sext=0), v7, 0x3c003c00, v8"), + "V_FMAMK_F16 did not lower with the literal in source 1"); + Check(Common::ContainsStr(result.ir_dump, "FmaF16 v72.sdwa(sel=4,sext=0), v9, v10, 0x40004000"), + "V_FMAAK_F16 did not lower with the literal in source 2"); + Check(SpirvContainsOpcode(result.spirv, 130), + "SPIR-V binary does not contain OpISub for S_SUBB_U32"); + Check(SpirvContainsOpcode(result.spirv, 166), + "SPIR-V binary does not contain OpLogicalOr for S_SUBB_U32 borrow-out"); + Check(SpirvContainsOpcode(result.spirv, 172), + "SPIR-V binary does not contain OpUGreaterThan for S_SUBB_U32 borrow-out"); + Check(SpirvContainsOpcode(result.spirv, 196), + "SPIR-V binary does not contain OpShiftLeftLogical for S_BITSET0_B32"); + Check(SpirvContainsOpcode(result.spirv, 199), + "SPIR-V binary does not contain OpBitwiseAnd for S_BITSET0_B32"); + Check(SpirvContainsOpcode(result.spirv, 200), + "SPIR-V binary does not contain OpNot for S_BITSET0_B32"); + Check(SpirvContainsExtInst(result.spirv, 50), + "SPIR-V binary does not contain GLSL.std.450 Fma for VOP2 F16 FMA ops"); + Check(SpirvContainsExtInst(result.spirv, 58), + "SPIR-V binary does not contain GLSL.std.450 PackHalf2x16 for VOP2 F16 FMA ops"); + Check(SpirvContainsExtInst(result.spirv, 62), + "SPIR-V binary does not contain GLSL.std.450 UnpackHalf2x16 for VOP2 F16 FMA ops"); + CheckSpirvBinaryValidates(result.spirv); +} + +void TestNewShaderRecompilerBootF16UnaryOpcodes() { + const uint32_t shader[] = { + EncodeVop1(0x55, 4, 249), + EncodeVop1Sdwa(3, 5, 2, 6), + EncodeVop1(0x5b, 5, 4 + 256), + 0x7e12b8f9u, + 0x0006150bu, + 0x7e12b90cu, + EncodeVop1(0x5d, 6, 5 + 256), + 0x7e0e02f9u, + 0x008616c1u, + 0x7e1002f9u, + 0x008616c1u, + EncodeVop1(0x5e, 3, 3 + 256), + 0xbf810000u, + }; + + ShaderRecompiler::CompileOptions options; + options.stage = ShaderType::Compute; + options.dump_ir = true; + + ShaderRecompiler::CompileResult result; + std::string error; + Check(ShaderRecompiler::TryRecompile(shader, options, &result, &error), error.c_str()); + Check(Common::ContainsStr(result.decoded_dump, "v_sqrt_f16 v4.sdwa(sel=5"), + "new decoder did not decode V_SQRT_F16 SDWA high-half destination"); + Check(Common::ContainsStr(result.decoded_dump, "v_rndne_f16 v3"), + "new decoder did not decode V_RNDNE_F16"); + Check(Common::ContainsStr(result.decoded_dump, "v_floor_f16 v5"), + "new decoder did not decode V_FLOOR_F16"); + Check(Common::ContainsStr(result.decoded_dump, "v_ceil_f16 v9.sdwa(sel=5"), + "new decoder did not decode boot V_CEIL_F16 SDWA high-half destination"); + Check(Common::ContainsStr(result.decoded_dump, "v_ceil_f16 v9"), + "new decoder did not decode boot V_CEIL_F16"); + Check(Common::ContainsStr(result.decoded_dump, "v_trunc_f16 v6"), + "new decoder did not decode V_TRUNC_F16"); + Check(Common::ContainsStr(result.decoded_dump, "v_mov_b32 v7, -1") && + Common::ContainsStr(result.decoded_dump, "v_mov_b32 v8, -1"), + "new decoder did not accept full-width V_MOV_B32 SDWA with DST_U=PRESERVE"); + Check(!Common::ContainsStr(result.decoded_dump, + "VOP1 SDWA destination selector is not supported"), + "full-width V_MOV_B32 SDWA with DST_U=PRESERVE was rejected"); + Check(!Common::ContainsStr(result.decoded_dump, "unsupported family=VOP2 opcode=0x00"), + "VOP1 SDWA extension word was decoded as a phantom VOP2 instruction"); + Check(Common::ContainsStr(result.ir_dump, "SqrtF16 v4.sdwa(sel=5"), + "V_SQRT_F16 did not lower to f16 sqrt IR"); + Check(Common::ContainsStr(result.ir_dump, "FloorF16 v5"), + "V_FLOOR_F16 did not lower to f16 floor IR"); + Check(Common::ContainsStr(result.ir_dump, "CeilF16 v9.sdwa(sel=5"), + "boot V_CEIL_F16 SDWA did not lower to f16 ceil IR"); + Check(Common::ContainsStr(result.ir_dump, "CeilF16 v9"), + "V_CEIL_F16 did not lower to f16 ceil IR"); + Check(Common::ContainsStr(result.ir_dump, "TruncF16 v6"), + "V_TRUNC_F16 did not lower to f16 trunc IR"); + Check(Common::ContainsStr(result.ir_dump, "MoveU32 v7, 0xffffffff") && + Common::ContainsStr(result.ir_dump, "MoveU32 v8, 0xffffffff"), + "full-width V_MOV_B32 SDWA with DST_U=PRESERVE did not lower to move IR"); + Check(Common::ContainsStr(result.ir_dump, "RoundEvenF16 v3"), + "V_RNDNE_F16 did not lower to f16 round-even IR"); + Check(SpirvContainsExtInst(result.spirv, 31), + "SPIR-V binary does not contain GLSL.std.450 Sqrt for V_SQRT_F16"); + Check(SpirvContainsExtInst(result.spirv, 8), + "SPIR-V binary does not contain GLSL.std.450 Floor for V_FLOOR_F16"); + Check(SpirvContainsExtInst(result.spirv, 9), + "SPIR-V binary does not contain GLSL.std.450 Ceil for V_CEIL_F16"); + Check(SpirvContainsExtInst(result.spirv, 3), + "SPIR-V binary does not contain GLSL.std.450 Trunc for V_TRUNC_F16"); + Check(SpirvContainsExtInst(result.spirv, 2), + "SPIR-V binary does not contain GLSL.std.450 RoundEven for V_RNDNE_F16"); + CheckSpirvBinaryValidates(result.spirv); +} + +void TestNewShaderRecompilerBootB16PackedAndSdwaOpcodes() { + const uint32_t shader[] = { + 0xd7070001u, + 0x00020081u, // v_lshrrev_b16 v1.lo, 1, v0.lo + 0xd7071008u, + 0x0002128fu, // v_lshrrev_b16 with selected high source lane + 0xd7144005u, + 0x00020481u, // v_lshlrev_b16 v5.hi, 1, v2.lo + 0xd7034001u, + 0x0002066au, // v_add_nc_u16 v1.hi, v106.lo, v3.lo + 0xd70e4011u, + 0x00020e80u, // v_sub_nc_i16 v17.hi, 0, v7.lo + 0xcc03400eu, + 0x10020e80u, // v_pk_sub_i16 from boot shader 0x0000001980eb0000 + 0xcc0a4104u, + 0x300202ffu, + 0x00007fffu, // v_pk_add_u16 with literal + 0x360e04f9u, + 0x04861482u, // v_and_b32 SDWA partial low-half destination + 0x381212f9u, + 0x04041508u, // v_or_b32 SDWA partial high-half destination + 0xcc10c00cu, + 0x18021d0du, // clamped v_pk_mul_f16 + 0x7e08a0f9u, + 0x00061500u, // v_cvt_f16_u16 v4.hi, v0 from boot shader + EncodeVop1(0x50, 16, 10 + 256), // v_cvt_f16_u16 v16, v10 + 0x7e1ca307u, // v_cvt_f16_i16 v14, v7 + 0x7e1ca2f9u, + 0x00051511u, // v_cvt_f16_i16 v14.hi, v17.hi + 0x7e08a704u, // v_cvt_i16_f16 v4, v4 from boot shader + EncodeVop1(0x52, 15, 4 + 256), // v_cvt_u16_f16 v15, v4 + 0x4a0a08f9u, + 0x0c860688u, // v_add_nc_u32 v5, 8, sign-extended v4.lo + 0x4a0c0cf9u, + 0x0d860688u, // v_add_nc_u32 v6, 8, sign-extended v6.hi + 0xbf810000u, + }; + + ShaderRecompiler::CompileOptions options; + options.stage = ShaderType::Compute; + options.dump_ir = true; + + ShaderRecompiler::CompileResult result; + std::string error; + Check(ShaderRecompiler::TryRecompile(shader, options, &result, &error), error.c_str()); + Check(Common::ContainsStr(result.decoded_dump, "v_lshrrev_b16 v1.sdwa(sel=4"), + "new decoder did not decode low-half V_LSHRREV_B16"); + Check(Common::ContainsStr(result.decoded_dump, "v_lshlrev_b16 v5.sdwa(sel=5"), + "new decoder did not decode high-half V_LSHLREV_B16"); + Check(Common::ContainsStr(result.decoded_dump, "v_add_nc_u16 v1.sdwa(sel=5"), + "new decoder did not decode V_ADD_NC_U16 op_sel destination"); + Check(Common::ContainsStr(result.decoded_dump, "v_sub_nc_i16 v17.sdwa(sel=5"), + "new decoder did not decode V_SUB_NC_I16 op_sel destination"); + Check(Common::ContainsStr(result.decoded_dump, "v_pk_sub_i16 v14"), + "new decoder did not decode V_PK_SUB_I16"); + Check(Common::ContainsStr(result.decoded_dump, "v_pk_add_u16 v4"), + "new decoder did not decode V_PK_ADD_U16"); + Check(Common::ContainsStr(result.decoded_dump, "v_pk_add_u16 v4, 0x00007fff"), + "V_PK_ADD_U16 literal was not consumed as a source operand"); + Check(Common::ContainsStr(result.decoded_dump, "v_and_b32 v7.sdwa(sel=4"), + "new decoder did not decode partial-destination V_AND_B32 SDWA"); + Check(Common::ContainsStr(result.decoded_dump, "v_or_b32 v9.sdwa(sel=5"), + "new decoder did not decode partial-destination V_OR_B32 SDWA"); + Check(Common::ContainsStr(result.decoded_dump, "v_pk_mul_f16 v12"), + "new decoder did not decode clamped V_PK_MUL_F16"); + Check(Common::ContainsStr(result.decoded_dump, "v_cvt_f16_u16 v4.sdwa(sel=5"), + "new decoder did not decode/consume SDWA V_CVT_F16_U16"); + Check(Common::ContainsStr(result.decoded_dump, "v_cvt_f16_u16 v16"), + "new decoder did not decode plain V_CVT_F16_U16"); + Check(Common::ContainsStr(result.decoded_dump, "v_cvt_f16_i16 v14"), + "new decoder did not decode plain V_CVT_F16_I16"); + Check(Common::ContainsStr(result.decoded_dump, "v_cvt_f16_i16 v14.sdwa(sel=5"), + "new decoder did not decode V_CVT_F16_I16 SDWA high-half destination"); + Check(Common::ContainsStr(result.decoded_dump, "v_cvt_i16_f16 v4"), + "new decoder did not decode plain V_CVT_I16_F16"); + Check(Common::ContainsStr(result.decoded_dump, "v_cvt_u16_f16 v15"), + "new decoder still rejects plain V_CVT_U16_F16"); + Check(Common::ContainsStr(result.decoded_dump, "v_add_nc_u32 v5") && + Common::ContainsStr(result.decoded_dump, "v4.sdwa(sel=4,sext=1"), + "new decoder did not decode V_ADD_NC_U32 SDWA sign-extended low word"); + Check(Common::ContainsStr(result.decoded_dump, "v_add_nc_u32 v6") && + Common::ContainsStr(result.decoded_dump, "v6.sdwa(sel=5,sext=1"), + "new decoder did not decode V_ADD_NC_U32 SDWA sign-extended high word"); + Check(!Common::ContainsStr(result.decoded_dump, "unsupported family=VOP2 opcode=0x00"), + "literal/SDWA extension words were decoded as phantom VOP2 instructions"); + Check(!Common::ContainsStr(result.decoded_dump, + "VOP2 SDWA destination selector is not supported"), + "partial bitwise SDWA destination is still rejected"); + Check(Common::ContainsStr(result.ir_dump, "ShiftRightLogicalU16"), + "V_LSHRREV_B16 did not lower to 16-bit logical right shift IR"); + Check(Common::ContainsStr(result.ir_dump, "ShiftLeftLogicalU16"), + "V_LSHLREV_B16 did not lower to 16-bit logical left shift IR"); + Check(Common::ContainsStr(result.ir_dump, "IAddU16 v1.sdwa(sel=5"), + "V_ADD_NC_U16 did not lower to 16-bit add IR"); + Check(Common::ContainsStr(result.ir_dump, "ISubI16 v17.sdwa(sel=5"), + "V_SUB_NC_I16 did not lower to 16-bit subtract IR"); + Check(Common::ContainsStr(result.ir_dump, "PackedSubI16 v14"), + "V_PK_SUB_I16 did not lower to packed I16 subtract IR"); + Check(Common::ContainsStr(result.ir_dump, "PackedAddU16 v4"), + "V_PK_ADD_U16 did not lower to packed U16 add IR"); + Check(Common::ContainsStr(result.ir_dump, "ConvertI16ToF16 v14"), + "V_CVT_F16_I16 did not lower to signed I16-to-F16 conversion IR"); + Check(Common::ContainsStr(result.ir_dump, "ConvertU16ToF16 v4.sdwa(sel=5"), + "V_CVT_F16_U16 SDWA did not lower to unsigned U16-to-F16 conversion IR"); + Check(Common::ContainsStr(result.ir_dump, "ConvertU16ToF16 v16"), + "V_CVT_F16_U16 plain form did not lower to unsigned U16-to-F16 conversion IR"); + Check(Common::ContainsStr(result.ir_dump, "ConvertF16ToI16 v4"), + "V_CVT_I16_F16 did not lower to signed F16-to-I16 conversion IR"); + Check(Common::ContainsStr(result.ir_dump, "ConvertF16ToU16 v15"), + "V_CVT_U16_F16 did not lower to unsigned F16-to-U16 conversion IR"); + Check(Common::ContainsStr(result.ir_dump, "IAddU32 v5") && + Common::ContainsStr(result.ir_dump, "v4.sdwa(sel=4,sext=1"), + "V_ADD_NC_U32 SDWA sign-extended low word did not lower to IR"); + Check(Common::ContainsStr(result.ir_dump, "IAddU32 v6") && + Common::ContainsStr(result.ir_dump, "v6.sdwa(sel=5,sext=1"), + "V_ADD_NC_U32 SDWA sign-extended high word did not lower to IR"); + Check(SpirvContainsOpcode(result.spirv, 128), + "SPIR-V binary does not contain OpIAdd for U16 operations"); + Check(SpirvContainsOpcode(result.spirv, 202), + "SPIR-V binary does not contain OpBitFieldSExtract for SDWA sign extension"); + Check(SpirvContainsOpcode(result.spirv, 130), + "SPIR-V binary does not contain OpISub for I16 operations"); + Check(SpirvContainsOpcode(result.spirv, 194), + "SPIR-V binary does not contain OpShiftRightLogical for B16 shift"); + Check(SpirvContainsOpcode(result.spirv, 196), + "SPIR-V binary does not contain OpShiftLeftLogical for B16 shift"); + Check(SpirvContainsOpcode(result.spirv, 197), + "SPIR-V binary does not contain OpBitwiseOr for packed U16 result"); + Check(SpirvContainsOpcode(result.spirv, 124), + "SPIR-V binary does not contain OpConvertSToF for V_CVT_F16_I16"); + Check(SpirvContainsOpcode(result.spirv, 109), + "SPIR-V binary does not contain OpConvertFToU for V_CVT_U16_F16"); + Check(SpirvContainsOpcode(result.spirv, 110), + "SPIR-V binary does not contain OpConvertFToS for V_CVT_I16_F16"); + Check(SpirvContainsOpcode(result.spirv, 112), + "SPIR-V binary does not contain OpConvertUToF for V_CVT_F16_U16"); + Check(SpirvContainsExtInst(result.spirv, 43), + "SPIR-V binary does not contain GLSL.std.450 FClamp for clamped V_PK_MUL_F16"); + CheckSpirvBinaryValidates(result.spirv); +} + +void TestNewShaderRecompilerScalarB64Alu() { + const uint32_t shader[] = { + EncodeSMovB32(0, 129), // s0 = 1 + EncodeSMovB32(1, 130), // s1 = 2 + EncodeSop1(0x07, 41, 0), // s_not_b32 s41, s0 + EncodeSop1(0x04, 2, 0), // s_mov_b64 s[2:3], s[0:1] + EncodeSop1(0x08, 4, 2), // s_not_b64 s[4:5], s[2:3] + EncodeSop2(0x0f, 6, 2, 4), // s_and_b64 s[6:7], s[2:3], s[4:5] + EncodeSop2(0x11, 8, 6, 2), // s_or_b64 s[8:9], s[6:7], s[2:3] + EncodeSop2(0x13, 10, 8, 4), // s_xor_b64 s[10:11], s[8:9], s[4:5] + EncodeSop2(0x14, 36, 0, 1), // s_andn2_b32 s36, s0, s1 + EncodeSop2(0x16, 37, 0, 1), // s_orn2_b32 s37, s0, s1 + EncodeSop2(0x18, 38, 0, 1), // s_nand_b32 s38, s0, s1 + EncodeSop2(0x1a, 39, 0, 1), // s_nor_b32 s39, s0, s1 + EncodeSop2(0x1c, 40, 0, 1), // s_xnor_b32 s40, s0, s1 + EncodeSopc(0x06, 0, 0), // s_cmp_eq_u32 s0, s0 + EncodeSop2(0x0b, 12, 10, 2), // s_cselect_b64 s[12:13], s[10:11], s[2:3] + EncodeSop2(0x15, 14, 10, 4), // s_andn2_b64 s[14:15], s[10:11], s[4:5] + EncodeSop2(0x17, 16, 14, 6), // s_orn2_b64 s[16:17], s[14:15], s[6:7] + EncodeSop2(0x19, 18, 16, 8), // s_nand_b64 s[18:19], s[16:17], s[8:9] + EncodeSop2(0x1b, 20, 18, 10), // s_nor_b64 s[20:21], s[18:19], s[10:11] + EncodeSop2(0x1d, 22, 20, 12), // s_xnor_b64 s[22:23], s[20:21], s[12:13] + EncodeSop2(0x1f, 24, 22, 1), // s_lshl_b64 s[24:25], s[22:23], s1 + EncodeSop2(0x21, 26, 24, 0), // s_lshr_b64 s[26:27], s[24:25], s0 + EncodeSop2(0x25, 28, 132, 130), // s_bfm_b64 s[28:29], 4, 2 + EncodeSop1(0x10, 30, 26), // s_bcnt1_i32_b64 s30, s[26:27] + EncodeSop1(0x16, 106, 26), // s_flbit_i32_b64 vcc_lo, s[26:27] + EncodeSop1(0x3b, 32, 30), // s_bitreplicate_b64_b32 s[32:33], s30 + EncodeSop2(0x29, 34, 32, 255), // s_bfe_u64 s[34:35], s[32:33], 0x00040002 + 0x00040002u, + EncodeSop1(0x0a, 126, 126), // s_wqm_b64 exec, exec + 0xbf810000u, + }; + + ShaderRecompiler::CompileOptions options; + options.stage = ShaderType::Compute; + options.dump_ir = true; + + ShaderRecompiler::CompileResult result; + std::string error; + Check(ShaderRecompiler::TryRecompile(shader, options, &result, &error), error.c_str()); + Check(Common::ContainsStr(result.decoded_dump, "s_mov_b64 s2, s0"), + "new decoder did not decode old-backed S_MOV_B64"); + Check(Common::ContainsStr(result.decoded_dump, "s_not_b32 s41, s0"), + "new decoder did not decode RDNA2 S_NOT_B32"); + Check(Common::ContainsStr(result.decoded_dump, "s_not_b64 s4, s2"), + "new decoder did not decode old-backed S_NOT_B64"); + Check(Common::ContainsStr(result.decoded_dump, "s_and_b64 s6, s2, s4"), + "new decoder did not decode old-backed S_AND_B64"); + Check(Common::ContainsStr(result.decoded_dump, "s_or_b64 s8, s6, s2"), + "new decoder did not decode old-backed S_OR_B64"); + Check(Common::ContainsStr(result.decoded_dump, "s_xor_b64 s10, s8, s4"), + "new decoder did not decode old-backed S_XOR_B64"); + Check(Common::ContainsStr(result.decoded_dump, "s_andn2_b32 s36, s0, s1"), + "new decoder did not decode RDNA2 S_ANDN2_B32"); + Check(Common::ContainsStr(result.decoded_dump, "s_orn2_b32 s37, s0, s1"), + "new decoder did not decode RDNA2 S_ORN2_B32"); + Check(Common::ContainsStr(result.decoded_dump, "s_nand_b32 s38, s0, s1"), + "new decoder did not decode RDNA2 S_NAND_B32"); + Check(Common::ContainsStr(result.decoded_dump, "s_nor_b32 s39, s0, s1"), + "new decoder did not decode RDNA2 S_NOR_B32"); + Check(Common::ContainsStr(result.decoded_dump, "s_xnor_b32 s40, s0, s1"), + "new decoder did not decode RDNA2 S_XNOR_B32"); + Check(Common::ContainsStr(result.decoded_dump, "s_cselect_b64 s12, s10, s2"), + "new decoder did not decode old-backed S_CSELECT_B64"); + Check(Common::ContainsStr(result.decoded_dump, "s_andn2_b64 s14, s10, s4"), + "new decoder did not decode old-backed S_ANDN2_B64"); + Check(Common::ContainsStr(result.decoded_dump, "s_orn2_b64 s16, s14, s6"), + "new decoder did not decode old-backed S_ORN2_B64"); + Check(Common::ContainsStr(result.decoded_dump, "s_nand_b64 s18, s16, s8"), + "new decoder did not decode old-backed S_NAND_B64"); + Check(Common::ContainsStr(result.decoded_dump, "s_nor_b64 s20, s18, s10"), + "new decoder did not decode old-backed S_NOR_B64"); + Check(Common::ContainsStr(result.decoded_dump, "s_xnor_b64 s22, s20, s12"), + "new decoder did not decode old-backed S_XNOR_B64"); + Check(Common::ContainsStr(result.decoded_dump, "s_lshl_b64 s24, s22, s1"), + "new decoder did not decode old-backed S_LSHL_B64"); + Check(Common::ContainsStr(result.decoded_dump, "s_lshr_b64 s26, s24, s0"), + "new decoder did not decode old-backed S_LSHR_B64"); + Check(Common::ContainsStr(result.decoded_dump, "s_bfm_b64 s28, 4, 2"), + "new decoder did not decode old-backed S_BFM_B64"); + Check(Common::ContainsStr(result.decoded_dump, "s_bcnt1_i32_b64 s30, s26"), + "new decoder did not decode old-backed S_BCNT1_I32_B64"); + Check(Common::ContainsStr(result.decoded_dump, "s_flbit_i32_b64 vcc_lo, s26"), + "new decoder did not decode RDNA2 S_FLBIT_I32_B64"); + Check(Common::ContainsStr(result.decoded_dump, "s_bitreplicate_b64_b32 s32, s30"), + "new decoder did not decode old-backed S_BITREPLICATE_B64_B32"); + Check(Common::ContainsStr(result.decoded_dump, "s_bfe_u64 s34, s32, 0x00040002"), + "new decoder did not decode old-backed S_BFE_U64"); + Check(Common::ContainsStr(result.decoded_dump, "s_wqm_b64 exec_lo, exec_lo"), + "new decoder did not decode old-backed S_WQM_B64"); + Check(Common::ContainsStr(result.ir_dump, "MoveU64 s2, s0"), + "S_MOV_B64 did not lower to paired-dword IR"); + Check(Common::ContainsStr(result.ir_dump, "BitwiseNotU32 s41, s0"), + "S_NOT_B32 did not lower to scalar bitwise-not IR"); + Check(Common::ContainsStr(result.ir_dump, "BitwiseNotU64 s4, s2"), + "S_NOT_B64 did not lower to paired-dword IR"); + Check(Common::ContainsStr(result.ir_dump, "BitwiseAndU64 s6, s2, s4"), + "S_AND_B64 did not lower to paired-dword IR"); + Check(Common::ContainsStr(result.ir_dump, "BitwiseOrU64 s8, s6, s2"), + "S_OR_B64 did not lower to paired-dword IR"); + Check(Common::ContainsStr(result.ir_dump, "BitwiseXorU64 s10, s8, s4"), + "S_XOR_B64 did not lower to paired-dword IR"); + Check(Common::ContainsStr(result.ir_dump, "BitwiseAndNotU32 s36, s0, s1"), + "S_ANDN2_B32 did not lower to scalar bitwise-and-not IR"); + Check(Common::ContainsStr(result.ir_dump, "BitwiseOrNotU32 s37, s0, s1"), + "S_ORN2_B32 did not lower to scalar bitwise-or-not IR"); + Check(Common::ContainsStr(result.ir_dump, "BitwiseNandU32 s38, s0, s1"), + "S_NAND_B32 did not lower to scalar bitwise-nand IR"); + Check(Common::ContainsStr(result.ir_dump, "BitwiseNorU32 s39, s0, s1"), + "S_NOR_B32 did not lower to scalar bitwise-nor IR"); + Check(Common::ContainsStr(result.ir_dump, "BitwiseXnorU32 s40, s0, s1"), + "S_XNOR_B32 did not lower to scalar bitwise-xnor IR"); + Check(Common::ContainsStr(result.ir_dump, "SelectU64 s12"), + "S_CSELECT_B64 did not lower to paired-dword select IR"); + Check(Common::ContainsStr(result.ir_dump, "BitwiseAndNotU64 s14, s10, s4"), + "S_ANDN2_B64 did not lower to paired-dword IR"); + Check(Common::ContainsStr(result.ir_dump, "BitwiseOrNotU64 s16, s14, s6"), + "S_ORN2_B64 did not lower to paired-dword IR"); + Check(Common::ContainsStr(result.ir_dump, "BitwiseNandU64 s18, s16, s8"), + "S_NAND_B64 did not lower to paired-dword IR"); + Check(Common::ContainsStr(result.ir_dump, "BitwiseNorU64 s20, s18, s10"), + "S_NOR_B64 did not lower to paired-dword IR"); + Check(Common::ContainsStr(result.ir_dump, "BitwiseXnorU64 s22, s20, s12"), + "S_XNOR_B64 did not lower to paired-dword IR"); + Check(Common::ContainsStr(result.ir_dump, "ShiftLeftLogicalU64 s24, s22, s1"), + "S_LSHL_B64 did not lower to paired-dword IR"); + Check(Common::ContainsStr(result.ir_dump, "ShiftRightLogicalU64 s26, s24, s0"), + "S_LSHR_B64 did not lower to paired-dword IR"); + Check(Common::ContainsStr(result.ir_dump, "BitFieldMaskU64 s28"), + "S_BFM_B64 did not lower to paired-dword IR"); + Check(Common::ContainsStr(result.ir_dump, "BitCountU64 s30, s26"), + "S_BCNT1_I32_B64 did not lower to paired-source bit count IR"); + Check(Common::ContainsStr(result.ir_dump, "FindMsbFromHighU64 vcc_lo, s26"), + "S_FLBIT_I32_B64 did not lower to paired-source leading-zero IR"); + Check(Common::ContainsStr(result.ir_dump, "BitReplicateB64B32 s32, s30"), + "S_BITREPLICATE_B64_B32 did not lower to paired-dword IR"); + Check(Common::ContainsStr(result.ir_dump, "BitFieldExtractU64 s34, s32, 0x00040002"), + "S_BFE_U64 did not lower to paired-dword IR"); + Check(Common::ContainsStr(result.ir_dump, "WqmB64 exec_lo, exec_lo"), + "S_WQM_B64 did not lower to whole-quad-mask IR"); + Check(SpirvContainsOpcode(result.spirv, 61), + "SPIR-V binary does not contain OpLoad for scalar B64 ops"); + Check(SpirvContainsOpcode(result.spirv, 62), + "SPIR-V binary does not contain OpStore for scalar B64 ops"); + Check(SpirvContainsOpcode(result.spirv, 197), + "SPIR-V binary does not contain OpBitwiseOr for scalar B64 ops"); + Check(SpirvContainsOpcode(result.spirv, 198), + "SPIR-V binary does not contain OpBitwiseXor for scalar B64 ops"); + Check(SpirvContainsOpcode(result.spirv, 199), + "SPIR-V binary does not contain OpBitwiseAnd for scalar B64 ops"); + Check(SpirvContainsOpcode(result.spirv, 200), + "SPIR-V binary does not contain OpNot for scalar B64 ops"); + Check(SpirvContainsOpcode(result.spirv, 169), + "SPIR-V binary does not contain OpSelect for scalar B64 select"); + Check(SpirvContainsOpcode(result.spirv, 171), + "SPIR-V binary does not contain OpINotEqual for scalar B64 whole-quad mask"); + Check(SpirvContainsOpcode(result.spirv, 196), + "SPIR-V binary does not contain OpShiftLeftLogical for scalar B64 shifts"); + Check(SpirvContainsOpcode(result.spirv, 194), + "SPIR-V binary does not contain OpShiftRightLogical for scalar B64 shifts"); + Check(SpirvContainsOpcode(result.spirv, 201), + "SPIR-V binary does not contain OpBitFieldInsert for scalar B64 mask"); + Check(SpirvContainsOpcode(result.spirv, 203), + "SPIR-V binary does not contain OpBitFieldUExtract for scalar B64 extract"); + Check(SpirvContainsOpcode(result.spirv, 205), + "SPIR-V binary does not contain OpBitCount for scalar B64 bit count"); + CheckSpirvBinaryValidates(result.spirv); +} + +void TestNewShaderRecompilerSignedCompareAlu() { + const uint32_t shader[] = { + EncodeSMovB32(0, 193), // s0 = -1 + EncodeSMovB32(1, 129), // s1 = 1 + EncodeSopc(0x02, 0, 1), // s_cmp_gt_i32 s0, s1 + EncodeSopk(0x07, 0, -2), // s_cmp_lt_i32 s0, -2 + EncodeVopc(0x84, 0, 1), // v_cmp_gt_i32 s0, v1 + EncodeVopc(0x81, 0, 1), // v_cmp_lt_i32 s0, v1 + 0x7d130ef9u, + 0x86068413u, // v_cmp_lt_i16 vcc_lo.sdwa, v19.word0, v7.word1 + 0x7d1d02f9u, + 0x86068213u, // v_cmp_ge_i16 vcc_lo.sdwa, v19.word0, v1.word1 + EncodeVopc(0xa9, 0, 1), // v_cmp_lt_u16 s0, v1 + EncodeVopc(0x94, 0, 1), // v_cmpx_gt_i32 s0, v1 + 0xbf810000u, + }; + + ShaderRecompiler::CompileOptions options; + options.stage = ShaderType::Compute; + options.dump_ir = true; + + ShaderRecompiler::CompileResult result; + std::string error; + Check(ShaderRecompiler::TryRecompile(shader, options, &result, &error), error.c_str()); + Check(Common::ContainsStr(result.decoded_dump, "s_cmp_gt_i32"), + "new decoder did not decode SOPC signed compare"); + Check(Common::ContainsStr(result.decoded_dump, "s_cmp_lt_i32"), + "new decoder did not decode SOPK signed compare"); + Check(Common::ContainsStr(result.decoded_dump, "v_cmp_gt_i32"), + "new decoder did not decode VOPC signed greater-than compare"); + Check(Common::ContainsStr(result.decoded_dump, "v_cmp_lt_i32"), + "new decoder did not decode VOPC signed less-than compare"); + Check(Common::ContainsStr(result.decoded_dump, "v_cmp_lt_i16"), + "new decoder did not decode VOPC signed halfword less-than compare"); + Check(Common::ContainsStr(result.decoded_dump, "v_cmp_ge_i16"), + "new decoder did not decode VOPC signed halfword greater-or-equal compare"); + Check(Common::ContainsStr(result.decoded_dump, "v_cmp_lt_u16"), + "new decoder did not decode VOPC unsigned halfword less-than compare"); + Check(Common::ContainsStr(result.decoded_dump, "v_cmpx_gt_i32"), + "new decoder did not decode VOPC signed compare-and-mask"); + Check(Common::ContainsStr(result.ir_dump, "CompareGtI32"), + "signed greater-than compare did not lower to IR"); + Check(Common::ContainsStr(result.ir_dump, "CompareLtI32"), + "signed less-than compare did not lower to IR"); + Check(Common::ContainsStr(result.ir_dump, "CompareLtI16"), + "signed halfword less-than compare did not lower to IR"); + Check(Common::ContainsStr(result.ir_dump, "CompareGeI16"), + "signed halfword greater-or-equal compare did not lower to IR"); + Check(Common::ContainsStr(result.ir_dump, "CompareLtU16"), + "unsigned halfword less-than compare did not lower to IR"); + Check(Common::ContainsStr(result.ir_dump, "CompareMaskGtI32 exec_lo"), + "signed compare-and-mask did not lower to exec mask IR"); + Check(SpirvContainsOpcode(result.spirv, 173), "SPIR-V binary does not contain OpSGreaterThan"); + Check(SpirvContainsOpcode(result.spirv, 177), "SPIR-V binary does not contain OpSLessThan"); + Check(SpirvContainsOpcode(result.spirv, 202), + "SPIR-V binary does not contain OpBitFieldSExtract for I16 compare"); + CheckSpirvBinaryValidates(result.spirv); +} + +void TestNewShaderRecompilerSignedMinShiftAlu() { + const uint32_t shader[] = { + EncodeSMovB32(0, 193), // s0 = -1 + EncodeSMovB32(1, 130), // s1 = 2 + EncodeSop2(0x06, 2, 0, 1), // s_min_i32 s2, s0, s1 + EncodeSop2(0x08, 3, 0, 1), // s_max_i32 s3, s0, s1 + EncodeSop2(0x22, 4, 0, 129), // s_ashr_i32 s4, s0, 1 + EncodeVop2(0x11, 1, 0, 0), // v_min_i32 v1, s0, v0 + EncodeVop2(0x12, 2, 1, 1), // v_max_i32 v2, s1, v1 + EncodeVop2(0x17, 3, 1 + 256, 1), // v_ashr_i32 v3, v1, v1 + EncodeVop2(0x18, 4, 129, 3), // v_ashrrev_i32 v4, 1, v3 + 0xbf810000u, + }; + + ShaderRecompiler::CompileOptions options; + options.stage = ShaderType::Compute; + options.dump_ir = true; + + ShaderRecompiler::CompileResult result; + std::string error; + Check(ShaderRecompiler::TryRecompile(shader, options, &result, &error), error.c_str()); + Check(Common::ContainsStr(result.decoded_dump, "s_min_i32"), + "new decoder did not decode S_MIN_I32"); + Check(Common::ContainsStr(result.decoded_dump, "s_ashr_i32"), + "new decoder did not decode S_ASHR_I32"); + Check(Common::ContainsStr(result.decoded_dump, "v_min_i32"), + "new decoder did not decode V_MIN_I32"); + Check(Common::ContainsStr(result.decoded_dump, "v_ashrrev_i32"), + "new decoder did not decode V_ASHRREV_I32"); + Check(Common::ContainsStr(result.ir_dump, "IMinI32 s2"), "S_MIN_I32 did not lower to IR"); + Check(Common::ContainsStr(result.ir_dump, "IMaxI32 s3"), "S_MAX_I32 did not lower to IR"); + Check(Common::ContainsStr(result.ir_dump, "ShiftRightArithmeticI32 s4"), + "S_ASHR_I32 did not lower to IR"); + Check(Common::ContainsStr(result.ir_dump, "IMinI32 v1"), "V_MIN_I32 did not lower to IR"); + Check(Common::ContainsStr(result.ir_dump, "IMaxI32 v2"), "V_MAX_I32 did not lower to IR"); + Check(Common::ContainsStr(result.ir_dump, "ShiftRightArithmeticI32 v4, v3, 0x00000001"), + "V_ASHRREV_I32 did not reverse source order in IR"); + Check(SpirvContainsOpcode(result.spirv, 173), "SPIR-V binary does not contain OpSGreaterThan"); + Check(SpirvContainsOpcode(result.spirv, 177), "SPIR-V binary does not contain OpSLessThan"); + Check(SpirvContainsOpcode(result.spirv, 195), + "SPIR-V binary does not contain OpShiftRightArithmetic"); + CheckSpirvBinaryValidates(result.spirv); +} + +void TestNewShaderRecompilerScalarBitfieldAlu() { + const uint32_t shader[] = { + EncodeSMovB32(0, 193), // s0 = -1 + EncodeSMovB32(1, 129), // s1 = 1 + EncodeSopc(0x02, 1, 0), // s_cmp_gt_i32 s1, s0 + EncodeSop2(0x0a, 2, 1, 0), // s_cselect_b32 s2, s1, s0 + EncodeSop1(0x34, 3, 0), // s_abs_i32 s3, s0 + EncodeSop1(0x0b, 4, 1), // s_brev_b32 s4, s1 + EncodeSop2(0x24, 5, 132, 129), // s_bfm_b32 s5, 4, 1 + EncodeSop2(0x27, 6, 4, 255), // s_bfe_u32 s6, s4, literal + 0x00080004u, + EncodeSop2(0x32, 7, 0, 1), // s_pack_ll_b32_b16 s7, s0, s1 + EncodeSop2(0x33, 8, 0, 1), // s_pack_lh_b32_b16 s8, s0, s1 + EncodeSop2(0x34, 9, 0, 1), // s_pack_hh_b32_b16 s9, s0, s1 + EncodeSopc(0x0c, 0, 1), // s_bitcmp0_b32 s0, s1 + EncodeSopc(0x0d, 1, 0), // s_bitcmp1_b32 s1, s0 + EncodeSopc(0x13, 0, 2), // s_cmp_lg_u64 s[0:1], s[2:3] + 0xbf810000u, + }; + + ShaderRecompiler::CompileOptions options; + options.stage = ShaderType::Compute; + options.dump_ir = true; + + ShaderRecompiler::CompileResult result; + std::string error; + Check(ShaderRecompiler::TryRecompile(shader, options, &result, &error), error.c_str()); + Check(Common::ContainsStr(result.decoded_dump, "s_cselect_b32"), + "new decoder did not decode S_CSELECT_B32"); + Check(Common::ContainsStr(result.decoded_dump, "s_abs_i32"), + "new decoder did not decode S_ABS_I32"); + Check(Common::ContainsStr(result.decoded_dump, "s_brev_b32"), + "new decoder did not decode S_BREV_B32"); + Check(Common::ContainsStr(result.decoded_dump, "s_bfm_b32"), + "new decoder did not decode S_BFM_B32"); + Check(Common::ContainsStr(result.decoded_dump, "s_bfe_u32"), + "new decoder did not decode S_BFE_U32"); + Check(Common::ContainsStr(result.decoded_dump, "s_pack_hh_b32_b16"), + "new decoder did not decode S_PACK_HH_B32_B16"); + Check(Common::ContainsStr(result.decoded_dump, "s_bitcmp0_b32"), + "new decoder did not decode S_BITCMP0_B32"); + Check(Common::ContainsStr(result.decoded_dump, "s_bitcmp1_b32"), + "new decoder did not decode S_BITCMP1_B32"); + Check(Common::ContainsStr(result.decoded_dump, "s_cmp_lg_u64"), + "new decoder did not decode S_CMP_LG_U64"); + Check(Common::ContainsStr(result.ir_dump, "SelectU32 s2"), "S_CSELECT_B32 did not lower to IR"); + Check(Common::ContainsStr(result.ir_dump, "AbsI32 s3"), "S_ABS_I32 did not lower to IR"); + Check(Common::ContainsStr(result.ir_dump, "BitReverseU32 s4"), + "S_BREV_B32 did not lower to IR"); + Check(Common::ContainsStr(result.ir_dump, "BitFieldMaskU32 s5"), + "S_BFM_B32 did not lower to IR"); + Check(Common::ContainsStr(result.ir_dump, "BitFieldExtractU32 s6"), + "S_BFE_U32 did not lower to IR"); + Check(Common::ContainsStr(result.ir_dump, "PackLowLowU16 s7"), + "S_PACK_LL_B32_B16 did not lower to IR"); + Check(Common::ContainsStr(result.ir_dump, "PackLowHighU16 s8"), + "S_PACK_LH_B32_B16 did not lower to IR"); + Check(Common::ContainsStr(result.ir_dump, "PackHighHighU16 s9"), + "S_PACK_HH_B32_B16 did not lower to IR"); + Check(Common::ContainsStr(result.ir_dump, "BitCompare0B32"), + "S_BITCMP0_B32 did not lower to IR"); + Check(Common::ContainsStr(result.ir_dump, "BitCompare1B32"), + "S_BITCMP1_B32 did not lower to IR"); + Check(Common::ContainsStr(result.ir_dump, "CompareNeU64"), "S_CMP_LG_U64 did not lower to IR"); + Check(SpirvContainsOpcode(result.spirv, 126), "SPIR-V binary does not contain OpSNegate"); + Check(SpirvContainsOpcode(result.spirv, 169), "SPIR-V binary does not contain OpSelect"); + Check(SpirvContainsOpcode(result.spirv, 201), + "SPIR-V binary does not contain OpBitFieldInsert"); + Check(SpirvContainsOpcode(result.spirv, 203), + "SPIR-V binary does not contain OpBitFieldUExtract"); + Check(SpirvContainsOpcode(result.spirv, 171), "SPIR-V binary does not contain OpINotEqual"); + Check(SpirvContainsOpcode(result.spirv, 166), "SPIR-V binary does not contain OpLogicalOr"); + Check(SpirvContainsOpcode(result.spirv, 204), "SPIR-V binary does not contain OpBitReverse"); + CheckSpirvBinaryValidates(result.spirv); +} + +void CheckNewDecoderUnsupported(const uint32_t* shader, uint32_t words, const char* family, + const char* opcode_name) { + ShaderRecompiler::Decoder::Program program; + std::string error; + const std::span code {shader, words}; + Check(ShaderRecompiler::Decoder::DecodeProgram(code, &program, &error), error.c_str()); + Check(program.instructions.size() >= 2, "decoder did not return instruction plus endpgm"); + const auto text = ShaderRecompiler::Decoder::ProgramToString(program); + Check(Common::ContainsStr(text, family), + "decoder unsupported text did not include opcode family"); + Check(Common::ContainsStr(text, opcode_name), + "decoder unsupported text did not include opcode name"); + + ShaderRecompiler::CompileOptions options; + options.stage = ShaderType::Compute; + options.dump_ir = true; + ShaderRecompiler::CompileResult result; + Check(!ShaderRecompiler::TryRecompile(code, options, &result, &error), + "unsupported opcode unexpectedly lowered without implementation"); + Check(Common::ContainsStr(error, "no IR lowering") || + Common::ContainsStr(error, "unsupported decoded"), + "unsupported lowering error was not explicit"); +} + +void TestNewShaderRecompilerMemoryFamilyLowering() { + const uint32_t shader[] = { + EncodeSmem0(0x00, 0, 4), + 0u, // s_load_dword s0 + EncodeSmem0(0x08, 1, 4), + 0u, // s_buffer_load_dword s1 + EncodeMubuf0(0x0c, 4), + EncodeMubuf1(0, 0, 1), // buffer_load_dword v0 + EncodeMubuf0(0x1c, 8), + EncodeMubuf1(0, 0, 1), // buffer_store_dword v0 + EncodeDs0(0x0d), + EncodeDs1(0, 0, 1), // ds_write_b32 v0, v1 + EncodeDs0(0x36), + EncodeDs1(2, 0, 1), // ds_read_b32 v2, v1 + EncodeMimg0(0x0e, 0x1), + EncodeMimg1(5, 0, 0, 1), // image_get_resinfo v5 + EncodeMimg0(0x00, 0xf), + EncodeMimg1(4, 0, 0, 1), // image_load v4 + EncodeMimg0(0x20, 0xf), + EncodeMimg1(3, 0, 0, 1), // image_sample v3 + 0xbf810000u, + }; + + auto user_data = ImageTestUserData(); + ShaderRecompiler::CompileOptions options; + options.stage = ShaderType::Compute; + options.dump_ir = true; + options.user_data = user_data.data(); + options.read_memory = ReadZeroTestMemory; + + ShaderRecompiler::CompileResult result; + std::string error; + const auto compiled = ShaderRecompiler::TryRecompile(shader, options, &result, &error); + Check(compiled, error.c_str()); + Check(Common::ContainsStr(result.decoded_dump, "s_load_dword"), + "new decoder did not decode SMEM dword load"); + Check(Common::ContainsStr(result.decoded_dump, "s_buffer_load_dword"), + "new decoder did not decode SMEM scalar-buffer dword load"); + Check(Common::ContainsStr(result.decoded_dump, "s_buffer_load_dword s1, s8"), + "SMEM scalar-buffer SBASE was not decoded as an SGPR-pair index"); + Check(Common::ContainsStr(result.decoded_dump, "buffer_load_dword"), + "new decoder did not decode MUBUF dword load"); + Check(Common::ContainsStr(result.decoded_dump, "buffer_store_dword"), + "new decoder did not decode MUBUF dword store"); + Check(Common::ContainsStr(result.decoded_dump, "ds_read_b32"), + "new decoder did not decode DS dword read"); + Check(Common::ContainsStr(result.decoded_dump, "image_get_resinfo"), + "new decoder did not decode MIMG resinfo query"); + Check(Common::ContainsStr(result.decoded_dump, "image_load"), + "new decoder did not decode MIMG load"); + Check(Common::ContainsStr(result.decoded_dump, "image_sample"), + "new decoder did not decode MIMG sample"); + Check(!Common::ContainsStr(result.decoded_dump, "lowering is not implemented"), + "implemented memory decode still reports unsupported lowering"); + Check(Common::ContainsStr(result.ir_dump, "SLoadDword s0"), "SMEM load did not lower to IR"); + Check(Common::ContainsStr(result.ir_dump, "SBufferLoadDword s1"), + "SMEM scalar-buffer load did not lower to IR"); + Check(Common::ContainsStr(result.ir_dump, "BufferLoadDword v0"), + "MUBUF load did not lower to IR"); + Check(Common::ContainsStr(result.ir_dump, "BufferStoreDword null, v0"), + "MUBUF store did not lower to IR"); + Check(Common::ContainsStr(result.ir_dump, "DsWriteB32 null, v0"), + "DS write did not lower to IR"); + Check(Common::ContainsStr(result.ir_dump, "DsReadB32 v2"), "DS read did not lower to IR"); + Check(Common::ContainsStr(result.ir_dump, "ImageGetResinfo v5"), + "MIMG resinfo query did not lower to IR"); + Check(Common::ContainsStr(result.ir_dump, "ImageLoad v4"), "MIMG load did not lower to IR"); + Check(Common::ContainsStr(result.ir_dump, "ImageSample v3"), "MIMG sample did not lower to IR"); + Check(SpirvContainsOpcode(result.spirv, 65), "SPIR-V binary does not contain OpAccessChain"); + Check(SpirvContainsOpcode(result.spirv, 61), "SPIR-V binary does not contain OpLoad"); + Check(SpirvContainsOpcode(result.spirv, 62), "SPIR-V binary does not contain OpStore"); + Check(SpirvContainsOpcode(result.spirv, 95), "SPIR-V binary does not contain OpImageFetch"); + Check(SpirvContainsOpcode(result.spirv, 100), "SPIR-V binary does not contain OpImage"); + Check(SpirvContainsOpcode(result.spirv, 103), + "SPIR-V binary does not contain OpImageQuerySizeLod"); + Check(SpirvContainsOpcode(result.spirv, 88), + "SPIR-V binary does not contain OpImageSampleExplicitLod"); + CheckSpirvBinaryValidates(result.spirv); +} + +void TestNewShaderRecompilerImageQueryLowering() { + const uint32_t shader[] = { + EncodeMimg0(0x60, 0x3), + EncodeMimg1(6, 0, 0, 1), // image_get_lod + 0xbf810000u, + }; + + auto user_data = ImageTestUserData(); + ShaderRecompiler::CompileOptions options; + options.stage = ShaderType::Compute; + options.dump_ir = true; + options.user_data = user_data.data(); + + ShaderRecompiler::CompileResult result; + std::string error; + Check(ShaderRecompiler::TryRecompile(shader, options, &result, &error), error.c_str()); + Check(Common::ContainsStr(result.decoded_dump, "image_get_lod"), + "new decoder did not decode MIMG image get-lod query"); + Check(Common::ContainsStr(result.decoded_dump, "dmask=0x3"), + "image_get_lod decode did not preserve dmask metadata"); + Check(Common::ContainsStr(result.ir_dump, "ImageGetLod v6"), + "image_get_lod did not lower to explicit query IR"); + Check(Common::ContainsStr(result.ir_dump, "data_dwords=2"), + "image_get_lod did not preserve two-component result metadata"); + Check(Common::ContainsStr(result.ir_dump, "image_addr=2"), + "image_get_lod did not preserve address component metadata"); + Check(SpirvContainsOpcode(result.spirv, 105), "SPIR-V binary does not contain OpImageQueryLod"); + Check(SpirvContainsOpcode(result.spirv, 80), + "SPIR-V binary does not contain coordinate composite for image_get_lod"); + Check(SpirvContainsOpcode(result.spirv, 81), + "SPIR-V binary does not contain dmask extraction for image_get_lod"); + Check(SpirvContainsOpcode(result.spirv, 124), + "SPIR-V binary does not contain result bitcast for image_get_lod"); + Check(std::find(result.spirv.begin(), result.spirv.end(), 5288u) != result.spirv.end(), + "SPIR-V binary does not request compute derivative group capability"); + Check(std::find(result.spirv.begin(), result.spirv.end(), 5289u) != result.spirv.end(), + "SPIR-V binary does not request compute derivative group execution mode"); + CheckSpirvBinaryValidates(result.spirv); +} + +void TestNewShaderRecompilerImageSampleVariants() { + const uint32_t shader[] = { + EncodeMimg0(0x24, 0xf), + EncodeMimg1(8, 0, 0, 1), // image_sample_l + EncodeMimg0(0x25, 0xf), + EncodeMimg1(12, 0, 0, 1), // image_sample_b + EncodeMimg0(0x27, 0xf), + EncodeMimg1(16, 0, 0, 1), // image_sample_lz + EncodeMimg0(0x28, 0xf), + EncodeMimg1(20, 0, 0, 1), // image_sample_c + EncodeMimg0(0x30, 0xf), + EncodeMimg1(24, 0, 0, 1), // image_sample_o + EncodeMimg0(0x22, 0xf), + EncodeMimg1(28, 0, 0, 1), // image_sample_d + EncodeMimg0(0x2f, 0x1), + EncodeMimg1(32, 0, 0, 1), // image_sample_c_lz + EncodeMimg0(0x38, 0x1), + EncodeMimg1(36, 0, 0, 1), // image_sample_c_o + EncodeMimg0(0x2a, 0x1), + EncodeMimg1(40, 0, 0, 1), // image_sample_c_d + EncodeMimg0(0x20, 0x7), + EncodeMimg1(48, 0, 0, 1, true), // image_sample with A16 bit + 0xbf810000u, + }; + + auto user_data = ImageTestUserData(); + ShaderRecompiler::CompileOptions options; + options.stage = ShaderType::Compute; + options.dump_ir = true; + options.user_data = user_data.data(); + + ShaderRecompiler::CompileResult result; + std::string error; + Check(ShaderRecompiler::TryRecompile(shader, options, &result, &error), error.c_str()); + Check(Common::ContainsStr(result.decoded_dump, "image_sample_l"), + "new decoder did not decode IMAGE_SAMPLE_L through shared MIMG path"); + Check(Common::ContainsStr(result.decoded_dump, "image_sample_b"), + "new decoder did not decode IMAGE_SAMPLE_B through shared MIMG path"); + Check(Common::ContainsStr(result.decoded_dump, "image_sample_lz"), + "new decoder did not decode IMAGE_SAMPLE_LZ through shared MIMG path"); + Check(Common::ContainsStr(result.decoded_dump, "image_sample_c"), + "new decoder did not decode IMAGE_SAMPLE_C through shared MIMG path"); + Check(Common::ContainsStr(result.decoded_dump, "image_sample_o"), + "new decoder did not decode IMAGE_SAMPLE_O through shared MIMG path"); + Check(Common::ContainsStr(result.decoded_dump, "image_sample_d"), + "new decoder did not decode IMAGE_SAMPLE_D through shared MIMG path"); + Check(Common::ContainsStr(result.decoded_dump, "image_sample_c_lz"), + "new decoder did not decode IMAGE_SAMPLE_C_LZ through shared MIMG path"); + Check(Common::ContainsStr(result.decoded_dump, "image_sample_c_o"), + "new decoder did not decode IMAGE_SAMPLE_C_O through shared MIMG path"); + Check(Common::ContainsStr(result.decoded_dump, "image_sample_c_d"), + "new decoder did not decode IMAGE_SAMPLE_C_D through shared MIMG path"); + Check(Common::ContainsStr(result.decoded_dump, "sample_flags=a16"), + "IMAGE_SAMPLE with MIMG A16 bit did not expose A16 sample flag"); + Check(Common::ContainsStr(result.decoded_dump, "sample_flags=lod"), + "IMAGE_SAMPLE_L did not expose lod sample flag"); + Check(Common::ContainsStr(result.decoded_dump, "sample_flags=bias"), + "IMAGE_SAMPLE_B did not expose bias sample flag"); + Check(Common::ContainsStr(result.decoded_dump, "sample_flags=level_zero"), + "IMAGE_SAMPLE_LZ did not expose level-zero sample flag"); + Check(Common::ContainsStr(result.decoded_dump, "sample_flags=compare"), + "IMAGE_SAMPLE_C did not expose compare sample flag"); + Check(Common::ContainsStr(result.decoded_dump, "sample_flags=offset addr_components=3"), + "IMAGE_SAMPLE_O did not expose offset sample flag/address width"); + Check(Common::ContainsStr(result.decoded_dump, "sample_flags=derivative addr_components=6"), + "IMAGE_SAMPLE_D did not expose derivative sample flag/address width"); + Check(Common::ContainsStr(result.decoded_dump, "sample_flags=compare|level_zero"), + "IMAGE_SAMPLE_C_LZ did not expose compare+level-zero flags"); + Check(Common::ContainsStr(result.decoded_dump, "sample_flags=compare|offset addr_components=4"), + "IMAGE_SAMPLE_C_O did not expose compare+offset flags/address width"); + Check(Common::ContainsStr(result.decoded_dump, + "sample_flags=derivative|compare addr_components=7"), + "IMAGE_SAMPLE_C_D did not expose compare+derivative address width"); + Check(Common::ContainsStr(result.ir_dump, "ImageSample v8"), + "IMAGE_SAMPLE_L did not lower to shared IR ImageSample"); + Check(Common::ContainsStr(result.ir_dump, "ImageSample v28"), + "IMAGE_SAMPLE_D did not lower to shared IR ImageSample"); + Check(Common::ContainsStr(result.ir_dump, "image_flags=0x4"), + "derivative sample flag did not survive into IR memory metadata"); + Check(Common::ContainsStr(result.ir_dump, "image_addr=6"), + "derivative sample address width did not survive into IR memory metadata"); + Check(Common::ContainsStr(result.ir_dump, "image_addr=7"), + "compare+derivative sample address width did not survive into IR memory metadata"); + Check(Common::ContainsStr(result.ir_dump, "image_flags=0x80"), + "MIMG A16 bit did not survive into IR memory metadata"); + Check(SpirvContainsOpcode(result.spirv, 88), + "SPIR-V binary does not contain shared OpImageSampleExplicitLod emission"); + Check(SpirvContainsOpcode(result.spirv, 90), + "SPIR-V binary does not contain shared OpImageSampleDrefExplicitLod emission"); + Check(SpirvContainsOpcode(result.spirv, 202), + "SPIR-V binary does not contain packed offset extraction"); + Check(SpirvContainsOpcode(result.spirv, 80), + "SPIR-V binary does not contain coordinate/gradient composite construction"); + Check(SpirvContainsExtInst(result.spirv, 62), + "SPIR-V binary does not unpack A16 image address halves"); + CheckSpirvBinaryValidates(result.spirv); +} + +void TestNewShaderRecompilerImageSampleA16SamplerCoords() { + constexpr uint32_t MimgDim3D = 2; + + const uint32_t shader[] = { + EncodeMimg0(0x20, 0x7, false, MimgDim3D), + EncodeMimg1(8, 0, 0, 4, true), // image_sample, A16 bit, 3D xyz coords + 0xbf810000u, + }; + + auto user_data = ImageTestUserData(Prospero::ImageType::kColor3D); + ShaderRecompiler::CompileOptions options; + options.stage = ShaderType::Compute; + options.dump_ir = true; + options.user_data = user_data.data(); + + ShaderRecompiler::CompileResult result; + std::string error; + const auto compiled = ShaderRecompiler::TryRecompile(shader, options, &result, &error); + Check(compiled, error.c_str()); + Check( + Common::ContainsStr(result.decoded_dump, "image_dim=3d sample_flags=a16 addr_components=3"), + "3D IMAGE_SAMPLE with MIMG A16 bit did not decode as three A16 sampler coords"); + Check(Common::ContainsStr(result.ir_dump, "ImageSample v8"), + "3D A16 IMAGE_SAMPLE did not lower to sample IR"); + Check(Common::ContainsStr(result.ir_dump, "image_flags=0x80"), + "3D A16 IMAGE_SAMPLE did not preserve A16 memory metadata"); + Check(Common::ContainsStr(result.ir_dump, "image_dim=3d"), + "3D A16 IMAGE_SAMPLE did not preserve image dimension metadata"); + Check(Common::ContainsStr(result.ir_dump, "image_addr=3"), + "3D A16 IMAGE_SAMPLE did not preserve three logical address components"); + Check(SpirvExtInstCount(result.spirv, 62) == 3, + "sampler A16 xyz coordinates should be converted from three packed f16 components"); + Check(SpirvContainsOpcode(result.spirv, 194), + "sampler A16 high-half coordinate extraction is missing"); + Check(SpirvContainsOpcode(result.spirv, 199), + "sampler A16 low-half coordinate masking is missing"); + CheckSpirvBinaryValidates(result.spirv); +} + +void TestNewShaderRecompilerImageSampleOpcodeAliases() { + const uint32_t shader[] = { + 0xf0800109u, + 0x00c00602u, // observed image_sample_a v6, v2, s0, s24 + EncodeMimg0(0xa5, 0xf), + EncodeMimg1(12, 0, 0, 4), // image_sample_b_a + EncodeMimg0(0xa8, 0x1), + EncodeMimg1(16, 0, 0, 4), // image_sample_c_a + EncodeMimg0(0xad, 0x1), + EncodeMimg1(20, 0, 0, 4), // image_sample_c_b_a + 0xbf810000u, + }; + + auto user_data = ImageTestUserData(); + ShaderRecompiler::CompileOptions options; + options.stage = ShaderType::Compute; + options.dump_ir = true; + options.user_data = user_data.data(); + + ShaderRecompiler::CompileResult result; + std::string error; + Check(ShaderRecompiler::TryRecompile(shader, options, &result, &error), error.c_str()); + Check(Common::ContainsStr(result.decoded_dump, "image_sample_a"), + "MIMG opcode 0xa0 should decode as image_sample_a alias"); + Check(Common::ContainsStr(result.decoded_dump, "image_sample_b_a"), + "MIMG opcode 0xa5 should decode as image_sample_b_a alias"); + Check(Common::ContainsStr(result.decoded_dump, "image_sample_c_a"), + "MIMG opcode 0xa8 should decode as image_sample_c_a alias"); + Check(Common::ContainsStr(result.decoded_dump, "image_sample_c_b_a"), + "MIMG opcode 0xad should decode as image_sample_c_b_a alias"); + Check(Common::ContainsStr(result.decoded_dump, "sample_flags=none"), + "opcode 0xa0 alias should use normal 32-bit sample coordinates"); + Check(Common::ContainsStr(result.decoded_dump, "sample_flags=bias|compare"), + "compare+bias opcode alias did not expose expected sample flags"); + Check(!Common::ContainsStr(result.decoded_dump, "a16"), + "A16 must come from MIMG bit 62, not from the opcode alias"); + Check(!SpirvContainsExtInst(result.spirv, 62), + "opcode aliases without bit 62 must not unpack sampled f16 address halves"); + CheckSpirvBinaryValidates(result.spirv); +} + +void TestNewShaderRecompilerImageSampleA16ExceptionComponents() { + auto compile = [](uint32_t opcode, uint32_t dmask, uint32_t vdata) { + const uint32_t shader[] = { + EncodeMimg0(opcode, dmask), + EncodeMimg1(vdata, 0, 0, 4, true), + 0xbf810000u, + }; + + auto user_data = ImageTestUserData(); + ShaderRecompiler::CompileOptions options; + options.stage = ShaderType::Compute; + options.dump_ir = true; + options.user_data = user_data.data(); + + ShaderRecompiler::CompileResult result; + std::string error; + Check(ShaderRecompiler::TryRecompile(shader, options, &result, &error), error.c_str()); + CheckSpirvBinaryValidates(result.spirv); + return result; + }; + + { + const auto result = compile(0x28, 0x1, 12); // image_sample_c with A16 bit + Check( + Common::ContainsStr(result.decoded_dump, "sample_flags=compare|a16 addr_components=3"), + "A16 IMAGE_SAMPLE_C did not preserve compare+A16 metadata"); + Check(Common::ContainsStr(result.ir_dump, "image_flags=0x88"), + "A16 IMAGE_SAMPLE_C did not preserve compare+A16 IR flags"); + Check(SpirvInstructionOpcodeCount(result.spirv, 90) == 1, + "A16 IMAGE_SAMPLE_C should emit one dref sample"); + Check(SpirvExtInstCount(result.spirv, 62) == 2, + "PCF reference must remain 32-bit while only xy sampler coords use A16"); + } + + { + const auto result = compile(0x30, 0xf, 16); // image_sample_o with A16 bit + Check(Common::ContainsStr(result.decoded_dump, "sample_flags=offset|a16 addr_components=3"), + "A16 IMAGE_SAMPLE_O did not preserve offset+A16 metadata"); + Check(Common::ContainsStr(result.ir_dump, "image_flags=0x90"), + "A16 IMAGE_SAMPLE_O did not preserve offset+A16 IR flags"); + Check(SpirvContainsOpcode(result.spirv, 202), + "texel offset should still be decoded from its packed 6-bit fields"); + Check(SpirvExtInstCount(result.spirv, 62) == 2, + "texel offset must remain 32-bit while only xy sampler coords use A16"); + } +} + +void TestNewShaderRecompilerImageSampleA16Rdna2AddressOrder() { + namespace Decoder = Libs::Graphics::ShaderRecompiler::Decoder; + namespace Emitter = Libs::Graphics::ShaderRecompiler::Spirv::Emitter; + namespace IR = Libs::Graphics::ShaderRecompiler::IR; + + auto make_inst = [](uint32_t flags) { + IR::Instruction inst {}; + inst.src[0] = Emitter::MakeRegisterOperand(IR::RegisterFile::Vector, 20); + inst.memory.image_sample_flags = flags | Decoder::ImageSampleFlagA16; + inst.memory.image_address_components = 5; + return inst; + }; + auto check_vgpr = [](const IR::Operand& operand, uint32_t expected, const char* message) { + Check(operand.kind == IR::OperandKind::Register && + operand.reg.file == IR::RegisterFile::Vector && operand.reg.index == expected, + message); + }; + + { + auto inst = make_inst(Decoder::ImageSampleFlagBias | Decoder::ImageSampleFlagCompare); + const auto layout = Emitter::MakeImageSampleLayout(inst, Emitter::ImageViewKind::Dim2D); + Check(layout.bias == 0u, "RDNA2 sample_c_b A16 bias must precede PCF reference"); + Check(layout.dref == 1u, "RDNA2 sample_c_b A16 PCF reference must follow bias"); + Check(layout.coord == 2u, "RDNA2 sample_c_b A16 body must follow bias and PCF"); + Check(Emitter::ImageAddressHalfComponent(inst, layout.bias) == 0u, + "A16 bias should use the first packed half"); + Check(Emitter::ImageAddressHalfComponent(inst, layout.dref) == 2u, + "32-bit PCF reference must align to the next VGPR after A16 bias"); + Check(Emitter::ImageAddressHalfComponent(inst, layout.coord) == 4u, + "A16 coordinates must start after the aligned PCF reference"); + check_vgpr(Emitter::ImageAddressOperand(inst, inst.src[0], layout.bias), 20, + "A16 sample_c_b bias should read v20"); + check_vgpr(Emitter::ImageAddressOperand(inst, inst.src[0], layout.dref), 21, + "A16 sample_c_b PCF reference should read v21"); + check_vgpr(Emitter::ImageAddressOperand(inst, inst.src[0], layout.coord), 22, + "A16 sample_c_b x/y coordinates should start in v22"); + } + + { + auto inst = make_inst(Decoder::ImageSampleFlagOffset | Decoder::ImageSampleFlagBias | + Decoder::ImageSampleFlagCompare); + const auto layout = Emitter::MakeImageSampleLayout(inst, Emitter::ImageViewKind::Dim2D); + Check(layout.offset == 0u && layout.bias == 1u && layout.dref == 2u && layout.coord == 3u, + "RDNA2 sample_c_b_o order must be offset, bias, PCF, body"); + Check(Emitter::ImageAddressHalfComponent(inst, layout.offset) == 0u, + "texel offset should stay a packed 32-bit VGPR"); + Check(Emitter::ImageAddressHalfComponent(inst, layout.bias) == 2u, + "A16 bias should follow the 32-bit texel offset"); + Check(Emitter::ImageAddressHalfComponent(inst, layout.dref) == 4u, + "PCF reference should align after A16 bias"); + Check(Emitter::ImageAddressHalfComponent(inst, layout.coord) == 6u, + "A16 coordinates should start after offset, bias, and PCF"); + check_vgpr(Emitter::ImageAddressOperand(inst, inst.src[0], layout.offset), 20, + "A16 sample_c_b_o offset should read v20"); + check_vgpr(Emitter::ImageAddressOperand(inst, inst.src[0], layout.bias), 21, + "A16 sample_c_b_o bias should read v21"); + check_vgpr(Emitter::ImageAddressOperand(inst, inst.src[0], layout.dref), 22, + "A16 sample_c_b_o PCF reference should read v22"); + check_vgpr(Emitter::ImageAddressOperand(inst, inst.src[0], layout.coord), 23, + "A16 sample_c_b_o x/y coordinates should start in v23"); + } +} + +void TestNewShaderRecompilerImageLoadA16UintCoords() { + const uint32_t shader[] = { + EncodeMimg0(0x00, 0x3), + EncodeMimg1(20, 0, 0, 4, true), // image_load, A16 bit + 0xbf810000u, + }; + + auto user_data = ImageTestUserData(); + ShaderRecompiler::CompileOptions options; + options.stage = ShaderType::Compute; + options.dump_ir = true; + options.user_data = user_data.data(); + + ShaderRecompiler::CompileResult result; + std::string error; + Check(ShaderRecompiler::TryRecompile(shader, options, &result, &error), error.c_str()); + Check(Common::ContainsStr(result.decoded_dump, "image_load"), "A16 IMAGE_LOAD did not decode"); + Check(Common::ContainsStr(result.ir_dump, "ImageLoad v20"), + "A16 IMAGE_LOAD did not lower to image-load IR"); + Check(Common::ContainsStr(result.ir_dump, "image_flags=0x80"), + "A16 IMAGE_LOAD did not preserve A16 memory metadata"); + Check(Common::ContainsStr(result.ir_dump, "image_addr=2"), + "A16 IMAGE_LOAD did not preserve logical address component count"); + Check(!SpirvContainsExtInst(result.spirv, 62), + "image ops without sampler use u16 A16 addresses and must not unpack f16"); + Check(SpirvContainsOpcode(result.spirv, 194), + "u16 A16 high-half coordinate extraction is missing"); + Check(SpirvContainsOpcode(result.spirv, 199), "u16 A16 low-half coordinate masking is missing"); + Check(SpirvContainsOpcode(result.spirv, 95), "A16 IMAGE_LOAD did not emit an image fetch"); + CheckSpirvBinaryValidates(result.spirv); +} + +void TestNewShaderRecompilerPixelImageSampleLodSelection() { + constexpr uint32_t OpImageSampleImplicitLod = 87; + constexpr uint32_t OpImageSampleExplicitLod = 88; + constexpr uint32_t OpImageSampleDrefImplicitLod = 89; + constexpr uint32_t OpImageSampleDrefExplicitLod = 90; + + auto compile = [](uint32_t opcode, uint32_t dmask) { + const uint32_t shader[] = { + EncodeMimg0(opcode, dmask), + EncodeMimg1(0, 0, 0, 4), + 0xbf810000u, + }; + + auto ps_info = RegressionPixelInputInfo(); + + auto user_data = ImageTestUserData(); + ShaderRecompiler::CompileOptions options; + options.stage = ShaderType::Pixel; + options.pixel_input_info = &ps_info; + options.dump_ir = true; + options.user_data = user_data.data(); + + ShaderRecompiler::CompileResult result; + std::string error; + Check(ShaderRecompiler::TryRecompile(shader, options, &result, &error), error.c_str()); + CheckSpirvBinaryValidates(result.spirv); + return result; + }; + + { + const auto result = compile(0x20, 0xf); // image_sample + Check(Common::ContainsStr(result.decoded_dump, "image_sample"), + "plain pixel IMAGE_SAMPLE did not decode"); + Check(SpirvInstructionOpcodeCount(result.spirv, OpImageSampleImplicitLod) == 1, + "plain pixel IMAGE_SAMPLE must use OpImageSampleImplicitLod"); + Check(SpirvInstructionOpcodeCount(result.spirv, OpImageSampleExplicitLod) == 0, + "plain pixel IMAGE_SAMPLE unexpectedly used explicit lod"); + } + { + const auto result = compile(0x25, 0xf); // image_sample_b + Check(Common::ContainsStr(result.decoded_dump, "sample_flags=bias"), + "pixel IMAGE_SAMPLE_B did not preserve bias metadata"); + Check(SpirvInstructionOpcodeCount(result.spirv, OpImageSampleImplicitLod) == 1, + "pixel IMAGE_SAMPLE_B must use implicit lod with a bias operand"); + Check(SpirvInstructionOpcodeCount(result.spirv, OpImageSampleExplicitLod) == 0, + "pixel IMAGE_SAMPLE_B unexpectedly used explicit lod"); + } + { + const auto result = compile(0x27, 0xf); // image_sample_lz + Check(SpirvInstructionOpcodeCount(result.spirv, OpImageSampleExplicitLod) == 1, + "pixel IMAGE_SAMPLE_LZ must use explicit lod"); + } + { + const auto result = compile(0x28, 0x1); // image_sample_c + Check(SpirvInstructionOpcodeCount(result.spirv, OpImageSampleDrefImplicitLod) == 1, + "pixel IMAGE_SAMPLE_C must use OpImageSampleDrefImplicitLod"); + Check(SpirvInstructionOpcodeCount(result.spirv, OpImageSampleDrefExplicitLod) == 0, + "pixel IMAGE_SAMPLE_C unexpectedly used explicit dref lod"); + } + { + const auto result = compile(0x2f, 0x1); // image_sample_c_lz + Check(SpirvInstructionOpcodeCount(result.spirv, OpImageSampleDrefExplicitLod) == 1, + "pixel IMAGE_SAMPLE_C_LZ must use explicit dref lod"); + } +} + +void TestNewShaderRecompilerImageViewDimensions() { + constexpr uint32_t MimgDim3D = 2; + constexpr uint32_t MimgDim2DArray = 5; + constexpr uint32_t SpirvDim2D = 1; + constexpr uint32_t SpirvDim3D = 2; + + const uint32_t shader[] = { + EncodeMimg0(0x20, 0xf, false, MimgDim2DArray), + EncodeMimg1(0, 0, 1, 0), // image_sample 2D array + EncodeMimg0(0x24, 0xf, false, MimgDim2DArray), + EncodeMimg1(4, 1, 1, 4), // image_sample_l 2D array + EncodeMimg0(0x20, 0xf, false, MimgDim3D), + EncodeMimg1(8, 2, 1, 8), // image_sample 3D + EncodeMimg0(0x01, 0xf, false, MimgDim2DArray), + EncodeMimg1(12, 3, 0, 12), // image_load_mip 2D array + 0xbf810000u, + }; + + auto ps_info = RegressionPixelInputInfo(); + auto user_data = ImageTestUserData(); + SetImageTestType(&user_data, 0, Prospero::ImageType::kColor2DArray); + SetImageTestType(&user_data, 1, Prospero::ImageType::kColor2DArray); + SetImageTestType(&user_data, 2, Prospero::ImageType::kColor3D); + SetImageTestType(&user_data, 3, Prospero::ImageType::kColor2DArray); + + ShaderRecompiler::CompileOptions options; + options.stage = ShaderType::Pixel; + options.pixel_input_info = &ps_info; + options.dump_ir = true; + options.user_data = user_data.data(); + + ShaderRecompiler::CompileResult result; + std::string error; + const auto compiled = ShaderRecompiler::TryRecompile(shader, options, &result, &error); + Check(compiled, error.c_str()); + Check(Common::ContainsStr(result.decoded_dump, "image_dim=2d_array"), + "MIMG DIM did not decode 2D-array image view"); + Check(Common::ContainsStr(result.decoded_dump, "image_dim=3d"), + "MIMG DIM did not decode 3D image view"); + Check(Common::ContainsStr(result.decoded_dump, "image_sample_l"), + "2D-array LOD sample did not decode"); + Check(Common::ContainsStr(result.decoded_dump, + "image_dim=2d_array sample_flags=lod addr_components=4"), + "2D-array LOD sample did not include slice plus LOD address components"); + Check(Common::ContainsStr(result.ir_dump, "image_dim=2d_array"), + "2D-array image view did not survive into IR metadata"); + Check(Common::ContainsStr(result.ir_dump, "image_dim=3d"), + "3D image view did not survive into IR metadata"); + Check(SpirvContainsTypeImage(result.spirv, SpirvDim2D, 1, 1), + "SPIR-V binary does not contain sampled 2D-array image type"); + Check(SpirvContainsTypeImage(result.spirv, SpirvDim3D, 0, 1), + "SPIR-V binary does not contain sampled 3D image type"); + Check(SpirvContainsOpcode(result.spirv, 95), + "SPIR-V binary does not contain array image fetch"); + CheckSpirvBinaryValidates(result.spirv); +} + +void TestNewShaderRecompilerImageGatherVariants() { + const uint32_t shader[] = { + EncodeMimg0(0x47, 0x1), + EncodeMimg1(60, 0, 1, 4), // image_gather4_lz + EncodeMimg0(0x48, 0x2), + EncodeMimg1(64, 0, 1, 8), // image_gather4_c + EncodeMimg0(0x4f, 0x1), + EncodeMimg1(68, 0, 1, 12), // image_gather4_c_lz + EncodeMimg0(0x57, 0x4), + EncodeMimg1(72, 0, 1, 16), // image_gather4_lz_o + EncodeMimg0(0x58, 0x1), + EncodeMimg1(76, 0, 1, 20), // image_gather4_c_o + EncodeMimg0(0x5f, 0x1), + EncodeMimg1(80, 0, 1, 24), // image_gather4_c_lz_o + 0xbf810000u, + }; + + auto user_data = ImageTestUserData(); + ShaderRecompiler::CompileOptions options; + options.stage = ShaderType::Compute; + options.dump_ir = true; + options.user_data = user_data.data(); + + ShaderRecompiler::CompileResult result; + std::string error; + Check(ShaderRecompiler::TryRecompile(shader, options, &result, &error), error.c_str()); + Check(Common::ContainsStr(result.decoded_dump, "image_gather4_lz"), + "new decoder did not decode IMAGE_GATHER4_LZ"); + Check(Common::ContainsStr(result.decoded_dump, "image_gather4_lz_o"), + "new decoder did not decode IMAGE_GATHER4_LZ_O"); + Check(Common::ContainsStr(result.decoded_dump, "image_gather4_c"), + "new decoder did not decode IMAGE_GATHER4_C"); + Check(Common::ContainsStr(result.decoded_dump, "image_gather4_c_lz"), + "new decoder did not decode IMAGE_GATHER4_C_LZ"); + Check(Common::ContainsStr(result.decoded_dump, "image_gather4_c_o"), + "new decoder did not decode IMAGE_GATHER4_C_O"); + Check(Common::ContainsStr(result.decoded_dump, "image_gather4_c_lz_o"), + "new decoder did not decode IMAGE_GATHER4_C_LZ_O"); + Check(Common::ContainsStr(result.decoded_dump, "dmask=0x4"), + "IMAGE_GATHER4_LZ_O did not preserve gather component dmask"); + Check(Common::ContainsStr(result.decoded_dump, + "sample_flags=offset|level_zero addr_components=3"), + "IMAGE_GATHER4_LZ_O did not expose shared offset sample metadata"); + Check(Common::ContainsStr(result.decoded_dump, "sample_flags=compare addr_components=3"), + "IMAGE_GATHER4_C did not expose compare sample metadata"); + Check(Common::ContainsStr(result.decoded_dump, + "sample_flags=compare|level_zero addr_components=3"), + "IMAGE_GATHER4_C_LZ did not expose compare+level-zero sample metadata"); + Check(Common::ContainsStr(result.decoded_dump, "sample_flags=compare|offset addr_components=4"), + "IMAGE_GATHER4_C_O did not expose compare+offset sample metadata"); + Check(Common::ContainsStr(result.decoded_dump, + "sample_flags=compare|offset|level_zero addr_components=4"), + "IMAGE_GATHER4_C_LZ_O did not expose compare+offset+level-zero sample metadata"); + Check(Common::ContainsStr(result.ir_dump, "ImageGather4 v60"), + "IMAGE_GATHER4_LZ did not lower to shared IR ImageGather4"); + Check(Common::ContainsStr(result.ir_dump, "ImageGather4 v64"), + "IMAGE_GATHER4_C did not lower to shared IR ImageGather4"); + Check(Common::ContainsStr(result.ir_dump, "ImageGather4 v68"), + "IMAGE_GATHER4_C_LZ did not lower to shared IR ImageGather4"); + Check(Common::ContainsStr(result.ir_dump, "ImageGather4 v72"), + "IMAGE_GATHER4_LZ_O did not lower to shared IR ImageGather4"); + Check(Common::ContainsStr(result.ir_dump, "ImageGather4 v76"), + "IMAGE_GATHER4_C_O did not lower to shared IR ImageGather4"); + Check(Common::ContainsStr(result.ir_dump, "ImageGather4 v80"), + "IMAGE_GATHER4_C_LZ_O did not lower to shared IR ImageGather4"); + Check(Common::ContainsStr(result.ir_dump, "data_dwords=4"), + "IMAGE_GATHER4 did not preserve four-component result metadata"); + Check(Common::ContainsStr(result.ir_dump, "image_flags=0x30"), + "IMAGE_GATHER4_LZ_O flags did not survive into IR memory metadata"); + Check(Common::ContainsStr(result.ir_dump, "image_flags=0x8"), + "IMAGE_GATHER4_C flags did not survive into IR memory metadata"); + Check(Common::ContainsStr(result.ir_dump, "image_flags=0x28"), + "IMAGE_GATHER4_C_LZ flags did not survive into IR memory metadata"); + Check(Common::ContainsStr(result.ir_dump, "image_flags=0x18"), + "IMAGE_GATHER4_C_O flags did not survive into IR memory metadata"); + Check(Common::ContainsStr(result.ir_dump, "image_flags=0x38"), + "IMAGE_GATHER4_C_LZ_O flags did not survive into IR memory metadata"); + Check(SpirvContainsCapability(result.spirv, 25), + "SPIR-V binary does not request ImageGatherExtended"); + Check(SpirvContainsOpcode(result.spirv, 96), "SPIR-V binary does not contain OpImageGather"); + Check(SpirvContainsOpcode(result.spirv, 97), + "SPIR-V binary does not contain OpImageDrefGather"); + Check(SpirvContainsOpcode(result.spirv, 202), + "SPIR-V binary does not contain packed gather offset extraction"); + Check(SpirvContainsOpcode(result.spirv, 81), + "SPIR-V binary does not contain gather result extraction"); + Check(SpirvContainsOpcode(result.spirv, 124), + "SPIR-V binary does not contain gather result bitcast"); + CheckSpirvBinaryValidates(result.spirv); +} + +void TestNewShaderRecompilerImageLoadVariants() { + const uint32_t shader[] = { + EncodeMimg0(0x00, 0x3), + EncodeMimg1(8, 0, 0, 1), // image_load dmask xy + EncodeMimg0(0x01, 0xf), + EncodeMimg1(12, 1, 0, 4), // image_load_mip dmask xyzw + 0xbf810000u, + }; + + auto user_data = ImageTestUserData(); + ShaderRecompiler::CompileOptions options; + options.stage = ShaderType::Compute; + options.dump_ir = true; + options.user_data = user_data.data(); + + ShaderRecompiler::CompileResult result; + std::string error; + Check(ShaderRecompiler::TryRecompile(shader, options, &result, &error), error.c_str()); + Check(Common::ContainsStr(result.decoded_dump, "image_load"), + "new decoder did not decode MIMG image load"); + Check(Common::ContainsStr(result.decoded_dump, "image_load_mip"), + "new decoder did not decode MIMG image load mip"); + Check(Common::ContainsStr(result.decoded_dump, "dmask=0x3"), + "image_load decode did not preserve partial dmask"); + Check(Common::ContainsStr(result.decoded_dump, "dmask=0xf"), + "image_load_mip decode did not preserve full dmask"); + Check(Common::ContainsStr(result.ir_dump, "ImageLoad v8"), + "image_load did not lower through shared image-load IR"); + Check(Common::ContainsStr(result.ir_dump, "ImageLoad v12"), + "image_load_mip did not lower through shared image-load IR"); + Check(Common::ContainsStr(result.ir_dump, "data_dwords=2"), + "image_load dmask xy did not preserve two-component result metadata"); + Check(Common::ContainsStr(result.ir_dump, "data_dwords=4"), + "image_load_mip dmask xyzw did not preserve four-component result metadata"); + Check(Common::ContainsStr(result.ir_dump, "image_addr=3 image_mip=1"), + "image_load_mip did not preserve mip address component metadata"); + Check(SpirvContainsOpcode(result.spirv, 95), "SPIR-V binary does not contain OpImageFetch"); + Check(SpirvContainsOpcode(result.spirv, 81), + "SPIR-V binary does not contain OpCompositeExtract for image-load dmask"); + Check(SpirvContainsOpcode(result.spirv, 124), + "SPIR-V binary does not contain OpBitcast for image-load result bits"); + CheckSpirvBinaryValidates(result.spirv); +} + +void TestNewShaderRecompilerImageStoreLowering() { + const uint32_t shader[] = { + EncodeMimg0(0x08, 0xf), + EncodeMimg1(20, 0, 0, 4), // image_store + EncodeMimg0(0x09, 0x3), + EncodeMimg1(24, 1, 0, 8), // image_store_mip + 0xbf810000u, + }; + + auto user_data = ImageTestUserData(); + ShaderRecompiler::CompileOptions options; + options.stage = ShaderType::Compute; + options.dump_ir = true; + options.user_data = user_data.data(); + + ShaderRecompiler::CompileResult result; + std::string error; + Check(ShaderRecompiler::TryRecompile(shader, options, &result, &error), error.c_str()); + Check(Common::ContainsStr(result.decoded_dump, "image_store"), + "new decoder did not decode MIMG image store"); + Check(Common::ContainsStr(result.decoded_dump, "image_store_mip"), + "new decoder did not decode MIMG image store mip"); + Check(Common::ContainsStr(result.decoded_dump, "dmask=0xf"), + "image store decode did not preserve full dmask"); + Check(Common::ContainsStr(result.decoded_dump, "dmask=0x3"), + "image store mip decode did not preserve partial dmask"); + Check(Common::ContainsStr(result.ir_dump, "ImageStore null, v20, v4"), + "image_store did not lower to shared image-store IR"); + Check(Common::ContainsStr(result.ir_dump, "ImageStore null, v24, v8"), + "image_store_mip did not lower to shared image-store IR"); + Check(Common::ContainsStr(result.ir_dump, "storage_image"), + "image store IR did not use storage-image resource metadata"); + Check(Common::ContainsStr(result.ir_dump, "data_dwords=4"), + "full-dmask image store did not preserve data component count"); + Check(Common::ContainsStr(result.ir_dump, "image_addr=3 image_mip=1"), + "mip image store did not preserve mip address component count"); + Check(SpirvContainsOpcode(result.spirv, 99), "SPIR-V binary does not contain OpImageWrite"); + Check(SpirvContainsOpcode(result.spirv, 80), + "SPIR-V binary does not contain image texel/coordinate composites"); + CheckSpirvBinaryValidates(result.spirv); +} + +void TestNewShaderRecompilerStorageImage3DDescriptorVariant() { + constexpr uint32_t ImageTypeColor3D = 10; + constexpr uint32_t ImageFormatRgba16f = 71; + constexpr uint32_t SpirvDim3D = 2; + + constexpr uint32_t MimgDim3D = 2; + + const uint32_t shader[] = { + EncodeMimg0(0x08, 0xf, false, MimgDim3D), + EncodeMimg1(20, 5, 0, 4), // image_store 3D + 0xbf810000u, + }; + + ShaderComputeInputInfo input_info = RegressionComputeInputInfo(); + std::array user_data {}; + user_data[20] = 0x1000u; + user_data[21] = ImageFormatRgba16f << 20u; + user_data[22] = 255u | (255u << 14u); + user_data[23] = ImageTypeColor3D << 28u; + + ShaderRecompiler::CompileOptions options; + options.stage = ShaderType::Compute; + options.dump_ir = true; + options.compute_input_info = &input_info; + options.user_data = user_data.data(); + + ShaderRecompiler::CompileResult result; + std::string error; + Check(ShaderRecompiler::TryRecompile(shader, options, &result, &error), error.c_str()); + Check(Common::ContainsStr(result.decoded_dump, "image_dim=3d"), + "MIMG store did not decode the RDNA2 3D instruction dimension"); + const auto has_3d_store = std::any_of( + result.program.blocks.begin(), result.program.blocks.end(), [](const auto& block) { + return std::any_of(block.instructions.begin(), block.instructions.end(), + [](const auto& inst) { + return inst.op == ShaderRecompiler::IR::Opcode::ImageStore && + inst.memory.image_dimension == + ShaderRecompiler::Decoder::ImageDimension::Dim3D && + inst.memory.image_address_components == 3; + }); + }); + Check(has_3d_store, "3D storage image store did not preserve three address components"); + Check(SpirvContainsTypeImage(result.spirv, SpirvDim3D, 0, 2), + "SPIR-V binary does not contain storage 3D image type"); + CheckSpirvBinaryValidates(result.spirv); + + const auto source = DisassembleSpirvBinary(result.spirv); + Check(SpirvSourceHasInstructionUsing(source, "OpAccessChain", "textures2D_L_3D"), + "storage image store did not access the 3D storage descriptor binding"); +} + +void TestNewShaderRecompilerStorageImage2DDescriptorOverridesMimg3D() { + constexpr uint32_t ImageTypeColor2D = 9; + constexpr uint32_t ImageFormatRgba16f = 71; + constexpr uint32_t MimgDim3D = 2; + constexpr uint32_t SpirvDim2D = 1; + + const uint32_t shader[] = { + EncodeMimg0(0x08, 0xf, false, MimgDim3D), + EncodeMimg1(20, 5, 0, 4), + 0xbf810000u, + }; + + ShaderComputeInputInfo input_info = RegressionComputeInputInfo(); + std::array user_data {}; + user_data[20] = 0x1000u; + user_data[21] = ImageFormatRgba16f << 20u; + user_data[22] = 255u | (255u << 14u); + user_data[23] = ImageTypeColor2D << 28u; + + ShaderRecompiler::CompileOptions options; + options.stage = ShaderType::Compute; + options.dump_ir = true; + options.compute_input_info = &input_info; + options.user_data = user_data.data(); + + ShaderRecompiler::CompileResult result; + std::string error; + Check(ShaderRecompiler::TryRecompile(shader, options, &result, &error), error.c_str()); + Check(Common::ContainsStr(result.decoded_dump, "image_dim=3d"), + "test MIMG store should decode as a 3D instruction"); + Check(SpirvContainsTypeImage(result.spirv, SpirvDim2D, 0, 2), + "SPIR-V binary does not contain storage 2D image type"); + CheckSpirvBinaryValidates(result.spirv); + + const auto source = DisassembleSpirvBinary(result.spirv); + Check(SpirvSourceHasInstructionUsing(source, "OpAccessChain", "textures2D_L"), + "2D descriptor storage image store did not access the base storage binding"); + Check(!SpirvSourceHasInstructionUsing(source, "OpAccessChain", "textures2D_L_A"), + "2D descriptor storage image store unexpectedly used the array storage binding"); + Check(!SpirvSourceHasInstructionUsing(source, "OpAccessChain", "textures2D_L_3D"), + "2D descriptor storage image store unexpectedly used the 3D storage binding"); +} + +void TestNewShaderRecompilerImageAtomicLowering() { + const uint32_t shader[] = { + EncodeMimg0(0x11, 0x1, true), + EncodeMimg1(52, 0, 0, 1), // image_atomic_add + EncodeMimg0(0x15, 0x1, true), + EncodeMimg1(53, 0, 0, 1), // image_atomic_umin + EncodeMimg0(0x17, 0x1, true), + EncodeMimg1(57, 0, 0, 1), // image_atomic_umax + EncodeMimg0(0x18, 0x1, true), + EncodeMimg1(54, 0, 0, 1), // image_atomic_and + EncodeMimg0(0x19, 0x1, true), + EncodeMimg1(55, 0, 0, 1), // image_atomic_or + EncodeMimg0(0x1a, 0x1, true), + EncodeMimg1(56, 0, 0, 1), // image_atomic_xor + 0xbf810000u, + }; + + auto user_data = ImageTestUserData(); + ShaderRecompiler::CompileOptions options; + options.stage = ShaderType::Compute; + options.dump_ir = true; + options.user_data = user_data.data(); + + ShaderRecompiler::CompileResult result; + std::string error; + Check(ShaderRecompiler::TryRecompile(shader, options, &result, &error), error.c_str()); + Check(Common::ContainsStr(result.decoded_dump, "image_atomic_add"), + "new decoder did not decode MIMG image atomic add"); + Check(Common::ContainsStr(result.decoded_dump, "image_atomic_umin"), + "new decoder did not decode MIMG image atomic umin"); + Check(Common::ContainsStr(result.decoded_dump, "image_atomic_umax"), + "new decoder did not decode MIMG image atomic umax"); + Check(Common::ContainsStr(result.decoded_dump, "image_atomic_and"), + "new decoder did not decode MIMG image atomic and"); + Check(Common::ContainsStr(result.decoded_dump, "image_atomic_or"), + "new decoder did not decode MIMG image atomic or"); + Check(Common::ContainsStr(result.decoded_dump, "image_atomic_xor"), + "new decoder did not decode MIMG image atomic xor"); + Check(Common::ContainsStr(result.decoded_dump, "dmask=0x1"), + "image atomic decode did not preserve dmask metadata"); + Check(Common::ContainsStr(result.ir_dump, "AtomicAddU32 v52, v52, v1"), + "image_atomic_add did not lower through shared atomic IR"); + Check(Common::ContainsStr(result.ir_dump, "AtomicUMinU32 v53, v53, v1"), + "image_atomic_umin did not lower through shared atomic IR"); + Check(Common::ContainsStr(result.ir_dump, "AtomicUMaxU32 v57, v57, v1"), + "image_atomic_umax did not lower through shared atomic IR"); + Check(Common::ContainsStr(result.ir_dump, "AtomicAndU32 v54, v54, v1"), + "image_atomic_and did not lower through shared atomic IR"); + Check(Common::ContainsStr(result.ir_dump, "AtomicOrU32 v55, v55, v1"), + "image_atomic_or did not lower through shared atomic IR"); + Check(Common::ContainsStr(result.ir_dump, "AtomicXorU32 v56, v56, v1"), + "image_atomic_xor did not lower through shared atomic IR"); + Check(Common::ContainsStr(result.ir_dump, "storage_image_uint"), + "image atomic IR did not use uint storage-image metadata"); + Check(SpirvContainsOpcode(result.spirv, 60), + "SPIR-V binary does not contain OpImageTexelPointer"); + Check(SpirvContainsOpcode(result.spirv, 234), "SPIR-V binary does not contain OpAtomicIAdd"); + Check(SpirvContainsOpcode(result.spirv, 237), "SPIR-V binary does not contain OpAtomicUMin"); + Check(SpirvContainsOpcode(result.spirv, 239), "SPIR-V binary does not contain OpAtomicUMax"); + Check(SpirvContainsOpcode(result.spirv, 240), "SPIR-V binary does not contain OpAtomicAnd"); + Check(SpirvContainsOpcode(result.spirv, 241), "SPIR-V binary does not contain OpAtomicOr"); + Check(SpirvContainsOpcode(result.spirv, 242), "SPIR-V binary does not contain OpAtomicXor"); + Check(SpirvContainsOpcode(result.spirv, 225), + "image atomic SPIR-V binary does not contain OpMemoryBarrier"); + CheckSpirvBinaryValidates(result.spirv); +} + +void TestNewShaderRecompilerVintrpLowering() { + const uint32_t shader[] = { + EncodeVintrp(0, 10, 1, 2, 4), // v_interp_p1_f32 v10, v4, attr1.z + EncodeVintrp(1, 11, 1, 2, 4), // v_interp_p2_f32 v11, v4, attr1.z + EncodeVintrp(2, 12, 0, 3, 2), // v_interp_mov_f32 v12, p0, attr0.w + 0xbf810000u, + }; + + ShaderPixelInputInfo ps_info {}; + ps_info.input_num = 2; + SetIdentityInterpolatorSettings(&ps_info); + + ShaderRecompiler::CompileOptions options; + options.stage = ShaderType::Pixel; + options.pixel_input_info = &ps_info; + options.dump_ir = true; + + ShaderRecompiler::CompileResult result; + std::string error; + Check(ShaderRecompiler::TryRecompile(shader, options, &result, &error), error.c_str()); + Check(Common::ContainsStr(result.decoded_dump, "v_interp_p1_f32"), + "new decoder did not decode VINTRP P1"); + Check(Common::ContainsStr(result.decoded_dump, "v_interp_p2_f32"), + "new decoder did not decode VINTRP P2"); + Check(Common::ContainsStr(result.decoded_dump, "v_interp_mov_f32"), + "new decoder did not decode VINTRP MOV"); + Check(Common::ContainsStr(result.ir_dump, "ControlNop"), + "VINTRP P1 did not lower to an explicit no-op marker"); + Check(Common::ContainsStr(result.ir_dump, "LoadInputF32 v11"), + "VINTRP P2 did not lower to input-load IR"); + Check(Common::ContainsStr(result.ir_dump, "input_attr=1 input_chan=2"), + "VINTRP P2 did not preserve attr/channel metadata"); + Check(Common::ContainsStr(result.ir_dump, "LoadInputF32 v12"), + "VINTRP MOV did not lower to input-load IR"); + Check(Common::ContainsStr(result.ir_dump, "input_attr=0 input_chan=3"), + "VINTRP MOV did not preserve attr/channel metadata"); + Check(ProgramInputCount(result.program, ShaderRecompiler::IR::StageInputKind::Parameter) == 2, + "VINTRP pixel shader did not reflect parameter inputs"); + Check(SpirvContainsOpcode(result.spirv, 61), "SPIR-V binary does not contain OpLoad"); + Check(SpirvContainsOpcode(result.spirv, 62), "SPIR-V binary does not contain OpStore"); + Check(SpirvContainsOpcode(result.spirv, 81), + "SPIR-V binary does not contain input component extraction"); + Check(SpirvContainsOpcode(result.spirv, 124), "SPIR-V binary does not contain input bitcast"); + CheckSpirvBinaryValidates(result.spirv); + + const uint32_t remapped_shader[] = { + EncodeVintrp(1, 11, 2, 0, 4), // v_interp_p2_f32 v11, v4, attr2.x + 0xbf810000u, + }; + ShaderPixelInputInfo remapped_ps_info {}; + remapped_ps_info.input_num = 3; + SetIdentityInterpolatorSettings(&remapped_ps_info); + remapped_ps_info.interpolator_settings[2] = 3; + + options.pixel_input_info = &remapped_ps_info; + + ShaderRecompiler::CompileResult remapped_result; + Check(ShaderRecompiler::TryRecompile(remapped_shader, options, &remapped_result, &error), + error.c_str()); + Check(Common::ContainsStr(remapped_result.ir_dump, "input_attr=2 input_chan=0"), + "remapped VINTRP did not preserve raw pixel attribute metadata"); + Check(SpirvHasDecorationValue(remapped_result.spirv, 30u, 3u), + "remapped VINTRP did not use SPI_PS_INPUT_CNTL input offset as Location"); + Check(!SpirvHasDecorationValue(remapped_result.spirv, 30u, 2u), + "remapped VINTRP still emitted the raw attribute as Location"); + CheckSpirvBinaryValidates(remapped_result.spirv); + + const uint32_t duplicate_location_shader[] = { + EncodeVintrp(0, 6, 1, 0, 4), // v_interp_p1_f32 v6, v4, attr1.x + EncodeVintrp(0, 10, 0, 0, 4), // v_interp_p1_f32 v10, v4, attr0.x + EncodeVintrp(1, 6, 1, 0, 4), // v_interp_p2_f32 v6, v4, attr1.x + EncodeVintrp(1, 10, 0, 0, 4), // v_interp_p2_f32 v10, v4, attr0.x + 0xbf810000u, + }; + ShaderPixelInputInfo duplicate_ps_info {}; + duplicate_ps_info.input_num = 2; + + options.pixel_input_info = &duplicate_ps_info; + + ShaderRecompiler::CompileResult duplicate_result; + Check(ShaderRecompiler::TryRecompile(duplicate_location_shader, options, &duplicate_result, + &error), + error.c_str()); + Check(ProgramInputCount(duplicate_result.program, + ShaderRecompiler::IR::StageInputKind::Parameter) == 2, + "duplicate-location VINTRP shader did not reflect both raw parameter attrs"); + Check(SpirvDecorationValueCount(duplicate_result.spirv, 30u, 0u) == 1, + "duplicate-location VINTRP emitted more than one Location 0 input"); + Check(SpirvDecorationValueCount(duplicate_result.spirv, 30u, 1u) == 1, + "duplicate-location VINTRP did not fall back attr1 to Location 1"); + CheckSpirvBinaryValidates(duplicate_result.spirv); + + const uint32_t flat_shader[] = { + EncodeVintrp(2, 12, 0, 0, 2), // v_interp_mov_f32 v12, p0, attr0.x + 0xbf810000u, + }; + ShaderPixelInputInfo flat_ps_info {}; + flat_ps_info.input_num = 1; + SetIdentityInterpolatorSettings(&flat_ps_info); + flat_ps_info.interpolator_settings[0] = 0x00000400u; + + options.pixel_input_info = &flat_ps_info; + + ShaderRecompiler::CompileResult flat_result; + Check(ShaderRecompiler::TryRecompile(flat_shader, options, &flat_result, &error), + error.c_str()); + Check(SpirvHasDecorationValueWithDecoration(flat_result.spirv, 30u, 0u, 14u), + "flat VINTRP input did not emit a Flat decoration"); + Check(!SpirvHasDecorationValueWithDecoration(flat_result.spirv, 30u, 0u, 13u), + "flat VINTRP input should not also emit NoPerspective"); + Check(SpirvHasDecorationValue(flat_result.spirv, 30u, 0u), + "flat VINTRP input did not preserve Location 0"); + CheckSpirvBinaryValidates(flat_result.spirv); + + ShaderPixelInputInfo no_perspective_ps_info {}; + no_perspective_ps_info.input_num = 1; + no_perspective_ps_info.ps_no_perspective = true; + SetIdentityInterpolatorSettings(&no_perspective_ps_info); + + options.pixel_input_info = &no_perspective_ps_info; + + ShaderRecompiler::CompileResult no_perspective_result; + Check(ShaderRecompiler::TryRecompile(flat_shader, options, &no_perspective_result, &error), + error.c_str()); + Check(SpirvHasDecorationValueWithDecoration(no_perspective_result.spirv, 30u, 0u, 13u), + "no-perspective VINTRP input did not emit a NoPerspective decoration"); + Check(!SpirvHasDecorationValueWithDecoration(no_perspective_result.spirv, 30u, 0u, 14u), + "non-flat no-perspective VINTRP input should not emit Flat"); + CheckSpirvBinaryValidates(no_perspective_result.spirv); +} + +void TestGraphicsCreateInterpolantMapping() { + ShaderRegister regs[32] = {}; + + Check(Gen5::GraphicsCreateInterpolantMapping(regs, nullptr, nullptr) == 0, + "null pixel shader interpolant mapping failed"); + for (uint32_t i = 0; i < 32u; i++) { + Check(regs[i].offset == Pm4::SPI_PS_INPUT_CNTL_0 + i, + "identity interpolant register offset was unexpected"); + Check(regs[i].value == i, "identity interpolant register value was unexpected"); + } + + ShaderSemantic gs_semantics[3] = {}; + gs_semantics[0].semantic = 7; + gs_semantics[0].hardware_mapping = 5; + gs_semantics[1].semantic = 9; + gs_semantics[1].hardware_mapping = 12; + gs_semantics[2].semantic = 10; + gs_semantics[2].hardware_mapping = 4; + gs_semantics[2].is_f16 = 1; + + Shader gs {}; + gs.output_semantics = gs_semantics; + gs.num_output_semantics = static_cast(std::size(gs_semantics)); + + ShaderSemantic ps_semantics[4] = {}; + ps_semantics[0].semantic = 7; + ps_semantics[1].semantic = 8; + ps_semantics[1].default_value = 2; + ps_semantics[2].semantic = 9; + ps_semantics[2].is_flat_shaded = 1; + ps_semantics[2].is_custom = 1; + ps_semantics[2].default_value = 1; + ps_semantics[3].semantic = 10; + ps_semantics[3].is_f16 = 1; + ps_semantics[3].default_value = 3; + ps_semantics[3].default_value_hi = 2; + + Shader ps {}; + ps.input_semantics = ps_semantics; + ps.num_input_semantics = static_cast(std::size(ps_semantics)); + + Check(Gen5::GraphicsCreateInterpolantMapping(regs, &gs, &ps) == 0, + "shader interpolant mapping failed"); + Check(regs[0].value == 0x00000005u, + "matched interpolant mapping did not use GS hardware mapping"); + Check(regs[1].value == 0x00000220u, "missing interpolant mapping did not use PS default value"); + Check(regs[2].value == 0x0000052cu, "flat/custom interpolant mapping bits were unexpected"); + Check(regs[3].value == 0x01580304u, "f16 interpolant mapping bits were unexpected"); + Check(regs[4].offset == Pm4::SPI_PS_INPUT_CNTL_0 + 4u && regs[4].value == 4u, + "interpolant identity tail was not filled"); +} + +void TestNewShaderRecompilerWideMemoryLowering() { + const uint32_t shader[] = { + EncodeSmem0(0x02, 4, 4), + 0u, // s_load_dwordx4 s[4:7] + EncodeSmem0(0x0a, 8, 4), + 0u, // s_buffer_load_dwordx4 s[8:11] + EncodeSmem0(0x09, 106, 4), + 0u, // s_buffer_load_dwordx2 vcc_lo + EncodeMubuf0(0x0d, 16), + EncodeMubuf1(20, 0, 1), // buffer_load_dwordx2 v[20:21] + EncodeMubuf0(0x0f, 32), + EncodeMubuf1(24, 0, 1), // buffer_load_dwordx3 v[24:26] + EncodeMubuf0(0x0e, 48), + EncodeMubuf1(28, 0, 1), // buffer_load_dwordx4 v[28:31] + EncodeMubuf0(0x1d, 64), + EncodeMubuf1(32, 0, 1), // buffer_store_dwordx2 v[32:33] + EncodeMubuf0(0x1f, 80), + EncodeMubuf1(36, 0, 1), // buffer_store_dwordx3 v[36:38] + EncodeMubuf0(0x1e, 96), + EncodeMubuf1(40, 0, 1), // buffer_store_dwordx4 v[40:43] + EncodeMubuf0(0x08, 2), + EncodeMubuf1(44, 0, 1), // buffer_load_ubyte v44 + EncodeMubuf0(0x0a, 4), + EncodeMubuf1(45, 0, 1), // buffer_load_ushort v45 + EncodeFlat0(0x0c, 0, 4), + EncodeFlat1(50, 0x7d, 0, 1), // flat_load_dword + EncodeFlat0(0x0d, 1, 8), + EncodeFlat1(52, 0x7d, 0, 1), // scratch_load_dwordx2 + EncodeFlat0(0x0e, 2, 12), + EncodeFlat1(56, 0x7d, 0, 1), // global_load_dwordx4 + 0xbf810000u, + }; + + ShaderRecompiler::CompileOptions options; + options.stage = ShaderType::Compute; + options.dump_ir = true; + options.flat_memory_base = 0; + + ShaderRecompiler::CompileResult result; + std::string error; + Check(ShaderRecompiler::TryRecompile(shader, options, &result, &error), error.c_str()); + Check(Common::ContainsStr(result.decoded_dump, "s_load_dwordx4"), + "new decoder did not decode SMEM x4 load"); + Check(Common::ContainsStr(result.decoded_dump, "s_buffer_load_dwordx4"), + "new decoder did not decode scalar-buffer x4 load"); + Check(Common::ContainsStr(result.decoded_dump, "s_buffer_load_dwordx2 vcc_lo"), + "new decoder did not decode scalar-buffer x2 load into VCC"); + Check(Common::ContainsStr(result.decoded_dump, "buffer_load_dwordx2"), + "new decoder did not decode buffer x2 load"); + Check(Common::ContainsStr(result.decoded_dump, "buffer_load_dwordx3"), + "new decoder did not decode buffer x3 load"); + Check(Common::ContainsStr(result.decoded_dump, "buffer_load_dwordx4"), + "new decoder did not decode buffer x4 load"); + Check(Common::ContainsStr(result.decoded_dump, "buffer_store_dwordx4"), + "new decoder did not decode buffer x4 store"); + Check(Common::ContainsStr(result.decoded_dump, "buffer_load_ubyte"), + "new decoder did not decode buffer ubyte load"); + Check(Common::ContainsStr(result.decoded_dump, "buffer_load_ushort"), + "new decoder did not decode buffer ushort load"); + Check(Common::ContainsStr(result.decoded_dump, "flat_load_dword"), + "new decoder did not decode flat dword load"); + Check(Common::ContainsStr(result.decoded_dump, "segment=1"), + "new decoder did not preserve scratch segment metadata"); + Check(Common::ContainsStr(result.decoded_dump, "segment=2"), + "new decoder did not preserve global segment metadata"); + Check(Common::ContainsStr(result.decoded_dump, "dwords=4 bits=32"), + "new decoder did not expose width metadata for wide memory ops"); + Check(Common::ContainsStr(result.ir_dump, "SLoadDword s4"), + "s_load_dwordx4 did not expand to first scalar dword IR load"); + Check(Common::ContainsStr(result.ir_dump, "SLoadDword s7"), + "s_load_dwordx4 did not expand to last scalar dword IR load"); + Check(Common::ContainsStr(result.ir_dump, "SBufferLoadDword s8"), + "s_buffer_load_dwordx4 did not expand to first scalar-buffer IR load"); + Check(Common::ContainsStr(result.ir_dump, "SBufferLoadDword s11"), + "s_buffer_load_dwordx4 did not expand to last scalar-buffer IR load"); + Check(Common::ContainsStr(result.ir_dump, "SBufferLoadDword vcc_lo"), + "s_buffer_load_dwordx2 did not expand to VCC low dword"); + Check(Common::ContainsStr(result.ir_dump, "SBufferLoadDword vcc_hi"), + "s_buffer_load_dwordx2 did not expand to VCC high dword"); + Check(Common::ContainsStr(result.ir_dump, "BufferLoadDword v20"), + "buffer_load_dwordx2 did not expand to dword IR loads"); + Check(Common::ContainsStr(result.ir_dump, "BufferLoadDword v31"), + "buffer_load_dwordx4 did not expand to the last dword IR load"); + Check(Common::ContainsStr(result.ir_dump, "BufferStoreDword null, v40"), + "buffer_store_dwordx4 did not expand to first dword IR store"); + Check(Common::ContainsStr(result.ir_dump, "BufferStoreDword null, v43"), + "buffer_store_dwordx4 did not expand to last dword IR store"); + Check(Common::ContainsStr(result.ir_dump, "BufferLoadUbyte v44"), + "buffer_load_ubyte did not lower to sub-dword IR load"); + Check(Common::ContainsStr(result.ir_dump, "BufferLoadUshort v45"), + "buffer_load_ushort did not lower to sub-dword IR load"); + Check(Common::ContainsStr(result.ir_dump, "FlatLoadDword v50"), + "flat_load_dword did not lower to flat dword IR load"); + Check(Common::ContainsStr(result.ir_dump, "scratch"), + "scratch segment load did not lower with scratch metadata"); + Check(Common::ContainsStr(result.ir_dump, "global"), + "global segment load did not lower with global metadata"); + Check(SpirvContainsOpcode(result.spirv, 65), "SPIR-V binary does not contain OpAccessChain"); + Check(SpirvContainsOpcode(result.spirv, 61), "SPIR-V binary does not contain OpLoad"); + Check(SpirvContainsOpcode(result.spirv, 62), "SPIR-V binary does not contain OpStore"); + Check(SpirvContainsOpcode(result.spirv, 194), + "SPIR-V binary does not contain OpShiftRightLogical"); + Check(SpirvContainsOpcode(result.spirv, 196), + "SPIR-V binary does not contain OpShiftLeftLogical"); + Check(SpirvContainsOpcode(result.spirv, 199), "SPIR-V binary does not contain OpBitwiseAnd"); + CheckSpirvBinaryValidates(result.spirv); +} + +void TestNewShaderRecompilerBufferSignedLoadLowering() { + const uint32_t shader[] = { + EncodeMubuf0(0x09, 3), EncodeMubuf1(46, 0, 1), // buffer_load_sbyte + EncodeMubuf0(0x0b, 6), EncodeMubuf1(47, 0, 1), // buffer_load_sshort + 0xbf810000u, + }; + + ShaderRecompiler::CompileOptions options; + options.stage = ShaderType::Compute; + options.dump_ir = true; + + ShaderRecompiler::CompileResult result; + std::string error; + Check(ShaderRecompiler::TryRecompile(shader, options, &result, &error), error.c_str()); + Check(Common::ContainsStr(result.decoded_dump, "buffer_load_sbyte"), + "new decoder did not decode buffer signed byte load"); + Check(Common::ContainsStr(result.decoded_dump, "buffer_load_sshort"), + "new decoder did not decode buffer signed short load"); + Check(Common::ContainsStr(result.decoded_dump, "bits=8"), + "signed byte buffer load did not preserve bit width metadata"); + Check(Common::ContainsStr(result.decoded_dump, "bits=16"), + "signed short buffer load did not preserve bit width metadata"); + Check(Common::ContainsStr(result.decoded_dump, "signed=1"), + "signed buffer loads did not preserve signed metadata"); + Check(Common::ContainsStr(result.ir_dump, "BufferLoadSbyte v46"), + "buffer_load_sbyte did not lower to signed byte IR load"); + Check(Common::ContainsStr(result.ir_dump, "BufferLoadSshort v47"), + "buffer_load_sshort did not lower to signed short IR load"); + Check(SpirvContainsOpcode(result.spirv, 61), "SPIR-V binary does not contain OpLoad"); + Check(SpirvContainsOpcode(result.spirv, 195), + "SPIR-V binary does not contain OpShiftRightArithmetic for sign extension"); + Check(SpirvContainsOpcode(result.spirv, 196), + "SPIR-V binary does not contain OpShiftLeftLogical for sign extension"); + Check(SpirvContainsOpcode(result.spirv, 199), "SPIR-V binary does not contain OpBitwiseAnd"); + CheckSpirvBinaryValidates(result.spirv); +} + +void TestNewShaderRecompilerBufferSubDwordStoreLowering() { + const uint32_t shader[] = { + EncodeMubuf0(0x18, 2), EncodeMubuf1(48, 0, 1), // buffer_store_byte + EncodeMubuf0(0x1a, 4), EncodeMubuf1(49, 0, 1), // buffer_store_short + 0xbf810000u, + }; + + ShaderRecompiler::CompileOptions options; + options.stage = ShaderType::Compute; + options.dump_ir = true; + + ShaderRecompiler::CompileResult result; + std::string error; + Check(ShaderRecompiler::TryRecompile(shader, options, &result, &error), error.c_str()); + Check(Common::ContainsStr(result.decoded_dump, "buffer_store_byte"), + "new decoder did not decode buffer byte store"); + Check(Common::ContainsStr(result.decoded_dump, "buffer_store_short"), + "new decoder did not decode buffer short store"); + Check(Common::ContainsStr(result.decoded_dump, "bits=8"), + "buffer byte store did not preserve bit width metadata"); + Check(Common::ContainsStr(result.decoded_dump, "bits=16"), + "buffer short store did not preserve bit width metadata"); + Check(Common::ContainsStr(result.ir_dump, "BufferStoreByte null, v48"), + "buffer_store_byte did not lower to byte store IR"); + Check(Common::ContainsStr(result.ir_dump, "BufferStoreShort null, v49"), + "buffer_store_short did not lower to short store IR"); + Check(SpirvContainsOpcode(result.spirv, 61), + "SPIR-V binary does not contain OpLoad for sub-dword store RMW"); + Check(SpirvContainsOpcode(result.spirv, 62), + "SPIR-V binary does not contain OpStore for sub-dword store RMW"); + Check(SpirvContainsOpcode(result.spirv, 197), "SPIR-V binary does not contain OpBitwiseOr"); + Check(SpirvContainsOpcode(result.spirv, 199), "SPIR-V binary does not contain OpBitwiseAnd"); + Check(SpirvContainsOpcode(result.spirv, 200), "SPIR-V binary does not contain OpNot"); + Check(SpirvContainsOpcode(result.spirv, 196), + "SPIR-V binary does not contain OpShiftLeftLogical"); + CheckSpirvBinaryValidates(result.spirv); +} + +void TestNewShaderRecompilerMubufFormatLowering() { + const uint32_t shader[] = { + EncodeMubuf0(0x00, 4), + EncodeMubuf1(84, 0, 1), // buffer_load_format_x + EncodeMubuf0(0x01, 8), + EncodeMubuf1(88, 0, 1), // buffer_load_format_xy + EncodeMubuf0(0x02, 12), + EncodeMubuf1(92, 0, 1), // buffer_load_format_xyz + EncodeMubuf0(0x03, 16), + EncodeMubuf1(96, 0, 1), // buffer_load_format_xyzw + EncodeMubuf0(0x04, 20), + EncodeMubuf1(100, 0, 1), // buffer_store_format_x + EncodeMubuf0(0x05, 24), + EncodeMubuf1(104, 0, 1), // buffer_store_format_xy + EncodeMubuf0(0x06, 28), + EncodeMubuf1(108, 0, 1), // buffer_store_format_xyz + EncodeMubuf0(0x07, 32), + EncodeMubuf1(112, 0, 1), // buffer_store_format_xyzw + 0xbf810000u, + }; + + ShaderRecompiler::CompileOptions options; + options.stage = ShaderType::Compute; + options.dump_ir = true; + + ShaderRecompiler::CompileResult result; + std::string error; + Check(ShaderRecompiler::TryRecompile(shader, options, &result, &error), error.c_str()); + Check(Common::ContainsStr(result.decoded_dump, "buffer_load_format_x"), + "new decoder did not decode MUBUF format-x load"); + Check(Common::ContainsStr(result.decoded_dump, "buffer_load_format_xyzw"), + "new decoder did not decode MUBUF format-xyzw load"); + Check(Common::ContainsStr(result.decoded_dump, "buffer_store_format_x"), + "new decoder did not decode MUBUF format-x store"); + Check(Common::ContainsStr(result.decoded_dump, "buffer_store_format_xyzw"), + "new decoder did not decode MUBUF format-xyzw store"); + Check(Common::ContainsStr(result.decoded_dump, "typed=0 formatted=1"), + "MUBUF format decode did not preserve formatted non-typed metadata"); + Check(Common::ContainsStr(result.ir_dump, "BufferLoadDword v84"), + "MUBUF format-x load did not lower through shared buffer load IR"); + Check(Common::ContainsStr(result.ir_dump, "BufferLoadDword v99"), + "MUBUF format-xyzw load did not expand to the last shared dword load"); + Check(Common::ContainsStr(result.ir_dump, "BufferStoreDword null, v100"), + "MUBUF format-x store did not lower through shared buffer store IR"); + Check(Common::ContainsStr(result.ir_dump, "BufferStoreDword null, v115"), + "MUBUF format-xyzw store did not expand to the last shared dword store"); + Check(Common::ContainsStr(result.ir_dump, "typed=0 formatted=1"), + "MUBUF format metadata did not survive into IR"); + Check(SpirvContainsOpcode(result.spirv, 65), "SPIR-V binary does not contain OpAccessChain"); + Check(SpirvContainsOpcode(result.spirv, 61), "SPIR-V binary does not contain OpLoad"); + Check(SpirvContainsOpcode(result.spirv, 62), "SPIR-V binary does not contain OpStore"); + CheckSpirvBinaryValidates(result.spirv); +} + +void TestNewShaderRecompilerFormattedStoreUsesRuntimeArrayLengthOnly() { + const uint32_t shader[] = { + EncodeMubuf0(0x04), + EncodeMubuf1(0, 0, 1), // buffer_store_format_x + 0xbf810000u, + }; + + std::array user_data {}; + user_data[1] = 4u << 16u; + user_data[2] = 5u; + user_data[3] = Prospero::GpuEnumValue(Prospero::BufferFormat::k32UInt) << 12u; + + ShaderRecompiler::CompileOptions options; + options.stage = ShaderType::Compute; + options.dump_ir = true; + options.user_data = user_data.data(); + + ShaderRecompiler::CompileResult result; + std::string error; + Check(ShaderRecompiler::TryRecompile(shader, options, &result, &error), error.c_str()); + Check(result.resources.buffers.size() == 1 && result.resources.buffers[0].dwords[2] == 5u, + "formatted store test did not preserve descriptor NumRecords"); + Check(Common::ContainsStr(result.decoded_dump, "buffer_store_format_x"), + "formatted store regression did not decode buffer_store_format_x"); + Check(Common::ContainsStr(result.ir_dump, "typed=0 formatted=1"), + "formatted store regression did not preserve formatted metadata"); + CheckSpirvBinaryValidates(result.spirv); + + const auto source = DisassembleSpirvBinary(result.spirv); + Check(Common::ContainsStr(source, "OpArrayLength"), + "formatted store SPIR-V lacks runtime storage-buffer bounds check"); + Check(!SpirvSourceHasInstructionUsing(source, "OpULessThan", "%uint_5"), + "formatted store SPIR-V baked descriptor NumRecords into a store guard"); +} + +void TestNewShaderRecompilerTypedBufferLowering() { + const uint32_t shader[] = { + EncodeMtbuf0(0x00, 14, 7, 4), + EncodeMtbuf1(0x00, 60, 0, 1), // tbuffer_load_format_x + EncodeMtbuf0(0x01, 13, 7, 8), + EncodeMtbuf1(0x01, 61, 0, 1), // tbuffer_load_format_xy + EncodeMtbuf0(0x02, 13, 4, 12), + EncodeMtbuf1(0x02, 64, 0, 1), // tbuffer_load_format_xyz + EncodeMtbuf0(0x03, 14, 7, 16, true, true), + EncodeMtbuf1(0x03, 68, 0, 1), // tbuffer_load_format_xyzw + EncodeMtbuf0(0x04, 14, 7, 20), + EncodeMtbuf1(0x04, 72, 0, 1), // tbuffer_store_format_x + EncodeMtbuf0(0x05, 13, 7, 24), + EncodeMtbuf1(0x05, 76, 0, 1), // tbuffer_store_format_xy + EncodeMtbuf0(0x07, 14, 7, 28), + EncodeMtbuf1(0x07, 80, 0, 1), // tbuffer_store_format_xyzw + 0xbf810000u, + }; + + ShaderRecompiler::CompileOptions options; + options.stage = ShaderType::Compute; + options.dump_ir = true; + + ShaderRecompiler::CompileResult result; + std::string error; + Check(ShaderRecompiler::TryRecompile(shader, options, &result, &error), error.c_str()); + Check(Common::ContainsStr(result.decoded_dump, "tbuffer_load_format_x"), + "new decoder did not decode typed buffer format-x load"); + Check(Common::ContainsStr(result.decoded_dump, "tbuffer_load_format_xy"), + "new decoder did not decode typed buffer format-xy load"); + Check(Common::ContainsStr(result.decoded_dump, "tbuffer_load_format_xyz"), + "new decoder did not decode typed buffer format-xyz load"); + Check(Common::ContainsStr(result.decoded_dump, "tbuffer_load_format_xyzw"), + "new decoder did not decode typed buffer format-xyzw load"); + Check(Common::ContainsStr(result.decoded_dump, "tbuffer_store_format_x"), + "new decoder did not decode typed buffer format-x store"); + Check(Common::ContainsStr(result.decoded_dump, "tbuffer_store_format_xy"), + "new decoder did not decode typed buffer format-xy store"); + Check(Common::ContainsStr(result.decoded_dump, "tbuffer_store_format_xyzw"), + "new decoder did not decode typed buffer format-xyzw store"); + Check(Common::ContainsStr(result.decoded_dump, "dfmt=14 nfmt=7"), + "MTBUF decode did not expose dfmt/nfmt metadata"); + Check(Common::ContainsStr(result.decoded_dump, "typed=1"), + "MTBUF decode did not preserve typed metadata"); + Check(Common::ContainsStr(result.decoded_dump, "offen=1"), + "MTBUF decode did not preserve offen metadata"); + Check(Common::ContainsStr(result.ir_dump, "BufferLoadDword v60"), + "typed buffer x load did not lower through shared buffer load IR"); + Check(Common::ContainsStr(result.ir_dump, "BufferLoadDword v71"), + "typed buffer xyzw load did not expand to last shared dword load"); + Check(Common::ContainsStr(result.ir_dump, "BufferStoreDword null, v72"), + "typed buffer x store did not lower through shared buffer store IR"); + Check(Common::ContainsStr(result.ir_dump, "BufferStoreDword null, v83"), + "typed buffer xyzw store did not expand to last shared dword store"); + Check(Common::ContainsStr(result.ir_dump, "typed=1"), + "typed buffer metadata did not survive into IR"); + Check(Common::ContainsStr(result.ir_dump, "dfmt=13 nfmt=7"), + "typed buffer format metadata did not survive into IR"); + Check(SpirvContainsOpcode(result.spirv, 65), "SPIR-V binary does not contain OpAccessChain"); + Check(SpirvContainsOpcode(result.spirv, 61), "SPIR-V binary does not contain OpLoad"); + Check(SpirvContainsOpcode(result.spirv, 62), "SPIR-V binary does not contain OpStore"); + CheckSpirvBinaryValidates(result.spirv); +} + +void TestNewShaderRecompilerFlatOldBackedLowering() { + const uint32_t shader[] = { + EncodeFlat0(0x08, 0, 4), + EncodeFlat1(9, 0x7d, 0, 1), // flat_load_ubyte + 0xbf810000u, + }; + + ShaderRecompiler::CompileOptions options; + options.stage = ShaderType::Compute; + options.dump_ir = true; + options.flat_memory_base = 0; + + ShaderRecompiler::CompileResult result; + std::string error; + Check(ShaderRecompiler::TryRecompile(shader, options, &result, &error), error.c_str()); + Check(Common::ContainsStr(result.decoded_dump, "flat_load_ubyte"), + "new decoder did not decode old-backed FLAT ubyte load"); + Check(Common::ContainsStr(result.ir_dump, "FlatLoadUbyte v9"), + "old-backed FLAT ubyte load did not lower to IR"); + Check(SpirvContainsOpcode(result.spirv, 65), "SPIR-V binary does not contain OpAccessChain"); + Check(SpirvContainsOpcode(result.spirv, 61), "SPIR-V binary does not contain OpLoad"); + Check(SpirvContainsOpcode(result.spirv, 194), + "SPIR-V binary does not contain OpShiftRightLogical"); + Check(SpirvContainsOpcode(result.spirv, 196), + "SPIR-V binary does not contain OpShiftLeftLogical"); + Check(SpirvContainsOpcode(result.spirv, 199), "SPIR-V binary does not contain OpBitwiseAnd"); + CheckSpirvBinaryValidates(result.spirv); +} + +void TestNewShaderRecompilerUnbasedFlatRequiresTranslator() { + const uint32_t shader[] = { + EncodeFlat0(0x0c, 0, 0), + EncodeFlat1(0, 0x7d, 0, 1), + 0xbf810000u, + }; + + ShaderRecompiler::CompileOptions options; + options.stage = ShaderType::Compute; + + ShaderRecompiler::CompileResult result; + std::string error; + Check(!ShaderRecompiler::TryRecompile(shader, options, &result, &error) && + Common::ContainsStr(error, "requires runtime guest-address translation"), + "unbased FLAT compiled without an explicit translator"); +} + +void TestNewShaderRecompilerFlatSignedLoadLowering() { + const uint32_t shader[] = { + EncodeFlat0(0x09, 0, 4), + EncodeFlat1(10, 0x7d, 0, 1), // flat_load_sbyte + EncodeFlat0(0x0b, 1, 8), + EncodeFlat1(11, 0x7d, 0, 1), // scratch_load_sshort + EncodeFlat0(0x09, 2, 12), + EncodeFlat1(12, 0x7d, 0, 1), // global_load_sbyte + 0xbf810000u, + }; + + ShaderRecompiler::CompileOptions options; + options.stage = ShaderType::Compute; + options.dump_ir = true; + options.flat_memory_base = 0; + + ShaderRecompiler::CompileResult result; + std::string error; + Check(ShaderRecompiler::TryRecompile(shader, options, &result, &error), error.c_str()); + Check(Common::ContainsStr(result.decoded_dump, "flat_load_sbyte"), + "new decoder did not decode flat signed byte load"); + Check(Common::ContainsStr(result.decoded_dump, "flat_load_sshort"), + "new decoder did not decode flat signed short load"); + Check(Common::ContainsStr(result.decoded_dump, "bits=8"), + "signed byte flat load did not preserve bit width metadata"); + Check(Common::ContainsStr(result.decoded_dump, "bits=16"), + "signed short flat load did not preserve bit width metadata"); + Check(Common::ContainsStr(result.decoded_dump, "signed=1"), + "signed flat loads did not preserve signed metadata"); + Check(Common::ContainsStr(result.ir_dump, "FlatLoadSbyte v10"), + "flat_load_sbyte did not lower to signed byte IR load"); + Check(Common::ContainsStr(result.ir_dump, "FlatLoadSshort v11"), + "flat_load_sshort did not lower to signed short IR load"); + Check(Common::ContainsStr(result.ir_dump, "scratch"), + "signed scratch load did not preserve scratch metadata"); + Check(Common::ContainsStr(result.ir_dump, "global"), + "signed global load did not preserve global metadata"); + Check(SpirvContainsOpcode(result.spirv, 61), "SPIR-V binary does not contain OpLoad"); + Check(SpirvContainsOpcode(result.spirv, 195), + "SPIR-V binary does not contain OpShiftRightArithmetic for sign extension"); + Check(SpirvContainsOpcode(result.spirv, 196), + "SPIR-V binary does not contain OpShiftLeftLogical for sign extension"); + Check(SpirvContainsOpcode(result.spirv, 199), "SPIR-V binary does not contain OpBitwiseAnd"); + CheckSpirvBinaryValidates(result.spirv); +} + +void TestNewShaderRecompilerFlatStoreLowering() { + const uint32_t shader[] = { + EncodeFlat0(0x18, 0, 2), + EncodeFlat1(0, 0x7d, 72, 1), // flat_store_byte + EncodeFlat0(0x1a, 1, 6), + EncodeFlat1(0, 0x7d, 73, 1), // scratch_store_short + EncodeFlat0(0x1c, 0, 4), + EncodeFlat1(0, 0x7d, 60, 1), // flat_store_dword + EncodeFlat0(0x1d, 1, 8), + EncodeFlat1(0, 0x7d, 61, 1), // scratch_store_dwordx2 + EncodeFlat0(0x1f, 2, 12), + EncodeFlat1(0, 0x7d, 64, 1), // global_store_dwordx3 + EncodeFlat0(0x1e, 2, 16), + EncodeFlat1(0, 0x7d, 68, 1), // global_store_dwordx4 + 0xbf810000u, + }; + + ShaderRecompiler::CompileOptions options; + options.stage = ShaderType::Compute; + options.dump_ir = true; + options.flat_memory_base = 0; + + ShaderRecompiler::CompileResult result; + std::string error; + Check(ShaderRecompiler::TryRecompile(shader, options, &result, &error), error.c_str()); + Check(Common::ContainsStr(result.decoded_dump, "flat_store_byte"), + "new decoder did not decode flat byte store"); + Check(Common::ContainsStr(result.decoded_dump, "flat_store_short"), + "new decoder did not decode flat short store"); + Check(Common::ContainsStr(result.decoded_dump, "flat_store_dword"), + "new decoder did not decode flat dword store"); + Check(Common::ContainsStr(result.decoded_dump, "flat_store_dwordx2"), + "new decoder did not decode flat dwordx2 store"); + Check(Common::ContainsStr(result.decoded_dump, "flat_store_dwordx3"), + "new decoder did not decode flat dwordx3 store"); + Check(Common::ContainsStr(result.decoded_dump, "flat_store_dwordx4"), + "new decoder did not decode flat dwordx4 store"); + Check(Common::ContainsStr(result.decoded_dump, "segment=1"), + "flat store decode did not preserve scratch segment metadata"); + Check(Common::ContainsStr(result.decoded_dump, "segment=2"), + "flat store decode did not preserve global segment metadata"); + Check(Common::ContainsStr(result.decoded_dump, "dwords=1 bits=8"), + "flat byte store decode did not expose byte-width metadata"); + Check(Common::ContainsStr(result.decoded_dump, "dwords=1 bits=16"), + "flat short store decode did not expose short-width metadata"); + Check(Common::ContainsStr(result.decoded_dump, "dwords=4 bits=32"), + "flat store decode did not expose wide width metadata"); + Check(Common::ContainsStr(result.ir_dump, "FlatStoreByte null, v72"), + "flat_store_byte did not lower through shared flat store IR"); + Check(Common::ContainsStr(result.ir_dump, "FlatStoreShort null, v73"), + "flat_store_short did not lower through shared flat store IR"); + Check(Common::ContainsStr(result.ir_dump, "FlatStoreDword null, v60"), + "flat_store_dword did not lower through shared flat store IR"); + Check(Common::ContainsStr(result.ir_dump, "FlatStoreDword null, v62"), + "scratch_store_dwordx2 did not expand to the last dword store"); + Check(Common::ContainsStr(result.ir_dump, "FlatStoreDword null, v66"), + "global_store_dwordx3 did not expand to the last dword store"); + Check(Common::ContainsStr(result.ir_dump, "FlatStoreDword null, v71"), + "global_store_dwordx4 did not expand to the last dword store"); + Check(Common::ContainsStr(result.ir_dump, "flat"), + "flat store IR did not preserve flat resource metadata"); + Check(Common::ContainsStr(result.ir_dump, "scratch"), + "flat store IR did not preserve scratch resource metadata"); + Check(Common::ContainsStr(result.ir_dump, "global"), + "flat store IR did not preserve global resource metadata"); + Check(SpirvContainsOpcode(result.spirv, 61), + "SPIR-V binary does not contain OpLoad for sub-dword flat store merge"); + Check(SpirvContainsOpcode(result.spirv, 65), "SPIR-V binary does not contain OpAccessChain"); + Check(SpirvContainsOpcode(result.spirv, 62), "SPIR-V binary does not contain OpStore"); + Check(SpirvContainsOpcode(result.spirv, 197), + "SPIR-V binary does not contain OpBitwiseOr for sub-dword flat store merge"); + Check(SpirvContainsOpcode(result.spirv, 200), + "SPIR-V binary does not contain OpNot for sub-dword flat store mask"); + CheckSpirvBinaryValidates(result.spirv); +} + +void TestNewShaderRecompilerAtomicLowering() { + const uint32_t shader[] = { + EncodeMubuf0(0x30, 4, true, true), + EncodeMubuf1(0, 0, 1), // buffer_atomic_swap + EncodeMubuf0(0x32, 8, true, true), + EncodeMubuf1(1, 0, 1), // buffer_atomic_add + EncodeMubuf0(0x33, 12, true, true), + EncodeMubuf1(7, 0, 1), // buffer_atomic_sub + EncodeMubuf0(0x35, 16, true, true), + EncodeMubuf1(8, 0, 1), // buffer_atomic_smin + EncodeMubuf0(0x36, 20, true, true), + EncodeMubuf1(2, 0, 1), // buffer_atomic_umin + EncodeMubuf0(0x37, 24, true, true), + EncodeMubuf1(9, 0, 1), // buffer_atomic_smax + EncodeMubuf0(0x38, 28, true, true), + EncodeMubuf1(3, 0, 1), // buffer_atomic_umax + EncodeMubuf0(0x39, 32, true, true), + EncodeMubuf1(4, 0, 1), // buffer_atomic_and + EncodeMubuf0(0x3a, 36, true, true), + EncodeMubuf1(5, 0, 1), // buffer_atomic_or + EncodeMubuf0(0x3b, 40, true, true), + EncodeMubuf1(6, 0, 1), // buffer_atomic_xor + EncodeDs0(0x00), + EncodeDs1(0, 2, 1), // ds_add_u32 + EncodeDs0(0x01), + EncodeDs1(0, 10, 1), // ds_sub_u32 + EncodeDs0(0x05), + EncodeDs1(0, 11, 1), // ds_min_i32 + EncodeDs0(0x06), + EncodeDs1(0, 12, 1), // ds_max_i32 + EncodeDs0(0x09), + EncodeDs1(0, 13, 1), // ds_and_b32 + EncodeDs0(0x0b), + EncodeDs1(0, 14, 1), // ds_xor_b32 + EncodeDs0(0x20), + EncodeDs1(15, 2, 1), // ds_add_rtn_u32 + EncodeDs0(0x21), + EncodeDs1(16, 10, 1), // ds_sub_rtn_u32 + EncodeDs0(0x25), + EncodeDs1(17, 11, 1), // ds_min_rtn_i32 + EncodeDs0(0x26), + EncodeDs1(18, 12, 1), // ds_max_rtn_i32 + EncodeDs0(0x27), + EncodeDs1(19, 2, 1), // ds_min_rtn_u32 + EncodeDs0(0x28), + EncodeDs1(20, 2, 1), // ds_max_rtn_u32 + EncodeDs0(0x29), + EncodeDs1(21, 13, 1), // ds_and_rtn_b32 + EncodeDs0(0x2a), + EncodeDs1(22, 2, 1), // ds_or_rtn_b32 + EncodeDs0(0x2b), + EncodeDs1(23, 14, 1), // ds_xor_rtn_b32 + EncodeDs0(0x2d), + EncodeDs1(24, 15, 1), // ds_wrxchg_rtn_b32 + 0xbf810000u, + }; + + ShaderRecompiler::CompileOptions options; + options.stage = ShaderType::Compute; + options.dump_ir = true; + + ShaderRecompiler::CompileResult result; + std::string error; + Check(ShaderRecompiler::TryRecompile(shader, options, &result, &error), error.c_str()); + Check(Common::ContainsStr(result.decoded_dump, "buffer_atomic_swap"), + "new decoder did not decode buffer atomic swap"); + Check(Common::ContainsStr(result.decoded_dump, "buffer_atomic_add"), + "new decoder did not decode buffer atomic add"); + Check(Common::ContainsStr(result.decoded_dump, "buffer_atomic_sub"), + "new decoder did not decode buffer atomic sub"); + Check(Common::ContainsStr(result.decoded_dump, "buffer_atomic_smin"), + "new decoder did not decode buffer atomic signed min"); + Check(Common::ContainsStr(result.decoded_dump, "buffer_atomic_smax"), + "new decoder did not decode buffer atomic signed max"); + Check(Common::ContainsStr(result.decoded_dump, "buffer_atomic_xor"), + "new decoder did not decode buffer atomic xor"); + Check(Common::ContainsStr(result.decoded_dump, "ds_add_u32"), + "new decoder did not decode DS atomic add"); + Check(Common::ContainsStr(result.decoded_dump, "ds_sub_u32"), + "new decoder did not decode DS atomic sub"); + Check(Common::ContainsStr(result.decoded_dump, "ds_min_i32"), + "new decoder did not decode DS signed min"); + Check(Common::ContainsStr(result.decoded_dump, "ds_xor_rtn_b32"), + "new decoder did not decode DS xor-return"); + Check(Common::ContainsStr(result.decoded_dump, "ds_wrxchg_rtn_b32"), + "new decoder did not decode DS write-exchange-return"); + Check(Common::ContainsStr(result.decoded_dump, "ds_add_rtn_u32"), + "new decoder did not decode DS atomic add-return"); + Check(Common::ContainsStr(result.ir_dump, "AtomicSwapU32 v0"), + "buffer atomic swap did not lower to IR"); + Check(Common::ContainsStr(result.ir_dump, "AtomicAddU32 v1"), + "buffer atomic add did not lower to IR"); + Check(Common::ContainsStr(result.ir_dump, "AtomicSubU32 v7"), + "buffer atomic sub did not lower to IR"); + Check(Common::ContainsStr(result.ir_dump, "AtomicSMinI32 v8"), + "buffer atomic signed min did not lower to IR"); + Check(Common::ContainsStr(result.ir_dump, "AtomicUMinU32 v2"), + "buffer atomic umin did not lower to IR"); + Check(Common::ContainsStr(result.ir_dump, "AtomicSMaxI32 v9"), + "buffer atomic signed max did not lower to IR"); + Check(Common::ContainsStr(result.ir_dump, "AtomicUMaxU32 v3"), + "buffer atomic umax did not lower to IR"); + Check(Common::ContainsStr(result.ir_dump, "AtomicAndU32 v4"), + "buffer atomic and did not lower to IR"); + Check(Common::ContainsStr(result.ir_dump, "AtomicOrU32 v5"), + "buffer atomic or did not lower to IR"); + Check(Common::ContainsStr(result.ir_dump, "AtomicXorU32 v6"), + "buffer atomic xor did not lower to IR"); + Check(Common::ContainsStr(result.ir_dump, "AtomicAddU32 null, v2"), + "DS no-return atomic add did not lower to IR"); + Check(Common::ContainsStr(result.ir_dump, "AtomicSubU32 null, v10"), + "DS no-return atomic sub did not lower to IR"); + Check(Common::ContainsStr(result.ir_dump, "AtomicSMinI32 null, v11"), + "DS no-return signed min did not lower to IR"); + Check(Common::ContainsStr(result.ir_dump, "AtomicSMaxI32 null, v12"), + "DS no-return signed max did not lower to IR"); + Check(Common::ContainsStr(result.ir_dump, "AtomicAndU32 null, v13"), + "DS no-return and did not lower to IR"); + Check(Common::ContainsStr(result.ir_dump, "AtomicXorU32 null, v14"), + "DS no-return xor did not lower to IR"); + Check(Common::ContainsStr(result.ir_dump, "AtomicAddU32 v15"), + "DS add-return did not lower to IR"); + Check(Common::ContainsStr(result.ir_dump, "AtomicSubU32 v16"), + "DS sub-return did not lower to IR"); + Check(Common::ContainsStr(result.ir_dump, "AtomicSMinI32 v17"), + "DS signed min-return did not lower to IR"); + Check(Common::ContainsStr(result.ir_dump, "AtomicSMaxI32 v18"), + "DS signed max-return did not lower to IR"); + Check(Common::ContainsStr(result.ir_dump, "AtomicUMinU32 v19"), + "DS unsigned min-return did not lower to IR"); + Check(Common::ContainsStr(result.ir_dump, "AtomicUMaxU32 v20"), + "DS unsigned max-return did not lower to IR"); + Check(Common::ContainsStr(result.ir_dump, "AtomicAndU32 v21"), + "DS and-return did not lower to IR"); + Check(Common::ContainsStr(result.ir_dump, "AtomicOrU32 v22"), + "DS or-return did not lower to IR"); + Check(Common::ContainsStr(result.ir_dump, "AtomicXorU32 v23"), + "DS xor-return did not lower to IR"); + Check(Common::ContainsStr(result.ir_dump, "AtomicSwapU32 v24"), + "DS write-exchange-return did not lower to shared atomic swap IR"); + Check(SpirvContainsOpcode(result.spirv, 229), + "SPIR-V binary does not contain OpAtomicExchange"); + Check(SpirvContainsOpcode(result.spirv, 234), "SPIR-V binary does not contain OpAtomicIAdd"); + Check(SpirvContainsOpcode(result.spirv, 235), "SPIR-V binary does not contain OpAtomicISub"); + Check(SpirvContainsOpcode(result.spirv, 236), "SPIR-V binary does not contain OpAtomicSMin"); + Check(SpirvContainsOpcode(result.spirv, 237), "SPIR-V binary does not contain OpAtomicUMin"); + Check(SpirvContainsOpcode(result.spirv, 238), "SPIR-V binary does not contain OpAtomicSMax"); + Check(SpirvContainsOpcode(result.spirv, 239), "SPIR-V binary does not contain OpAtomicUMax"); + Check(SpirvContainsOpcode(result.spirv, 240), "SPIR-V binary does not contain OpAtomicAnd"); + Check(SpirvContainsOpcode(result.spirv, 241), "SPIR-V binary does not contain OpAtomicOr"); + Check(SpirvContainsOpcode(result.spirv, 242), "SPIR-V binary does not contain OpAtomicXor"); + Check(SpirvContainsOpcode(result.spirv, 225), + "buffer atomic SPIR-V binary does not contain OpMemoryBarrier"); + CheckSpirvBinaryValidates(result.spirv); +} + +void TestNewShaderRecompilerDsReadWrite2Lowering() { + const uint32_t shader[] = { + EncodeDs0(0x0e, (3u << 8u) | 1u), + EncodeDs1Ex(0, 61, 60, 1), + EncodeDs0(0x37, (4u << 8u) | 2u), + EncodeDs1Ex(70, 0, 0, 1), + EncodeDs0(0x38, (5u << 8u) | 1u), + EncodeDs1Ex(72, 0, 0, 1), + EncodeDs0(0x77, (6u << 8u) | 2u), + EncodeDs1Ex(80, 0, 0, 1), + 0xbf810000u, + }; + + ShaderRecompiler::CompileOptions options; + options.stage = ShaderType::Compute; + options.dump_ir = true; + + ShaderRecompiler::CompileResult result; + std::string error; + Check(ShaderRecompiler::TryRecompile(shader, options, &result, &error), error.c_str()); + Check(Common::ContainsStr(result.decoded_dump, "ds_write2_b32"), + "new decoder did not decode old-backed DS write2"); + Check(Common::ContainsStr(result.decoded_dump, "ds_read2_b32"), + "new decoder did not decode old-backed DS read2"); + Check(Common::ContainsStr(result.decoded_dump, "ds_read2_b64"), + "new decoder did not decode old-backed DS read2 b64"); + Check(Common::ContainsStr(result.decoded_dump, "offset=4"), + "DS write2 decode did not scale offset0 to bytes"); + Check(Common::ContainsStr(result.decoded_dump, "offset2=12"), + "DS write2 decode did not scale offset1 to bytes"); + Check(Common::ContainsStr(result.decoded_dump, "offset=8"), + "DS read2 decode did not scale offset0 to bytes"); + Check(Common::ContainsStr(result.decoded_dump, "offset2=16"), + "DS read2 decode did not scale offset1 to bytes"); + Check(Common::ContainsStr(result.decoded_dump, "offset=256"), + "DS read2 st64 decode did not scale offset0 to bytes"); + Check(Common::ContainsStr(result.decoded_dump, "offset2=1280"), + "DS read2 st64 decode did not scale offset1 to bytes"); + Check(Common::ContainsStr(result.decoded_dump, "offset=16"), + "DS read2 b64 decode did not scale offset0 to bytes"); + Check(Common::ContainsStr(result.decoded_dump, "offset2=48"), + "DS read2 b64 decode did not scale offset1 to bytes"); + Check(Common::ContainsStr(result.decoded_dump, "dwords=2 bits=32"), + "DS read/write2 decode did not preserve two-dword metadata"); + Check(Common::ContainsStr(result.decoded_dump, "dwords=4 bits=32"), + "DS read2 b64 decode did not preserve four-dword metadata"); + Check(Common::ContainsStr(result.ir_dump, "DsWriteB32 null, v60"), + "DS write2 did not lower first dword through shared LDS store IR"); + Check(Common::ContainsStr(result.ir_dump, "DsWriteB32 null, v61"), + "DS write2 did not lower second dword through shared LDS store IR"); + Check(Common::ContainsStr(result.ir_dump, "DsReadB32 v70"), + "DS read2 did not lower first dword through shared LDS load IR"); + Check(Common::ContainsStr(result.ir_dump, "DsReadB32 v71"), + "DS read2 did not lower second dword through shared LDS load IR"); + Check(Common::ContainsStr(result.ir_dump, "DsReadB32 v72"), + "DS read2 st64 did not lower first dword through shared LDS load IR"); + Check(Common::ContainsStr(result.ir_dump, "DsReadB32 v73"), + "DS read2 st64 did not lower second dword through shared LDS load IR"); + Check(Common::ContainsStr(result.ir_dump, "DsReadB32 v80"), + "DS read2 b64 did not lower first dword through shared LDS load IR"); + Check(Common::ContainsStr(result.ir_dump, "DsReadB32 v83"), + "DS read2 b64 did not lower fourth dword through shared LDS load IR"); + Check(SpirvContainsOpcode(result.spirv, 61), "SPIR-V binary does not contain OpLoad"); + Check(SpirvContainsOpcode(result.spirv, 62), "SPIR-V binary does not contain OpStore"); + Check(SpirvContainsOpcode(result.spirv, 65), "SPIR-V binary does not contain OpAccessChain"); + CheckSpirvBinaryValidates(result.spirv); +} + +void TestNewShaderRecompilerDsSubDwordLowering() { + const uint32_t shader[] = { + EncodeDs0(0x1e, 1), EncodeDs1(0, 40, 1), // ds_write_b8 v40, v1 + EncodeDs0(0x1f, 2), EncodeDs1(0, 41, 1), // ds_write_b16 v41, v1 + EncodeDs0(0x39, 3), EncodeDs1(42, 0, 1), // ds_read_i8 v42, v1 + EncodeDs0(0x3a, 4), EncodeDs1(43, 0, 1), // ds_read_u8 v43, v1 + EncodeDs0(0x3b, 6), EncodeDs1(44, 0, 1), // ds_read_i16 v44, v1 + EncodeDs0(0x3c, 8), EncodeDs1(45, 0, 1), // ds_read_u16 v45, v1 + 0xbf810000u, + }; + + ShaderRecompiler::CompileOptions options; + options.stage = ShaderType::Compute; + options.dump_ir = true; + + ShaderRecompiler::CompileResult result; + std::string error; + Check(ShaderRecompiler::TryRecompile(shader, options, &result, &error), error.c_str()); + Check(Common::ContainsStr(result.decoded_dump, "ds_write_b8"), + "new decoder did not decode DS byte write"); + Check(Common::ContainsStr(result.decoded_dump, "ds_write_b16"), + "new decoder did not decode DS short write"); + Check(Common::ContainsStr(result.decoded_dump, "ds_read_i8"), + "new decoder did not decode DS signed byte read"); + Check(Common::ContainsStr(result.decoded_dump, "ds_read_u8"), + "new decoder did not decode DS unsigned byte read"); + Check(Common::ContainsStr(result.decoded_dump, "ds_read_i16"), + "new decoder did not decode DS signed short read"); + Check(Common::ContainsStr(result.decoded_dump, "ds_read_u16"), + "new decoder did not decode DS unsigned short read"); + Check(Common::ContainsStr(result.decoded_dump, "dwords=1 bits=8"), + "DS byte decode did not preserve byte-width metadata"); + Check(Common::ContainsStr(result.decoded_dump, "dwords=1 bits=16"), + "DS short decode did not preserve short-width metadata"); + Check(Common::ContainsStr(result.ir_dump, "DsWriteByte null, v40"), + "DS byte write did not lower to explicit IR"); + Check(Common::ContainsStr(result.ir_dump, "DsWriteShort null, v41"), + "DS short write did not lower to explicit IR"); + Check(Common::ContainsStr(result.ir_dump, "DsReadSbyte v42"), + "DS signed byte read did not lower to explicit IR"); + Check(Common::ContainsStr(result.ir_dump, "DsReadUbyte v43"), + "DS unsigned byte read did not lower to explicit IR"); + Check(Common::ContainsStr(result.ir_dump, "DsReadSshort v44"), + "DS signed short read did not lower to explicit IR"); + Check(Common::ContainsStr(result.ir_dump, "DsReadUshort v45"), + "DS unsigned short read did not lower to explicit IR"); + Check(SpirvContainsOpcode(result.spirv, 61), "SPIR-V binary does not contain OpLoad"); + Check(SpirvContainsOpcode(result.spirv, 62), "SPIR-V binary does not contain OpStore"); + Check(SpirvContainsOpcode(result.spirv, 65), "SPIR-V binary does not contain OpAccessChain"); + Check(SpirvContainsOpcode(result.spirv, 194), + "SPIR-V binary does not contain OpShiftRightLogical"); + Check(SpirvContainsOpcode(result.spirv, 195), + "SPIR-V binary does not contain OpShiftRightArithmetic"); + Check(SpirvContainsOpcode(result.spirv, 196), + "SPIR-V binary does not contain OpShiftLeftLogical"); + Check(SpirvContainsOpcode(result.spirv, 199), "SPIR-V binary does not contain OpBitwiseAnd"); + Check(SpirvContainsOpcode(result.spirv, 200), "SPIR-V binary does not contain OpNot"); + CheckSpirvBinaryValidates(result.spirv); +} + +void TestNewShaderRecompilerDsWideAndAtomicLowering() { + const uint32_t shader[] = { + EncodeDs0(0x4d, 4), EncodeDs1(0, 10, 1), // ds_write_b64 v[10:11], v1 + EncodeDs0(0xde, 8), EncodeDs1(0, 12, 1), // ds_write_b96 v[12:14], v1 + EncodeDs0(0xdf, 12), EncodeDs1(0, 16, 1), // ds_write_b128 v[16:19], v1 + EncodeDs0(0x76, 16), EncodeDs1(20, 0, 1), // ds_read_b64 v[20:21], v1 + EncodeDs0(0xfe, 20), EncodeDs1(24, 0, 1), // ds_read_b96 v[24:26], v1 + EncodeDs0(0xff, 24), EncodeDs1(28, 0, 1), // ds_read_b128 v[28:31], v1 + EncodeDs0(0x07, 28), EncodeDs1(0, 32, 1), // ds_min_u32 v32, v1 + EncodeDs0(0x08, 32), EncodeDs1(0, 33, 1), // ds_max_u32 v33, v1 + EncodeDs0(0x0a, 36), EncodeDs1(0, 34, 1), // ds_or_b32 v34, v1 + 0xbf810000u, + }; + + ShaderRecompiler::CompileOptions options; + options.stage = ShaderType::Compute; + options.dump_ir = true; + + ShaderRecompiler::CompileResult result; + std::string error; + Check(ShaderRecompiler::TryRecompile(shader, options, &result, &error), error.c_str()); + Check(Common::ContainsStr(result.decoded_dump, "ds_write_b64"), + "new decoder did not decode DS b64 write"); + Check(Common::ContainsStr(result.decoded_dump, "ds_write_b96"), + "new decoder did not decode DS b96 write"); + Check(Common::ContainsStr(result.decoded_dump, "ds_write_b128"), + "new decoder did not decode DS b128 write"); + Check(Common::ContainsStr(result.decoded_dump, "ds_read_b64"), + "new decoder did not decode DS b64 read"); + Check(Common::ContainsStr(result.decoded_dump, "ds_read_b96"), + "new decoder did not decode DS b96 read"); + Check(Common::ContainsStr(result.decoded_dump, "ds_read_b128"), + "new decoder did not decode DS b128 read"); + Check(Common::ContainsStr(result.decoded_dump, "ds_min_u32"), + "new decoder did not decode DS min atomic"); + Check(Common::ContainsStr(result.decoded_dump, "ds_max_u32"), + "new decoder did not decode DS max atomic"); + Check(Common::ContainsStr(result.decoded_dump, "ds_or_b32"), + "new decoder did not decode DS or atomic"); + Check(Common::ContainsStr(result.decoded_dump, "dwords=4 bits=32"), + "new decoder did not expose DS wide width metadata"); + Check(Common::ContainsStr(result.ir_dump, "DsWriteB32 null, v10"), + "DS b64 write did not expand to first dword store"); + Check(Common::ContainsStr(result.ir_dump, "DsWriteB32 null, v19"), + "DS b128 write did not expand to last dword store"); + Check(Common::ContainsStr(result.ir_dump, "DsReadB32 v20"), + "DS b64 read did not expand to first dword load"); + Check(Common::ContainsStr(result.ir_dump, "DsReadB32 v31"), + "DS b128 read did not expand to last dword load"); + Check(Common::ContainsStr(result.ir_dump, "AtomicUMinU32 null, v32"), + "DS min atomic did not lower to shared atomic IR"); + Check(Common::ContainsStr(result.ir_dump, "AtomicUMaxU32 null, v33"), + "DS max atomic did not lower to shared atomic IR"); + Check(Common::ContainsStr(result.ir_dump, "AtomicOrU32 null, v34"), + "DS or atomic did not lower to shared atomic IR"); + Check(SpirvContainsOpcode(result.spirv, 61), "SPIR-V binary does not contain OpLoad"); + Check(SpirvContainsOpcode(result.spirv, 62), "SPIR-V binary does not contain OpStore"); + Check(SpirvContainsOpcode(result.spirv, 237), "SPIR-V binary does not contain OpAtomicUMin"); + Check(SpirvContainsOpcode(result.spirv, 239), "SPIR-V binary does not contain OpAtomicUMax"); + Check(SpirvContainsOpcode(result.spirv, 241), "SPIR-V binary does not contain OpAtomicOr"); + CheckSpirvBinaryValidates(result.spirv); +} + +void TestNewShaderRecompilerDsSwizzleLowering() { + const uint32_t shader[] = { + EncodeDs0(0x35, 0x001f), + EncodeDs1(8, 0, 5), // ds_swizzle_b32 v8, v5 + EncodeDs0(0x35, 0x801b), + EncodeDs1(9, 0, 6), // ds_swizzle_b32 v9, v6 + 0xd8d4c480u, + 0x45000045u, // ds_swizzle_b32 rotate mode from boot shader + 0xbf810000u, + }; + + ShaderRecompiler::CompileOptions options; + options.stage = ShaderType::Compute; + options.dump_ir = true; + + ShaderRecompiler::CompileResult result; + std::string error; + Check(ShaderRecompiler::TryRecompile(shader, options, &result, &error), error.c_str()); + Check(Common::ContainsStr(result.decoded_dump, "ds_swizzle_b32"), + "new decoder did not decode DS swizzle"); + Check(Common::ContainsStr(result.decoded_dump, "offset=31"), + "new decoder did not expose DS swizzle control"); + Check(Common::ContainsStr(result.decoded_dump, "offset=32795"), + "new decoder did not expose DS swizzle quad control"); + Check(Common::ContainsStr(result.decoded_dump, "offset=50304"), + "new decoder did not expose DS swizzle rotate control"); + Check(Common::ContainsStr(result.ir_dump, "DsSwizzleB32 v8, v5, 0x0000001f"), + "DS swizzle did not lower to explicit IR"); + Check(Common::ContainsStr(result.ir_dump, "DsSwizzleB32 v9, v6, 0x0000801b"), + "DS quad swizzle did not lower to explicit IR"); + Check(Common::ContainsStr(result.ir_dump, "DsSwizzleB32 v69, v69, 0x0000c480"), + "DS rotate swizzle did not lower to explicit IR"); + Check(SpirvContainsOpcode(result.spirv, 345), + "SPIR-V binary does not contain OpGroupNonUniformShuffle"); + Check(SpirvContainsOpcode(result.spirv, 128), + "SPIR-V binary does not contain OpIAdd for DS rotate swizzle"); + Check(SpirvContainsOpcode(result.spirv, 61), "SPIR-V binary does not contain OpLoad"); + Check(SpirvContainsOpcode(result.spirv, 62), "SPIR-V binary does not contain OpStore"); + Check(SpirvContainsOpcode(result.spirv, 196), + "SPIR-V binary does not contain OpShiftLeftLogical"); + Check(SpirvContainsOpcode(result.spirv, 198), "SPIR-V binary does not contain OpBitwiseOr"); + Check(SpirvContainsOpcode(result.spirv, 199), "SPIR-V binary does not contain OpBitwiseAnd"); + CheckSpirvBinaryValidates(result.spirv); +} + +void TestNewShaderRecompilerDsAddtidLowering() { + const uint32_t shader[] = { + EncodeSMovB32(124, 132), // m0 = 4 + EncodeDs0(0xb0, 8), + EncodeDs1(0, 7, 0), // ds_write_addtid_b32 v7 + EncodeDs0(0xb1, 12), + EncodeDs1(8, 0, 0), // ds_read_addtid_b32 v8 + 0xbf810000u, + }; + + ShaderComputeInputInfo input_info = RegressionComputeInputInfo(); + input_info.thread_ids_num = 1; + + ShaderRecompiler::CompileOptions options; + options.stage = ShaderType::Compute; + options.dump_ir = true; + options.compute_input_info = &input_info; + + ShaderRecompiler::CompileResult result; + std::string error; + Check(ShaderRecompiler::TryRecompile(shader, options, &result, &error), error.c_str()); + Check(Common::ContainsStr(result.decoded_dump, "ds_write_addtid_b32"), + "new decoder did not decode DS write addtid"); + Check(Common::ContainsStr(result.decoded_dump, "ds_read_addtid_b32"), + "new decoder did not decode DS read addtid"); + Check(Common::ContainsStr(result.ir_dump, "DsWriteAddtidB32 null, v7, m0"), + "DS write addtid did not lower to explicit IR"); + Check(Common::ContainsStr(result.ir_dump, "DsReadAddtidB32 v8, m0"), + "DS read addtid did not lower to explicit IR"); + Check( + ProgramHasInput(result.program, ShaderRecompiler::IR::StageInputKind::LocalInvocationIndex), + "DS addtid did not request LocalInvocationIndex input"); + Check(SpirvContainsOpcode(result.spirv, 61), "SPIR-V binary does not contain OpLoad"); + Check(SpirvContainsOpcode(result.spirv, 62), "SPIR-V binary does not contain OpStore"); + Check(SpirvContainsOpcode(result.spirv, 65), "SPIR-V binary does not contain OpAccessChain"); + Check(SpirvContainsOpcode(result.spirv, 128), "SPIR-V binary does not contain OpIAdd"); + Check(SpirvContainsOpcode(result.spirv, 196), + "SPIR-V binary does not contain OpShiftLeftLogical"); + Check(SpirvContainsOpcode(result.spirv, 199), "SPIR-V binary does not contain OpBitwiseAnd"); + CheckSpirvBinaryValidates(result.spirv); +} + +void TestNewShaderRecompilerDsFloatMinMaxLowering() { + const uint32_t shader[] = { + EncodeDs0(0x12, 4), EncodeDs1Ex(0, 9, 7, 1), // ds_min_f32 v7, v9, v1 + EncodeDs0(0x13, 8), EncodeDs1Ex(0, 10, 8, 1), // ds_max_f32 v8, v10, v1 + 0xbf810000u, + }; + + ShaderRecompiler::CompileOptions options; + options.stage = ShaderType::Compute; + options.dump_ir = true; + + ShaderRecompiler::CompileResult result; + std::string error; + Check(ShaderRecompiler::TryRecompile(shader, options, &result, &error), error.c_str()); + Check(Common::ContainsStr(result.decoded_dump, "ds_min_f32"), + "new decoder did not decode DS float min"); + Check(Common::ContainsStr(result.decoded_dump, "ds_max_f32"), + "new decoder did not decode DS float max"); + Check(Common::ContainsStr(result.ir_dump, "DsMinF32 null, v7, v1"), + "DS float min did not lower to explicit IR"); + Check(Common::ContainsStr(result.ir_dump, "DsMaxF32 null, v8, v1"), + "DS float max did not lower to explicit IR"); + Check(Common::ContainsStr(result.ir_dump, "v9"), + "DS float min did not retain DATA1 compare operand"); + Check(Common::ContainsStr(result.ir_dump, "v10"), + "DS float max did not retain DATA1 compare operand"); + Check(SpirvContainsOpcode(result.spirv, 12), "SPIR-V binary does not contain OpExtInst"); + Check(SpirvContainsOpcode(result.spirv, 61), "SPIR-V binary does not contain OpLoad"); + Check(SpirvContainsOpcode(result.spirv, 62), "SPIR-V binary does not contain OpStore"); + Check(SpirvContainsOpcode(result.spirv, 65), "SPIR-V binary does not contain OpAccessChain"); + Check(SpirvContainsOpcode(result.spirv, 124), "SPIR-V binary does not contain OpBitcast"); + CheckSpirvBinaryValidates(result.spirv); +} + +void TestNewShaderRecompilerCfgStraightLine() { + const uint32_t shader[] = { + EncodeSMovB32(0, 129), + 0xbf810000u, + }; + + ShaderRecompiler::CompileOptions options; + options.stage = ShaderType::Compute; + options.dump_ir = true; + + ShaderRecompiler::CompileResult result; + std::string error; + Check(ShaderRecompiler::TryRecompile(shader, options, &result, &error), error.c_str()); + Check(Common::ContainsStr(result.ir_dump, "CFG:"), "CFG dump was not emitted"); + Check(Common::ContainsStr(result.ir_dump, "block_0"), "straight-line CFG block missing"); + Check(Common::ContainsStr(result.ir_dump, "successors=["), "CFG successors were not dumped"); + CheckSpirvBinaryValidates(result.spirv); +} + +void TestNewShaderRecompilerCfgIfElse() { + const uint32_t shader[] = { + EncodeSopc(0x06, 0, 0), // s_cmp_eq_u32 s0, s0 + EncodeSopp(0x05, 2), // s_cbranch_scc1 else + EncodeSMovB32(1, 129), // then + EncodeSopp(0x02, 1), // s_branch merge + EncodeSMovB32(1, 130), // else + 0xbf810000u, + }; + + ShaderRecompiler::CompileOptions options; + options.stage = ShaderType::Compute; + options.dump_ir = true; + + ShaderRecompiler::CompileResult result; + std::string error; + Check(ShaderRecompiler::TryRecompile(shader, options, &result, &error), error.c_str()); + Check(Common::ContainsStr(result.ir_dump, "condition=scc1"), + "if/else branch condition missing"); + Check(SpirvContainsOpcode(result.spirv, 247), "if/else SPIR-V lacks OpSelectionMerge"); + Check(SpirvContainsOpcode(result.spirv, 250), "if/else SPIR-V lacks OpBranchConditional"); + CheckSpirvBinaryValidates(result.spirv); +} + +void TestNewShaderRecompilerCfgTerminalExitMergePS() { + const uint32_t shader[] = { + EncodeSopp(0x04, 2), // s_cbranch_scc0 end + EncodeSMovB32(0, 129), // fallthrough work + 0xbf810000u, + }; + + ShaderRecompiler::CompileOptions options; + options.stage = ShaderType::Pixel; + options.dump_ir = true; + + ShaderRecompiler::CompileResult result; + std::string error; + Check(ShaderRecompiler::TryRecompile(shader, options, &result, &error), error.c_str()); + Check(Common::ContainsStr(result.ir_dump, "mode=structured"), + "terminal PS branch should stay on structured path"); + Check(!Common::ContainsStr(result.ir_dump, "conditional block 0 has no structured merge"), + "known terminal PS branch shape still reports missing merge"); + Check(SpirvContainsOpcode(result.spirv, 247), + "terminal PS branch SPIR-V lacks OpSelectionMerge"); + CheckSpirvBinaryValidates(result.spirv); +} + +void TestNewShaderRecompilerCfgPostEndTargetMergePS() { + const uint32_t shader[] = { + EncodeSopp(0x04, 2), // s_cbranch_scc0 label_000c + EncodeSMovB32(0, 129), // fallthrough work + 0xbf810000u, EncodeSMovB32(1, 129), // branch target after first s_endpgm + 0xbf810000u, + }; + + ShaderRecompiler::CompileOptions options; + options.stage = ShaderType::Pixel; + options.dump_ir = true; + + ShaderRecompiler::CompileResult result; + std::string error; + const bool ok = ShaderRecompiler::TryRecompile(shader, options, &result, &error); + if (!ok) { + std::fprintf(stderr, "PostEndTargetMergePS compile error: %s\n", error.c_str()); + } + Check(ok, "post-end target PS shader failed to compile"); + Check(Common::ContainsStr(result.decoded_dump, "0x0000000c: s_mov_b32"), + "post-end branch target was not decoded"); + Check(Common::ContainsStr(result.ir_dump, "mode=structured"), + "post-end terminal branch should stay on structured path"); + Check(Common::ContainsStr(result.ir_dump, "pc=0x0000000c"), + "post-end branch target did not reach IR blocks"); + Check(!Common::ContainsStr(result.ir_dump, "conditional block 0 has no structured merge"), + "known post-end PS branch shape still reports missing merge"); + Check(SpirvContainsOpcode(result.spirv, 247), + "post-end terminal branch SPIR-V lacks OpSelectionMerge"); + CheckSpirvBinaryValidates(result.spirv); +} + +void TestNewShaderRecompilerCfgLoopBreakContinue() { + const uint32_t shader[] = { + EncodeSMovB32(0, 128), // s0 = 0 + EncodeSopc(0x0a, 0, 129), // s_cmp_lt_u32 s0, 1 + EncodeSopp(0x04, 2), // break when scc == 0 + EncodeSop2(0x00, 0, 0, 129), // s_add_u32 s0, s0, 1 + EncodeSopp(0x02, 0xfffcu), // continue/backedge + 0xbf810000u, + }; + + ShaderRecompiler::CompileOptions options; + options.stage = ShaderType::Compute; + options.dump_ir = true; + + ShaderRecompiler::CompileResult result; + std::string error; + Check(ShaderRecompiler::TryRecompile(shader, options, &result, &error), error.c_str()); + Check(Common::ContainsStr(result.ir_dump, "backedge"), "loop backedge was not detected"); + Check(Common::ContainsStr(result.ir_dump, "loop_header=1"), "loop header was not marked"); + Check(Common::ContainsStr(result.ir_dump, "continue="), + "loop continue block was not identified"); + Check(SpirvContainsOpcode(result.spirv, 246), "loop SPIR-V lacks OpLoopMerge"); + CheckSpirvBinaryValidates(result.spirv); +} + +void TestNewShaderRecompilerCfgLoopHeaderDynamicScalarBufferLoadStructured() { + const uint32_t shader[] = { + EncodeSMovB32(0, 128), // preheader: s0 = 0 + EncodeSmem0(0x08, 8, 4), + 0u, // loop: s_buffer_load_dword s8, s[4:7] + EncodeSop2(0x00, 0, 0, 129), // s_add_u32 s0, s0, 1 + EncodeSopc(0x0a, 0, 130), // s_cmp_lt_u32 s0, 2 + EncodeSopp(0x05, 0xfffbu), // s_cbranch_scc1 loop + 0xbf810000u, + }; + + ShaderRecompiler::CompileOptions options; + options.stage = ShaderType::Compute; + options.dump_ir = true; + + ShaderRecompiler::CompileResult result; + std::string error; + Check(!ShaderRecompiler::TryRecompile(shader, options, &result, &error) && + Common::ContainsStr(error, "unsupported GPU selection"), + "self-modifying scalar-buffer descriptor should fail explicitly"); +} + +void TestNewShaderRecompilerCfgLoopHeaderBufferLoadDispatcher() { + const uint32_t shader[] = { + EncodeSMovB32(0, 128), // preheader: s0 = 0 + EncodeMubuf0(0x0c), + EncodeMubuf1(0, 0, 1), // loop: buffer_load_dword v0 + EncodeSop2(0x00, 0, 0, 129), // s_add_u32 s0, s0, 1 + EncodeSopc(0x0a, 0, 130), // s_cmp_lt_u32 s0, 2 + EncodeSopp(0x05, 0xfffbu), // s_cbranch_scc1 loop + 0xbf810000u, + }; + + ShaderRecompiler::CompileOptions options; + options.stage = ShaderType::Compute; + options.dump_ir = true; + + ShaderRecompiler::CompileResult result; + std::string error; + Check(!ShaderRecompiler::TryRecompile(shader, options, &result, &error) && + Common::ContainsStr(error, "unsupported GPU selection"), + "self-modifying vector-buffer descriptor should fail explicitly"); +} + +void TestNewShaderRecompilerCfgLoopHeaderDsAppendConsumeDispatcher() { + const uint32_t shader[] = { + EncodeSMovB32(124, 129), // m0 = one counter + EncodeDs0(0x3e), // loop: ds_append + EncodeDs1(0, 0, 0), + EncodeDs0(0x3d), // ds_consume + EncodeDs1(1, 0, 0), + EncodeSop2(0x00, 2, 2, 129), // s_add_u32 s2, s2, 1 + EncodeSopc(0x0a, 2, 130), // s_cmp_lt_u32 s2, 2 + EncodeSopp(0x05, 0xfff9u), // s_cbranch_scc1 loop + 0xbf810000u, + }; + + ShaderRecompiler::CompileOptions options; + options.stage = ShaderType::Compute; + options.dump_ir = true; + + ShaderRecompiler::CompileResult result; + std::string error; + Check(ShaderRecompiler::TryRecompile(shader, options, &result, &error), error.c_str()); + Check(Common::ContainsStr(result.ir_dump, "mode=dispatcher"), + "DS append/consume loop header did not select dispatcher fallback"); + Check(SpirvContainsOpcode(result.spirv, 246), "DS dispatcher SPIR-V lacks OpLoopMerge"); + Check(SpirvContainsOpcode(result.spirv, 251), "DS dispatcher SPIR-V lacks OpSwitch"); + CheckSpirvBinaryValidates(result.spirv); +} + +void TestNewShaderRecompilerCfgSharedOuterAndLoopMerge() { + const uint32_t shader[] = { + EncodeSopc(0x06, 0, 0), // s_cmp_eq_u32 s0, s0 + EncodeSopp(0x05, 5), // outer early exit -> end + EncodeSMovB32(2, 128), // s2 = 0 + EncodeSopc(0x0a, 2, 129), // loop: s_cmp_lt_u32 s2, 1 + EncodeSopp(0x04, 2), // loop exit -> same real end block + EncodeSop2(0x00, 2, 2, 129), // s_add_u32 s2, s2, 1 + EncodeSopp(0x02, 0xfffcu), // continue/backedge + 0xbf810000u, + }; + + ShaderRecompiler::CompileOptions options; + options.stage = ShaderType::Compute; + options.dump_ir = true; + + ShaderRecompiler::CompileResult result; + std::string error; + Check(ShaderRecompiler::TryRecompile(shader, options, &result, &error), error.c_str()); + Check(Common::ContainsStr(result.ir_dump, "mode=structured"), + "shared outer/loop merge should stay on structured path"); + Check(!Common::ContainsStr(result.ir_dump, "duplicate structured merge block"), + "shared outer/loop merge was not split before structurization"); + Check(SpirvContainsOpcode(result.spirv, 246), + "shared outer/loop merge SPIR-V lacks OpLoopMerge"); + Check(SpirvContainsOpcode(result.spirv, 247), + "shared outer/loop merge SPIR-V lacks OpSelectionMerge"); + Check(!SpirvContainsOpcode(result.spirv, 251), + "shared outer/loop merge unexpectedly used dispatcher OpSwitch"); + CheckSpirvBinaryValidates(result.spirv); +} + +void TestNewShaderRecompilerCfgLoopSharedContinueSelectionMerges() { + const uint32_t shader[] = { + EncodeSMovB32(0, 128), // s0 = 0 + EncodeSopc(0x0a, 0, 130), // loop: s_cmp_lt_u32 s0, 2 + EncodeSopp(0x04, 9), // loop exit -> end + EncodeSopc(0x06, 1, 1), // s_cmp_eq_u32 s1, s1 + EncodeSopp(0x05, 5), // first early continue -> continue block + EncodeSopc(0x06, 2, 2), // s_cmp_eq_u32 s2, s2 + EncodeSopp(0x05, 3), // second early continue -> continue block + EncodeSopc(0x06, 3, 3), // s_cmp_eq_u32 s3, s3 + EncodeSopp(0x05, 1), // third early continue -> continue block + EncodeSMovB32(4, 129), // fallthrough work + EncodeSop2(0x00, 0, 0, 129), // continue: s_add_u32 s0, s0, 1 + EncodeSopp(0x02, 0xfff5u), // backedge -> loop header + 0xbf810000u, + }; + + ShaderRecompiler::CompileOptions options; + options.stage = ShaderType::Compute; + options.dump_ir = true; + + ShaderRecompiler::CompileResult result; + std::string error; + Check(ShaderRecompiler::TryRecompile(shader, options, &result, &error), error.c_str()); + Check(Common::ContainsStr(result.ir_dump, "mode=structured"), + "shared loop continue selections should stay on structured path"); + Check(!Common::ContainsStr(result.ir_dump, "duplicate structured merge block"), + "shared loop continue selections were not split before structurization"); + Check(SpirvContainsOpcode(result.spirv, 246), + "shared loop continue selections SPIR-V lacks OpLoopMerge"); + Check(SpirvContainsOpcode(result.spirv, 247), + "shared loop continue selections SPIR-V lacks OpSelectionMerge"); + Check(!SpirvContainsOpcode(result.spirv, 251), + "shared loop continue selections unexpectedly used dispatcher OpSwitch"); + CheckSpirvBinaryValidates(result.spirv); +} + +void TestNewShaderRecompilerCfgDuplicateMergeStructuredSplit() { + const uint32_t shader[] = { + EncodeSopc(0x06, 0, 0), // s_cmp_eq_u32 s0, s0 + EncodeSopp(0x05, 2), // first early exit -> end + EncodeSopp(0x05, 1), // second early exit -> same end merge + EncodeSMovB32(0, 129), // fallthrough work + 0xbf810000u, + }; + + ShaderRecompiler::CompileOptions options; + options.stage = ShaderType::Compute; + options.dump_ir = true; + + ShaderRecompiler::CompileResult result; + std::string error; + Check(ShaderRecompiler::TryRecompile(shader, options, &result, &error), error.c_str()); + Check(Common::ContainsStr(result.ir_dump, "mode=structured"), + "duplicate merge CFG did not stay on structured path"); + Check(!Common::ContainsStr(result.ir_dump, "duplicate structured merge block"), + "duplicate merge CFG was not split before structurization"); + Check(SpirvContainsOpcode(result.spirv, 247), "duplicate merge SPIR-V lacks OpSelectionMerge"); + Check(!SpirvContainsOpcode(result.spirv, 251), + "duplicate merge CFG unexpectedly used dispatcher OpSwitch"); + CheckSpirvBinaryValidates(result.spirv); +} + +void TestNewShaderRecompilerCfgIrreducibleDispatcher() { + const uint32_t shader[] = { + EncodeSopp(0x05, 2), // entry -> B, fallthrough A + EncodeSopp(0x02, 0), // A -> C + EncodeSopp(0x05, 0xfffeu), // C -> A, fallthrough B + EncodeSopp(0x02, 0xfffeu), // B -> C + 0xbf810000u, + }; + + ShaderRecompiler::CompileOptions options; + options.stage = ShaderType::Compute; + options.dump_ir = true; + + ShaderRecompiler::CompileResult result; + std::string error; + Check(ShaderRecompiler::TryRecompile(shader, options, &result, &error), error.c_str()); + Check(Common::ContainsStr(result.ir_dump, "irreducible CFG"), + "irreducible CFG reason was not retained"); + Check(Common::ContainsStr(result.ir_dump, "mode=dispatcher"), + "irreducible CFG did not select dispatcher fallback"); + Check(SpirvContainsOpcode(result.spirv, 246), "dispatcher SPIR-V lacks OpLoopMerge"); + Check(SpirvContainsOpcode(result.spirv, 251), "dispatcher SPIR-V lacks OpSwitch"); + CheckSpirvBinaryValidates(result.spirv); +} + +void TestNewShaderRecompilerExecMaskHelpers() { + using namespace ShaderRecompiler::Exec; + + State wave32; + wave32.wave_size = 32; + WriteExec(wave32, FullMask(64)); + Check(ReadExec(wave32).low == 0xffffffffu && ReadExec(wave32).high == 0u, + "wave32 EXEC high half was not masked off"); + AndExec(wave32, {0x0fu, 0xffffffffu}); + Check(ReadExec(wave32).low == 0x0fu && ReadExec(wave32).high == 0u, + "wave32 AndExec did not preserve wave size"); + InvertExec(wave32); + Check(ReadExec(wave32).low == 0xfffffff0u && ReadExec(wave32).high == 0u, + "wave32 InvertExec did not preserve wave size"); + WriteExec(wave32, {0u, 0xffffffffu}); + Check(IsExecZero(wave32), "wave32 IsExecZero used inactive high lanes"); + + State wave64; + wave64.wave_size = 64; + WriteExec(wave64, FullMask(64)); + Check(ReadExec(wave64).low == 0xffffffffu && ReadExec(wave64).high == 0xffffffffu, + "wave64 EXEC full mask was wrong"); + XorExec(wave64, {0xffffffffu, 0u}); + Check(ReadExec(wave64).low == 0u && ReadExec(wave64).high == 0xffffffffu, + "wave64 XorExec did not update both halves correctly"); + wave64.vcc = {0u, 1u}; + Check(!IsVccZero(wave64), "wave64 VCC high half was ignored"); +} + +void TestComputeShaderInputWaveSize() { + const auto decode_wave_size = [](uint32_t rsrc1) { + const bool wave32 = (((rsrc1 >> Pm4::COMPUTE_PGM_RSRC1_W32_EN_SHIFT) & + Pm4::COMPUTE_PGM_RSRC1_W32_EN_MASK) != 0u); + return wave32 ? 32u : 64u; + }; + + constexpr uint32_t observed_wave32_rsrc1_a = 0x402c0146u; + constexpr uint32_t observed_wave32_rsrc1_b = 0x402c00c1u; + Check(decode_wave_size(observed_wave32_rsrc1_a) == 32u, + "COMPUTE_PGM_RSRC1 W32_EN bit did not match observed PS5 value A"); + Check(decode_wave_size(observed_wave32_rsrc1_b) == 32u, + "COMPUTE_PGM_RSRC1 W32_EN bit did not match observed PS5 value B"); + + constexpr uint32_t w32_en_bit = 1u << Pm4::COMPUTE_PGM_RSRC1_W32_EN_SHIFT; + Check(decode_wave_size(observed_wave32_rsrc1_a & ~w32_en_bit) == 64u, + "cleared COMPUTE_PGM_RSRC1 W32_EN bit did not decode as wave64"); +} + +void TestNewShaderRecompilerWave32MasksExecHighStores() { + const uint32_t shader[] = { + EncodeSop1(0x04, 126, 193), // s_mov_b64 exec, -1 + EncodeSopp(0x01), + }; + + ShaderRecompiler::CompileOptions options; + options.stage = ShaderType::Compute; + options.dump_ir = true; + options.wave_size = 32; + + ShaderRecompiler::CompileResult result; + std::string error; + Check(ShaderRecompiler::TryRecompile(shader, options, &result, &error), error.c_str()); + Check(Common::ContainsStr(result.decoded_dump, "s_mov_b64 exec_lo, -1"), + "wave32 EXEC mask regression did not decode s_mov_b64 exec, -1"); + CheckSpirvBinaryValidates(result.spirv); + + const auto source = DisassembleSpirvBinary(result.spirv); + Check(CountSourceOccurrences(source, "OpStore %exec_hi %uint_0") >= 2u, + "wave32 EXEC high half was not clamped to zero in SPIR-V stores"); + Check(!Common::ContainsStr(source, "OpStore %exec_hi %uint_4294967295"), + "wave32 EXEC high half was stored as a full 64-lane mask"); +} + +void TestNewShaderRecompilerWave32VccHighScalarStores() { + const uint32_t shader[] = { + EncodeSmem0(0x09, 106, 4), + 0u, // s_buffer_load_dwordx2 vcc_lo + EncodeSopp(0x01), + }; + + ShaderRecompiler::CompileOptions options; + options.stage = ShaderType::Compute; + options.dump_ir = true; + options.wave_size = 32; + + ShaderRecompiler::CompileResult result; + std::string error; + Check(ShaderRecompiler::TryRecompile(shader, options, &result, &error), error.c_str()); + Check(Common::ContainsStr(result.ir_dump, "SBufferLoadDword vcc_hi"), + "wave32 VCC scalar regression did not lower the high VCC dword load"); + CheckSpirvBinaryValidates(result.spirv); + + const auto source = DisassembleSpirvBinary(result.spirv); + Check(CountSourceOccurrences(source, "OpStore %vcc_hi") >= 2u, + "wave32 VCC high scalar load did not store to vcc_hi"); + Check(CountSourceOccurrences(source, "OpStore %vcc_hi %uint_0") == 1u, + "wave32 VCC high scalar load was clamped to zero instead of preserving data"); +} + +void TestNewShaderRecompilerCompareMaskIsFullWaveBallot() { + const uint32_t shader[] = { + EncodeVopc(0xc1, 0 + 256, 1), // v_cmp_lt_u32 vcc, v0, v1 + EncodeSop2(0x0f, 2, 126, 106), // s_and_b64 s[2:3], exec, vcc + EncodeSop1(0x04, 126, 2), // s_mov_b64 exec, s[2:3] + EncodeSopp(0x01), + }; + + ShaderRecompiler::CompileOptions options; + options.stage = ShaderType::Compute; + options.dump_ir = true; + options.wave_size = 32; + + ShaderRecompiler::CompileResult result; + std::string error; + Check(ShaderRecompiler::TryRecompile(shader, options, &result, &error), error.c_str()); + Check(Common::ContainsStr(result.decoded_dump, "v_cmp_lt_u32"), + "compare-mask ballot regression did not decode V_CMP_LT_U32"); + Check(Common::ContainsStr(result.ir_dump, "CompareLtU32 vcc_lo"), + "compare-mask ballot regression did not lower compare to VCC"); + Check(Common::ContainsStr(result.ir_dump, "BitwiseAndU64 s2, exec_lo, vcc_lo"), + "compare-mask ballot regression did not consume VCC through scalar mask ALU"); + CheckSpirvBinaryValidates(result.spirv); + + const auto source = DisassembleSpirvBinary(result.spirv); + Check(Common::ContainsStr(source, "OpGroupNonUniformBallot"), + "compare-mask result was not materialized with a subgroup ballot"); + Check(Common::ContainsStr(source, "OpStore %vcc_lo"), + "compare-mask ballot regression did not store VCC low mask"); + Check(Common::ContainsStr(source, "OpStore %vcc_hi %uint_0"), + "wave32 compare-mask VCC high half was not cleared"); +} + +void TestNewShaderRecompilerBufferLoadsGuardedByExec() { + const uint32_t shader[] = { + EncodeMubuf0(0x0c), + EncodeMubuf1(0, 0, 1), // buffer_load_dword v0 + EncodeSopp(0x01), + }; + + ShaderRecompiler::CompileOptions options; + options.stage = ShaderType::Compute; + options.dump_ir = true; + + ShaderRecompiler::CompileResult result; + std::string error; + Check(ShaderRecompiler::TryRecompile(shader, options, &result, &error), error.c_str()); + Check(Common::ContainsStr(result.decoded_dump, "buffer_load_dword"), + "buffer load guard regression did not decode MUBUF load"); + CheckSpirvBinaryValidates(result.spirv); + + const auto source = DisassembleSpirvBinary(result.spirv); + const auto exec_branch = Common::FindIndex(source, std::string("OpBranchConditional"), 0); + const auto array_length = Common::FindIndex(source, std::string("OpArrayLength"), 0); + const auto bounds_branch = + Common::FindIndex(source, std::string("OpBranchConditional"), array_length); + const auto element_access = + Common::FindIndex(source, std::string("OpAccessChain %_ptr_StorageBuffer_uint"), 0); + Check(exec_branch != Common::FIND_INVALID_INDEX, "buffer load SPIR-V lacks EXEC guard branch"); + Check(array_length != Common::FIND_INVALID_INDEX, + "buffer load SPIR-V lacks storage buffer array-length bounds check"); + Check(bounds_branch != Common::FIND_INVALID_INDEX, + "buffer load SPIR-V lacks storage buffer bounds branch"); + Check(element_access != Common::FIND_INVALID_INDEX, + "buffer load SPIR-V lacks storage element access"); + Check(exec_branch < array_length, "buffer load bounds check was emitted outside EXEC guard"); + Check(bounds_branch < element_access, + "buffer load storage element pointer was formed before bounds guard"); +} + +void TestNewShaderRecompilerBufferAtomicsGuardedByBounds() { + const uint32_t shader[] = { + EncodeMubuf0(0x32, 0, true, true), + EncodeMubuf1(0, 0, 1), // buffer_atomic_add + EncodeSopp(0x01), + }; + + ShaderRecompiler::CompileOptions options; + options.stage = ShaderType::Compute; + options.dump_ir = true; + + ShaderRecompiler::CompileResult result; + std::string error; + Check(ShaderRecompiler::TryRecompile(shader, options, &result, &error), error.c_str()); + Check(Common::ContainsStr(result.decoded_dump, "buffer_atomic_add"), + "buffer atomic bounds regression did not decode MUBUF atomic"); + CheckSpirvBinaryValidates(result.spirv); + + const auto source = DisassembleSpirvBinary(result.spirv); + const auto array_length = Common::FindIndex(source, std::string("OpArrayLength"), 0); + const auto bounds_branch = + Common::FindIndex(source, std::string("OpBranchConditional"), array_length); + const auto atomic = Common::FindIndex(source, std::string("OpAtomicIAdd"), 0); + const auto memory_barrier = Common::FindIndex(source, std::string("OpMemoryBarrier"), atomic); + Check(array_length != Common::FIND_INVALID_INDEX, + "buffer atomic SPIR-V lacks storage buffer array-length bounds check"); + Check(bounds_branch != Common::FIND_INVALID_INDEX, + "buffer atomic SPIR-V lacks storage buffer bounds branch"); + Check(atomic != Common::FIND_INVALID_INDEX, "buffer atomic SPIR-V lacks atomic operation"); + Check(memory_barrier != Common::FIND_INVALID_INDEX, + "buffer atomic SPIR-V lacks memory barrier after atomic operation"); + Check(bounds_branch < atomic, "buffer atomic was emitted before bounds guard"); + Check(atomic < memory_barrier, "buffer atomic memory barrier was emitted before atomic"); +} + +void TestNewShaderRecompilerBranchConditionForms() { + struct Case { + uint32_t opcode; + const char* condition; + }; + const Case cases[] = { + {0x04, "condition=scc0"}, + {0x08, "condition=execz"}, + {0x07, "condition=vccnz"}, + }; + + for (const auto& c: cases) { + const uint32_t shader[] = { + EncodeSopp(c.opcode, 1), + EncodeSMovB32(0, 129), + 0xbf810000u, + }; + + ShaderRecompiler::CompileOptions options; + options.stage = ShaderType::Compute; + options.dump_ir = true; + + ShaderRecompiler::CompileResult result; + std::string error; + Check(ShaderRecompiler::TryRecompile(shader, options, &result, &error), error.c_str()); + Check(Common::ContainsStr(result.ir_dump, c.condition), + "branch condition was not preserved"); + Check(SpirvContainsOpcode(result.spirv, 250), + "branch condition SPIR-V lacks OpBranchConditional"); + CheckSpirvBinaryValidates(result.spirv); + if (c.opcode == 0x04) { + const auto source = DisassembleSpirvBinary(result.spirv); + Check(!Common::ContainsStr(source, "OpGroupNonUniformBallot"), + "structured SCC branch retained a host-subgroup reduction"); + } + } +} + +void TestNewShaderRecompilerSetpcBranch() { + const uint32_t shader[] = { + EncodeSop1(0x1f, 4, 0), // s_getpc_b64 s[4:5] + EncodeSop2(0x00, 4, 4, 140), // s_add_u32 s4, s4, 12 + EncodeSop1(0x20, 0, 4), // s_setpc_b64 s[4:5] + EncodeSMovB32(0, 129), 0xbf810000u, + }; + + ShaderRecompiler::CompileOptions options; + options.stage = ShaderType::Compute; + options.dump_ir = true; + + ShaderRecompiler::CompileResult result; + std::string error; + Check(ShaderRecompiler::TryRecompile(shader, options, &result, &error), error.c_str()); + Check(Common::ContainsStr(result.decoded_dump, "s_setpc_b64"), "S_SETPC_B64 was not decoded"); + Check(Common::ContainsStr(result.ir_dump, "successors=["), + "S_SETPC_B64 did not participate in CFG"); + CheckSpirvBinaryValidates(result.spirv); +} + +void TestNewShaderRecompilerSetpcJumpTable() { + const uint32_t shader[] = { + EncodeSop2(0x07, 0, 0, 129), // s_min_u32 s0, s0, 1 + EncodeSop2(0x1e, 0, 0, 131), // s_lshl_b32 s0, s0, 3 + EncodeSop1(0x1f, 8, 0), // s_getpc_b64 s[8:9] + EncodeSop2(0x00, 8, 8, 255), // s_add_u32 s8, s8, literal + 0x00000038u, + EncodeSop2(0x04, 9, 9, 128), // s_addc_u32 s9, s9, 0 + EncodeSmem0(0x01, 4, 4), + 0u, // s_load_dwordx2 s[4:5], s[8:9], s0 + EncodeSopp(0x0c, 0), // s_waitcnt 0 + EncodeSop1(0x1f, 10, 0), // s_getpc_b64 s[10:11] + EncodeSop2(0x00, 10, 10, 4), // s_add_u32 s10, s10, s4 + EncodeSop2(0x04, 11, 11, 5), // s_addc_u32 s11, s11, s5 + EncodeSop1(0x20, 0, 10), // s_setpc_b64 s[10:11] + EncodeSMovB32(1, 129), // case 0 + EncodeSopp(0x02, 1), + EncodeSMovB32(2, 129), // case 1 + EncodeSopp(0x01, 0), + 0x0000000cu, + 0x00000000u, // table: target case 0 relative to pc 0x28 + 0x00000014u, + 0x00000000u, // table: target case 1 relative to pc 0x28 + }; + + ShaderRecompiler::CompileOptions options; + options.stage = ShaderType::Compute; + options.dump_ir = true; + + ShaderRecompiler::CompileResult result; + std::string error; + Check(ShaderRecompiler::TryRecompile(shader, options, &result, &error), error.c_str()); + Check(Common::ContainsStr(result.ir_dump, "mode=dispatcher"), + "S_SETPC_B64 jump table did not select dispatcher fallback"); + Check(Common::ContainsStr(result.ir_dump, "indirect_targets=2"), + "S_SETPC_B64 jump table did not retain both targets"); + Check(Common::ContainsStr(result.ir_dump, "selector_values=2"), + "S_SETPC_B64 jump table did not retain selector mapping"); + Check(!Common::ContainsStr(result.ir_dump, "SLoadDword"), + "S_SETPC_B64 jump table load was reflected as a raw scalar buffer load"); + Check(SpirvContainsOpcode(result.spirv, 251), "dispatcher SPIR-V lacks OpSwitch"); + CheckSpirvBinaryValidates(result.spirv); +} + +void TestNewShaderRecompilerExpVertexOutputs() { + const uint32_t shader[] = { + EncodeExp0(0x0c, 0xf), EncodeExp1(0, 1, 2, 3), // POS0 + EncodeExp0(0x20, 0xf), EncodeExp1(4, 5, 6, 7), // PARAM0 + EncodeExp0(0x14, 0x1), EncodeExp1(8, 0, 0, 0), // PRIM + 0xbf810000u, + }; + + ShaderRecompiler::CompileOptions options; + options.stage = ShaderType::Vertex; + options.dump_ir = true; + + ShaderRecompiler::CompileResult result; + std::string error; + Check(ShaderRecompiler::TryRecompile(shader, options, &result, &error), error.c_str()); + Check(Common::ContainsStr(result.decoded_dump, "target=0x0c"), "POS export was not decoded"); + Check(Common::ContainsStr(result.decoded_dump, "target=0x20"), "PARAM export was not decoded"); + Check(Common::ContainsStr(result.decoded_dump, "target=0x14"), "PRIM export was not decoded"); + Check(Common::ContainsStr(result.ir_dump, "position"), "POS export did not reach IR"); + Check(Common::ContainsStr(result.ir_dump, "parameter"), "PARAM export did not reach IR"); + Check(Common::ContainsStr(result.ir_dump, "primitive"), "PRIM export did not reach IR"); + Check(SpirvContainsOpcode(result.spirv, 62), "vertex export SPIR-V lacks OpStore"); + CheckSpirvBinaryValidates(result.spirv); +} + +void TestNewShaderRecompilerVertexSystemVgprs() { + using StageInputKind = ShaderRecompiler::IR::StageInputKind; + + const uint32_t shader[] = { + EncodeVop1(0x01, 0, 5 + 256), // v_mov_b32 v0, v5 + EncodeVop1(0x01, 1, 8 + 256), // v_mov_b32 v1, v8 + EncodeExp0(0x0c, 0xf), + EncodeExp1(0, 1, 2, 3), // POS0 + EncodeExp0(0x20, 0x3), + EncodeExp1(0, 1, 0, 0), // PARAM0 + 0xbf810000u, + }; + + ShaderRecompiler::CompileOptions options; + options.stage = ShaderType::Vertex; + options.dump_ir = true; + + ShaderRecompiler::CompileResult result; + std::string error; + Check(ShaderRecompiler::TryRecompile(shader, options, &result, &error), error.c_str()); + Check(ProgramHasInput(result.program, StageInputKind::VertexIndex), + "vertex shader missing VertexIndex input"); + Check(ProgramHasInput(result.program, StageInputKind::InstanceIndex), + "vertex shader missing InstanceIndex input"); + Check(Common::ContainsStr(result.ir_dump, "MoveU32 v0, v5"), + "vertex shader did not keep v5 as a guest VGPR source"); + Check(Common::ContainsStr(result.ir_dump, "MoveU32 v1, v8"), + "vertex shader did not keep v8 as a guest VGPR source"); + CheckSpirvBinaryValidates(result.spirv); + + const auto source = DisassembleSpirvBinary(result.spirv); + Check(CountSourceOccurrences(source, "OpLoad %int %gl_VertexIndex") == 1u, + "vertex SPIR-V does not load gl_VertexIndex"); + Check(CountSourceOccurrences(source, "OpLoad %int %gl_InstanceIndex") == 1u, + "vertex SPIR-V does not load gl_InstanceIndex"); + Check(CountSourceOccurrences(source, "OpStore %v5") >= 2u, + "vertex SPIR-V does not seed guest v5 from gl_VertexIndex"); + Check(CountSourceOccurrences(source, "OpStore %v8") >= 2u, + "vertex SPIR-V does not seed guest v8 from gl_InstanceIndex"); +} + +void TestNewShaderRecompilerVertexExportUsesLaneExecMask() { + const uint32_t shader[] = { + EncodeSop1(0x04, 126, 129), // s_mov_b64 exec, 1 + EncodeExp0(0x0c, 0xf), + EncodeExp1(0, 1, 2, 3), // POS0 + 0xbf810000u, + }; + + ShaderRecompiler::CompileOptions options; + options.stage = ShaderType::Vertex; + options.dump_ir = true; + + ShaderRecompiler::CompileResult result; + std::string error; + Check(ShaderRecompiler::TryRecompile(shader, options, &result, &error), error.c_str()); + CheckSpirvBinaryValidates(result.spirv); + + const auto source = DisassembleSpirvBinary(result.spirv); + Check(Common::ContainsStr(source, "gl_SubgroupInvocationID"), + "vertex export EXEC guard did not test the current lane"); + Check(Common::ContainsStr(source, "OpBranchConditional"), + "vertex export EXEC guard did not branch on the lane-active mask"); + + options.lane_mask_mode = ShaderLaneMaskMode::PerInvocation; + ShaderRecompiler::CompileResult local_result; + Check(ShaderRecompiler::TryRecompile(shader, options, &local_result, &error), error.c_str()); + Check(local_result.program.lane_mask_mode == ShaderLaneMaskMode::PerInvocation, + "per-invocation mask mode was not preserved in the immutable program"); + CheckSpirvBinaryValidates(local_result.spirv); + const auto local_source = DisassembleSpirvBinary(local_result.spirv); + Check(!Common::ContainsStr(local_source, "OpLoad %uint %gl_SubgroupInvocationID"), + "per-invocation EXEC still depends on the native subgroup lane"); + Check(Common::ContainsStr(local_source, "OpBranchConditional"), + "per-invocation vertex export lost its EXEC guard"); +} + +void TestNewShaderRecompilerPerInvocationMasks() { + const uint32_t local_shader[] = { + EncodeVopc(0xc1, 0 + 256, 1), // v_cmp_lt_u32 vcc, v0, v1 + EncodeSop2(0x0f, 2, 126, 106), // s_and_b64 s[2:3], exec, vcc + EncodeSop1(0x24, 4, 126), // s_and_saveexec_b64 s[4:5], exec + EncodeSop2(0x25, 126, 132, 128), // s_bfm_b64 exec, 4, 0 + EncodeSop1(0x08, 126, 126), // s_not_b64 exec, exec + EncodeSop1(0x08, 126, 126), // s_not_b64 exec, exec + EncodeExp0(0x0c, 0xf), + EncodeExp1(0, 1, 2, 3), // POS0 + EncodeSopp(0x01), + }; + + ShaderRecompiler::CompileOptions options; + options.stage = ShaderType::Vertex; + options.lane_mask_mode = ShaderLaneMaskMode::PerInvocation; + options.dump_ir = true; + + ShaderRecompiler::CompileResult result; + std::string error; + Check(ShaderRecompiler::TryRecompile(local_shader, options, &result, &error), error.c_str()); + CheckSpirvBinaryValidates(result.spirv); + const auto source = DisassembleSpirvBinary(result.spirv); + Check(!Common::ContainsStr(source, "OpGroupNonUniformBallot"), + "per-invocation VCC producer still materialized a shared subgroup mask"); + Check(!Common::ContainsStr(source, "OpLoad %uint %gl_SubgroupInvocationID"), + "per-invocation BFM EXEC prefix still selected native subgroup lanes"); + Check(Common::ContainsStr(source, "OpLogicalAnd") && + Common::ContainsStr(source, "OpLogicalNot"), + "per-invocation wide mask ALU did not use Boolean operations"); + Check(Common::ContainsStr(source, "OpStore %vcc_lo") && + Common::ContainsStr(source, "OpStore %vcc_hi %uint_0"), + "per-invocation comparison did not store a local Boolean VCC pair"); + Check(Common::ContainsStr(result.ir_dump, "SaveexecB64 s4, exec_lo"), + "per-invocation SAVEEXEC regression did not reach IR"); + + const uint32_t wqm_shader[] = { + EncodeSop1(0x0a, 2, 126), // s_wqm_b64 s[2:3], exec + EncodeExp0(0x0c, 0xf), + EncodeExp1(0, 1, 2, 3), // POS0 + EncodeSopp(0x01), + }; + Check(ShaderRecompiler::TryRecompile(wqm_shader, options, &result, &error), error.c_str()); + CheckSpirvBinaryValidates(result.spirv); + const auto wqm_source = DisassembleSpirvBinary(result.spirv); + Check(Common::ContainsStr(wqm_source, "OpCapability GroupNonUniformBallot") && + Common::ContainsStr(wqm_source, "OpGroupNonUniformBallot"), + "per-invocation scalar WQM omitted its subgroup ballot capability"); + + const uint32_t cross_lane_shader[] = { + EncodeSop2(0x25, 126, 132, 128), // s_bfm_b64 exec, 4, 0 + EncodeVop1(0x01, 0, 250), + EncodeVop1Dpp(1), // v_mov_b32 v0, v1 dpp + EncodeExp0(0x0c, 0xf), + EncodeExp1(0, 1, 2, 3), // POS0 + EncodeSopp(0x01), + }; + Check(ShaderRecompiler::TryRecompile(cross_lane_shader, options, &result, &error), + error.c_str()); + CheckSpirvBinaryValidates(result.spirv); + Check(Common::ContainsStr(DisassembleSpirvBinary(result.spirv), "OpGroupNonUniformBallot"), + "per-invocation cross-lane EXEC was not reconstructed as a subgroup ballot"); +} + +void TestNewShaderRecompilerExpPixelOutputs() { + const uint32_t shader[] = { + EncodeExp0(0x00, 0xf), + EncodeExp1(0, 1, 2, 3), // MRT0 + EncodeExp0(0x08, 0x1), + EncodeExp1(4, 0, 0, 0), // MRTZ depth + EncodeExp0(0x09, 0x0), + EncodeExp1(0, 0, 0, 0), // NULL + EncodeExp0(0x00, 0xf, true, true), + EncodeExp1(5, 6, 0, 0), // compressed MRT0 + 0xbf810000u, + }; + + ShaderRecompiler::CompileOptions options; + options.stage = ShaderType::Pixel; + options.dump_ir = true; + + ShaderRecompiler::CompileResult result; + std::string error; + Check(ShaderRecompiler::TryRecompile(shader, options, &result, &error), error.c_str()); + Check(Common::ContainsStr(result.ir_dump, "mrt"), "MRT export did not reach IR"); + Check(Common::ContainsStr(result.ir_dump, "mrtz"), "MRTZ export did not reach IR"); + Check(Common::ContainsStr(result.ir_dump, "null"), "NULL export did not reach IR"); + Check(Common::ContainsStr(result.ir_dump, "compr=1"), "compressed MRT export did not reach IR"); + Check(SpirvContainsOpcode(result.spirv, 62), "pixel export SPIR-V lacks OpStore"); + Check(SpirvContainsExtInst(result.spirv, 62), + "compressed pixel export SPIR-V lacks GLSL.std.450 UnpackHalf2x16"); + Check(SpirvContainsOpcode(result.spirv, 81), + "compressed pixel export SPIR-V lacks OpCompositeExtract"); + CheckSpirvBinaryValidates(result.spirv); +} + +void TestNewShaderRecompilerEarlyZDisabledWhenPixelKillEnabled() { + constexpr uint32_t ExecutionModeEarlyFragmentTests = 9; + + const uint32_t shader[] = { + EncodeExp0(0x00, 0xf, true, false, true), + EncodeExp1(0, 1, 2, 3), + 0xbf810000u, + }; + + ShaderPixelInputInfo ps_info; + ps_info.ps_early_z = true; + ps_info.ps_pixel_kill_enable = true; + + ShaderRecompiler::CompileOptions options; + options.stage = ShaderType::Pixel; + options.pixel_input_info = &ps_info; + + ShaderRecompiler::CompileResult result; + std::string error; + Check(ShaderRecompiler::TryRecompile(shader, options, &result, &error), error.c_str()); + Check(SpirvContainsOpcode(result.spirv, 252), + "pixel valid-mask export should still lower to OpKill"); + Check(!SpirvContainsExecutionMode(result.spirv, ExecutionModeEarlyFragmentTests), + "pixel shaders that may kill fragments must not request EarlyFragmentTests"); + CheckSpirvBinaryValidates(result.spirv); +} + +void TestNewShaderRecompilerNativeBindingPlan() { + using BindingKind = ShaderRecompiler::IR::DescriptorBindingKind; + + const uint32_t shader[] = { + EncodeSmem0(0x08, 0, 4), + 0u, // s_buffer_load_dword s0 + EncodeMubuf0(0x1c, 8), + EncodeMubuf1(0, 1, 1), // buffer_store_dword + EncodeMimg0(0x20, 0xf), + EncodeMimg1(8, 2, 3, 1), // image_sample + EncodeMimg0(0x08, 0xf), + EncodeMimg1(12, 5, 0, 1), // image_store + EncodeMimg0(0x11, 0x1), + EncodeMimg1(16, 6, 0, 1), // image_atomic_add + 0xbf810000u, + }; + + auto user_data = ImageTestUserData(); + ShaderRecompiler::CompileOptions options; + options.stage = ShaderType::Compute; + options.dump_ir = true; + options.user_data = user_data.data(); + + ShaderRecompiler::CompileResult result; + std::string error; + const auto compiled = ShaderRecompiler::TryRecompile(shader, options, &result, &error); + Check(compiled, error.c_str()); + const auto* buffers = + ShaderRecompiler::IR::FindBinding(result.program.bindings, BindingKind::Buffers); + const auto* sampled = + ShaderRecompiler::IR::FindBinding(result.program.bindings, BindingKind::Sampled2D); + const auto* storage = + ShaderRecompiler::IR::FindBinding(result.program.bindings, BindingKind::Storage2D); + const auto* uint_storage = + ShaderRecompiler::IR::FindBinding(result.program.bindings, BindingKind::StorageUint2D); + const auto* samplers = + ShaderRecompiler::IR::FindBinding(result.program.bindings, BindingKind::Samplers); + Check(buffers != nullptr && buffers->resources.size() == 2, + "native binding plan did not allocate scalar/vector buffers"); + Check(sampled != nullptr && sampled->resources.size() == 1, + "native binding plan did not allocate the sampled image"); + Check(storage != nullptr && storage->resources.size() == 1, + "native binding plan did not allocate the float storage image"); + Check(uint_storage != nullptr && uint_storage->resources.size() == 1, + "native binding plan did not allocate the uint storage image"); + Check(samplers != nullptr && samplers->resources.size() == 1, + "native binding plan did not allocate the sampler"); + Check(SpirvContainsOpcode(result.spirv, 86), + "SPIR-V binary does not combine separate image/sampler descriptors"); + Check(SpirvHasDecorationValue(result.spirv, 33u, buffers->binding), + "SPIR-V lacks storage-buffer Binding decoration"); + Check(SpirvHasDecorationValue(result.spirv, 33u, sampled->binding), + "SPIR-V lacks sampled-image Binding decoration"); + Check(SpirvHasDecorationValue(result.spirv, 33u, samplers->binding), + "SPIR-V lacks sampler Binding decoration"); + Check(SpirvHasDecorationValue(result.spirv, 34u, result.program.bindings.descriptor_set), + "SPIR-V lacks DescriptorSet decoration"); + CheckSpirvBinaryValidates(result.spirv); + + auto malformed = result.program; + auto malformed_buffers = + std::find_if(malformed.bindings.descriptors.begin(), malformed.bindings.descriptors.end(), + [](const auto& binding) { return binding.kind == BindingKind::Buffers; }); + Check(malformed_buffers != malformed.bindings.descriptors.end(), + "native validation fixture lacks a buffer group"); + malformed_buffers->resources.clear(); + std::vector rejected_spirv = {0xdeadbeefu}; + const auto rejected_before = rejected_spirv; + std::string rejected_error; + Check(!ShaderRecompiler::Spirv::EmitProgram(malformed, result.resources, nullptr, nullptr, + nullptr, &rejected_spirv, &rejected_error) && + rejected_spirv == rejected_before && + rejected_error.find("topology") != std::string::npos, + "malformed native binding plan was not rejected transactionally"); + const auto FindBufferInstruction = [](auto* program) { + for (auto& block: program->blocks) { + const auto found = std::find_if( + block.instructions.begin(), block.instructions.end(), [](const auto& inst) { + return inst.memory.kind == ShaderRecompiler::IR::ResourceKind::Buffer; + }); + if (found != block.instructions.end()) { + return &*found; + } + } + return static_cast(nullptr); + }; + + auto malformed_inst = result.program; + auto* buffer_inst = FindBufferInstruction(&malformed_inst); + Check(buffer_inst != nullptr, "native validation fixture lacks a buffer instruction"); + buffer_inst->op = ShaderRecompiler::IR::Opcode::FlatLoadDword; + buffer_inst->memory.kind = ShaderRecompiler::IR::ResourceKind::Flat; + buffer_inst->src_count = 1; + rejected_spirv = rejected_before; + rejected_error.clear(); + Check(!ShaderRecompiler::Spirv::EmitProgram(malformed_inst, result.resources, nullptr, nullptr, + nullptr, &rejected_spirv, &rejected_error) && + rejected_spirv == rejected_before && + rejected_error.find("contract") != std::string::npos, + "malformed native instruction was not rejected before fatal emitter paths"); + + auto missing_buffer_operand = result.program; + buffer_inst = FindBufferInstruction(&missing_buffer_operand); + Check(buffer_inst != nullptr, "native validation fixture lacks a buffer-arity candidate"); + buffer_inst->src_count--; + rejected_spirv = rejected_before; + rejected_error.clear(); + Check(!ShaderRecompiler::Spirv::EmitProgram(missing_buffer_operand, result.resources, nullptr, + nullptr, nullptr, &rejected_spirv, + &rejected_error) && + rejected_spirv == rejected_before && + rejected_error.find("contract") != std::string::npos, + "buffer instruction with a missing address operand was accepted"); + + auto extra_buffer_operand = result.program; + buffer_inst = FindBufferInstruction(&extra_buffer_operand); + Check(buffer_inst != nullptr, "native validation fixture lacks an extra-operand candidate"); + buffer_inst->src_count++; + rejected_spirv = rejected_before; + rejected_error.clear(); + Check(!ShaderRecompiler::Spirv::EmitProgram(extra_buffer_operand, result.resources, nullptr, + nullptr, nullptr, &rejected_spirv, + &rejected_error) && + rejected_spirv == rejected_before && + rejected_error.find("contract") != std::string::npos, + "buffer instruction with an extra address operand was accepted"); + + auto malformed_atomic = result.program; + auto* atomic_inst = FindBufferInstruction(&malformed_atomic); + Check(atomic_inst != nullptr, "native validation fixture lacks an atomic candidate"); + atomic_inst->op = ShaderRecompiler::IR::Opcode::AtomicAddU32; + atomic_inst->memory.kind = ShaderRecompiler::IR::ResourceKind::ScalarBuffer; + atomic_inst->src_count = 2; + rejected_spirv = rejected_before; + rejected_error.clear(); + Check( + !ShaderRecompiler::Spirv::EmitProgram(malformed_atomic, result.resources, nullptr, nullptr, + nullptr, &rejected_spirv, &rejected_error) && + rejected_spirv == rejected_before && rejected_error.find("atomic") != std::string::npos, + "malformed atomic instruction reached a fatal descriptor path"); + + auto overwide_atomic = result.program; + atomic_inst = FindBufferInstruction(&overwide_atomic); + Check(atomic_inst != nullptr, "native validation fixture lacks an overwide atomic candidate"); + atomic_inst->op = ShaderRecompiler::IR::Opcode::AtomicAddU32; + atomic_inst->src_count = 5; + rejected_spirv = rejected_before; + rejected_error.clear(); + Check(!ShaderRecompiler::Spirv::EmitProgram(overwide_atomic, result.resources, nullptr, nullptr, + nullptr, &rejected_spirv, &rejected_error) && + rejected_spirv == rejected_before && + rejected_error.find("fixed IR storage") != std::string::npos, + "overwide atomic operands reached fixed emitter storage"); + + auto overwide_move = result.program; + auto& move_inst = overwide_move.blocks.front().instructions.front(); + move_inst.op = ShaderRecompiler::IR::Opcode::MoveU32; + move_inst.memory = {}; + move_inst.src_count = 5; + rejected_spirv = rejected_before; + rejected_error.clear(); + Check(!ShaderRecompiler::Spirv::EmitProgram(overwide_move, result.resources, nullptr, nullptr, + nullptr, &rejected_spirv, &rejected_error) && + rejected_spirv == rejected_before && + rejected_error.find("fixed IR storage") != std::string::npos, + "overwide non-memory operands reached fixed emitter storage"); + + auto malformed_image = result.program; + auto* image_inst = static_cast(nullptr); + for (auto& block: malformed_image.blocks) { + const auto found = std::find_if( + block.instructions.begin(), block.instructions.end(), + [](const auto& inst) { return inst.op == ShaderRecompiler::IR::Opcode::ImageSample; }); + if (found != block.instructions.end()) { + image_inst = &*found; + break; + } + } + Check(image_inst != nullptr, "native validation fixture lacks a sampled image"); + image_inst->src_count = 0; + rejected_spirv = rejected_before; + rejected_error.clear(); + Check(!ShaderRecompiler::Spirv::EmitProgram(malformed_image, result.resources, nullptr, nullptr, + nullptr, &rejected_spirv, &rejected_error) && + rejected_spirv == rejected_before && + rejected_error.find("sampled image") != std::string::npos, + "malformed image operands reached fixed emitter storage"); + + auto stale_resources = result.resources; + uint32_t float_storage = UINT32_MAX; + for (uint32_t i = 0; i < result.program.info.images.size(); i++) { + if (result.program.info.images[i].kind == + ShaderRecompiler::IR::ResourceKind::StorageImage) { + float_storage = i; + break; + } + } + Check(float_storage != UINT32_MAX, "native validation fixture lacks float storage image"); + stale_resources.images[float_storage].dwords[1] = 20u << 20u; + rejected_spirv = rejected_before; + rejected_error.clear(); + Check(!ShaderRecompiler::Spirv::EmitProgram(result.program, stale_resources, nullptr, nullptr, + nullptr, &rejected_spirv, &rejected_error) && + rejected_spirv == rejected_before && + rejected_error.find("specialized format") != std::string::npos, + "stale runtime image format re-selected an absent descriptor group"); + + auto stale_dimension = result.resources; + stale_dimension.images[float_storage].dwords[3] &= 0x0fffffffu; + rejected_spirv = rejected_before; + rejected_error.clear(); + Check(!ShaderRecompiler::Spirv::EmitProgram(result.program, stale_dimension, nullptr, nullptr, + nullptr, &rejected_spirv, &rejected_error) && + rejected_spirv == rejected_before && + rejected_error.find("specialized dimension") != std::string::npos, + "unsupported runtime image type bypassed specialization validation"); +} + +bool LowerProvenanceOnly(const uint32_t* code, uint32_t words, ShaderRecompiler::IR::Program* ir, + std::string* error) { + ShaderRecompiler::Decoder::Program decoded; + if (!ShaderRecompiler::Decoder::DecodeProgram(std::span {code, words}, &decoded, error)) { + return false; + } + ShaderRecompiler::CFG::Graph cfg; + if (!ShaderRecompiler::CFG::BuildGraph(decoded, &cfg, error) || + !ShaderRecompiler::IR::LowerProgram(decoded, cfg, ShaderType::Compute, 64, ir, error)) { + return false; + } + return ShaderRecompiler::IR::BuildScalarProvenance(ir, error) && + ShaderRecompiler::IR::BuildSrtPlan(ir, error); +} + +bool ReadSrtHostDword(void*, uint64_t address, uint32_t* value) { + if (address == 0 || value == nullptr) { + return false; + } + std::memcpy(value, reinterpret_cast(address), sizeof(*value)); + return true; +} + +void TestScalarProvenanceRealWideMoveLowering() { + const uint32_t shader[] = { + EncodeSop1(0x04, 0, 20), // s_mov_b64 s[0:1], s[20:21] + EncodeSop1(0x04, 2, 22), // s_mov_b64 s[2:3], s[22:23] + EncodeMubuf0(0x0c), EncodeMubuf1(0, 0, 1), // buffer_load_dword via copied s[0:3] + EncodeSopp(0x01), + }; + + std::string error; + ShaderRecompiler::IR::Program ir; + const auto lowered = + LowerProvenanceOnly(shader, static_cast(std::size(shader)), &ir, &error); + Check(lowered, error.c_str()); + + const ShaderRecompiler::IR::Instruction* use = nullptr; + for (const auto& block: ir.blocks) { + for (const auto& inst: block.instructions) { + if (inst.op == ShaderRecompiler::IR::Opcode::BufferLoadDword) { + use = &inst; + } + } + } + Check(use != nullptr, "real wide-move shader did not lower a buffer use"); + const auto* source = ShaderRecompiler::IR::GetDescriptorSource(ir, use->memory.resource_source); + Check(source != nullptr && + ShaderRecompiler::IR::DescriptorSourceResolved(ir, use->memory.resource_source), + "real wide-move descriptor source was unresolved"); + for (uint32_t i = 0; i < 4; i++) { + const auto& value = ir.provenance.values[source->dwords[i]]; + Check(value.op == ShaderRecompiler::IR::ScalarValueOp::UserData && value.imm == 20 + i, + "real s_mov_b64 lowering did not copy both descriptor pairs"); + } +} + +void TestScalarProvenanceRealCarryAndScalarLoads() { + const uint32_t carry_shader[] = { + EncodeSop1(0x1f, 0, 0), // s_getpc_b64 s[0:1] + EncodeSop2(0x00, 0, 0, 129), // s_add_u32 s0, s0, 1 + EncodeSop2(0x04, 1, 1, 128), // s_addc_u32 s1, s1, 0 + EncodeMubuf0(0x0c), + EncodeMubuf1(0, 0, 1), // buffer_load_dword via computed s[0:3] + EncodeSopp(0x01), + }; + std::string error; + ShaderRecompiler::IR::Program carry_ir; + const auto carry_lowered = LowerProvenanceOnly( + carry_shader, static_cast(std::size(carry_shader)), &carry_ir, &error); + Check(carry_lowered, error.c_str()); + const ShaderRecompiler::IR::DescriptorValue* carry_source = nullptr; + for (const auto& block: carry_ir.blocks) { + for (const auto& inst: block.instructions) { + if (inst.op == ShaderRecompiler::IR::Opcode::BufferLoadDword) { + carry_source = ShaderRecompiler::IR::GetDescriptorSource( + carry_ir, inst.memory.resource_source); + } + } + } + Check(carry_source != nullptr, "real add/addc descriptor source was not attached"); + const auto& high = carry_ir.provenance.values[carry_source->dwords[1]]; + Check(high.op == ShaderRecompiler::IR::ScalarValueOp::AddCarry && + carry_ir.provenance.values[high.args[2]].op == + ShaderRecompiler::IR::ScalarValueOp::Carry, + "real s_add_u32/s_addc_u32 lowering lost SCC carry provenance"); + + const uint32_t load_shader[] = { + EncodeSmem0(0x02, 0, 4), + 0u, // s_load_dwordx4 s[0:3] + EncodeMubuf0(0x0c), + EncodeMubuf1(0, 0, 1), + EncodeSmem0(0x0a, 4, 4), + 0u, // s_buffer_load_dwordx4 s[4:7] + EncodeMubuf0(0x0c), + EncodeMubuf1(0, 1, 1), + EncodeSopp(0x01), + }; + ShaderRecompiler::IR::Program load_ir; + const auto load_lowered = LowerProvenanceOnly( + load_shader, static_cast(std::size(load_shader)), &load_ir, &error); + Check(load_lowered, error.c_str()); + uint32_t uses = 0; + for (const auto& block: load_ir.blocks) { + for (const auto& inst: block.instructions) { + if (inst.op != ShaderRecompiler::IR::Opcode::BufferLoadDword) { + continue; + } + const auto* source = + ShaderRecompiler::IR::GetDescriptorSource(load_ir, inst.memory.resource_source); + Check(source != nullptr, "real scalar-load descriptor source was not attached"); + const auto expected = uses++ == 0 + ? ShaderRecompiler::IR::ScalarValueOp::ReadConst + : ShaderRecompiler::IR::ScalarValueOp::ReadConstBuffer; + for (uint32_t i = 0; i < 4; i++) { + Check(load_ir.provenance.values[source->dwords[i]].op == expected, + "real scalar load used the wrong provenance operation"); + } + } + } + Check(uses == 2, "real scalar-load provenance test did not find both buffer uses"); + + const uint32_t inline_sampler_shader[] = { + EncodeSop2(0x25, 12, 128 + 12, 128 + 44), // s_bfm_b64 s[12:13], 12, 44 + EncodeSop1(0x04, 14, 255), + 0x09500000u, // s_mov_b64 s[14:15], 0x09500000 + EncodeMimg0(0xa0, 0xf), + EncodeMimg1(2, 1, 3, 6), // image_sample_a v2, v6, s[4:11], s[12:15] + EncodeSopp(0x01), + }; + ShaderRecompiler::IR::Program inline_sampler_ir; + error.clear(); + Check(LowerProvenanceOnly(inline_sampler_shader, + static_cast(std::size(inline_sampler_shader)), + &inline_sampler_ir, &error), + error.c_str()); + uint32_t sampler_source = 0; + for (const auto& block: inline_sampler_ir.blocks) { + for (const auto& inst: block.instructions) { + if (inst.op == ShaderRecompiler::IR::Opcode::ImageSample) { + sampler_source = inst.memory.sampler_source; + } + } + } + ShaderRecompiler::IR::DescriptorValue sampler; + ShaderRecompiler::IR::SrtRuntime runtime; + Check(sampler_source >= 2 && + ShaderRecompiler::IR::DescriptorSourceResolved(inline_sampler_ir, sampler_source) && + ShaderRecompiler::IR::EvaluateDescriptorSource(inline_sampler_ir, sampler_source, + 0x10, runtime, &sampler, &error) && + sampler.dwords[0] == 0 && sampler.dwords[1] == 0x00fff000u && + sampler.dwords[2] == 0x09500000u && sampler.dwords[3] == 0, + "real inline sampler construction was unresolved or evaluated incorrectly"); + + auto user_data = ImageTestUserData(); + ShaderRecompiler::CompileOptions options; + options.stage = ShaderType::Pixel; + options.user_data = user_data.data(); + ShaderRecompiler::CompileResult result; + error.clear(); + Check(ShaderRecompiler::TryRecompile(inline_sampler_shader, options, &result, &error), + error.c_str()); +} + +void TestSrtWalkerRealSmemLowering() { + const uint32_t shader[] = { + EncodeSMovB32(124, 130), // m0 = 2 + EncodeSmem0(0x02, 0, 4), + (124u << 25u) | 2u, // s_load_dwordx4 s[0:3], s[8:9], m0 offset + immediate 2 + EncodeMubuf0(0x0c), EncodeMubuf1(0, 0, 1), EncodeSopp(0x01), + }; + std::string error; + ShaderRecompiler::IR::Program ir; + const auto lowered = + LowerProvenanceOnly(shader, static_cast(std::size(shader)), &ir, &error); + Check(lowered, error.c_str()); + Check(ir.srt.reads.size() == 4 && ir.srt.dynamic_reads.empty(), + "real SMEM lowering did not build four compact SRT reads"); + + const std::array table = {0x11111111u, 0x22222222u, 0x33333333u, 0x44444444u}; + std::array user_data = {}; + const auto address = reinterpret_cast(table.data()); + user_data[8] = static_cast(address); + user_data[9] = static_cast(address >> 32u); + std::vector flat; + const ShaderRecompiler::IR::SrtRuntime runtime {user_data, 0, ReadSrtHostDword, nullptr}; + const auto walked = ShaderRecompiler::IR::WalkSrt(ir, runtime, &flat, &error); + Check(walked, error.c_str()); + Check(flat.size() == table.size() && std::equal(flat.begin(), flat.end(), table.begin()), + "real SMEM SRT walk did not apply component-level alignment"); + Check(ShaderRecompiler::IR::PatchSrtReads(&ir, &error), error.c_str()); + uint32_t patched = 0; + for (const auto& block: ir.blocks) { + for (const auto& inst: block.instructions) { + if (inst.op == ShaderRecompiler::IR::Opcode::LoadSrtDword) { + Check(inst.src[0].imm == patched++, "real SMEM patch used the wrong flat offset"); + } + } + } + Check(patched == 4, "real SMEM patch did not rewrite every component"); +} + +void TestSrtWalkerRealSBufferLowering() { + const uint32_t shader[] = { + EncodeSMovB32(124, 130), // m0 = 2 + EncodeSmem0(0x0a, 0, 4), + (124u << 25u) | 2u, // s_buffer_load_dwordx4 s[0:3], s[8:11], m0 + 2 + EncodeMubuf0(0x0c), EncodeMubuf1(0, 0, 1), EncodeSopp(0x01), + }; + std::string error; + ShaderRecompiler::IR::Program ir; + const auto lowered = + LowerProvenanceOnly(shader, static_cast(std::size(shader)), &ir, &error); + Check(lowered, error.c_str()); + Check(ir.srt.reads.size() == 4 && ir.srt.dynamic_reads.empty(), + "real S_BUFFER_LOAD lowering did not build four compact reads"); + + const std::array table = {0x11111111u, 0x22222222u, 0x33333333u, 0x44444444u, + 0x55555555u}; + std::array user_data = {}; + const auto address = reinterpret_cast(table.data()); + user_data[8] = static_cast(address); + user_data[9] = static_cast(address >> 32u); + user_data[10] = sizeof(table); + std::vector flat; + const ShaderRecompiler::IR::SrtRuntime runtime {user_data, 0, ReadSrtHostDword, nullptr}; + const auto walked = ShaderRecompiler::IR::WalkSrt(ir, runtime, &flat, &error); + Check(walked, error.c_str()); + Check(flat.size() == 4 && std::equal(flat.begin(), flat.end(), table.begin() + 1), + "real S_BUFFER_LOAD walk used the wrong final alignment"); + + user_data[10] = 4 * sizeof(uint32_t); + const auto flat_before_failure = flat; + const auto bounds_walked = ShaderRecompiler::IR::WalkSrt(ir, runtime, &flat, &error); + Check(!bounds_walked && error.find("exceeds size=16") != std::string::npos, + "real S_BUFFER_LOAD walk ignored descriptor bounds"); + Check(flat == flat_before_failure, + "failed real S_BUFFER_LOAD walk changed the prior flat snapshot"); + Check(ShaderRecompiler::IR::PatchSrtReads(&ir, &error), error.c_str()); + uint32_t patched = 0; + for (const auto& block: ir.blocks) { + for (const auto& inst: block.instructions) { + if (inst.op == ShaderRecompiler::IR::Opcode::LoadSrtDword) { + Check(inst.src[0].imm == patched++, + "real S_BUFFER_LOAD patch used the wrong flat offset"); + } + } + } + Check(patched == 4, "real S_BUFFER_LOAD patch did not rewrite every component"); + + const uint32_t negative_shader[] = { + EncodeSmem0(0x0a, 0, 4), + (125u << 25u) | 0x1ffffeu, // illegal negative S_BUFFER immediate + EncodeMubuf0(0x0c), EncodeMubuf1(0, 0, 1), EncodeSopp(0x01), + }; + ShaderRecompiler::IR::Program negative_ir; + const auto negative_lowered = LowerProvenanceOnly( + negative_shader, static_cast(std::size(negative_shader)), &negative_ir, &error); + Check(negative_lowered, error.c_str()); + user_data[10] = sizeof(table); + Check(!ShaderRecompiler::IR::WalkSrt(negative_ir, runtime, &flat, &error) && + error.find("negative immediate") != std::string::npos, + "real S_BUFFER_LOAD walk accepted a negative immediate"); +} + +void TestScalarMemoryLoadsSnapshotOverlappingOperands() { + const auto CheckOverlap = [](uint32_t opcode) { + const uint32_t shader[] = { + EncodeSmem0(opcode, 0, 0), + 125u << 25u, // load s[0:3] through overlapping s[0:*], null SOFFSET + EncodeMubuf0(0x0c), + EncodeMubuf1(0, 0, 1), + EncodeSopp(0x01), + }; + std::string error; + ShaderRecompiler::IR::Program ir; + Check(LowerProvenanceOnly(shader, static_cast(std::size(shader)), &ir, &error), + error.c_str()); + Check(ir.srt.reads.size() == 4 && ir.srt.dynamic_reads.empty(), + opcode == 0x02 + ? "overlapping S_LOAD operands were evaluated after a component write" + : "overlapping S_BUFFER_LOAD operands were evaluated after a component write"); + Check(ShaderRecompiler::IR::PatchSrtReads(&ir, &error), error.c_str()); + uint32_t patched = 0; + for (const auto& block: ir.blocks) { + for (const auto& inst: block.instructions) { + if (inst.op == ShaderRecompiler::IR::Opcode::LoadSrtDword) { + Check(inst.src[0].imm == patched++, + "overlapping scalar-memory patch used the wrong flat offset"); + } + } + } + Check(patched == 4, "overlapping scalar-memory load was not fully patched"); + }; + CheckOverlap(0x02); // s_load_dwordx4 + CheckOverlap(0x0a); // s_buffer_load_dwordx4 +} + +void TestScalarMemoryLoadCrossesIntoVcc() { + const uint32_t shader[] = { + EncodeSmem0(0x02, 104, 4), + 125u << 25u, // s_load_dwordx4 s[104:105], vcc_lo, vcc_hi + EncodeMubuf0(0x0c), + EncodeMubuf1(0, 26, 1), + EncodeSopp(0x01), + }; + std::string error; + ShaderRecompiler::IR::Program ir; + Check(LowerProvenanceOnly(shader, static_cast(std::size(shader)), &ir, &error), + error.c_str()); + Check(ir.srt.reads.size() == 4 && ir.srt.dynamic_reads.empty(), + "wide SMEM destination crossing into VCC lost scalar provenance"); + Check(ShaderRecompiler::IR::PatchSrtReads(&ir, &error), error.c_str()); + uint32_t patched = 0; + for (const auto& block: ir.blocks) { + for (const auto& inst: block.instructions) { + if (inst.op == ShaderRecompiler::IR::Opcode::LoadSrtDword) { + Check(inst.src[0].imm == patched++, + "wide SMEM destination crossing into VCC used the wrong flat offset"); + } + } + } + Check(patched == 4, "wide SMEM destination crossing into VCC was not fully patched"); +} + +void TestResourceTrackingRealDensePatching() { + const uint32_t shader[] = { + EncodeMubuf0(0x0c), EncodeMubuf1(0, 0, 1), // buffer load s[0:3] + EncodeMubuf0(0x0c), EncodeMubuf1(1, 0, 1), // same buffer + EncodeMubuf0(0x1c), EncodeMubuf1(2, 1, 1), // buffer store s[4:7] + EncodeMimg0(0x20, 0xf), EncodeMimg1(8, 0, 2, 1), // sampled image s[0:7], sampler s[8:11] + EncodeMimg0(0x08, 0xf), EncodeMimg1(12, 0, 0, 1), // storage view of the same image source + EncodeSopp(0x01), + }; + std::string error; + ShaderRecompiler::IR::Program ir; + const auto lowered = + LowerProvenanceOnly(shader, static_cast(std::size(shader)), &ir, &error); + Check(lowered, error.c_str()); + Check(ShaderRecompiler::IR::PatchSrtReads(&ir, &error), error.c_str()); + const auto tracked = ShaderRecompiler::IR::TrackResources(&ir, &error); + Check(tracked, error.c_str()); + Check(ir.info.buffers.size() == 2 && ir.info.images.size() == 2 && ir.info.samplers.size() == 1, + "real resource tracking produced the wrong dense list sizes"); + + uint32_t buffer_use = 0; + uint32_t image_use = 0; + for (const auto& block: ir.blocks) { + for (const auto& inst: block.instructions) { + switch (inst.memory.kind) { + case ShaderRecompiler::IR::ResourceKind::Buffer: { + const uint32_t expected = buffer_use++ < 2 ? 0 : 1; + Check(inst.memory.resource == expected, + "real buffer operand was not patched densely"); + break; + } + case ShaderRecompiler::IR::ResourceKind::Image: + case ShaderRecompiler::IR::ResourceKind::StorageImage: + Check(inst.memory.resource == image_use++, + "real image operand was not patched by view class"); + if (inst.op == ShaderRecompiler::IR::Opcode::ImageSample) { + Check(inst.memory.sampler == 0, + "real sampler operand was not patched densely"); + } + break; + default: break; + } + } + } + Check(buffer_use == 3 && image_use == 2 && ir.info.buffers[0].read && + ir.info.buffers[1].written, + "real tracked resource access facts were incomplete"); +} + +void TestLowerProgramResetsAnalysisState() { + const uint32_t first_shader[] = {EncodeMubuf0(0x0c), EncodeMubuf1(0, 0, 1), EncodeSopp(0x01)}; + std::string error; + ShaderRecompiler::IR::Program ir; + Check(LowerProvenanceOnly(first_shader, static_cast(std::size(first_shader)), &ir, + &error), + error.c_str()); + Check(ShaderRecompiler::IR::PatchSrtReads(&ir, &error), error.c_str()); + Check(ShaderRecompiler::IR::TrackResources(&ir, &error), error.c_str()); + ShaderComputeInputInfo compute; + Check(ShaderRecompiler::IR::CollectShaderInfo(&ir, {.compute = &compute}, &error), + error.c_str()); + Check(ir.resource_tracking_complete && ir.shader_info_complete && !ir.info.buffers.empty(), + "analysis-reset fixture did not reach completed state"); + ir.shader_hash = 0xdeadbeef; + + const uint32_t second_shader[] = {EncodeSopp(0x01)}; + Check(LowerProvenanceOnly(second_shader, static_cast(std::size(second_shader)), &ir, + &error), + error.c_str()); + Check(!ir.resource_tracking_complete && !ir.shader_info_complete && ir.srt_plan_complete && + ir.shader_hash == 0 && ir.info.buffers.empty() && ir.info.images.empty() && + ir.info.samplers.empty() && ir.info.sampled_pairs.empty() && ir.info.inputs.empty() && + ir.info.outputs.empty(), + "LowerProgram reused stale provenance/resource/interface state"); +} + +void TestNewShaderRecompilerStageInputInfo() { + using StageInputKind = ShaderRecompiler::IR::StageInputKind; + + const uint32_t shader[] = {0xbf810000u}; + + ShaderComputeInputInfo cs_info {}; + cs_info.group_id[0] = true; + cs_info.thread_ids_num = 3; + cs_info.dispatch_thread_dimensions = true; + + ShaderRecompiler::CompileOptions cs_options; + cs_options.stage = ShaderType::Compute; + cs_options.compute_input_info = &cs_info; + + ShaderRecompiler::CompileResult cs_result; + std::string error; + Check(ShaderRecompiler::TryRecompile(shader, cs_options, &cs_result, &error), error.c_str()); + Check(ProgramHasInput(cs_result.program, StageInputKind::WorkgroupId), + "compute WorkgroupId input missing from reflection"); + Check(ProgramHasInput(cs_result.program, StageInputKind::LocalInvocationId), + "compute LocalInvocationId input missing from reflection"); + Check(ProgramHasInput(cs_result.program, StageInputKind::LocalInvocationIndex), + "compute LocalInvocationIndex input missing from reflection"); + Check(ProgramHasInput(cs_result.program, StageInputKind::GlobalInvocationId), + "compute GlobalInvocationId input missing from reflection"); + Check(SpirvHasDecorationValue(cs_result.spirv, 11u, 26u), + "SPIR-V lacks WorkgroupId BuiltIn decoration"); + Check(SpirvHasDecorationValue(cs_result.spirv, 11u, 27u), + "SPIR-V lacks LocalInvocationId BuiltIn decoration"); + Check(SpirvHasDecorationValue(cs_result.spirv, 11u, 28u), + "SPIR-V lacks GlobalInvocationId BuiltIn decoration"); + Check(SpirvHasDecorationValue(cs_result.spirv, 11u, 29u), + "SPIR-V lacks LocalInvocationIndex BuiltIn decoration"); + CheckSpirvBinaryValidates(cs_result.spirv); + + ShaderRecompiler::CompileOptions vs_options; + vs_options.stage = ShaderType::Vertex; + + ShaderRecompiler::CompileResult vs_result; + Check(ShaderRecompiler::TryRecompile(shader, vs_options, &vs_result, &error), error.c_str()); + Check(ProgramHasInput(vs_result.program, StageInputKind::VertexIndex), + "vertex VertexIndex input missing from reflection"); + Check(ProgramHasInput(vs_result.program, StageInputKind::InstanceIndex), + "vertex InstanceIndex input missing from reflection"); + Check(SpirvHasDecorationValue(vs_result.spirv, 11u, 42u), + "SPIR-V lacks VertexIndex BuiltIn decoration"); + Check(SpirvHasDecorationValue(vs_result.spirv, 11u, 43u), + "SPIR-V lacks InstanceIndex BuiltIn decoration"); + CheckSpirvBinaryValidates(vs_result.spirv); + + ShaderPixelInputInfo ps_info {}; + ps_info.input_num = 2; + ps_info.ps_pos_x = true; + ps_info.ps_pos_y = true; + ps_info.ps_pos_z = true; + SetIdentityInterpolatorSettings(&ps_info); + + ShaderRecompiler::CompileOptions ps_options; + ps_options.stage = ShaderType::Pixel; + ps_options.pixel_input_info = &ps_info; + + ShaderRecompiler::CompileResult ps_result; + Check(ShaderRecompiler::TryRecompile(shader, ps_options, &ps_result, &error), error.c_str()); + Check(ProgramHasInput(ps_result.program, StageInputKind::FragCoord), + "pixel FragCoord input missing from reflection"); + Check(ProgramInputCount(ps_result.program, StageInputKind::Parameter) == 2, + "pixel interpolant inputs missing from reflection"); + Check(SpirvHasDecorationValue(ps_result.spirv, 11u, 15u), + "SPIR-V lacks FragCoord BuiltIn decoration"); + Check(SpirvHasDecorationValue(ps_result.spirv, 30u, 0u), + "SPIR-V lacks interpolant Location 0 decoration"); + Check(SpirvHasDecorationValue(ps_result.spirv, 30u, 1u), + "SPIR-V lacks interpolant Location 1 decoration"); + CheckSpirvBinaryValidates(ps_result.spirv); + + ShaderPixelInputInfo ps_pos_y_info {}; + ps_pos_y_info.input_num = 1; + ps_pos_y_info.ps_system_input_base = 2; + ps_pos_y_info.ps_pos_y = true; + SetIdentityInterpolatorSettings(&ps_pos_y_info); + ps_options.pixel_input_info = &ps_pos_y_info; + + ShaderRecompiler::CompileResult ps_pos_y_result; + Check(ShaderRecompiler::TryRecompile(shader, ps_options, &ps_pos_y_result, &error), + error.c_str()); + Check(ProgramHasInput(ps_pos_y_result.program, StageInputKind::FragCoord), + "pixel POS_Y-only FragCoord input missing from reflection"); + Check(SpirvHasDecorationValue(ps_pos_y_result.spirv, 11u, 15u), + "SPIR-V lacks POS_Y-only FragCoord BuiltIn decoration"); + CheckSpirvBinaryValidates(ps_pos_y_result.spirv); +} + +void TestNewShaderRecompilerPixelPipelineEntry() { + const uint32_t shader[] = {0xbf810000u}; + + EnsureConfigInitialized(); + + HW::PixelShaderInfo regs {}; + regs.ps_regs.data_addr = reinterpret_cast(shader); + regs.ps_regs.chksum = 0x1234567800000001ull; + ShaderMappedData mapped {}; + mapped.code_size_bytes = sizeof(shader); + ShaderMapUserData(regs.ps_regs.data_addr, mapped); + + HW::ShaderRegisters sh {}; + ShaderPixelInputInfo input_info {}; + std::vector spirv; + Check(ShaderCompileSpirvPS(®s, &sh, ShaderLaneMaskMode::NativeWave, &input_info, &spirv), + "new pixel shader recompiler wrapper did not produce SPIR-V"); + Check(!spirv.empty(), "new pixel shader recompiler wrapper returned empty SPIR-V"); + Check(spirv.front() == 0x07230203u, + "new pixel shader recompiler wrapper did not emit SPIR-V binary"); + CheckSpirvBinaryValidates(spirv); + + const uint32_t vcc_load_shader[] = { + EncodeSop1(0x24, 0, 106), // s_and_saveexec_b64 s[0:1], vcc + EncodeSMovB32(106, 27), EncodeSMovB32(107, 28), EncodeSmem0(0x02, 44, 53), + (0x7du << 25u) | 160u, EncodeMubuf0(0x0c), EncodeMubuf1(0, 11, 1), + EncodeSopp(0x01), + }; + std::array table {}; + std::array buffer {}; + const auto table_address = reinterpret_cast(table.data()); + const auto buffer_address = reinterpret_cast(buffer.data()); + table[40] = static_cast(buffer_address); + table[41] = static_cast(buffer_address >> 32u); + table[42] = 1; + + HW::PixelShaderInfo vcc_regs {}; + vcc_regs.ps_regs.data_addr = reinterpret_cast(vcc_load_shader); + vcc_regs.ps_regs.chksum = 0x6306606500000001ull; + vcc_regs.ps_regs.rsrc2.user_sgpr = 29; + vcc_regs.ps_user_sgpr.value[27] = static_cast(table_address); + vcc_regs.ps_user_sgpr.value[28] = static_cast(table_address >> 32u); + ShaderMappedData vcc_mapped {}; + vcc_mapped.code_size_bytes = sizeof(vcc_load_shader); + ShaderMapUserData(vcc_regs.ps_regs.data_addr, vcc_mapped); + + ShaderPixelInputInfo vcc_input {}; + std::vector vcc_spirv; + Check(ShaderCompileSpirvPS(&vcc_regs, &sh, ShaderLaneMaskMode::NativeWave, &vcc_input, + &vcc_spirv), + "VCC raw-load pointer was lost"); + CheckSpirvBinaryValidates(vcc_spirv); +} + +void TestPixelProgramCacheDescriptorSetIdentity() { + const uint32_t shader_01[] = {0xbf810000u}; + const uint32_t shader_10[] = {0xbf810000u}; + const uint32_t shader_mask[] = {0xbf810000u}; + HW::ShaderRegisters sh {}; + + auto check_transition = [&](const uint32_t* shader, uint64_t checksum, + bool first_has_vs_descriptors, bool second_has_vs_descriptors) { + HW::PixelShaderInfo regs {}; + regs.ps_regs.data_addr = reinterpret_cast(shader); + regs.ps_regs.chksum = checksum; + ShaderMappedData mapped {}; + mapped.code_size_bytes = sizeof(uint32_t); + ShaderMapUserData(regs.ps_regs.data_addr, mapped); + + auto compile = [&](bool has_vs_descriptors) { + auto vs_program = std::make_shared(); + if (has_vs_descriptors) { + vs_program->bindings.descriptors.emplace_back(); + } + ShaderVertexInputInfo vs_info {}; + vs_info.stage.program = std::move(vs_program); + ShaderPixelInputInfo ps_info {}; + std::span spirv; + Check(ShaderCompileInfoPS(®s, &sh, ShaderLaneMaskMode::NativeWave, &vs_info, + &ps_info, &spirv), + "pixel program-cache transition failed to compile"); + const auto expected_set = has_vs_descriptors ? 1u : 0u; + Check(ps_info.descriptor_set == expected_set && ps_info.stage.program != nullptr && + ps_info.stage.program->bindings.descriptor_set == expected_set, + "pixel program cache reused SPIR-V with the paired VS descriptor set"); + }; + + compile(first_has_vs_descriptors); + compile(second_has_vs_descriptors); + }; + + check_transition(shader_01, 0x91a27e6300000001ull, false, true); + check_transition(shader_10, 0x91a27e6300000002ull, true, false); + + HW::PixelShaderInfo mask_regs {}; + mask_regs.ps_regs.data_addr = reinterpret_cast(shader_mask); + mask_regs.ps_regs.chksum = 0x91a27e6300000003ull; + ShaderMappedData mapped {}; + mapped.code_size_bytes = sizeof(shader_mask); + ShaderMapUserData(mask_regs.ps_regs.data_addr, mapped); + auto compile_mask_mode = [&](ShaderLaneMaskMode mode) { + ShaderVertexInputInfo vs_info {}; + vs_info.stage.program = std::make_shared(); + ShaderPixelInputInfo ps_info {}; + std::span spirv; + Check(ShaderCompileInfoPS(&mask_regs, &sh, mode, &vs_info, &ps_info, &spirv), + "pixel lane-mask cache transition failed to compile"); + Check(ps_info.stage.program != nullptr && ps_info.stage.program->lane_mask_mode == mode, + "pixel program cache reused a different lane-mask lowering"); + }; + compile_mask_mode(ShaderLaneMaskMode::NativeWave); + compile_mask_mode(ShaderLaneMaskMode::PerInvocation); +} + +void TestNewShaderRecompilerUnsupportedMemoryDecode() { + const uint32_t mubuf_unknown[] = {EncodeMubuf0(0x7b), EncodeMubuf1(0, 0, 1), 0xbf810000u}; + CheckNewDecoderUnsupported(mubuf_unknown, static_cast(std::size(mubuf_unknown)), + "MUBUF", "opcode=0x7b"); + + const uint32_t flat_unknown[] = {EncodeFlat0(0x7e, 0, 4), EncodeFlat1(9, 0x7d, 0, 1), + 0xbf810000u}; + CheckNewDecoderUnsupported(flat_unknown, static_cast(std::size(flat_unknown)), "FLAT", + "opcode=0x7e"); + + const uint32_t mtbuf_unknown[] = {EncodeMtbuf0(0x08, 14, 7, 4), EncodeMtbuf1(0x08, 9, 0, 1), + 0xbf810000u}; + CheckNewDecoderUnsupported(mtbuf_unknown, static_cast(std::size(mtbuf_unknown)), + "MTBUF", "opcode=0x08"); +} + +void TestNewShaderRecompilerFlatUserPointerProvenance() { + const uint32_t shader[] = { + EncodeVop2(0x25, 0, 0, 2), // v_add_nc_u32 v0, s0, v2 + EncodeVop1(0x01, 1, 1), // v_mov_b32 v1, s1 + EncodeFlat0(0x0c, 0, 0), + EncodeFlat1(3, 0x7d, 0, 0), // flat_load_dword v3, v[0:1] + 0xbf810000u, + }; + const uint32_t user_data[] = {0x34567000u, 0x00000012u}; + + ShaderRecompiler::CompileOptions options; + options.stage = ShaderType::Compute; + options.user_data = user_data; + options.user_data_count = static_cast(std::size(user_data)); + + ShaderRecompiler::CompileResult result; + std::string error; + const bool compiled = ShaderRecompiler::TryRecompile(shader, options, &result, &error); + Check(compiled, error.c_str()); + Check(result.program.info.addresses.size() == 1 && + result.program.info.addresses[0].source != + ShaderRecompiler::IR::ScalarProvenance::Unknown, + "FLAT user pointer provenance was not attached"); + Check(result.resources.addresses.size() == 1 && + result.resources.addresses[0].guest_base == 0x0000001234567000ull && + result.resources.addresses[0].binding_base == 0x0000001234560000ull && + result.program.info.addresses[0].specialized_base == 0x0000001234560000ull, + "FLAT user pointer did not materialize its guest base"); + CheckSpirvBinaryValidates(result.spirv); +} + +void TestNewShaderRecompilerFlatAddressProvenanceBoundaries() { + const uint32_t segmented_shader[] = { + EncodeVop2(0x25, 0, 0, 2), + EncodeVop1(0x01, 1, 1), + EncodeFlat0(0x0c, 2, 0), + EncodeFlat1(3, 0x7d, 0, 0), // global_load_dword v3, v[0:1] + EncodeFlat0(0x0c, 1, 0), + EncodeFlat1(4, 0x7d, 0, 0), // scratch_load_dword v4, v[0:1] + 0xbf810000u, + }; + const uint32_t user_data[] = {0x34567000u, 0u, 0x1000u}; + + ShaderRecompiler::CompileOptions options; + options.stage = ShaderType::Compute; + options.user_data = user_data; + options.user_data_count = static_cast(std::size(user_data)); + options.flat_memory_base = 0; + ShaderRecompiler::CompileResult result; + std::string error; + const bool compiled = + ShaderRecompiler::TryRecompile(segmented_shader, options, &result, &error); + Check(compiled, error.c_str()); + Check(result.program.info.addresses.size() == 2, + "segmented address resources were not tracked independently"); + for (const auto& address: result.program.info.addresses) { + Check(address.source == ShaderRecompiler::IR::ScalarProvenance::Unknown, + "GLOBAL/SCRATCH null-SADDR incorrectly inherited FLAT VGPR provenance"); + } +} + +} // namespace +} // namespace Libs::Graphics + +int main() { + using namespace Libs::Graphics; + + EnsureConfigInitialized(); + + TestResourceDescriptorClassification(); + TestNativeShaderResourceDependencies(); + TestNativeSubgroupPolicy(); + TestNewShaderRecompilerSMovB32(); + TestNewShaderRecompilerSoppMarkers(); + TestNewShaderRecompilerSopkWaitcntMarkers(); + TestNewShaderRecompilerRdna2ScalarOpcodes(); + TestNewShaderRecompilerScalarVectorAlu(); + TestNewShaderRecompilerVop3LaneReadDestinationEncoding(); + TestNewShaderRecompilerMoreAluFamilies(); + TestNewShaderRecompilerExpandedAluBatch(); + TestNewShaderRecompilerVop3pPackedF16(); + TestNewShaderRecompilerStagedShaderOps(); + TestNewShaderRecompilerBootF16UnaryOpcodes(); + TestNewShaderRecompilerBootB16PackedAndSdwaOpcodes(); + TestNewShaderRecompilerScalarB64Alu(); + TestNewShaderRecompilerSignedCompareAlu(); + TestNewShaderRecompilerSignedMinShiftAlu(); + TestNewShaderRecompilerScalarBitfieldAlu(); + TestNewShaderRecompilerMemoryFamilyLowering(); + TestNewShaderRecompilerImageQueryLowering(); + TestNewShaderRecompilerImageSampleVariants(); + TestNewShaderRecompilerImageSampleA16SamplerCoords(); + TestNewShaderRecompilerImageSampleOpcodeAliases(); + TestNewShaderRecompilerImageSampleA16ExceptionComponents(); + TestNewShaderRecompilerImageSampleA16Rdna2AddressOrder(); + TestNewShaderRecompilerPixelImageSampleLodSelection(); + TestNewShaderRecompilerImageViewDimensions(); + TestNewShaderRecompilerImageGatherVariants(); + TestNewShaderRecompilerImageLoadA16UintCoords(); + TestNewShaderRecompilerImageLoadVariants(); + TestNewShaderRecompilerImageStoreLowering(); + TestNewShaderRecompilerStorageImage3DDescriptorVariant(); + TestNewShaderRecompilerStorageImage2DDescriptorOverridesMimg3D(); + TestNewShaderRecompilerImageAtomicLowering(); + TestNewShaderRecompilerVintrpLowering(); + TestNewShaderRecompilerWideMemoryLowering(); + TestNewShaderRecompilerBufferSignedLoadLowering(); + TestNewShaderRecompilerBufferSubDwordStoreLowering(); + TestNewShaderRecompilerMubufFormatLowering(); + TestNewShaderRecompilerFormattedStoreUsesRuntimeArrayLengthOnly(); + TestNewShaderRecompilerTypedBufferLowering(); + TestNewShaderRecompilerFlatOldBackedLowering(); + TestNewShaderRecompilerUnbasedFlatRequiresTranslator(); + TestNewShaderRecompilerFlatUserPointerProvenance(); + TestNewShaderRecompilerFlatAddressProvenanceBoundaries(); + TestNewShaderRecompilerFlatSignedLoadLowering(); + TestNewShaderRecompilerFlatStoreLowering(); + TestNewShaderRecompilerAtomicLowering(); + TestNewShaderRecompilerDsReadWrite2Lowering(); + TestNewShaderRecompilerDsSubDwordLowering(); + TestNewShaderRecompilerDsWideAndAtomicLowering(); + TestNewShaderRecompilerDsSwizzleLowering(); + TestNewShaderRecompilerDsAddtidLowering(); + TestNewShaderRecompilerDsFloatMinMaxLowering(); + TestNewShaderRecompilerCfgStraightLine(); + TestNewShaderRecompilerCfgIfElse(); + TestNewShaderRecompilerCfgTerminalExitMergePS(); + TestNewShaderRecompilerCfgPostEndTargetMergePS(); + TestNewShaderRecompilerCfgLoopBreakContinue(); + TestNewShaderRecompilerCfgLoopHeaderDynamicScalarBufferLoadStructured(); + TestNewShaderRecompilerCfgLoopHeaderBufferLoadDispatcher(); + TestNewShaderRecompilerCfgLoopHeaderDsAppendConsumeDispatcher(); + TestNewShaderRecompilerCfgSharedOuterAndLoopMerge(); + TestNewShaderRecompilerCfgLoopSharedContinueSelectionMerges(); + TestNewShaderRecompilerCfgDuplicateMergeStructuredSplit(); + TestNewShaderRecompilerCfgIrreducibleDispatcher(); + TestNewShaderRecompilerExecMaskHelpers(); + TestComputeShaderInputWaveSize(); + TestNewShaderRecompilerWave32MasksExecHighStores(); + TestNewShaderRecompilerWave32VccHighScalarStores(); + TestNewShaderRecompilerCompareMaskIsFullWaveBallot(); + TestNewShaderRecompilerBufferLoadsGuardedByExec(); + TestNewShaderRecompilerBufferAtomicsGuardedByBounds(); + TestNewShaderRecompilerBranchConditionForms(); + TestNewShaderRecompilerSetpcBranch(); + TestNewShaderRecompilerSetpcJumpTable(); + TestNewShaderRecompilerExpVertexOutputs(); + TestNewShaderRecompilerVertexSystemVgprs(); + TestNewShaderRecompilerVertexExportUsesLaneExecMask(); + TestNewShaderRecompilerPerInvocationMasks(); + TestNewShaderRecompilerExpPixelOutputs(); + TestNewShaderRecompilerEarlyZDisabledWhenPixelKillEnabled(); + TestScalarProvenanceRealWideMoveLowering(); + TestScalarProvenanceRealCarryAndScalarLoads(); + TestSrtWalkerRealSmemLowering(); + TestSrtWalkerRealSBufferLowering(); + TestScalarMemoryLoadsSnapshotOverlappingOperands(); + TestScalarMemoryLoadCrossesIntoVcc(); + TestResourceTrackingRealDensePatching(); + TestLowerProgramResetsAnalysisState(); + TestNewShaderRecompilerNativeBindingPlan(); + TestNewShaderRecompilerStageInputInfo(); + TestGraphicsCreateInterpolantMapping(); + TestNewShaderRecompilerPixelPipelineEntry(); + TestPixelProgramCacheDescriptorSetIdentity(); + TestNewShaderRecompilerUnsupportedMemoryDecode(); + + return 0; +}